implemented device placement preset system more places. Vastly improved speed on setting network multiplier and activating network. Fixed timing issues on progress bar

This commit is contained in:
Jaret Burkett
2023-09-14 08:31:54 -06:00
parent 4e945917df
commit 569d7464d5
9 changed files with 173 additions and 91 deletions

View File

@@ -29,6 +29,7 @@ class SDTrainer(BaseSDTrainProcess):
else: else:
# offload it. Already cached # offload it. Already cached
self.sd.vae.to('cpu') self.sd.vae.to('cpu')
flush()
def hook_train_loop(self, batch): def hook_train_loop(self, batch):
@@ -110,7 +111,5 @@ class SDTrainer(BaseSDTrainProcess):
loss_dict = OrderedDict( loss_dict = OrderedDict(
{'loss': loss.item()} {'loss': loss.item()}
) )
# reset network multiplier
network.multiplier = 1.0
return loss_dict return loss_dict

View File

@@ -7,6 +7,8 @@ from typing import Union
# from lycoris.config import PRESET # from lycoris.config import PRESET
from torch.utils.data import DataLoader from torch.utils.data import DataLoader
import torch
import torch.backends.cuda
from toolkit.basic import value_map from toolkit.basic import value_map
from toolkit.data_loader import get_dataloader_from_datasets from toolkit.data_loader import get_dataloader_from_datasets
@@ -21,6 +23,7 @@ from toolkit.progress_bar import ToolkitProgressBar
from toolkit.sampler import get_sampler from toolkit.sampler import get_sampler
from toolkit.scheduler import get_lr_scheduler from toolkit.scheduler import get_lr_scheduler
from toolkit.sd_device_states_presets import get_train_sd_device_state_preset
from toolkit.stable_diffusion_model import StableDiffusion from toolkit.stable_diffusion_model import StableDiffusion
from jobs.process import BaseTrainProcess from jobs.process import BaseTrainProcess
@@ -28,7 +31,6 @@ from toolkit.metadata import get_meta_for_safetensors, load_metadata_from_safete
from toolkit.train_tools import get_torch_dtype from toolkit.train_tools import get_torch_dtype
import gc import gc
import torch
from tqdm import tqdm from tqdm import tqdm
from toolkit.config_modules import SaveConfig, LogingConfig, SampleConfig, NetworkConfig, TrainConfig, ModelConfig, \ from toolkit.config_modules import SaveConfig, LogingConfig, SampleConfig, NetworkConfig, TrainConfig, ModelConfig, \
@@ -135,6 +137,16 @@ class BaseSDTrainProcess(BaseTrainProcess):
self.network: Union[Network, None] = None self.network: Union[Network, None] = None
self.embedding: Union[Embedding, None] = None self.embedding: Union[Embedding, None] = None
# get the device state preset based on what we are training
self.train_device_state_preset = get_train_sd_device_state_preset(
device=self.device_torch,
train_unet=self.train_config.train_unet,
train_text_encoder=self.train_config.train_text_encoder,
cached_latents=self.is_latents_cached,
train_lora=self.network_config is not None,
train_embedding=self.embed_config is not None,
)
def sample(self, step=None, is_first=False): def sample(self, step=None, is_first=False):
sample_folder = os.path.join(self.save_root, 'samples') sample_folder = os.path.join(self.save_root, 'samples')
gen_img_config_list = [] gen_img_config_list = []
@@ -477,6 +489,10 @@ class BaseSDTrainProcess(BaseTrainProcess):
# if it has it # if it has it
if hasattr(te, 'enable_xformers_memory_efficient_attention'): if hasattr(te, 'enable_xformers_memory_efficient_attention'):
te.enable_xformers_memory_efficient_attention() te.enable_xformers_memory_efficient_attention()
if self.train_config.sdp:
torch.backends.cuda.enable_math_sdp(True)
torch.backends.cuda.enable_flash_sdp(True)
torch.backends.cuda.enable_mem_efficient_sdp(True)
if self.train_config.gradient_checkpointing: if self.train_config.gradient_checkpointing:
unet.enable_gradient_checkpointing() unet.enable_gradient_checkpointing()
@@ -513,7 +529,8 @@ class BaseSDTrainProcess(BaseTrainProcess):
if self.datasets is not None: if self.datasets is not None:
self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd) self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd)
if self.datasets_reg is not None: if self.datasets_reg is not None:
self.data_loader_reg = get_dataloader_from_datasets(self.datasets_reg, self.train_config.batch_size, self.sd) self.data_loader_reg = get_dataloader_from_datasets(self.datasets_reg, self.train_config.batch_size,
self.sd)
if self.network_config is not None: if self.network_config is not None:
# TODO should we completely switch to LycorisSpecialNetwork? # TODO should we completely switch to LycorisSpecialNetwork?
@@ -547,6 +564,7 @@ class BaseSDTrainProcess(BaseTrainProcess):
self.network.force_to(self.device_torch, dtype=dtype) self.network.force_to(self.device_torch, dtype=dtype)
# give network to sd so it can use it # give network to sd so it can use it
self.sd.network = self.network self.sd.network = self.network
self.network._update_torch_multiplier()
self.network.apply_to( self.network.apply_to(
text_encoder, text_encoder,
@@ -621,32 +639,12 @@ class BaseSDTrainProcess(BaseTrainProcess):
if not params: if not params:
# set trainable params # set trainable params
params = self.embedding.get_trainable_params() params = self.embedding.get_trainable_params()
flush() flush()
else: else:
# set them to train or not
if self.train_config.train_unet:
self.sd.unet.requires_grad_(True)
self.sd.unet.train()
else:
self.sd.unet.requires_grad_(False)
self.sd.unet.eval()
if self.train_config.train_text_encoder: # set the device state preset before getting params
if isinstance(self.sd.text_encoder, list): self.sd.set_device_state(self.train_device_state_preset)
for te in self.sd.text_encoder:
te.requires_grad_(True)
te.train()
else:
self.sd.text_encoder.requires_grad_(True)
self.sd.text_encoder.train()
else:
if isinstance(self.sd.text_encoder, list):
for te in self.sd.text_encoder:
te.requires_grad_(False)
te.eval()
else:
self.sd.text_encoder.requires_grad_(False)
self.sd.text_encoder.eval()
params = self.get_params() params = self.get_params()
@@ -729,25 +727,9 @@ class BaseSDTrainProcess(BaseTrainProcess):
# zero any gradients # zero any gradients
optimizer.zero_grad() optimizer.zero_grad()
self.lr_scheduler.step(self.step_num) self.lr_scheduler.step(self.step_num)
if self.embedding is not None or self.train_config.train_text_encoder: self.sd.set_device_state(self.train_device_state_preset)
if isinstance(self.sd.text_encoder, list):
for te in self.sd.text_encoder:
te.train()
else:
self.sd.text_encoder.train()
else:
if isinstance(self.sd.text_encoder, list):
for te in self.sd.text_encoder:
te.eval()
else:
self.sd.text_encoder.eval()
if self.train_config.train_unet or self.embedding:
self.sd.unet.train()
else:
self.sd.unet.eval()
flush() flush()
# self.step_num = 0 # self.step_num = 0
for step in range(self.step_num, self.train_config.steps): for step in range(self.step_num, self.train_config.steps):

View File

@@ -85,6 +85,7 @@ class TrainConfig:
self.batch_size: int = kwargs.get('batch_size', 1) self.batch_size: int = kwargs.get('batch_size', 1)
self.dtype: str = kwargs.get('dtype', 'fp32') self.dtype: str = kwargs.get('dtype', 'fp32')
self.xformers = kwargs.get('xformers', False) self.xformers = kwargs.get('xformers', False)
self.sdp = kwargs.get('sdp', False)
self.train_unet = kwargs.get('train_unet', True) self.train_unet = kwargs.get('train_unet', True)
self.train_text_encoder = kwargs.get('train_text_encoder', True) self.train_text_encoder = kwargs.get('train_text_encoder', True)
self.min_snr_gamma = kwargs.get('min_snr_gamma', None) self.min_snr_gamma = kwargs.get('min_snr_gamma', None)

View File

@@ -4,7 +4,6 @@ import os
import re import re
import sys import sys
from typing import List, Optional, Dict, Type, Union from typing import List, Optional, Dict, Type, Union
import torch import torch
from transformers import CLIPTextModel from transformers import CLIPTextModel
@@ -46,11 +45,12 @@ class LoRAModule(ToolkitModuleMixin, torch.nn.Module):
dropout=None, dropout=None,
rank_dropout=None, rank_dropout=None,
module_dropout=None, module_dropout=None,
network: 'LoRASpecialNetwork' = None,
parent=None, parent=None,
**kwargs **kwargs
): ):
"""if alpha == 0 or None, alpha is rank (no scaling).""" """if alpha == 0 or None, alpha is rank (no scaling)."""
super().__init__() super().__init__(network=network)
self.lora_name = lora_name self.lora_name = lora_name
self.scalar = torch.tensor(1.0) self.scalar = torch.tensor(1.0)
@@ -150,7 +150,7 @@ class LoRASpecialNetwork(ToolkitNetworkMixin, LoRANetwork):
5. modules_dimとmodules_alphaを指定 (推論用) 5. modules_dimとmodules_alphaを指定 (推論用)
""" """
# call the parent of the parent we are replacing (LoRANetwork) init # call the parent of the parent we are replacing (LoRANetwork) init
super(LoRANetwork, self).__init__() torch.nn.Module.__init__(self)
self.lora_dim = lora_dim self.lora_dim = lora_dim
self.alpha = alpha self.alpha = alpha
@@ -163,6 +163,7 @@ class LoRASpecialNetwork(ToolkitNetworkMixin, LoRANetwork):
self._multiplier: float = 1.0 self._multiplier: float = 1.0
self.is_active: bool = False self.is_active: bool = False
self._is_normalizing: bool = False self._is_normalizing: bool = False
self.torch_multiplier = None
# triggers the state updates # triggers the state updates
self.multiplier = multiplier self.multiplier = multiplier
self.is_sdxl = is_sdxl self.is_sdxl = is_sdxl
@@ -258,6 +259,7 @@ class LoRASpecialNetwork(ToolkitNetworkMixin, LoRANetwork):
dropout=dropout, dropout=dropout,
rank_dropout=rank_dropout, rank_dropout=rank_dropout,
module_dropout=module_dropout, module_dropout=module_dropout,
network=self,
parent=module, parent=module,
) )
loras.append(lora) loras.append(lora)

View File

@@ -29,6 +29,7 @@ class LoConSpecialModule(ToolkitModuleMixin, LoConModule):
lora_dim=4, alpha=1, lora_dim=4, alpha=1,
dropout=0., rank_dropout=0., module_dropout=0., dropout=0., rank_dropout=0., module_dropout=0.,
use_cp=False, use_cp=False,
network: 'LycorisSpecialNetwork' = None,
parent=None, parent=None,
**kwargs, **kwargs,
): ):
@@ -36,7 +37,7 @@ class LoConSpecialModule(ToolkitModuleMixin, LoConModule):
# call super of super # call super of super
torch.nn.Module.__init__(self) torch.nn.Module.__init__(self)
# call super of # call super of
super().__init__(call_super_init=False) super().__init__(call_super_init=False, network=network)
self.lora_name = lora_name self.lora_name = lora_name
self.lora_dim = lora_dim self.lora_dim = lora_dim
self.cp = False self.cp = False
@@ -170,6 +171,8 @@ class LycorisSpecialNetwork(ToolkitNetworkMixin, LycorisNetwork):
if module_dropout is None: if module_dropout is None:
module_dropout = 0 module_dropout = 0
self.torch_multiplier = None
# triggers a tensor update
self.multiplier = multiplier self.multiplier = multiplier
self.lora_dim = lora_dim self.lora_dim = lora_dim
@@ -229,6 +232,7 @@ class LycorisSpecialNetwork(ToolkitNetworkMixin, LycorisNetwork):
self.lora_dim, self.alpha, self.lora_dim, self.alpha,
self.dropout, self.rank_dropout, self.module_dropout, self.dropout, self.rank_dropout, self.module_dropout,
use_cp, use_cp,
network=self,
parent=module, parent=module,
**kwargs **kwargs
) )
@@ -240,6 +244,7 @@ class LycorisSpecialNetwork(ToolkitNetworkMixin, LycorisNetwork):
self.lora_dim, self.alpha, self.lora_dim, self.alpha,
self.dropout, self.rank_dropout, self.module_dropout, self.dropout, self.rank_dropout, self.module_dropout,
use_cp, use_cp,
network=self,
parent=module, parent=module,
**kwargs **kwargs
) )
@@ -249,6 +254,7 @@ class LycorisSpecialNetwork(ToolkitNetworkMixin, LycorisNetwork):
self.conv_lora_dim, self.conv_alpha, self.conv_lora_dim, self.conv_alpha,
self.dropout, self.rank_dropout, self.module_dropout, self.dropout, self.rank_dropout, self.module_dropout,
use_cp, use_cp,
network=self,
parent=module, parent=module,
**kwargs **kwargs
) )
@@ -271,6 +277,7 @@ class LycorisSpecialNetwork(ToolkitNetworkMixin, LycorisNetwork):
self.dropout, self.rank_dropout, self.module_dropout, self.dropout, self.rank_dropout, self.module_dropout,
use_cp, use_cp,
parent=module, parent=module,
network=self,
**kwargs **kwargs
) )
elif module.__class__.__name__ == 'Conv2d': elif module.__class__.__name__ == 'Conv2d':
@@ -281,6 +288,7 @@ class LycorisSpecialNetwork(ToolkitNetworkMixin, LycorisNetwork):
self.lora_dim, self.alpha, self.lora_dim, self.alpha,
self.dropout, self.rank_dropout, self.module_dropout, self.dropout, self.rank_dropout, self.module_dropout,
use_cp, use_cp,
network=self,
parent=module, parent=module,
**kwargs **kwargs
) )
@@ -290,6 +298,7 @@ class LycorisSpecialNetwork(ToolkitNetworkMixin, LycorisNetwork):
self.conv_lora_dim, self.conv_alpha, self.conv_lora_dim, self.conv_alpha,
self.dropout, self.rank_dropout, self.module_dropout, self.dropout, self.rank_dropout, self.module_dropout,
use_cp, use_cp,
network=self,
parent=module, parent=module,
**kwargs **kwargs
) )

View File

@@ -4,10 +4,8 @@ from collections import OrderedDict
from typing import Optional, Union, List, Type, TYPE_CHECKING, Dict, Any from typing import Optional, Union, List, Type, TYPE_CHECKING, Dict, Any
import torch import torch
from diffusers.utils import is_torch_version
from torch import nn from torch import nn
from torch.utils.checkpoint import checkpoint import weakref
from toolkit.metadata import add_model_hash_to_meta from toolkit.metadata import add_model_hash_to_meta
from toolkit.paths import KEYMAPS_ROOT from toolkit.paths import KEYMAPS_ROOT
@@ -47,11 +45,13 @@ class ToolkitModuleMixin:
def __init__( def __init__(
self: Module, self: Module,
*args, *args,
network: Network,
call_super_init: bool = True, call_super_init: bool = True,
**kwargs **kwargs
): ):
if call_super_init: if call_super_init:
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.network_ref: weakref.ref = weakref.ref(network)
self.is_checkpointing = False self.is_checkpointing = False
self.is_normalizing = False self.is_normalizing = False
self.normalize_scaler = 1.0 self.normalize_scaler = 1.0
@@ -125,13 +125,13 @@ class ToolkitModuleMixin:
# this may get an additional positional arg or not # this may get an additional positional arg or not
def forward(self: Module, x, *args, **kwargs): def forward(self: Module, x, *args, **kwargs):
# diffusers added scale to resnet.. not sure what it does if not self.network_ref().is_active:
if self._multiplier is None: # network is not active, avoid doing anything
self.set_multiplier(0.0) return self.org_forward(x, *args, **kwargs)
org_forwarded = self.org_forward(x, *args, **kwargs) org_forwarded = self.org_forward(x, *args, **kwargs)
lora_output = self._call_forward(x) lora_output = self._call_forward(x)
multiplier = self._multiplier.clone().detach() multiplier = self.network_ref().torch_multiplier
lora_output_batch_size = lora_output.size(0) lora_output_batch_size = lora_output.size(0)
multiplier_batch_size = multiplier.size(0) multiplier_batch_size = multiplier.size(0)
@@ -328,35 +328,52 @@ class ToolkitNetworkMixin:
extra_dict = None extra_dict = None
return extra_dict return extra_dict
def _update_torch_multiplier(self: Network):
# builds a tensor for fast usage in the forward pass of the network modules
# without having to set it in every single module every time it changes
multiplier = self._multiplier
# get first module
first_module = self.get_all_modules()[0]
device = first_module.lora_down.weight.device
dtype = first_module.lora_down.weight.dtype
with torch.no_grad():
tensor_multiplier = None
if isinstance(multiplier, int) or isinstance(multiplier, float):
tensor_multiplier = torch.tensor((multiplier,)).to(device, dtype=dtype)
elif isinstance(multiplier, list):
tensor_list = []
for m in multiplier:
if isinstance(m, int) or isinstance(m, float):
tensor_list.append(torch.tensor((m,)).to(device, dtype=dtype))
elif isinstance(m, torch.Tensor):
tensor_list.append(m.clone().detach().to(device, dtype=dtype))
tensor_multiplier = torch.cat(tensor_list)
elif isinstance(multiplier, torch.Tensor):
tensor_multiplier = multiplier.clone().detach().to(device, dtype=dtype)
self.torch_multiplier = tensor_multiplier.clone().detach()
@property @property
def multiplier(self) -> Union[float, List[float]]: def multiplier(self) -> Union[float, List[float]]:
return self._multiplier return self._multiplier
@multiplier.setter @multiplier.setter
def multiplier(self, value: Union[float, List[float]]): def multiplier(self, value: Union[float, List[float]]):
# only update if the value has changed # it takes time to update all the multipliers, so we only do it if the value has changed
if self._multiplier == value: if self._multiplier == value:
return return
# if we are setting a single value but have a list, keep the list if every item is the same as value
self._multiplier = value self._multiplier = value
self._update_lora_multiplier() self._update_torch_multiplier()
def _update_lora_multiplier(self: Network):
if self.is_active:
for lora in self.get_all_modules():
lora.set_multiplier(self._multiplier)
else:
for lora in self.get_all_modules():
lora.set_multiplier(0)
# called when the context manager is entered # called when the context manager is entered
# ie: with network: # ie: with network:
def __enter__(self: Network): def __enter__(self: Network):
self.is_active = True self.is_active = True
self._update_lora_multiplier()
def __exit__(self: Network, exc_type, exc_value, tb): def __exit__(self: Network, exc_type, exc_value, tb):
self.is_active = False self.is_active = False
self._update_lora_multiplier()
def force_to(self: Network, device, dtype): def force_to(self: Network, device, dtype):
self.to(device, dtype) self.to(device, dtype)

View File

@@ -6,6 +6,7 @@ class ToolkitProgressBar(tqdm):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self.paused = False self.paused = False
self.last_time = self._time()
def pause(self): def pause(self):
if not self.paused: if not self.paused:
@@ -15,7 +16,9 @@ class ToolkitProgressBar(tqdm):
def unpause(self): def unpause(self):
if self.paused: if self.paused:
self.paused = False self.paused = False
self.start_t += self._time() - self.last_time cur_t = self._time()
self.start_t += cur_t - self.last_time
self.last_print_t = cur_t
def update(self, *args, **kwargs): def update(self, *args, **kwargs):
if not self.paused: if not self.paused:

View File

@@ -0,0 +1,59 @@
import torch
import copy
empty_preset = {
'vae': {
'training': False,
'device': 'cpu',
},
'unet': {
'training': False,
'requires_grad': False,
'device': 'cpu',
},
'text_encoder': {
'training': False,
'requires_grad': False,
'device': 'cpu',
}
}
def get_train_sd_device_state_preset (
device: torch.DeviceObjType,
train_unet: bool = False,
train_text_encoder: bool = False,
cached_latents: bool = False,
train_lora: bool = False,
train_embedding: bool = False,
):
preset = copy.deepcopy(empty_preset)
if not cached_latents:
preset['vae']['device'] = device
if train_unet:
preset['unet']['training'] = True
preset['unet']['requires_grad'] = True
preset['unet']['device'] = device
else:
preset['unet']['device'] = device
if train_text_encoder:
preset['text_encoder']['training'] = True
preset['text_encoder']['requires_grad'] = True
preset['text_encoder']['device'] = device
else:
preset['text_encoder']['device'] = device
if train_embedding:
preset['text_encoder']['training'] = True
preset['text_encoder']['requires_grad'] = True
preset['text_encoder']['training'] = True
preset['unet']['training'] = True
if train_lora:
preset['text_encoder']['requires_grad'] = False
preset['unet']['requires_grad'] = False
return preset

View File

@@ -24,7 +24,6 @@ from toolkit.sampler import get_sampler
from toolkit.saving import save_ldm_model_from_diffusers from toolkit.saving import save_ldm_model_from_diffusers
from toolkit.train_tools import get_torch_dtype, apply_noise_offset from toolkit.train_tools import get_torch_dtype, apply_noise_offset
import torch import torch
from diffusers.schedulers import DDPMScheduler
from toolkit.pipelines import CustomStableDiffusionXLPipeline, CustomStableDiffusionPipeline, \ from toolkit.pipelines import CustomStableDiffusionXLPipeline, CustomStableDiffusionPipeline, \
StableDiffusionKDiffusionXLPipeline StableDiffusionKDiffusionXLPipeline
from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline from diffusers import StableDiffusionPipeline, StableDiffusionXLPipeline
@@ -48,7 +47,7 @@ DO_NOT_TRAIN_WEIGHTS = [
"unet_time_embedding.linear_2.weight", "unet_time_embedding.linear_2.weight",
] ]
DeviceStatePreset = Literal['cache_latents'] DeviceStatePreset = Literal['cache_latents', 'generate']
class BlankNetwork: class BlankNetwork:
@@ -111,7 +110,7 @@ class StableDiffusion:
self.unet: Union[None, 'UNet2DConditionModel'] self.unet: Union[None, 'UNet2DConditionModel']
self.text_encoder: Union[None, 'CLIPTextModel', List[Union['CLIPTextModel', 'CLIPTextModelWithProjection']]] self.text_encoder: Union[None, 'CLIPTextModel', List[Union['CLIPTextModel', 'CLIPTextModelWithProjection']]]
self.tokenizer: Union[None, 'CLIPTokenizer', List['CLIPTokenizer']] self.tokenizer: Union[None, 'CLIPTokenizer', List['CLIPTokenizer']]
self.noise_scheduler: Union[None, 'KarrasDiffusionSchedulers', 'DDPMScheduler'] = noise_scheduler self.noise_scheduler: Union[None, 'KarrasDiffusionSchedulers'] = noise_scheduler
# sdxl stuff # sdxl stuff
self.logit_scale = None self.logit_scale = None
@@ -247,7 +246,7 @@ class StableDiffusion:
# pipe.unet = prepare_unet_for_training(pipe.unet) # pipe.unet = prepare_unet_for_training(pipe.unet)
self.unet = pipe.unet self.unet = pipe.unet
self.vae = pipe.vae.to(self.device_torch, dtype=dtype) self.vae: 'AutoencoderKL' = pipe.vae.to(self.device_torch, dtype=dtype)
self.vae.eval() self.vae.eval()
self.vae.requires_grad_(False) self.vae.requires_grad_(False)
self.unet.to(self.device_torch, dtype=dtype) self.unet.to(self.device_torch, dtype=dtype)
@@ -275,26 +274,12 @@ class StableDiffusion:
network.is_normalizing = False network.is_normalizing = False
self.save_device_state() self.save_device_state()
self.set_device_state_preset('generate')
# save current seed state for training # save current seed state for training
rng_state = torch.get_rng_state() rng_state = torch.get_rng_state()
cuda_rng_state = torch.cuda.get_rng_state() if torch.cuda.is_available() else None cuda_rng_state = torch.cuda.get_rng_state() if torch.cuda.is_available() else None
# handle sdxl text encoder
if isinstance(self.text_encoder, list):
for encoder, i in zip(self.text_encoder, range(len(self.text_encoder))):
encoder.to(self.device_torch)
encoder.eval()
else:
self.text_encoder.to(self.device_torch)
self.text_encoder.eval()
self.vae.to(self.device_torch)
self.vae.eval()
self.unet.to(self.device_torch)
self.unet.eval()
flush()
noise_scheduler = self.noise_scheduler noise_scheduler = self.noise_scheduler
if sampler is not None: if sampler is not None:
if sampler.startswith("sample_"): # sample_dpmpp_2m if sampler.startswith("sample_"): # sample_dpmpp_2m
@@ -346,6 +331,7 @@ class StableDiffusion:
start_multiplier = self.network.multiplier start_multiplier = self.network.multiplier
pipeline.to(self.device_torch) pipeline.to(self.device_torch)
with network: with network:
with torch.no_grad(): with torch.no_grad():
if self.network is not None: if self.network is not None:
@@ -876,6 +862,7 @@ class StableDiffusion:
'unet': { 'unet': {
'training': self.unet.training, 'training': self.unet.training,
'device': self.unet.device, 'device': self.unet.device,
'requires_grad': self.unet.conv_in.weight.requires_grad,
}, },
} }
if isinstance(self.text_encoder, list): if isinstance(self.text_encoder, list):
@@ -884,11 +871,14 @@ class StableDiffusion:
self.device_state['text_encoder'].append({ self.device_state['text_encoder'].append({
'training': encoder.training, 'training': encoder.training,
'device': encoder.device, 'device': encoder.device,
# todo there has to be a better way to do this
'requires_grad': encoder.text_model.final_layer_norm.weight.requires_grad
}) })
else: else:
self.device_state['text_encoder'] = { self.device_state['text_encoder'] = {
'training': self.text_encoder.training, 'training': self.text_encoder.training,
'device': self.text_encoder.device, 'device': self.text_encoder.device,
'requires_grad': self.text_encoder.text_model.final_layer_norm.weight.requires_grad
} }
def restore_device_state(self): def restore_device_state(self):
@@ -910,19 +900,33 @@ class StableDiffusion:
else: else:
self.unet.eval() self.unet.eval()
self.unet.to(state['unet']['device']) self.unet.to(state['unet']['device'])
if state['unet']['requires_grad']:
self.unet.requires_grad_(True)
else:
self.unet.requires_grad_(False)
if isinstance(self.text_encoder, list): if isinstance(self.text_encoder, list):
for i, encoder in enumerate(self.text_encoder): for i, encoder in enumerate(self.text_encoder):
if state['text_encoder'][i]['training']: if isinstance(state['text_encoder'], list):
encoder.train() if state['text_encoder'][i]['training']:
encoder.train()
else:
encoder.eval()
encoder.to(state['text_encoder'][i]['device'])
encoder.requires_grad_(state['text_encoder'][i]['requires_grad'])
else: else:
encoder.eval() if state['text_encoder']['training']:
encoder.to(state['text_encoder'][i]['device']) encoder.train()
else:
encoder.eval()
encoder.to(state['text_encoder']['device'])
encoder.requires_grad_(state['text_encoder']['requires_grad'])
else: else:
if state['text_encoder']['training']: if state['text_encoder']['training']:
self.text_encoder.train() self.text_encoder.train()
else: else:
self.text_encoder.eval() self.text_encoder.eval()
self.text_encoder.to(state['text_encoder']['device']) self.text_encoder.to(state['text_encoder']['device'])
self.text_encoder.requires_grad_(state['text_encoder']['requires_grad'])
flush() flush()
def set_device_state_preset(self, device_state_preset: DeviceStatePreset): def set_device_state_preset(self, device_state_preset: DeviceStatePreset):
@@ -935,18 +939,22 @@ class StableDiffusion:
training_modules = [] training_modules = []
if device_state_preset in ['cache_latents']: if device_state_preset in ['cache_latents']:
active_modules = ['vae'] active_modules = ['vae']
if device_state_preset in ['generate']:
active_modules = ['vae', 'unet', 'text_encoder']
state = {} state = {}
# vae # vae
state['vae'] = { state['vae'] = {
'training': 'vae' in training_modules, 'training': 'vae' in training_modules,
'device': self.device_torch if 'vae' in active_modules else 'cpu', 'device': self.device_torch if 'vae' in active_modules else 'cpu',
'requires_grad': 'vae' in training_modules,
} }
# unet # unet
state['unet'] = { state['unet'] = {
'training': 'unet' in training_modules, 'training': 'unet' in training_modules,
'device': self.device_torch if 'unet' in active_modules else 'cpu', 'device': self.device_torch if 'unet' in active_modules else 'cpu',
'requires_grad': 'unet' in training_modules,
} }
# text encoder # text encoder
@@ -956,11 +964,13 @@ class StableDiffusion:
state['text_encoder'].append({ state['text_encoder'].append({
'training': 'text_encoder' in training_modules, 'training': 'text_encoder' in training_modules,
'device': self.device_torch if 'text_encoder' in active_modules else 'cpu', 'device': self.device_torch if 'text_encoder' in active_modules else 'cpu',
'requires_grad': 'text_encoder' in training_modules,
}) })
else: else:
state['text_encoder'] = { state['text_encoder'] = {
'training': 'text_encoder' in training_modules, 'training': 'text_encoder' in training_modules,
'device': self.device_torch if 'text_encoder' in active_modules else 'cpu', 'device': self.device_torch if 'text_encoder' in active_modules else 'cpu',
'requires_grad': 'text_encoder' in training_modules,
} }
self.set_device_state(state) self.set_device_state(state)