Added a Sample Next Step in the job gear dropdown to force a sample on the next step.
This commit is contained in:
1 parent
7a3d94ed03
commit
988d891102
5 files changed
+88
-3
No files matched your search
@@ -203,6 +203,40 @@ class DiffusionTrainer(SDTrainer):
|
||||
if self.progress_bar is not None:
|
||||
self.progress_bar.unpause()
|
||||
|
||||
def should_sample(self):
|
||||
if not self.is_ui_trainer:
|
||||
return False
|
||||
def _check_sample():
|
||||
with self._db_connect() as conn:
|
||||
cursor = conn.cursor()
|
||||
cursor.execute(
|
||||
"SELECT sample_now FROM Job WHERE id = ?", (self.job_id,))
|
||||
sample_now = cursor.fetchone()
|
||||
return False if sample_now is None else sample_now[0] == 1
|
||||
|
||||
return self._retry_db_operation(_check_sample)
|
||||
|
||||
def maybe_sample(self):
|
||||
if not self.is_ui_trainer:
|
||||
return
|
||||
if self.should_sample():
|
||||
self.update_db_key("sample_now", 0)
|
||||
if self.progress_bar is not None:
|
||||
self.progress_bar.pause()
|
||||
print_acc(f"\nSampling at step {self.step_num}")
|
||||
# clear any grads
|
||||
self.optimizer.zero_grad()
|
||||
if self.train_config.free_u:
|
||||
self.sd.pipeline.disable_freeu()
|
||||
self.sample(self.step_num)
|
||||
if self.train_config.unload_text_encoder:
|
||||
# make sure the text encoder is unloaded
|
||||
self.sd.text_encoder_to('cpu')
|
||||
self.ensure_params_requires_grad()
|
||||
flush()
|
||||
if self.progress_bar is not None:
|
||||
self.progress_bar.unpause()
|
||||
|
||||
async def _update_key(self, key, value):
|
||||
if not self.accelerator.is_main_process:
|
||||
return
|
||||
@@ -319,6 +353,7 @@ class DiffusionTrainer(SDTrainer):
|
||||
self.update_step()
|
||||
self.maybe_stop()
|
||||
self.maybe_save()
|
||||
self.maybe_sample()
|
||||
|
||||
def hook_before_model_load(self):
|
||||
super().hook_before_model_load()
|
||||
|
||||
Reference in new issue
Block a user