Added a Sample Next Step in the job gear dropdown to force a sample on the next step.

This commit is contained in:
Jaret Burkett committed 2026-07-17 07:59:30 -06:00
1 parent 7a3d94ed03
commit 988d891102
5 files changed
+88 -3

No files matched your search

@@ -203,6 +203,40 @@ class DiffusionTrainer(SDTrainer):
if self.progress_bar is not None:
self.progress_bar.unpause()
def should_sample(self):
if not self.is_ui_trainer:
return False
def _check_sample():
with self._db_connect() as conn:
cursor = conn.cursor()
cursor.execute(
"SELECT sample_now FROM Job WHERE id = ?", (self.job_id,))
sample_now = cursor.fetchone()
return False if sample_now is None else sample_now[0] == 1
return self._retry_db_operation(_check_sample)
def maybe_sample(self):
if not self.is_ui_trainer:
return
if self.should_sample():
self.update_db_key("sample_now", 0)
if self.progress_bar is not None:
self.progress_bar.pause()
print_acc(f"\nSampling at step {self.step_num}")
# clear any grads
self.optimizer.zero_grad()
if self.train_config.free_u:
self.sd.pipeline.disable_freeu()
self.sample(self.step_num)
if self.train_config.unload_text_encoder:
# make sure the text encoder is unloaded
self.sd.text_encoder_to('cpu')
self.ensure_params_requires_grad()
flush()
if self.progress_bar is not None:
self.progress_bar.unpause()
async def _update_key(self, key, value):
if not self.accelerator.is_main_process:
return
@@ -319,6 +353,7 @@ class DiffusionTrainer(SDTrainer):
self.update_step()
self.maybe_stop()
self.maybe_save()
self.maybe_sample()
def hook_before_model_load(self):
super().hook_before_model_load()