Built out the ui trainer plugin with db comminication

This commit is contained in:
Jaret Burkett
2025-02-21 05:53:35 -07:00
parent f778d979b5
commit adcf884c0f
13 changed files with 245 additions and 22 deletions

View File

@@ -439,6 +439,12 @@ class BaseSDTrainProcess(BaseTrainProcess):
def post_save_hook(self, save_path):
# override in subclass
pass
def done_hook(self):
pass
def end_step_hook(self):
pass
def save(self, step=None):
if not self.accelerator.is_main_process:
@@ -648,6 +654,8 @@ class BaseSDTrainProcess(BaseTrainProcess):
self.logger.start()
self.prepare_accelerator()
def sample_step_hook(self, img_num, total_imgs):
pass
def prepare_accelerator(self):
# set some config
@@ -1419,6 +1427,8 @@ class BaseSDTrainProcess(BaseTrainProcess):
)
# run base sd process run
self.sd.load_model()
self.sd.add_after_sample_image_hook(self.after_sample_image_hook)
dtype = get_torch_dtype(self.train_config.dtype)
@@ -2091,6 +2101,7 @@ class BaseSDTrainProcess(BaseTrainProcess):
# update various steps
self.step_num = step + 1
self.grad_accumulation_step += 1
self.end_step_hook()
###################################################################
@@ -2110,13 +2121,15 @@ class BaseSDTrainProcess(BaseTrainProcess):
self.logger.finish()
self.accelerator.end_training()
if self.save_config.push_to_hub:
if("HF_TOKEN" not in os.environ):
interpreter_login(new_session=False, write_permission=True)
self.push_to_hub(
repo_id=self.save_config.hf_repo_id,
private=self.save_config.hf_private
)
if self.accelerator.is_main_process:
# push to hub
if self.save_config.push_to_hub:
if("HF_TOKEN" not in os.environ):
interpreter_login(new_session=False, write_permission=True)
self.push_to_hub(
repo_id=self.save_config.hf_repo_id,
private=self.save_config.hf_private
)
del (
self.sd,
unet,
@@ -2128,6 +2141,7 @@ class BaseSDTrainProcess(BaseTrainProcess):
)
flush()
self.done_hook()
def push_to_hub(
self,