Built out the ui trainer plugin with db comminication
This commit is contained in:
@@ -439,6 +439,12 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
def post_save_hook(self, save_path):
|
||||
# override in subclass
|
||||
pass
|
||||
|
||||
def done_hook(self):
|
||||
pass
|
||||
|
||||
def end_step_hook(self):
|
||||
pass
|
||||
|
||||
def save(self, step=None):
|
||||
if not self.accelerator.is_main_process:
|
||||
@@ -648,6 +654,8 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
self.logger.start()
|
||||
self.prepare_accelerator()
|
||||
|
||||
def sample_step_hook(self, img_num, total_imgs):
|
||||
pass
|
||||
|
||||
def prepare_accelerator(self):
|
||||
# set some config
|
||||
@@ -1419,6 +1427,8 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
)
|
||||
# run base sd process run
|
||||
self.sd.load_model()
|
||||
|
||||
self.sd.add_after_sample_image_hook(self.after_sample_image_hook)
|
||||
|
||||
dtype = get_torch_dtype(self.train_config.dtype)
|
||||
|
||||
@@ -2091,6 +2101,7 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
# update various steps
|
||||
self.step_num = step + 1
|
||||
self.grad_accumulation_step += 1
|
||||
self.end_step_hook()
|
||||
|
||||
|
||||
###################################################################
|
||||
@@ -2110,13 +2121,15 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
self.logger.finish()
|
||||
self.accelerator.end_training()
|
||||
|
||||
if self.save_config.push_to_hub:
|
||||
if("HF_TOKEN" not in os.environ):
|
||||
interpreter_login(new_session=False, write_permission=True)
|
||||
self.push_to_hub(
|
||||
repo_id=self.save_config.hf_repo_id,
|
||||
private=self.save_config.hf_private
|
||||
)
|
||||
if self.accelerator.is_main_process:
|
||||
# push to hub
|
||||
if self.save_config.push_to_hub:
|
||||
if("HF_TOKEN" not in os.environ):
|
||||
interpreter_login(new_session=False, write_permission=True)
|
||||
self.push_to_hub(
|
||||
repo_id=self.save_config.hf_repo_id,
|
||||
private=self.save_config.hf_private
|
||||
)
|
||||
del (
|
||||
self.sd,
|
||||
unet,
|
||||
@@ -2128,6 +2141,7 @@ class BaseSDTrainProcess(BaseTrainProcess):
|
||||
)
|
||||
|
||||
flush()
|
||||
self.done_hook()
|
||||
|
||||
def push_to_hub(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user