Allow control image for omnigen training and sampling
This commit is contained in:
1 parent
bbb57de6ec
commit
611969ec1f
6 files changed
+158
-103
No files matched your search
@@ -1,8 +1,6 @@
|
|||||||
import inspect
|
|
||||||
import os
|
import os
|
||||||
from typing import TYPE_CHECKING, List, Optional
|
from typing import TYPE_CHECKING, List, Optional
|
||||||
|
|
||||||
import einops
|
|
||||||
import torch
|
import torch
|
||||||
import yaml
|
import yaml
|
||||||
from toolkit.config_modules import GenerateImageConfig, ModelConfig
|
from toolkit.config_modules import GenerateImageConfig, ModelConfig
|
||||||
@@ -10,22 +8,29 @@ from toolkit.models.base_model import BaseModel
|
|||||||
from diffusers import AutoencoderKL
|
from diffusers import AutoencoderKL
|
||||||
from toolkit.basic import flush
|
from toolkit.basic import flush
|
||||||
from toolkit.prompt_utils import PromptEmbeds
|
from toolkit.prompt_utils import PromptEmbeds
|
||||||
from toolkit.samplers.custom_flowmatch_sampler import CustomFlowMatchEulerDiscreteScheduler
|
from toolkit.samplers.custom_flowmatch_sampler import (
|
||||||
|
CustomFlowMatchEulerDiscreteScheduler,
|
||||||
|
)
|
||||||
from toolkit.accelerator import unwrap_model
|
from toolkit.accelerator import unwrap_model
|
||||||
from optimum.quanto import freeze
|
from optimum.quanto import freeze
|
||||||
from toolkit.util.quantize import quantize, get_qtype
|
from toolkit.util.quantize import quantize, get_qtype
|
||||||
from .src.pipelines.omnigen2.pipeline_omnigen2 import OmniGen2Pipeline
|
from .src.pipelines.omnigen2.pipeline_omnigen2 import OmniGen2Pipeline
|
||||||
from .src.models.transformers import OmniGen2Transformer2DModel
|
from .src.models.transformers import OmniGen2Transformer2DModel
|
||||||
from .src.models.transformers.repo import OmniGen2RotaryPosEmbed
|
from .src.models.transformers.repo import OmniGen2RotaryPosEmbed
|
||||||
from .src.schedulers.scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler as OmniFlowMatchEuler
|
from .src.schedulers.scheduling_flow_match_euler_discrete import (
|
||||||
from transformers import CLIPProcessor, Qwen2_5_VLProcessor, Qwen2_5_VLForConditionalGeneration
|
FlowMatchEulerDiscreteScheduler as OmniFlowMatchEuler,
|
||||||
|
)
|
||||||
|
from PIL import Image
|
||||||
|
from transformers import (
|
||||||
|
CLIPProcessor,
|
||||||
|
Qwen2_5_VLForConditionalGeneration,
|
||||||
|
)
|
||||||
|
import torch.nn.functional as F
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from toolkit.data_transfer_object.data_loader import DataLoaderBatchDTO
|
from toolkit.data_transfer_object.data_loader import DataLoaderBatchDTO
|
||||||
|
|
||||||
scheduler_config = {
|
scheduler_config = {"num_train_timesteps": 1000}
|
||||||
"num_train_timesteps": 1000
|
|
||||||
}
|
|
||||||
|
|
||||||
BASE_MODEL_PATH = "OmniGen2/OmniGen2"
|
BASE_MODEL_PATH = "OmniGen2/OmniGen2"
|
||||||
|
|
||||||
@@ -34,25 +39,21 @@ class OmniGen2Model(BaseModel):
|
|||||||
arch = "omnigen2"
|
arch = "omnigen2"
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
device,
|
device,
|
||||||
model_config: ModelConfig,
|
model_config: ModelConfig,
|
||||||
dtype='bf16',
|
dtype="bf16",
|
||||||
custom_pipeline=None,
|
custom_pipeline=None,
|
||||||
noise_scheduler=None,
|
noise_scheduler=None,
|
||||||
**kwargs
|
**kwargs,
|
||||||
):
|
):
|
||||||
super().__init__(
|
super().__init__(
|
||||||
device,
|
device, model_config, dtype, custom_pipeline, noise_scheduler, **kwargs
|
||||||
model_config,
|
|
||||||
dtype,
|
|
||||||
custom_pipeline,
|
|
||||||
noise_scheduler,
|
|
||||||
**kwargs
|
|
||||||
)
|
)
|
||||||
self.is_flow_matching = True
|
self.is_flow_matching = True
|
||||||
self.is_transformer = True
|
self.is_transformer = True
|
||||||
self.target_lora_modules = ['OmniGen2Transformer2DModel']
|
self.target_lora_modules = ["OmniGen2Transformer2DModel"]
|
||||||
|
self._control_latent = None
|
||||||
|
|
||||||
# static method to get the noise scheduler
|
# static method to get the noise scheduler
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -74,15 +75,11 @@ class OmniGen2Model(BaseModel):
|
|||||||
|
|
||||||
self.print_and_status_update("Loading Qwen2.5 VL")
|
self.print_and_status_update("Loading Qwen2.5 VL")
|
||||||
processor = CLIPProcessor.from_pretrained(
|
processor = CLIPProcessor.from_pretrained(
|
||||||
extras_path,
|
extras_path, subfolder="processor", use_fast=True
|
||||||
subfolder="processor",
|
|
||||||
use_fast=True
|
|
||||||
)
|
)
|
||||||
|
|
||||||
mllm = Qwen2_5_VLForConditionalGeneration.from_pretrained(
|
mllm = Qwen2_5_VLForConditionalGeneration.from_pretrained(
|
||||||
extras_path,
|
extras_path, subfolder="mllm", torch_dtype=torch.bfloat16
|
||||||
subfolder="mllm",
|
|
||||||
torch_dtype=torch.bfloat16
|
|
||||||
)
|
)
|
||||||
mllm.to(self.device_torch, dtype=dtype)
|
mllm.to(self.device_torch, dtype=dtype)
|
||||||
if self.model_config.quantize_te:
|
if self.model_config.quantize_te:
|
||||||
@@ -93,16 +90,14 @@ class OmniGen2Model(BaseModel):
|
|||||||
|
|
||||||
if self.low_vram:
|
if self.low_vram:
|
||||||
# unload it for now
|
# unload it for now
|
||||||
mllm.to('cpu')
|
mllm.to("cpu")
|
||||||
|
|
||||||
flush()
|
flush()
|
||||||
|
|
||||||
self.print_and_status_update("Loading transformer")
|
self.print_and_status_update("Loading transformer")
|
||||||
|
|
||||||
transformer = OmniGen2Transformer2DModel.from_pretrained(
|
transformer = OmniGen2Transformer2DModel.from_pretrained(
|
||||||
model_path,
|
model_path, subfolder="transformer", torch_dtype=torch.bfloat16
|
||||||
subfolder="transformer",
|
|
||||||
torch_dtype=torch.bfloat16
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if not self.low_vram:
|
if not self.low_vram:
|
||||||
@@ -116,19 +111,16 @@ class OmniGen2Model(BaseModel):
|
|||||||
|
|
||||||
if self.low_vram:
|
if self.low_vram:
|
||||||
# unload it for now
|
# unload it for now
|
||||||
transformer.to('cpu')
|
transformer.to("cpu")
|
||||||
|
|
||||||
flush()
|
flush()
|
||||||
|
|
||||||
self.print_and_status_update("Loading vae")
|
self.print_and_status_update("Loading vae")
|
||||||
|
|
||||||
vae = AutoencoderKL.from_pretrained(
|
vae = AutoencoderKL.from_pretrained(
|
||||||
extras_path,
|
extras_path, subfolder="vae", torch_dtype=torch.bfloat16
|
||||||
subfolder="vae",
|
|
||||||
torch_dtype=torch.bfloat16
|
|
||||||
).to(self.device_torch, dtype=dtype)
|
).to(self.device_torch, dtype=dtype)
|
||||||
|
|
||||||
|
|
||||||
flush()
|
flush()
|
||||||
self.print_and_status_update("Loading Qwen2.5 VLProcessor")
|
self.print_and_status_update("Loading Qwen2.5 VLProcessor")
|
||||||
|
|
||||||
@@ -155,22 +147,11 @@ class OmniGen2Model(BaseModel):
|
|||||||
processor=processor,
|
processor=processor,
|
||||||
)
|
)
|
||||||
|
|
||||||
# pipe: OmniGen2Pipeline = OmniGen2Pipeline.from_pretrained(
|
|
||||||
# model_path,
|
|
||||||
# transformer=transformer,
|
|
||||||
# vae=vae,
|
|
||||||
# scheduler=scheduler,
|
|
||||||
# mllm=mllm,
|
|
||||||
# trust_remote_code=True,
|
|
||||||
# )
|
|
||||||
# processor = pipe.processor
|
|
||||||
|
|
||||||
flush()
|
flush()
|
||||||
|
|
||||||
text_encoder_list = [mllm]
|
text_encoder_list = [mllm]
|
||||||
tokenizer_list = [processor]
|
tokenizer_list = [processor]
|
||||||
|
|
||||||
|
|
||||||
flush()
|
flush()
|
||||||
|
|
||||||
# save it to the model class
|
# save it to the model class
|
||||||
@@ -190,8 +171,7 @@ class OmniGen2Model(BaseModel):
|
|||||||
|
|
||||||
def get_generation_pipeline(self):
|
def get_generation_pipeline(self):
|
||||||
scheduler = OmniFlowMatchEuler(
|
scheduler = OmniFlowMatchEuler(
|
||||||
dynamic_time_shift=True,
|
dynamic_time_shift=True, num_train_timesteps=1000
|
||||||
num_train_timesteps=1000
|
|
||||||
)
|
)
|
||||||
|
|
||||||
pipeline: OmniGen2Pipeline = OmniGen2Pipeline(
|
pipeline: OmniGen2Pipeline = OmniGen2Pipeline(
|
||||||
@@ -215,6 +195,17 @@ class OmniGen2Model(BaseModel):
|
|||||||
generator: torch.Generator,
|
generator: torch.Generator,
|
||||||
extra: dict,
|
extra: dict,
|
||||||
):
|
):
|
||||||
|
input_images = []
|
||||||
|
if gen_config.ctrl_img is not None:
|
||||||
|
control_img = Image.open(gen_config.ctrl_img)
|
||||||
|
control_img = control_img.convert("RGB")
|
||||||
|
# resize to width and height
|
||||||
|
if control_img.size != (gen_config.width, gen_config.height):
|
||||||
|
control_img = control_img.resize(
|
||||||
|
(gen_config.width, gen_config.height), Image.BILINEAR
|
||||||
|
)
|
||||||
|
input_images = [control_img]
|
||||||
|
|
||||||
img = pipeline(
|
img = pipeline(
|
||||||
prompt_embeds=conditional_embeds.text_embeds,
|
prompt_embeds=conditional_embeds.text_embeds,
|
||||||
prompt_attention_mask=conditional_embeds.attention_mask,
|
prompt_attention_mask=conditional_embeds.attention_mask,
|
||||||
@@ -224,10 +215,12 @@ class OmniGen2Model(BaseModel):
|
|||||||
width=gen_config.width,
|
width=gen_config.width,
|
||||||
num_inference_steps=gen_config.num_inference_steps,
|
num_inference_steps=gen_config.num_inference_steps,
|
||||||
text_guidance_scale=gen_config.guidance_scale,
|
text_guidance_scale=gen_config.guidance_scale,
|
||||||
image_guidance_scale=1.0, # reference image guidance scale. Add this for controls
|
image_guidance_scale=1.0, # reference image guidance scale. Add this for controls
|
||||||
latents=gen_config.latents,
|
latents=gen_config.latents,
|
||||||
|
align_res=False,
|
||||||
generator=generator,
|
generator=generator,
|
||||||
**extra
|
input_images=input_images,
|
||||||
|
**extra,
|
||||||
).images[0]
|
).images[0]
|
||||||
return img
|
return img
|
||||||
|
|
||||||
@@ -236,18 +229,16 @@ class OmniGen2Model(BaseModel):
|
|||||||
latent_model_input: torch.Tensor,
|
latent_model_input: torch.Tensor,
|
||||||
timestep: torch.Tensor, # 0 to 1000 scale
|
timestep: torch.Tensor, # 0 to 1000 scale
|
||||||
text_embeddings: PromptEmbeds,
|
text_embeddings: PromptEmbeds,
|
||||||
**kwargs
|
**kwargs,
|
||||||
):
|
):
|
||||||
# broadcast to batch dimension in a way that's compatible with ONNX/Core ML
|
# broadcast to batch dimension in a way that's compatible with ONNX/Core ML
|
||||||
try:
|
try:
|
||||||
timestep = timestep.expand(latent_model_input.shape[0]).to(latent_model_input.dtype)
|
timestep = timestep.expand(latent_model_input.shape[0]).to(
|
||||||
|
latent_model_input.dtype
|
||||||
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# optional_kwargs = {}
|
|
||||||
# if 'ref_image_hidden_states' in set(inspect.signature(self.model.forward).parameters.keys()):
|
|
||||||
# optional_kwargs['ref_image_hidden_states'] = ref_image_hidden_states
|
|
||||||
|
|
||||||
timesteps = timestep / 1000 # convert to 0 to 1 scale
|
timesteps = timestep / 1000 # convert to 0 to 1 scale
|
||||||
# timestep for model starts at 0 instead of 1. So we need to reverse them
|
# timestep for model starts at 0 instead of 1. So we need to reverse them
|
||||||
timestep = 1 - timesteps
|
timestep = 1 - timesteps
|
||||||
@@ -257,18 +248,60 @@ class OmniGen2Model(BaseModel):
|
|||||||
text_embeddings.text_embeds,
|
text_embeddings.text_embeds,
|
||||||
self.freqs_cis,
|
self.freqs_cis,
|
||||||
text_embeddings.attention_mask,
|
text_embeddings.attention_mask,
|
||||||
ref_image_hidden_states=None, # todo add ref latent ability
|
ref_image_hidden_states=self._control_latent,
|
||||||
)
|
)
|
||||||
|
|
||||||
return model_pred
|
return model_pred
|
||||||
|
|
||||||
|
def condition_noisy_latents(
|
||||||
|
self, latents: torch.Tensor, batch: "DataLoaderBatchDTO"
|
||||||
|
):
|
||||||
|
# reset the control latent
|
||||||
|
self._control_latent = None
|
||||||
|
with torch.no_grad():
|
||||||
|
control_tensor = batch.control_tensor
|
||||||
|
if control_tensor is not None:
|
||||||
|
self.vae.to(self.device_torch)
|
||||||
|
# we are not packed here, so we just need to pass them so we can pack them later
|
||||||
|
control_tensor = control_tensor * 2 - 1
|
||||||
|
control_tensor = control_tensor.to(
|
||||||
|
self.vae_device_torch, dtype=self.torch_dtype
|
||||||
|
)
|
||||||
|
|
||||||
|
# if it is not the size of batch.tensor, (bs,ch,h,w) then we need to resize it
|
||||||
|
# todo, we may not need to do this, check
|
||||||
|
if batch.tensor is not None:
|
||||||
|
target_h, target_w = batch.tensor.shape[2], batch.tensor.shape[3]
|
||||||
|
else:
|
||||||
|
# When caching latents, batch.tensor is None. We get the size from the file_items instead.
|
||||||
|
target_h = batch.file_items[0].crop_height
|
||||||
|
target_w = batch.file_items[0].crop_width
|
||||||
|
|
||||||
|
if (
|
||||||
|
control_tensor.shape[2] != target_h
|
||||||
|
or control_tensor.shape[3] != target_w
|
||||||
|
):
|
||||||
|
control_tensor = F.interpolate(
|
||||||
|
control_tensor, size=(target_h, target_w), mode="bilinear"
|
||||||
|
)
|
||||||
|
|
||||||
|
control_latent = self.encode_images(control_tensor).to(
|
||||||
|
latents.device, latents.dtype
|
||||||
|
)
|
||||||
|
self._control_latent = [
|
||||||
|
[x.squeeze(0)]
|
||||||
|
for x in torch.chunk(control_latent, control_latent.shape[0], dim=0)
|
||||||
|
]
|
||||||
|
|
||||||
|
return latents.detach()
|
||||||
|
|
||||||
def get_prompt_embeds(self, prompt: str) -> PromptEmbeds:
|
def get_prompt_embeds(self, prompt: str) -> PromptEmbeds:
|
||||||
prompt = [prompt] if isinstance(prompt, str) else prompt
|
prompt = [prompt] if isinstance(prompt, str) else prompt
|
||||||
prompt = [self.pipeline._apply_chat_template(_prompt) for _prompt in prompt]
|
prompt = [self.pipeline._apply_chat_template(_prompt) for _prompt in prompt]
|
||||||
self.text_encoder_to(self.device_torch, dtype=self.torch_dtype)
|
self.text_encoder_to(self.device_torch, dtype=self.torch_dtype)
|
||||||
max_sequence_length = 256
|
max_sequence_length = 256
|
||||||
prompt_embeds, prompt_attention_mask, _, _ = self.pipeline.encode_prompt(
|
prompt_embeds, prompt_attention_mask, _, _ = self.pipeline.encode_prompt(
|
||||||
prompt = prompt,
|
prompt=prompt,
|
||||||
do_classifier_free_guidance=False,
|
do_classifier_free_guidance=False,
|
||||||
device=self.device_torch,
|
device=self.device_torch,
|
||||||
max_sequence_length=max_sequence_length,
|
max_sequence_length=max_sequence_length,
|
||||||
@@ -289,25 +322,26 @@ class OmniGen2Model(BaseModel):
|
|||||||
# only save the transformer
|
# only save the transformer
|
||||||
transformer: OmniGen2Transformer2DModel = unwrap_model(self.model)
|
transformer: OmniGen2Transformer2DModel = unwrap_model(self.model)
|
||||||
transformer.save_pretrained(
|
transformer.save_pretrained(
|
||||||
save_directory=os.path.join(output_path, 'transformer'),
|
save_directory=os.path.join(output_path, "transformer"),
|
||||||
safe_serialization=True,
|
safe_serialization=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
meta_path = os.path.join(output_path, 'aitk_meta.yaml')
|
meta_path = os.path.join(output_path, "aitk_meta.yaml")
|
||||||
with open(meta_path, 'w') as f:
|
with open(meta_path, "w") as f:
|
||||||
yaml.dump(meta, f)
|
yaml.dump(meta, f)
|
||||||
|
|
||||||
def get_loss_target(self, *args, **kwargs):
|
def get_loss_target(self, *args, **kwargs):
|
||||||
noise = kwargs.get('noise')
|
noise = kwargs.get("noise")
|
||||||
batch = kwargs.get('batch')
|
batch = kwargs.get("batch")
|
||||||
# return (noise - batch.latents).detach()
|
# return (noise - batch.latents).detach()
|
||||||
return (batch.latents - noise).detach()
|
return (batch.latents - noise).detach()
|
||||||
|
|
||||||
def get_transformer_block_names(self) -> Optional[List[str]]:
|
def get_transformer_block_names(self) -> Optional[List[str]]:
|
||||||
# omnigen2 had a few blocks for things like noise_refiner, ref_image_refiner, context_refiner, and layers.
|
# omnigen2 had a few blocks for things like noise_refiner, ref_image_refiner, context_refiner, and layers.
|
||||||
# lets do all but image refiner until we add it
|
# lets do all but image refiner until we add it
|
||||||
return ['noise_refiner', 'context_refiner', 'layers']
|
if self.model_config.model_kwargs.get("use_image_refiner", False):
|
||||||
# return ['layers']
|
return ["noise_refiner", "context_refiner", "ref_image_refiner", "layers"]
|
||||||
|
return ["noise_refiner", "context_refiner", "layers"]
|
||||||
|
|
||||||
def convert_lora_weights_before_save(self, state_dict):
|
def convert_lora_weights_before_save(self, state_dict):
|
||||||
# currently starte with transformer. but needs to start with diffusion_model. for comfyui
|
# currently starte with transformer. but needs to start with diffusion_model. for comfyui
|
||||||
@@ -327,4 +361,3 @@ class OmniGen2Model(BaseModel):
|
|||||||
|
|
||||||
def get_base_model_version(self):
|
def get_base_model_version(self):
|
||||||
return "omnigen2"
|
return "omnigen2"
|
||||||
|
|
||||||
+2
-1
@@ -676,7 +676,8 @@ class OmniGen2Pipeline(DiffusionPipeline):
|
|||||||
prompt_embeds=negative_prompt_embeds,
|
prompt_embeds=negative_prompt_embeds,
|
||||||
freqs_cis=freqs_cis,
|
freqs_cis=freqs_cis,
|
||||||
prompt_attention_mask=negative_prompt_attention_mask,
|
prompt_attention_mask=negative_prompt_attention_mask,
|
||||||
ref_image_hidden_states=None,
|
ref_image_hidden_states=ref_latents,
|
||||||
|
# ref_image_hidden_states=None,
|
||||||
)
|
)
|
||||||
model_pred = model_pred_uncond + text_guidance_scale * (model_pred - model_pred_uncond)
|
model_pred = model_pred_uncond + text_guidance_scale * (model_pred - model_pred_uncond)
|
||||||
|
|
||||||
|
|||||||
@@ -310,6 +310,7 @@ export default function SimpleJob({
|
|||||||
{ value: 'sigmoid', label: 'Sigmoid' },
|
{ value: 'sigmoid', label: 'Sigmoid' },
|
||||||
{ value: 'linear', label: 'Linear' },
|
{ value: 'linear', label: 'Linear' },
|
||||||
{ value: 'shift', label: 'Shift' },
|
{ value: 'shift', label: 'Shift' },
|
||||||
|
{ value: 'weighted', label: 'Weighted' },
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
)}
|
)}
|
||||||
@@ -541,13 +542,12 @@ export default function SimpleJob({
|
|||||||
{ value: 'ddpm', label: 'DDPM' },
|
{ value: 'ddpm', label: 'DDPM' },
|
||||||
]}
|
]}
|
||||||
/>
|
/>
|
||||||
</div>
|
|
||||||
<div>
|
|
||||||
<NumberInput
|
<NumberInput
|
||||||
label="Guidance Scale"
|
label="Guidance Scale"
|
||||||
value={jobConfig.config.process[0].sample.guidance_scale}
|
value={jobConfig.config.process[0].sample.guidance_scale}
|
||||||
onChange={value => setJobConfig(value, 'config.process[0].sample.guidance_scale')}
|
onChange={value => setJobConfig(value, 'config.process[0].sample.guidance_scale')}
|
||||||
placeholder="eg. 1.0"
|
placeholder="eg. 1.0"
|
||||||
|
className="pt-2"
|
||||||
min={0}
|
min={0}
|
||||||
required
|
required
|
||||||
/>
|
/>
|
||||||
@@ -579,6 +579,26 @@ export default function SimpleJob({
|
|||||||
min={0}
|
min={0}
|
||||||
required
|
required
|
||||||
/>
|
/>
|
||||||
|
{isVideoModel && (
|
||||||
|
<div>
|
||||||
|
<NumberInput
|
||||||
|
label="Num Frames"
|
||||||
|
value={jobConfig.config.process[0].sample.num_frames}
|
||||||
|
onChange={value => setJobConfig(value, 'config.process[0].sample.num_frames')}
|
||||||
|
placeholder="eg. 0"
|
||||||
|
min={0}
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
<NumberInput
|
||||||
|
label="FPS"
|
||||||
|
value={jobConfig.config.process[0].sample.fps}
|
||||||
|
onChange={value => setJobConfig(value, 'config.process[0].sample.fps')}
|
||||||
|
placeholder="eg. 0"
|
||||||
|
min={0}
|
||||||
|
required
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div>
|
<div>
|
||||||
@@ -597,40 +617,36 @@ export default function SimpleJob({
|
|||||||
onChange={value => setJobConfig(value, 'config.process[0].sample.walk_seed')}
|
onChange={value => setJobConfig(value, 'config.process[0].sample.walk_seed')}
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
{isVideoModel && (
|
<div>
|
||||||
<div>
|
<FormGroup label="Advanced Sampling" className="pt-2">
|
||||||
<NumberInput
|
<div>
|
||||||
label="Num Frames"
|
<Checkbox
|
||||||
value={jobConfig.config.process[0].sample.num_frames}
|
label="Skip First Sample"
|
||||||
onChange={value => setJobConfig(value, 'config.process[0].sample.num_frames')}
|
className="pt-4"
|
||||||
placeholder="eg. 0"
|
checked={jobConfig.config.process[0].train.skip_first_sample || false}
|
||||||
min={0}
|
onChange={value => setJobConfig(value, 'config.process[0].train.skip_first_sample')}
|
||||||
required
|
/>
|
||||||
/>
|
</div>
|
||||||
<NumberInput
|
<div>
|
||||||
label="FPS"
|
<Checkbox
|
||||||
value={jobConfig.config.process[0].sample.fps}
|
label="Disable Sampling"
|
||||||
onChange={value => setJobConfig(value, 'config.process[0].sample.fps')}
|
className="pt-1"
|
||||||
placeholder="eg. 0"
|
checked={jobConfig.config.process[0].train.disable_sampling || false}
|
||||||
min={0}
|
onChange={value => setJobConfig(value, 'config.process[0].train.disable_sampling')}
|
||||||
required
|
/>
|
||||||
/>
|
</div>
|
||||||
</div>
|
</FormGroup>
|
||||||
)}
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<FormGroup label={`Sample Prompts (${jobConfig.config.process[0].sample.prompts.length})`} className="pt-2">
|
<FormGroup label={`Sample Prompts (${jobConfig.config.process[0].sample.prompts.length})`} className="pt-2">
|
||||||
{
|
{modelArch?.additionalSections?.includes('sample.ctrl_img') && (
|
||||||
modelArch?.additionalSections?.includes('sample.ctrl_img') && (
|
<div className="text-sm text-gray-100 mb-2 py-2 px-4 bg-yellow-700 rounded-lg">
|
||||||
<div className='text-sm text-gray-100 mb-2 py-2 px-4 bg-yellow-700 rounded-lg'>
|
<p className="font-semibold mb-1">Control Images</p>
|
||||||
<p className='font-semibold mb-1'>
|
To use control images on samples, add --ctrl_img to the prompts below.
|
||||||
Control Images
|
<br />
|
||||||
</p>
|
Example: <code className="bg-yellow-900 p-1">make this a cartoon --ctrl_img /path/to/image.png</code>
|
||||||
To use control images on samples, add --ctrl_img to the prompts below.
|
</div>
|
||||||
<br />
|
)}
|
||||||
Example: <code className='bg-yellow-900 p-1'>make this a cartoon --ctrl_img /path/to/image.png</code>
|
|
||||||
</div>
|
|
||||||
)
|
|
||||||
}
|
|
||||||
{jobConfig.config.process[0].sample.prompts.map((prompt, i) => (
|
{jobConfig.config.process[0].sample.prompts.map((prompt, i) => (
|
||||||
<div key={i} className="flex items-center space-x-2">
|
<div key={i} className="flex items-center space-x-2">
|
||||||
<div className="flex-1">
|
<div className="flex-1">
|
||||||
|
|||||||
@@ -68,6 +68,8 @@ export const defaultJobConfig: JobConfig = {
|
|||||||
use_ema: false,
|
use_ema: false,
|
||||||
ema_decay: 0.99,
|
ema_decay: 0.99,
|
||||||
},
|
},
|
||||||
|
skip_first_sample: false,
|
||||||
|
disable_sampling: false,
|
||||||
dtype: 'bf16',
|
dtype: 'bf16',
|
||||||
diff_output_preservation: false,
|
diff_output_preservation: false,
|
||||||
diff_output_preservation_multiplier: 1.0,
|
diff_output_preservation_multiplier: 1.0,
|
||||||
|
|||||||
@@ -200,6 +200,7 @@ export const modelArchs: ModelArch[] = [
|
|||||||
'config.process[0].model.quantize_te': [true, false],
|
'config.process[0].model.quantize_te': [true, false],
|
||||||
},
|
},
|
||||||
disableSections: ['network.conv'],
|
disableSections: ['network.conv'],
|
||||||
|
additionalSections: ['datasets.control_path', 'sample.ctrl_img'],
|
||||||
},
|
},
|
||||||
].sort((a, b) => {
|
].sort((a, b) => {
|
||||||
// Sort by label, case-insensitive
|
// Sort by label, case-insensitive
|
||||||
|
|||||||
@@ -110,6 +110,8 @@ export interface TrainConfig {
|
|||||||
optimizer_params: {
|
optimizer_params: {
|
||||||
weight_decay: number;
|
weight_decay: number;
|
||||||
};
|
};
|
||||||
|
skip_first_sample: boolean;
|
||||||
|
disable_sampling: boolean;
|
||||||
diff_output_preservation: boolean;
|
diff_output_preservation: boolean;
|
||||||
diff_output_preservation_multiplier: number;
|
diff_output_preservation_multiplier: number;
|
||||||
diff_output_preservation_class: string;
|
diff_output_preservation_class: string;
|
||||||
|
|||||||
Reference in new issue
Block a user