From 6ea281973d1ee498085eca86ae1493666159e712 Mon Sep 17 00:00:00 2001 From: Jaret Burkett Date: Thu, 13 Aug 2026 12:36:03 -0600 Subject: [PATCH] Add support for MiniMax H3 Ref2Vid training --- .../diffusion_models/__init__.py | 3 +- .../diffusion_models/minimax_h3/__init__.py | 2 +- .../diffusion_models/minimax_h3/minimax_h3.py | 241 +++++++-- .../minimax_h3/src/packing.py | 71 ++- .../minimax_h3/src/pipeline.py | 29 +- ui/src/app/jobs/new/options.tsx | 457 +++++++++++------- 6 files changed, 574 insertions(+), 229 deletions(-) diff --git a/extensions_built_in/diffusion_models/__init__.py b/extensions_built_in/diffusion_models/__init__.py index 510424b..688cafd 100644 --- a/extensions_built_in/diffusion_models/__init__.py +++ b/extensions_built_in/diffusion_models/__init__.py @@ -19,7 +19,7 @@ from .prx_pixel_t2i import PRXPixelT2IModel from .krea2 import Krea2Model from .boogu_image import BooguImageModel, BooguImageEditModel from .mageflow import MageFlowModel, MageFlowEditModel -from .minimax_h3 import MinimaxH3Model +from .minimax_h3 import MinimaxH3Model, MinimaxH3Ref2VAModel AI_TOOLKIT_MODELS = [ # put a list of models here @@ -57,4 +57,5 @@ AI_TOOLKIT_MODELS = [ MageFlowModel, MageFlowEditModel, MinimaxH3Model, + MinimaxH3Ref2VAModel, ] diff --git a/extensions_built_in/diffusion_models/minimax_h3/__init__.py b/extensions_built_in/diffusion_models/minimax_h3/__init__.py index 80eb9f4..11d2c61 100644 --- a/extensions_built_in/diffusion_models/minimax_h3/__init__.py +++ b/extensions_built_in/diffusion_models/minimax_h3/__init__.py @@ -1 +1 @@ -from .minimax_h3 import MinimaxH3Model +from .minimax_h3 import MinimaxH3Model, MinimaxH3Ref2VAModel diff --git a/extensions_built_in/diffusion_models/minimax_h3/minimax_h3.py b/extensions_built_in/diffusion_models/minimax_h3/minimax_h3.py index c08fff2..7cac9cd 100644 --- a/extensions_built_in/diffusion_models/minimax_h3/minimax_h3.py +++ b/extensions_built_in/diffusion_models/minimax_h3/minimax_h3.py @@ -818,6 +818,47 @@ class MinimaxH3Model(BaseModel): # ------------------------------------------------------------------ # Training forward # ------------------------------------------------------------------ + def _build_condition( + self, batch: "DataLoaderBatchDTO", latent_shape, device, dtype + ): + """Build the packed sequence's condition segment for one train step. + + ``latent_shape`` is the target's ``(t_lat, h_lat, w_lat)``. Returns + ``(cond_rows, keyframe_anchors, image_ref_shapes)`` where ``cond_rows`` + is ``(B, num_condition_rows, 96)`` or None. The base model implements + fl2va: the clip's first frame as a keyframe when the dataset asks for + i2v. MinimaxH3Ref2VAModel overrides this with image references from + the control images.""" + do_i2v = ( + batch is not None + and batch.dataset_config.do_i2v + and getattr(batch, "num_frames", 1) > 1 + ) + if not do_i2v: + return None, (), () + + if batch.first_frame_latents is not None: + first_latents = batch.first_frame_latents.to(device, torch.float32) + else: + frames = batch.tensor + if frames is None: + raise ValueError( + "do_i2v needs the first frame; no cached " + "first_frame_latents or raw tensors in batch" + ) + first_frames = frames[:, 0] if frames.ndim == 5 else frames + first_latents = self.encode_keyframe_latents( + first_frames.unsqueeze(2).to(device) + ) + if first_latents.ndim == 4: + first_latents = first_latents.unsqueeze(2) + cond_noise = torch.randn_like(first_latents) + first_latents = ( + KEYFRAME_NOISE_AUG_T * first_latents + + (1.0 - KEYFRAME_NOISE_AUG_T) * cond_noise + ) + return patchify_video_latents(first_latents).to(dtype), ("first",), () + def get_noise_prediction( self, latent_model_input: torch.Tensor, # (B, 24, t, h, w) noisy latents @@ -854,35 +895,10 @@ class MinimaxH3Model(BaseModel): t_v = 1.0 - sigma_v t_a = 1.0 - sigma_a - # --- i2v first-frame conditioning rows ------------------------- - do_i2v = ( - batch is not None - and batch.dataset_config.do_i2v - and getattr(batch, "num_frames", 1) > 1 + # --- conditioning rows (fl2va keyframe / ref2va references) ---- + cond_rows, keyframe_anchors, image_ref_shapes = self._build_condition( + batch, (t_lat, h_lat, w_lat), device, dtype ) - cond_rows = None - if do_i2v: - if batch.first_frame_latents is not None: - first_latents = batch.first_frame_latents.to(device, torch.float32) - else: - frames = batch.tensor - if frames is None: - raise ValueError( - "do_i2v needs the first frame; no cached " - "first_frame_latents or raw tensors in batch" - ) - first_frames = frames[:, 0] if frames.ndim == 5 else frames - first_latents = self.encode_keyframe_latents( - first_frames.unsqueeze(2).to(device) - ) - if first_latents.ndim == 4: - first_latents = first_latents.unsqueeze(2) - cond_noise = torch.randn_like(first_latents) - first_latents = ( - KEYFRAME_NOISE_AUG_T * first_latents - + (1.0 - KEYFRAME_NOISE_AUG_T) * cond_noise - ) - cond_rows = patchify_video_latents(first_latents).to(dtype) # --- audio rows ------------------------------------------------- if batch is not None and getattr(batch, "num_frames", None): @@ -958,7 +974,6 @@ class MinimaxH3Model(BaseModel): # --- packed layout (per item: text lengths differ) -------------- layouts = [] - anchors = ("first",) if cond_rows is not None else () for i in range(batch_size): layouts.append( build_packed_sequence( @@ -967,7 +982,8 @@ class MinimaxH3Model(BaseModel): latent_height=h_lat, latent_width=w_lat, num_audio_latents=a_lat, - keyframe_anchors=anchors, + keyframe_anchors=keyframe_anchors, + image_ref_shapes=image_ref_shapes, ) ) ( @@ -1158,3 +1174,168 @@ class MinimaxH3Model(BaseModel): k.replace("diffusion_model.", "transformer."): v for k, v in state_dict.items() } + + +class MinimaxH3Ref2VAModel(MinimaxH3Model): + """Reference-to-video (ref2va): the control images ride along as reference + blocks in the packed sequence (plus ``: `` vision blocks in the + Qwen3-VL conditioning) instead of anchoring the first frame. + + Image references only for now. Every reference keeps its OWN aspect ratio + and is resized to the TARGET's pixel area (axes snapped to /32), then sits + on its own aspect-normalized rotary grid — like the released ref2va's + per-reference grids, but area-matched to the target instead of the 2048px + reference short edge. Each reference block advances the rotary media + clock by 1.0. + + At sampling, ctrl images are ALWAYS references, never first frames. + """ + + arch = "minimax_h3_ref2va" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # references arrive as a list per sample (multi_control_paths), at + # their ORIGINAL size — this model does its own area-matched resize + self.has_multiple_control_images = True + self.use_raw_control_images = True + + def _dit_component(self) -> str: + partition = str( + self.model_config.model_kwargs.get("partition", "ref2va_pruned") + ).lower() + if partition not in ("ref2va", "ref2va_pruned"): + raise ValueError( + f"model_kwargs.partition must be ref2va or ref2va_pruned for " + f"{self.arch}, got {partition}" + ) + return f"dit_{partition}" + + def get_base_model_version(self): + return "minimax_h3_ref2va" + + def _build_condition( + self, batch: "DataLoaderBatchDTO", latent_shape, device, dtype + ): + # raw-size control tensors in [0, 1]; each is resized to the TARGET's + # pixel area with its own aspect kept, then encoded on its own grid + if batch is None: + return None, (), () + controls_per_item = None + if batch.control_tensor_list is not None: + controls_per_item = batch.control_tensor_list + elif batch.control_tensor is not None: + controls_per_item = [ + [batch.control_tensor[b]] for b in range(batch.control_tensor.shape[0]) + ] + if not controls_per_item: + return None, (), () + ref_count = len(controls_per_item[0]) + if any(len(c) != ref_count for c in controls_per_item): + raise ValueError( + "ref2va: every item in a batch must have the same number of " + "reference images" + ) + + _, h_lat, w_lat = latent_shape + target_h, target_w = h_lat * 16, w_lat * 16 + + all_rows = [] + ref_shapes = [] + for ref_idx in range(ref_count): + resized = [] + for c in controls_per_item: + img = c[ref_idx] + if img.ndim == 4: + img = img[0] + ph, pw = packing.reference_pixel_size( + img.shape[2], img.shape[1], target_h, target_w + ) + resized.append( + torch.nn.functional.interpolate( + img[None].to(device, torch.float32), + size=(ph, pw), + mode="bilinear", + antialias=True, + )[0] + ) + shapes = {tuple(r.shape[1:]) for r in resized} + if len(shapes) > 1: + raise ValueError( + "ref2va: reference aspect ratios must match across a batch " + f"(got pixel shapes {sorted(shapes)}); use batch_size 1 for " + "mixed-aspect references" + ) + frames = torch.stack(resized) + # [0, 1] control -> [-1, 1] pixels, single-frame keyframe encode + ref_latents = self.encode_keyframe_latents( + (frames * 2.0 - 1.0).unsqueeze(2) + ) + ref_noise = torch.randn_like(ref_latents) + ref_latents = ( + KEYFRAME_NOISE_AUG_T * ref_latents + + (1.0 - KEYFRAME_NOISE_AUG_T) * ref_noise + ) + ref_shapes.append((ref_latents.shape[3], ref_latents.shape[4])) + all_rows.append(patchify_video_latents(ref_latents).to(dtype)) + return torch.cat(all_rows, dim=1), (), tuple(ref_shapes) + + def generate_single_image( + self, + pipeline: MiniMaxH3Pipeline, + gen_config: GenerateImageConfig, + conditional_embeds: AdvancedPromptEmbeds, + unconditional_embeds: AdvancedPromptEmbeds, + generator: torch.Generator, + extra: dict, + ): + if self.model.device == torch.device("cpu"): + self.model.to(self.device_torch) + + sc = self.get_bucket_divisibility() + gen_config.width = max(sc, int(gen_config.width // sc * sc)) + gen_config.height = max(sc, int(gen_config.height // sc * sc)) + + is_video = gen_config.num_frames > 1 + if is_video: + gen_config.num_frames = packing.align_num_frames_down(gen_config.num_frames) + gen_config.fps = packing.FPS + gen_config.save_image = partial(new_save_image_function, gen_config) + gen_config.log_image = partial(blank_log_image_function, gen_config) + gen_config.output_ext = "mp4" + + # every ctrl image is a REFERENCE (never a first frame): resized to + # the target's pixel area with its own aspect kept + ref_images = [] + for path in ( + gen_config.ctrl_img, + gen_config.ctrl_img_1, + gen_config.ctrl_img_2, + gen_config.ctrl_img_3, + ): + if path is not None: + img = Image.open(path).convert("RGB") + ref_images.append( + packing.prepare_reference_image( + img, gen_config.height, gen_config.width + ) + ) + + with_audio = bool(self.model_config.model_kwargs.get("sample_audio", True)) + + result = pipeline( + conditional_embeds=conditional_embeds, + unconditional_embeds=unconditional_embeds, + height=gen_config.height, + width=gen_config.width, + num_frames=gen_config.num_frames, + num_inference_steps=gen_config.num_inference_steps, + guidance_scale=gen_config.guidance_scale, + latents=gen_config.latents, + generator=generator, + ref_images=ref_images or None, + with_audio=with_audio and is_video, + ) + if is_video: + return result # dict consumed by new_save_image_function + return result[0] diff --git a/extensions_built_in/diffusion_models/minimax_h3/src/packing.py b/extensions_built_in/diffusion_models/minimax_h3/src/packing.py index 58e5653..a0b960f 100644 --- a/extensions_built_in/diffusion_models/minimax_h3/src/packing.py +++ b/extensions_built_in/diffusion_models/minimax_h3/src/packing.py @@ -109,6 +109,31 @@ def resolve_canvas_size(aspect_width: float, aspect_height: float) -> Tuple[int, return max(m, round(height / m) * m), max(m, round(width / m) * m) +def reference_pixel_size( + ref_width: int, ref_height: int, target_height: int, target_width: int +) -> Tuple[int, int]: + """Reference images match the TARGET's pixel area while keeping their own + aspect ratio; both axes snap to the canvas multiple. Returns (height, width).""" + scale = math.sqrt((target_height * target_width) / float(ref_width * ref_height)) + m = CANVAS_MULTIPLE + height = max(m, round(ref_height * scale / m) * m) + width = max(m, round(ref_width * scale / m) * m) + return height, width + + +def prepare_reference_image( + image: Image.Image, target_height: int, target_width: int +) -> Image.Image: + """Resize a reference onto the target's pixel budget, aspect preserved + (up to the /32 snap).""" + height, width = reference_pixel_size( + image.size[0], image.size[1], target_height, target_width + ) + if image.size == (width, height): + return image + return image.resize((width, height), Image.Resampling.LANCZOS) + + def prepare_keyframe_image( image: Image.Image, height: int, width: int, stretch: bool = True ): @@ -254,13 +279,24 @@ def build_packed_sequence( num_audio_latents: int, patch_size=(1, 2, 2), keyframe_anchors: Tuple[str, ...] = (), + image_ref_shapes: Tuple[Tuple[int, int], ...] = (), ) -> PackedLayout: - """Build the [text | keyframe conditions | target audio | target video] - layout used by t2va and fl2va.""" + """Build the [text | conditions | target audio | target video] layout. + + The condition segment holds either fl2va keyframes (``keyframe_anchors``: + rows pinned at the first/last target frame's rotary time, on the target's + spatial grid) or ref2va image references (``image_ref_shapes``: one + ``(latent_height, latent_width)`` per reference — references keep their + OWN aspect on their own aspect-normalized grid; each block sits at its + own rotary time and advances the shared media clock by 1.0, so the target + streams start at ``num_text + len(image_ref_shapes)``).""" + if keyframe_anchors and image_ref_shapes: + raise ValueError("keyframe_anchors and image_ref_shapes are mutually exclusive") _, ph, pw = patch_size rows_per_frame = (latent_height // ph) * (latent_width // pw) num_text = int(text_token_tags.shape[0]) - num_cond = len(keyframe_anchors) * rows_per_frame + ref_rows = [(h // ph) * (w // pw) for h, w in image_ref_shapes] + num_cond = len(keyframe_anchors) * rows_per_frame + sum(ref_rows) num_audio_rows = num_audio_latents * AUDIO_CHANNELS num_video_rows = num_latent_frames * rows_per_frame seq_len = num_text + num_cond + num_audio_rows + num_video_rows @@ -270,7 +306,9 @@ def build_packed_sequence( video_start = audio_start + num_audio_rows # text rows sit on the time axis at their row index; the media clock - # continues from there, so prompt length shifts the whole media clock + # continues from there (past the reference blocks, which advance it 1.0 + # each), so prompt length shifts the whole media clock + media_origin = float(num_text + len(image_ref_shapes)) position_ids = torch.zeros(seq_len, 3, dtype=torch.float64) position_ids[:num_text, 0] = torch.arange(num_text, dtype=torch.float64) @@ -301,9 +339,30 @@ def build_packed_sequence( position_ids[rows, 0] = anchor_time position_ids[rows, 1:] = frame_grid + ref_cursor = cond_start + for i, (ref_h, ref_w) in enumerate(image_ref_shapes): + # each reference on its own aspect-normalized grid (area-matched to + # the target, so the grids span comparable ranges) + ref_sqrt_area = math.sqrt(ref_h * ref_w) + ref_grid = torch.stack( + [ + g.reshape(-1) + for g in torch.meshgrid( + _spatial_position_grid(ref_h, ph, ref_sqrt_area), + _spatial_position_grid(ref_w, pw, ref_sqrt_area), + indexing="ij", + ) + ], + dim=-1, + ) + rows = slice(ref_cursor, ref_cursor + ref_rows[i]) + position_ids[rows, 0] = float(num_text + i) + position_ids[rows, 1:] = ref_grid + ref_cursor += ref_rows[i] + # audio rows: channel-major, one rotary unit per latent (40/s = 24fps*5/3), # no height coordinate, width pinned to the grid extremes per channel - audio_time = float(num_text) + torch.arange(num_audio_latents, dtype=torch.float64) + audio_time = media_origin + torch.arange(num_audio_latents, dtype=torch.float64) position_ids[audio_start:video_start, 0] = audio_time.repeat(AUDIO_CHANNELS) position_ids[audio_start:video_start, 2] = torch.cat( [ @@ -315,7 +374,7 @@ def build_packed_sequence( ) video_pos = torch.empty(num_latent_frames, rows_per_frame, 3, dtype=torch.float64) - video_pos[:, :, 0] = _temporal_position_grid(num_latent_frames, float(num_text))[ + video_pos[:, :, 0] = _temporal_position_grid(num_latent_frames, media_origin)[ :, None ] video_pos[:, :, 1:] = frame_grid[None] diff --git a/extensions_built_in/diffusion_models/minimax_h3/src/pipeline.py b/extensions_built_in/diffusion_models/minimax_h3/src/pipeline.py index f0b8ab7..4aa86ee 100644 --- a/extensions_built_in/diffusion_models/minimax_h3/src/pipeline.py +++ b/extensions_built_in/diffusion_models/minimax_h3/src/pipeline.py @@ -75,6 +75,9 @@ class MiniMaxH3Pipeline: ctrl_img: Optional[ Image.Image ] = None, # first-frame keyframe, already canvas-sized + ref_images: Optional[ + list + ] = None, # ref2va references, already area-matched (own aspect, /32) with_audio: bool = True, **kwargs, ): @@ -101,7 +104,13 @@ class MiniMaxH3Pipeline: token_tags = conditional_embeds.text_token_tags[0].to("cpu", torch.long) # --- packed layout ------------------------------------------------- + if ctrl_img is not None and ref_images: + raise ValueError("ctrl_img (first frame) and ref_images are exclusive") anchors = ("first",) if ctrl_img is not None else () + # references keep their own aspect: latent dims come from each image + ref_shapes = tuple( + (img.size[1] // 16, img.size[0] // 16) for img in (ref_images or []) + ) layout = build_packed_sequence( text_token_tags=token_tags, num_latent_frames=t_lat, @@ -109,16 +118,18 @@ class MiniMaxH3Pipeline: latent_width=w_lat, num_audio_latents=a_lat, keyframe_anchors=anchors, + image_ref_shapes=ref_shapes, ) num_cond = layout.num_condition_video_rows # --- conditioning rows (draw order: condition noise, video, audio) -- - cond_rows = None - if ctrl_img is not None: + def encode_condition_image(img: Image.Image) -> torch.Tensor: cond_noise = randn_tensor( - (1, 24, 1, h_lat, w_lat), generator=generator, dtype=torch.float32 + (1, 24, 1, img.size[1] // 16, img.size[0] // 16), + generator=generator, + dtype=torch.float32, ).to(device) - frame = torch.from_numpy(np.array(ctrl_img)).float() + frame = torch.from_numpy(np.array(img)).float() frame = (frame / 255.0) * 2.0 - 1.0 # (H, W, 3) -> [-1, 1] frame = frame.permute(2, 0, 1)[None, :, None] # (1, 3, 1, H, W) cond_latents = model.encode_keyframe_latents(frame) # (1, 24, 1, h, w) fp32 @@ -127,7 +138,15 @@ class MiniMaxH3Pipeline: KEYFRAME_NOISE_AUG_T * cond_latents.to(device) + (1.0 - KEYFRAME_NOISE_AUG_T) * cond_noise ) - cond_rows = patchify_video_latents(cond_latents) # (1, rows, 96) + return patchify_video_latents(cond_latents) # (1, rows, 96) + + cond_rows = None + if ctrl_img is not None: + cond_rows = encode_condition_image(ctrl_img) + elif ref_images: + cond_rows = torch.cat( + [encode_condition_image(img) for img in ref_images], dim=1 + ) # --- initial noise ------------------------------------------------- if latents is None: diff --git a/ui/src/app/jobs/new/options.tsx b/ui/src/app/jobs/new/options.tsx index 30fd2a7..a4bdf92 100644 --- a/ui/src/app/jobs/new/options.tsx +++ b/ui/src/app/jobs/new/options.tsx @@ -45,14 +45,14 @@ export interface CustomModelSelectOption { options: SelectOption[]; getValue: (config: JobConfig) => string | undefined; onChange: (value: string, config: JobConfig, setJobConfig: (value: any, key: string) => void) => void; - doc?: ConfigDoc + doc?: ConfigDoc; } export type SampleTag = { title: string; - type: 'text' | 'multiline' | 'number' + type: 'text' | 'multiline' | 'number'; full?: boolean; -} +}; export interface SampleTags { [key: string]: SampleTag; @@ -78,6 +78,64 @@ export interface ModelArch { const defaultNameOrPath = ''; const defaultLinearRank = 32; +// used by the MiniMax-H3 fl2va arch (ref2va is contrastive-guidance only) +const minimaxH3DistillationHandling = { + label: 'Distillation Handling Method', + options: [ + { value: 'cg', label: 'Contrastive Guidance (default)' }, + { value: 'ta', label: 'Training Adapter' }, + { value: 'both', label: 'Contrastive Guidance + Training Adapter' }, + ], + getValue: (config: JobConfig) => { + const assistantLoraPath = config?.config?.process?.[0]?.model?.assistant_lora_path; + const hasAssistantLoraPath = assistantLoraPath && assistantLoraPath.trim() !== ''; + const hasContrastiveGuidance = config?.config?.process?.[0]?.train?.do_guidance_loss; + if (hasAssistantLoraPath && hasContrastiveGuidance) { + return 'both'; + } + if (hasAssistantLoraPath) { + return 'ta'; + } + return 'cg'; + }, + onChange: (value: string, config: JobConfig, setJobConfig: (value: any, key: string) => void) => { + if (value === 'cg') { + setJobConfig(true, 'config.process[0].train.do_guidance_loss'); + setJobConfig(undefined, 'config.process[0].model.assistant_lora_path'); + if (!config?.config?.process?.[0]?.train?.guidance_loss_target) { + setJobConfig(3.5, 'config.process[0].train.guidance_loss_target'); + } + } else if (value === 'ta') { + setJobConfig(undefined, 'config.process[0].train.do_guidance_loss'); + setJobConfig(undefined, 'config.process[0].train.guidance_loss_target'); + setJobConfig( + 'ostris/minimax_h3_training_adapter/minimax_h3_training_adapter_v1.safetensors', + 'config.process[0].model.assistant_lora_path', + ); + } else if (value === 'both') { + setJobConfig(true, 'config.process[0].train.do_guidance_loss'); + setJobConfig( + 'ostris/minimax_h3_training_adapter/minimax_h3_training_adapter_v1.safetensors', + 'config.process[0].model.assistant_lora_path', + ); + if (!config?.config?.process?.[0]?.train?.guidance_loss_target) { + setJobConfig(3.5, 'config.process[0].train.guidance_loss_target'); + } + } + }, + doc: { + title: 'MiniMax-H3 Distillation Handling', + description: ( +
+ MiniMax H3 is a guidance distilled model, so training on it directly will make the guidance distillation break + down. There are two different ways to train on this model without breaking the guidance distillation: + Contrastive Guidance and Training Adapter. Both have their pros and cons. The adapter is faster, but will still + break down over a long run. Contrastive Guidance is slower, but is less likely to break down. +
+ ), + }, +}; + export const modelArchs: ModelArch[] = [ { name: 'anima', @@ -196,7 +254,10 @@ export const modelArchs: ModelArch[] = [ group: 'experimental', defaults: { // default updates when [selected, unselected] in the UI - 'config.process[0].model.name_or_path': ['lodestones/Zeta-Chroma/zeta-chroma-base-x0-pixel-dino-distance.safetensors', defaultNameOrPath], + 'config.process[0].model.name_or_path': [ + 'lodestones/Zeta-Chroma/zeta-chroma-base-x0-pixel-dino-distance.safetensors', + defaultNameOrPath, + ], 'config.process[0].model.extras_name_or_path': ['Tongyi-MAI/Z-Image-Turbo', undefined], 'config.process[0].model.quantize': [true, false], 'config.process[0].model.quantize_te': [true, false], @@ -309,7 +370,13 @@ export const modelArchs: ModelArch[] = [ ], }, disableSections: ['network.conv'], - additionalSections: ['datasets.num_frames', 'model.low_vram', 'model.multistage', 'model.layer_offloading', 'datasets.auto_frame_count'], + additionalSections: [ + 'datasets.num_frames', + 'model.low_vram', + 'model.multistage', + 'model.layer_offloading', + 'datasets.auto_frame_count', + ], accuracyRecoveryAdapters: { // '3 bit with ARA': 'uint3|ostris/accuracy_recovery_adapters/wan22_14b_t2i_torchao_uint3.safetensors', '4 bit with ARA': 'uint4|ostris/accuracy_recovery_adapters/wan22_14b_t2i_torchao_uint4.safetensors', @@ -375,7 +442,13 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].datasets[x].fps': [24, undefined], }, disableSections: ['network.conv'], - additionalSections: ['sample.ctrl_img', 'datasets.num_frames', 'model.low_vram', 'datasets.do_i2v', 'datasets.auto_frame_count'], + additionalSections: [ + 'sample.ctrl_img', + 'datasets.num_frames', + 'model.low_vram', + 'datasets.do_i2v', + 'datasets.auto_frame_count', + ], }, { name: 'lumina2', @@ -747,59 +820,20 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].datasets[x].auto_frame_count': [true, undefined], }, disableSections: ['network.conv'], - additionalSections: ['sample.ctrl_img', 'datasets.num_frames', 'model.layer_offloading', 'model.low_vram', 'datasets.do_audio', 'datasets.audio_normalize', 'datasets.audio_preserve_pitch', 'datasets.do_i2v', 'train.audio_loss_multiplier', 'datasets.auto_frame_count', 'model.assistant_lora_path'], - customModelSelectOptions: [ - { - label: 'Distillation Handling Method', - options: [ - { value: 'cg', label: 'Contrastive Guidance (default)' }, - { value: 'ta', label: 'Training Adapter' }, - { value: 'both', label: 'Contrastive Guidance + Training Adapter' }, - ], - getValue: (config: JobConfig) => { - const assistantLoraPath = config?.config?.process?.[0]?.model?.assistant_lora_path; - const hasAssistantLoraPath = assistantLoraPath && assistantLoraPath.trim() !== ''; - const hasContrastiveGuidance = config?.config?.process?.[0]?.train?.do_guidance_loss; - if (hasAssistantLoraPath && hasContrastiveGuidance) { - return 'both'; - } - if (hasAssistantLoraPath) { - return 'ta'; - } - return 'cg'; - }, - onChange: (value: string, config: JobConfig, setJobConfig: (value: any, key: string) => void) => { - if (value === 'cg') { - setJobConfig(true, 'config.process[0].train.do_guidance_loss'); - setJobConfig(undefined, 'config.process[0].model.assistant_lora_path'); - if (!(config?.config?.process?.[0]?.train?.guidance_loss_target)) { - setJobConfig(3.5, 'config.process[0].train.guidance_loss_target'); - } - } else if (value === 'ta') { - setJobConfig(undefined, 'config.process[0].train.do_guidance_loss'); - setJobConfig(undefined, 'config.process[0].train.guidance_loss_target'); - setJobConfig("ostris/minimax_h3_training_adapter/minimax_h3_training_adapter_v1.safetensors", 'config.process[0].model.assistant_lora_path'); - } else if (value === 'both') { - setJobConfig(true, 'config.process[0].train.do_guidance_loss'); - setJobConfig("ostris/minimax_h3_training_adapter/minimax_h3_training_adapter_v1.safetensors", 'config.process[0].model.assistant_lora_path'); - if (!(config?.config?.process?.[0]?.train?.guidance_loss_target)) { - setJobConfig(3.5, 'config.process[0].train.guidance_loss_target'); - } - } - }, - doc: { - title: 'MiniMax-H3 Distillation Handling', - description: ( -
- MiniMax H3 is a guidance distilled model, so training on it directly will make the guidance distillation break down. - There are two different ways to train on this model without breaking the guidance distillation: - Contrastive Guidance and Training Adapter. Both have their pros and cons. The adapter is faster, but will still - break down over a long run. Contrastive Guidance is slower, but is less likely to break down. -
- ) - } - }, + additionalSections: [ + 'sample.ctrl_img', + 'datasets.num_frames', + 'model.layer_offloading', + 'model.low_vram', + 'datasets.do_audio', + 'datasets.audio_normalize', + 'datasets.audio_preserve_pitch', + 'datasets.do_i2v', + 'train.audio_loss_multiplier', + 'datasets.auto_frame_count', + 'model.assistant_lora_path', ], + customModelSelectOptions: [minimaxH3DistillationHandling], modelNotes: (

@@ -835,6 +869,77 @@ export const modelArchs: ModelArch[] = [

), }, + { + name: 'minimax_h3_ref2va', + label: 'MiniMax-H3 Ref2V', + group: 'video', + isVideoModel: true, + defaults: { + // default updates when [selected, unselected] in the UI + 'config.process[0].model.name_or_path': ['Comfy-Org/MiniMax-H3', defaultNameOrPath], + // pre-quantized weights: matching qtypes keep the load unchanged + 'config.process[0].model.quantize': [true, false], + 'config.process[0].model.qtype': ['convrot8', 'qfloat8'], + 'config.process[0].model.quantize_te': [true, false], + 'config.process[0].model.qtype_te': ['nvfp4', 'qfloat8'], + 'config.process[0].model.low_vram': [true, false], + 'config.process[0].sample.sampler': ['flowmatch', 'flowmatch'], + 'config.process[0].train.noise_scheduler': ['flowmatch', 'flowmatch'], + 'config.process[0].train.cache_text_embeddings': [true, false], + 'config.process[0].train.do_guidance_loss': [true, undefined], + 'config.process[0].train.guidance_loss_target': [3.5, undefined], + 'config.process[0].network.linear': [16, defaultLinearRank], + 'config.process[0].network.linear_alpha': [16, defaultLinearRank], + 'config.process[0].network.network_kwargs.ignore_if_contains': [['adaln_proj'], []], + 'config.process[0].sample.num_frames': [107, 1], + 'config.process[0].sample.fps': [24, 1], + 'config.process[0].sample.width': [768, 1024], + 'config.process[0].sample.height': [768, 1024], + 'config.process[0].sample.guidance_scale': [1, 4], + 'config.process[0].sample.sample_steps': [28, 25], + 'config.process[0].train.audio_loss_multiplier': [1.0, undefined], + 'config.process[0].train.timestep_type': ['shift', 'sigmoid'], + 'config.process[0].datasets[x].do_audio': [true, undefined], + 'config.process[0].datasets[x].cache_latents_to_disk': [true, false], + 'config.process[0].datasets[x].fps': [24, undefined], + 'config.process[0].datasets[x].num_frames': [39, undefined], + 'config.process[0].datasets[x].auto_frame_count': [true, undefined], + }, + disableSections: ['network.conv'], + additionalSections: [ + 'sample.multi_ctrl_imgs', + 'datasets.multi_control_paths', + 'datasets.num_frames', + 'model.layer_offloading', + 'model.low_vram', + 'datasets.do_audio', + 'datasets.audio_normalize', + 'datasets.audio_preserve_pitch', + 'train.audio_loss_multiplier', + 'datasets.auto_frame_count', + ], + modelNotes: ( +
+

+ Reference-to-video: control images condition the output as subject/style references (never as a first frame). + Each reference keeps its own aspect ratio, is resized to the target's pixel area, rides into the packed + sequence as a reference block, and is also shown to the Qwen3-VL conditioner as a{' '} + <Picture i> vision block. Training references come from the dataset control path(s); + sampling uses the sample ctrl images — always as references. Image references only for now (no reference + video/audio). +

+

+ Weights load like MiniMax-H3 (see that arch's notes) from the{' '} + + Models Folder Path + + , using diffusion_models/minimax_h3_ref2va_pruned_int8_convrot.safetensors — the ref2va partition + of the same release; text encoder and VAEs are shared with the fl2va arch. Everything else (pre-quantized + load, 24 fps, 17n+5 frame grid, guidance scale 1, single-image mode) matches MiniMax-H3. +

+
+ ), + }, { name: 'ltx2', label: 'LTX-2', @@ -860,7 +965,18 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].datasets[x].auto_frame_count': [false, undefined], }, disableSections: ['network.conv'], - additionalSections: ['sample.ctrl_img', 'datasets.num_frames', 'model.layer_offloading', 'model.low_vram', 'datasets.do_audio', 'datasets.audio_normalize', 'datasets.audio_preserve_pitch', 'datasets.do_i2v', 'train.audio_loss_multiplier', 'datasets.auto_frame_count'], + additionalSections: [ + 'sample.ctrl_img', + 'datasets.num_frames', + 'model.layer_offloading', + 'model.low_vram', + 'datasets.do_audio', + 'datasets.audio_normalize', + 'datasets.audio_preserve_pitch', + 'datasets.do_i2v', + 'train.audio_loss_multiplier', + 'datasets.auto_frame_count', + ], }, { name: 'ltx2.3', @@ -888,7 +1004,18 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].datasets[x].auto_frame_count': [false, undefined], }, disableSections: ['network.conv'], - additionalSections: ['sample.ctrl_img', 'datasets.num_frames', 'model.layer_offloading', 'model.low_vram', 'datasets.do_audio', 'datasets.audio_normalize', 'datasets.audio_preserve_pitch', 'datasets.do_i2v', 'train.audio_loss_multiplier', 'datasets.auto_frame_count'], + additionalSections: [ + 'sample.ctrl_img', + 'datasets.num_frames', + 'model.layer_offloading', + 'model.low_vram', + 'datasets.do_audio', + 'datasets.audio_normalize', + 'datasets.audio_preserve_pitch', + 'datasets.do_i2v', + 'train.audio_loss_multiplier', + 'datasets.auto_frame_count', + ], }, { name: 'ltx2.5', @@ -921,7 +1048,18 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].datasets[x].auto_frame_count': [false, undefined], }, disableSections: ['network.conv'], - additionalSections: ['sample.ctrl_img', 'datasets.num_frames', 'model.layer_offloading', 'model.low_vram', 'datasets.do_audio', 'datasets.audio_normalize', 'datasets.audio_preserve_pitch', 'datasets.do_i2v', 'train.audio_loss_multiplier', 'datasets.auto_frame_count'], + additionalSections: [ + 'sample.ctrl_img', + 'datasets.num_frames', + 'model.layer_offloading', + 'model.low_vram', + 'datasets.do_audio', + 'datasets.audio_normalize', + 'datasets.audio_preserve_pitch', + 'datasets.do_i2v', + 'train.audio_loss_multiplier', + 'datasets.auto_frame_count', + ], }, { name: 'flux2_klein_4b', @@ -971,10 +1109,7 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].model.qtype': ['qfloat8', 'qfloat8'], }, disableSections: ['network.conv'], - additionalSections: [ - 'model.low_vram', - 'model.layer_offloading', - ], + additionalSections: ['model.low_vram', 'model.layer_offloading'], }, { name: 'flux2_klein_9b', @@ -1014,7 +1149,10 @@ export const modelArchs: ModelArch[] = [ group: 'audio', defaults: { // default updates when [selected, unselected] in the UI - 'config.process[0].model.name_or_path': ['ostris/ace_step_1.5_ComfyUI_files/ace_step_1.5_xl_base_aio.safetensors', defaultNameOrPath], + 'config.process[0].model.name_or_path': [ + 'ostris/ace_step_1.5_ComfyUI_files/ace_step_1.5_xl_base_aio.safetensors', + defaultNameOrPath, + ], 'config.process[0].model.quantize': [true, false], 'config.process[0].model.quantize_te': [true, false], 'config.process[0].model.low_vram': [true, false], @@ -1025,43 +1163,39 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].sample': [defaultAudioSampleConfig, defaultSampleConfig], }, sampleTags: { - "CAPTION": { - title: "Audio Prompt", - type: "text", + CAPTION: { + title: 'Audio Prompt', + type: 'text', full: true, }, - "LYRICS": { - title: "Lyrics", - type: "multiline", + LYRICS: { + title: 'Lyrics', + type: 'multiline', full: true, }, - "BPM": { - title: "BPM", - type: "number", + BPM: { + title: 'BPM', + type: 'number', }, - "KEYSCALE": { - title: "Key Scale", - type: "text", + KEYSCALE: { + title: 'Key Scale', + type: 'text', }, - "TIMESIGNATURE": { - title: "Time Signature", - type: "text", + TIMESIGNATURE: { + title: 'Time Signature', + type: 'text', }, - "DURATION": { - title: "Duration (sec)", - type: "number", + DURATION: { + title: 'Duration (sec)', + type: 'number', }, - "LANGUAGE": { - title: "Language", - type: "text", + LANGUAGE: { + title: 'Language', + type: 'text', }, }, disableSections: ['network.conv'], - additionalSections: [ - 'sample.multi_ctrl_imgs', - 'model.low_vram', - 'model.layer_offloading', - ], + additionalSections: ['sample.multi_ctrl_imgs', 'model.low_vram', 'model.layer_offloading'], }, { name: 'ace_step_15', @@ -1069,7 +1203,10 @@ export const modelArchs: ModelArch[] = [ group: 'audio', defaults: { // default updates when [selected, unselected] in the UI - 'config.process[0].model.name_or_path': ['ostris/ace_step_1.5_ComfyUI_files/ace_step_1.5_base_aio.safetensors', defaultNameOrPath], + 'config.process[0].model.name_or_path': [ + 'ostris/ace_step_1.5_ComfyUI_files/ace_step_1.5_base_aio.safetensors', + defaultNameOrPath, + ], 'config.process[0].model.quantize': [true, false], 'config.process[0].model.quantize_te': [true, false], 'config.process[0].model.low_vram': [true, false], @@ -1080,43 +1217,39 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].sample': [defaultAudioSampleConfig, defaultSampleConfig], }, sampleTags: { - "CAPTION": { - title: "Audio Prompt", - type: "text", + CAPTION: { + title: 'Audio Prompt', + type: 'text', full: true, }, - "LYRICS": { - title: "Lyrics", - type: "multiline", + LYRICS: { + title: 'Lyrics', + type: 'multiline', full: true, }, - "BPM": { - title: "BPM", - type: "number", + BPM: { + title: 'BPM', + type: 'number', }, - "KEYSCALE": { - title: "Key Scale", - type: "text", + KEYSCALE: { + title: 'Key Scale', + type: 'text', }, - "TIMESIGNATURE": { - title: "Time Signature", - type: "text", + TIMESIGNATURE: { + title: 'Time Signature', + type: 'text', }, - "DURATION": { - title: "Duration (sec)", - type: "number", + DURATION: { + title: 'Duration (sec)', + type: 'number', }, - "LANGUAGE": { - title: "Language", - type: "text", + LANGUAGE: { + title: 'Language', + type: 'text', }, }, disableSections: ['network.conv'], - additionalSections: [ - 'sample.multi_ctrl_imgs', - 'model.low_vram', - 'model.layer_offloading', - ], + additionalSections: ['sample.multi_ctrl_imgs', 'model.low_vram', 'model.layer_offloading'], }, { name: 'nucleus_image', @@ -1158,15 +1291,8 @@ export const modelArchs: ModelArch[] = [ {}, ], }, - disableSections: [ - 'network.conv', - 'model.quantize_te', - 'train.unload_text_encoder', - ], - additionalSections: [ - 'model.low_vram', - 'model.layer_offloading', - ], + disableSections: ['network.conv', 'model.quantize_te', 'train.unload_text_encoder'], + additionalSections: ['model.low_vram', 'model.layer_offloading'], }, { name: 'zimage_l2p', @@ -1182,13 +1308,8 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].network.conv_alpha': [undefined, 16], 'config.process[0].model.low_vram': [true, false], }, - disableSections: [ - 'network.conv', - ], - additionalSections: [ - 'model.low_vram', - 'model.layer_offloading', - ], + disableSections: ['network.conv'], + additionalSections: ['model.low_vram', 'model.layer_offloading'], }, { name: 'ideogram4', @@ -1208,9 +1329,7 @@ export const modelArchs: ModelArch[] = [ undefined, ], }, - disableSections: [ - 'network.conv', - ], + disableSections: ['network.conv'], additionalSections: [ 'model.low_vram', 'model.layer_offloading', @@ -1233,13 +1352,8 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].network.conv_alpha': [undefined, 16], 'config.process[0].model.low_vram': [true, false], }, - disableSections: [ - 'network.conv', - ], - additionalSections: [ - 'model.low_vram', - 'model.layer_offloading', - ], + disableSections: ['network.conv'], + additionalSections: ['model.low_vram', 'model.layer_offloading'], }, { name: 'krea2', @@ -1255,13 +1369,8 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].network.conv_alpha': [undefined, 16], 'config.process[0].model.low_vram': [true, false], }, - disableSections: [ - 'network.conv', - ], - additionalSections: [ - 'model.low_vram', - 'model.layer_offloading', - ], + disableSections: ['network.conv'], + additionalSections: ['model.low_vram', 'model.layer_offloading'], }, { name: 'krea2:turbo', @@ -1283,14 +1392,8 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].sample.guidance_scale': [1, 4], 'config.process[0].sample.sample_steps': [9, 25], }, - disableSections: [ - 'network.conv', - ], - additionalSections: [ - 'model.low_vram', - 'model.layer_offloading', - 'model.assistant_lora_path', - ], + disableSections: ['network.conv'], + additionalSections: ['model.low_vram', 'model.layer_offloading', 'model.assistant_lora_path'], }, { name: 'krea2:o_edit', @@ -1315,9 +1418,7 @@ export const modelArchs: ModelArch[] = [ {}, ], }, - disableSections: [ - 'network.conv', 'train.unload_text_encoder' - ], + disableSections: ['network.conv', 'train.unload_text_encoder'], additionalSections: [ 'datasets.multi_control_paths', 'sample.multi_ctrl_imgs', @@ -1356,9 +1457,7 @@ export const modelArchs: ModelArch[] = [ {}, ], }, - disableSections: [ - 'network.conv', 'train.unload_text_encoder' - ], + disableSections: ['network.conv', 'train.unload_text_encoder'], additionalSections: [ 'datasets.multi_control_paths', 'sample.multi_ctrl_imgs', @@ -1384,13 +1483,8 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].sample.guidance_scale': [4, 4], 'config.process[0].sample.sample_steps': [25, 25], }, - disableSections: [ - 'network.conv', - ], - additionalSections: [ - 'model.low_vram', - 'model.layer_offloading', - ], + disableSections: ['network.conv'], + additionalSections: ['model.low_vram', 'model.layer_offloading'], }, { name: 'mageflow_edit', @@ -1408,9 +1502,7 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].sample.sample_steps': [25, 25], 'config.process[0].train.unload_text_encoder': [false, false], }, - disableSections: [ - 'network.conv', 'train.unload_text_encoder', - ], + disableSections: ['network.conv', 'train.unload_text_encoder'], additionalSections: [ 'datasets.multi_control_paths', 'sample.multi_ctrl_imgs', @@ -1431,13 +1523,8 @@ export const modelArchs: ModelArch[] = [ 'config.process[0].network.conv_alpha': [undefined, 16], 'config.process[0].model.low_vram': [true, false], }, - disableSections: [ - 'network.conv', - ], - additionalSections: [ - 'model.low_vram', - 'model.layer_offloading', - ], + disableSections: ['network.conv'], + additionalSections: ['model.low_vram', 'model.layer_offloading'], }, { name: 'boogu_image_edit', @@ -1459,9 +1546,7 @@ export const modelArchs: ModelArch[] = [ {}, ], }, - disableSections: [ - 'network.conv', 'train.unload_text_encoder', - ], + disableSections: ['network.conv', 'train.unload_text_encoder'], additionalSections: [ 'datasets.multi_control_paths', 'sample.multi_ctrl_imgs',