Add support for MiniMax H3 Ref2Vid training

This commit is contained in:
Jaret Burkett
2026-08-13 12:36:03 -06:00
parent ab18528fdb
commit 6ea281973d
6 changed files with 574 additions and 229 deletions

View File

@@ -45,14 +45,14 @@ export interface CustomModelSelectOption {
options: SelectOption[];
getValue: (config: JobConfig) => string | undefined;
onChange: (value: string, config: JobConfig, setJobConfig: (value: any, key: string) => void) => void;
doc?: ConfigDoc
doc?: ConfigDoc;
}
export type SampleTag = {
title: string;
type: 'text' | 'multiline' | 'number'
type: 'text' | 'multiline' | 'number';
full?: boolean;
}
};
export interface SampleTags {
[key: string]: SampleTag;
@@ -78,6 +78,64 @@ export interface ModelArch {
const defaultNameOrPath = '';
const defaultLinearRank = 32;
// used by the MiniMax-H3 fl2va arch (ref2va is contrastive-guidance only)
const minimaxH3DistillationHandling = {
label: 'Distillation Handling Method',
options: [
{ value: 'cg', label: 'Contrastive Guidance (default)' },
{ value: 'ta', label: 'Training Adapter' },
{ value: 'both', label: 'Contrastive Guidance + Training Adapter' },
],
getValue: (config: JobConfig) => {
const assistantLoraPath = config?.config?.process?.[0]?.model?.assistant_lora_path;
const hasAssistantLoraPath = assistantLoraPath && assistantLoraPath.trim() !== '';
const hasContrastiveGuidance = config?.config?.process?.[0]?.train?.do_guidance_loss;
if (hasAssistantLoraPath && hasContrastiveGuidance) {
return 'both';
}
if (hasAssistantLoraPath) {
return 'ta';
}
return 'cg';
},
onChange: (value: string, config: JobConfig, setJobConfig: (value: any, key: string) => void) => {
if (value === 'cg') {
setJobConfig(true, 'config.process[0].train.do_guidance_loss');
setJobConfig(undefined, 'config.process[0].model.assistant_lora_path');
if (!config?.config?.process?.[0]?.train?.guidance_loss_target) {
setJobConfig(3.5, 'config.process[0].train.guidance_loss_target');
}
} else if (value === 'ta') {
setJobConfig(undefined, 'config.process[0].train.do_guidance_loss');
setJobConfig(undefined, 'config.process[0].train.guidance_loss_target');
setJobConfig(
'ostris/minimax_h3_training_adapter/minimax_h3_training_adapter_v1.safetensors',
'config.process[0].model.assistant_lora_path',
);
} else if (value === 'both') {
setJobConfig(true, 'config.process[0].train.do_guidance_loss');
setJobConfig(
'ostris/minimax_h3_training_adapter/minimax_h3_training_adapter_v1.safetensors',
'config.process[0].model.assistant_lora_path',
);
if (!config?.config?.process?.[0]?.train?.guidance_loss_target) {
setJobConfig(3.5, 'config.process[0].train.guidance_loss_target');
}
}
},
doc: {
title: 'MiniMax-H3 Distillation Handling',
description: (
<div>
MiniMax H3 is a guidance distilled model, so training on it directly will make the guidance distillation break
down. There are two different ways to train on this model without breaking the guidance distillation:
Contrastive Guidance and Training Adapter. Both have their pros and cons. The adapter is faster, but will still
break down over a long run. Contrastive Guidance is slower, but is less likely to break down.
</div>
),
},
};
export const modelArchs: ModelArch[] = [
{
name: 'anima',
@@ -196,7 +254,10 @@ export const modelArchs: ModelArch[] = [
group: 'experimental',
defaults: {
// default updates when [selected, unselected] in the UI
'config.process[0].model.name_or_path': ['lodestones/Zeta-Chroma/zeta-chroma-base-x0-pixel-dino-distance.safetensors', defaultNameOrPath],
'config.process[0].model.name_or_path': [
'lodestones/Zeta-Chroma/zeta-chroma-base-x0-pixel-dino-distance.safetensors',
defaultNameOrPath,
],
'config.process[0].model.extras_name_or_path': ['Tongyi-MAI/Z-Image-Turbo', undefined],
'config.process[0].model.quantize': [true, false],
'config.process[0].model.quantize_te': [true, false],
@@ -309,7 +370,13 @@ export const modelArchs: ModelArch[] = [
],
},
disableSections: ['network.conv'],
additionalSections: ['datasets.num_frames', 'model.low_vram', 'model.multistage', 'model.layer_offloading', 'datasets.auto_frame_count'],
additionalSections: [
'datasets.num_frames',
'model.low_vram',
'model.multistage',
'model.layer_offloading',
'datasets.auto_frame_count',
],
accuracyRecoveryAdapters: {
// '3 bit with ARA': 'uint3|ostris/accuracy_recovery_adapters/wan22_14b_t2i_torchao_uint3.safetensors',
'4 bit with ARA': 'uint4|ostris/accuracy_recovery_adapters/wan22_14b_t2i_torchao_uint4.safetensors',
@@ -375,7 +442,13 @@ export const modelArchs: ModelArch[] = [
'config.process[0].datasets[x].fps': [24, undefined],
},
disableSections: ['network.conv'],
additionalSections: ['sample.ctrl_img', 'datasets.num_frames', 'model.low_vram', 'datasets.do_i2v', 'datasets.auto_frame_count'],
additionalSections: [
'sample.ctrl_img',
'datasets.num_frames',
'model.low_vram',
'datasets.do_i2v',
'datasets.auto_frame_count',
],
},
{
name: 'lumina2',
@@ -747,59 +820,20 @@ export const modelArchs: ModelArch[] = [
'config.process[0].datasets[x].auto_frame_count': [true, undefined],
},
disableSections: ['network.conv'],
additionalSections: ['sample.ctrl_img', 'datasets.num_frames', 'model.layer_offloading', 'model.low_vram', 'datasets.do_audio', 'datasets.audio_normalize', 'datasets.audio_preserve_pitch', 'datasets.do_i2v', 'train.audio_loss_multiplier', 'datasets.auto_frame_count', 'model.assistant_lora_path'],
customModelSelectOptions: [
{
label: 'Distillation Handling Method',
options: [
{ value: 'cg', label: 'Contrastive Guidance (default)' },
{ value: 'ta', label: 'Training Adapter' },
{ value: 'both', label: 'Contrastive Guidance + Training Adapter' },
],
getValue: (config: JobConfig) => {
const assistantLoraPath = config?.config?.process?.[0]?.model?.assistant_lora_path;
const hasAssistantLoraPath = assistantLoraPath && assistantLoraPath.trim() !== '';
const hasContrastiveGuidance = config?.config?.process?.[0]?.train?.do_guidance_loss;
if (hasAssistantLoraPath && hasContrastiveGuidance) {
return 'both';
}
if (hasAssistantLoraPath) {
return 'ta';
}
return 'cg';
},
onChange: (value: string, config: JobConfig, setJobConfig: (value: any, key: string) => void) => {
if (value === 'cg') {
setJobConfig(true, 'config.process[0].train.do_guidance_loss');
setJobConfig(undefined, 'config.process[0].model.assistant_lora_path');
if (!(config?.config?.process?.[0]?.train?.guidance_loss_target)) {
setJobConfig(3.5, 'config.process[0].train.guidance_loss_target');
}
} else if (value === 'ta') {
setJobConfig(undefined, 'config.process[0].train.do_guidance_loss');
setJobConfig(undefined, 'config.process[0].train.guidance_loss_target');
setJobConfig("ostris/minimax_h3_training_adapter/minimax_h3_training_adapter_v1.safetensors", 'config.process[0].model.assistant_lora_path');
} else if (value === 'both') {
setJobConfig(true, 'config.process[0].train.do_guidance_loss');
setJobConfig("ostris/minimax_h3_training_adapter/minimax_h3_training_adapter_v1.safetensors", 'config.process[0].model.assistant_lora_path');
if (!(config?.config?.process?.[0]?.train?.guidance_loss_target)) {
setJobConfig(3.5, 'config.process[0].train.guidance_loss_target');
}
}
},
doc: {
title: 'MiniMax-H3 Distillation Handling',
description: (
<div>
MiniMax H3 is a guidance distilled model, so training on it directly will make the guidance distillation break down.
There are two different ways to train on this model without breaking the guidance distillation:
Contrastive Guidance and Training Adapter. Both have their pros and cons. The adapter is faster, but will still
break down over a long run. Contrastive Guidance is slower, but is less likely to break down.
</div>
)
}
},
additionalSections: [
'sample.ctrl_img',
'datasets.num_frames',
'model.layer_offloading',
'model.low_vram',
'datasets.do_audio',
'datasets.audio_normalize',
'datasets.audio_preserve_pitch',
'datasets.do_i2v',
'train.audio_loss_multiplier',
'datasets.auto_frame_count',
'model.assistant_lora_path',
],
customModelSelectOptions: [minimaxH3DistillationHandling],
modelNotes: (
<div className="space-y-2">
<p>
@@ -835,6 +869,77 @@ export const modelArchs: ModelArch[] = [
</div>
),
},
{
name: 'minimax_h3_ref2va',
label: 'MiniMax-H3 Ref2V',
group: 'video',
isVideoModel: true,
defaults: {
// default updates when [selected, unselected] in the UI
'config.process[0].model.name_or_path': ['Comfy-Org/MiniMax-H3', defaultNameOrPath],
// pre-quantized weights: matching qtypes keep the load unchanged
'config.process[0].model.quantize': [true, false],
'config.process[0].model.qtype': ['convrot8', 'qfloat8'],
'config.process[0].model.quantize_te': [true, false],
'config.process[0].model.qtype_te': ['nvfp4', 'qfloat8'],
'config.process[0].model.low_vram': [true, false],
'config.process[0].sample.sampler': ['flowmatch', 'flowmatch'],
'config.process[0].train.noise_scheduler': ['flowmatch', 'flowmatch'],
'config.process[0].train.cache_text_embeddings': [true, false],
'config.process[0].train.do_guidance_loss': [true, undefined],
'config.process[0].train.guidance_loss_target': [3.5, undefined],
'config.process[0].network.linear': [16, defaultLinearRank],
'config.process[0].network.linear_alpha': [16, defaultLinearRank],
'config.process[0].network.network_kwargs.ignore_if_contains': [['adaln_proj'], []],
'config.process[0].sample.num_frames': [107, 1],
'config.process[0].sample.fps': [24, 1],
'config.process[0].sample.width': [768, 1024],
'config.process[0].sample.height': [768, 1024],
'config.process[0].sample.guidance_scale': [1, 4],
'config.process[0].sample.sample_steps': [28, 25],
'config.process[0].train.audio_loss_multiplier': [1.0, undefined],
'config.process[0].train.timestep_type': ['shift', 'sigmoid'],
'config.process[0].datasets[x].do_audio': [true, undefined],
'config.process[0].datasets[x].cache_latents_to_disk': [true, false],
'config.process[0].datasets[x].fps': [24, undefined],
'config.process[0].datasets[x].num_frames': [39, undefined],
'config.process[0].datasets[x].auto_frame_count': [true, undefined],
},
disableSections: ['network.conv'],
additionalSections: [
'sample.multi_ctrl_imgs',
'datasets.multi_control_paths',
'datasets.num_frames',
'model.layer_offloading',
'model.low_vram',
'datasets.do_audio',
'datasets.audio_normalize',
'datasets.audio_preserve_pitch',
'train.audio_loss_multiplier',
'datasets.auto_frame_count',
],
modelNotes: (
<div className="space-y-2">
<p>
Reference-to-video: control images condition the output as subject/style references (never as a first frame).
Each reference keeps its own aspect ratio, is resized to the target's pixel area, rides into the packed
sequence as a reference block, and is also shown to the Qwen3-VL conditioner as a{' '}
<code>&lt;Picture i&gt;</code> vision block. Training references come from the dataset control path(s);
sampling uses the sample ctrl images — always as references. Image references only for now (no reference
video/audio).
</p>
<p>
Weights load like MiniMax-H3 (see that arch's notes) from the{' '}
<Link href="/settings" className="text-blue-400 hover:underline">
Models Folder Path
</Link>
, using <code>diffusion_models/minimax_h3_ref2va_pruned_int8_convrot.safetensors</code> — the ref2va partition
of the same release; text encoder and VAEs are shared with the fl2va arch. Everything else (pre-quantized
load, 24 fps, 17n+5 frame grid, guidance scale 1, single-image mode) matches MiniMax-H3.
</p>
</div>
),
},
{
name: 'ltx2',
label: 'LTX-2',
@@ -860,7 +965,18 @@ export const modelArchs: ModelArch[] = [
'config.process[0].datasets[x].auto_frame_count': [false, undefined],
},
disableSections: ['network.conv'],
additionalSections: ['sample.ctrl_img', 'datasets.num_frames', 'model.layer_offloading', 'model.low_vram', 'datasets.do_audio', 'datasets.audio_normalize', 'datasets.audio_preserve_pitch', 'datasets.do_i2v', 'train.audio_loss_multiplier', 'datasets.auto_frame_count'],
additionalSections: [
'sample.ctrl_img',
'datasets.num_frames',
'model.layer_offloading',
'model.low_vram',
'datasets.do_audio',
'datasets.audio_normalize',
'datasets.audio_preserve_pitch',
'datasets.do_i2v',
'train.audio_loss_multiplier',
'datasets.auto_frame_count',
],
},
{
name: 'ltx2.3',
@@ -888,7 +1004,18 @@ export const modelArchs: ModelArch[] = [
'config.process[0].datasets[x].auto_frame_count': [false, undefined],
},
disableSections: ['network.conv'],
additionalSections: ['sample.ctrl_img', 'datasets.num_frames', 'model.layer_offloading', 'model.low_vram', 'datasets.do_audio', 'datasets.audio_normalize', 'datasets.audio_preserve_pitch', 'datasets.do_i2v', 'train.audio_loss_multiplier', 'datasets.auto_frame_count'],
additionalSections: [
'sample.ctrl_img',
'datasets.num_frames',
'model.layer_offloading',
'model.low_vram',
'datasets.do_audio',
'datasets.audio_normalize',
'datasets.audio_preserve_pitch',
'datasets.do_i2v',
'train.audio_loss_multiplier',
'datasets.auto_frame_count',
],
},
{
name: 'ltx2.5',
@@ -921,7 +1048,18 @@ export const modelArchs: ModelArch[] = [
'config.process[0].datasets[x].auto_frame_count': [false, undefined],
},
disableSections: ['network.conv'],
additionalSections: ['sample.ctrl_img', 'datasets.num_frames', 'model.layer_offloading', 'model.low_vram', 'datasets.do_audio', 'datasets.audio_normalize', 'datasets.audio_preserve_pitch', 'datasets.do_i2v', 'train.audio_loss_multiplier', 'datasets.auto_frame_count'],
additionalSections: [
'sample.ctrl_img',
'datasets.num_frames',
'model.layer_offloading',
'model.low_vram',
'datasets.do_audio',
'datasets.audio_normalize',
'datasets.audio_preserve_pitch',
'datasets.do_i2v',
'train.audio_loss_multiplier',
'datasets.auto_frame_count',
],
},
{
name: 'flux2_klein_4b',
@@ -971,10 +1109,7 @@ export const modelArchs: ModelArch[] = [
'config.process[0].model.qtype': ['qfloat8', 'qfloat8'],
},
disableSections: ['network.conv'],
additionalSections: [
'model.low_vram',
'model.layer_offloading',
],
additionalSections: ['model.low_vram', 'model.layer_offloading'],
},
{
name: 'flux2_klein_9b',
@@ -1014,7 +1149,10 @@ export const modelArchs: ModelArch[] = [
group: 'audio',
defaults: {
// default updates when [selected, unselected] in the UI
'config.process[0].model.name_or_path': ['ostris/ace_step_1.5_ComfyUI_files/ace_step_1.5_xl_base_aio.safetensors', defaultNameOrPath],
'config.process[0].model.name_or_path': [
'ostris/ace_step_1.5_ComfyUI_files/ace_step_1.5_xl_base_aio.safetensors',
defaultNameOrPath,
],
'config.process[0].model.quantize': [true, false],
'config.process[0].model.quantize_te': [true, false],
'config.process[0].model.low_vram': [true, false],
@@ -1025,43 +1163,39 @@ export const modelArchs: ModelArch[] = [
'config.process[0].sample': [defaultAudioSampleConfig, defaultSampleConfig],
},
sampleTags: {
"CAPTION": {
title: "Audio Prompt",
type: "text",
CAPTION: {
title: 'Audio Prompt',
type: 'text',
full: true,
},
"LYRICS": {
title: "Lyrics",
type: "multiline",
LYRICS: {
title: 'Lyrics',
type: 'multiline',
full: true,
},
"BPM": {
title: "BPM",
type: "number",
BPM: {
title: 'BPM',
type: 'number',
},
"KEYSCALE": {
title: "Key Scale",
type: "text",
KEYSCALE: {
title: 'Key Scale',
type: 'text',
},
"TIMESIGNATURE": {
title: "Time Signature",
type: "text",
TIMESIGNATURE: {
title: 'Time Signature',
type: 'text',
},
"DURATION": {
title: "Duration (sec)",
type: "number",
DURATION: {
title: 'Duration (sec)',
type: 'number',
},
"LANGUAGE": {
title: "Language",
type: "text",
LANGUAGE: {
title: 'Language',
type: 'text',
},
},
disableSections: ['network.conv'],
additionalSections: [
'sample.multi_ctrl_imgs',
'model.low_vram',
'model.layer_offloading',
],
additionalSections: ['sample.multi_ctrl_imgs', 'model.low_vram', 'model.layer_offloading'],
},
{
name: 'ace_step_15',
@@ -1069,7 +1203,10 @@ export const modelArchs: ModelArch[] = [
group: 'audio',
defaults: {
// default updates when [selected, unselected] in the UI
'config.process[0].model.name_or_path': ['ostris/ace_step_1.5_ComfyUI_files/ace_step_1.5_base_aio.safetensors', defaultNameOrPath],
'config.process[0].model.name_or_path': [
'ostris/ace_step_1.5_ComfyUI_files/ace_step_1.5_base_aio.safetensors',
defaultNameOrPath,
],
'config.process[0].model.quantize': [true, false],
'config.process[0].model.quantize_te': [true, false],
'config.process[0].model.low_vram': [true, false],
@@ -1080,43 +1217,39 @@ export const modelArchs: ModelArch[] = [
'config.process[0].sample': [defaultAudioSampleConfig, defaultSampleConfig],
},
sampleTags: {
"CAPTION": {
title: "Audio Prompt",
type: "text",
CAPTION: {
title: 'Audio Prompt',
type: 'text',
full: true,
},
"LYRICS": {
title: "Lyrics",
type: "multiline",
LYRICS: {
title: 'Lyrics',
type: 'multiline',
full: true,
},
"BPM": {
title: "BPM",
type: "number",
BPM: {
title: 'BPM',
type: 'number',
},
"KEYSCALE": {
title: "Key Scale",
type: "text",
KEYSCALE: {
title: 'Key Scale',
type: 'text',
},
"TIMESIGNATURE": {
title: "Time Signature",
type: "text",
TIMESIGNATURE: {
title: 'Time Signature',
type: 'text',
},
"DURATION": {
title: "Duration (sec)",
type: "number",
DURATION: {
title: 'Duration (sec)',
type: 'number',
},
"LANGUAGE": {
title: "Language",
type: "text",
LANGUAGE: {
title: 'Language',
type: 'text',
},
},
disableSections: ['network.conv'],
additionalSections: [
'sample.multi_ctrl_imgs',
'model.low_vram',
'model.layer_offloading',
],
additionalSections: ['sample.multi_ctrl_imgs', 'model.low_vram', 'model.layer_offloading'],
},
{
name: 'nucleus_image',
@@ -1158,15 +1291,8 @@ export const modelArchs: ModelArch[] = [
{},
],
},
disableSections: [
'network.conv',
'model.quantize_te',
'train.unload_text_encoder',
],
additionalSections: [
'model.low_vram',
'model.layer_offloading',
],
disableSections: ['network.conv', 'model.quantize_te', 'train.unload_text_encoder'],
additionalSections: ['model.low_vram', 'model.layer_offloading'],
},
{
name: 'zimage_l2p',
@@ -1182,13 +1308,8 @@ export const modelArchs: ModelArch[] = [
'config.process[0].network.conv_alpha': [undefined, 16],
'config.process[0].model.low_vram': [true, false],
},
disableSections: [
'network.conv',
],
additionalSections: [
'model.low_vram',
'model.layer_offloading',
],
disableSections: ['network.conv'],
additionalSections: ['model.low_vram', 'model.layer_offloading'],
},
{
name: 'ideogram4',
@@ -1208,9 +1329,7 @@ export const modelArchs: ModelArch[] = [
undefined,
],
},
disableSections: [
'network.conv',
],
disableSections: ['network.conv'],
additionalSections: [
'model.low_vram',
'model.layer_offloading',
@@ -1233,13 +1352,8 @@ export const modelArchs: ModelArch[] = [
'config.process[0].network.conv_alpha': [undefined, 16],
'config.process[0].model.low_vram': [true, false],
},
disableSections: [
'network.conv',
],
additionalSections: [
'model.low_vram',
'model.layer_offloading',
],
disableSections: ['network.conv'],
additionalSections: ['model.low_vram', 'model.layer_offloading'],
},
{
name: 'krea2',
@@ -1255,13 +1369,8 @@ export const modelArchs: ModelArch[] = [
'config.process[0].network.conv_alpha': [undefined, 16],
'config.process[0].model.low_vram': [true, false],
},
disableSections: [
'network.conv',
],
additionalSections: [
'model.low_vram',
'model.layer_offloading',
],
disableSections: ['network.conv'],
additionalSections: ['model.low_vram', 'model.layer_offloading'],
},
{
name: 'krea2:turbo',
@@ -1283,14 +1392,8 @@ export const modelArchs: ModelArch[] = [
'config.process[0].sample.guidance_scale': [1, 4],
'config.process[0].sample.sample_steps': [9, 25],
},
disableSections: [
'network.conv',
],
additionalSections: [
'model.low_vram',
'model.layer_offloading',
'model.assistant_lora_path',
],
disableSections: ['network.conv'],
additionalSections: ['model.low_vram', 'model.layer_offloading', 'model.assistant_lora_path'],
},
{
name: 'krea2:o_edit',
@@ -1315,9 +1418,7 @@ export const modelArchs: ModelArch[] = [
{},
],
},
disableSections: [
'network.conv', 'train.unload_text_encoder'
],
disableSections: ['network.conv', 'train.unload_text_encoder'],
additionalSections: [
'datasets.multi_control_paths',
'sample.multi_ctrl_imgs',
@@ -1356,9 +1457,7 @@ export const modelArchs: ModelArch[] = [
{},
],
},
disableSections: [
'network.conv', 'train.unload_text_encoder'
],
disableSections: ['network.conv', 'train.unload_text_encoder'],
additionalSections: [
'datasets.multi_control_paths',
'sample.multi_ctrl_imgs',
@@ -1384,13 +1483,8 @@ export const modelArchs: ModelArch[] = [
'config.process[0].sample.guidance_scale': [4, 4],
'config.process[0].sample.sample_steps': [25, 25],
},
disableSections: [
'network.conv',
],
additionalSections: [
'model.low_vram',
'model.layer_offloading',
],
disableSections: ['network.conv'],
additionalSections: ['model.low_vram', 'model.layer_offloading'],
},
{
name: 'mageflow_edit',
@@ -1408,9 +1502,7 @@ export const modelArchs: ModelArch[] = [
'config.process[0].sample.sample_steps': [25, 25],
'config.process[0].train.unload_text_encoder': [false, false],
},
disableSections: [
'network.conv', 'train.unload_text_encoder',
],
disableSections: ['network.conv', 'train.unload_text_encoder'],
additionalSections: [
'datasets.multi_control_paths',
'sample.multi_ctrl_imgs',
@@ -1431,13 +1523,8 @@ export const modelArchs: ModelArch[] = [
'config.process[0].network.conv_alpha': [undefined, 16],
'config.process[0].model.low_vram': [true, false],
},
disableSections: [
'network.conv',
],
additionalSections: [
'model.low_vram',
'model.layer_offloading',
],
disableSections: ['network.conv'],
additionalSections: ['model.low_vram', 'model.layer_offloading'],
},
{
name: 'boogu_image_edit',
@@ -1459,9 +1546,7 @@ export const modelArchs: ModelArch[] = [
{},
],
},
disableSections: [
'network.conv', 'train.unload_text_encoder',
],
disableSections: ['network.conv', 'train.unload_text_encoder'],
additionalSections: [
'datasets.multi_control_paths',
'sample.multi_ctrl_imgs',