Add support for MiniMax H3 T2V and I2V training

2026-08-03 10:17:39 -06:00
parent 73cab2acf5
commit 8502a845b1
28 changed files with 4095 additions and 41 deletions
--- a/ui/src/app/api/settings/route.ts
+++ b/ui/src/app/api/settings/route.ts
@@ -1,6 +1,6 @@
 import { NextResponse } from 'next/server';
 import prisma from '@/server/prisma';
-import { defaultTrainFolder, defaultDatasetsFolder } from '@/paths';
+import { defaultTrainFolder, defaultDatasetsFolder, defaultModelsFolder } from '@/paths';
 import { flushCache } from '@/server/settings';

 export async function GET() {
@@ -18,6 +18,13 @@ export async function GET() {
    if (!settingsObject.DATASETS_FOLDER || settingsObject.DATASETS_FOLDER === '') {
      settingsObject.DATASETS_FOLDER = defaultDatasetsFolder;
    }
+    // MODELS_PATH from the env file always takes precedence over the setting
+    if (process.env.MODELS_PATH && process.env.MODELS_PATH.trim() !== '') {
+      settingsObject.MODELS_PATH = process.env.MODELS_PATH;
+    } else if (!settingsObject.MODELS_PATH || settingsObject.MODELS_PATH === '') {
+      // if MODELS_PATH is not set, use default
+      settingsObject.MODELS_PATH = defaultModelsFolder;
+    }
    return NextResponse.json(settingsObject);
  } catch (error) {
    return NextResponse.json({ error: 'Failed to fetch settings' }, { status: 500 });
@@ -27,7 +34,7 @@ export async function GET() {
 export async function POST(request: Request) {
  try {
    const body = await request.json();
-    const { HF_TOKEN, TRAINING_FOLDER, DATASETS_FOLDER } = body;
+    const { HF_TOKEN, TRAINING_FOLDER, DATASETS_FOLDER, MODELS_PATH } = body;

    // Upsert both settings
    await Promise.all([
@@ -46,6 +53,11 @@ export async function POST(request: Request) {
        update: { value: DATASETS_FOLDER },
        create: { key: 'DATASETS_FOLDER', value: DATASETS_FOLDER },
      }),
+      prisma.settings.upsert({
+        where: { key: 'MODELS_PATH' },
+        update: { value: MODELS_PATH },
+        create: { key: 'MODELS_PATH', value: MODELS_PATH },
+      }),
    ]);

    flushCache();
--- a/ui/src/app/jobs/new/SimpleJob.tsx
+++ b/ui/src/app/jobs/new/SimpleJob.tsx
@@ -324,6 +324,30 @@ export default function SimpleJob({
                placeholder=""
              />
            )}
+            {modelArch?.modelNotes && (
+              <div className="pt-2">
+                <button
+                  type="button"
+                  onClick={() => {
+                    const gateUrl = modelArch.gateUrl as string;
+                    openDoc({
+                      title: `Notes - ${modelArch.label}`,
+                      description: (
+                        <div className="space-y-3">
+                          {modelArch.modelNotes}
+                        </div>
+                      ),
+                    });
+                  }}
+                  className="w-full flex items-center gap-2 rounded-md bg-blue-950/60 border border-blue-800 px-3 py-2 text-sm text-blue-200 hover:bg-blue-900/60 text-left"
+                >
+                  <Info className="w-4 h-4 shrink-0 text-blue-400" />
+                  <span>
+                    Model notes
+                  </span>
+                </button>
+              </div>
+            )}
            {modelArch?.gateUrl && (
              <div className="pt-2">
                <button
@@ -368,9 +392,9 @@ export default function SimpleJob({
                      ),
                    });
                  }}
-                  className="w-full flex items-center gap-2 rounded-md bg-blue-950/60 border border-blue-800 px-3 py-2 text-sm text-blue-200 hover:bg-blue-900/60 text-left"
+                  className="w-full flex items-center gap-2 rounded-md bg-yellow-950/60 border border-yellow-800 px-3 py-2 text-sm text-yellow-200 hover:bg-yellow-900/60 text-left"
                >
-                  <Info className="w-4 h-4 shrink-0 text-blue-400" />
+                  <Info className="w-4 h-4 shrink-0 text-yellow-400" />
                  <span>
                    Gated model. <span className="underline">Learn more.</span>
                  </span>
--- a/ui/src/app/jobs/new/options.tsx
+++ b/ui/src/app/jobs/new/options.tsx
@@ -1,3 +1,5 @@
+import React from 'react';
+import Link from 'next/link';
 import { GroupedSelectOption, SelectOption, JobConfig } from '@/types';
 import { defaultSliderConfig } from './jobConfig';
 import { defaultAudioSampleConfig, defaultSampleConfig, defaultIdeogramSamplesConfig } from '@/helpers/defaultSamples';
@@ -61,6 +63,7 @@ export interface ModelArch {
  accuracyRecoveryAdapters?: { [key: string]: string };
  sampleTags?: SampleTags;
  gateUrl?: string;
+  modelNotes?: React.ReactNode;
 }

 const defaultNameOrPath = '';
@@ -695,6 +698,77 @@ export const modelArchs: ModelArch[] = [
    disableSections: ['network.conv'],
    additionalSections: ['model.low_vram', 'model.layer_offloading'],
  },
+  {
+    name: 'minimax_h3',
+    label: 'MiniMax-H3',
+    group: 'video',
+    isVideoModel: true,
+    defaults: {
+      // default updates when [selected, unselected] in the UI
+      'config.process[0].model.name_or_path': ['Comfy-Org/MiniMax-H3', defaultNameOrPath],
+      // the Comfy-Org weights are pre-quantized (int8 convrot DiT, nvfp4 TE); these
+      // qtypes match the checkpoints exactly, so the load is unchanged. Picking a
+      // different qtype re-quantizes layer by layer into that format.
+      'config.process[0].model.quantize': [true, false],
+      'config.process[0].model.qtype': ['convrot8', 'qfloat8'],
+      'config.process[0].model.quantize_te': [true, false],
+      'config.process[0].model.qtype_te': ['nvfp4', 'qfloat8'],
+      'config.process[0].model.low_vram': [true, false],
+      'config.process[0].sample.sampler': ['flowmatch', 'flowmatch'],
+      'config.process[0].train.noise_scheduler': ['flowmatch', 'flowmatch'],
+      'config.process[0].train.cache_text_embeddings': [true, false],
+      'config.process[0].network.linear': [16, defaultLinearRank],
+      'config.process[0].sample.num_frames': [107, 1],
+      'config.process[0].sample.fps': [24, 1],
+      'config.process[0].sample.width': [768, 1024],
+      'config.process[0].sample.height': [768, 1024],
+      'config.process[0].sample.guidance_scale': [1, 4],
+      'config.process[0].sample.sample_steps': [28, 25],
+      'config.process[0].train.audio_loss_multiplier': [1.0, undefined],
+      'config.process[0].train.timestep_type': ['shift', 'sigmoid'],
+      'config.process[0].datasets[x].do_i2v': [false, undefined],
+      'config.process[0].datasets[x].do_audio': [true, undefined],
+      'config.process[0].datasets[x].cache_latents_to_disk': [true, false],
+      'config.process[0].datasets[x].fps': [24, undefined],
+      'config.process[0].datasets[x].num_frames': [39, undefined],
+      'config.process[0].datasets[x].auto_frame_count': [true, undefined],
+    },
+    disableSections: ['network.conv'],
+    additionalSections: ['sample.ctrl_img', 'datasets.num_frames', 'model.layer_offloading', 'model.low_vram', 'datasets.do_audio', 'datasets.audio_normalize', 'datasets.audio_preserve_pitch', 'datasets.do_i2v', 'train.audio_loss_multiplier', 'datasets.auto_frame_count'],
+    modelNotes: (
+      <div className="space-y-2">
+        <p>
+          Weights load from the{' '}
+          <Link href="/settings" className="text-blue-400 hover:underline">
+            Models Folder Path
+          </Link>{' '}
+          set in settings. Anything missing is downloaded there from <code>Comfy-Org/MiniMax-H3</code> on first load
+          (~43GB total). Files used:
+        </p>
+        <pre className="bg-gray-900 border border-gray-700 rounded-lg p-3 text-xs overflow-x-auto">
+          <code>{`<MODELS_PATH>/
+├── diffusion_models/
+│   └── minimax_h3_fl2va_pruned_int8_convrot.safetensors
+├── text_encoders/
+│   └── qwen3vl_32b_minimax_h3_nvfp4_awq.safetensors
+└── vae/
+    ├── minimax_h3_video_vae_fp16.safetensors
+    └── minimax_h3_audio_vae_fp32.safetensors`}</code>
+        </pre>
+        <p>
+          The checkpoints are pre-quantized and load directly: int8 ConvRot DiT (~21GB) and nvfp4 Qwen3-VL text encoder
+          (~16GB). The default qtypes (<code>convrot8</code> / <code>nvfp4</code>) match the files exactly, so nothing
+          is re-quantized on load. Picking a different quantization re-quantizes the pre-quantized layers into that
+          format, one layer at a time.
+        </p>
+        <p>
+          Supports t2v and first-frame i2v (ctrl img / i2v datasets) with joint audio. The model is guidance-distilled —
+          keep guidance scale at 1. Video is fixed 24 fps and frame counts snap down to the 17n+5 grid (5, 22, 39, 56,
+          ..., 107, 124 ≈ 5s).
+        </p>
+      </div>
+    ),
+  },
  {
    name: 'ltx2',
    label: 'LTX-2',
@@ -1321,6 +1395,7 @@ export const quantizationOptions: SelectOption[] = [
  { value: 'float8', label: 'float8' },
  { value: 'convrot8', label: '8bit convrot' },
  { value: 'convrot4', label: '4bit convrot (nvfp4)' },
+  { value: 'nvfp4', label: 'nvfp4 (4bit weight only)' },
  { value: 'convrotint7', label: '7bit convrot' },
  { value: 'convrotint6', label: '6bit convrot' },
  { value: 'convrotint5', label: '5bit convrot' },
--- a/ui/src/app/settings/page.tsx
+++ b/ui/src/app/settings/page.tsx
@@ -108,6 +108,26 @@ export default function Settings() {
                    placeholder="Enter datasets folder path"
                  />
                </div>
+
+                <div>
+                  <label htmlFor="MODELS_PATH" className="block text-sm font-medium mb-2">
+                    Models Folder Path
+                    <div className="text-gray-500 text-sm ml-1">
+                      Some models support loading ComfyUI model weights directly. Models that do will be loaded
+                      from/downloaded to this path. Must be an absolute path. If blank, it will default to the models
+                      folder in the project root.
+                    </div>
+                  </label>
+                  <input
+                    type="text"
+                    id="MODELS_PATH"
+                    name="MODELS_PATH"
+                    value={settings.MODELS_PATH}
+                    onChange={handleChange}
+                    className="w-full px-4 py-2 bg-gray-800 border border-gray-700 rounded-lg focus:ring-2 focus:ring-gray-600 focus:border-transparent"
+                    placeholder="Enter models folder path"
+                  />
+                </div>
              </div>
            </div>
          </div>
--- a/ui/src/hooks/useSettings.tsx
+++ b/ui/src/hooks/useSettings.tsx
@@ -7,6 +7,7 @@ export interface Settings {
  HF_TOKEN: string;
  TRAINING_FOLDER: string;
  DATASETS_FOLDER: string;
+  MODELS_PATH: string;
 }

 export default function useSettings() {
@@ -14,6 +15,7 @@ export default function useSettings() {
    HF_TOKEN: '',
    TRAINING_FOLDER: '',
    DATASETS_FOLDER: '',
+    MODELS_PATH: '',
  });
  const [isSettingsLoaded, setIsLoaded] = useState(false);
  useEffect(() => {
@@ -26,6 +28,7 @@ export default function useSettings() {
          HF_TOKEN: data.HF_TOKEN || '',
          TRAINING_FOLDER: data.TRAINING_FOLDER || '',
          DATASETS_FOLDER: data.DATASETS_FOLDER || '',
+          MODELS_PATH: data.MODELS_PATH || '',
        });
        setIsLoaded(true);
      })
--- a/ui/src/paths.ts
+++ b/ui/src/paths.ts
@@ -3,3 +3,4 @@ export const TOOLKIT_ROOT = path.resolve('@', '..', '..');
 export const defaultTrainFolder = path.join(TOOLKIT_ROOT, 'output');
 export const defaultDatasetsFolder = path.join(TOOLKIT_ROOT, 'datasets');
 export const defaultDataRoot = path.join(TOOLKIT_ROOT, 'data');
+export const defaultModelsFolder = path.join(TOOLKIT_ROOT, 'models');