github/ai-toolkit
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Adjust MiniMah h3 sizing for videos to downscale to match

Browse Source
This commit is contained in:
Jaret Burkett
2026-08-15 07:50:42 -06:00
parent 70b1089359
commit 151ad0e959
5 changed files with 36 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
extensions_built_in/diffusion_models/minimax_h3/minimax_h3.py
View File

@@ -1330,7 +1330,7 @@ class MinimaxH3Ref2VAModel(MinimaxH3Model):
blocks = [(1, h, w, 0) for h, w in ref_shapes]
audio_rows = []
self._append_video_ref_blocks(
batch, all_rows, audio_rows, blocks, device, dtype
batch, all_rows, audio_rows, blocks, device, dtype, target_h, target_w
)
if not all_rows:
return None, None, (), ()
@@ -1364,8 +1364,10 @@ class MinimaxH3Ref2VAModel(MinimaxH3Model):
n = packing.align_num_frames_down(max(len(frames), 5))
frames = frames[:n]
h0, w0 = frames[0].shape[:2]
# ComfyUI reference-video sizing (independent of the sample canvas)
ph, pw = packing.reference_video_pixel_size(w0, h0)
# match the sample canvas's pixel area, own aspect kept
ph, pw = packing.reference_video_pixel_size(
w0, h0, gen_config.height, gen_config.width
)
pixels = torch.from_numpy(np.stack(frames)).float() / 255.0 * 2.0 - 1.0
pixels = pixels.permute(3, 0, 1, 2)[None] # (1, 3, T, H, W)
pixels = (
@@ -1405,10 +1407,11 @@ class MinimaxH3Ref2VAModel(MinimaxH3Model):
return {"latent": latents[0].float(), "audio_rows": audio_rows}
def _append_video_ref_blocks(
self, batch, all_rows, audio_rows, blocks, device, dtype
self, batch, all_rows, audio_rows, blocks, device, dtype, target_h, target_w
):
# control videos get dataset-identical treatment (frame count, fps,
# bucket) and one VAE encode, disk-cached next to the video; the
# control videos get the dataset's temporal treatment (frame count,
# fps), are area-matched to the target, and get one VAE encode
# disk-cached next to the video; the
# resulting latents become multi-frame reference blocks packed after
# the image references
paths_per_item = getattr(batch, "control_video_paths_list", None)
@@ -1425,7 +1428,7 @@ class MinimaxH3Ref2VAModel(MinimaxH3Model):
auds = []
for per_item in paths_per_item:
entry = load_ref_video_latent(
self, per_item[ref_idx], batch.dataset_config
self, per_item[ref_idx], batch.dataset_config, target_h, target_w
)
lats.append(entry["latent"].to(device, torch.float32))
auds.append(entry.get("audio_rows"))

21
extensions_built_in/diffusion_models/minimax_h3/src/packing.py
View File

@@ -124,16 +124,17 @@ def reference_pixel_size(
return height, width
def reference_video_pixel_size(ref_width: int, ref_height: int) -> Tuple[int, int]:
"""Reference VIDEO sizing (ComfyUI): the 768-short-edge canvas with the
768*1344 area cap, or the native size (rounded to /32) when the source is
smaller than that canvas. Returns (height, width)."""
ch, cw = resolve_canvas_size(ref_width, ref_height)
if ref_width * ref_height < cw * ch:
m = CANVAS_MULTIPLE
cw = max(m, round(ref_width / m) * m)
ch = max(m, round(ref_height / m) * m)
return ch, cw
def reference_video_pixel_size(
ref_width: int, ref_height: int, target_height: int, target_width: int
) -> Tuple[int, int]:
"""Reference VIDEO sizing: match the TARGET's pixel area with the ref's own
aspect kept (same aspect -> exactly the target size; other aspects -> the
same pixel budget on the /32 grid). Returns (height, width)."""
scale = math.sqrt((target_height * target_width) / float(ref_width * ref_height))
m = CANVAS_MULTIPLE
height = max(m, round(ref_height * scale / m) * m)
width = max(m, round(ref_width * scale / m) * m)
return height, width
def prepare_reference_image(

20
extensions_built_in/diffusion_models/minimax_h3/src/ref_video_cache.py
View File

@@ -1,8 +1,8 @@
"""Reference-video latents for ref2va, without dataloader machinery.
A control VIDEO gets the dataset's temporal treatment — num_frames /
auto_frame_count, fps — and ComfyUI's reference sizing (768-short-edge canvas
or native when smaller, aspect kept), then a single VAE encode whose
auto_frame_count, fps — and is area-matched to the target (own aspect kept, /32
grid), then a single VAE encode whose
result is cached next to the video in ``_latent_cache/`` (keyed like normal
latent caches: file signature + the config values that shape the latent).
Everything is deterministic (even frame spread, no random start) so the cache
@@ -112,7 +112,9 @@ def _cache_path(path: str, hash_dict: dict) -> str:
@torch.no_grad()
def load_ref_video_latent(model, path: str, dataset_config) -> dict:
def load_ref_video_latent(
model, path: str, dataset_config, target_height: int, target_width: int
) -> dict:
"""Returns {"latent": (C, T, h, w) cpu tensor, "num_frames": int},
encoding + disk-caching on first use. ``model`` is the MinimaxH3 model
(used for the VAE, audio encode and the frame-count snapper)."""
@@ -145,7 +147,10 @@ def load_ref_video_latent(model, path: str, dataset_config) -> dict:
)
hash_dict = {
"signature": get_quick_signature_string(path),
"ref_sizing": "comfy_canvas",
"ref_sizing": "match_target_area",
"target_area": int(target_height * target_width)
if target_height and target_width
else 0,
"num_frames": num_frames,
"fps": dataset_config.fps,
"trim_tail": trim_tail,
@@ -164,9 +169,10 @@ def load_ref_video_latent(model, path: str, dataset_config) -> dict:
mem_cache[path] = entry
return entry
# ComfyUI reference-video sizing: 768-short-edge canvas (768*1344 area
# cap) or native size when smaller; aspect-preserving resize, no crop
out_h, out_w = reference_video_pixel_size(src_w, src_h)
# match the target's pixel area (the dataset bucket the target trains at)
# with the ref's own aspect: same aspect -> identical size; aspect-
# preserving resize, no crop
out_h, out_w = reference_video_pixel_size(src_w, src_h, target_height, target_width)
indices = ref_frame_indices(
total, src_fps, num_frames, dataset_config.fps, trim_tail