Fix quantization issues

This commit is contained in:
Jaret Burkett
2026-08-28 10:48:34 -06:00
parent 85a6880643
commit 92df289931
7 changed files with 470 additions and 19 deletions

View File

@@ -1,5 +1,5 @@
import os
from toolkit.models.v2.text_encoders.qwen3_vl import Qwen3VLTextEncoder
from toolkit.models.v2._mixin import OstrisTransformersMixin
from typing import List, Optional
import torch
@@ -28,6 +28,17 @@ from .src.hidream_o1.model_config import model_config
if TYPE_CHECKING:
from toolkit.data_transfer_object.data_loader import DataLoaderBatchDTO
class HidreamO1Transformer(Qwen3VLForConditionalGeneration, OstrisTransformersMixin):
"""The o1 DiT-in-LLM: the vendored Qwen3VL with the image-diffusion heads
(x_embedder / t_embedder1 / final_layer2). The generic Qwen3VLTextEncoder
must NOT be used here — it drops those keys as unexpected."""
@classmethod
def get_transformer_block_names(cls):
return ["model.language_model.layers"]
scheduler_config = {
"num_train_timesteps": 1000,
"shift": 3.0,
@@ -129,7 +140,10 @@ class HidreamO1Model(BaseModel):
self.use_old_lokr_format = False
self.is_flow_matching = True
self.is_transformer = True
self.target_lora_modules = ["Qwen3VLForConditionalGeneration"]
self.target_lora_modules = [
"Qwen3VLForConditionalGeneration",
"HidreamO1Transformer",
]
self.noise_scale = self.model_config.model_kwargs.get(
"noise_scale", DEFAULT_NOISE_SCALE
)
@@ -188,7 +202,7 @@ class HidreamO1Model(BaseModel):
)
# transformer.load_state_dict(state_dict, assign=True)
transformer = Qwen3VLTextEncoder.from_pretrained(
transformer = HidreamO1Transformer.from_pretrained(
None,
config=Qwen3VLConfig(**model_config),
state_dict=state_dict,
@@ -196,7 +210,7 @@ class HidreamO1Model(BaseModel):
)
del state_dict # free memory
else:
transformer = Qwen3VLTextEncoder.from_pretrained(
transformer = HidreamO1Transformer.from_pretrained(
model_path,
torch_dtype=self.torch_dtype,
)