Move the offload and quantization to the base model

This commit is contained in:
Jaret Burkett
2026-08-28 08:37:56 -06:00
parent 702254688d
commit 85a6880643
45 changed files with 725 additions and 1140 deletions

View File

@@ -1,10 +1,7 @@
from .flux2_model import Flux2Model
from transformers import Qwen3ForCausalLM, Qwen2Tokenizer
from toolkit.models.v2.text_encoders.qwen3 import Qwen3TextEncoder
from optimum.quanto import freeze
from toolkit.util.quantize import quantize, get_qtype
from toolkit.config_modules import ModelConfig
from toolkit.memory_management.manager import MemoryManager
from toolkit.basic import flush
from .src.model import Klein9BParams, Klein4BParams
@@ -41,27 +38,11 @@ class Flux2KleinModel(Flux2Model):
dtype = self.torch_dtype
self.print_and_status_update("Loading Qwen3")
text_encoder = Qwen3TextEncoder.load_model(
self.flux2_klein_te_path, dtype=dtype, subfolder=""
# load + quantize + offload + placement, all driven by model_config
text_encoder = Qwen3TextEncoder.load(
self.flux2_klein_te_path, subfolder="", **self.component_load_kwargs("te")
)
if self.model_config.quantize_te:
self.print_and_status_update("Quantizing Qwen3")
quantize(text_encoder, weights=get_qtype(self.model_config.qtype_te))
freeze(text_encoder)
flush()
elif not self.model_config.low_vram:
text_encoder.to(self.device_torch, dtype=dtype)
flush()
if (
self.model_config.layer_offloading
and self.model_config.layer_offloading_text_encoder_percent > 0
):
MemoryManager.attach(
text_encoder,
self.device_torch,
offload_percent=self.model_config.layer_offloading_text_encoder_percent,
)
flush()
tokenizer = Qwen2Tokenizer.from_pretrained(self.flux2_klein_te_path)
return text_encoder, tokenizer