Move the offload and quantization to the base model
This commit is contained in:
@@ -1,10 +1,7 @@
|
||||
from .flux2_model import Flux2Model
|
||||
from transformers import Qwen3ForCausalLM, Qwen2Tokenizer
|
||||
from toolkit.models.v2.text_encoders.qwen3 import Qwen3TextEncoder
|
||||
from optimum.quanto import freeze
|
||||
from toolkit.util.quantize import quantize, get_qtype
|
||||
from toolkit.config_modules import ModelConfig
|
||||
from toolkit.memory_management.manager import MemoryManager
|
||||
from toolkit.basic import flush
|
||||
from .src.model import Klein9BParams, Klein4BParams
|
||||
|
||||
@@ -41,27 +38,11 @@ class Flux2KleinModel(Flux2Model):
|
||||
dtype = self.torch_dtype
|
||||
self.print_and_status_update("Loading Qwen3")
|
||||
|
||||
text_encoder = Qwen3TextEncoder.load_model(
|
||||
self.flux2_klein_te_path, dtype=dtype, subfolder=""
|
||||
# load + quantize + offload + placement, all driven by model_config
|
||||
text_encoder = Qwen3TextEncoder.load(
|
||||
self.flux2_klein_te_path, subfolder="", **self.component_load_kwargs("te")
|
||||
)
|
||||
if self.model_config.quantize_te:
|
||||
self.print_and_status_update("Quantizing Qwen3")
|
||||
quantize(text_encoder, weights=get_qtype(self.model_config.qtype_te))
|
||||
freeze(text_encoder)
|
||||
flush()
|
||||
elif not self.model_config.low_vram:
|
||||
text_encoder.to(self.device_torch, dtype=dtype)
|
||||
flush()
|
||||
|
||||
if (
|
||||
self.model_config.layer_offloading
|
||||
and self.model_config.layer_offloading_text_encoder_percent > 0
|
||||
):
|
||||
MemoryManager.attach(
|
||||
text_encoder,
|
||||
self.device_torch,
|
||||
offload_percent=self.model_config.layer_offloading_text_encoder_percent,
|
||||
)
|
||||
flush()
|
||||
|
||||
tokenizer = Qwen2Tokenizer.from_pretrained(self.flux2_klein_te_path)
|
||||
return text_encoder, tokenizer
|
||||
|
||||
Reference in New Issue
Block a user