Phase 1
This commit is contained in:
27
toolkit/models/v2/text_encoders/clip.py
Normal file
27
toolkit/models/v2/text_encoders/clip.py
Normal file
@@ -0,0 +1,27 @@
|
||||
from transformers import CLIPTextModel, CLIPTextModelWithProjection
|
||||
|
||||
from .._mixin import OstrisTransformersMixin
|
||||
|
||||
|
||||
class CLIPTextEncoder(CLIPTextModel, OstrisTransformersMixin):
|
||||
"""CLIP-L text encoder (flux-style checkpoints: text_encoder/ +
|
||||
tokenizer/)."""
|
||||
|
||||
aitk_subfolder = "text_encoder"
|
||||
aitk_tokenizer_subfolder = "tokenizer"
|
||||
|
||||
@classmethod
|
||||
def get_transformer_block_names(cls):
|
||||
return ["text_model.encoder.layers"]
|
||||
|
||||
|
||||
class CLIPTextEncoderWithProjection(CLIPTextModelWithProjection, OstrisTransformersMixin):
|
||||
"""CLIP text encoder with the projection head (SDXL / SD3 / HiDream style
|
||||
checkpoints; the second encoder lives at text_encoder_2/ + tokenizer_2/)."""
|
||||
|
||||
aitk_subfolder = "text_encoder"
|
||||
aitk_tokenizer_subfolder = "tokenizer"
|
||||
|
||||
@classmethod
|
||||
def get_transformer_block_names(cls):
|
||||
return ["text_model.encoder.layers"]
|
||||
22
toolkit/models/v2/text_encoders/qwen25_vl.py
Normal file
22
toolkit/models/v2/text_encoders/qwen25_vl.py
Normal file
@@ -0,0 +1,22 @@
|
||||
from transformers import Qwen2_5_VLForConditionalGeneration
|
||||
|
||||
from .._mixin import OstrisTransformersMixin
|
||||
|
||||
|
||||
class Qwen25VLTextEncoder(Qwen2_5_VLForConditionalGeneration, OstrisTransformersMixin):
|
||||
"""Qwen2.5-VL conditioning stack (qwen_image family). Loads from a
|
||||
checkpoint's text_encoder/ subfolder; the edit variants keep the vision
|
||||
tower, plain t2i drops it."""
|
||||
|
||||
aitk_subfolder = "text_encoder"
|
||||
aitk_tokenizer_subfolder = "tokenizer"
|
||||
|
||||
@classmethod
|
||||
def get_transformer_block_names(cls):
|
||||
return ["model.language_model.layers"]
|
||||
|
||||
def drop_vision_tower(self):
|
||||
"""Text-only conditioning: the vision tower is dead weight."""
|
||||
if getattr(self.model, "visual", None) is not None:
|
||||
self.model.visual = None
|
||||
return self
|
||||
17
toolkit/models/v2/text_encoders/qwen3.py
Normal file
17
toolkit/models/v2/text_encoders/qwen3.py
Normal file
@@ -0,0 +1,17 @@
|
||||
from transformers import Qwen3ForCausalLM
|
||||
|
||||
from .._mixin import OstrisTransformersMixin
|
||||
|
||||
|
||||
class Qwen3TextEncoder(Qwen3ForCausalLM, OstrisTransformersMixin):
|
||||
"""Qwen3 causal-LM text encoder (Z-Image family, Zeta-Chroma, ...). Loads
|
||||
from a checkpoint's text_encoder/ subfolder, a hub repo, or a single
|
||||
.safetensors file; the tokenizer rides in the checkpoint's tokenizer/
|
||||
subfolder."""
|
||||
|
||||
aitk_subfolder = "text_encoder"
|
||||
aitk_tokenizer_subfolder = "tokenizer"
|
||||
|
||||
@classmethod
|
||||
def get_transformer_block_names(cls):
|
||||
return ["model.layers"]
|
||||
53
toolkit/models/v2/text_encoders/qwen3_vl.py
Normal file
53
toolkit/models/v2/text_encoders/qwen3_vl.py
Normal file
@@ -0,0 +1,53 @@
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from transformers import Qwen3VLForConditionalGeneration
|
||||
|
||||
from .._mixin import OstrisTransformersMixin
|
||||
|
||||
|
||||
def patch_qwen_vl_patch_embed(model) -> int:
|
||||
"""Qwen-VL's vision patch_embed is a Conv3d whose kernel == stride, i.e. a plain
|
||||
linear projection of each flattened patch. bf16 Conv3d has no fast cuDNN kernel and
|
||||
falls back to a slow, GPU-underutilizing path. Swap it for the equivalent F.linear
|
||||
(a GEMM). The weight is read lazily so this survives later .to(device)/dtype moves.
|
||||
Returns the number of patch_embed modules patched."""
|
||||
patched = 0
|
||||
for module in model.modules():
|
||||
proj = getattr(module, "proj", None)
|
||||
if isinstance(proj, torch.nn.Conv3d) and tuple(proj.kernel_size) == tuple(
|
||||
proj.stride
|
||||
):
|
||||
|
||||
def fast_forward(hidden_states, _proj=proj):
|
||||
w = _proj.weight.reshape(_proj.weight.shape[0], -1)
|
||||
x = hidden_states.view(-1, w.shape[1]).to(w.dtype)
|
||||
return F.linear(x, w, _proj.bias)
|
||||
|
||||
module.forward = fast_forward
|
||||
patched += 1
|
||||
return patched
|
||||
|
||||
|
||||
class Qwen3VLTextEncoder(Qwen3VLForConditionalGeneration, OstrisTransformersMixin):
|
||||
"""Qwen3-VL conditioning stack (krea2, mageflow, nucleus_image, ideogram4,
|
||||
minimax_h3, ...). Loads from a checkpoint's text_encoder/ subfolder or a
|
||||
raw Qwen repo (pass subfolder=\"\" for the latter)."""
|
||||
|
||||
aitk_subfolder = "text_encoder"
|
||||
aitk_processor_subfolder = "processor"
|
||||
|
||||
@classmethod
|
||||
def get_transformer_block_names(cls):
|
||||
return ["model.language_model.layers"]
|
||||
|
||||
def drop_vision_tower(self):
|
||||
"""Text-only conditioning: the vision tower is dead weight — drop it to
|
||||
free VRAM and skip loading its (bf16-slow) Conv3d patch_embed."""
|
||||
if getattr(self.model, "visual", None) is not None:
|
||||
self.model.visual = None
|
||||
return self
|
||||
|
||||
def patch_vision_patch_embed(self) -> int:
|
||||
"""Keep the vision tower (reference images ride into the embeddings)
|
||||
but swap its Conv3d patch_embed for an equivalent GEMM."""
|
||||
return patch_qwen_vl_patch_embed(self)
|
||||
16
toolkit/models/v2/text_encoders/t5.py
Normal file
16
toolkit/models/v2/text_encoders/t5.py
Normal file
@@ -0,0 +1,16 @@
|
||||
from transformers import T5EncoderModel
|
||||
|
||||
from .._mixin import OstrisTransformersMixin
|
||||
|
||||
|
||||
class T5TextEncoder(T5EncoderModel, OstrisTransformersMixin):
|
||||
"""T5-XXL text encoder. Defaults to the flux-style checkpoint layout
|
||||
(text_encoder_2/ + tokenizer_2/); pass subfolder overrides for checkpoints
|
||||
that keep it at text_encoder/ + tokenizer/ (e.g. f-lite)."""
|
||||
|
||||
aitk_subfolder = "text_encoder_2"
|
||||
aitk_tokenizer_subfolder = "tokenizer_2"
|
||||
|
||||
@classmethod
|
||||
def get_transformer_block_names(cls):
|
||||
return ["encoder.block"]
|
||||
49
toolkit/models/v2/text_encoders/umt5.py
Normal file
49
toolkit/models/v2/text_encoders/umt5.py
Normal file
@@ -0,0 +1,49 @@
|
||||
import torch
|
||||
from transformers import T5Tokenizer, UMT5EncoderModel
|
||||
|
||||
from .._mixin import OstrisTransformersMixin
|
||||
|
||||
|
||||
class PatchedT5Tokenizer(T5Tokenizer):
|
||||
def __init__(
|
||||
self,
|
||||
vocab=None,
|
||||
eos_token="</s>",
|
||||
unk_token="<unk>",
|
||||
pad_token="<pad>",
|
||||
_spm_precompiled_charsmap=None,
|
||||
extra_ids=100,
|
||||
additional_special_tokens=None,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(
|
||||
vocab=vocab,
|
||||
eos_token=eos_token,
|
||||
unk_token=unk_token,
|
||||
pad_token=pad_token,
|
||||
_spm_precompiled_charsmap=None, # this is passing a empty byte string for some reason now.
|
||||
extra_ids=extra_ids,
|
||||
additional_special_tokens=additional_special_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
class UMT5TextEncoder(UMT5EncoderModel, OstrisTransformersMixin):
|
||||
"""UMT5-XXL text encoder (wan family)."""
|
||||
|
||||
aitk_subfolder = "text_encoder"
|
||||
aitk_tokenizer_subfolder = "tokenizer"
|
||||
|
||||
@classmethod
|
||||
def get_transformer_block_names(cls):
|
||||
return ["encoder.block"]
|
||||
|
||||
@classmethod
|
||||
def load_tokenizer(cls, name_or_path=None, subfolder=None, **kwargs):
|
||||
# T5's tokenizer needs the _spm_precompiled_charsmap patch
|
||||
source = name_or_path if name_or_path is not None else cls.aitk_tokenizer_repo
|
||||
if subfolder is None:
|
||||
subfolder = cls.aitk_tokenizer_subfolder
|
||||
return PatchedT5Tokenizer.from_pretrained(
|
||||
source, subfolder=subfolder or "", **kwargs
|
||||
)
|
||||
Reference in New Issue
Block a user