This commit is contained in:
Jaret Burkett
2026-08-27 11:53:08 -06:00
parent e8d9cf6d35
commit 8db198ec0a
51 changed files with 985 additions and 1273 deletions

View File

@@ -0,0 +1,27 @@
from transformers import CLIPTextModel, CLIPTextModelWithProjection
from .._mixin import OstrisTransformersMixin
class CLIPTextEncoder(CLIPTextModel, OstrisTransformersMixin):
"""CLIP-L text encoder (flux-style checkpoints: text_encoder/ +
tokenizer/)."""
aitk_subfolder = "text_encoder"
aitk_tokenizer_subfolder = "tokenizer"
@classmethod
def get_transformer_block_names(cls):
return ["text_model.encoder.layers"]
class CLIPTextEncoderWithProjection(CLIPTextModelWithProjection, OstrisTransformersMixin):
"""CLIP text encoder with the projection head (SDXL / SD3 / HiDream style
checkpoints; the second encoder lives at text_encoder_2/ + tokenizer_2/)."""
aitk_subfolder = "text_encoder"
aitk_tokenizer_subfolder = "tokenizer"
@classmethod
def get_transformer_block_names(cls):
return ["text_model.encoder.layers"]

View File

@@ -0,0 +1,22 @@
from transformers import Qwen2_5_VLForConditionalGeneration
from .._mixin import OstrisTransformersMixin
class Qwen25VLTextEncoder(Qwen2_5_VLForConditionalGeneration, OstrisTransformersMixin):
"""Qwen2.5-VL conditioning stack (qwen_image family). Loads from a
checkpoint's text_encoder/ subfolder; the edit variants keep the vision
tower, plain t2i drops it."""
aitk_subfolder = "text_encoder"
aitk_tokenizer_subfolder = "tokenizer"
@classmethod
def get_transformer_block_names(cls):
return ["model.language_model.layers"]
def drop_vision_tower(self):
"""Text-only conditioning: the vision tower is dead weight."""
if getattr(self.model, "visual", None) is not None:
self.model.visual = None
return self

View File

@@ -0,0 +1,17 @@
from transformers import Qwen3ForCausalLM
from .._mixin import OstrisTransformersMixin
class Qwen3TextEncoder(Qwen3ForCausalLM, OstrisTransformersMixin):
"""Qwen3 causal-LM text encoder (Z-Image family, Zeta-Chroma, ...). Loads
from a checkpoint's text_encoder/ subfolder, a hub repo, or a single
.safetensors file; the tokenizer rides in the checkpoint's tokenizer/
subfolder."""
aitk_subfolder = "text_encoder"
aitk_tokenizer_subfolder = "tokenizer"
@classmethod
def get_transformer_block_names(cls):
return ["model.layers"]

View File

@@ -0,0 +1,53 @@
import torch
import torch.nn.functional as F
from transformers import Qwen3VLForConditionalGeneration
from .._mixin import OstrisTransformersMixin
def patch_qwen_vl_patch_embed(model) -> int:
"""Qwen-VL's vision patch_embed is a Conv3d whose kernel == stride, i.e. a plain
linear projection of each flattened patch. bf16 Conv3d has no fast cuDNN kernel and
falls back to a slow, GPU-underutilizing path. Swap it for the equivalent F.linear
(a GEMM). The weight is read lazily so this survives later .to(device)/dtype moves.
Returns the number of patch_embed modules patched."""
patched = 0
for module in model.modules():
proj = getattr(module, "proj", None)
if isinstance(proj, torch.nn.Conv3d) and tuple(proj.kernel_size) == tuple(
proj.stride
):
def fast_forward(hidden_states, _proj=proj):
w = _proj.weight.reshape(_proj.weight.shape[0], -1)
x = hidden_states.view(-1, w.shape[1]).to(w.dtype)
return F.linear(x, w, _proj.bias)
module.forward = fast_forward
patched += 1
return patched
class Qwen3VLTextEncoder(Qwen3VLForConditionalGeneration, OstrisTransformersMixin):
"""Qwen3-VL conditioning stack (krea2, mageflow, nucleus_image, ideogram4,
minimax_h3, ...). Loads from a checkpoint's text_encoder/ subfolder or a
raw Qwen repo (pass subfolder=\"\" for the latter)."""
aitk_subfolder = "text_encoder"
aitk_processor_subfolder = "processor"
@classmethod
def get_transformer_block_names(cls):
return ["model.language_model.layers"]
def drop_vision_tower(self):
"""Text-only conditioning: the vision tower is dead weight — drop it to
free VRAM and skip loading its (bf16-slow) Conv3d patch_embed."""
if getattr(self.model, "visual", None) is not None:
self.model.visual = None
return self
def patch_vision_patch_embed(self) -> int:
"""Keep the vision tower (reference images ride into the embeddings)
but swap its Conv3d patch_embed for an equivalent GEMM."""
return patch_qwen_vl_patch_embed(self)

View File

@@ -0,0 +1,16 @@
from transformers import T5EncoderModel
from .._mixin import OstrisTransformersMixin
class T5TextEncoder(T5EncoderModel, OstrisTransformersMixin):
"""T5-XXL text encoder. Defaults to the flux-style checkpoint layout
(text_encoder_2/ + tokenizer_2/); pass subfolder overrides for checkpoints
that keep it at text_encoder/ + tokenizer/ (e.g. f-lite)."""
aitk_subfolder = "text_encoder_2"
aitk_tokenizer_subfolder = "tokenizer_2"
@classmethod
def get_transformer_block_names(cls):
return ["encoder.block"]

View File

@@ -0,0 +1,49 @@
import torch
from transformers import T5Tokenizer, UMT5EncoderModel
from .._mixin import OstrisTransformersMixin
class PatchedT5Tokenizer(T5Tokenizer):
def __init__(
self,
vocab=None,
eos_token="</s>",
unk_token="<unk>",
pad_token="<pad>",
_spm_precompiled_charsmap=None,
extra_ids=100,
additional_special_tokens=None,
**kwargs,
):
super().__init__(
vocab=vocab,
eos_token=eos_token,
unk_token=unk_token,
pad_token=pad_token,
_spm_precompiled_charsmap=None, # this is passing a empty byte string for some reason now.
extra_ids=extra_ids,
additional_special_tokens=additional_special_tokens,
**kwargs,
)
class UMT5TextEncoder(UMT5EncoderModel, OstrisTransformersMixin):
"""UMT5-XXL text encoder (wan family)."""
aitk_subfolder = "text_encoder"
aitk_tokenizer_subfolder = "tokenizer"
@classmethod
def get_transformer_block_names(cls):
return ["encoder.block"]
@classmethod
def load_tokenizer(cls, name_or_path=None, subfolder=None, **kwargs):
# T5's tokenizer needs the _spm_precompiled_charsmap patch
source = name_or_path if name_or_path is not None else cls.aitk_tokenizer_repo
if subfolder is None:
subfolder = cls.aitk_tokenizer_subfolder
return PatchedT5Tokenizer.from_pretrained(
source, subfolder=subfolder or "", **kwargs
)