Move the offload and quantization to the base model

This commit is contained in:
Jaret Burkett committed 2026-08-28 08:37:56 -06:00
1 parent 702254688d
commit 85a6880643
45 files changed
+725 -1140

No files matched your search

+10
View File
@@ -14,6 +14,9 @@ class Gemma3TextEncoder(Gemma3ForConditionalGeneration, OstrisTransformersMixin)
# both layouts seen across transformers versions; missing paths skip
return ["model.language_model.layers", "language_model.model.layers"]
def get_offload_ignore_modules(self):
return [self.model.language_model.base_model.embed_tokens]
try:
from transformers.models.gemma4.modeling_gemma4 import Gemma4TextModel
@@ -28,5 +31,12 @@ try:
def get_transformer_block_names(cls):
return ["layers"]
def get_offload_ignore_modules(self):
# layer_scalar is a bare tensor buffer on each decoder layer; the
# manager never enumerates it, so it must ride along explicitly
return [self.embed_tokens] + [
layer.layer_scalar for layer in self.layers
]
except ImportError:
Gemma4TextEncoder = None