Move the offload and quantization to the base model
This commit is contained in:
1 parent
702254688d
commit
85a6880643
45 files changed
+725
-1140
No files matched your search
@@ -14,6 +14,9 @@ class Gemma3TextEncoder(Gemma3ForConditionalGeneration, OstrisTransformersMixin)
|
||||
# both layouts seen across transformers versions; missing paths skip
|
||||
return ["model.language_model.layers", "language_model.model.layers"]
|
||||
|
||||
def get_offload_ignore_modules(self):
|
||||
return [self.model.language_model.base_model.embed_tokens]
|
||||
|
||||
|
||||
try:
|
||||
from transformers.models.gemma4.modeling_gemma4 import Gemma4TextModel
|
||||
@@ -28,5 +31,12 @@ try:
|
||||
def get_transformer_block_names(cls):
|
||||
return ["layers"]
|
||||
|
||||
def get_offload_ignore_modules(self):
|
||||
# layer_scalar is a bare tensor buffer on each decoder layer; the
|
||||
# manager never enumerates it, so it must ride along explicitly
|
||||
return [self.embed_tokens] + [
|
||||
layer.layer_scalar for layer in self.layers
|
||||
]
|
||||
|
||||
except ImportError:
|
||||
Gemma4TextEncoder = None
|
||||
Reference in new issue
Block a user