Rework merge_network_on_save to handle dequantization on merging and saving for much more efficient full finetuning.

This commit is contained in:
Jaret Burkett
2026-06-25 13:19:38 -06:00
parent 6ecff36f26
commit 5f04ae7ad5
8 changed files with 147 additions and 41 deletions

View File

@@ -337,6 +337,13 @@ class ToolkitModuleMixin:
def disable_gradient_checkpointing(self: Module):
self.is_checkpointing = False
def _get_base_qtype(self: Module):
# the qtype string the base model was quantized with (so we can re-quantize after merging), or None
network = self.network_ref()
base_ref = getattr(network, 'base_model_ref', None)
base = base_ref() if base_ref is not None else None
return getattr(getattr(base, 'model_config', None), 'qtype', None)
@torch.no_grad()
def merge_out(self: Module, merge_out_weight=1.0):
# make sure it is positive
@@ -363,12 +370,12 @@ class ToolkitModuleMixin:
return
weight_key = "weight"
if 'weight._data' in org_sd:
# quantized weight
weight_key = "weight._data"
orig_dtype = org_sd[weight_key].dtype
weight = org_sd[weight_key].float()
from toolkit.util.quantize import is_quantized_tensor
org_weight = self.org_module[0].weight
is_ao_quantized = is_quantized_tensor(org_weight)
orig_dtype = org_weight.dtype
# dequantize torchao weights so the delta can be merged in full precision
weight = (org_weight.dequantize() if is_ao_quantized else org_weight).float()
multiplier = merge_weight
scale = self.scale
@@ -401,10 +408,19 @@ class ToolkitModuleMixin:
# print(conved.size(), weight.size(), module.stride, module.padding)
weight = weight + multiplier * conved * scale
# set weight to org_module
org_sd[weight_key] = weight.to(weight_device, orig_dtype)
self.org_module[0].load_state_dict(org_sd)
# write the merged weight back, re-quantizing if the original was torchao quantized so the
# model stays quantized across continuous merge/reset cycles
if is_ao_quantized:
from toolkit.util.quantize import get_torchao_config, requantize_module_weight
config = get_torchao_config(self._get_base_qtype())
if config is None:
print_once(f"Warning: merging into quantized layer {getattr(self, 'lora_name', '?')} "
f"without a known qtype; it will be left dequantized")
requantize_module_weight(self.org_module[0], weight.to(weight_device), orig_dtype, config)
else:
org_sd[weight_key] = weight.to(weight_device, orig_dtype)
self.org_module[0].load_state_dict(org_sd)
def reset_weights(self: Module):
# reset the weights to zero
org_sd = self.state_dict()