Rework merge_network_on_save to handle dequantization on merging and saving for much more efficient full finetuning.
This commit is contained in:
@@ -337,6 +337,13 @@ class ToolkitModuleMixin:
|
||||
def disable_gradient_checkpointing(self: Module):
|
||||
self.is_checkpointing = False
|
||||
|
||||
def _get_base_qtype(self: Module):
|
||||
# the qtype string the base model was quantized with (so we can re-quantize after merging), or None
|
||||
network = self.network_ref()
|
||||
base_ref = getattr(network, 'base_model_ref', None)
|
||||
base = base_ref() if base_ref is not None else None
|
||||
return getattr(getattr(base, 'model_config', None), 'qtype', None)
|
||||
|
||||
@torch.no_grad()
|
||||
def merge_out(self: Module, merge_out_weight=1.0):
|
||||
# make sure it is positive
|
||||
@@ -363,12 +370,12 @@ class ToolkitModuleMixin:
|
||||
return
|
||||
|
||||
weight_key = "weight"
|
||||
if 'weight._data' in org_sd:
|
||||
# quantized weight
|
||||
weight_key = "weight._data"
|
||||
|
||||
orig_dtype = org_sd[weight_key].dtype
|
||||
weight = org_sd[weight_key].float()
|
||||
from toolkit.util.quantize import is_quantized_tensor
|
||||
org_weight = self.org_module[0].weight
|
||||
is_ao_quantized = is_quantized_tensor(org_weight)
|
||||
orig_dtype = org_weight.dtype
|
||||
# dequantize torchao weights so the delta can be merged in full precision
|
||||
weight = (org_weight.dequantize() if is_ao_quantized else org_weight).float()
|
||||
|
||||
multiplier = merge_weight
|
||||
scale = self.scale
|
||||
@@ -401,10 +408,19 @@ class ToolkitModuleMixin:
|
||||
# print(conved.size(), weight.size(), module.stride, module.padding)
|
||||
weight = weight + multiplier * conved * scale
|
||||
|
||||
# set weight to org_module
|
||||
org_sd[weight_key] = weight.to(weight_device, orig_dtype)
|
||||
self.org_module[0].load_state_dict(org_sd)
|
||||
|
||||
# write the merged weight back, re-quantizing if the original was torchao quantized so the
|
||||
# model stays quantized across continuous merge/reset cycles
|
||||
if is_ao_quantized:
|
||||
from toolkit.util.quantize import get_torchao_config, requantize_module_weight
|
||||
config = get_torchao_config(self._get_base_qtype())
|
||||
if config is None:
|
||||
print_once(f"Warning: merging into quantized layer {getattr(self, 'lora_name', '?')} "
|
||||
f"without a known qtype; it will be left dequantized")
|
||||
requantize_module_weight(self.org_module[0], weight.to(weight_device), orig_dtype, config)
|
||||
else:
|
||||
org_sd[weight_key] = weight.to(weight_device, orig_dtype)
|
||||
self.org_module[0].load_state_dict(org_sd)
|
||||
|
||||
def reset_weights(self: Module):
|
||||
# reset the weights to zero
|
||||
org_sd = self.state_dict()
|
||||
|
||||
Reference in New Issue
Block a user