Restore device runtime scales for compiled adapters (#963)

Co-authored-by: Rydén Johan <johan.ryden@bostad.uppsala.se>
This commit is contained in:
ZironicandRydén Johan authored and GitHub committed 2026-07-18 08:19:57 -06:00
1 parent cfdc9033a6
commit a0224793ce
7 files changed
+202 -16

No files matched your search

+18 -3
View File
@@ -158,7 +158,7 @@ class ExtractableModuleMixin:
# set up alphas
self.alpha = (self.alpha * 0) + down_weight.shape[0]
self.scale = float(self.alpha.detach().float().item()) / self.lora_dim
self._set_runtime_scale(float(self.alpha.detach().float().item()) / self.lora_dim)
# assign them
@@ -179,6 +179,21 @@ class ToolkitModuleMixin:
self.is_checkpointing = False
self._multiplier: Union[float, list, torch.Tensor] = None
def _set_runtime_scale(self: Module, value) -> None:
"""Keep float metadata while using a device tensor in compiled math."""
self.scale = float(value)
runtime_scale = getattr(self, "_runtime_scale", None)
if runtime_scale is None:
reference = next(self.parameters(), None)
if reference is None:
runtime_scale = torch.tensor(self.scale, dtype=torch.float32)
else:
runtime_scale = reference.new_tensor(self.scale, dtype=torch.float32)
self.register_buffer("_runtime_scale", runtime_scale, persistent=False)
else:
with torch.no_grad():
runtime_scale.fill_(self.scale)
def _call_forward(self: Module, x):
# module dropout
if self.module_dropout is not None and self.training:
@@ -211,9 +226,9 @@ class ToolkitModuleMixin:
# scaling for rank dropout: treat as if the rank is changed
# maskから計算することも考えられるが、augmentation的な効果を期待してrank_dropoutを用いる
scale = self.scale * (1.0 / (1.0 - self.rank_dropout)) # redundant for readability
scale = self._runtime_scale * (1.0 / (1.0 - self.rank_dropout)) # redundant for readability
else:
scale = self.scale
scale = self._runtime_scale
lx = self.lora_up(lx)