Restore device runtime scales for compiled adapters (#963)
Co-authored-by: Rydén Johan <johan.ryden@bostad.uppsala.se>
This commit is contained in:
1 parent
cfdc9033a6
commit
a0224793ce
7 files changed
+202
-16
No files matched your search
@@ -158,7 +158,7 @@ class ExtractableModuleMixin:
|
||||
|
||||
# set up alphas
|
||||
self.alpha = (self.alpha * 0) + down_weight.shape[0]
|
||||
self.scale = float(self.alpha.detach().float().item()) / self.lora_dim
|
||||
self._set_runtime_scale(float(self.alpha.detach().float().item()) / self.lora_dim)
|
||||
|
||||
# assign them
|
||||
|
||||
@@ -179,6 +179,21 @@ class ToolkitModuleMixin:
|
||||
self.is_checkpointing = False
|
||||
self._multiplier: Union[float, list, torch.Tensor] = None
|
||||
|
||||
def _set_runtime_scale(self: Module, value) -> None:
|
||||
"""Keep float metadata while using a device tensor in compiled math."""
|
||||
self.scale = float(value)
|
||||
runtime_scale = getattr(self, "_runtime_scale", None)
|
||||
if runtime_scale is None:
|
||||
reference = next(self.parameters(), None)
|
||||
if reference is None:
|
||||
runtime_scale = torch.tensor(self.scale, dtype=torch.float32)
|
||||
else:
|
||||
runtime_scale = reference.new_tensor(self.scale, dtype=torch.float32)
|
||||
self.register_buffer("_runtime_scale", runtime_scale, persistent=False)
|
||||
else:
|
||||
with torch.no_grad():
|
||||
runtime_scale.fill_(self.scale)
|
||||
|
||||
def _call_forward(self: Module, x):
|
||||
# module dropout
|
||||
if self.module_dropout is not None and self.training:
|
||||
@@ -211,9 +226,9 @@ class ToolkitModuleMixin:
|
||||
|
||||
# scaling for rank dropout: treat as if the rank is changed
|
||||
# maskから計算することも考えられるが、augmentation的な効果を期待してrank_dropoutを用いる
|
||||
scale = self.scale * (1.0 / (1.0 - self.rank_dropout)) # redundant for readability
|
||||
scale = self._runtime_scale * (1.0 / (1.0 - self.rank_dropout)) # redundant for readability
|
||||
else:
|
||||
scale = self.scale
|
||||
scale = self._runtime_scale
|
||||
|
||||
lx = self.lora_up(lx)
|
||||
|
||||
|
||||
Reference in new issue
Block a user