Restore device runtime scales for compiled adapters (#963)

Co-authored-by: Rydén Johan <johan.ryden@bostad.uppsala.se>
This commit is contained in:
ZironicandRydén Johan authored and GitHub committed 2026-07-18 08:19:57 -06:00
1 parent cfdc9033a6
commit a0224793ce
7 files changed
+202 -16

No files matched your search

+22 -7
View File
@@ -65,7 +65,7 @@ class LoRAModule(torch.nn.Module):
if type(alpha) == torch.Tensor:
alpha = float(alpha.detach().float().item())
alpha = self.lora_dim if alpha is None or alpha == 0 else alpha
self.scale = float(alpha) / self.lora_dim
self._set_runtime_scale(float(alpha) / self.lora_dim)
self.register_buffer("alpha", torch.tensor(alpha)) # 定数として扱える
# same as microsoft's
@@ -78,6 +78,21 @@ class LoRAModule(torch.nn.Module):
self.rank_dropout = rank_dropout
self.module_dropout = module_dropout
def _set_runtime_scale(self, value) -> None:
"""Keep float metadata while using a device tensor in compiled math."""
self.scale = float(value)
runtime_scale = getattr(self, "_runtime_scale", None)
if runtime_scale is None:
reference = next(self.parameters(), None)
if reference is None:
runtime_scale = torch.tensor(self.scale, dtype=torch.float32)
else:
runtime_scale = reference.new_tensor(self.scale, dtype=torch.float32)
self.register_buffer("_runtime_scale", runtime_scale, persistent=False)
else:
with torch.no_grad():
runtime_scale.fill_(self.scale)
def apply_to(self):
self.org_forward = self.org_module.forward
self.org_module.forward = self.forward
@@ -108,9 +123,9 @@ class LoRAModule(torch.nn.Module):
# scaling for rank dropout: treat as if the rank is changed
# maskから計算することも考えられるが、augmentation的な効果を期待してrank_dropoutを用いる
scale = self.scale * (1.0 / (1.0 - self.rank_dropout)) # redundant for readability
scale = self._runtime_scale * (1.0 / (1.0 - self.rank_dropout)) # redundant for readability
else:
scale = self.scale
scale = self._runtime_scale
lx = self.lora_up(lx)
@@ -219,7 +234,7 @@ class LoRAInfModule(LoRAModule):
def default_forward(self, x):
# print("default_forward", self.lora_name, x.size())
return self.org_forward(x) + self.lora_up(self.lora_down(x)) * self.multiplier * self.scale
return self.org_forward(x) + self.lora_up(self.lora_down(x)) * self.multiplier * self._runtime_scale
def forward(self, x):
if not self.enabled:
@@ -258,7 +273,7 @@ class LoRAInfModule(LoRAModule):
return self.default_forward(x)
# apply mask for LoRA result
lx = self.lora_up(self.lora_down(x)) * self.multiplier * self.scale
lx = self.lora_up(self.lora_down(x)) * self.multiplier * self._runtime_scale
mask = self.get_mask_for_x(lx)
# print("regional", self.lora_name, self.network.sub_prompt_index, lx.size(), mask.size())
lx = lx * mask
@@ -302,7 +317,7 @@ class LoRAInfModule(LoRAModule):
# apply sub prompt of X
lx = x[emb_idx :: self.network.num_sub_prompts]
lx = self.lora_up(self.lora_down(lx)) * self.multiplier * self.scale
lx = self.lora_up(self.lora_down(lx)) * self.multiplier * self._runtime_scale
# print("sub_prompt_forward", self.lora_name, x.size(), lx.size(), emb_idx)
@@ -322,7 +337,7 @@ class LoRAInfModule(LoRAModule):
# call own LoRA
x1 = x[self.network.batch_size + self.network.sub_prompt_index :: self.network.num_sub_prompts]
lx1 = self.lora_up(self.lora_down(x1)) * self.multiplier * self.scale
lx1 = self.lora_up(self.lora_down(x1)) * self.multiplier * self._runtime_scale
if self.network.is_last_network:
lx = torch.zeros(