Fixed issue where a buffer was stuck on cpu when offloading ideogram4

This commit is contained in:
Jaret Burkett committed 2026-06-07 13:31:03 -06:00
1 parent 1dc9a797cf
commit 35588726de
2 files changed
+5 -1

No files matched your search

@@ -289,7 +289,7 @@ class Ideogram4Model(BaseModel):
transformer, transformer,
self.device_torch, self.device_torch,
offload_percent=self.model_config.layer_offloading_transformer_percent, offload_percent=self.model_config.layer_offloading_transformer_percent,
ignore_modules=[transformer.input_proj, transformer.llm_cond_proj], ignore_modules=[transformer.rotary_emb.inv_freq, transformer.input_proj, transformer.llm_cond_proj],
) )
elif self.model_config.low_vram: elif self.model_config.low_vram:
self.print_and_status_update("Moving transformer to CPU") self.print_and_status_update("Moving transformer to CPU")
@@ -94,6 +94,10 @@ class Ideogram4MRoPE(nn.Module):
assert position_ids.ndim == 3 and position_ids.shape[-1] == 3 assert position_ids.ndim == 3 and position_ids.shape[-1] == 3
batch_size, seq_len, _ = position_ids.shape batch_size, seq_len, _ = position_ids.shape
if self.inv_freq.device == torch.device("cpu"):
# sometimes it gets stuck on CPU
self.inv_freq = self.inv_freq.to(position_ids.device)
# (3, B, inv_freq_size, L) # (3, B, inv_freq_size, L)
pos = position_ids.permute(2, 0, 1).to(dtype=torch.float32) pos = position_ids.permute(2, 0, 1).to(dtype=torch.float32)
inv_freq = self.inv_freq.to(dtype=torch.float32)[None, None, :, None].expand( inv_freq = self.inv_freq.to(dtype=torch.float32)[None, None, :, None].expand(