github/ai-toolkit
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

WIP to add the caption_proj weight to pixart sigma TE adapter

Browse Source
This commit is contained in:
Jaret Burkett
2024-07-06 13:00:21 -06:00
parent acb06d6ff3
commit cab8a1c7b8
8 changed files with 500 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
toolkit/models/ilora.py
View File

@@ -156,10 +156,10 @@ class InstantLoRAMidModule(torch.nn.Module):
weight_chunk = weight_chunk.view(self.down_shape)
# check if is conv or linear
if len(weight_chunk.shape) == 4:
padding = 0
if weight_chunk.shape[-1] == 3:
padding = 1
x_chunk = nn.functional.conv2d(x_chunk, weight_chunk, padding=padding)
org_module = self.lora_module_ref().orig_module_ref()
stride = org_module.stride
padding = org_module.padding
x_chunk = nn.functional.conv2d(x_chunk, weight_chunk, padding=padding, stride=stride)
else:
# run a simple linear layer with the down weight
x_chunk = x_chunk @ weight_chunk.T