Prepwork for ilora

toolkit/config_modules.py

@@ -127,7 +127,7 @@ class NetworkConfig:
                 self.conv = 4
 
 
-AdapterTypes = Literal['t2i', 'ip', 'ip+']
+AdapterTypes = Literal['t2i', 'ip', 'ip+', 'clip', 'ilora']
 
 
 class AdapterConfig:

toolkit/ip_adapter.py

@@ -288,6 +288,9 @@ class IPAdapter(torch.nn.Module):
                 output_dim=sd.unet.config['cross_attention_dim'],
                 ff_mult=4
             )
+        elif adapter_config.type == 'ilora':
+            # we apply the clip encodings to the LoRA
+            image_proj_model = None
         else:
             raise ValueError(f"unknown adapter type: {adapter_config.type}")