Added ability to target parts of lora for ilora

2024-07-20 22:45:52 +00:00
parent 4c249cf607
commit c2c4e8cf34
3 changed files with 59 additions and 22 deletions
--- a/toolkit/config_modules.py
+++ b/toolkit/config_modules.py
@@ -194,6 +194,9 @@ class AdapterConfig:
        # for ilora
        self.head_dim: int = kwargs.get('head_dim', 1024)
        self.num_heads: int = kwargs.get('num_heads', 1)
+        self.ilora_down: bool = kwargs.get('ilora_down', True)
+        self.ilora_mid: bool = kwargs.get('ilora_mid', True)
+        self.ilora_up: bool = kwargs.get('ilora_up', True)


 class EmbeddingConfig: