Various bug fixes and optimizations for quantized training. Added untested custom adam8bit optimizer. Did some work on LoRM (dont use)

2024-11-20 09:16:55 -07:00
parent 6509ba4484
commit 894374b2e9
7 changed files with 241 additions and 18 deletions
--- a/toolkit/lorm.py
+++ b/toolkit/lorm.py
@@ -354,7 +354,8 @@ def convert_diffusers_unet_to_lorm(
                elif child_module.__class__.__name__ in LINEAR_MODULES:
                    if count_parameters(child_module) > parameter_threshold:

-                        dtype = child_module.weight.dtype
+                        # dtype = child_module.weight.dtype
+                        dtype = torch.float32
                        # extract and convert
                        down_weight, up_weight, lora_dim, diff = extract_linear(
                            weight=child_module.weight.clone().detach().float(),