github/ai-toolkit
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Bug fixes and little improvements here and there.

Browse Source
This commit is contained in:
Jaret Burkett
2024-06-08 06:24:20 -06:00
parent 833c833f28
commit 3f3636b788
12 changed files with 358 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
toolkit/optimizer.py
View File

@@ -1,5 +1,5 @@
import torch
from transformers import Adafactor
from transformers import Adafactor, AdamW
def get_optimizer(
@@ -69,7 +69,7 @@ def get_optimizer(
if 'relative_step' not in optimizer_params:
optimizer_params['relative_step'] = False
if 'scale_parameter' not in optimizer_params:
optimizer_params['scale_parameter'] = True
optimizer_params['scale_parameter'] = False
if 'warmup_init' not in optimizer_params:
optimizer_params['warmup_init'] = False
optimizer = Adafactor(params, lr=float(learning_rate), eps=1e-6, **optimizer_params)