423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-14 10:56:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

[optim] clean apollo (#6645)

Browse Source 
* clean apollo code

* update readme

Former-commit-id: 7a04021d04
This commit is contained in:
hoshi-hiyouga
2025-01-15 01:42:50 +08:00
committed by GitHub
parent 763f9b9df0
commit 9ef85f8fc4
14 changed files with 110 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/extras/galore/llama3_full_sft.yaml
View File

@@ -7,8 +7,8 @@ stage: sft
do_train: true
finetuning_type: full
use_galore: true
galore_layerwise: true
galore_target: mlp,self_attn
galore_layerwise: true # choices: [true, false], use false for DDP training
galore_target: all
galore_rank: 128
galore_scale: 2.0
@@ -29,7 +29,7 @@ overwrite_output_dir: true
### train
per_device_train_batch_size: 1
gradient_accumulation_steps: 1
gradient_accumulation_steps: 1 # use 1 for layerwise galore
learning_rate: 1.0e-5
num_train_epochs: 3.0
lr_scheduler_type: cosine