mirror of
https://github.com/hiyouga/LLaMA-Factory.git
synced 2026-07-28 11:46:09 +08:00
[v1] add muon optimizer (#10618)
Co-authored-by: mhh111 <mahonghao1@huawei.com>
This commit is contained in:
committed by
GitHub
parent
d1049d650a
commit
5f653cb96a
29
examples/v1/train_full/train_full_muon.yaml
Normal file
29
examples/v1/train_full/train_full_muon.yaml
Normal file
@@ -0,0 +1,29 @@
|
||||
model: Qwen/Qwen3-0.6B
|
||||
model_class: llm
|
||||
|
||||
|
||||
dist_config:
|
||||
name: fsdp2
|
||||
|
||||
optim_config:
|
||||
name: muon
|
||||
wd: 0.1
|
||||
momentum: 0.95
|
||||
nesterov: true
|
||||
ns_steps: 5
|
||||
adamw_betas: [0.9, 0.95]
|
||||
adamw_eps: 1.0e-8
|
||||
|
||||
### data
|
||||
train_dataset: data/v1_sft_demo.yaml
|
||||
|
||||
### training
|
||||
output_dir: outputs/test_muon
|
||||
micro_batch_size: 1
|
||||
cutoff_len: 2048
|
||||
learning_rate: 1.0e-5
|
||||
max_steps: 10
|
||||
|
||||
### sample
|
||||
sample_backend: hf
|
||||
max_new_tokens: 128
|
||||
Reference in New Issue
Block a user