[v1] add muon optimizer (#10618)

Co-authored-by: mhh111 <mahonghao1@huawei.com>
This commit is contained in:
HelloWorldBeginner
2026-07-17 21:44:28 +08:00
committed by GitHub
parent d1049d650a
commit 5f653cb96a
7 changed files with 422 additions and 20 deletions

View File

@@ -0,0 +1,29 @@
model: Qwen/Qwen3-0.6B
model_class: llm
dist_config:
name: fsdp2
optim_config:
name: muon
wd: 0.1
momentum: 0.95
nesterov: true
ns_steps: 5
adamw_betas: [0.9, 0.95]
adamw_eps: 1.0e-8
### data
train_dataset: data/v1_sft_demo.yaml
### training
output_dir: outputs/test_muon
micro_batch_size: 1
cutoff_len: 2048
learning_rate: 1.0e-5
max_steps: 10
### sample
sample_backend: hf
max_new_tokens: 128