[v1] add FSDPTurbo EP/EFSDP plugin for MoE training (#10676)

This commit is contained in:
Hazeldxq
2026-08-13 20:45:55 +08:00
committed by GitHub
parent bc4b42cefc
commit f28afaf635
21 changed files with 1341 additions and 29 deletions

View File

@@ -0,0 +1,32 @@
model: Qwen/Qwen3.5-35B-A3B
model_class: llm
kernel_config:
name: auto, flash-linear-attention
include_kernels: chunk_gated_delta_rule, fused_recurrent_gated_delta_rule
chunk_size: 64
dist_config:
name: fsdpturbo
ep_size: 16
ep_dispatcher: eager
cp_size: 1
init_config:
name: init_on_meta
### data
train_dataset: data/v1_sft_demo.yaml
### training
output_dir: outputs/Qwen3.5-35B-A3B/full/sft
micro_batch_size: 1
cutoff_len: 256
learning_rate: 1.0e-4
bf16: true
max_steps: 5
### sample
sample_backend: hf
max_new_tokens: 128