423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2026-05-28 02:48:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

[v1] fix padding free with sp (#10513)

Browse Source
This commit is contained in:
jiaqiw09
2026-05-26 23:49:21 +08:00
committed by GitHub
parent 8e68764b65
commit 01398eb18d
6 changed files with 80 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
examples/v1/train_batching_strategy/train_full_fsdp2_dynamic_padding_free.yaml Normal file
View File

@@ -0,0 +1,30 @@
model: Qwen/Qwen3-0.6B
model_class: llm
template: qwen3_nothink
kernel_config:
name: auto
include_kernels: auto # choice: null/true/false/auto/kernel_id1,kernel_id2,kernel_id3, default is null
quant_config: null
dist_config:
name: fsdp2
dcp_path: null # /mnt/f/pretrain_models/Qwen3-0.6B-dcp
### data
train_dataset: data/v1_sft_demo.yaml
### training
output_dir: outputs/test_fsdp2
micro_batch_size: 4
batching_strategy: dynamic_padding_free
flash_attn: flash_attention2
cutoff_len: 2048
learning_rate: 1.0e-4
max_steps: 10
### sample
sample_backend: hf
max_new_tokens: 128

2
examples/v1/train_batching_strategy/train_full_fsdp2_padding_free.yaml
View File

@@ -20,7 +20,7 @@ train_dataset: data/v1_sft_demo.yaml
output_dir: outputs/test_fsdp2
micro_batch_size: 4
batching_strategy: padding_free
flash_attn: fa2
flash_attn: flash_attention2
cutoff_len: 2048
learning_rate: 1.0e-4
max_steps: 10