423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-15 03:10:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix examples

Browse Source
This commit is contained in:
hiyouga
2024-05-15 00:26:10 +08:00
parent 5bdad46387
commit 7e69e71a52
2 changed files with 11 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
examples/deepspeed/ds_z0_config.json
View File

@@ -14,5 +14,15 @@
}, },
"bf16": { "bf16": {
"enabled": "auto" "enabled": "auto"
},
"zero_optimization": {
"stage": 0,
"allgather_partitions": true,
"allgather_bucket_size": 5e8,
"overlap_comm": true,
"reduce_scatter": true,
"reduce_bucket_size": 5e8,
"contiguous_gradients": true,
"round_robin_gradients": true
} }
} }

2
examples/lora_multi_npu/ds_zero0.sh
View File

@@ -12,4 +12,4 @@ ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 torchrun \
--node_rank $RANK \ --node_rank $RANK \
--master_addr $MASTER_ADDR \ --master_addr $MASTER_ADDR \
--master_port $MASTER_PORT \ --master_port $MASTER_PORT \
src/train.py examples/lora_multi_gpu/llama3_lora_sft_ds.yaml src/train.py examples/lora_multi_npu/llama3_lora_sft_ds.yaml