423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-11-05 18:32:14 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix examples

Browse Source 
Former-commit-id: 910ffaf46e3dde87d2dbb48b82a59a9898a90847
This commit is contained in:
hiyouga 2024-05-15 00:26:10 +08:00
parent eeb8ac3ca0
commit add54e3aaa
2 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
examples/deepspeed/ds_z0_config.json
View File

@ -14,5 +14,15 @@
}, },
"bf16": { "bf16": {
"enabled": "auto" "enabled": "auto"
},
"zero_optimization": {
"stage": 0,
"allgather_partitions": true,
"allgather_bucket_size": 5e8,
"overlap_comm": true,
"reduce_scatter": true,
"reduce_bucket_size": 5e8,
"contiguous_gradients": true,
"round_robin_gradients": true
} }
} }

2
examples/lora_multi_npu/ds_zero0.sh
View File

@ -12,4 +12,4 @@ ASCEND_RT_VISIBLE_DEVICES=0,1,2,3 torchrun \
--node_rank $RANK \ --node_rank $RANK \
--master_addr $MASTER_ADDR \ --master_addr $MASTER_ADDR \
--master_port $MASTER_PORT \ --master_port $MASTER_PORT \
src/train.py examples/lora_multi_gpu/llama3_lora_sft_ds.yaml src/train.py examples/lora_multi_npu/llama3_lora_sft_ds.yaml