From 90fb5605c12b9205fc69170b40f2efc0004dd0d4 Mon Sep 17 00:00:00 2001 From: hiyouga Date: Thu, 5 Dec 2024 08:48:25 +0000 Subject: [PATCH] update examples Former-commit-id: e5584dc7ba356264034c14ff7d8efcd3d50fa9a3 --- examples/README.md | 6 +-- examples/README_zh.md | 6 +-- ...full_sft_ds3.yaml => llama3_full_sft.yaml} | 2 +- examples/train_full/qwen2vl_full_sft.yaml | 7 ++-- examples/train_lora/llama3_lora_sft_ds0.yaml | 40 ------------------- examples/train_lora/llama3_lora_sft_ds3.yaml | 2 +- examples/train_lora/qwen2vl_lora_sft.yaml | 2 +- 7 files changed, 13 insertions(+), 52 deletions(-) rename examples/train_full/{llama3_full_sft_ds3.yaml => llama3_full_sft.yaml} (85%) delete mode 100644 examples/train_lora/llama3_lora_sft_ds0.yaml diff --git a/examples/README.md b/examples/README.md index 9413ef2a..d79518df 100644 --- a/examples/README.md +++ b/examples/README.md @@ -126,14 +126,14 @@ llamafactory-cli train examples/train_qlora/llama3_lora_sft_aqlm.yaml #### Supervised Fine-Tuning on Single Node ```bash -FORCE_TORCHRUN=1 llamafactory-cli train examples/train_full/llama3_full_sft_ds3.yaml +FORCE_TORCHRUN=1 llamafactory-cli train examples/train_full/llama3_full_sft.yaml ``` #### Supervised Fine-Tuning on Multiple Nodes ```bash -FORCE_TORCHRUN=1 NNODES=2 RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft_ds3.yaml -FORCE_TORCHRUN=1 NNODES=2 RANK=1 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft_ds3.yaml +FORCE_TORCHRUN=1 NNODES=2 RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft.yaml +FORCE_TORCHRUN=1 NNODES=2 RANK=1 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft.yaml ``` #### Multimodal Supervised Fine-Tuning diff --git a/examples/README_zh.md b/examples/README_zh.md index 9aa4ca90..6fa935fe 100644 --- a/examples/README_zh.md +++ b/examples/README_zh.md @@ -126,14 +126,14 @@ llamafactory-cli train examples/train_qlora/llama3_lora_sft_aqlm.yaml #### 在单机上进行指令监督微调 ```bash -FORCE_TORCHRUN=1 llamafactory-cli train examples/train_full/llama3_full_sft_ds3.yaml +FORCE_TORCHRUN=1 llamafactory-cli train examples/train_full/llama3_full_sft.yaml ``` #### 在多机上进行指令监督微调 ```bash -FORCE_TORCHRUN=1 NNODES=2 RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft_ds3.yaml -FORCE_TORCHRUN=1 NNODES=2 RANK=1 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft_ds3.yaml +FORCE_TORCHRUN=1 NNODES=2 RANK=0 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft.yaml +FORCE_TORCHRUN=1 NNODES=2 RANK=1 MASTER_ADDR=192.168.0.1 MASTER_PORT=29500 llamafactory-cli train examples/train_full/llama3_full_sft.yaml ``` #### 多模态指令监督微调 diff --git a/examples/train_full/llama3_full_sft_ds3.yaml b/examples/train_full/llama3_full_sft.yaml similarity index 85% rename from examples/train_full/llama3_full_sft_ds3.yaml rename to examples/train_full/llama3_full_sft.yaml index e8b151b5..d1afe101 100644 --- a/examples/train_full/llama3_full_sft_ds3.yaml +++ b/examples/train_full/llama3_full_sft.yaml @@ -5,7 +5,7 @@ model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct stage: sft do_train: true finetuning_type: full -deepspeed: examples/deepspeed/ds_z3_config.json +deepspeed: examples/deepspeed/ds_z3_config.json # choices: [ds_z0_config.json, ds_z2_config.json, ds_z3_config.json] ### dataset dataset: identity,alpaca_en_demo diff --git a/examples/train_full/qwen2vl_full_sft.yaml b/examples/train_full/qwen2vl_full_sft.yaml index 60057d42..d3f8d2b3 100644 --- a/examples/train_full/qwen2vl_full_sft.yaml +++ b/examples/train_full/qwen2vl_full_sft.yaml @@ -5,10 +5,11 @@ model_name_or_path: Qwen/Qwen2-VL-7B-Instruct stage: sft do_train: true finetuning_type: full -deepspeed: examples/deepspeed/ds_z3_config.json +freeze_vision_tower: true # choices: [true, false] +deepspeed: examples/deepspeed/ds_z3_config.json # choices: [ds_z0_config.json, ds_z2_config.json, ds_z3_config.json] ### dataset -dataset: mllm_demo,identity +dataset: mllm_demo,identity,alpaca_en_demo template: qwen2_vl cutoff_len: 2048 max_samples: 1000 @@ -26,7 +27,7 @@ overwrite_output_dir: true per_device_train_batch_size: 1 gradient_accumulation_steps: 2 learning_rate: 1.0e-5 -num_train_epochs: 3.0 +num_train_epochs: 30.0 lr_scheduler_type: cosine warmup_ratio: 0.1 bf16: true diff --git a/examples/train_lora/llama3_lora_sft_ds0.yaml b/examples/train_lora/llama3_lora_sft_ds0.yaml deleted file mode 100644 index d5cae8fd..00000000 --- a/examples/train_lora/llama3_lora_sft_ds0.yaml +++ /dev/null @@ -1,40 +0,0 @@ -### model -model_name_or_path: meta-llama/Meta-Llama-3-8B-Instruct - -### method -stage: sft -do_train: true -finetuning_type: lora -lora_target: all -deepspeed: examples/deepspeed/ds_z0_config.json - -### dataset -dataset: identity,alpaca_en_demo -template: llama3 -cutoff_len: 2048 -max_samples: 1000 -overwrite_cache: true -preprocessing_num_workers: 16 - -### output -output_dir: saves/llama3-8b/lora/sft -logging_steps: 10 -save_steps: 500 -plot_loss: true -overwrite_output_dir: true - -### train -per_device_train_batch_size: 1 -gradient_accumulation_steps: 2 -learning_rate: 1.0e-4 -num_train_epochs: 3.0 -lr_scheduler_type: cosine -warmup_ratio: 0.1 -bf16: true -ddp_timeout: 180000000 - -### eval -val_size: 0.1 -per_device_eval_batch_size: 1 -eval_strategy: steps -eval_steps: 500 diff --git a/examples/train_lora/llama3_lora_sft_ds3.yaml b/examples/train_lora/llama3_lora_sft_ds3.yaml index e13d1a76..89a686d7 100644 --- a/examples/train_lora/llama3_lora_sft_ds3.yaml +++ b/examples/train_lora/llama3_lora_sft_ds3.yaml @@ -6,7 +6,7 @@ stage: sft do_train: true finetuning_type: lora lora_target: all -deepspeed: examples/deepspeed/ds_z3_config.json +deepspeed: examples/deepspeed/ds_z3_config.json # choices: [ds_z0_config.json, ds_z2_config.json, ds_z3_config.json] ### dataset dataset: identity,alpaca_en_demo diff --git a/examples/train_lora/qwen2vl_lora_sft.yaml b/examples/train_lora/qwen2vl_lora_sft.yaml index c8a18db2..9c7e6742 100644 --- a/examples/train_lora/qwen2vl_lora_sft.yaml +++ b/examples/train_lora/qwen2vl_lora_sft.yaml @@ -8,7 +8,7 @@ finetuning_type: lora lora_target: all ### dataset -dataset: mllm_demo,identity # video: mllm_video_demo +dataset: mllm_demo,identity,alpaca_en_demo # video: mllm_video_demo template: qwen2_vl cutoff_len: 2048 max_samples: 1000