423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-11-04 18:02:19 +08:00
Code Issues Packages Projects Releases Wiki Activity

support qwen2vl train proj only

Browse Source 
Former-commit-id: 0e949ef03455726e907c6f1039e93ebe480c897a
This commit is contained in:
hiyouga 2024-12-05 10:37:42 +00:00
parent be1ec97c8e
commit bac2c64f87
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
examples/train_full/qwen2vl_full_sft.yaml
View File

@ -6,6 +6,7 @@ stage: sft
do_train: true
finetuning_type: full
freeze_vision_tower: true # choices: [true, false]
train_mm_proj_only: false # choices: [true, false]
deepspeed: examples/deepspeed/ds_z3_config.json # choices: [ds_z0_config.json, ds_z2_config.json, ds_z3_config.json]
### dataset

7
src/llamafactory/model/model_utils/visual.py
View File

@ -138,11 +138,10 @@ def get_forbidden_modules(config: "PretrainedConfig", finetuning_args: "Finetuni
forbidden_modules.add("language_model")
elif model_type == "qwen2_vl":
if finetuning_args.freeze_vision_tower:
forbidden_modules.add("visual")
if finetuning_args.train_mm_proj_only:
raise ValueError("Qwen2-VL models do not support `train_mm_proj_only`.")
forbidden_modules.update({"visual.patch_embed", "visual.blocks", "model", "lm_head"})
elif finetuning_args.freeze_vision_tower:
forbidden_modules.add("visual")
return forbidden_modules