support pretraining of llava

Former-commit-id: 6a4c8cf0a6a1674c693b9337f018ff8df7477f8f

src/llamafactory/hparams/model_args.py

@@ -85,6 +85,10 @@ class ModelArguments:
         default=False,
         metadata={"help": "Whethor or not to use multimodal LLM that accepts visual inputs."},
     )
+    tune_mm_proj: bool = field(
+        default=False,
+        metadata={"help": "Whethor or not only finetune mm_projector for MLLM."},
+    )
     moe_aux_loss_coef: Optional[float] = field(
         default=None,
         metadata={"help": "Coefficient of the auxiliary router loss in mixture-of-experts model."},

src/llamafactory/model/loader.py

@@ -163,6 +163,11 @@ def load_model(
     else:
         model.train()
 
+    if model_args.visual_inputs and model_args.tune_mm_proj:
+        lm_params = [param for name, param in model.named_parameters() if "language_model" in name]
+        for param in lm_params:
+            param.requires_grad_(False)
+
     trainable_params, all_param = count_parameters(model)
     if is_trainable:
         param_stats = "trainable params: {:d} || all params: {:d} || trainable%: {:.4f}".format(