[model] Add DeepSpeed Z3 leaf module for Qwen3-Next (#10194)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

src/llamafactory/extras/constants.py

@@ -65,6 +65,7 @@ MCA_SUPPORTED_MODELS = {
     "qwen2_vl",
     "qwen2_5_vl",
     "qwen3_vl",
+    "qwen3_vl_moe",
     "qwen3",
     "qwen3_moe",
     "qwen3_next",

src/llamafactory/model/model_utils/moe.py

@@ -142,6 +142,10 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None:
 
         _set_z3_leaf_modules(model, [Qwen3OmniMoeThinkerTextSparseMoeBlock])
 
+    if model_type == "qwen3_next":
+        from transformers.models.qwen3_next.modeling_qwen3_next import Qwen3NextSparseMoeBlock
+
+        _set_z3_leaf_modules(model, [Qwen3NextSparseMoeBlock])
 
 def configure_moe(config: "PretrainedConfig", model_args: "ModelArguments", is_trainable: bool) -> None:
     if not is_trainable or not model_args.moe_aux_loss_coef:

src/llamafactory/train/mca/workflow.py

@@ -82,9 +82,34 @@ def _check_model_support(model_args: "ModelArguments"):
         model_args.model_name_or_path, trust_remote_code=model_args.trust_remote_code
     )
     if config.model_type not in MCA_SUPPORTED_MODELS:
-        raise ValueError(f"Model {config.model_type} is not supported by MCA.")
+        raise ValueError(
+            f"Model {config.model_type} is not supported by mcore_adapter."
+            "You can try to upgrade mcore_adapter to the latest version for more supported models."
+        )
 
 
+def _freeze_model_parameters(model: Any, finetuning_args: "FinetuningArguments"):
+    """Freeze model parameters for qwen_vl series models based on finetuning arguments."""
+    if getattr(model.config, "hf_model_type", None) not in ["qwen2_vl", "qwen2_5_vl", "qwen3_vl", "qwen3_vl_moe"]:
+        return
+
+    params_to_freeze = []
+    if finetuning_args.freeze_vision_tower:
+        params_to_freeze.extend(["vision_model.blocks", "vision_model.patch_embed"])
+        if getattr(model.config, "hf_model_type", None) in ["qwen3_vl", "qwen3_vl_moe"]:
+            params_to_freeze.extend(["vision_model.pos_embed"])
+
+    if finetuning_args.freeze_multi_modal_projector:
+        params_to_freeze.extend(["multi_modal_projector"])
+
+    if finetuning_args.freeze_language_model:
+        params_to_freeze.extend(["embedding", "decoder", "output_layer"])
+
+    if params_to_freeze:
+        for name, p in model.named_parameters():
+            if any(name.startswith(k) for k in params_to_freeze):
+                p.requires_grad_(False)
+
 def run_pt(
     model_args: "ModelArguments",
     data_args: "DataArguments",
@@ -161,22 +186,8 @@ def run_sft(
     _check_model_support(model_args)
     model = AutoModel.from_pretrained(model_args.model_name_or_path, training_args)
 
-    # optional freezing for qwen2_vl, qwen2_5_vl
-    if getattr(model.config, "hf_model_type", None) in ["qwen2_vl", "qwen2_5_vl", "qwen3_vl"]:
-        params_to_freeze = []
-        if finetuning_args.freeze_vision_tower:
-            params_to_freeze.extend(["vision_model.blocks", "vision_model.patch_embed"])
-
-        if finetuning_args.freeze_multi_modal_projector:
-            params_to_freeze.extend(["multi_modal_projector"])
-
-        if finetuning_args.freeze_language_model:
-            params_to_freeze.extend(["embedding", "decoder", "output_layer"])
-
-        if params_to_freeze:
-            for name, p in model.named_parameters():
-                if any(name.startswith(k) for k in params_to_freeze):
-                    p.requires_grad_(False)
+    # optional freezing for qwen_vl series
+    _freeze_model_parameters(model, finetuning_args)
 
     pad_to_max = training_args.expert_model_parallel_size is not None and training_args.expert_model_parallel_size > 1
     data_collator = SFTDataCollatorWith4DAttentionMask(
@@ -229,6 +240,8 @@ def run_dpo(
     _check_model_support(model_args)
     model = AutoModel.from_pretrained(model_args.model_name_or_path, training_args)
 
+    _freeze_model_parameters(model, finetuning_args)
+
     if finetuning_args.use_ref_model:
         ref_config = AutoConfig.from_pretrained(model_args.model_name_or_path, training_args)
         ref_model = AutoModel.from_config(ref_config)

src/llamafactory/train/tuner.py

@@ -24,7 +24,7 @@ from ..data import get_template_and_fix_tokenizer
 from ..extras import logging
 from ..extras.constants import V_HEAD_SAFE_WEIGHTS_NAME, V_HEAD_WEIGHTS_NAME
 from ..extras.misc import find_available_port, get_device_name, get_torch_device, infer_optim_dtype
-from ..extras.packages import is_mcore_adapter_available, is_ray_available
+from ..extras.packages import is_mcore_adapter_available, is_ray_available, is_transformers_version_greater_than
 from ..hparams import RayArguments, get_infer_args, get_ray_args, get_train_args, read_args
 from ..model import load_model, load_tokenizer
 from .callbacks import LogCallback, PissaConvertCallback, ReporterCallback
@@ -160,17 +160,28 @@ def export_model(args: Optional[dict[str, Any]] = None) -> None:
         model = model.to(output_dtype)
         logger.info_rank0(f"Convert model dtype to: {output_dtype}.")
 
-    model.save_pretrained(
-        save_directory=model_args.export_dir,
-        max_shard_size=f"{model_args.export_size}GB",
-        safe_serialization=(not model_args.export_legacy_format),
-    )
+    # Prepare save arguments (safe_serialization removed in transformers v5.0.0)
+    save_kwargs = {
+        "save_directory": model_args.export_dir,
+        "max_shard_size": f"{model_args.export_size}GB",
+    }
+    if not is_transformers_version_greater_than("5.0.0"):
+        save_kwargs["safe_serialization"] = not model_args.export_legacy_format
+
+    model.save_pretrained(**save_kwargs)
+
     if model_args.export_hub_model_id is not None:
+        # Prepare push arguments (safe_serialization removed in transformers v5.0.0)
+        push_kwargs = {
+            "max_shard_size": f"{model_args.export_size}GB",
+        }
+        if not is_transformers_version_greater_than("5.0.0"):
+            push_kwargs["safe_serialization"] = not model_args.export_legacy_format
+
         model.push_to_hub(
             model_args.export_hub_model_id,
             token=model_args.hf_hub_token,
-            max_shard_size=f"{model_args.export_size}GB",
-            safe_serialization=(not model_args.export_legacy_format),
+            **push_kwargs,
         )
 
     if finetuning_args.stage == "rm":