From 33525a34b6fe17c70f356f468a298d3d569c24c7 Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Fri, 17 Jan 2025 13:46:09 +0800 Subject: [PATCH] fix qwen2 moe (#6684) Former-commit-id: ab624419fa0ab23ef7a331a0ec14e393328772b5 --- src/llamafactory/model/model_utils/moe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llamafactory/model/model_utils/moe.py b/src/llamafactory/model/model_utils/moe.py index 58039e2a..cd5558d3 100644 --- a/src/llamafactory/model/model_utils/moe.py +++ b/src/llamafactory/model/model_utils/moe.py @@ -61,7 +61,7 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None: _set_z3_leaf_modules(model, [MixtralSparseMoeBlock]) - if model_type == "qwen2moe": + if model_type == "qwen2_moe": from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeSparseMoeBlock _set_z3_leaf_modules(model, [Qwen2MoeSparseMoeBlock])