From 788accb601851d31f1f6ff706a42d94891c33d98 Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Fri, 17 Jan 2025 13:46:09 +0800 Subject: [PATCH] fix qwen2 moe (#6684) Former-commit-id: 7bf09abf1c4d971cda33daed933c75f391e79294 --- src/llamafactory/model/model_utils/moe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llamafactory/model/model_utils/moe.py b/src/llamafactory/model/model_utils/moe.py index 58039e2a..cd5558d3 100644 --- a/src/llamafactory/model/model_utils/moe.py +++ b/src/llamafactory/model/model_utils/moe.py @@ -61,7 +61,7 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None: _set_z3_leaf_modules(model, [MixtralSparseMoeBlock]) - if model_type == "qwen2moe": + if model_type == "qwen2_moe": from transformers.models.qwen2_moe.modeling_qwen2_moe import Qwen2MoeSparseMoeBlock _set_z3_leaf_modules(model, [Qwen2MoeSparseMoeBlock])