423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2026-03-01 17:26:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

[model] efficient tuning for gpt-oss (#9354)

Browse Source 
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
thulyubh22
2025-12-23 16:28:38 +08:00
committed by GitHub
parent 1f1f5a7d1b
commit 7901b2f32e
3 changed files with 18 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/llamafactory/model/model_utils/moe.py
View File

@@ -129,9 +129,13 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None:
if model_type in ("qwen3_omni_moe", "qwen3_omni_moe_thinker"):
from transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe import Qwen3OmniMoeThinkerTextSparseMoeBlock
_set_z3_leaf_modules(model, [Qwen3OmniMoeThinkerTextSparseMoeBlock])
if model_type == "gpt_oss":
from transformers.models.gpt_oss.modeling_gpt_oss import GptOssMLP
_set_z3_leaf_modules(model, [GptOssMLP])
def configure_moe(config: "PretrainedConfig", model_args: "ModelArguments", is_trainable: bool) -> None:
if not is_trainable or not model_args.moe_aux_loss_coef: