[model] add Qwen3.8 model support (#10749)

Co-authored-by: hiyouga <hiyouga@buaa.edu.cn>

src/llamafactory/data/template.py

@@ -522,6 +522,80 @@ class ReasoningTemplate(Template):
         return [(encoded_messages[i], encoded_messages[i + 1]) for i in range(0, len(encoded_messages), 2)]
 
 
+@dataclass
+class Qwen38ReasoningTemplate(ReasoningTemplate):
+    r"""Qwen3.8 template with reasoning-effort instructions and official system ordering."""
+
+    reasoning_effort: str = "xhigh"
+
+    def _get_reasoning_instruction(self) -> str:
+        if self.enable_thinking is False:
+            return ""
+
+        if self.reasoning_effort == "xhigh":
+            return (
+                "Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, "
+                "consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final "
+                "answer."
+            )
+        elif self.reasoning_effort == "medium":
+            return ""
+        elif self.reasoning_effort == "low":
+            return (
+                "Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the "
+                "conclusion without unnecessary elaboration."
+            )
+        else:
+            # Defensive validation for callers that configure the template without DataArguments.
+            raise ValueError(
+                f"Unexpected reasoning effort {self.reasoning_effort}. "
+                "Supported types are xhigh (default), medium, and low."
+            )
+
+    @override
+    def _encode(
+        self,
+        tokenizer: "PreTrainedTokenizer",
+        messages: list[dict[str, str]],
+        system: Optional[str],
+        tools: Optional[str],
+    ) -> list[list[int]]:
+        system = (system or self.default_system).strip()
+        reasoning_instruction = self._get_reasoning_instruction()
+        encoded_messages = []
+        for i, message in enumerate(messages):
+            elements = []
+
+            if i == 0:
+                elements += self.format_prefix.apply()
+                system_parts = []
+                if reasoning_instruction:
+                    system_parts.append(reasoning_instruction)
+                if tools:
+                    system_parts.append(self.format_tools.apply(content=tools)[0].lstrip("\n"))
+                if system:
+                    system_parts.append(system)
+                if system_parts:
+                    elements += self.format_system.apply(content="\n\n".join(system_parts))
+
+            if message["role"] == Role.USER:
+                elements += self.format_user.apply(content=message["content"], idx=str(i // 2))
+            elif message["role"] == Role.ASSISTANT:
+                elements += self.format_assistant.apply(content=message["content"])
+            elif message["role"] == Role.OBSERVATION:
+                elements += self.format_observation.apply(content=message["content"])
+            elif message["role"] == Role.FUNCTION:
+                elements += self.format_function.apply(
+                    content=message["content"], thought_words=self.thought_words, tool_call_words=self.tool_call_words
+                )
+            else:
+                raise NotImplementedError("Unexpected role: {}".format(message["role"]))
+
+            encoded_messages.append(self._convert_elements_to_ids(tokenizer, elements))
+
+        return encoded_messages
+
+
 @dataclass
 class Glm47ReasoningTemplate(ReasoningTemplate):
     r"""GLM-4.7 uses only the closing </think> tag for empty thinking blocks."""
@@ -693,6 +767,9 @@ def get_template_and_fix_tokenizer(tokenizer: "PreTrainedTokenizer", data_args:
     if data_args.train_on_prompt and template.efficient_eos:
         raise ValueError("Current template does not support `train_on_prompt`.")
 
+    if isinstance(template, Qwen38ReasoningTemplate) and data_args.tool_format not in {None, "qwen3_8"}:
+        raise ValueError("Template `qwen3_8` uses its built-in tool format; remove the incompatible `tool_format`.")
+
     if data_args.tool_format is not None:
         logger.info_rank0(f"Using tool format: {data_args.tool_format}.")
         default_slots = ["{{content}}"] if template.efficient_eos else ["{{content}}", {"eos_token"}]
@@ -712,7 +789,11 @@ def get_template_and_fix_tokenizer(tokenizer: "PreTrainedTokenizer", data_args:
             "e.g., qwen3_vl_nothink"
         )
         template.enable_thinking = data_args.enable_thinking
-        template.preserve_thinking = data_args.preserve_thinking
+        if isinstance(template, Qwen38ReasoningTemplate):
+            template.reasoning_effort = data_args.reasoning_effort
+            template.preserve_thinking = True if data_args.preserve_thinking is None else data_args.preserve_thinking
+        elif data_args.preserve_thinking is not None:
+            template.preserve_thinking = data_args.preserve_thinking
 
     template.fix_special_tokens(tokenizer)
     template.fix_jinja_template(tokenizer)
@@ -2291,6 +2372,24 @@ register_template(
 )
 
 
+register_template(
+    name="qwen3_8",
+    format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
+    format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
+    format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
+    format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen3_8"),
+    format_observation=StringFormatter(
+        slots=["<|im_start|>user\n<tool_response>\n{{content}}\n</tool_response><|im_end|>\n<|im_start|>assistant\n"]
+    ),
+    format_tools=ToolFormatter(tool_format="qwen3_8"),
+    stop_words=["<|im_end|>"],
+    replace_eos=True,
+    preserve_thinking=True,
+    mm_plugin=get_mm_plugin(name="qwen3_vl", image_token="<|image_pad|>", video_token="<|video_pad|>"),
+    template_class=Qwen38ReasoningTemplate,
+)
+
+
 register_template(
     name="sailor",
     format_user=StringFormatter(slots=["<|im_start|>question\n{{content}}<|im_end|>\n<|im_start|>answer\n"]),

src/llamafactory/data/tool_utils.py

@@ -758,6 +758,20 @@ class Qwen35ToolUtils(ToolUtils):
         return results if results else content
 
 
+class Qwen38ToolUtils(Qwen35ToolUtils):
+    r"""Qwen 3.8 tool template preserving the OpenAI function wrapper."""
+
+    @override
+    @staticmethod
+    def tool_formatter(tools: list[dict[str, Any]]) -> str:
+        tool_text = ""
+        for tool in tools:
+            wrapped_tool = tool if tool.get("type") == "function" else {"type": "function", "function": tool}
+            tool_text += "\n" + json.dumps(wrapped_tool, ensure_ascii=False)
+
+        return QWEN35_TOOL_PROMPT.format(tool_text=tool_text)
+
+
 class GLM4MOEToolUtils(QwenToolUtils):
     r"""GLM-4-MOE tool using template."""
 
@@ -970,6 +984,7 @@ TOOLS = {
     "mistral": MistralToolUtils(),
     "qwen": QwenToolUtils(),
     "qwen3_5": Qwen35ToolUtils(),
+    "qwen3_8": Qwen38ToolUtils(),
     "glm4_moe": GLM4MOEToolUtils(),
     "seed_oss": SeedToolUtils(),
     "ling": LingToolUtils(),

src/llamafactory/extras/constants.py

@@ -71,6 +71,8 @@ MCA_SUPPORTED_MODELS = {
     "qwen3_next",
     "qwen3_5",
     "qwen3_5_moe",
+    "qwen3_5_moe_text",
+    "qwen3_5_text",
 }
 
 # Text LLM model_types supported by the Megatron Bridge PT/SFT path (gpt_step).
@@ -3000,6 +3002,37 @@ register_model_group(
 )
 
 
+register_model_group(
+    models={
+        "Qwen3.8-27B": {
+            DownloadSource.DEFAULT: "Qwen/Qwen3.8-27B",
+            DownloadSource.MODELSCOPE: "Qwen/Qwen3.8-27B",
+        },
+        "Qwen3.8-27B-FP8": {
+            DownloadSource.DEFAULT: "Qwen/Qwen3.8-27B-FP8",
+            DownloadSource.MODELSCOPE: "Qwen/Qwen3.8-27B-FP8",
+        },
+    },
+    template="qwen3_8",
+    multimodal=True,
+)
+
+
+register_model_group(
+    models={
+        "Qwen3.8-2.4T-A95B-Thinking": {
+            DownloadSource.DEFAULT: "Qwen/Qwen3.8-2.4T-A95B",
+            DownloadSource.MODELSCOPE: "Qwen/Qwen3.8-2.4T-A95B",
+        },
+        "Qwen3.8-2.4T-A95B-Thinking-FP8": {
+            DownloadSource.DEFAULT: "Qwen/Qwen3.8-2.4T-A95B-FP8",
+            DownloadSource.MODELSCOPE: "Qwen/Qwen3.8-2.4T-A95B-FP8",
+        },
+    },
+    template="qwen3_8",
+)
+
+
 register_model_group(
     models={
         "Qwen2-Audio-7B": {

src/llamafactory/hparams/data_args.py

@@ -125,9 +125,18 @@ class DataArguments:
         default=True,
         metadata={"help": "Whether or not to enable thinking mode for reasoning models."},
     )
-    preserve_thinking: bool = field(
-        default=False,
-        metadata={"help": "Whether or not to preserve thinking content in historical turns for reasoning models."},
+    reasoning_effort: str = field(
+        default="xhigh",
+        metadata={"help": "Reasoning effort for supported reasoning models (xhigh, medium, or low)."},
+    )
+    preserve_thinking: bool | None = field(
+        default=None,
+        metadata={
+            "help": (
+                "Whether or not to preserve thinking content in historical turns for reasoning models. "
+                "Uses the template default when unspecified."
+            )
+        },
     )
     tokenized_path: str | None = field(
         default=None,
@@ -182,6 +191,9 @@ class DataArguments:
         if self.mask_history and self.train_on_prompt:
             raise ValueError("`mask_history` is incompatible with `train_on_prompt`.")
 
+        if self.reasoning_effort not in {"xhigh", "medium", "low"}:
+            raise ValueError("`reasoning_effort` must be one of xhigh, medium, or low.")
+
         if self.neat_packing:
             self.packing = True
 

src/llamafactory/model/model_utils/liger_kernel.py

@@ -82,8 +82,12 @@ def apply_liger_kernel(
         from liger_kernel.transformers import apply_liger_kernel_to_qwen3_next as apply_liger_kernel
     elif model_type == "qwen3_5":
         from liger_kernel.transformers import apply_liger_kernel_to_qwen3_5 as apply_liger_kernel
+    elif model_type == "qwen3_5_text":
+        from liger_kernel.transformers import apply_liger_kernel_to_qwen3_5_text as apply_liger_kernel
     elif model_type == "qwen3_5_moe":
         from liger_kernel.transformers import apply_liger_kernel_to_qwen3_5_moe as apply_liger_kernel
+    elif model_type == "qwen3_5_moe_text":
+        from liger_kernel.transformers import apply_liger_kernel_to_qwen3_5_moe_text as apply_liger_kernel
     elif model_type == "gpt_oss":
         try:
             from liger_kernel.transformers import apply_liger_kernel_to_gpt_oss as apply_liger_kernel

src/llamafactory/model/model_utils/moe.py

@@ -151,7 +151,7 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None:
 
         _set_z3_leaf_modules(model, [Qwen3NextSparseMoeBlock])
 
-    if model_type == "qwen3_5_moe":
+    if model_type in ("qwen3_5_moe", "qwen3_5_moe_text"):
         from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5MoeSparseMoeBlock
 
         _set_z3_leaf_modules(model, [Qwen3_5MoeSparseMoeBlock])

src/llamafactory/model/patcher.py

@@ -139,7 +139,7 @@ def patch_qwen3_5_forward_npu(model: "PreTrainedModel") -> None:
 
 
 def patch_qwen3_5_forward_gpu(model: "PreTrainedModel") -> None:
-    """Patch the forward method of Qwen3_5ForConditionalGeneration to support cu_seqlens input only patch when do training.
+    """Patch Qwen3.5 decoder forward methods to support cu_seqlens during training.
 
     Refer to: https://github.com/axolotl-ai-cloud/axolotl/blob/main/src/axolotl/monkeypatch/models/qwen3_5/modeling.py.
     """
@@ -286,12 +286,12 @@ def patch_qwen3_5_forward_gpu(model: "PreTrainedModel") -> None:
 
         return output
 
-    if model.config.architectures[0] == "Qwen3_5ForConditionalGeneration":
+    if model.config.architectures[0] in ("Qwen3_5ForCausalLM", "Qwen3_5ForConditionalGeneration"):
         from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5GatedDeltaNet
 
         Qwen3_5DecoderLayer.forward = _patched_decoder_forward
         Qwen3_5GatedDeltaNet.forward = _patch_gdn_forward
-    elif model.config.architectures[0] == "Qwen3_5MoeForConditionalGeneration":
+    elif model.config.architectures[0] in ("Qwen3_5MoeForCausalLM", "Qwen3_5MoeForConditionalGeneration"):
         from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import (
             Qwen3_5MoeDecoderLayer,
             Qwen3_5MoeGatedDeltaNet,
@@ -484,7 +484,12 @@ def patch_model(
         autocast_projector_dtype(model, model_args)
         add_z3_leaf_module(model)
 
-        if getattr(model.config, "model_type", None) in ["qwen3_5", "qwen3_5_moe"]:
+        if getattr(model.config, "model_type", None) in [
+            "qwen3_5",
+            "qwen3_5_moe",
+            "qwen3_5_moe_text",
+            "qwen3_5_text",
+        ]:
             if is_torch_npu_available():
                 patch_qwen3_5_forward_npu(model)
             elif is_torch_cuda_available() and model_args.flash_attn == "fa2":

src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/npu_fused_moe.py

@@ -363,6 +363,9 @@ _V5_MODEL_TYPE_TO_PATCHES = {
     "qwen3_5_moe": {
         "Qwen3_5MoeExperts": NpuMoeFusedV5.experts_forward,
     },
+    "qwen3_5_moe_text": {
+        "Qwen3_5MoeExperts": NpuMoeFusedV5.experts_forward,
+    },
 }
 
 _MODEL_TYPE_TO_PATCHES = (

src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/npu_swiglu.py

@@ -83,9 +83,15 @@ _MODEL_TYPE_TO_PATCHES = {
     "qwen3_5": {
         "Qwen3_5MLP": npu_swiglu_forward,
     },
+    "qwen3_5_text": {
+        "Qwen3_5MLP": npu_swiglu_forward,
+    },
     "qwen3_5_moe": {
         "Qwen3_5MoeMLP": npu_swiglu_forward,
     },
+    "qwen3_5_moe_text": {
+        "Qwen3_5MoeMLP": npu_swiglu_forward,
+    },
 }
 
 

src/llamafactory/v1/plugins/model_plugins/kernels/ops/rms_norm/npu_rms_norm.py

@@ -148,10 +148,18 @@ _MODEL_TYPE_TO_PATCHES = {
         "Qwen3_5RMSNorm": npu_residual_rms_norm_forward,
         "Qwen3_5RMSNormGated": npu_gated_rms_norm_forward,
     },
+    "qwen3_5_text": {
+        "Qwen3_5RMSNorm": npu_residual_rms_norm_forward,
+        "Qwen3_5RMSNormGated": npu_gated_rms_norm_forward,
+    },
     "qwen3_5_moe": {
         "Qwen3_5MoeRMSNorm": npu_residual_rms_norm_forward,
         "Qwen3_5MoeRMSNormGated": npu_gated_rms_norm_forward,
     },
+    "qwen3_5_moe_text": {
+        "Qwen3_5MoeRMSNorm": npu_residual_rms_norm_forward,
+        "Qwen3_5MoeRMSNormGated": npu_gated_rms_norm_forward,
+    },
 }
 
 

src/llamafactory/v1/plugins/model_plugins/kernels/ops/rope/npu_rope.py

@@ -121,7 +121,9 @@ _MODEL_TYPE_TO_PATCHES = {
     "qwen3_vl": _default_rope_patch("qwen3_vl"),
     "qwen3_vl_moe": _default_rope_patch("qwen3_vl_moe"),
     "qwen3_5": _default_rope_patch("qwen3_5"),
+    "qwen3_5_text": _default_rope_patch("qwen3_5"),
     "qwen3_5_moe": _default_rope_patch("qwen3_5_moe"),
+    "qwen3_5_moe_text": _default_rope_patch("qwen3_5_moe"),
 }
 
 

tests/data/test_formatter.py

@@ -281,6 +281,14 @@ def test_qwen_tool_extractor():
     assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")]
 
 
+@pytest.mark.runs_on(["cpu", "mps"])
+def test_qwen38_tool_formatter():
+    formatter = ToolFormatter(tool_format="qwen3_8")
+    wrapped_tool = {"type": "function", "function": TOOLS[0]}
+    output = formatter.apply(content=json.dumps(TOOLS))[0]
+    assert json.dumps(wrapped_tool, ensure_ascii=False) in output
+
+
 @pytest.mark.runs_on(["cpu", "mps"])
 def test_qwen_multi_tool_extractor():
     formatter = ToolFormatter(tool_format="qwen")

tests/data/test_template.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import os
+from copy import deepcopy
 from typing import TYPE_CHECKING
 
 import pytest
@@ -22,6 +23,7 @@ from llamafactory.data import get_template_and_fix_tokenizer
 from llamafactory.data.template import TEMPLATES, parse_template
 from llamafactory.extras.constants import (
     DEFAULT_TEMPLATE,
+    MCA_SUPPORTED_MODELS,
     MULTIMODAL_SUPPORTED_MODELS,
     SUPPORTED_MODELS,
     DownloadSource,
@@ -54,6 +56,16 @@ MESSAGES_WITH_THOUGHT = [
 ]
 
 
+class CharTokenizer:
+    r"""Minimal reversible tokenizer for testing rendered template text."""
+
+    def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
+        return [ord(char) for char in text]
+
+    def decode(self, token_ids: list[int]) -> str:
+        return "".join(chr(token_id) for token_id in token_ids)
+
+
 def _check_tokenization(
     tokenizer: "PreTrainedTokenizer", batch_input_ids: list[list[int]], batch_text: list[str]
 ) -> None:
@@ -113,6 +125,20 @@ def test_moss_vl_registration():
     assert TEMPLATES["moss_vl"].mm_plugin.time_eos_token == "<|time_end|>"
 
 
+def test_qwen38_registration():
+    multimodal_model = "Qwen3.8-27B"
+    text_model = "Qwen3.8-2.4T-A95B-Thinking"
+
+    assert SUPPORTED_MODELS[multimodal_model][DownloadSource.DEFAULT] == "Qwen/Qwen3.8-27B"
+    assert DEFAULT_TEMPLATE[multimodal_model] == "qwen3_8"
+    assert multimodal_model in MULTIMODAL_SUPPORTED_MODELS
+
+    assert SUPPORTED_MODELS[text_model][DownloadSource.DEFAULT] == "Qwen/Qwen3.8-2.4T-A95B"
+    assert DEFAULT_TEMPLATE[text_model] == "qwen3_8"
+    assert text_model not in MULTIMODAL_SUPPORTED_MODELS
+    assert {"qwen3_5_moe_text", "qwen3_5_text"} <= MCA_SUPPORTED_MODELS
+
+
 @pytest.mark.runs_on(["cpu", "mps"])
 def test_encode_oneturn():
     tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)
@@ -240,6 +266,48 @@ def test_reasoning_encode_multiturn_discarding_history_cot(enable_thinking: bool
     )
 
 
+@pytest.mark.runs_on(["cpu", "mps"])
+def test_qwen38_reasoning_instruction():
+    tokenizer = CharTokenizer()
+    template = deepcopy(TEMPLATES["qwen3_8"])
+    encoded_pairs = template.encode_multiturn(tokenizer, MESSAGES[:2], system="Medical assistant")
+
+    prompt_text = tokenizer.decode(encoded_pairs[0][0])
+    answer_text = tokenizer.decode(encoded_pairs[0][1])
+    assert prompt_text == (
+        "<|im_start|>system\n"
+        "Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, "
+        "consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer."
+        "\n\nMedical assistant<|im_end|>\n"
+        "<|im_start|>user\nHow are you<|im_end|>\n<|im_start|>assistant\n"
+    )
+    assert answer_text == "<think>\n\n</think>\n\nI am fine!<|im_end|>\n"
+
+
+@pytest.mark.runs_on(["cpu", "mps"])
+def test_qwen38_tool_system_order():
+    tokenizer = CharTokenizer()
+    template = deepcopy(TEMPLATES["qwen3_8"])
+    tools = '[{"name":"get_weather","parameters":{"type":"object","properties":{}}}]'
+    messages = [
+        {"role": "user", "content": "How is the weather?"},
+        {"role": "function", "content": '{"name":"get_weather","arguments":{}}'},
+    ]
+    encoded_pairs = template.encode_multiturn(tokenizer, messages, system="Use tools safely", tools=tools)
+
+    prompt_text = tokenizer.decode(encoded_pairs[0][0])
+    answer_text = tokenizer.decode(encoded_pairs[0][1])
+    reasoning_text = template._get_reasoning_instruction()
+    tool_text = template.format_tools.apply(content=tools)[0].lstrip("\n")
+    assert prompt_text == (
+        f"<|im_start|>system\n{reasoning_text}\n\n{tool_text}\n\nUse tools safely<|im_end|>\n"
+        "<|im_start|>user\nHow is the weather?<|im_end|>\n<|im_start|>assistant\n"
+    )
+    assert answer_text == (
+        "<think>\n\n</think>\n\n<tool_call>\n<function=get_weather>\n</function>\n</tool_call><|im_end|>\n"
+    )
+
+
 @pytest.mark.runs_on(["cpu", "mps"])
 def test_jinja_template():
     tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)