From 4451765a6b04ff08a6c5650f5953513608ae9e64 Mon Sep 17 00:00:00 2001 From: Hertz <2267379130@qq.com> Date: Wed, 2 Sep 2026 15:52:22 +0800 Subject: [PATCH] [model] add Qwen3.8 model support (#10749) Co-authored-by: hiyouga --- src/llamafactory/data/template.py | 101 +++++++++++++++++- src/llamafactory/data/tool_utils.py | 15 +++ src/llamafactory/extras/constants.py | 33 ++++++ src/llamafactory/hparams/data_args.py | 18 +++- .../model/model_utils/liger_kernel.py | 4 + src/llamafactory/model/model_utils/moe.py | 2 +- src/llamafactory/model/patcher.py | 13 ++- .../kernels/ops/mlp/npu_fused_moe.py | 3 + .../kernels/ops/mlp/npu_swiglu.py | 6 ++ .../kernels/ops/rms_norm/npu_rms_norm.py | 8 ++ .../kernels/ops/rope/npu_rope.py | 2 + tests/data/test_formatter.py | 8 ++ tests/data/test_template.py | 68 ++++++++++++ 13 files changed, 272 insertions(+), 9 deletions(-) diff --git a/src/llamafactory/data/template.py b/src/llamafactory/data/template.py index 37952fc06..08af15464 100644 --- a/src/llamafactory/data/template.py +++ b/src/llamafactory/data/template.py @@ -522,6 +522,80 @@ class ReasoningTemplate(Template): return [(encoded_messages[i], encoded_messages[i + 1]) for i in range(0, len(encoded_messages), 2)] +@dataclass +class Qwen38ReasoningTemplate(ReasoningTemplate): + r"""Qwen3.8 template with reasoning-effort instructions and official system ordering.""" + + reasoning_effort: str = "xhigh" + + def _get_reasoning_instruction(self) -> str: + if self.enable_thinking is False: + return "" + + if self.reasoning_effort == "xhigh": + return ( + "Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, " + "consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final " + "answer." + ) + elif self.reasoning_effort == "medium": + return "" + elif self.reasoning_effort == "low": + return ( + "Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the " + "conclusion without unnecessary elaboration." + ) + else: + # Defensive validation for callers that configure the template without DataArguments. + raise ValueError( + f"Unexpected reasoning effort {self.reasoning_effort}. " + "Supported types are xhigh (default), medium, and low." + ) + + @override + def _encode( + self, + tokenizer: "PreTrainedTokenizer", + messages: list[dict[str, str]], + system: Optional[str], + tools: Optional[str], + ) -> list[list[int]]: + system = (system or self.default_system).strip() + reasoning_instruction = self._get_reasoning_instruction() + encoded_messages = [] + for i, message in enumerate(messages): + elements = [] + + if i == 0: + elements += self.format_prefix.apply() + system_parts = [] + if reasoning_instruction: + system_parts.append(reasoning_instruction) + if tools: + system_parts.append(self.format_tools.apply(content=tools)[0].lstrip("\n")) + if system: + system_parts.append(system) + if system_parts: + elements += self.format_system.apply(content="\n\n".join(system_parts)) + + if message["role"] == Role.USER: + elements += self.format_user.apply(content=message["content"], idx=str(i // 2)) + elif message["role"] == Role.ASSISTANT: + elements += self.format_assistant.apply(content=message["content"]) + elif message["role"] == Role.OBSERVATION: + elements += self.format_observation.apply(content=message["content"]) + elif message["role"] == Role.FUNCTION: + elements += self.format_function.apply( + content=message["content"], thought_words=self.thought_words, tool_call_words=self.tool_call_words + ) + else: + raise NotImplementedError("Unexpected role: {}".format(message["role"])) + + encoded_messages.append(self._convert_elements_to_ids(tokenizer, elements)) + + return encoded_messages + + @dataclass class Glm47ReasoningTemplate(ReasoningTemplate): r"""GLM-4.7 uses only the closing tag for empty thinking blocks.""" @@ -693,6 +767,9 @@ def get_template_and_fix_tokenizer(tokenizer: "PreTrainedTokenizer", data_args: if data_args.train_on_prompt and template.efficient_eos: raise ValueError("Current template does not support `train_on_prompt`.") + if isinstance(template, Qwen38ReasoningTemplate) and data_args.tool_format not in {None, "qwen3_8"}: + raise ValueError("Template `qwen3_8` uses its built-in tool format; remove the incompatible `tool_format`.") + if data_args.tool_format is not None: logger.info_rank0(f"Using tool format: {data_args.tool_format}.") default_slots = ["{{content}}"] if template.efficient_eos else ["{{content}}", {"eos_token"}] @@ -712,7 +789,11 @@ def get_template_and_fix_tokenizer(tokenizer: "PreTrainedTokenizer", data_args: "e.g., qwen3_vl_nothink" ) template.enable_thinking = data_args.enable_thinking - template.preserve_thinking = data_args.preserve_thinking + if isinstance(template, Qwen38ReasoningTemplate): + template.reasoning_effort = data_args.reasoning_effort + template.preserve_thinking = True if data_args.preserve_thinking is None else data_args.preserve_thinking + elif data_args.preserve_thinking is not None: + template.preserve_thinking = data_args.preserve_thinking template.fix_special_tokens(tokenizer) template.fix_jinja_template(tokenizer) @@ -2291,6 +2372,24 @@ register_template( ) +register_template( + name="qwen3_8", + format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]), + format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]), + format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]), + format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen3_8"), + format_observation=StringFormatter( + slots=["<|im_start|>user\n\n{{content}}\n<|im_end|>\n<|im_start|>assistant\n"] + ), + format_tools=ToolFormatter(tool_format="qwen3_8"), + stop_words=["<|im_end|>"], + replace_eos=True, + preserve_thinking=True, + mm_plugin=get_mm_plugin(name="qwen3_vl", image_token="<|image_pad|>", video_token="<|video_pad|>"), + template_class=Qwen38ReasoningTemplate, +) + + register_template( name="sailor", format_user=StringFormatter(slots=["<|im_start|>question\n{{content}}<|im_end|>\n<|im_start|>answer\n"]), diff --git a/src/llamafactory/data/tool_utils.py b/src/llamafactory/data/tool_utils.py index 773fd35d8..d2f322d32 100644 --- a/src/llamafactory/data/tool_utils.py +++ b/src/llamafactory/data/tool_utils.py @@ -758,6 +758,20 @@ class Qwen35ToolUtils(ToolUtils): return results if results else content +class Qwen38ToolUtils(Qwen35ToolUtils): + r"""Qwen 3.8 tool template preserving the OpenAI function wrapper.""" + + @override + @staticmethod + def tool_formatter(tools: list[dict[str, Any]]) -> str: + tool_text = "" + for tool in tools: + wrapped_tool = tool if tool.get("type") == "function" else {"type": "function", "function": tool} + tool_text += "\n" + json.dumps(wrapped_tool, ensure_ascii=False) + + return QWEN35_TOOL_PROMPT.format(tool_text=tool_text) + + class GLM4MOEToolUtils(QwenToolUtils): r"""GLM-4-MOE tool using template.""" @@ -970,6 +984,7 @@ TOOLS = { "mistral": MistralToolUtils(), "qwen": QwenToolUtils(), "qwen3_5": Qwen35ToolUtils(), + "qwen3_8": Qwen38ToolUtils(), "glm4_moe": GLM4MOEToolUtils(), "seed_oss": SeedToolUtils(), "ling": LingToolUtils(), diff --git a/src/llamafactory/extras/constants.py b/src/llamafactory/extras/constants.py index e0167dc0b..f3a521fef 100644 --- a/src/llamafactory/extras/constants.py +++ b/src/llamafactory/extras/constants.py @@ -71,6 +71,8 @@ MCA_SUPPORTED_MODELS = { "qwen3_next", "qwen3_5", "qwen3_5_moe", + "qwen3_5_moe_text", + "qwen3_5_text", } # Text LLM model_types supported by the Megatron Bridge PT/SFT path (gpt_step). @@ -3000,6 +3002,37 @@ register_model_group( ) +register_model_group( + models={ + "Qwen3.8-27B": { + DownloadSource.DEFAULT: "Qwen/Qwen3.8-27B", + DownloadSource.MODELSCOPE: "Qwen/Qwen3.8-27B", + }, + "Qwen3.8-27B-FP8": { + DownloadSource.DEFAULT: "Qwen/Qwen3.8-27B-FP8", + DownloadSource.MODELSCOPE: "Qwen/Qwen3.8-27B-FP8", + }, + }, + template="qwen3_8", + multimodal=True, +) + + +register_model_group( + models={ + "Qwen3.8-2.4T-A95B-Thinking": { + DownloadSource.DEFAULT: "Qwen/Qwen3.8-2.4T-A95B", + DownloadSource.MODELSCOPE: "Qwen/Qwen3.8-2.4T-A95B", + }, + "Qwen3.8-2.4T-A95B-Thinking-FP8": { + DownloadSource.DEFAULT: "Qwen/Qwen3.8-2.4T-A95B-FP8", + DownloadSource.MODELSCOPE: "Qwen/Qwen3.8-2.4T-A95B-FP8", + }, + }, + template="qwen3_8", +) + + register_model_group( models={ "Qwen2-Audio-7B": { diff --git a/src/llamafactory/hparams/data_args.py b/src/llamafactory/hparams/data_args.py index 9267657c1..68ee772cc 100644 --- a/src/llamafactory/hparams/data_args.py +++ b/src/llamafactory/hparams/data_args.py @@ -125,9 +125,18 @@ class DataArguments: default=True, metadata={"help": "Whether or not to enable thinking mode for reasoning models."}, ) - preserve_thinking: bool = field( - default=False, - metadata={"help": "Whether or not to preserve thinking content in historical turns for reasoning models."}, + reasoning_effort: str = field( + default="xhigh", + metadata={"help": "Reasoning effort for supported reasoning models (xhigh, medium, or low)."}, + ) + preserve_thinking: bool | None = field( + default=None, + metadata={ + "help": ( + "Whether or not to preserve thinking content in historical turns for reasoning models. " + "Uses the template default when unspecified." + ) + }, ) tokenized_path: str | None = field( default=None, @@ -182,6 +191,9 @@ class DataArguments: if self.mask_history and self.train_on_prompt: raise ValueError("`mask_history` is incompatible with `train_on_prompt`.") + if self.reasoning_effort not in {"xhigh", "medium", "low"}: + raise ValueError("`reasoning_effort` must be one of xhigh, medium, or low.") + if self.neat_packing: self.packing = True diff --git a/src/llamafactory/model/model_utils/liger_kernel.py b/src/llamafactory/model/model_utils/liger_kernel.py index 4f63e10c1..0eece68f8 100644 --- a/src/llamafactory/model/model_utils/liger_kernel.py +++ b/src/llamafactory/model/model_utils/liger_kernel.py @@ -82,8 +82,12 @@ def apply_liger_kernel( from liger_kernel.transformers import apply_liger_kernel_to_qwen3_next as apply_liger_kernel elif model_type == "qwen3_5": from liger_kernel.transformers import apply_liger_kernel_to_qwen3_5 as apply_liger_kernel + elif model_type == "qwen3_5_text": + from liger_kernel.transformers import apply_liger_kernel_to_qwen3_5_text as apply_liger_kernel elif model_type == "qwen3_5_moe": from liger_kernel.transformers import apply_liger_kernel_to_qwen3_5_moe as apply_liger_kernel + elif model_type == "qwen3_5_moe_text": + from liger_kernel.transformers import apply_liger_kernel_to_qwen3_5_moe_text as apply_liger_kernel elif model_type == "gpt_oss": try: from liger_kernel.transformers import apply_liger_kernel_to_gpt_oss as apply_liger_kernel diff --git a/src/llamafactory/model/model_utils/moe.py b/src/llamafactory/model/model_utils/moe.py index 478dbf9e8..339e9dc99 100644 --- a/src/llamafactory/model/model_utils/moe.py +++ b/src/llamafactory/model/model_utils/moe.py @@ -151,7 +151,7 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None: _set_z3_leaf_modules(model, [Qwen3NextSparseMoeBlock]) - if model_type == "qwen3_5_moe": + if model_type in ("qwen3_5_moe", "qwen3_5_moe_text"): from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import Qwen3_5MoeSparseMoeBlock _set_z3_leaf_modules(model, [Qwen3_5MoeSparseMoeBlock]) diff --git a/src/llamafactory/model/patcher.py b/src/llamafactory/model/patcher.py index 79c57603e..a3e2d54b6 100644 --- a/src/llamafactory/model/patcher.py +++ b/src/llamafactory/model/patcher.py @@ -139,7 +139,7 @@ def patch_qwen3_5_forward_npu(model: "PreTrainedModel") -> None: def patch_qwen3_5_forward_gpu(model: "PreTrainedModel") -> None: - """Patch the forward method of Qwen3_5ForConditionalGeneration to support cu_seqlens input only patch when do training. + """Patch Qwen3.5 decoder forward methods to support cu_seqlens during training. Refer to: https://github.com/axolotl-ai-cloud/axolotl/blob/main/src/axolotl/monkeypatch/models/qwen3_5/modeling.py. """ @@ -286,12 +286,12 @@ def patch_qwen3_5_forward_gpu(model: "PreTrainedModel") -> None: return output - if model.config.architectures[0] == "Qwen3_5ForConditionalGeneration": + if model.config.architectures[0] in ("Qwen3_5ForCausalLM", "Qwen3_5ForConditionalGeneration"): from transformers.models.qwen3_5.modeling_qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5GatedDeltaNet Qwen3_5DecoderLayer.forward = _patched_decoder_forward Qwen3_5GatedDeltaNet.forward = _patch_gdn_forward - elif model.config.architectures[0] == "Qwen3_5MoeForConditionalGeneration": + elif model.config.architectures[0] in ("Qwen3_5MoeForCausalLM", "Qwen3_5MoeForConditionalGeneration"): from transformers.models.qwen3_5_moe.modeling_qwen3_5_moe import ( Qwen3_5MoeDecoderLayer, Qwen3_5MoeGatedDeltaNet, @@ -484,7 +484,12 @@ def patch_model( autocast_projector_dtype(model, model_args) add_z3_leaf_module(model) - if getattr(model.config, "model_type", None) in ["qwen3_5", "qwen3_5_moe"]: + if getattr(model.config, "model_type", None) in [ + "qwen3_5", + "qwen3_5_moe", + "qwen3_5_moe_text", + "qwen3_5_text", + ]: if is_torch_npu_available(): patch_qwen3_5_forward_npu(model) elif is_torch_cuda_available() and model_args.flash_attn == "fa2": diff --git a/src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/npu_fused_moe.py b/src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/npu_fused_moe.py index 756ffe84b..e377acca9 100644 --- a/src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/npu_fused_moe.py +++ b/src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/npu_fused_moe.py @@ -363,6 +363,9 @@ _V5_MODEL_TYPE_TO_PATCHES = { "qwen3_5_moe": { "Qwen3_5MoeExperts": NpuMoeFusedV5.experts_forward, }, + "qwen3_5_moe_text": { + "Qwen3_5MoeExperts": NpuMoeFusedV5.experts_forward, + }, } _MODEL_TYPE_TO_PATCHES = ( diff --git a/src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/npu_swiglu.py b/src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/npu_swiglu.py index ae0ad7c96..e56e09fe2 100644 --- a/src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/npu_swiglu.py +++ b/src/llamafactory/v1/plugins/model_plugins/kernels/ops/mlp/npu_swiglu.py @@ -83,9 +83,15 @@ _MODEL_TYPE_TO_PATCHES = { "qwen3_5": { "Qwen3_5MLP": npu_swiglu_forward, }, + "qwen3_5_text": { + "Qwen3_5MLP": npu_swiglu_forward, + }, "qwen3_5_moe": { "Qwen3_5MoeMLP": npu_swiglu_forward, }, + "qwen3_5_moe_text": { + "Qwen3_5MoeMLP": npu_swiglu_forward, + }, } diff --git a/src/llamafactory/v1/plugins/model_plugins/kernels/ops/rms_norm/npu_rms_norm.py b/src/llamafactory/v1/plugins/model_plugins/kernels/ops/rms_norm/npu_rms_norm.py index 0e960b6c7..b6c2cf8ce 100644 --- a/src/llamafactory/v1/plugins/model_plugins/kernels/ops/rms_norm/npu_rms_norm.py +++ b/src/llamafactory/v1/plugins/model_plugins/kernels/ops/rms_norm/npu_rms_norm.py @@ -148,10 +148,18 @@ _MODEL_TYPE_TO_PATCHES = { "Qwen3_5RMSNorm": npu_residual_rms_norm_forward, "Qwen3_5RMSNormGated": npu_gated_rms_norm_forward, }, + "qwen3_5_text": { + "Qwen3_5RMSNorm": npu_residual_rms_norm_forward, + "Qwen3_5RMSNormGated": npu_gated_rms_norm_forward, + }, "qwen3_5_moe": { "Qwen3_5MoeRMSNorm": npu_residual_rms_norm_forward, "Qwen3_5MoeRMSNormGated": npu_gated_rms_norm_forward, }, + "qwen3_5_moe_text": { + "Qwen3_5MoeRMSNorm": npu_residual_rms_norm_forward, + "Qwen3_5MoeRMSNormGated": npu_gated_rms_norm_forward, + }, } diff --git a/src/llamafactory/v1/plugins/model_plugins/kernels/ops/rope/npu_rope.py b/src/llamafactory/v1/plugins/model_plugins/kernels/ops/rope/npu_rope.py index adb177a74..74fdefde0 100644 --- a/src/llamafactory/v1/plugins/model_plugins/kernels/ops/rope/npu_rope.py +++ b/src/llamafactory/v1/plugins/model_plugins/kernels/ops/rope/npu_rope.py @@ -121,7 +121,9 @@ _MODEL_TYPE_TO_PATCHES = { "qwen3_vl": _default_rope_patch("qwen3_vl"), "qwen3_vl_moe": _default_rope_patch("qwen3_vl_moe"), "qwen3_5": _default_rope_patch("qwen3_5"), + "qwen3_5_text": _default_rope_patch("qwen3_5"), "qwen3_5_moe": _default_rope_patch("qwen3_5_moe"), + "qwen3_5_moe_text": _default_rope_patch("qwen3_5_moe"), } diff --git a/tests/data/test_formatter.py b/tests/data/test_formatter.py index b60621fca..8a9748c5b 100644 --- a/tests/data/test_formatter.py +++ b/tests/data/test_formatter.py @@ -281,6 +281,14 @@ def test_qwen_tool_extractor(): assert formatter.extract(result) == [("test_tool", """{"foo": "bar", "size": 10}""")] +@pytest.mark.runs_on(["cpu", "mps"]) +def test_qwen38_tool_formatter(): + formatter = ToolFormatter(tool_format="qwen3_8") + wrapped_tool = {"type": "function", "function": TOOLS[0]} + output = formatter.apply(content=json.dumps(TOOLS))[0] + assert json.dumps(wrapped_tool, ensure_ascii=False) in output + + @pytest.mark.runs_on(["cpu", "mps"]) def test_qwen_multi_tool_extractor(): formatter = ToolFormatter(tool_format="qwen") diff --git a/tests/data/test_template.py b/tests/data/test_template.py index b45df8ee5..a08cfe0ca 100644 --- a/tests/data/test_template.py +++ b/tests/data/test_template.py @@ -13,6 +13,7 @@ # limitations under the License. import os +from copy import deepcopy from typing import TYPE_CHECKING import pytest @@ -22,6 +23,7 @@ from llamafactory.data import get_template_and_fix_tokenizer from llamafactory.data.template import TEMPLATES, parse_template from llamafactory.extras.constants import ( DEFAULT_TEMPLATE, + MCA_SUPPORTED_MODELS, MULTIMODAL_SUPPORTED_MODELS, SUPPORTED_MODELS, DownloadSource, @@ -54,6 +56,16 @@ MESSAGES_WITH_THOUGHT = [ ] +class CharTokenizer: + r"""Minimal reversible tokenizer for testing rendered template text.""" + + def encode(self, text: str, add_special_tokens: bool = False) -> list[int]: + return [ord(char) for char in text] + + def decode(self, token_ids: list[int]) -> str: + return "".join(chr(token_id) for token_id in token_ids) + + def _check_tokenization( tokenizer: "PreTrainedTokenizer", batch_input_ids: list[list[int]], batch_text: list[str] ) -> None: @@ -113,6 +125,20 @@ def test_moss_vl_registration(): assert TEMPLATES["moss_vl"].mm_plugin.time_eos_token == "<|time_end|>" +def test_qwen38_registration(): + multimodal_model = "Qwen3.8-27B" + text_model = "Qwen3.8-2.4T-A95B-Thinking" + + assert SUPPORTED_MODELS[multimodal_model][DownloadSource.DEFAULT] == "Qwen/Qwen3.8-27B" + assert DEFAULT_TEMPLATE[multimodal_model] == "qwen3_8" + assert multimodal_model in MULTIMODAL_SUPPORTED_MODELS + + assert SUPPORTED_MODELS[text_model][DownloadSource.DEFAULT] == "Qwen/Qwen3.8-2.4T-A95B" + assert DEFAULT_TEMPLATE[text_model] == "qwen3_8" + assert text_model not in MULTIMODAL_SUPPORTED_MODELS + assert {"qwen3_5_moe_text", "qwen3_5_text"} <= MCA_SUPPORTED_MODELS + + @pytest.mark.runs_on(["cpu", "mps"]) def test_encode_oneturn(): tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3) @@ -240,6 +266,48 @@ def test_reasoning_encode_multiturn_discarding_history_cot(enable_thinking: bool ) +@pytest.mark.runs_on(["cpu", "mps"]) +def test_qwen38_reasoning_instruction(): + tokenizer = CharTokenizer() + template = deepcopy(TEMPLATES["qwen3_8"]) + encoded_pairs = template.encode_multiturn(tokenizer, MESSAGES[:2], system="Medical assistant") + + prompt_text = tokenizer.decode(encoded_pairs[0][0]) + answer_text = tokenizer.decode(encoded_pairs[0][1]) + assert prompt_text == ( + "<|im_start|>system\n" + "Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, " + "consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer." + "\n\nMedical assistant<|im_end|>\n" + "<|im_start|>user\nHow are you<|im_end|>\n<|im_start|>assistant\n" + ) + assert answer_text == "\n\n\n\nI am fine!<|im_end|>\n" + + +@pytest.mark.runs_on(["cpu", "mps"]) +def test_qwen38_tool_system_order(): + tokenizer = CharTokenizer() + template = deepcopy(TEMPLATES["qwen3_8"]) + tools = '[{"name":"get_weather","parameters":{"type":"object","properties":{}}}]' + messages = [ + {"role": "user", "content": "How is the weather?"}, + {"role": "function", "content": '{"name":"get_weather","arguments":{}}'}, + ] + encoded_pairs = template.encode_multiturn(tokenizer, messages, system="Use tools safely", tools=tools) + + prompt_text = tokenizer.decode(encoded_pairs[0][0]) + answer_text = tokenizer.decode(encoded_pairs[0][1]) + reasoning_text = template._get_reasoning_instruction() + tool_text = template.format_tools.apply(content=tools)[0].lstrip("\n") + assert prompt_text == ( + f"<|im_start|>system\n{reasoning_text}\n\n{tool_text}\n\nUse tools safely<|im_end|>\n" + "<|im_start|>user\nHow is the weather?<|im_end|>\n<|im_start|>assistant\n" + ) + assert answer_text == ( + "\n\n\n\n\n\n\n<|im_end|>\n" + ) + + @pytest.mark.runs_on(["cpu", "mps"]) def test_jinja_template(): tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)