diff --git a/README.md b/README.md index 6de8a1600..59c04d268 100644 --- a/README.md +++ b/README.md @@ -338,6 +338,7 @@ Read technical notes: | [Seed (OSS/Coder)](https://huggingface.co/ByteDance-Seed) | 8B/36B | seed_oss/seed_coder | | [StarCoder 2](https://huggingface.co/bigcode) | 3B/7B/15B | - | | [TeleChat 2-2.5](https://huggingface.co/Tele-AI) | 3B/7B/35B/115B | telechat2 | +| [Xing4.0-29B-A4B](https://huggingface.co/XingChen-AGI) | 29B-A4B | xing4_0 | | [Yuan 2](https://huggingface.co/IEITYuan) | 2B/51B/102B | yuan | > [!NOTE] diff --git a/README_zh.md b/README_zh.md index e57b6be6c..4a8966d77 100644 --- a/README_zh.md +++ b/README_zh.md @@ -339,6 +339,7 @@ https://github.com/user-attachments/assets/43b700c6-a178-41db-b1f8-8190a5d3fcfc | [Seed (OSS/Coder)](https://huggingface.co/ByteDance-Seed) | 8B/36B | seed_oss/seed_coder | | [StarCoder 2](https://huggingface.co/bigcode) | 3B/7B/15B | - | | [TeleChat 2-2.5](https://huggingface.co/Tele-AI) | 3B/7B/35B/115B | telechat2 | +| [Xing4.0-29B-A4B](https://huggingface.co/XingChen-AGI) | 29B-A4B | xing4_0 | | [Yuan 2](https://huggingface.co/IEITYuan) | 2B/51B/102B | yuan | > [!NOTE] diff --git a/src/llamafactory/data/template.py b/src/llamafactory/data/template.py index d499b4e00..f5ff81072 100644 --- a/src/llamafactory/data/template.py +++ b/src/llamafactory/data/template.py @@ -675,6 +675,62 @@ class Glm47ReasoningTemplate(ReasoningTemplate): return self.thought_words[0] + content + self.thought_words[1] +@dataclass +class Xing4ReasoningTemplate(ReasoningTemplate): + r"""Xing4.0 places the thinking-mode marker in the generation prompt.""" + + @override + def add_thought(self, content: str = "") -> str: + if not self.enable_thinking: + return self.thought_words[1] + content + + return self.thought_words[0] + "\n" + self.thought_words[1] + content + + def _move_thought_prefix( + self, + tokenizer: "PreTrainedTokenizer", + prompt_ids: list[int], + response_ids: list[int], + ) -> tuple[list[int], list[int]]: + if not self.enable_thinking: + return prompt_ids, response_ids + + thought_prefix_ids = tokenizer.encode(self.thought_words[0], add_special_tokens=False) + if response_ids[: len(thought_prefix_ids)] == thought_prefix_ids: + prompt_ids += thought_prefix_ids + response_ids = response_ids[len(thought_prefix_ids) :] + + return prompt_ids, response_ids + + @override + def encode_oneturn( + self, + tokenizer: "PreTrainedTokenizer", + messages: list[dict[str, str]], + system: Optional[str] = None, + tools: Optional[str] = None, + ) -> tuple[list[int], list[int]]: + prompt_ids, response_ids = super().encode_oneturn(tokenizer, messages, system, tools) + return self._move_thought_prefix(tokenizer, prompt_ids, response_ids) + + @override + def encode_multiturn( + self, + tokenizer: "PreTrainedTokenizer", + messages: list[dict[str, str]], + system: Optional[str] = None, + tools: Optional[str] = None, + discarding_history_cot: bool = False, + ) -> list[tuple[list[int], list[int]]]: + encoded_pairs = super().encode_multiturn( + tokenizer, messages, system, tools, discarding_history_cot=discarding_history_cot + ) + return [ + self._move_thought_prefix(tokenizer, prompt_ids, response_ids) + for prompt_ids, response_ids in encoded_pairs + ] + + TEMPLATES: dict[str, "Template"] = {} @@ -2561,6 +2617,25 @@ register_template( ) +register_template( + name="xing4_0", + format_user=StringFormatter(slots=["<_user>{{content}}<_bot>"]), + format_assistant=StringFormatter(slots=["{{content}}", {"eos_token"}, "\n"]), + format_function=FunctionFormatter( + slots=["{{content}}", {"eos_token"}, "\n"], tool_format="xing4" + ), + format_system=StringFormatter(slots=["{{content}}"]), + format_observation=StringFormatter( + slots=["<_observation>{{content}}<_bot>"] + ), + format_prefix=EmptyFormatter(slots=["<_system>"]), + format_tools=ToolFormatter(tool_format="xing4"), + thought_words=("\n", ""), + tool_call_words=("", ""), + template_class=Xing4ReasoningTemplate, +) + + register_template( name="vicuna", format_user=StringFormatter(slots=["USER: {{content}} ASSISTANT:"]), diff --git a/src/llamafactory/data/tool_utils.py b/src/llamafactory/data/tool_utils.py index b74fdc65c..772a21cb3 100644 --- a/src/llamafactory/data/tool_utils.py +++ b/src/llamafactory/data/tool_utils.py @@ -63,6 +63,15 @@ GLM5_NEXT_TOOL_PROMPT = ( "{{arg-key-2}}{{arg-value-2}}..." ) +XING4_TOOL_PROMPT = ( + "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\n" + "You are provided with function signatures within XML tags:\n{tool_text}" + "\n\n\nFor each function call, output the function name and arguments within the following XML format:\n" + "{{function-name}}{{param-key-1}}" + "{{param-value-1}}{{param-key-2}}" + "{{param-value-2}}..." +) + LLAMA3_TOOL_PROMPT = ( "Cutting Knowledge Date: December 2023\nToday Date: {date}\n\n" "You have access to the following functions. To call a function, please respond with JSON for a function call. " @@ -850,6 +859,88 @@ class GLM5NextToolUtils(GLM4MOEToolUtils): return "".join(calls) +class Xing4ToolUtils(ToolUtils): + r"""Xing4 tool using template.""" + + @override + @staticmethod + def tool_formatter(tools: list[dict[str, Any]]) -> str: + tool_text = "".join("\n" + json.dumps(tool, ensure_ascii=False) for tool in tools) + return XING4_TOOL_PROMPT.format(tool_text=tool_text) + + @override + @staticmethod + def function_formatter(functions: list["FunctionCall"]) -> str: + calls = [] + for name, arguments in functions: + if not isinstance(name, str) or not re.fullmatch(r"[^\s<>]+", name): + raise ValueError("Invalid xing4 function name.") + + arguments = json.loads(arguments) + if not isinstance(arguments, dict): + raise ValueError("Xing4 function arguments must be a JSON object.") + + call = f"{name}" + for key, value in arguments.items(): + if not re.fullmatch(r"[^<>]+", key): + raise ValueError("Invalid xing4 parameter name.") + + value = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False) + call += f"{key}{value}" + + calls.append(call + "") + + return "".join(calls) + + @override + @staticmethod + def tool_extractor(content: str) -> Union[str, list["FunctionCall"]]: + call_pattern = re.compile(r"(.*?)", re.DOTALL) + call_matches = list(re.finditer(call_pattern, content)) + if not call_matches: + return content + + if content.count("") != len(call_matches) or content.count("") != len(call_matches): + return content + + results = [] + param_pattern = re.compile(r"(.*?)(.*?)", re.DOTALL) + for call_match in call_matches: + call_content = call_match.group(1) + name_match = re.match(r"\s*([^\s<>]+)", call_content) + if name_match is None: + return content + + arguments = {} + params_content = call_content[name_match.end() :] + position = 0 + for param_match in re.finditer(param_pattern, params_content): + if params_content[position : param_match.start()].strip(): + return content + + key, raw_value = param_match.groups() + key = key.strip() + if not key: + return content + + try: + value = json.loads(raw_value) + except json.JSONDecodeError: + value = raw_value + + arguments[key] = value + position = param_match.end() + + if params_content[position:].strip(): + return content + + results.append( + FunctionCall(name_match.group(1), json.dumps(arguments, ensure_ascii=False)) + ) + + return results + + class SeedToolUtils(ToolUtils): r"""Seed tool using template.""" @@ -1031,6 +1122,7 @@ TOOLS = { "qwen": QwenToolUtils(), "qwen3_5": Qwen35ToolUtils(), "qwen3_8": Qwen38ToolUtils(), + "xing4": Xing4ToolUtils(), "glm4_moe": GLM4MOEToolUtils(), "glm5_next": GLM5NextToolUtils(), "seed_oss": SeedToolUtils(), diff --git a/src/llamafactory/extras/constants.py b/src/llamafactory/extras/constants.py index f3a521fef..cff08b98b 100644 --- a/src/llamafactory/extras/constants.py +++ b/src/llamafactory/extras/constants.py @@ -3422,6 +3422,17 @@ register_model_group( ) +register_model_group( + models={ + "Xing4.0-29B-A4B": { + DownloadSource.DEFAULT: "XingChen-AGI/Xing4.0-29B-A4B", + DownloadSource.MODELSCOPE: "XingChen-AGI/Xing4.0-29B-A4B", + }, + }, + template="xing4_0", +) + + register_model_group( models={ "VibeThinker-1.5B": { diff --git a/src/llamafactory/model/loader.py b/src/llamafactory/model/loader.py index b162c8820..9ff0ce0a6 100644 --- a/src/llamafactory/model/loader.py +++ b/src/llamafactory/model/loader.py @@ -60,6 +60,7 @@ def _get_init_kwargs(model_args: "ModelArguments") -> dict[str, Any]: """ skip_check_imports() model_args.model_name_or_path = try_download_model_from_other_hub(model_args) + return { "trust_remote_code": model_args.trust_remote_code, "cache_dir": model_args.cache_dir, diff --git a/src/llamafactory/model/model_utils/moe.py b/src/llamafactory/model/model_utils/moe.py index 339e9dc99..fde24b418 100644 --- a/src/llamafactory/model/model_utils/moe.py +++ b/src/llamafactory/model/model_utils/moe.py @@ -156,6 +156,10 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None: _set_z3_leaf_modules(model, [Qwen3_5MoeSparseMoeBlock]) + if model_type == "xing4_0": + # Xing4.0 uses custom code + _set_z3_leaf_modules(model, ["Xing4_0MoE"]) + def configure_moe(config: "PretrainedConfig", model_args: "ModelArguments", is_trainable: bool) -> None: if not is_trainable or not model_args.moe_aux_loss_coef: diff --git a/src/llamafactory/model/patcher.py b/src/llamafactory/model/patcher.py index 0b198838a..c97f39a02 100644 --- a/src/llamafactory/model/patcher.py +++ b/src/llamafactory/model/patcher.py @@ -369,6 +369,9 @@ def patch_config( init_kwargs: dict[str, Any], is_trainable: bool, ) -> None: + if getattr(config, "model_type", None) == "xing4_0": + check_version("transformers>=5.4.0", mandatory=True) + if model_args.compute_dtype is None: # priority: bf16 > fp16 > fp32 if model_args.infer_dtype != "auto" and not is_trainable: model_args.compute_dtype = getattr(torch, model_args.infer_dtype) diff --git a/src/llamafactory/v1/core/model_engine.py b/src/llamafactory/v1/core/model_engine.py index 00124d487..a4e23fac2 100644 --- a/src/llamafactory/v1/core/model_engine.py +++ b/src/llamafactory/v1/core/model_engine.py @@ -31,13 +31,14 @@ Init workflow: import torch from accelerate import init_empty_weights -from transformers import AutoConfig, AutoProcessor +from transformers import AutoConfig, AutoProcessor, AutoTokenizer from ..accelerator.helper import DeviceType from ..accelerator.interface import DistributedInterface from ..config.model_args import ModelArguments, ModelClass from ..utils import logging from ..utils.helper import get_tokenizer, is_tokenizer +from ..utils.packages import check_version from ..utils.types import HFConfig, HFModel, Processor from .rendering import Renderer @@ -97,10 +98,17 @@ class ModelEngine: NOTE: Transformers v5 always use fast tokenizer. https://github.com/huggingface/transformers/blob/v5.0.0rc1/src/transformers/models/auto/tokenization_auto.py#L642 """ - return AutoProcessor.from_pretrained( - self.args.model, - trust_remote_code=self.args.trust_remote_code, - ) + try: + return AutoProcessor.from_pretrained( + self.args.model, + trust_remote_code=self.args.trust_remote_code, + ) + except Exception: + logger.info_rank0("Failed to load processor, falling back to tokenizer.") + return AutoTokenizer.from_pretrained( + self.args.model, + trust_remote_code=self.args.trust_remote_code, + ) def _sync_chat_template(self) -> None: """Sync chat_template and inject custom_chat_template.""" @@ -116,10 +124,14 @@ class ModelEngine: def _init_model_config(self) -> HFConfig: """Init model config.""" - return AutoConfig.from_pretrained( + config = AutoConfig.from_pretrained( self.args.model, trust_remote_code=self.args.trust_remote_code, ) + if getattr(config, "model_type", None) == "xing4_0": + check_version("transformers>=5.4.0", mandatory=True) + + return config def _init_model(self) -> HFModel: """Init model. diff --git a/tests/data/test_formatter.py b/tests/data/test_formatter.py index 1d9e9ef2e..bfe095e98 100644 --- a/tests/data/test_formatter.py +++ b/tests/data/test_formatter.py @@ -302,6 +302,65 @@ def test_qwen_multi_tool_extractor(): ] +@pytest.mark.runs_on(["cpu", "mps", "xpu"]) +def test_xing4_function_formatter(): + formatter = FunctionFormatter(slots=["{{content}}", "<_end>\n"], tool_format="xing4") + tool_calls = json.dumps( + { + "name": "tool_name", + "arguments": {"foo": "bar", "size": 10, "enabled": True, "items": [1, 2]}, + } + ) + assert formatter.apply(content=tool_calls) == [ + "tool_namefoobar" + "size10" + "enabledtrue" + "items[1, 2]", + "<_end>\n", + ] + + +@pytest.mark.runs_on(["cpu", "mps", "xpu"]) +def test_xing4_multi_function_formatter(): + formatter = FunctionFormatter(slots=["{{content}}"], tool_format="xing4") + tool_calls = json.dumps([FUNCTION, {"name": "another_tool", "arguments": {"city": "北京"}}]) + assert formatter.apply(content=tool_calls) == [ + "tool_namefoobar" + "size10" + "another_toolcity北京" + ] + + +@pytest.mark.runs_on(["cpu", "mps", "xpu"]) +def test_xing4_tool_formatter(): + formatter = ToolFormatter(tool_format="xing4") + assert formatter.apply(content=json.dumps(TOOLS)) == [ + "\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\n" + "You are provided with function signatures within XML tags:\n" + f"\n{json.dumps(TOOLS[0], ensure_ascii=False)}" + "\n\n\nFor each function call, output the function name and arguments within the following " + "XML format:\n{function-name}{param-key-1}" + "{param-value-1}{param-key-2}" + "{param-value-2}..." + ] + + +@pytest.mark.runs_on(["cpu", "mps", "xpu"]) +def test_xing4_multi_tool_extractor(): + formatter = ToolFormatter(tool_format="xing4") + result = ( + "check tools" + "test_toolfoobar" + "size10" + "another_toolenabledtrue" + "items[1, 2]" + ) + assert formatter.extract(result) == [ + ("test_tool", """{"foo": "bar", "size": 10}"""), + ("another_tool", """{"enabled": true, "items": [1, 2]}"""), + ] + + @pytest.mark.runs_on(["cpu", "mps", "xpu"]) def test_lfm2_function_formatter(): formatter = FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="lfm2")