add codegemma

Former-commit-id: 9324176525c2eda22962b0ca1895009b6237e6e3
2025-12-17 04:10:36 +08:00 · 2024-04-16 00:11:15 +08:00
parent 19874e39ee
commit 2aa1d1476e
4 changed files with 23 additions and 7 deletions
--- a/src/llmtuner/model/loader.py
+++ b/src/llmtuner/model/loader.py
@@ -48,7 +48,6 @@ def load_tokenizer(model_args: "ModelArguments") -> "PreTrainedTokenizer":
        tokenizer = AutoTokenizer.from_pretrained(
            model_args.model_name_or_path,
            use_fast=True,
-            split_special_tokens=model_args.split_special_tokens,
            padding_side="right",
            **init_kwargs,
        )