add codegemma

Former-commit-id: 9324176525c2eda22962b0ca1895009b6237e6e3
2026-01-12 09:00:35 +08:00 · 2024-04-16 00:11:15 +08:00
parent b638c65519
commit b5c5283dd6
4 changed files with 23 additions and 7 deletions
--- a/src/llmtuner/model/loader.py
+++ b/src/llmtuner/model/loader.py
@@ -48,7 +48,6 @@ def load_tokenizer(model_args: "ModelArguments") -> "PreTrainedTokenizer":
        tokenizer = AutoTokenizer.from_pretrained(
            model_args.model_name_or_path,
            use_fast=True,
-            split_special_tokens=model_args.split_special_tokens,
            padding_side="right",
            **init_kwargs,
        )