add codegemma

Former-commit-id: 6543f3d449

src/llmtuner/extras/constants.py

@@ -385,6 +385,23 @@ register_model_group(
 )
 
 
+register_model_group(
+    models={
+        "CodeGemma-2B": {
+            DownloadSource.DEFAULT: "google/codegemma-2b",
+        },
+        "CodeGemma-7B": {
+            DownloadSource.DEFAULT: "google/codegemma-7b",
+        },
+        "CodeGemma-7B-Chat": {
+            DownloadSource.DEFAULT: "google/codegemma-7b-it",
+            DownloadSource.MODELSCOPE: "AI-ModelScope/codegemma-7b-it",
+        },
+    },
+    template="gemma",
+)
+
+
 register_model_group(
     models={
         "InternLM-7B": {

src/llmtuner/model/loader.py

@@ -48,7 +48,6 @@ def load_tokenizer(model_args: "ModelArguments") -> "PreTrainedTokenizer":
         tokenizer = AutoTokenizer.from_pretrained(
             model_args.model_name_or_path,
             use_fast=True,
-            split_special_tokens=model_args.split_special_tokens,
             padding_side="right",
             **init_kwargs,
         )