patch for gemma cpt

Former-commit-id: fc0b19c62f52a90d78b63761dda3d8970a42f2da

src/llmtuner/data/preprocess.py

@@ -37,6 +37,10 @@ def preprocess_pretrain_dataset(
         k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
         for k, t in concatenated_examples.items()
     }
+    if data_args.template == "gemma":
+        for i in range(len(result["input_ids"])):
+            result["input_ids"][i][0] = tokenizer.bos_token_id
+
     return result
 
 

src/llmtuner/extras/constants.py

@@ -503,7 +503,7 @@ register_model_group(
         },
         "OLMo-7B-Chat": {
             DownloadSource.DEFAULT: "allenai/OLMo-7B-Instruct",
-        }
+        },
     },
     module="att_proj",
     template="olmo",