423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-15 08:08:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

patch for gemma cpt

Browse Source 
Former-commit-id: fc0b19c62f52a90d78b63761dda3d8970a42f2da
This commit is contained in:
hiyouga 2024-03-12 21:21:54 +08:00
parent 282edb9161
commit e6ab1a57ea
2 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/data/preprocess.py
View File

@ -37,6 +37,10 @@ def preprocess_pretrain_dataset(
k: [t[i : i + block_size] for i in range(0, total_length, block_size)] k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
for k, t in concatenated_examples.items() for k, t in concatenated_examples.items()
} }
if data_args.template == "gemma":
for i in range(len(result["input_ids"])):
result["input_ids"][i][0] = tokenizer.bos_token_id
return result return result

2
src/llmtuner/extras/constants.py
View File

@ -503,7 +503,7 @@ register_model_group(
}, },
"OLMo-7B-Chat": { "OLMo-7B-Chat": {
DownloadSource.DEFAULT: "allenai/OLMo-7B-Instruct", DownloadSource.DEFAULT: "allenai/OLMo-7B-Instruct",
} },
}, },
module="att_proj", module="att_proj",
template="olmo", template="olmo",