423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-19 21:30:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

tiny fix

Browse Source
This commit is contained in:
hiyouga
2024-09-05 23:41:16 +08:00
parent eb5aa9adce
commit 76f2e59504
5 changed files with 7 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llamafactory/data/processors/pretrain.py
View File

@@ -36,7 +36,7 @@ def preprocess_pretrain_dataset(
if data_args.template == "gemma":
text_examples = [tokenizer.bos_token + example for example in text_examples]
result = tokenizer(text_examples, add_special_tokens=False, max_length=data_args.cutoff_len, truncation=True)
result = tokenizer(text_examples, add_special_tokens=False, truncation=True, max_length=data_args.cutoff_len)
else:
tokenized_examples = tokenizer(text_examples, add_special_tokens=False)
concatenated_examples = {k: list(chain(*tokenized_examples[k])) for k in tokenized_examples.keys()}