423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-15 03:10:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix #2516

Browse Source 
Former-commit-id: 02c8c55ce3
This commit is contained in:
hiyouga
2024-02-20 20:44:24 +08:00
parent ba48084086
commit e87a17464f
2 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/llmtuner/data/preprocess.py
View File

@@ -112,9 +112,9 @@ def preprocess_packed_supervised_dataset(
input_ids += source_ids + target_ids
labels += source_mask + target_ids
if template.efficient_eos:
input_ids += [tokenizer.eos_token_id]
labels += [tokenizer.eos_token_id]
if template.efficient_eos:
input_ids += [tokenizer.eos_token_id]
labels += [tokenizer.eos_token_id]
total_length = len(input_ids)
block_size = data_args.cutoff_len