423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-23 14:22:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix bug in packed sft dataset

Browse Source 
Former-commit-id: de196143064772db770a45235424b3c911b2e147
This commit is contained in:
hiyouga 2023-09-28 01:16:46 +08:00
parent f61a000e73
commit f88088c43d
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/dsets/preprocess.py
View File

@ -116,7 +116,7 @@ def preprocess_dataset(
# split by chunks of cutoff_len # split by chunks of cutoff_len
for i in range(0, total_length, block_size): for i in range(0, total_length, block_size):
model_inputs["input_ids"].append(input_ids[i: i + block_size]) model_inputs["input_ids"].append(input_ids[i: i + block_size])
model_inputs["attention_mask"].append([1] * len(block_size)) model_inputs["attention_mask"].append([1] * block_size)
model_inputs["labels"].append(labels[i: i + block_size]) model_inputs["labels"].append(labels[i: i + block_size])
return model_inputs return model_inputs