423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-16 00:28:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix sft encode

Browse Source 
Former-commit-id: 2369a96a3200593421ae9afb06e08e2ac8010bb2
This commit is contained in:
hiyouga 2023-07-11 19:50:33 +08:00
parent cc290a41e6
commit 8de7a01887
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/utils/common.py
View File

@ -505,7 +505,7 @@ def preprocess_data(
input_ids, labels = [], [] input_ids, labels = [], []
for i in range(len(dialog) // 2): for i in range(len(dialog) // 2):
source_ids = tokenizer.encode(text=dialog[2*i], add_special_tokens=True) source_ids = tokenizer.encode(text=dialog[2*i], add_special_tokens=(i == 0))
target_ids = tokenizer.encode(text=dialog[2*i+1], add_special_tokens=False) target_ids = tokenizer.encode(text=dialog[2*i+1], add_special_tokens=False)
if len(source_ids) > data_args.max_source_length: if len(source_ids) > data_args.max_source_length: