423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-23 22:32:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix sft encode

Browse Source 
Former-commit-id: b2f7cb446591e3722b5be8d250ddfe0caa226384
This commit is contained in:
hiyouga 2023-07-11 19:50:33 +08:00
parent 685ba1e02e
commit 925a026853
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/utils/common.py
View File

@ -505,7 +505,7 @@ def preprocess_data(
input_ids, labels = [], [] input_ids, labels = [], []
for i in range(len(dialog) // 2): for i in range(len(dialog) // 2):
source_ids = tokenizer.encode(text=dialog[2*i], add_special_tokens=True) source_ids = tokenizer.encode(text=dialog[2*i], add_special_tokens=(i == 0))
target_ids = tokenizer.encode(text=dialog[2*i+1], add_special_tokens=False) target_ids = tokenizer.encode(text=dialog[2*i+1], add_special_tokens=False)
if len(source_ids) > data_args.max_source_length: if len(source_ids) > data_args.max_source_length: