423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-15 03:10:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix encode

Browse Source
This commit is contained in:
hiyouga
2023-08-04 23:27:55 +08:00
parent b4852f9406
commit 8172ad1b5e
2 changed files with 19 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/dsets/preprocess.py
View File

@@ -55,7 +55,7 @@ def preprocess_dataset(
for query, response, history, prefix in construct_example(examples):
input_ids, labels = [], []
for source_ids, target_ids in template.get_dialog(tokenizer, query, response, history, prefix):
for source_ids, target_ids in template.get_dialog(tokenizer, query, response, history, prefix): # TODO: fix bos
if len(source_ids) > data_args.max_source_length:
source_ids = source_ids[:data_args.max_source_length]
if len(target_ids) > data_args.max_target_length - 1: # eos token