423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 20:00:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

add tool test

Browse Source
This commit is contained in:
hiyouga
2024-01-18 10:26:26 +08:00
parent d9f1cae351
commit 83dbfce8c3
9 changed files with 63 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/data/preprocess.py
View File

@@ -97,7 +97,7 @@ def preprocess_packed_supervised_dataset(
messages = examples["prompt"][i] + examples["response"][i]
for turn_idx, (source_ids, target_ids) in enumerate(template.encode_multiturn(
tokenizer, messages, examples["system"][i], examples["tool"][i], 1_000_000
tokenizer, messages, examples["system"][i], examples["tool"][i]
)):
if data_args.train_on_prompt:
source_mask = source_ids