423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-23 22:32:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix bug in preprocessing

Browse Source 
Former-commit-id: 968ce0dcce6bfef582ce37aea6566a65f5aac811
This commit is contained in:
hiyouga 2023-08-02 01:10:28 +08:00
parent 40d277ae5e
commit de407b59ea
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/dsets/preprocess.py
View File

@ -25,8 +25,8 @@ def preprocess_dataset(
for i in range(len(examples["prompt"])): for i in range(len(examples["prompt"])):
query, response = examples["prompt"][i], examples["response"][i] query, response = examples["prompt"][i], examples["response"][i]
query = query + "\n" + examples["query"][i] if "query" in examples and examples["query"][i] else query query = query + "\n" + examples["query"][i] if "query" in examples and examples["query"][i] else query
history = history if "history" in examples and examples["history"][i] else [] history = examples["history"][i] if "history" in examples else None
prefix = prefix if "prefix" in examples and examples["prefix"][i] else "" prefix = examples["prefix"][i] if "prefix" in examples else None
yield query, response, history, prefix yield query, response, history, prefix
def preprocess_pretrain_dataset(examples: Dict[str, List[Any]]) -> Dict[str, Any]: def preprocess_pretrain_dataset(examples: Dict[str, List[Any]]) -> Dict[str, Any]: