423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 11:50:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

add kto

Browse Source
This commit is contained in:
enji.zhou
2024-05-17 13:09:17 +08:00
parent 84415492bf
commit db1d5a4f51
14 changed files with 5923 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/llamafactory/data/parser.py
View File

@@ -28,6 +28,7 @@ class DatasetAttr:
""" columns """
system: Optional[str] = None
images: Optional[str] = None
tag: Optional[bool] = None
""" columns for the alpaca format """
prompt: Optional[str] = "instruction"
query: Optional[str] = "input"
@@ -106,7 +107,7 @@ def get_dataset_list(data_args: "DataArguments") -> List["DatasetAttr"]:
dataset_attr.set_attr("formatting", dataset_info[name], default="alpaca")
if "columns" in dataset_info[name]:
column_names = ["system", "images"]
column_names = ["system", "images", "tag"]
if dataset_attr.formatting == "alpaca":
column_names.extend(["prompt", "query", "response", "history"])
else: