423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-18 12:50:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

enable cutoff len

Browse Source
This commit is contained in:
hiyouga
2024-01-18 12:25:42 +08:00
parent 83dbfce8c3
commit f1067d2b58
8 changed files with 297254 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/data/parser.py
View File

@@ -29,7 +29,7 @@ class DatasetAttr:
history: Optional[str] = None
messages: Optional[str] = "conversations"
tool: Optional[str] = None
tools: Optional[str] = None
role_tag: Optional[str] = "from"
content_tag: Optional[str] = "value"
@@ -86,7 +86,7 @@ def get_dataset_list(data_args: "DataArguments") -> List["DatasetAttr"]:
if dataset_attr.formatting == "alpaca":
column_names = ["prompt", "query", "response", "history"]
else:
column_names = ["messages", "tool"]
column_names = ["messages", "tools"]
column_names += ["system"]
for column_name in column_names: