423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-29 18:20:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

enable cutoff len

Browse Source 
Former-commit-id: e9513d300c338dfcae98eee7d057bfd00da2da0e
This commit is contained in:
hiyouga
2024-01-18 12:25:42 +08:00
parent d8affd3967
commit e4a424cb6a
6 changed files with 46 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/data/parser.py
View File

@@ -29,7 +29,7 @@ class DatasetAttr:
history: Optional[str] = None
messages: Optional[str] = "conversations"
tool: Optional[str] = None
tools: Optional[str] = None
role_tag: Optional[str] = "from"
content_tag: Optional[str] = "value"
@@ -86,7 +86,7 @@ def get_dataset_list(data_args: "DataArguments") -> List["DatasetAttr"]:
if dataset_attr.formatting == "alpaca":
column_names = ["prompt", "query", "response", "history"]
else:
column_names = ["messages", "tool"]
column_names = ["messages", "tools"]
column_names += ["system"]
for column_name in column_names: