423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-20 05:40:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

support new special token #3420

Browse Source
This commit is contained in:
hiyouga
2024-04-24 23:39:31 +08:00
parent e5d23c053a
commit 297fb8ead3
8 changed files with 47 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/hparams/data_args.py
View File

@@ -26,11 +26,11 @@ class DataArguments:
)
cutoff_len: int = field(
default=1024,
metadata={"help": "The cutoff length of the model inputs after tokenization."},
metadata={"help": "The cutoff length of the tokenized inputs in the dataset."},
)
reserved_label_len: int = field(
default=1,
metadata={"help": "The minimum cutoff length reserved for label after tokenization."},
metadata={"help": "The minimum cutoff length reserved for the tokenized labels in the dataset."},
)
train_on_prompt: bool = field(
default=False,