refactor dataset_attr, add eos in pt, fix #757

This commit is contained in:
hiyouga
2023-09-01 19:00:45 +08:00
parent cf106e9d65
commit a9d1fb72f7
20 changed files with 160 additions and 198 deletions

View File

@@ -16,7 +16,7 @@ class ModelArguments:
metadata={"help": "Where to store the pretrained models downloaded from huggingface.co."}
)
use_fast_tokenizer: Optional[bool] = field(
default=False,
default=True,
metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}
)
use_auth_token: Optional[bool] = field(