423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-15 03:10:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix tokenizer for Yi chat models #1617 #1875

Browse Source
This commit is contained in:
hiyouga
2023-12-18 17:18:11 +08:00
parent 2b4e5f0d32
commit 71a9c16171
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/hparams/model_args.py
View File

@@ -19,7 +19,7 @@ class ModelArguments:
metadata={"help": "Where to store the pre-trained models downloaded from huggingface.co or modelscope.cn."} metadata={"help": "Where to store the pre-trained models downloaded from huggingface.co or modelscope.cn."}
) )
use_fast_tokenizer: Optional[bool] = field( use_fast_tokenizer: Optional[bool] = field(
default=True, default=False,
metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."} metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}
) )
split_special_tokens: Optional[bool] = field( split_special_tokens: Optional[bool] = field(