423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-15 16:18:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix tokenizer for Yi chat models #1617 #1875

Browse Source 
Former-commit-id: 9485692c8d367a0b25d3e653db413aa01cb9ad7d
This commit is contained in:
hiyouga 2023-12-18 17:18:11 +08:00
parent 74575f8922
commit 13fd751a78
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/hparams/model_args.py
View File

@ -19,7 +19,7 @@ class ModelArguments:
metadata={"help": "Where to store the pre-trained models downloaded from huggingface.co or modelscope.cn."} metadata={"help": "Where to store the pre-trained models downloaded from huggingface.co or modelscope.cn."}
) )
use_fast_tokenizer: Optional[bool] = field( use_fast_tokenizer: Optional[bool] = field(
default=True, default=False,
metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."} metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}
) )
split_special_tokens: Optional[bool] = field( split_special_tokens: Optional[bool] = field(