423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-22 22:02:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix tokenizer for Yi chat models #1617 #1875

Browse Source 
Former-commit-id: 71a9c1617181b7df46cfb193464fb7e56e6399b1
This commit is contained in:
hiyouga 2023-12-18 17:18:11 +08:00
parent dee19b11ba
commit 5a199af387
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/hparams/model_args.py
View File

@ -19,7 +19,7 @@ class ModelArguments:
metadata={"help": "Where to store the pre-trained models downloaded from huggingface.co or modelscope.cn."} metadata={"help": "Where to store the pre-trained models downloaded from huggingface.co or modelscope.cn."}
) )
use_fast_tokenizer: Optional[bool] = field( use_fast_tokenizer: Optional[bool] = field(
default=True, default=False,
metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."} metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}
) )
split_special_tokens: Optional[bool] = field( split_special_tokens: Optional[bool] = field(