From 5a199af3872afec4daae2c6e2ae0fb19a12a8b5e Mon Sep 17 00:00:00 2001 From: hiyouga Date: Mon, 18 Dec 2023 17:18:11 +0800 Subject: [PATCH] fix tokenizer for Yi chat models #1617 #1875 Former-commit-id: 71a9c1617181b7df46cfb193464fb7e56e6399b1 --- src/llmtuner/hparams/model_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llmtuner/hparams/model_args.py b/src/llmtuner/hparams/model_args.py index 330c48c8..f46f291f 100644 --- a/src/llmtuner/hparams/model_args.py +++ b/src/llmtuner/hparams/model_args.py @@ -19,7 +19,7 @@ class ModelArguments: metadata={"help": "Where to store the pre-trained models downloaded from huggingface.co or modelscope.cn."} ) use_fast_tokenizer: Optional[bool] = field( - default=True, + default=False, metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."} ) split_special_tokens: Optional[bool] = field(