From 13fd751a78833ca9d40729dd9aa3161365c427b2 Mon Sep 17 00:00:00 2001 From: hiyouga Date: Mon, 18 Dec 2023 17:18:11 +0800 Subject: [PATCH] fix tokenizer for Yi chat models #1617 #1875 Former-commit-id: 9485692c8d367a0b25d3e653db413aa01cb9ad7d --- src/llmtuner/hparams/model_args.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llmtuner/hparams/model_args.py b/src/llmtuner/hparams/model_args.py index 330c48c8..f46f291f 100644 --- a/src/llmtuner/hparams/model_args.py +++ b/src/llmtuner/hparams/model_args.py @@ -19,7 +19,7 @@ class ModelArguments: metadata={"help": "Where to store the pre-trained models downloaded from huggingface.co or modelscope.cn."} ) use_fast_tokenizer: Optional[bool] = field( - default=True, + default=False, metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."} ) split_special_tokens: Optional[bool] = field(