Former-commit-id: aa5e921c00f60074eceb2f9d4d8837cc713edba6
This commit is contained in:
hoshi-hiyouga 2024-04-20 10:34:09 +08:00
parent 0cb596fee1
commit 3a5e68b7d9

View File

@ -44,7 +44,7 @@ def load_tokenizer(model_args: "ModelArguments") -> "PreTrainedTokenizer":
padding_side="right",
**init_kwargs,
)
except ValueError: # try the fast one
except Exception: # try the fast one
tokenizer = AutoTokenizer.from_pretrained(
model_args.model_name_or_path,
use_fast=True,