From a37e1c11c93454c8e4de0c719e8a8f1ca7ebb6c7 Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Wed, 9 Aug 2023 00:54:54 +0800 Subject: [PATCH] fix tokenizer Former-commit-id: 32fa5e8d706050a30a3eb49f9a6bc2591f9c21ea --- src/llmtuner/extras/template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llmtuner/extras/template.py b/src/llmtuner/extras/template.py index 95d34d47..77d2dd4c 100644 --- a/src/llmtuner/extras/template.py +++ b/src/llmtuner/extras/template.py @@ -67,12 +67,12 @@ class Template: self, tokenizer: "PreTrainedTokenizer" ) -> Tuple[List[int], List[int]]: - if tokenizer.bos_token_id and getattr(tokenizer, "add_bos_token", True): + if tokenizer.bos_token_id: bos_ids = [tokenizer.bos_token_id] else: bos_ids = [] # bos token is optional - if tokenizer.eos_token_id and getattr(tokenizer, "add_eos_token", True): + if tokenizer.eos_token_id: eos_ids = [tokenizer.eos_token_id] else: raise ValueError("EOS token is required.")