From eb4b4e3c8cd0eb39e9f31a0c307a255cf9caf5bf Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Wed, 9 Aug 2023 00:54:54 +0800 Subject: [PATCH] fix tokenizer Former-commit-id: fa463ef279b596d5d53cc169831f51b42031fc05 --- src/llmtuner/extras/template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/llmtuner/extras/template.py b/src/llmtuner/extras/template.py index 95d34d47..77d2dd4c 100644 --- a/src/llmtuner/extras/template.py +++ b/src/llmtuner/extras/template.py @@ -67,12 +67,12 @@ class Template: self, tokenizer: "PreTrainedTokenizer" ) -> Tuple[List[int], List[int]]: - if tokenizer.bos_token_id and getattr(tokenizer, "add_bos_token", True): + if tokenizer.bos_token_id: bos_ids = [tokenizer.bos_token_id] else: bos_ids = [] # bos token is optional - if tokenizer.eos_token_id and getattr(tokenizer, "add_eos_token", True): + if tokenizer.eos_token_id: eos_ids = [tokenizer.eos_token_id] else: raise ValueError("EOS token is required.")