423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-15 08:08:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix tokenizer

Browse Source 
Former-commit-id: fa463ef279b596d5d53cc169831f51b42031fc05
This commit is contained in:
hoshi-hiyouga 2023-08-09 00:54:54 +08:00 committed by GitHub
parent d2e1fe9b1d
commit eb4b4e3c8c
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/extras/template.py
View File

@ -67,12 +67,12 @@ class Template:
self, self,
tokenizer: "PreTrainedTokenizer" tokenizer: "PreTrainedTokenizer"
) -> Tuple[List[int], List[int]]: ) -> Tuple[List[int], List[int]]:
if tokenizer.bos_token_id and getattr(tokenizer, "add_bos_token", True): if tokenizer.bos_token_id:
bos_ids = [tokenizer.bos_token_id] bos_ids = [tokenizer.bos_token_id]
else: else:
bos_ids = [] # bos token is optional bos_ids = [] # bos token is optional
if tokenizer.eos_token_id and getattr(tokenizer, "add_eos_token", True): if tokenizer.eos_token_id:
eos_ids = [tokenizer.eos_token_id] eos_ids = [tokenizer.eos_token_id]
else: else:
raise ValueError("EOS token is required.") raise ValueError("EOS token is required.")