423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-14 23:58:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix tokenizer

Browse Source 
Former-commit-id: fa463ef279b596d5d53cc169831f51b42031fc05
This commit is contained in:
hoshi-hiyouga 2023-08-09 00:54:54 +08:00 committed by GitHub
parent d2e1fe9b1d
commit eb4b4e3c8c
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/extras/template.py
View File

@ -67,12 +67,12 @@ class Template:
self,
tokenizer: "PreTrainedTokenizer"
) -> Tuple[List[int], List[int]]:
if tokenizer.bos_token_id and getattr(tokenizer, "add_bos_token", True):
if tokenizer.bos_token_id:
bos_ids = [tokenizer.bos_token_id]
else:
bos_ids = [] # bos token is optional
if tokenizer.eos_token_id and getattr(tokenizer, "add_eos_token", True):
if tokenizer.eos_token_id:
eos_ids = [tokenizer.eos_token_id]
else:
raise ValueError("EOS token is required.")