423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 20:00:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

alert pad_token source

Browse Source
This commit is contained in:
hiyouga
2023-08-15 00:07:56 +08:00
parent 9d0f6214b6
commit 80b4053602
2 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/llmtuner/extras/template.py
View File

@@ -204,7 +204,10 @@ def get_template_and_fix_tokenizer(
logger.info("Add eos token: {}".format(tokenizer.eos_token))
if tokenizer.pad_token_id is None:
tokenizer.pad_token = tokenizer.eos_token
if tokenizer.unk_token_id is not None:
tokenizer.pad_token = tokenizer.unk_token
else:
tokenizer.pad_token = tokenizer.eos_token
logger.info("Add pad token: {}".format(tokenizer.pad_token))
tokenizer.add_special_tokens(dict(additional_special_tokens=template.stop_words))