423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 11:50:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

change to right-padding, update reward score #803

Browse Source
This commit is contained in:
hiyouga
2023-09-08 20:04:31 +08:00
parent 8aaaa132d4
commit 8ea32e4046
15 changed files with 97 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/llmtuner/extras/template.py
View File

@@ -214,10 +214,7 @@ def get_template_and_fix_tokenizer(
logger.info("Add eos token: {}".format(tokenizer.eos_token))
if tokenizer.pad_token_id is None:
if tokenizer.unk_token_id is not None:
tokenizer.pad_token = tokenizer.unk_token
else:
tokenizer.pad_token = tokenizer.eos_token
tokenizer.pad_token = tokenizer.eos_token
logger.info("Add pad token: {}".format(tokenizer.pad_token))
tokenizer.add_special_tokens(