423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 03:40:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

improve aligner

Browse Source
This commit is contained in:
hiyouga
2024-02-10 16:39:19 +08:00
parent 388b705a8d
commit 7d2dc83c5e
11 changed files with 127 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/data/template.py
View File

@@ -247,7 +247,7 @@ def _add_or_replace_eos_token(tokenizer: "PreTrainedTokenizer", eos_token: str)
logger.info("Replace eos token: {}".format(tokenizer.eos_token))
if is_oov:
logger.warning("New token is added, you must enable `resize_vocab` to activate it.")
logger.warning("New tokens have been added, make sure `resize_vocab` is True.")
def get_template_and_fix_tokenizer(