423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-04 20:52:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix #608

Browse Source 
Former-commit-id: 02d69b6fdefa6b303b84fb8195a159006fe3f50a
This commit is contained in:
hiyouga 2023-08-21 17:49:36 +08:00
parent d3bef03dc6
commit e6f4eab4ab
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/llmtuner/extras/template.py
View File

@ -224,7 +224,10 @@ def get_template_and_fix_tokenizer(
tokenizer.pad_token = tokenizer.eos_token tokenizer.pad_token = tokenizer.eos_token
logger.info("Add pad token: {}".format(tokenizer.pad_token)) logger.info("Add pad token: {}".format(tokenizer.pad_token))
tokenizer.add_special_tokens(dict(additional_special_tokens=additional_special_tokens)) tokenizer.add_special_tokens(
dict(additional_special_tokens=additional_special_tokens),
replace_additional_special_tokens=False
)
return template return template
@ -472,7 +475,7 @@ register_template(
r""" r"""
Supports: https://huggingface.co/baichuan-inc/Baichuan-13B-Chat Supports: https://huggingface.co/baichuan-inc/Baichuan-13B-Chat
Used for training. Used for training and inference of the fine-tuned models.
""" """
register_template( register_template(
name="baichuan", name="baichuan",