From e6f4eab4ab61499c0404d86d4b1e5316308fffd5 Mon Sep 17 00:00:00 2001 From: hiyouga Date: Mon, 21 Aug 2023 17:49:36 +0800 Subject: [PATCH] fix #608 Former-commit-id: 02d69b6fdefa6b303b84fb8195a159006fe3f50a --- src/llmtuner/extras/template.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/llmtuner/extras/template.py b/src/llmtuner/extras/template.py index c4cdb70a..b53c5893 100644 --- a/src/llmtuner/extras/template.py +++ b/src/llmtuner/extras/template.py @@ -224,7 +224,10 @@ def get_template_and_fix_tokenizer( tokenizer.pad_token = tokenizer.eos_token logger.info("Add pad token: {}".format(tokenizer.pad_token)) - tokenizer.add_special_tokens(dict(additional_special_tokens=additional_special_tokens)) + tokenizer.add_special_tokens( + dict(additional_special_tokens=additional_special_tokens), + replace_additional_special_tokens=False + ) return template @@ -472,7 +475,7 @@ register_template( r""" Supports: https://huggingface.co/baichuan-inc/Baichuan-13B-Chat -Used for training. +Used for training and inference of the fine-tuned models. """ register_template( name="baichuan",