diff --git a/src/llamafactory/data/template.py b/src/llamafactory/data/template.py index bf07ec95..6acdebac 100644 --- a/src/llamafactory/data/template.py +++ b/src/llamafactory/data/template.py @@ -398,11 +398,11 @@ def get_template_and_fix_tokenizer(tokenizer: "PreTrainedTokenizer", data_args: if num_added_tokens > 0: logger.warning("New tokens have been added, make sure `resize_vocab` is True.") - if template.replace_jinja_template: + if tokenizer.chat_template is None or template.replace_jinja_template: try: tokenizer.chat_template = _get_jinja_template(template, tokenizer) - except ValueError: - logger.info("Cannot add this chat template to tokenizer.") + except ValueError as e: + logger.info(f"Cannot add this chat template to tokenizer: {e}.") return template diff --git a/src/llamafactory/model/loader.py b/src/llamafactory/model/loader.py index 5fdfd8c1..e117976f 100644 --- a/src/llamafactory/model/loader.py +++ b/src/llamafactory/model/loader.py @@ -100,7 +100,7 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule": processor = AutoProcessor.from_pretrained(model_args.model_name_or_path, **init_kwargs) patch_processor(processor, config, tokenizer, model_args) except Exception as e: - logger.warning(f"Processor was not found: {e}.") + logger.debug(f"Processor was not found: {e}.") processor = None # Avoid load tokenizer, see: