diff --git a/src/llamafactory/model/loader.py b/src/llamafactory/model/loader.py index b4ef16e8..ee09db43 100644 --- a/src/llamafactory/model/loader.py +++ b/src/llamafactory/model/loader.py @@ -82,6 +82,8 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule": padding_side="right", **init_kwargs, ) + except Exception as e: + raise OSError("Failed to load tokenizer") from e if model_args.new_special_tokens is not None: num_added_tokens = tokenizer.add_special_tokens( @@ -97,12 +99,13 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule": try: processor = AutoProcessor.from_pretrained(model_args.model_name_or_path, **init_kwargs) patch_processor(processor, config, tokenizer, model_args) - except Exception: + except Exception as e: + logger.warning("Failed to load processor. Error: {}".format(e)) processor = None # Avoid load tokenizer, see: # https://github.com/huggingface/transformers/blob/v4.40.0/src/transformers/models/auto/processing_auto.py#L324 - if "Processor" not in processor.__class__.__name__: + if processor and "Processor" not in processor.__class__.__name__: processor = None return {"tokenizer": tokenizer, "processor": processor} diff --git a/src/llamafactory/train/tuner.py b/src/llamafactory/train/tuner.py index 9928e7b9..3c8a3b13 100644 --- a/src/llamafactory/train/tuner.py +++ b/src/llamafactory/train/tuner.py @@ -139,5 +139,5 @@ def export_model(args: Optional[Dict[str, Any]] = None) -> None: model_args.export_hub_model_id, token=model_args.hf_hub_token ) - except Exception: - logger.warning("Cannot save tokenizer, please copy the files manually.") + except Exception as e: + logger.warning("Cannot save tokenizer, please copy the files manually. Error: {}".format(e))