From e80c98367e0ea150f5f87ec779b618ed3c079415 Mon Sep 17 00:00:00 2001 From: Chengcheng Pei Date: Wed, 25 Sep 2024 12:59:48 -0700 Subject: [PATCH 1/2] 1, log exceptions in details; 2, check processor is None before calling it. Former-commit-id: 573e3183e644e8da61a409d96b9adcfacbfc3a7a --- src/llamafactory/model/loader.py | 8 ++++++-- src/llamafactory/train/tuner.py | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/llamafactory/model/loader.py b/src/llamafactory/model/loader.py index 030ce90f..e86e7955 100644 --- a/src/llamafactory/model/loader.py +++ b/src/llamafactory/model/loader.py @@ -82,6 +82,9 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule": padding_side="right", **init_kwargs, ) + except Exception as e: + logger.error("Failed to load tokenizer. Error: {}".format(e)) + raise e if model_args.new_special_tokens is not None: num_added_tokens = tokenizer.add_special_tokens( @@ -103,12 +106,13 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule": setattr(processor, "video_resolution", model_args.video_resolution) setattr(processor, "video_fps", model_args.video_fps) setattr(processor, "video_maxlen", model_args.video_maxlen) - except Exception: + except Exception as e: + logger.warning("Failed to load processor. Error: {}".format(e)) processor = None # Avoid load tokenizer, see: # https://github.com/huggingface/transformers/blob/v4.40.0/src/transformers/models/auto/processing_auto.py#L324 - if "Processor" not in processor.__class__.__name__: + if processor and "Processor" not in processor.__class__.__name__: processor = None return {"tokenizer": tokenizer, "processor": processor} diff --git a/src/llamafactory/train/tuner.py b/src/llamafactory/train/tuner.py index 9928e7b9..3c8a3b13 100644 --- a/src/llamafactory/train/tuner.py +++ b/src/llamafactory/train/tuner.py @@ -139,5 +139,5 @@ def export_model(args: Optional[Dict[str, Any]] = None) -> None: model_args.export_hub_model_id, token=model_args.hf_hub_token ) - except Exception: - logger.warning("Cannot save tokenizer, please copy the files manually.") + except Exception as e: + logger.warning("Cannot save tokenizer, please copy the files manually. Error: {}".format(e)) From bc7197dcfc45fbd97b61788c69ca9650320a2b37 Mon Sep 17 00:00:00 2001 From: Chengcheng Pei Date: Wed, 25 Sep 2024 21:07:51 -0700 Subject: [PATCH 2/2] address comments Former-commit-id: 6bfd8442671eec4c0fb63fd34667a44760fd84e2 --- src/llamafactory/model/loader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/llamafactory/model/loader.py b/src/llamafactory/model/loader.py index e86e7955..1fa04e0f 100644 --- a/src/llamafactory/model/loader.py +++ b/src/llamafactory/model/loader.py @@ -83,8 +83,7 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule": **init_kwargs, ) except Exception as e: - logger.error("Failed to load tokenizer. Error: {}".format(e)) - raise e + raise OSError("Failed to load tokenizer") from e if model_args.new_special_tokens is not None: num_added_tokens = tokenizer.add_special_tokens(