423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-24 06:42:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

1, log exceptions in details; 2, check processor is None before calling it.

Browse Source 
Former-commit-id: 573e3183e644e8da61a409d96b9adcfacbfc3a7a
This commit is contained in:
Chengcheng Pei 2024-09-25 12:59:48 -07:00
parent fb75821793
commit e80c98367e
2 changed files with 8 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/llamafactory/model/loader.py
View File

@ -82,6 +82,9 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule":
padding_side="right", padding_side="right",
**init_kwargs, **init_kwargs,
) )
except Exception as e:
logger.error("Failed to load tokenizer. Error: {}".format(e))
raise e
if model_args.new_special_tokens is not None: if model_args.new_special_tokens is not None:
num_added_tokens = tokenizer.add_special_tokens( num_added_tokens = tokenizer.add_special_tokens(
@ -103,12 +106,13 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule":
setattr(processor, "video_resolution", model_args.video_resolution) setattr(processor, "video_resolution", model_args.video_resolution)
setattr(processor, "video_fps", model_args.video_fps) setattr(processor, "video_fps", model_args.video_fps)
setattr(processor, "video_maxlen", model_args.video_maxlen) setattr(processor, "video_maxlen", model_args.video_maxlen)
except Exception: except Exception as e:
logger.warning("Failed to load processor. Error: {}".format(e))
processor = None processor = None
# Avoid load tokenizer, see: # Avoid load tokenizer, see:
# https://github.com/huggingface/transformers/blob/v4.40.0/src/transformers/models/auto/processing_auto.py#L324 # https://github.com/huggingface/transformers/blob/v4.40.0/src/transformers/models/auto/processing_auto.py#L324
if "Processor" not in processor.__class__.__name__: if processor and "Processor" not in processor.__class__.__name__:
processor = None processor = None
return {"tokenizer": tokenizer, "processor": processor} return {"tokenizer": tokenizer, "processor": processor}

4
src/llamafactory/train/tuner.py
View File

@ -139,5 +139,5 @@ def export_model(args: Optional[Dict[str, Any]] = None) -> None:
model_args.export_hub_model_id, token=model_args.hf_hub_token model_args.export_hub_model_id, token=model_args.hf_hub_token
) )
except Exception: except Exception as e:
logger.warning("Cannot save tokenizer, please copy the files manually.") logger.warning("Cannot save tokenizer, please copy the files manually. Error: {}".format(e))