423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-15 08:08:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

tiny fix

Browse Source 
Former-commit-id: 987df4c62f34026adfe2089910f4ff9ac6ebd9a6
This commit is contained in:
hoshi-hiyouga 2023-12-13 17:32:36 +08:00
parent 3ae479faae
commit 45abe361ba
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/llmtuner/model/utils.py
View File

@ -177,12 +177,12 @@ def resize_embedding_layer(model: "PreTrainedModel", tokenizer: "PreTrainedToken
r""" r"""
Resize token embeddings. Resize token embeddings.
""" """
if not isinstance(model.get_output_embeddings(), torch.nn.Linear):
logger.warning("Current model does not support resizing token embeddings.")
return
current_embedding_size = model.get_input_embeddings().weight.size(0) current_embedding_size = model.get_input_embeddings().weight.size(0)
if len(tokenizer) > current_embedding_size: if len(tokenizer) > current_embedding_size:
if not isinstance(model.get_output_embeddings(), torch.nn.Linear):
logger.warning("Current model does not support resizing token embeddings.")
return
model.resize_token_embeddings(len(tokenizer), pad_to_multiple_of=64) model.resize_token_embeddings(len(tokenizer), pad_to_multiple_of=64)
new_embedding_size = model.get_input_embeddings().weight.size(0) new_embedding_size = model.get_input_embeddings().weight.size(0)
logger.info("Resized token embeddings from {} to {}.".format(current_embedding_size, new_embedding_size)) logger.info("Resized token embeddings from {} to {}.".format(current_embedding_size, new_embedding_size))