From fa99ead86b8b4eacc2293c5ba4ffa83913bf8d3c Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Wed, 13 Dec 2023 17:32:36 +0800 Subject: [PATCH] tiny fix Former-commit-id: 81167cd19dac4926da9bd259f4e3cb064c22825c --- src/llmtuner/model/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/llmtuner/model/utils.py b/src/llmtuner/model/utils.py index 0eabae25..1fe51155 100644 --- a/src/llmtuner/model/utils.py +++ b/src/llmtuner/model/utils.py @@ -177,12 +177,12 @@ def resize_embedding_layer(model: "PreTrainedModel", tokenizer: "PreTrainedToken r""" Resize token embeddings. """ - if not isinstance(model.get_output_embeddings(), torch.nn.Linear): - logger.warning("Current model does not support resizing token embeddings.") - return - current_embedding_size = model.get_input_embeddings().weight.size(0) if len(tokenizer) > current_embedding_size: + if not isinstance(model.get_output_embeddings(), torch.nn.Linear): + logger.warning("Current model does not support resizing token embeddings.") + return + model.resize_token_embeddings(len(tokenizer), pad_to_multiple_of=64) new_embedding_size = model.get_input_embeddings().weight.size(0) logger.info("Resized token embeddings from {} to {}.".format(current_embedding_size, new_embedding_size))