From 0fc8612b97c64ae40be2405e4021597b7800a3c9 Mon Sep 17 00:00:00 2001 From: hiyouga Date: Sat, 3 Feb 2024 23:38:56 +0800 Subject: [PATCH] add hint for freeze #2412 Former-commit-id: 6545c02790e39395a87d664682ab73e0e3191099 --- src/llmtuner/hparams/finetuning_args.py | 1 + src/llmtuner/model/adapter.py | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/src/llmtuner/hparams/finetuning_args.py b/src/llmtuner/hparams/finetuning_args.py index 43e5e9e0..69403cc2 100644 --- a/src/llmtuner/hparams/finetuning_args.py +++ b/src/llmtuner/hparams/finetuning_args.py @@ -17,6 +17,7 @@ class FreezeArguments: BLOOM & Falcon & ChatGLM choices: ["mlp", "self_attention"], \ Qwen choices: ["mlp", "attn"], \ Phi choices: ["mlp", "mixer"], \ + InternLM2 choices: ["feed_forward", "attention"], \ Others choices: the same as LLaMA.' }, ) diff --git a/src/llmtuner/model/adapter.py b/src/llmtuner/model/adapter.py index c4a1a3b9..3d3f95c0 100644 --- a/src/llmtuner/model/adapter.py +++ b/src/llmtuner/model/adapter.py @@ -52,8 +52,18 @@ def init_adapter( else: # fine-tuning the first n layers if num_layer_trainable < 0 trainable_layer_ids = [k for k in range(-finetuning_args.num_layer_trainable)] # noqa: C416 + freeze_modules = set() + for name, _ in model.named_modules(): + if "0." in name: + freeze_modules.add(name.split("0.")[-1].split(".")[0]) + trainable_layers = [] for module_name in finetuning_args.name_module_trainable: + if module_name not in freeze_modules: + raise ValueError( + "Module {} is not found, please choose from {}".format(module_name, ", ".join(freeze_modules)) + ) + for idx in trainable_layer_ids: trainable_layers.append("{:d}.{}".format(idx, module_name))