423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-04 12:42:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

add hint for freeze #2412

Browse Source 
Former-commit-id: 6545c02790e39395a87d664682ab73e0e3191099
This commit is contained in:
hiyouga 2024-02-03 23:38:56 +08:00
parent a9e58740f5
commit 0fc8612b97
2 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/llmtuner/hparams/finetuning_args.py
View File

@ -17,6 +17,7 @@ class FreezeArguments:
BLOOM & Falcon & ChatGLM choices: ["mlp", "self_attention"], \ BLOOM & Falcon & ChatGLM choices: ["mlp", "self_attention"], \
Qwen choices: ["mlp", "attn"], \ Qwen choices: ["mlp", "attn"], \
Phi choices: ["mlp", "mixer"], \ Phi choices: ["mlp", "mixer"], \
InternLM2 choices: ["feed_forward", "attention"], \
Others choices: the same as LLaMA.' Others choices: the same as LLaMA.'
}, },
) )

10
src/llmtuner/model/adapter.py
View File

@ -52,8 +52,18 @@ def init_adapter(
else: # fine-tuning the first n layers if num_layer_trainable < 0 else: # fine-tuning the first n layers if num_layer_trainable < 0
trainable_layer_ids = [k for k in range(-finetuning_args.num_layer_trainable)] # noqa: C416 trainable_layer_ids = [k for k in range(-finetuning_args.num_layer_trainable)] # noqa: C416
freeze_modules = set()
for name, _ in model.named_modules():
if "0." in name:
freeze_modules.add(name.split("0.")[-1].split(".")[0])
trainable_layers = [] trainable_layers = []
for module_name in finetuning_args.name_module_trainable: for module_name in finetuning_args.name_module_trainable:
if module_name not in freeze_modules:
raise ValueError(
"Module {} is not found, please choose from {}".format(module_name, ", ".join(freeze_modules))
)
for idx in trainable_layer_ids: for idx in trainable_layer_ids:
trainable_layers.append("{:d}.{}".format(idx, module_name)) trainable_layers.append("{:d}.{}".format(idx, module_name))