423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 11:50:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Added Mixture of Depths

Browse Source
This commit is contained in:
Marco
2024-04-18 20:31:24 +02:00
parent 2aaaede247
commit 620add7b9f
10 changed files with 103 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/model/adapter.py
View File

@@ -69,6 +69,8 @@ def init_adapter(
for name, _ in model.named_modules():
if ".0." in name:
freeze_modules.add(name.split(".0.")[-1].split(".")[0])
elif ".1." in name: # here since MoD starts from layer 1
freeze_modules.add(name.split(".1.")[-1].split(".")[0])
trainable_layers = []
for module_name in finetuning_args.name_module_trainable: