423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-29 02:00:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

support lora target auto find

Browse Source 
Former-commit-id: bce9984733d88bf013847eed523d1c75fdf0995e
This commit is contained in:
hiyouga
2023-09-09 15:38:37 +08:00
parent 50e93392dd
commit 7143c551ab
11 changed files with 117 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/llmtuner/tuner/core/adapter.py
View File

@@ -12,6 +12,7 @@ from peft.utils import CONFIG_NAME, WEIGHTS_NAME
from llmtuner.extras.logging import get_logger
from llmtuner.extras.save_and_load import load_trainable_params
from llmtuner.tuner.core.utils import find_all_linear_modules
if TYPE_CHECKING:
from transformers.modeling_utils import PreTrainedModel
@@ -81,13 +82,18 @@ def init_adapter(
model = PeftModel.from_pretrained(model, latest_checkpoint, is_trainable=is_trainable)
if is_trainable and latest_checkpoint is None: # create new lora weights while training
if len(finetuning_args.lora_target) == 1 and finetuning_args.lora_target == "all":
target_modules = find_all_linear_modules(model, model_args.quantization_bit)
else:
target_modules = finetuning_args.lora_target
lora_config = LoraConfig(
task_type=TaskType.CAUSAL_LM,
inference_mode=False,
r=finetuning_args.lora_rank,
lora_alpha=finetuning_args.lora_alpha,
lora_dropout=finetuning_args.lora_dropout,
target_modules=finetuning_args.lora_target
target_modules=target_modules
)
model = get_peft_model(model, lora_config)