423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-23 14:22:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

a monkey patch for lora_target

Browse Source 
Former-commit-id: 262252d67bbe4ebcbb315b5d7a34f9a091f8af0c
This commit is contained in:
hiyouga 2023-07-18 00:31:40 +08:00
parent 091805d38e
commit 4e1997a343
2 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/llmtuner/extras/constants.py
View File

@ -29,3 +29,12 @@ SUPPORTED_MODELS = {
"InternLM-7B-Base": "internlm/internlm-7b", "InternLM-7B-Base": "internlm/internlm-7b",
"InternLM-7B-Chat": "internlm/internlm-chat-7b" "InternLM-7B-Chat": "internlm/internlm-chat-7b"
} }
DEFAULT_MODULE = { # will be deprecated
"LLaMA": "q_proj,v_proj",
"BLOOM": "query_key_value",
"BLOOMZ": "query_key_value",
"Falcon": "query_key_value",
"Baichuan": "W_pack",
"InternLM": "q_proj,v_proj"
}

2
src/llmtuner/webui/runner.py
View File

@ -6,6 +6,7 @@ import transformers
from typing import Optional, Tuple from typing import Optional, Tuple
from llmtuner.extras.callbacks import LogCallback from llmtuner.extras.callbacks import LogCallback
from llmtuner.extras.constants import DEFAULT_MODULE # will be deprecated
from llmtuner.extras.logging import LoggerHandler from llmtuner.extras.logging import LoggerHandler
from llmtuner.extras.misc import torch_gc from llmtuner.extras.misc import torch_gc
from llmtuner.tuner import get_train_args, run_sft from llmtuner.tuner import get_train_args, run_sft
@ -79,6 +80,7 @@ class Runner:
model_name_or_path=model_name_or_path, model_name_or_path=model_name_or_path,
do_train=True, do_train=True,
finetuning_type=finetuning_type, finetuning_type=finetuning_type,
lora_target=DEFAULT_MODULE.get(model_name.split("-")[0], None) or "q_proj,v_proj",
prompt_template=template, prompt_template=template,
dataset=",".join(dataset), dataset=",".join(dataset),
dataset_dir=dataset_dir, dataset_dir=dataset_dir,