423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 11:50:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

[FEATURE]: ADD LORA+ ALGORITHM

Browse Source
This commit is contained in:
齐保元
2024-03-13 19:43:27 +08:00
parent dfd451b722
commit a0965cd62c
4 changed files with 130 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/train/sft/workflow.py
View File

@@ -12,7 +12,7 @@ from ...model import load_model, load_tokenizer
from ...train.sft.metric import ComputeMetrics
from ...train.sft.trainer import CustomSeq2SeqTrainer
from ...train.utils import create_modelcard_and_push
from ..utils import create_custom_optimzer
from ..utils import create_custom_optimzer, create_lora_plus_optimizer
if TYPE_CHECKING:
@@ -51,6 +51,8 @@ def run_sft(
# Initialize our Trainer
optimizer = create_custom_optimzer(model, dataset, training_args, finetuning_args)
if finetuning_args.lora_lr_ratio:
optimizer = create_lora_plus_optimizer(model, training_args, finetuning_args)
trainer = CustomSeq2SeqTrainer(
model=model,
args=training_args,