423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-14 19:06:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

implement efficient packing without cross-contamination attention

Browse Source 
Former-commit-id: b2c367bc61
This commit is contained in:
ancv
2024-06-12 11:56:01 +07:00
parent 1a261add61
commit 045eb155a2
9 changed files with 287 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llamafactory/train/kto/workflow.py
View File

@@ -24,7 +24,7 @@ def run_kto(
):
tokenizer_module = load_tokenizer(model_args)
tokenizer = tokenizer_module["tokenizer"]
dataset = get_dataset(model_args, data_args, training_args, stage="kto", **tokenizer_module)
dataset = get_dataset(model_args, data_args, training_args, finetuning_args, stage="kto", **tokenizer_module)
model = load_model(tokenizer, model_args, finetuning_args, training_args.do_train)
data_collator = KTODataCollatorWithPadding(

5
src/llamafactory/train/sft/workflow.py
View File

@@ -12,7 +12,7 @@ from ...model import load_model, load_tokenizer
from ..trainer_utils import create_modelcard_and_push
from .metric import ComputeMetrics
from .trainer import CustomSeq2SeqTrainer
from ...model.model_utils.packing import configure_packing
if TYPE_CHECKING:
from transformers import Seq2SeqTrainingArguments, TrainerCallback
@@ -33,6 +33,9 @@ def run_sft(
dataset = get_dataset(model_args, data_args, training_args, stage="sft", **tokenizer_module)
model = load_model(tokenizer, model_args, finetuning_args, training_args.do_train)
if data_args.efficient_packing:
configure_packing(model.config)
if training_args.predict_with_generate:
tokenizer.padding_side = "left" # use left-padding in generation