From 7622300c4b13a97c0600358b9e5e8048879c7d20 Mon Sep 17 00:00:00 2001 From: hoshi-hiyouga Date: Wed, 15 May 2024 14:13:01 +0800 Subject: [PATCH] Update workflow.py Former-commit-id: c309605ff565dc34d043314269fce5881212c27c --- src/llmtuner/train/sft/workflow.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/llmtuner/train/sft/workflow.py b/src/llmtuner/train/sft/workflow.py index 3b7b909a..d9d7c8e9 100644 --- a/src/llmtuner/train/sft/workflow.py +++ b/src/llmtuner/train/sft/workflow.py @@ -30,7 +30,6 @@ def run_sft( ): tokenizer_module = load_tokenizer(model_args) tokenizer = tokenizer_module["tokenizer"] - processor = tokenizer_module["processor"] dataset = get_dataset(model_args, data_args, training_args, stage="sft", **tokenizer_module) model = load_model(tokenizer, model_args, finetuning_args, training_args.do_train) @@ -56,11 +55,10 @@ def run_sft( model=model, args=training_args, finetuning_args=finetuning_args, - processor=processor, - tokenizer=tokenizer, data_collator=data_collator, callbacks=callbacks, compute_metrics=ComputeMetrics(tokenizer) if training_args.predict_with_generate else None, + **tokenizer_module, **split_dataset(dataset, data_args, training_args), )