support val set in streaming mode

2025-12-17 04:10:36 +08:00 · 2023-08-09 23:00:26 +08:00
parent 572ea3bafb
commit d86ea314a1
10 changed files with 58 additions and 50 deletions
--- a/src/llmtuner/tuner/pt/workflow.py
+++ b/src/llmtuner/tuner/pt/workflow.py
@@ -38,7 +38,7 @@ def run_pt(
        tokenizer=tokenizer,
        data_collator=data_collator,
        callbacks=callbacks,
-        **split_dataset(dataset, data_args.dev_ratio, training_args.do_train)
+        **split_dataset(dataset, data_args, training_args)
    )

    # Training