lazy image load

Former-commit-id: 47ea97fb1b
2025-12-17 20:30:36 +08:00 · 2024-09-04 02:27:08 +08:00
parent 5ef58eb655
commit 22deca0e9e
19 changed files with 353 additions and 366 deletions
--- a/src/llamafactory/hparams/data_args.py
+++ b/src/llamafactory/hparams/data_args.py
@@ -73,6 +73,10 @@ class DataArguments:
        default=False,
        metadata={"help": "Overwrite the cached training and evaluation sets."},
    )
+    preprocessing_batch_size: int = field(
+        default=1000,
+        metadata={"help": "The number of examples in one group in pre-processing."},
+    )
    preprocessing_num_workers: Optional[int] = field(
        default=None,
        metadata={"help": "The number of processes to use for the pre-processing."},