fix #6149

Former-commit-id: b581b272793314a9602f4dc2fb646a988a6249df
2025-11-05 02:12:14 +08:00 · 2024-11-26 16:03:02 +00:00 · 2024-11-26 16:03:02 +00:00 · 0b08d5882a
commit 0b08d5882a
parent 5a52e41399
1 changed files with 1 additions and 0 deletions
--- a/src/llamafactory/data/loader.py
+++ b/src/llamafactory/data/loader.py
@ -128,6 +128,7 @@ def _load_single_dataset(
            cache_dir=model_args.cache_dir,
            token=model_args.hf_hub_token,
            streaming=data_args.streaming,
+            num_proc=data_args.preprocessing_num_workers,
            trust_remote_code=True,
        )