From 358708ee974170371d17ae9b1bc0e7ebc24a6a95 Mon Sep 17 00:00:00 2001
From: hiyouga <hiyouga@buaa.edu.cn>
Date: Tue, 26 Nov 2024 16:03:02 +0000
Subject: [PATCH] fix #6149

Former-commit-id: 362d579ce83e63007e6f89f264d06d2698671cc6
---
 src/llamafactory/data/loader.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/llamafactory/data/loader.py b/src/llamafactory/data/loader.py
index 540dff1c..1c3a7747 100644
--- a/src/llamafactory/data/loader.py
+++ b/src/llamafactory/data/loader.py
@@ -128,6 +128,7 @@ def _load_single_dataset(
             cache_dir=model_args.cache_dir,
             token=model_args.hf_hub_token,
             streaming=data_args.streaming,
+            num_proc=data_args.preprocessing_num_workers,
             trust_remote_code=True,
         )