From 0b08d5882ab8d47095b7e9327687eddd2daacd5f Mon Sep 17 00:00:00 2001 From: hiyouga Date: Tue, 26 Nov 2024 16:03:02 +0000 Subject: [PATCH] fix #6149 Former-commit-id: b581b272793314a9602f4dc2fb646a988a6249df --- src/llamafactory/data/loader.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/llamafactory/data/loader.py b/src/llamafactory/data/loader.py index 540dff1c..1c3a7747 100644 --- a/src/llamafactory/data/loader.py +++ b/src/llamafactory/data/loader.py @@ -128,6 +128,7 @@ def _load_single_dataset( cache_dir=model_args.cache_dir, token=model_args.hf_hub_token, streaming=data_args.streaming, + num_proc=data_args.preprocessing_num_workers, trust_remote_code=True, )