Former-commit-id: 362d579ce83e63007e6f89f264d06d2698671cc6
This commit is contained in:
hiyouga 2024-11-26 16:03:02 +00:00
parent e0325b1123
commit 358708ee97

View File

@ -128,6 +128,7 @@ def _load_single_dataset(
cache_dir=model_args.cache_dir,
token=model_args.hf_hub_token,
streaming=data_args.streaming,
num_proc=data_args.preprocessing_num_workers,
trust_remote_code=True,
)