add use_streaming

Former-commit-id: adc98c86dad64f1a793017fa628b5cf19abbdd01
This commit is contained in:
xingjun.wang 2023-12-12 14:23:05 +08:00
parent 1bd75afae8
commit 6cb2c99e7d

View File

@ -65,7 +65,7 @@ def get_dataset(
data_files=data_files,
cache_dir=cache_dir,
token=model_args.ms_hub_token,
streaming=(data_args.streaming and (dataset_attr.load_from != "file")),
use_streaming=(data_args.streaming and (dataset_attr.load_from != "file")),
).to_hf_dataset()
else:
dataset = load_dataset(