423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-03 12:12:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

add use_streaming

Browse Source 
Former-commit-id: adc98c86dad64f1a793017fa628b5cf19abbdd01
This commit is contained in:
xingjun.wang 2023-12-12 14:23:05 +08:00
parent 1bd75afae8
commit 6cb2c99e7d
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/data/loader.py
View File

@ -65,7 +65,7 @@ def get_dataset(
data_files=data_files, data_files=data_files,
cache_dir=cache_dir, cache_dir=cache_dir,
token=model_args.ms_hub_token, token=model_args.ms_hub_token,
streaming=(data_args.streaming and (dataset_attr.load_from != "file")), use_streaming=(data_args.streaming and (dataset_attr.load_from != "file")),
).to_hf_dataset() ).to_hf_dataset()
else: else:
dataset = load_dataset( dataset = load_dataset(