423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 03:40:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

add use_streaming

Browse Source 
Former-commit-id: 80388abdb7ee88eb4afad92d8c706370c0574039
This commit is contained in:
xingjun.wang
2023-12-12 14:23:05 +08:00
parent bfce8c9b35
commit cc7de28bbb
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/data/loader.py
View File

@@ -65,7 +65,7 @@ def get_dataset(
data_files=data_files,
cache_dir=cache_dir,
token=model_args.ms_hub_token,
streaming=(data_args.streaming and (dataset_attr.load_from != "file")),
use_streaming=(data_args.streaming and (dataset_attr.load_from != "file")),
).to_hf_dataset()
else:
dataset = load_dataset(