423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-16 00:28:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

add use_streaming

Browse Source 
Former-commit-id: 80388abdb7ee88eb4afad92d8c706370c0574039
This commit is contained in:
xingjun.wang 2023-12-12 14:23:05 +08:00
parent e54dad75da
commit 3ee3fe0bbb
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/data/loader.py
View File

@ -65,7 +65,7 @@ def get_dataset(
data_files=data_files,
cache_dir=cache_dir,
token=model_args.ms_hub_token,
streaming=(data_args.streaming and (dataset_attr.load_from != "file")),
use_streaming=(data_args.streaming and (dataset_attr.load_from != "file")),
).to_hf_dataset()
else:
dataset = load_dataset(