423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-14 19:06:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

better data streaming

Browse Source
This commit is contained in:
hiyouga
2023-11-19 23:32:47 +08:00
parent 211b2db5a8
commit 00baaa990e
2 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
assets/wechat.jpg
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 141 KiB

After

Width:  |  Height:  |  Size: 140 KiB

5
src/llmtuner/data/loader.py
View File

@@ -60,9 +60,12 @@ def get_dataset(
split=data_args.split, split=data_args.split,
cache_dir=model_args.cache_dir, cache_dir=model_args.cache_dir,
token=model_args.hf_hub_token, token=model_args.hf_hub_token,
streaming=data_args.streaming streaming=(data_args.streaming and (dataset_attr.load_from != "file"))
) )
if data_args.streaming and (dataset_attr.load_from == "file"):
dataset = dataset.to_iterable_dataset() # TODO: add num shards parameter
if max_samples is not None: # truncate dataset if max_samples is not None: # truncate dataset
dataset = dataset.select(range(min(len(dataset), max_samples))) dataset = dataset.select(range(min(len(dataset), max_samples)))