423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-03 04:02:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update loader.py

Browse Source 
Former-commit-id: ca5dd7c6c115a359e4b50e93f4ffcc9f2955ec2f
This commit is contained in:
hoshi-hiyouga 2024-05-30 00:20:20 +08:00 committed by GitHub
parent 9fc713da89
commit 7b83c550ab
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/llamafactory/data/loader.py
View File

@ -115,6 +115,7 @@ def load_single_dataset(
expand_indexes = np.random.choice(len(dataset), target_num) expand_indexes = np.random.choice(len(dataset), target_num)
indexes = np.concatenate((indexes, expand_indexes), axis=0) indexes = np.concatenate((indexes, expand_indexes), axis=0)
assert len(indexes) == dataset_attr.num_samples, "Sample num mismatched."
dataset = dataset.select(indexes) dataset = dataset.select(indexes)
logger.info("Sampled {} examples from dataset {}.".format(dataset_attr.num_samples, dataset_attr)) logger.info("Sampled {} examples from dataset {}.".format(dataset_attr.num_samples, dataset_attr))