423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 03:40:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Release v0.1.6

Browse Source
This commit is contained in:
hiyouga
2023-08-11 23:25:57 +08:00
parent 156710a995
commit a48cb0d474
18 changed files with 127 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/llmtuner/dsets/utils.py
View File

@@ -19,7 +19,8 @@ def split_dataset(
dataset = dataset.shuffle(buffer_size=data_args.buffer_size, seed=training_args.seed)
return {"train_dataset": train_set, "eval_dataset": val_set}
else:
dataset = dataset.train_test_split(test_size=data_args.val_size, seed=training_args.seed)
val_size = int(data_args.val_size) if data_args.val_size > 1 else data_args.val_size
dataset = dataset.train_test_split(test_size=val_size, seed=training_args.seed)
return {"train_dataset": dataset["train"], "eval_dataset": dataset["test"]}
else:
if data_args.streaming: