423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-11-05 10:22:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix possibly OOM error

Browse Source 
Former-commit-id: 0d590dffb41b0e832d9f87d20a23bcd0acd983aa
This commit is contained in:
hiyouga 2023-06-01 23:54:44 +08:00
parent 4edd01fc08
commit 7ef5821cba
1 changed files with 9 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
src/utils/common.py
View File

@ -477,13 +477,13 @@ def preprocess_data(
desc="Running tokenizer on dataset"
)
if stage == "pt":
print_unsupervised_dataset_example(dataset[0])
elif stage == "sft":
print_supervised_dataset_example(dataset[0])
elif stage == "rm":
print_pairwise_dataset_example(dataset[0])
elif stage == "ppo":
print_unsupervised_dataset_example(dataset[0])
if stage == "pt":
print_unsupervised_dataset_example(dataset[0])
elif stage == "sft":
print_supervised_dataset_example(dataset[0])
elif stage == "rm":
print_pairwise_dataset_example(dataset[0])
elif stage == "ppo":
print_unsupervised_dataset_example(dataset[0])
return dataset
return dataset