From 181c776b580f705fb82482ad2dad81fd34edcee7 Mon Sep 17 00:00:00 2001 From: hiyouga Date: Tue, 30 May 2023 16:28:00 +0800 Subject: [PATCH] remove dummy code Former-commit-id: e6bc89d280945bbf48281107145c40a41d7cbd56 --- src/utils/common.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/utils/common.py b/src/utils/common.py index 30f226f8..4ba573e4 100644 --- a/src/utils/common.py +++ b/src/utils/common.py @@ -316,9 +316,6 @@ def prepare_data( max_samples_temp = min(len(dataset), max_samples) dataset = dataset.select(range(max_samples_temp)) - if dataset.column_names[0] == "text": # for plaintext (in pre-training) - dataset = dataset.rename_column("text", getattr(dataset_attr, "prompt_column")) - dummy_data = [None] * len(dataset) for column_name, target_name in [ ("prompt_column", "prompt"),