Former-commit-id: e15fc417d897c3063a25d6eb7eb89d1916db3cc5
This commit is contained in:
xingjun.wang 2023-12-12 12:03:23 +08:00
parent 79a376d1db
commit 0baf32e219

View File

@ -59,13 +59,6 @@ def get_dataset(
dataset_name=data_path,
subset_name=data_name,
).to_hf_dataset()
def map_func(example):
# do something to example
example['input'] = example['input'] or ''
return example
dataset = dataset.map(map_func)
else:
dataset = load_dataset(
path=data_path,