Former-commit-id: 8a908a8c644f4a961001cdd8388a3a7fea992c55
This commit is contained in:
xingjun.wang 2023-12-12 11:47:59 +08:00
parent 9c30cdb53d
commit 2918743520

View File

@ -59,6 +59,13 @@ def get_dataset(
dataset_name=data_path, dataset_name=data_path,
subset_name=data_name, subset_name=data_name,
).to_hf_dataset() ).to_hf_dataset()
def map_func(example):
# do something to example
example['input'] = example['input'] or ''
return example
dataset = dataset.ds_instance.map(map_func)
else: else:
dataset = load_dataset( dataset = load_dataset(
path=data_path, path=data_path,