423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-04 12:42:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

update

Browse Source 
Former-commit-id: cfba1009d0fc31b5933b558b249d89248f723d6b
This commit is contained in:
xingjun.wang 2023-12-12 12:03:23 +08:00
parent 1d65d24071
commit 6520aecef1
1 changed files with 0 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/llmtuner/data/loader.py
View File

@ -59,13 +59,6 @@ def get_dataset(
dataset_name=data_path, dataset_name=data_path,
subset_name=data_name, subset_name=data_name,
).to_hf_dataset() ).to_hf_dataset()
def map_func(example):
# do something to example
example['input'] = example['input'] or ''
return example
dataset = dataset.map(map_func)
else: else:
dataset = load_dataset( dataset = load_dataset(
path=data_path, path=data_path,