423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-15 11:20:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

add llava and instructblip

Browse Source
This commit is contained in:
BUAADreamer
2024-04-25 00:22:43 +08:00
parent e1afbea68f
commit cfb485eddf
22 changed files with 343 additions and 4484 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/llmtuner/data/loader.py
View File

@@ -199,8 +199,7 @@ def get_mm_dataset(
with training_args.main_process_first(desc="load dataset"):
all_datasets = []
for dataset_attr in get_dataset_list(data_args):
local_path = os.path.join(data_args.dataset_dir, dataset_attr.dataset_name)
all_datasets.append(load_dataset("json", data_files=local_path)['train'])
all_datasets.append(load_dataset(dataset_attr.dataset_name)['train'])
dataset = merge_dataset(all_datasets, data_args, training_args)
return dataset