423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-15 19:30:36 +08:00
Code Issues Packages Projects Releases Wiki Activity

support layerwise galore

Browse Source
This commit is contained in:
hiyouga
2024-03-10 00:24:11 +08:00
parent 18ffce36b5
commit 8664262cde
14 changed files with 109 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llmtuner/data/loader.py
View File

@@ -29,7 +29,7 @@ def load_single_dataset(
dataset_attr: "DatasetAttr",
model_args: "ModelArguments",
data_args: "DataArguments",
):
) -> Union["Dataset", "IterableDataset"]:
logger.info("Loading dataset {}...".format(dataset_attr))
data_path, data_name, data_dir, data_files = None, None, None, None
if dataset_attr.load_from in ["hf_hub", "ms_hub"]: