mirror of
https://github.com/hiyouga/LLaMA-Factory.git
synced 2025-08-02 03:32:50 +08:00
38 lines
1.3 KiB
Python
38 lines
1.3 KiB
Python
import json
|
|
from typing import Any, Dict, Generator, List, Tuple
|
|
|
|
import datasets
|
|
|
|
|
|
_DESCRIPTION = "An example of dataset."
|
|
_CITATION = ""
|
|
_HOMEPAGE = ""
|
|
_LICENSE = ""
|
|
_URL = "examples.json"
|
|
|
|
|
|
class ExampleDataset(datasets.GeneratorBasedBuilder):
|
|
VERSION = datasets.Version("0.0.0")
|
|
|
|
def _info(self) -> datasets.DatasetInfo:
|
|
features = datasets.Features(
|
|
{
|
|
"instruction": datasets.Value("string"),
|
|
"input": datasets.Value("string"),
|
|
"output": datasets.Value("string"),
|
|
"history": datasets.Sequence(datasets.Sequence(datasets.Value("string"))),
|
|
}
|
|
)
|
|
return datasets.DatasetInfo(
|
|
description=_DESCRIPTION, features=features, homepage=_HOMEPAGE, license=_LICENSE, citation=_CITATION
|
|
)
|
|
|
|
def _split_generators(self, dl_manager: datasets.DownloadManager) -> List[datasets.SplitGenerator]:
|
|
file_path = dl_manager.download(_URL)
|
|
return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": file_path})]
|
|
|
|
def _generate_examples(self, filepath: str) -> Generator[Tuple[int, Dict[str, Any]], None, None]:
|
|
example_dataset = json.load(open(filepath, "r", encoding="utf-8"))
|
|
for key, example in enumerate(example_dataset):
|
|
yield key, example
|