diff --git a/data/belle_multiturn/belle_multiturn.py b/data/belle_multiturn/belle_multiturn.py index 86140ed9..6e31f0e6 100644 --- a/data/belle_multiturn/belle_multiturn.py +++ b/data/belle_multiturn/belle_multiturn.py @@ -2,6 +2,7 @@ import os import json import datasets + _HF_ENDPOINT = os.getenv("HF_ENDPOINT", "https://huggingface.co") _DESCRIPTION = "BELLE multiturn chat dataset." @@ -15,9 +16,9 @@ _CITATION = """\ } """ -_HOMEPAGE = f"{_HF_ENDPOINT}/datasets/BelleGroup/multiturn_chat_0.8M" +_HOMEPAGE = "{}/datasets/BelleGroup/multiturn_chat_0.8M".format(_HF_ENDPOINT) _LICENSE = "gpl-3.0" -_URL = f"{_HF_ENDPOINT}/datasets/BelleGroup/multiturn_chat_0.8M/resolve/main/multiturn_chat_0.8M.json" +_URL = "{}/datasets/BelleGroup/multiturn_chat_0.8M/resolve/main/multiturn_chat_0.8M.json".format(_HF_ENDPOINT) class BelleMultiturn(datasets.GeneratorBasedBuilder): diff --git a/data/hh_rlhf_en/hh_rlhf_en.py b/data/hh_rlhf_en/hh_rlhf_en.py index 3f596d43..2839af7d 100644 --- a/data/hh_rlhf_en/hh_rlhf_en.py +++ b/data/hh_rlhf_en/hh_rlhf_en.py @@ -3,15 +3,12 @@ import json import datasets from typing import List -_HF_ENDPOINT = os.getenv("_HF_ENDPOINT", "https://huggingface.co") - +_HF_ENDPOINT = os.getenv("HF_ENDPOINT", "https://huggingface.co") _DESCRIPTION = "Human preference data about helpfulness and harmlessness." - _CITATION = "" - -_HOMEPAGE = f"{_HF_ENDPOINT}/datasets/Anthropic/hh-rlhf" +_HOMEPAGE = "{}/datasets/Anthropic/hh-rlhf".format(_HF_ENDPOINT) _LICENSE = "mit" -_URL = f"{_HF_ENDPOINT}/datasets/Anthropic/hh-rlhf/resolve/main/" +_URL = "{}/datasets/Anthropic/hh-rlhf/resolve/main/".format(_HF_ENDPOINT) _URLS = { "train": [ _URL + "harmless-base/train.jsonl.gz", diff --git a/data/orca_rlhf.json.REMOVED.git-id b/data/orca_rlhf.json.REMOVED.git-id new file mode 100644 index 00000000..45f1a9ac --- /dev/null +++ b/data/orca_rlhf.json.REMOVED.git-id @@ -0,0 +1 @@ +736bcedea2b24a1414765c6d69cbdafaea839f3c \ No newline at end of file diff --git a/data/ultra_chat/ultra_chat.py b/data/ultra_chat/ultra_chat.py index 1a337812..2e8a75e1 100644 --- a/data/ultra_chat/ultra_chat.py +++ b/data/ultra_chat/ultra_chat.py @@ -3,7 +3,7 @@ import json import datasets from typing import List -_HF_ENDPOINT = os.getenv("_HF_ENDPOINT", "https://huggingface.co") +_HF_ENDPOINT = os.getenv("HF_ENDPOINT", "https://huggingface.co") _DESCRIPTION = "UltraChat: Large-scale, Informative, and Diverse Multi-round Dialogue Data." @@ -18,9 +18,9 @@ _CITATION = """\ } """ -_HOMEPAGE = f"{_HF_ENDPOINT}/datasets/stingning/ultrachat" +_HOMEPAGE = "{}/datasets/stingning/ultrachat".format(_HF_ENDPOINT) _LICENSE = "cc-by-nc-4.0" -_BASE_DATA_URL = "{_HF_ENDPOINT}/datasets/stingning/ultrachat/resolve/main/train_{idx}.jsonl" +_BASE_DATA_URL = "{}/datasets/stingning/ultrachat/resolve/main/train_{{idx}}.jsonl".format(_HF_ENDPOINT) class UltraChat(datasets.GeneratorBasedBuilder): @@ -40,7 +40,7 @@ class UltraChat(datasets.GeneratorBasedBuilder): ) def _split_generators(self, dl_manager: datasets.DownloadManager): - file_paths = [dl_manager.download(_BASE_DATA_URL.format(_HF_ENDPOINT=_HF_ENDPOINT,idx=idx)) for idx in range(10)] # multiple shards + file_paths = [dl_manager.download(_BASE_DATA_URL.format(idx=idx)) for idx in range(10)] # multiple shards return [ datasets.SplitGenerator( name=datasets.Split.TRAIN,