1. add model and dataset info to support webui

Former-commit-id: 0f669f221a31622ec7a53d0baab5da6a7891f9b6
2025-08-01 11:12:50 +08:00 · 2024-10-10 16:46:34 +08:00 · 2024-10-10 16:46:34 +08:00 · 132c1f1b0f
commit 132c1f1b0f
parent 26e897e861
7 changed files with 28 additions and 7 deletions
--- a/README.md
+++ b/README.md
@ -417,7 +417,7 @@ Download the pre-built Docker images: [32GB](http://mirrors.cn-central-221.ovaij

 ### Data Preparation

-Please refer to [data/README.md](data/README.md) for checking the details about the format of dataset files. You can use datasets on HuggingFace hub, ModelScope hub, modelers hub or load the dataset in local disk.
+Please refer to [data/README.md](data/README.md) for checking the details about the format of dataset files. You can either use datasets on HuggingFace / ModelScope / Modelers hub or load the dataset in local disk.

 > [!NOTE]
 > Please update `data/dataset_info.json` to use your custom dataset.
--- a/README_zh.md
+++ b/README_zh.md
@ -417,8 +417,7 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh

 ### 数据准备

-关于数据集文件的格式，请参考 [data/README_zh.md](data/README_zh.md) 的内容。你可以使用 HuggingFace , ModelScope 或者 Modelers 上的数据集或加载本地数据集。
-
+关于数据集文件的格式，请参考 [data/README_zh.md](data/README_zh.md) 的内容。你可以使用 HuggingFace / ModelScope / Modelers 上的数据集或加载本地数据集。
 > [!NOTE]
 > 使用自定义数据集时，请更新 `data/dataset_info.json` 文件。

--- a/data/dataset_info.json
+++ b/data/dataset_info.json
@ -54,7 +54,8 @@
  },
  "alpaca_en": {
    "hf_hub_url": "llamafactory/alpaca_en",
-    "ms_hub_url": "llamafactory/alpaca_en"
+    "ms_hub_url": "llamafactory/alpaca_en",
+    "om_hub_url": "HaM/alpaca_en"
  },
  "alpaca_zh": {
    "hf_hub_url": "llamafactory/alpaca_zh",
@ -66,7 +67,8 @@
  },
  "alpaca_gpt4_zh": {
    "hf_hub_url": "llamafactory/alpaca_gpt4_zh",
-    "ms_hub_url": "llamafactory/alpaca_gpt4_zh"
+    "ms_hub_url": "llamafactory/alpaca_gpt4_zh",
+    "om_hub_url": "State_Cloud/alpaca-gpt4-data-zh"
  },
  "glaive_toolcall_en": {
    "hf_hub_url": "llamafactory/glaive_toolcall_en",
--- a/src/llamafactory/data/loader.py
+++ b/src/llamafactory/data/loader.py
@ -94,7 +94,7 @@ def _load_single_dataset(
                name=data_name,
                data_dir=data_dir,
                data_files=data_files,
-                split=data_args.split,
+                split=dataset_attr.split,
                cache_dir=cache_dir,
                token=model_args.ms_hub_token,
                streaming=(data_args.streaming and (dataset_attr.load_from != "file")),
--- a/src/llamafactory/data/parser.py
+++ b/src/llamafactory/data/parser.py
@ -98,6 +98,7 @@ def get_dataset_list(dataset_names: Optional[Sequence[str]], dataset_dir: str) -
    dataset_list: List["DatasetAttr"] = []
    for name in dataset_names:
        if dataset_info is None: # dataset_dir is ONLINE
+            load_from = None
            if use_openmind():
                load_from = "om_hub"
            if use_modelscope():
--- a/src/llamafactory/extras/constants.py
+++ b/src/llamafactory/extras/constants.py
@ -107,6 +107,7 @@ VISION_MODELS = set()
 class DownloadSource(str, Enum):
    DEFAULT = "hf"
    MODELSCOPE = "ms"
+    MODELERS = "om"


 def register_model_group(
@ -163,14 +164,17 @@ register_model_group(
        "Baichuan2-13B-Base": {
            DownloadSource.DEFAULT: "baichuan-inc/Baichuan2-13B-Base",
            DownloadSource.MODELSCOPE: "baichuan-inc/Baichuan2-13B-Base",
+            DownloadSource.MODELERS: "Baichuan/Baichuan2_13b_base_pt"
        },
        "Baichuan2-7B-Chat": {
            DownloadSource.DEFAULT: "baichuan-inc/Baichuan2-7B-Chat",
            DownloadSource.MODELSCOPE: "baichuan-inc/Baichuan2-7B-Chat",
+            DownloadSource.MODELERS: "Baichuan/Baichuan2_7b_chat_pt"
        },
        "Baichuan2-13B-Chat": {
            DownloadSource.DEFAULT: "baichuan-inc/Baichuan2-13B-Chat",
            DownloadSource.MODELSCOPE: "baichuan-inc/Baichuan2-13B-Chat",
+            DownloadSource.MODELERS: "Baichuan/Baichuan2_13b_chat_pt"
        },
    },
    template="baichuan2",
@ -559,6 +563,7 @@ register_model_group(
        "Gemma-2-9B-Instruct": {
            DownloadSource.DEFAULT: "google/gemma-2-9b-it",
            DownloadSource.MODELSCOPE: "LLM-Research/gemma-2-9b-it",
+            DownloadSource.MODELERS: "LlamaFactory/Qwen2-VL-2B-Instruct"
        },
        "Gemma-2-27B-Instruct": {
            DownloadSource.DEFAULT: "google/gemma-2-27b-it",
@ -656,6 +661,7 @@ register_model_group(
        "InternLM2.5-20B-Chat": {
            DownloadSource.DEFAULT: "internlm/internlm2_5-20b-chat",
            DownloadSource.MODELSCOPE: "Shanghai_AI_Laboratory/internlm2_5-20b-chat",
+            DownloadSource.MODELERS: "Intern/internlm2_5-20b-chat"
        },
    },
    template="intern2",
@ -756,6 +762,7 @@ register_model_group(
        "Llama-3-8B-Chinese-Chat": {
            DownloadSource.DEFAULT: "shenzhi-wang/Llama3-8B-Chinese-Chat",
            DownloadSource.MODELSCOPE: "LLM-Research/Llama3-8B-Chinese-Chat",
+            DownloadSource.MODELERS: "HaM/Llama3-8B-Chinese-Chat",
        },
        "Llama-3-70B-Chinese-Chat": {
            DownloadSource.DEFAULT: "shenzhi-wang/Llama3-70B-Chinese-Chat",
@ -960,6 +967,7 @@ register_model_group(
        "MiniCPM3-4B-Chat": {
            DownloadSource.DEFAULT: "openbmb/MiniCPM3-4B",
            DownloadSource.MODELSCOPE: "OpenBMB/MiniCPM3-4B",
+            DownloadSource.MODELERS: "LlamaFactory/MiniCPM3-4B"
        },
    },
    template="cpm3",
@ -1699,6 +1707,7 @@ register_model_group(
        "Qwen2-VL-2B-Instruct": {
            DownloadSource.DEFAULT: "Qwen/Qwen2-VL-2B-Instruct",
            DownloadSource.MODELSCOPE: "qwen/Qwen2-VL-2B-Instruct",
+            DownloadSource.MODELERS: "LlamaFactory/Qwen2-VL-2B-Instruct"
        },
        "Qwen2-VL-7B-Instruct": {
            DownloadSource.DEFAULT: "Qwen/Qwen2-VL-7B-Instruct",
@ -1801,10 +1810,12 @@ register_model_group(
        "TeleChat-7B-Chat": {
            DownloadSource.DEFAULT: "Tele-AI/telechat-7B",
            DownloadSource.MODELSCOPE: "TeleAI/telechat-7B",
+            DownloadSource.MODELERS: "TeleAI/TeleChat-7B-pt"
        },
        "TeleChat-12B-Chat": {
            DownloadSource.DEFAULT: "Tele-AI/TeleChat-12B",
            DownloadSource.MODELSCOPE: "TeleAI/TeleChat-12B",
+            DownloadSource.MODELERS: "TeleAI/TeleChat-12B-pt",
        },
        "TeleChat-12B-v2-Chat": {
            DownloadSource.DEFAULT: "Tele-AI/TeleChat-12B-v2",
@ -2023,6 +2034,7 @@ register_model_group(
        "Yi-1.5-6B-Chat": {
            DownloadSource.DEFAULT: "01-ai/Yi-1.5-6B-Chat",
            DownloadSource.MODELSCOPE: "01ai/Yi-1.5-6B-Chat",
+            DownloadSource.MODELERS: "LlamaFactory/Yi-1.5-6B-Chat"
        },
        "Yi-1.5-9B-Chat": {
            DownloadSource.DEFAULT: "01-ai/Yi-1.5-9B-Chat",
--- a/src/llamafactory/webui/common.py
+++ b/src/llamafactory/webui/common.py
@ -31,7 +31,7 @@ from ..extras.constants import (
    DownloadSource,
 )
 from ..extras.logging import get_logger
-from ..extras.misc import use_modelscope
+from ..extras.misc import use_modelscope, use_openmind
 from ..extras.packages import is_gradio_available


@ -112,6 +112,13 @@ def get_model_path(model_name: str) -> str:
    ):  # replace path
        model_path = path_dict.get(DownloadSource.MODELSCOPE)

+    if (
+        use_openmind()
+        and path_dict.get(DownloadSource.MODELERS)
+        and model_path == path_dict.get(DownloadSource.DEFAULT)
+    ):  # replace path
+        model_path = path_dict.get(DownloadSource.MODELERS)
+
    return model_path