423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-14 23:58:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

1. add model and dataset info to support webui

Browse Source 
Former-commit-id: 92f6226f3fecbd9af744a7232dda2c68b2bb0d86
This commit is contained in:
huniu20 2024-10-10 16:46:34 +08:00
parent c42dcab32b
commit aad79127e6
6 changed files with 24 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -417,7 +417,7 @@ Download the pre-built Docker images: [32GB](http://mirrors.cn-central-221.ovaij
### Data Preparation
Please refer to [data/README.md](data/README.md) for checking the details about the format of dataset files. You can use datasets on HuggingFace hub, ModelScope hub, modelers hub or load the dataset in local disk.
Please refer to [data/README.md](data/README.md) for checking the details about the format of dataset files. You can either use datasets on HuggingFace / ModelScope / Modelers hub or load the dataset in local disk.
> [!NOTE]
> Please update `data/dataset_info.json` to use your custom dataset.

3
README_zh.md
View File

@ -417,8 +417,7 @@ source /usr/local/Ascend/ascend-toolkit/set_env.sh
### 数据准备
关于数据集文件的格式，请参考 [data/README_zh.md](data/README_zh.md) 的内容。你可以使用 HuggingFace , ModelScope 或者 Modelers 上的数据集或加载本地数据集。
关于数据集文件的格式，请参考 [data/README_zh.md](data/README_zh.md) 的内容。你可以使用 HuggingFace / ModelScope / Modelers 上的数据集或加载本地数据集。
> [!NOTE]
> 使用自定义数据集时，请更新 `data/dataset_info.json` 文件。

2
src/llamafactory/data/loader.py
View File

@ -94,7 +94,7 @@ def _load_single_dataset(
name=data_name,
data_dir=data_dir,
data_files=data_files,
split=data_args.split,
split=dataset_attr.split,
cache_dir=cache_dir,
token=model_args.ms_hub_token,
streaming=(data_args.streaming and (dataset_attr.load_from != "file")),

1
src/llamafactory/data/parser.py
View File

@ -98,6 +98,7 @@ def get_dataset_list(dataset_names: Optional[Sequence[str]], dataset_dir: str) -
dataset_list: List["DatasetAttr"] = []
for name in dataset_names:
if dataset_info is None: # dataset_dir is ONLINE
load_from = None
if use_openmind():
load_from = "om_hub"
if use_modelscope():

12
src/llamafactory/extras/constants.py
View File

@ -107,6 +107,7 @@ VISION_MODELS = set()
class DownloadSource(str, Enum):
DEFAULT = "hf"
MODELSCOPE = "ms"
MODELERS = "om"
def register_model_group(
@ -163,14 +164,17 @@ register_model_group(
"Baichuan2-13B-Base": {
DownloadSource.DEFAULT: "baichuan-inc/Baichuan2-13B-Base",
DownloadSource.MODELSCOPE: "baichuan-inc/Baichuan2-13B-Base",
DownloadSource.MODELERS: "Baichuan/Baichuan2_13b_base_pt"
},
"Baichuan2-7B-Chat": {
DownloadSource.DEFAULT: "baichuan-inc/Baichuan2-7B-Chat",
DownloadSource.MODELSCOPE: "baichuan-inc/Baichuan2-7B-Chat",
DownloadSource.MODELERS: "Baichuan/Baichuan2_7b_chat_pt"
},
"Baichuan2-13B-Chat": {
DownloadSource.DEFAULT: "baichuan-inc/Baichuan2-13B-Chat",
DownloadSource.MODELSCOPE: "baichuan-inc/Baichuan2-13B-Chat",
DownloadSource.MODELERS: "Baichuan/Baichuan2_13b_chat_pt"
},
},
template="baichuan2",
@ -559,6 +563,7 @@ register_model_group(
"Gemma-2-9B-Instruct": {
DownloadSource.DEFAULT: "google/gemma-2-9b-it",
DownloadSource.MODELSCOPE: "LLM-Research/gemma-2-9b-it",
DownloadSource.MODELERS: "LlamaFactory/Qwen2-VL-2B-Instruct"
},
"Gemma-2-27B-Instruct": {
DownloadSource.DEFAULT: "google/gemma-2-27b-it",
@ -656,6 +661,7 @@ register_model_group(
"InternLM2.5-20B-Chat": {
DownloadSource.DEFAULT: "internlm/internlm2_5-20b-chat",
DownloadSource.MODELSCOPE: "Shanghai_AI_Laboratory/internlm2_5-20b-chat",
DownloadSource.MODELERS: "Intern/internlm2_5-20b-chat"
},
},
template="intern2",
@ -756,6 +762,7 @@ register_model_group(
"Llama-3-8B-Chinese-Chat": {
DownloadSource.DEFAULT: "shenzhi-wang/Llama3-8B-Chinese-Chat",
DownloadSource.MODELSCOPE: "LLM-Research/Llama3-8B-Chinese-Chat",
DownloadSource.MODELERS: "HaM/Llama3-8B-Chinese-Chat",
},
"Llama-3-70B-Chinese-Chat": {
DownloadSource.DEFAULT: "shenzhi-wang/Llama3-70B-Chinese-Chat",
@ -960,6 +967,7 @@ register_model_group(
"MiniCPM3-4B-Chat": {
DownloadSource.DEFAULT: "openbmb/MiniCPM3-4B",
DownloadSource.MODELSCOPE: "OpenBMB/MiniCPM3-4B",
DownloadSource.MODELERS: "LlamaFactory/MiniCPM3-4B"
},
},
template="cpm3",
@ -1699,6 +1707,7 @@ register_model_group(
"Qwen2-VL-2B-Instruct": {
DownloadSource.DEFAULT: "Qwen/Qwen2-VL-2B-Instruct",
DownloadSource.MODELSCOPE: "qwen/Qwen2-VL-2B-Instruct",
DownloadSource.MODELERS: "LlamaFactory/Qwen2-VL-2B-Instruct"
},
"Qwen2-VL-7B-Instruct": {
DownloadSource.DEFAULT: "Qwen/Qwen2-VL-7B-Instruct",
@ -1801,10 +1810,12 @@ register_model_group(
"TeleChat-7B-Chat": {
DownloadSource.DEFAULT: "Tele-AI/telechat-7B",
DownloadSource.MODELSCOPE: "TeleAI/telechat-7B",
DownloadSource.MODELERS: "TeleAI/TeleChat-7B-pt"
},
"TeleChat-12B-Chat": {
DownloadSource.DEFAULT: "Tele-AI/TeleChat-12B",
DownloadSource.MODELSCOPE: "TeleAI/TeleChat-12B",
DownloadSource.MODELERS: "TeleAI/TeleChat-12B-pt",
},
"TeleChat-12B-v2-Chat": {
DownloadSource.DEFAULT: "Tele-AI/TeleChat-12B-v2",
@ -2023,6 +2034,7 @@ register_model_group(
"Yi-1.5-6B-Chat": {
DownloadSource.DEFAULT: "01-ai/Yi-1.5-6B-Chat",
DownloadSource.MODELSCOPE: "01ai/Yi-1.5-6B-Chat",
DownloadSource.MODELERS: "LlamaFactory/Yi-1.5-6B-Chat"
},
"Yi-1.5-9B-Chat": {
DownloadSource.DEFAULT: "01-ai/Yi-1.5-9B-Chat",

9
src/llamafactory/webui/common.py
View File

@ -31,7 +31,7 @@ from ..extras.constants import (
DownloadSource,
)
from ..extras.logging import get_logger
from ..extras.misc import use_modelscope
from ..extras.misc import use_modelscope, use_openmind
from ..extras.packages import is_gradio_available
@ -112,6 +112,13 @@ def get_model_path(model_name: str) -> str:
): # replace path
model_path = path_dict.get(DownloadSource.MODELSCOPE)
if (
use_openmind()
and path_dict.get(DownloadSource.MODELERS)
and model_path == path_dict.get(DownloadSource.DEFAULT)
): # replace path
model_path = path_dict.get(DownloadSource.MODELERS)
return model_path