423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 11:50:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/support_ms

Browse Source
This commit is contained in:
hoshi-hiyouga
2023-12-01 20:23:46 +08:00
committed by GitHub
parent 5a2392f105 a5a248d569
commit 00f5c9ee16
16 changed files with 121 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/llmtuner/webui/common.py
View File

@@ -11,14 +11,22 @@ from transformers.utils import (
ADAPTER_SAFE_WEIGHTS_NAME
)
from llmtuner.extras.constants import DEFAULT_MODULE, DEFAULT_TEMPLATE, SUPPORTED_MODELS, ALL_OFFICIAL_MODELS, TRAINING_STAGES
from llmtuner.extras.constants import (
DEFAULT_MODULE,
DEFAULT_TEMPLATE,
SUPPORTED_MODELS,
ALL_OFFICIAL_MODELS,
TRAINING_STAGES
)
from llmtuner.hparams.data_args import DATA_CONFIG
DEFAULT_CACHE_DIR = "cache"
DEFAULT_DATA_DIR = "data"
DEFAULT_SAVE_DIR = "saves"
USER_CONFIG = "user.config"
DATA_CONFIG = "dataset_info.json"
CKPT_NAMES = [
WEIGHTS_NAME,
WEIGHTS_INDEX_NAME,
@@ -92,12 +100,12 @@ def list_checkpoint(model_name: str, finetuning_type: str) -> Dict[str, Any]:
return gr.update(value=[], choices=checkpoints)
def load_dataset_info(dataset_dir: str) -> Dict[str, Any]:
def load_dataset_info(dataset_dir: str) -> Dict[str, Dict[str, Any]]:
try:
with open(os.path.join(dataset_dir, DATA_CONFIG), "r", encoding="utf-8") as f:
return json.load(f)
except:
print("Cannot find {} in {}.".format(DATA_CONFIG, dataset_dir))
except Exception as err:
print("Cannot open {} due to {}.".format(os.path.join(dataset_dir, DATA_CONFIG), str(err)))
return {}