423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-16 00:28:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

update args for MsDataset.load

Browse Source 
Former-commit-id: c5f69357a167cbf99a93607177526e787419ea05
This commit is contained in:
xingjun.wang 2023-12-12 13:02:54 +08:00
parent 0baf32e219
commit ed26bb3d82
2 changed files with 9 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/llmtuner/data/loader.py
View File

@ -58,6 +58,11 @@ def get_dataset(
dataset = MsDataset.load( dataset = MsDataset.load(
dataset_name=data_path, dataset_name=data_path,
subset_name=data_name, subset_name=data_name,
split=data_args.split,
data_files=data_files,
cache_dir=model_args.cache_dir,
token=model_args.ms_hub_token,
streaming=(data_args.streaming and (dataset_attr.load_from != "file")),
).to_hf_dataset() ).to_hf_dataset()
else: else:
dataset = load_dataset( dataset = load_dataset(

4
src/llmtuner/hparams/model_args.py
View File

@ -59,6 +59,10 @@ class ModelArguments:
default=None, default=None,
metadata={"help": "Auth token to log in with Hugging Face Hub."} metadata={"help": "Auth token to log in with Hugging Face Hub."}
) )
ms_hub_token: Optional[str] = field(
default=None,
metadata={"help": "Auth token to log in with ModelScope Hub."}
)
def __post_init__(self): def __post_init__(self):
self.compute_dtype = None self.compute_dtype = None