From e17f2a3f7ffe4edfe06fa9c113895bc9ab9ad121 Mon Sep 17 00:00:00 2001 From: "xingjun.wang" Date: Tue, 12 Dec 2023 13:08:18 +0800 Subject: [PATCH] update cache dir Former-commit-id: edc82b923a3fb03c5af100b5357e10f0c18b4523 --- src/llmtuner/data/loader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/llmtuner/data/loader.py b/src/llmtuner/data/loader.py index 7bd52caa..82130485 100644 --- a/src/llmtuner/data/loader.py +++ b/src/llmtuner/data/loader.py @@ -55,12 +55,14 @@ def get_dataset( if int(os.environ.get('USE_MODELSCOPE_HUB', '0')) and dataset_attr.load_from == "ms_hub": from modelscope import MsDataset + cache_dir = model_args.cache_dir + cache_dir = str(cache_dir) if cache_dir is not None else None dataset = MsDataset.load( dataset_name=data_path, subset_name=data_name, split=data_args.split, data_files=data_files, - cache_dir=model_args.cache_dir, + cache_dir=cache_dir, token=model_args.ms_hub_token, streaming=(data_args.streaming and (dataset_attr.load_from != "file")), ).to_hf_dataset()