423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-10-16 00:28:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

add code for reading from multi files in one directory

Browse Source 
Former-commit-id: 9b80cf08b9f0d4aee896b228fb76399e9a7c9d8b
This commit is contained in:
BUAADreamer 2023-06-10 16:27:30 +08:00
parent ef6c5ae18a
commit 5b93ca6c39
1 changed files with 0 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/utils/common.py
View File

@ -358,12 +358,10 @@ def prepare_data(
elif dataset_attr.load_from == "file":
data_file = os.path.join(data_args.dataset_dir, dataset_attr.file_name)
extension = dataset_attr.file_name.split(".")[-1]
if dataset_attr.file_sha1 is not None:
checksum(data_file, dataset_attr.file_sha1)
else:
logger.warning("Checksum failed: missing SHA-1 hash value in dataset_info.json.")
print(extension)
raw_datasets = load_dataset(
extension if extension in ["csv", "json"] else "text",
data_files=data_file,