423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 03:40:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

support quantization in export model

Browse Source
This commit is contained in:
hiyouga
2023-12-15 23:44:50 +08:00
parent 87ef3f47b5
commit 3524aa1e58
9 changed files with 120 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/llmtuner/data/utils.py
View File

@@ -12,16 +12,6 @@ if TYPE_CHECKING:
logger = get_logger(__name__)
EXT2TYPE = {
"arrow": "arrow",
"csv": "csv",
"json": "json",
"jsonl": "json",
"parquet": "parquet",
"txt": "text"
}
def checksum(data_files: List[str], file_sha1: Optional[str] = None) -> None:
if file_sha1 is None:
logger.warning("Checksum failed: missing SHA-1 hash value in dataset_info.json.")