support parquet format #1446

Former-commit-id: 3df90b988b
2025-12-14 19:06:26 +08:00 · 2023-11-09 14:17:40 +08:00
parent b9f42172dd
commit f9ebc718d0
1 changed files with 2 additions and 0 deletions
--- a/src/llmtuner/dsets/utils.py
+++ b/src/llmtuner/dsets/utils.py
@@ -13,9 +13,11 @@ logger = get_logger(__name__)


 EXT2TYPE = {
+    "arrow": "arrow",
    "csv": "csv",
    "json": "json",
    "jsonl": "json",
+    "parquet": "parquet",
    "txt": "text"
 }