add vllm_dtype arg #3387 #3717

Former-commit-id: 8fcc79e1e6
2025-12-14 19:06:26 +08:00 · 2024-06-06 02:53:27 +08:00
parent 3fcb678d00
commit c439c959f7
2 changed files with 6 additions and 5 deletions
--- a/src/llamafactory/hparams/model_args.py
+++ b/src/llamafactory/hparams/model_args.py
@@ -125,6 +125,10 @@ class ModelArguments:
        default=8,
        metadata={"help": "Maximum rank of all LoRAs in the vLLM engine."},
    )
+    vllm_dtype: Literal["auto", "float16", "bfloat16", "float32"] = field(
+        default="auto",
+        metadata={"help": "Data type for model weights and activations in the vLLM engine."},
+    )
    offload_folder: str = field(
        default="offload",
        metadata={"help": "Path to offload model weights."},