423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-22 13:42:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3794 from jue-jue-zi/main

Browse Source 
feat: pass the `max_lora_rank` parameter to vLLM backend
Former-commit-id: d7ff49f245cd34668cbe43366e5f1890876da5e7
This commit is contained in:
hoshi-hiyouga 2024-05-17 16:17:30 +08:00 committed by GitHub
commit 2d1583faba
2 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/llamafactory/chat/vllm_engine.py
View File

@ -59,6 +59,7 @@ class VllmEngine(BaseEngine):
"disable_log_requests": True, "disable_log_requests": True,
"enforce_eager": model_args.vllm_enforce_eager, "enforce_eager": model_args.vllm_enforce_eager,
"enable_lora": model_args.adapter_name_or_path is not None, "enable_lora": model_args.adapter_name_or_path is not None,
"max_lora_rank": model_args.vllm_max_lora_rank,
} }
if model_args.visual_inputs: if model_args.visual_inputs:

4
src/llamafactory/hparams/model_args.py
View File

@ -117,6 +117,10 @@ class ModelArguments:
default=False, default=False,
metadata={"help": "Whether or not to disable CUDA graph in the vLLM engine."}, metadata={"help": "Whether or not to disable CUDA graph in the vLLM engine."},
) )
vllm_max_lora_rank: int = field(
default=8,
metadata={"help": "Maximum rank of all LoRAs in the vLLM engine."},
)
offload_folder: str = field( offload_folder: str = field(
default="offload", default="offload",
metadata={"help": "Path to offload model weights."}, metadata={"help": "Path to offload model weights."},