423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-14 10:56:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix #2782 #2798

Browse Source 
Former-commit-id: eb3ab610610a0964bc8a1c9fa015805353f04c31
This commit is contained in:
hiyouga
2024-03-12 15:53:29 +08:00
parent 9ee416a8fc
commit c9ed3fc3a4
3 changed files with 19 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/llmtuner/chat/vllm_engine.py
View File

@@ -31,9 +31,11 @@ class VllmEngine(BaseEngine):
model=model_args.model_name_or_path,
trust_remote_code=True,
max_model_len=model_args.vllm_maxlen,
tensor_parallel_size=get_device_count(),
tensor_parallel_size=get_device_count() or 1,
gpu_memory_utilization=model_args.vllm_gpu_util,
disable_log_stats=True,
disable_log_requests=True,
enforce_eager=model_args.vllm_enforce_eager,
)
self.model = AsyncLLMEngine.from_engine_args(engine_args)
self.tokenizer = load_tokenizer(model_args)