423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-14 10:56:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

[model] fix kv cache (#7564)

Browse Source
This commit is contained in:
hoshi-hiyouga
2025-04-01 23:07:46 +08:00
committed by GitHub
parent 69b0c1cf4f
commit 37d783149d
16 changed files with 122 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/vllm_infer.py
View File

@@ -56,7 +56,7 @@ def vllm_infer(
Usage: python vllm_infer.py --model_name_or_path meta-llama/Llama-2-7b-hf --template llama --dataset alpaca_en_demo
"""
check_version("vllm>=0.4.3,<=0.7.3")
check_version("vllm>=0.4.3,<=0.8.2")
if pipeline_parallel_size > get_device_count():
raise ValueError("Pipeline parallel size should be smaller than the number of gpus.")