423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-23 22:32:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6123 from hiyouga/hiyouga/fix_qwen2vl_vllm

Browse Source 
[inference] fix qwen2vl vllm infer

Former-commit-id: 0ce173e2a41f070051cbb7d50180a2f3b7cae89d
This commit is contained in:
hoshi-hiyouga 2024-11-24 23:42:11 +08:00 committed by GitHub
commit 5a7ff02806
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/llamafactory/chat/vllm_engine.py
View File

@ -112,7 +112,15 @@ class VllmEngine(BaseEngine):
if not any(IMAGE_PLACEHOLDER in message["content"] for message in messages): if not any(IMAGE_PLACEHOLDER in message["content"] for message in messages):
messages[0]["content"] = IMAGE_PLACEHOLDER * len(images) + messages[0]["content"] messages[0]["content"] = IMAGE_PLACEHOLDER * len(images) + messages[0]["content"]
paired_messages = messages + [{"role": "assistant", "content": ""}] if self.template.mm_plugin.__class__.__name__ == "Qwen2vlPlugin": # temporary solution
image_str = "<|vision_start|>" + self.template.mm_plugin.image_token + "<|vision_end|>"
else:
image_str = self.template.mm_plugin.image_token
paired_messages = [
{"role": message["role"], "content": message["content"].replace(IMAGE_PLACEHOLDER, image_str)}
for message in messages
] + [{"role": "assistant", "content": ""}]
system = system or self.generating_args["default_system"] system = system or self.generating_args["default_system"]
prompt_ids, _ = self.template.encode_oneturn(self.tokenizer, paired_messages, system, tools) prompt_ids, _ = self.template.encode_oneturn(self.tokenizer, paired_messages, system, tools)
prompt_length = len(prompt_ids) prompt_length = len(prompt_ids)