423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-23 06:12:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5927 from hiyouga/hiyouga/dev_fixmmchat

Browse Source 
[fix] chat engines

Former-commit-id: bcbd1bebb0c2c1f9bfbe920c62008383cb044dba
This commit is contained in:
hoshi-hiyouga 2024-11-04 16:36:23 +08:00 committed by GitHub
commit 2e6dd72539
3 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/llamafactory/chat/hf_engine.py
View File

@ -86,12 +86,12 @@ class HuggingfaceEngine(BaseEngine):
mm_input_dict = {"images": [], "videos": [], "imglens": [0], "vidlens": [0]}
if images is not None:
mm_input_dict.update({"images": images, "imglens": [len(images)]})
if not any(IMAGE_PLACEHOLDER not in message["content"] for message in messages):
if not any(IMAGE_PLACEHOLDER in message["content"] for message in messages):
messages[0]["content"] = IMAGE_PLACEHOLDER * len(images) + messages[0]["content"]
if videos is not None:
mm_input_dict.update({"videos": videos, "vidlens": [len(videos)]})
if not any(VIDEO_PLACEHOLDER not in message["content"] for message in messages):
if not any(VIDEO_PLACEHOLDER in message["content"] for message in messages):
messages[0]["content"] = VIDEO_PLACEHOLDER * len(videos) + messages[0]["content"]
messages = template.mm_plugin.process_messages(

2
src/llamafactory/chat/vllm_engine.py
View File

@ -107,7 +107,7 @@ class VllmEngine(BaseEngine):
) -> AsyncIterator["RequestOutput"]:
request_id = f"chatcmpl-{uuid.uuid4().hex}"
if images is not None:
if not any(IMAGE_PLACEHOLDER not in message["content"] for message in messages):
if not any(IMAGE_PLACEHOLDER in message["content"] for message in messages):
messages[0]["content"] = IMAGE_PLACEHOLDER * len(images) + messages[0]["content"]
paired_messages = messages + [{"role": "assistant", "content": ""}]

4
src/llamafactory/data/mm_plugin.py
View File

@ -30,7 +30,7 @@ if TYPE_CHECKING:
path: Optional[str]
bytes: Optional[bytes]
ImageInput = Union[str, EncodedImage, ImageObject]
ImageInput = Union[str, bytes, EncodedImage, ImageObject]
VideoInput = str
@ -104,6 +104,8 @@ class BasePlugin:
for image in images:
if isinstance(image, str):
image = Image.open(image)
elif isinstance(image, bytes):
image = Image.open(BytesIO(image))
elif isinstance(image, dict):
if image["bytes"] is not None:
image = Image.open(BytesIO(image["bytes"]))