423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2026-07-07 09:35:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

[fix] Fixes Qwen3-VL prompt expansion for multiple videos (#10518)

Browse Source 
Co-authored-by: gemini-code-assist <200291788+gemini-code-assist@users.noreply.github.com>
This commit is contained in:
luca-888
2026-07-02 11:06:42 +08:00
committed by GitHub
parent a48af5cc69
commit c8a082e0e3
2 changed files with 28 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/llamafactory/data/mm_plugin.py
View File

@@ -2379,16 +2379,13 @@ class Qwen3VLPlugin(Qwen2VLPlugin):
image_grid_thw = mm_inputs.get("image_grid_thw", []) image_grid_thw = mm_inputs.get("image_grid_thw", [])
video_grid_thw = mm_inputs.get("video_grid_thw", []) video_grid_thw = mm_inputs.get("video_grid_thw", [])
num_frames = video_grid_thw[0][0] if len(video_grid_thw) > 0 else 0 # hard code for now
video_metadata = mm_inputs.get("video_metadata", []) video_metadata = mm_inputs.get("video_metadata", [])
else: else:
image_grid_thw = [None] * len(images) image_grid_thw = [None] * len(images)
video_grid_thw = [None] * len(videos) video_grid_thw = [None] * len(videos)
num_frames = 0
timestamps = [0]
for idx, message in enumerate(messages): for message in messages:
content = message["content"] content = message["content"]
while IMAGE_PLACEHOLDER in content: while IMAGE_PLACEHOLDER in content:
image_seqlen = ( image_seqlen = (
@@ -2403,19 +2400,17 @@ class Qwen3VLPlugin(Qwen2VLPlugin):
while VIDEO_PLACEHOLDER in content: while VIDEO_PLACEHOLDER in content:
if self.expand_mm_tokens: if self.expand_mm_tokens:
metadata = video_metadata[idx] video_grid = video_grid_thw[num_video_tokens]
num_frames = int(video_grid[0].item())
metadata = video_metadata[num_video_tokens]
timestamps = processor._calculate_timestamps( timestamps = processor._calculate_timestamps(
metadata.frames_indices, metadata.frames_indices,
metadata.fps, metadata.fps,
video_processor.merge_size, getattr(video_processor, "temporal_patch_size", 2),
) )
video_structure = "" video_structure = ""
video_seqlen = int((video_grid[1:].prod() // video_merge_length).item())
for frame_index in range(num_frames): for frame_index in range(num_frames):
video_seqlen = (
video_grid_thw[num_video_tokens][1:].prod() // video_merge_length
if self.expand_mm_tokens
else 1
)
timestamp_sec = timestamps[frame_index] timestamp_sec = timestamps[frame_index]
frame_structure = ( frame_structure = (
f"<{timestamp_sec:.1f} seconds>" f"<{timestamp_sec:.1f} seconds>"

32
tests/data/test_mm_plugin.py
View File

@@ -424,19 +424,31 @@ def test_qwen3_vl_plugin():
tokenizer_module = _load_tokenizer_module(model_name_or_path="Qwen/Qwen3-VL-30B-A3B-Instruct") tokenizer_module = _load_tokenizer_module(model_name_or_path="Qwen/Qwen3-VL-30B-A3B-Instruct")
qwen3_vl_plugin = get_mm_plugin(name="qwen3_vl", video_token="<|video_pad|>") qwen3_vl_plugin = get_mm_plugin(name="qwen3_vl", video_token="<|video_pad|>")
check_inputs = {"plugin": qwen3_vl_plugin, **tokenizer_module} check_inputs = {"plugin": qwen3_vl_plugin, **tokenizer_module}
check_inputs["expected_mm_messages"] = [ video_token = "<|video_pad|>" * frame_seqlen
{ first_video = (
key: value.replace( f"<0.2 seconds><|vision_start|>{video_token}<|vision_end|>"
"<video>", # little different with original processor for default `fps=2` in our repo f"<1.2 seconds><|vision_start|>{video_token}<|vision_end|>"
"<0.2 seconds><|vision_start|>{}<|vision_end|><1.2 seconds><|vision_start|>{}<|vision_end|>".format(
"<|video_pad|>" * frame_seqlen, "<|video_pad|>" * frame_seqlen
),
) )
for key, value in message.items() second_video = first_video + f"<2.2 seconds><|vision_start|>{video_token}<|vision_end|>"
} videos = [
for message in VIDEO_MESSAGES [Image.new("RGB", (32, 32), (255, 255, 255))] * 4,
[Image.new("RGB", (32, 32), (255, 255, 255))] * 6,
]
messages = [
{"role": "user", "content": "Compare these videos: <video> and <video>."},
{"role": "assistant", "content": "They are different."},
]
check_inputs["expected_mm_messages"] = [
{key: value.replace("<video>", first_video) for key, value in message.items()} for message in VIDEO_MESSAGES
] ]
_check_plugin(**check_inputs) _check_plugin(**check_inputs)
assert qwen3_vl_plugin.process_messages(messages, NO_IMAGES, videos, NO_AUDIOS, tokenizer_module["processor"]) == [
{
"role": "user",
"content": f"Compare these videos: {first_video} and {second_video}.",
},
{"role": "assistant", "content": "They are different."},
]
@pytest.mark.runs_on(["cpu", "mps"]) @pytest.mark.runs_on(["cpu", "mps"])