423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-11-08 06:16:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

add

Browse Source
This commit is contained in:
kingsley 2025-09-27 20:12:08 +08:00
parent 66ec2b6cda
commit 0c017bac01
2 changed files with 34 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/llamafactory/model/model_utils/visual.py
View File

@ -350,6 +350,32 @@ _register_composite_model(
lora_conflict_keys=["patch_embed"], lora_conflict_keys=["patch_embed"],
) )
_register_composite_model(
model_type="qwen3_vl",
projector_key="visual.merger",
vision_model_keys=["visual.patch_embed", "visual.blocks"],
language_model_keys=["language_model", "lm_head"],
lora_conflict_keys=["patch_embed"],
)
_register_composite_model(
model_type="qwen3_vl_moe",
projector_key="visual.merger",
vision_model_keys=["visual.patch_embed", "visual.blocks"],
language_model_keys=["language_model", "lm_head"],
lora_conflict_keys=["patch_embed"],
)
_register_composite_model(
model_type="qwen3_omni_moe_thinker",
projector_key="visual.merger",
vision_model_keys=["visual.patch_embed", "visual.blocks", "audio_tower"],
language_model_keys=["model", "lm_head"],
lora_conflict_keys=["patch_embed"],
)
_register_composite_model( _register_composite_model(
model_type="video_llava", model_type="video_llava",

11
tests/data/test_mm_plugin.py
View File

@ -332,9 +332,14 @@ def test_qwen2_omni_plugin():
image_seqlen, audio_seqlen = 4, 2 image_seqlen, audio_seqlen = 4, 2
tokenizer_module = _load_tokenizer_module(model_name_or_path="Qwen/Qwen2.5-Omni-7B") tokenizer_module = _load_tokenizer_module(model_name_or_path="Qwen/Qwen2.5-Omni-7B")
qwen2_omni_plugin = get_mm_plugin( qwen2_omni_plugin = get_mm_plugin(
name="qwen2_omni", audio_token="<|AUDIO|>", image_token="<|IMAGE|>", video_token="<|VIDEO|>", name="qwen2_omni",
vision_bos_token="<|vision_bos|>", vision_eos_token="<|vision_eos|>", audio_token="<|AUDIO|>",
audio_bos_token="<|audio_bos|>", audio_eos_token="<|audio_eos|>" image_token="<|IMAGE|>",
video_token="<|VIDEO|>",
vision_bos_token="<|vision_bos|>",
vision_eos_token="<|vision_eos|>",
audio_bos_token="<|audio_bos|>",
audio_eos_token="<|audio_eos|>",
) )
check_inputs = {"plugin": qwen2_omni_plugin, **tokenizer_module} check_inputs = {"plugin": qwen2_omni_plugin, **tokenizer_module}
check_inputs["expected_mm_messages"] = [ check_inputs["expected_mm_messages"] = [