423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2026-08-17 05:25:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

[model] add MOSS-VL support (#10708)

Browse Source
This commit is contained in:
SSSSuperC
2026-08-03 18:18:24 +08:00
committed by GitHub
parent 62ae362455
commit 713b5a3f95
16 changed files with 1432 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
examples/inference/mossvl.yaml Normal file
View File

@@ -0,0 +1,6 @@
### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt`
model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708
template: moss_vl
infer_backend: huggingface # choices: [huggingface, vllm, sglang, ktransformers]
trust_remote_code: true

14
examples/merge_lora/mossvl_lora_sft.yaml Normal file
View File

@@ -0,0 +1,14 @@
### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt`
### Note: DO NOT use quantized model or quantization_bit when merging lora adapters
### model
model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708
adapter_name_or_path: saves/moss-vl-11b/lora/sft
template: moss_vl
trust_remote_code: true
### export
export_dir: saves/moss_vl_sft_merged
export_size: 5
export_device: cpu # choices: [cpu, auto]
export_legacy_format: false

57
examples/train_full/mossvl_full_sft.yaml Normal file
View File

@@ -0,0 +1,57 @@
### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt`
### model
model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708
image_max_pixels: 262144
video_max_pixels: 16384
video_fps: 1.0
video_maxlen: 256
use_reentrant_gc: false
trust_remote_code: true
### method
stage: sft
do_train: true
finetuning_type: full
freeze_vision_tower: true
freeze_multi_modal_projector: true
freeze_language_model: false
deepspeed: examples/deepspeed/ds_z3_config.json
### dataset
dataset: mllm_demo,identity,alpaca_en_demo # video: mllm_video_demo
template: moss_vl
cutoff_len: 4096
max_samples: 1000
preprocessing_num_workers: 16
dataloader_num_workers: 4
packing: false
### output
output_dir: saves/moss-vl-11b/full/sft
logging_steps: 10
save_steps: 500
plot_loss: true
overwrite_output_dir: true
save_only_model: false
report_to: none # choices: [none, wandb, tensorboard, swanlab, mlflow]
### train
per_device_train_batch_size: 1
gradient_accumulation_steps: 1
gradient_checkpointing: true
gradient_checkpointing_kwargs:
use_reentrant: false
learning_rate: 1.0e-5
num_train_epochs: 3.0
lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null
### eval
# val_size: 0.1
# per_device_eval_batch_size: 1
# eval_strategy: steps
# eval_steps: 500

54
examples/train_lora/mossvl_lora_sft.yaml Normal file
View File

@@ -0,0 +1,54 @@
### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt`
### model
model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708
image_max_pixels: 262144
video_max_pixels: 16384
video_fps: 1.0
video_maxlen: 256
trust_remote_code: true
### method
stage: sft
do_train: true
finetuning_type: lora
lora_rank: 8
lora_target: all
freeze_vision_tower: true
freeze_multi_modal_projector: true
freeze_language_model: false
### dataset
dataset: mllm_demo,identity,alpaca_en_demo # video: mllm_video_demo
template: moss_vl
cutoff_len: 4096
max_samples: 1000
preprocessing_num_workers: 16
dataloader_num_workers: 4
packing: false
### output
output_dir: saves/moss-vl-11b/lora/sft
logging_steps: 10
save_steps: 500
plot_loss: true
overwrite_output_dir: true
save_only_model: false
report_to: none # choices: [none, wandb, tensorboard, swanlab, mlflow]
### train
per_device_train_batch_size: 2
gradient_accumulation_steps: 1
learning_rate: 1.0e-4
num_train_epochs: 3.0
lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: true
ddp_timeout: 180000000
resume_from_checkpoint: null
### eval
# val_size: 0.1
# per_device_eval_batch_size: 1
# eval_strategy: steps
# eval_steps: 500

3
requirements/moss-vl.txt Normal file
View File

@@ -0,0 +1,3 @@
transformers==4.57.1
torchcodec==0.7.0
joblib

19
src/llamafactory/data/collator.py
View File

@@ -150,7 +150,9 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
if isinstance(self.model, PeftModel):
self.model = self.model.base_model.model
if self.model is not None and hasattr(self.model, "get_rope_index"): # for qwen2vl mrope
if getattr(getattr(self.model, "config", None), "model_type", None) == "moss_vl":
self.get_rope_func = None # MOSS-VL computes its own XRoPE positions in model.forward.
elif self.model is not None and hasattr(self.model, "get_rope_index"): # for qwen2vl mrope
self.get_rope_func = self.model.get_rope_index # transformers < 4.52.0 or qwen2.5 omni
elif self.model is not None and hasattr(self.model, "model") and hasattr(self.model.model, "get_rope_index"):
self.get_rope_func = self.model.model.get_rope_index # transformers >= 4.52.0
@@ -322,6 +324,8 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
)
def __call__(self, features: list[dict[str, Any]]) -> dict[str, "torch.Tensor"]:
model_type = getattr(getattr(self.model, "config", None), "model_type", None)
is_moss_vl = model_type == "moss_vl"
batch_images, batch_videos, batch_audios = [], [], []
batch_imglens, batch_vidlens, batch_audlens, batch_input_ids = [], [], [], []
packing_params_list: list[dict[str, Any] | None] = []
@@ -341,7 +345,10 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
fake_input_ids = []
has_dummy_image = False
if (
self.template.mm_plugin.image_token is not None and sum(batch_imglens) == 0 and sum(batch_vidlens) == 0
self.template.mm_plugin.image_token is not None
and sum(batch_imglens) == 0
and sum(batch_vidlens) == 0
and not is_moss_vl # MOSS-VL builds one native zero-valued dummy per text-only sample in its plugin.
): # avoid process hanging in zero3/fsdp case
fake_messages = [{"role": "user", "content": IMAGE_PLACEHOLDER}]
fake_images = [Image.new("RGB", (64, 64), (255, 255, 255))]
@@ -416,7 +423,6 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
features: dict[str, torch.Tensor] = super().__call__(features)
bsz, seq_len = features["input_ids"].shape[:2]
model_type = getattr(self.model.config, "model_type", None) if self.model is not None else None
is_omni = model_type in [
"qwen2_5_omni_thinker",
"qwen3_omni_moe_thinker",
@@ -461,12 +467,17 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
):
raise ValueError(f"{self.model.config.model_type} requires 3D position ids for mrope.")
if "cross_attention_mask" in mm_inputs: # for mllama inputs when pad_to_multiple_of is enabled
if (
"cross_attention_mask" in mm_inputs and mm_inputs["cross_attention_mask"].dtype != torch.bool
): # for mllama inputs when pad_to_multiple_of is enabled
cross_attention_mask = mm_inputs.pop("cross_attention_mask")
seq_len = features["input_ids"].size(1)
orig_len = cross_attention_mask.size(1)
mm_inputs["cross_attention_mask"] = F.pad(cross_attention_mask, (0, 0, 0, 0, 0, seq_len - orig_len))
if is_moss_vl:
mm_inputs = self.template.mm_plugin.post_process_mossvl_inputs(features, mm_inputs, self.processor)
features.update(mm_inputs)
if "image_bound" in features: # for minicpmv inputs

349
src/llamafactory/data/mm_plugin.py
View File

@@ -472,6 +472,354 @@ class BasePlugin(MMPluginMixin):
return self._get_mm_inputs(images, videos, audios, processor)
@dataclass
class MossVLPlugin(BasePlugin):
vision_bos_token: str = "<|vision_start|>"
vision_eos_token: str = "<|vision_end|>"
time_bos_token: str = "<|time_start|>"
time_eos_token: str = "<|time_end|>"
@staticmethod
def _split_pixel_values(
pixel_values: "torch.Tensor",
grid_thw: "torch.Tensor",
) -> list["torch.Tensor"]:
patch_counts = [int(grid.prod().item()) for grid in grid_thw]
return list(torch.split(pixel_values, patch_counts))
@staticmethod
def _create_cross_attention_mask(
input_ids: Union[list[list[int]], "torch.Tensor"],
grid_thw: "torch.Tensor",
media_nums_per_sample: list[int],
image_token_id: int,
attention_mask: Optional["torch.Tensor"] = None,
padding_side: Literal["left", "right"] = "right",
) -> "torch.Tensor":
r"""Create the native MOSS-VL frame-level causal cross-attention mask."""
if isinstance(input_ids, list):
max_text_len = max(len(token_ids) for token_ids in input_ids)
input_ids_tensor = torch.full((len(input_ids), max_text_len), -1, dtype=torch.long)
attention_mask_tensor = torch.zeros_like(input_ids_tensor, dtype=torch.bool)
for batch_index, token_ids in enumerate(input_ids):
seq_len = len(token_ids)
start = max_text_len - seq_len if padding_side == "left" else 0
input_ids_tensor[batch_index, start : start + seq_len] = torch.tensor(token_ids, dtype=torch.long)
attention_mask_tensor[batch_index, start : start + seq_len] = True
else:
input_ids_tensor = input_ids
attention_mask_tensor = (
torch.ones_like(input_ids_tensor, dtype=torch.bool)
if attention_mask is None
else attention_mask.bool()
)
total_frames_per_sample = []
media_index = 0
for num_media in media_nums_per_sample:
sample_grid = grid_thw[media_index : media_index + num_media]
total_frames_per_sample.append(int(sample_grid[:, 0].sum().item()))
media_index += num_media
max_num_frames = max(total_frames_per_sample)
frame_indices = torch.arange(max_num_frames, device=input_ids_tensor.device).view(1, 1, -1)
visible_mask = (input_ids_tensor == image_token_id).cumsum(dim=1).unsqueeze(-1) > frame_indices
visible_mask &= attention_mask_tensor.unsqueeze(-1)
valid_frames = frame_indices < torch.tensor(
total_frames_per_sample,
device=input_ids_tensor.device,
).view(-1, 1, 1)
visible_mask &= valid_frames
return (~visible_mask).unsqueeze(1)
def _get_video_inputs(
self,
videos: list["VideoInput"],
processor: "MMProcessor",
return_metadata: bool,
) -> dict[str, Any]:
video_kwargs = {"return_tensors": "pt", "return_metadata": return_metadata}
if getattr(processor, "video_fps", None) is not None:
video_kwargs["video_fps"] = processor.video_fps
if getattr(processor, "video_maxlen", None) is not None:
video_kwargs["max_frames"] = processor.video_maxlen
video_min_pixels = getattr(processor, "video_min_pixels", None)
video_max_pixels = getattr(processor, "video_max_pixels", None)
if video_min_pixels is not None and video_max_pixels is not None:
video_kwargs["size"] = {
"shortest_edge": video_min_pixels,
"longest_edge": video_max_pixels,
}
return dict(processor.video_processor(videos=videos, **video_kwargs))
def _get_media_order_from_ids(
self,
input_ids: list[int],
processor: "MMProcessor",
num_images: int,
num_videos: int,
expected_video_frames: Optional[list[int]] = None,
) -> list[str]:
media_order = []
video_frame_counts = []
in_video = False
current_video_frames = 0
for token_id in input_ids:
if token_id == processor.vision_start_token_id:
if in_video:
raise ValueError(
"MOSS-VL encountered nested video token blocks after tokenization. "
"Please increase `cutoff_len` if a video placeholder was truncated."
)
media_order.append("video")
in_video = True
current_video_frames = 0
elif token_id == processor.vision_end_token_id:
if not in_video:
raise ValueError(
"MOSS-VL encountered a video end token without a matching start token after tokenization. "
"Please increase `cutoff_len` if a video placeholder was truncated."
)
video_frame_counts.append(current_video_frames)
in_video = False
elif token_id == processor.image_token_id:
if in_video:
current_video_frames += 1
else:
media_order.append("image")
if in_video:
raise ValueError(
"MOSS-VL encountered an incomplete video token block after tokenization. "
"Please increase `cutoff_len` or reduce `video_maxlen`."
)
if media_order.count("image") != num_images or media_order.count("video") != num_videos:
raise ValueError(
"MOSS-VL media tokens do not match the provided media after tokenization: "
f"order={media_order}, images={num_images}, videos={num_videos}. "
"Please increase `cutoff_len` if a visual placeholder was truncated."
)
if expected_video_frames is not None and video_frame_counts != expected_video_frames:
raise ValueError(
"MOSS-VL video frame tokens do not match the processed video after tokenization: "
f"tokens={video_frame_counts}, frames={expected_video_frames}. "
"Please increase `cutoff_len` or reduce `video_maxlen`."
)
return media_order
@override
def process_messages(
self,
messages: list[dict[str, str]],
images: list["ImageInput"],
videos: list["VideoInput"],
audios: list["AudioInput"],
processor: Optional["MMProcessor"],
) -> list[dict[str, str]]:
self._validate_input(processor, images, videos, audios)
self._validate_messages(messages, images, videos, audios)
messages = deepcopy(messages)
video_inputs = self._get_video_inputs(videos, processor, return_metadata=True) if videos else {}
video_grid_thw = video_inputs.get("video_grid_thw", [])
video_metadata = video_inputs.get("video_metadata", [])
video_index = 0
for message in messages:
content = message["content"]
content = content.replace(IMAGE_PLACEHOLDER, self.image_token)
while VIDEO_PLACEHOLDER in content:
metadata = video_metadata[video_index]
if metadata.fps is None:
metadata.fps = 24
timestamps = processor._calculate_timestamps(
metadata.frames_indices,
metadata.total_num_frames,
metadata.fps,
metadata.duration,
processor.video_processor.temporal_patch_size,
actual_timestamps=getattr(metadata, "actual_timestamps", None),
)
num_frames = int(video_grid_thw[video_index][0].item())
frame_tokens = [
f"{self.time_bos_token}{timestamps[frame_idx]:.1f} seconds{self.time_eos_token}{self.image_token}"
for frame_idx in range(num_frames)
]
video_tokens = f"{self.vision_bos_token}{''.join(frame_tokens)}{self.vision_eos_token}"
content = content.replace(VIDEO_PLACEHOLDER, video_tokens, 1)
video_index += 1
message["content"] = content
return messages
@override
def get_mm_inputs(
self,
images: list["ImageInput"],
videos: list["VideoInput"],
audios: list["AudioInput"],
imglens: list[int],
vidlens: list[int],
audlens: list[int],
batch_ids: list[list[int]],
processor: Optional["MMProcessor"],
) -> dict[str, Union[list[int], "torch.Tensor"]]:
self._validate_input(processor, images, videos, audios)
if audios:
raise ValueError("MOSS-VL does not support audio inputs.")
if not (len(imglens) == len(vidlens) == len(batch_ids)):
raise ValueError("MOSS-VL batch metadata must have one entry per sample.")
final_pixel_values = []
final_grid_thw = []
media_nums_per_sample = []
image_offset = 0
video_offset = 0
for imglen, vidlen, input_ids in zip(imglens, vidlens, batch_ids):
sample_images = images[image_offset : image_offset + imglen]
sample_videos = videos[video_offset : video_offset + vidlen]
image_offset += imglen
video_offset += vidlen
image_chunks, image_grids = [], []
if sample_images:
regularized_images = self._regularize_images(
sample_images,
image_max_pixels=2**63 - 1,
image_min_pixels=1,
)["images"]
image_kwargs = {"return_tensors": "pt"}
if getattr(processor, "image_min_pixels", None) is not None:
image_kwargs["min_pixels"] = processor.image_min_pixels
if getattr(processor, "image_max_pixels", None) is not None:
image_kwargs["max_pixels"] = processor.image_max_pixels
image_inputs = processor.image_processor(images=regularized_images, **image_kwargs)
image_grids = list(image_inputs["image_grid_thw"])
image_chunks = self._split_pixel_values(image_inputs["pixel_values"], image_inputs["image_grid_thw"])
video_chunks, video_grids = [], []
if sample_videos:
video_inputs = self._get_video_inputs(sample_videos, processor, return_metadata=False)
video_grids = list(video_inputs["video_grid_thw"])
video_chunks = self._split_pixel_values(
video_inputs["pixel_values_videos"],
video_inputs["video_grid_thw"],
)
media_order = self._get_media_order_from_ids(
input_ids,
processor,
imglen,
vidlen,
expected_video_frames=[int(grid[0].item()) for grid in video_grids],
)
if not media_order:
patch_size = getattr(processor.image_processor, "patch_size", None)
if patch_size is None: # lightweight/test processors without the native MOSS-VL contract
blank_image = Image.new("RGB", (128, 128), (255, 255, 255))
blank_inputs = processor.image_processor(images=[blank_image], return_tensors="pt")
final_pixel_values.append(blank_inputs["pixel_values"])
final_grid_thw.append(blank_inputs["image_grid_thw"][0])
else:
temporal_patch_size = getattr(processor.image_processor, "temporal_patch_size", None) or 1
merge_size = getattr(processor.image_processor, "merge_size", None) or 2
factor = patch_size * merge_size
side = math.ceil(128 / factor) * factor
grid_thw = torch.tensor([1, side // patch_size, side // patch_size])
feature_dim = 3 * temporal_patch_size * patch_size * patch_size
final_pixel_values.append(torch.zeros((int(grid_thw.prod()), feature_dim), dtype=torch.float32))
final_grid_thw.append(grid_thw)
media_nums_per_sample.append(1)
continue
image_index = 0
video_index = 0
for modality in media_order:
if modality == "image":
final_pixel_values.append(image_chunks[image_index])
final_grid_thw.append(image_grids[image_index])
image_index += 1
else:
final_pixel_values.append(video_chunks[video_index])
final_grid_thw.append(video_grids[video_index])
video_index += 1
media_nums_per_sample.append(len(media_order))
if image_offset != len(images) or video_offset != len(videos):
raise ValueError("MOSS-VL media lengths do not consume all provided inputs.")
mm_inputs = {
"pixel_values": torch.cat(final_pixel_values, dim=0),
"grid_thw": torch.stack(final_grid_thw),
"media_nums_per_sample": media_nums_per_sample,
}
mm_inputs["cross_attention_mask"] = self._create_cross_attention_mask(
batch_ids,
mm_inputs["grid_thw"],
media_nums_per_sample,
processor.image_token_id,
padding_side=processor.tokenizer.padding_side,
)
return mm_inputs
def post_process_mossvl_inputs(
self,
features: dict[str, "torch.Tensor"],
mm_inputs: dict[str, Any],
processor: "MMProcessor",
) -> dict[str, Any]:
r"""Create MOSS-VL batch-only inputs after the text batch has been padded."""
input_ids = features["input_ids"]
attention_mask = features["attention_mask"].bool()
mm_inputs["cross_attention_mask"] = self._create_cross_attention_mask(
input_ids,
mm_inputs["grid_thw"],
mm_inputs["media_nums_per_sample"],
processor.image_token_id,
attention_mask,
)
dummy_image_tokens = (input_ids == processor.image_token_id) & ~attention_mask
input_ids.masked_fill_(dummy_image_tokens, processor.tokenizer.pad_token_id)
labels = features.get("labels")
if labels is not None:
control_token_ids = {
processor.image_token_id,
processor.video_token_id,
processor.vision_start_token_id,
processor.vision_end_token_id,
processor.tokenizer.convert_tokens_to_ids(self.time_bos_token),
processor.tokenizer.convert_tokens_to_ids(self.time_eos_token),
}
for batch_index, token_ids in enumerate(input_ids):
in_vision = False
for token_index, token_id in enumerate(token_ids.tolist()):
if token_id == processor.vision_start_token_id:
in_vision = True
if in_vision or token_id in control_token_ids:
labels[batch_index, token_index] = IGNORE_INDEX
if token_id == processor.vision_end_token_id:
in_vision = False
# Native MOSS-VL labels_spans supervise through <|im_end|>, but not its trailing newline.
im_end_token_id = processor.tokenizer.convert_tokens_to_ids("<|im_end|>")
labels[:, 1:].masked_fill_(input_ids[:, :-1] == im_end_token_id, IGNORE_INDEX)
features.pop("position_ids", None)
return mm_inputs
@dataclass
class ErnieVLPlugin(BasePlugin):
@override
@@ -2911,6 +3259,7 @@ PLUGINS = {
"minicpm_v": MiniCPMVPlugin,
"minicpm_v_4_6": MiniCPMV4_6Plugin,
"mllama": MllamaPlugin,
"moss_vl": MossVLPlugin,
"paligemma": PaliGemmaPlugin,
"pixtral": PixtralPlugin,
"qwen2_audio": Qwen2AudioPlugin,

70
src/llamafactory/data/template.py
View File

@@ -333,6 +333,50 @@ class Template:
return modelfile
@dataclass
class MossVLTemplate(Template):
@override
def _encode(
self,
tokenizer: "PreTrainedTokenizer",
messages: list[dict[str, str]],
system: Optional[str],
tools: Optional[str],
) -> list[list[int]]:
system = system or self.default_system
encoded_messages = []
for i, message in enumerate(messages):
elements = []
if i == 0:
elements += self.format_prefix.apply()
if system or tools:
tool_text = self.format_tools.apply(content=tools)[0] if tools else ""
if tools and not system:
tool_text = tool_text.lstrip("\n")
elements += self.format_system.apply(content=(system + tool_text))
if message["role"] == Role.USER:
elements += self.format_user.apply(content=message["content"], idx=str(i // 2))
elif message["role"] == Role.ASSISTANT:
elements += self.format_assistant.apply(content=message["content"])
elif message["role"] == Role.OBSERVATION:
elements += self.format_observation.apply(content=message["content"])
elif message["role"] == Role.FUNCTION:
elements += self.format_function.apply(
content=message["content"],
thought_words=self.thought_words,
tool_call_words=self.tool_call_words,
)
else:
raise NotImplementedError("Unexpected role: {}".format(message["role"]))
encoded_messages.append(self._convert_elements_to_ids(tokenizer, elements))
return encoded_messages
@dataclass
class Llama2Template(Template):
r"""A template that fuse the system message to first user message."""
@@ -1526,6 +1570,32 @@ register_template(
)
# copied from qwen template
register_template(
name="moss_vl",
format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen"),
format_observation=StringFormatter(
slots=["<|im_start|>user\n<tool_response>\n{{content}}\n</tool_response><|im_end|>\n<|im_start|>assistant\n"]
),
format_tools=ToolFormatter(tool_format="qwen"),
stop_words=["<|im_end|>"],
replace_eos=True,
mm_plugin=get_mm_plugin(
name="moss_vl",
image_token="<|image_pad|>",
video_token="<|video_pad|>",
vision_bos_token="<|vision_start|>",
vision_eos_token="<|vision_end|>",
time_bos_token="<|time_start|>",
time_eos_token="<|time_end|>",
),
template_class=MossVLTemplate,
)
# copied from vicuna template
register_template(
name="llava",

11
src/llamafactory/extras/constants.py
View File

@@ -2198,6 +2198,17 @@ register_model_group(
)
register_model_group(
models={
"MOSS-VL-Instruct-0708": {
DownloadSource.DEFAULT: "OpenMOSS-Team/MOSS-VL-Instruct-0708",
},
},
template="moss_vl",
multimodal=True,
)
register_model_group(
models={
"OLMo-1B": {

11
src/llamafactory/model/model_utils/visual.py
View File

@@ -56,7 +56,7 @@ class CompositeModel:
)
break
if project_module is not None:
if isinstance(project_module, torch.nn.Module):
mm_projectors.append(project_module)
return mm_projectors
@@ -344,6 +344,15 @@ _register_composite_model(
)
_register_composite_model(
model_type="moss_vl",
projector_keys=["model.visual.merger", "model.separator_token"],
vision_model_keys=["model.visual.pos_embed", "model.visual.patch_embed", "model.visual.blocks"],
language_model_keys=["model.language_model", "lm_head"],
lora_conflict_keys=["patch_embed"],
)
_register_composite_model(
model_type="mllama",
vision_model_keys=["vision_model"],

6
src/llamafactory/model/patcher.py
View File

@@ -23,7 +23,7 @@ from transformers.modeling_utils import is_fsdp_enabled
from transformers.utils import is_torch_cuda_available, is_torch_npu_available
from ..extras import logging
from ..extras.misc import infer_optim_dtype
from ..extras.misc import check_version, infer_optim_dtype
from ..extras.packages import is_transformers_version_greater_than
from .model_utils.attention import configure_attn_implementation, print_attn_implementation
from .model_utils.checkpointing import prepare_model_for_training
@@ -418,6 +418,10 @@ def patch_config(
"pip install git+https://github.com/huggingface/transformers.git@3c2517727ce28a30f5044e01663ee204deb1cdbe"
)
if getattr(config, "model_type", None) == "moss_vl":
check_version("transformers==4.57.1", mandatory=True)
check_version("torchcodec==0.7.0", mandatory=True)
if getattr(config, "model_type", None) == "qwen3_omni_moe":
patch_qwen3_omni_moe_thinker_text_sparse_moe_block()

19
tests/data/test_mm_plugin.py
View File

@@ -13,6 +13,7 @@
# limitations under the License.
import os
from types import SimpleNamespace
from typing import TYPE_CHECKING, Any
import numpy as np
@@ -417,6 +418,24 @@ def test_qwen2_vl_plugin():
_check_plugin(**check_inputs)
def test_moss_vl_plugin():
messages = [
{"role": "user", "content": "First <image>, finally <image>."},
{"role": "assistant", "content": "Done."},
]
expected_messages = [
{"role": "user", "content": "First <|image_pad|>, finally <|image_pad|>."},
{"role": "assistant", "content": "Done."},
]
processor = SimpleNamespace(image_processor=object(), video_processor=object())
plugin = get_mm_plugin(name="moss_vl", image_token="<|image_pad|>", video_token="<|video_pad|>")
processed_messages = plugin.process_messages(messages, [object(), object()], [], [], processor)
assert processed_messages == expected_messages
assert messages[0]["content"] == "First <image>, finally <image>."
@pytest.mark.runs_on(["cpu", "mps"])
@pytest.mark.skipif(not is_transformers_version_greater_than("4.57.0"), reason="Requires transformers>=4.57.0")
def test_qwen3_vl_plugin():

631
tests/data/test_moss_vl_plugin.py Normal file
View File

@@ -0,0 +1,631 @@
# Copyright 2025 the LlamaFactory team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
from llamafactory.data.collator import MultiModalDataCollatorForSeq2Seq
from llamafactory.data.mm_plugin import get_mm_plugin
from llamafactory.data.processor.supervised import SupervisedDatasetProcessor
from llamafactory.extras.constants import IGNORE_INDEX
IMAGE_TOKEN_ID = 101
VIDEO_TOKEN_ID = 102
VISION_START_TOKEN_ID = 103
VISION_END_TOKEN_ID = 104
TIME_START_TOKEN_ID = 105
TIME_END_TOKEN_ID = 106
IM_END_TOKEN_ID = 107
class _ImageProcessor:
def __init__(self):
self.calls = []
def __call__(self, images, return_tensors, **kwargs):
self.calls.append({"return_tensors": return_tensors, **kwargs})
values = []
for image in images:
marker = image.getpixel((0, 0))[0] + 1
values.append(torch.full((1, 3), marker, dtype=torch.float32))
return {
"pixel_values": torch.cat(values),
"image_grid_thw": torch.tensor([[1, 1, 1]] * len(images)),
}
class _VideoProcessor:
temporal_patch_size = 1
def __init__(self):
self.calls = []
def __call__(self, videos, return_tensors, return_metadata, **kwargs):
self.calls.append(
{
"return_tensors": return_tensors,
"return_metadata": return_metadata,
**kwargs,
}
)
result = {
"pixel_values_videos": torch.cat(
[torch.full((2, 3), 9 + index, dtype=torch.float32) for index in range(len(videos))]
),
"video_grid_thw": torch.tensor([[2, 1, 1]] * len(videos)),
}
if return_metadata:
result["video_metadata"] = [
SimpleNamespace(frames_indices=[0, 2], total_num_frames=2, fps=2.0, duration=2.0) for _ in videos
]
return result
class _Tokenizer:
pad_token_id = 0
padding_side = "right"
_token_ids = {
"<|time_start|>": TIME_START_TOKEN_ID,
"<|time_end|>": TIME_END_TOKEN_ID,
"<|im_end|>": IM_END_TOKEN_ID,
}
def convert_tokens_to_ids(self, token):
return self._token_ids[token]
def pad(self, features, padding, max_length, pad_to_multiple_of, return_tensors):
del padding, max_length, return_tensors
sequence_length = max(len(feature["input_ids"]) for feature in features)
if pad_to_multiple_of is not None:
sequence_length = ((sequence_length + pad_to_multiple_of - 1) // pad_to_multiple_of) * pad_to_multiple_of
padded = {"input_ids": [], "attention_mask": []}
for feature in features:
pad_length = sequence_length - len(feature["input_ids"])
if self.padding_side == "right":
padded["input_ids"].append(feature["input_ids"] + [self.pad_token_id] * pad_length)
padded["attention_mask"].append(feature["attention_mask"] + [0] * pad_length)
else:
padded["input_ids"].append([self.pad_token_id] * pad_length + feature["input_ids"])
padded["attention_mask"].append([0] * pad_length + feature["attention_mask"])
return {key: torch.tensor(value) for key, value in padded.items()}
class _Processor:
image_token_id = IMAGE_TOKEN_ID
video_token_id = VIDEO_TOKEN_ID
vision_start_token_id = VISION_START_TOKEN_ID
vision_end_token_id = VISION_END_TOKEN_ID
def __init__(self):
self.image_processor = _ImageProcessor()
self.video_processor = _VideoProcessor()
self.tokenizer = _Tokenizer()
@staticmethod
def _calculate_timestamps(*args, **kwargs):
del args, kwargs
return [0.0, 1.0]
def _get_plugin():
return get_mm_plugin(
name="moss_vl",
image_token="<|image_pad|>",
video_token="<|video_pad|>",
vision_bos_token="<|vision_start|>",
vision_eos_token="<|vision_end|>",
time_bos_token="<|time_start|>",
time_eos_token="<|time_end|>",
)
def _video_ids(seed):
return [
VISION_START_TOKEN_ID,
TIME_START_TOKEN_ID,
seed,
TIME_END_TOKEN_ID,
IMAGE_TOKEN_ID,
TIME_START_TOKEN_ID,
seed + 1,
TIME_END_TOKEN_ID,
IMAGE_TOKEN_ID,
VISION_END_TOKEN_ID,
]
def _left_pad(sequences, pad_value):
max_len = max(map(len, sequences))
return torch.tensor([[pad_value] * (max_len - len(sequence)) + sequence for sequence in sequences])
def test_moss_vl_process_messages_expands_video_frames():
plugin = _get_plugin()
processor = _Processor()
messages = [
{"role": "user", "content": "First <image>, then <video>, finally <image>."},
{"role": "assistant", "content": "Done."},
]
images = [Image.new("RGB", (2, 2)), Image.new("RGB", (2, 2), (2, 0, 0))]
processed = plugin.process_messages(messages, images, ["video.mp4"], [], processor)
video_tokens = (
"<|vision_start|>"
"<|time_start|>0.0 seconds<|time_end|><|image_pad|>"
"<|time_start|>1.0 seconds<|time_end|><|image_pad|>"
"<|vision_end|>"
)
assert processed[0]["content"] == (f"First <|image_pad|>, then {video_tokens}, finally <|image_pad|>.")
assert messages[0]["content"] == "First <image>, then <video>, finally <image>."
@pytest.mark.parametrize(
("content", "images", "videos", "error"),
[
("Missing media: <image>.", [], [], "number of images does not match"),
("Missing media: <video>.", [], [], "number of videos does not match"),
],
)
def test_moss_vl_rejects_placeholder_count_mismatch(content, images, videos, error):
plugin = _get_plugin()
with pytest.raises(ValueError, match=error):
plugin.process_messages(
[{"role": "user", "content": content}],
images,
videos,
[],
_Processor(),
)
def test_moss_vl_process_messages_expands_multiple_videos_in_order():
plugin = _get_plugin()
messages = [{"role": "user", "content": "Compare <video> with <video>."}]
processed = plugin.process_messages(messages, [], ["first.mp4", "second.mp4"], [], _Processor())
frame_tokens = (
"<|vision_start|>"
"<|time_start|>0.0 seconds<|time_end|><|image_pad|>"
"<|time_start|>1.0 seconds<|time_end|><|image_pad|>"
"<|vision_end|>"
)
assert processed[0]["content"] == f"Compare {frame_tokens} with {frame_tokens}."
def test_moss_vl_forwards_spatial_pixel_limits_to_native_processors():
plugin = _get_plugin()
processor = _Processor()
processor.image_min_pixels = 1024
processor.image_max_pixels = 262144
processor.video_min_pixels = 256
processor.video_max_pixels = 16384
processor.video_fps = 1.0
processor.video_maxlen = 8
image = Image.new("RGB", (1024, 1024))
plugin.process_messages(
[{"role": "user", "content": "Compare <image> and <video>."}],
[image],
["video.mp4"],
[],
processor,
)
plugin.get_mm_inputs(
[image],
["video.mp4"],
[],
[1],
[1],
[0],
[[IMAGE_TOKEN_ID, *_video_ids(201)]],
processor,
)
assert processor.image_processor.calls == [
{
"return_tensors": "pt",
"min_pixels": 1024,
"max_pixels": 262144,
}
]
assert processor.video_processor.calls == [
{
"return_tensors": "pt",
"return_metadata": True,
"video_fps": 1.0,
"max_frames": 8,
"size": {"shortest_edge": 256, "longest_edge": 16384},
},
{
"return_tensors": "pt",
"return_metadata": False,
"video_fps": 1.0,
"max_frames": 8,
"size": {"shortest_edge": 256, "longest_edge": 16384},
},
]
def test_moss_vl_rejects_invalid_batch_metadata():
plugin = _get_plugin()
processor = _Processor()
image = Image.new("RGB", (2, 2))
with pytest.raises(ValueError, match="batch metadata must have one entry per sample"):
plugin.get_mm_inputs([image], [], [], [1], [], [0], [[IMAGE_TOKEN_ID]], processor)
with pytest.raises(ValueError, match="media lengths do not consume all provided inputs"):
plugin.get_mm_inputs([image], [], [], [0], [0], [0], [[201]], processor)
def test_moss_vl_rejects_truncated_media_tokens():
plugin = _get_plugin()
with pytest.raises(ValueError, match="increase `cutoff_len`"):
plugin.get_mm_inputs(
[Image.new("RGB", (2, 2))],
[],
[],
[1],
[0],
[0],
[[201, 202]],
_Processor(),
)
def test_moss_vl_rejects_incomplete_video_token_block():
plugin = _get_plugin()
truncated_video_ids = _video_ids(201)[:-1]
with pytest.raises(ValueError, match="incomplete video token block"):
plugin.get_mm_inputs(
[],
["video.mp4"],
[],
[0],
[1],
[0],
[truncated_video_ids],
_Processor(),
)
def test_moss_vl_rejects_video_frame_token_count_mismatch():
plugin = _get_plugin()
incomplete_frame_ids = [VISION_START_TOKEN_ID, IMAGE_TOKEN_ID, VISION_END_TOKEN_ID]
with pytest.raises(ValueError, match="video frame tokens do not match"):
plugin.get_mm_inputs(
[],
["video.mp4"],
[],
[0],
[1],
[0],
[incomplete_frame_ids],
_Processor(),
)
def test_moss_vl_media_order_batch_mask_and_labels():
plugin = _get_plugin()
processor = _Processor()
images = [Image.new("RGB", (2, 2)), Image.new("RGB", (2, 2), (2, 0, 0))]
first_ids = [
IMAGE_TOKEN_ID,
201,
VISION_START_TOKEN_ID,
TIME_START_TOKEN_ID,
202,
TIME_END_TOKEN_ID,
IMAGE_TOKEN_ID,
TIME_START_TOKEN_ID,
203,
TIME_END_TOKEN_ID,
IMAGE_TOKEN_ID,
VISION_END_TOKEN_ID,
IMAGE_TOKEN_ID,
204,
]
second_ids = [301, 302]
assert plugin._get_media_order_from_ids(first_ids, processor, 2, 1) == ["image", "video", "image"]
mm_inputs = plugin.get_mm_inputs(
images=images,
videos=["video.mp4"],
audios=[],
imglens=[2, 0],
vidlens=[1, 0],
audlens=[0, 0],
batch_ids=[first_ids, second_ids],
processor=processor,
)
assert mm_inputs["grid_thw"].tolist() == [[1, 1, 1], [2, 1, 1], [1, 1, 1], [1, 1, 1]]
assert mm_inputs["media_nums_per_sample"] == [3, 1]
assert mm_inputs["pixel_values"][:, 0].tolist() == [1.0, 9.0, 9.0, 3.0, 256.0]
pre_padding_mask = mm_inputs["cross_attention_mask"]
assert pre_padding_mask.shape == (2, 1, len(first_ids), 4)
assert pre_padding_mask[0, 0, 0].tolist() == [False, True, True, True]
assert pre_padding_mask[0, 0, 10].tolist() == [False, False, False, True]
assert pre_padding_mask[0, 0, 12].tolist() == [False, False, False, False]
assert pre_padding_mask[1].all()
seq_len = len(first_ids)
input_ids = torch.tensor([first_ids, [0] * (seq_len - 2) + second_ids])
attention_mask = torch.tensor([[1] * seq_len, [0] * (seq_len - 2) + [1, 1]])
labels = input_ids.clone()
labels[attention_mask == 0] = IGNORE_INDEX
features = {
"input_ids": input_ids,
"attention_mask": attention_mask,
"labels": labels,
"position_ids": torch.arange(seq_len).repeat(2, 1),
}
mm_inputs = plugin.post_process_mossvl_inputs(features, mm_inputs, processor)
mask = mm_inputs["cross_attention_mask"]
assert mask.shape == (2, 1, seq_len, 4)
assert mask[0, 0, 0].tolist() == [False, True, True, True]
assert mask[0, 0, 10].tolist() == [False, False, False, True]
assert mask[0, 0, 12].tolist() == [False, False, False, False]
assert mask[1].all()
assert "position_ids" not in features
assert not torch.any((features["input_ids"] == IMAGE_TOKEN_ID) & ~features["attention_mask"].bool())
assert features["labels"][0, 1].item() == 201
assert features["labels"][0, 13].item() == 204
assert features["labels"][0, 0].item() == IGNORE_INDEX
assert features["labels"][0, 12].item() == IGNORE_INDEX
assert torch.all(features["labels"][0, 2:12] == IGNORE_INDEX)
def test_moss_vl_complex_batch_keeps_media_and_masks_sample_local():
plugin = _get_plugin()
processor = _Processor()
batch_ids = [
[IMAGE_TOKEN_ID, 211, IMAGE_TOKEN_ID, 212],
[221, *_video_ids(222), 223, *_video_ids(224), 225],
[IMAGE_TOKEN_ID, 231, *_video_ids(232), 233, IMAGE_TOKEN_ID, 234],
[241, 242, 243],
]
images = [Image.new("RGB", (2, 2), (marker, 0, 0)) for marker in range(4)]
mm_inputs = plugin.get_mm_inputs(
images=images,
videos=["first.mp4", "second.mp4", "third.mp4"],
audios=[],
imglens=[2, 0, 2, 0],
vidlens=[0, 2, 1, 0],
audlens=[0, 0, 0, 0],
batch_ids=batch_ids,
processor=processor,
)
assert mm_inputs["grid_thw"].tolist() == [
[1, 1, 1],
[1, 1, 1],
[2, 1, 1],
[2, 1, 1],
[1, 1, 1],
[2, 1, 1],
[1, 1, 1],
[1, 1, 1],
]
assert mm_inputs["media_nums_per_sample"] == [2, 2, 3, 1]
assert mm_inputs["pixel_values"][:, 0].tolist() == [
1.0,
2.0,
9.0,
9.0,
10.0,
10.0,
3.0,
9.0,
9.0,
4.0,
256.0,
]
input_ids = _left_pad(batch_ids, 0)
attention_mask = _left_pad([[1] * len(ids) for ids in batch_ids], 0)
labels = input_ids.clone()
labels[attention_mask == 0] = IGNORE_INDEX
features = {
"input_ids": input_ids,
"attention_mask": attention_mask,
"labels": labels,
"position_ids": torch.arange(input_ids.shape[1]).repeat(len(batch_ids), 1),
}
plugin.post_process_mossvl_inputs(features, mm_inputs, processor)
cross_mask = mm_inputs["cross_attention_mask"]
assert cross_mask.shape == (4, 1, input_ids.shape[1], 4)
assert (~cross_mask[0]).sum().item() > 0
assert (~cross_mask[1]).sum().item() > 0
assert (~cross_mask[2]).sum().item() > 0
assert cross_mask[3].all()
assert cross_mask[0, ..., 2:].all()
assert not cross_mask[1, ..., :4].all()
assert not cross_mask[2, ..., :4].all()
assert features["labels"][3, -3:].tolist() == [241, 242, 243]
assert torch.all(features["labels"][features["attention_mask"] == 0] == IGNORE_INDEX)
assert "position_ids" not in features
def test_moss_vl_supervised_processor_to_collator_mixed_batch(monkeypatch):
plugin = _get_plugin()
processor = _Processor()
tokenizer = processor.tokenizer
template = SimpleNamespace(mm_plugin=plugin)
dataset_processor = SupervisedDatasetProcessor(
template=template,
tokenizer=tokenizer,
processor=processor,
data_args=SimpleNamespace(),
)
first_ids = [IMAGE_TOKEN_ID, 211, *_video_ids(212), IMAGE_TOKEN_ID, 214]
second_ids = [221, 222]
def encode_example(prompt, **kwargs):
del kwargs
input_ids = first_ids if "<image>" in prompt[0]["content"] else second_ids
return input_ids, input_ids.copy()
monkeypatch.setattr(dataset_processor, "_encode_data_example", encode_example)
examples = {
"_prompt": [
[{"role": "user", "content": "Compare <image>, <video>, and <image>."}],
[{"role": "user", "content": "Text-only question."}],
],
"_response": [
[{"role": "assistant", "content": "Mixed answer."}],
[{"role": "assistant", "content": "Text answer."}],
],
"_system": ["", ""],
"_tools": ["", ""],
"_images": [
[Image.new("RGB", (2, 2)), Image.new("RGB", (2, 2), (2, 0, 0))],
None,
],
"_videos": [["video.mp4"], None],
"_audios": [None, None],
}
model_inputs = dataset_processor.preprocess_dataset(examples)
assert "media_order" not in model_inputs
collator = MultiModalDataCollatorForSeq2Seq(
tokenizer=tokenizer,
model=SimpleNamespace(config=SimpleNamespace(model_type="moss_vl")),
template=template,
processor=processor,
label_pad_token_id=IGNORE_INDEX,
)
features = [
{key: values[index] for key, values in model_inputs.items()} for index in range(len(model_inputs["input_ids"]))
]
batch = collator(features)
assert batch["grid_thw"].tolist() == [[1, 1, 1], [2, 1, 1], [1, 1, 1], [1, 1, 1]]
assert batch["media_nums_per_sample"] == [3, 1]
assert batch["pixel_values"][:, 0].tolist() == [1.0, 9.0, 9.0, 3.0, 256.0]
assert batch["cross_attention_mask"].shape == (2, 1, len(first_ids), 4)
assert batch["cross_attention_mask"][1].all()
assert torch.all(batch["labels"][1, len(second_ids) :] == IGNORE_INDEX)
assert "position_ids" not in batch
def test_moss_vl_generate_collator_keeps_left_padded_cross_attention_mask():
plugin = _get_plugin()
processor = _Processor()
processor.tokenizer.padding_side = "left"
template = SimpleNamespace(mm_plugin=plugin)
batch_ids = [
[IMAGE_TOKEN_ID, 211],
[301, IMAGE_TOKEN_ID, 302, 303],
]
features = [
{
"input_ids": input_ids,
"attention_mask": [1] * len(input_ids),
"labels": input_ids.copy(),
"images": [Image.new("RGB", (2, 2))],
}
for input_ids in batch_ids
]
collator = MultiModalDataCollatorForSeq2Seq(
tokenizer=processor.tokenizer,
model=SimpleNamespace(config=SimpleNamespace(model_type="moss_vl")),
template=template,
processor=processor,
label_pad_token_id=IGNORE_INDEX,
pad_to_multiple_of=8,
)
batch = collator(features)
assert batch["cross_attention_mask"].shape == (2, 1, 8, 1)
assert batch["cross_attention_mask"][0, 0, :, 0].tolist() == [True] * 6 + [False, False]
assert batch["cross_attention_mask"][1, 0, :, 0].tolist() == [True] * 5 + [False, False, False]
def test_moss_vl_predict_collator_uses_precomputed_cross_attention_mask_without_model():
plugin = _get_plugin()
processor = _Processor()
template = SimpleNamespace(mm_plugin=plugin)
batch_ids = [
[IMAGE_TOKEN_ID, 211],
[301, IMAGE_TOKEN_ID, 302, 303],
]
features = [
{
"input_ids": input_ids,
"attention_mask": [1] * len(input_ids),
"labels": input_ids.copy(),
"images": [Image.new("RGB", (2, 2))],
}
for input_ids in batch_ids
]
collator = MultiModalDataCollatorForSeq2Seq(
tokenizer=processor.tokenizer,
model=None,
template=template,
processor=processor,
label_pad_token_id=IGNORE_INDEX,
)
batch = collator(features)
assert batch["cross_attention_mask"].shape == (2, 1, 4, 1)
assert batch["cross_attention_mask"][0, 0, :, 0].tolist() == [False, False, True, True]
assert batch["cross_attention_mask"][1, 0, :, 0].tolist() == [True, False, False, False]
def test_moss_vl_masks_only_the_token_after_im_end():
plugin = _get_plugin()
processor = _Processor()
input_ids = torch.tensor([[301, IM_END_TOKEN_ID, 302, 303]])
features = {
"input_ids": input_ids,
"attention_mask": torch.ones_like(input_ids),
"labels": input_ids.clone(),
}
mm_inputs = plugin.get_mm_inputs([], [], [], [0], [0], [0], [input_ids[0].tolist()], processor)
plugin.post_process_mossvl_inputs(features, mm_inputs, processor)
assert features["labels"].tolist() == [[301, IM_END_TOKEN_ID, IGNORE_INDEX, 303]]
def test_moss_vl_native_text_dummy_shape_and_values():
plugin = _get_plugin()
processor = _Processor()
processor.image_processor = SimpleNamespace(patch_size=16, temporal_patch_size=1, merge_size=2)
mm_inputs = plugin.get_mm_inputs([], [], [], [0], [0], [0], [[301]], processor)
assert mm_inputs["grid_thw"].tolist() == [[1, 8, 8]]
assert mm_inputs["pixel_values"].shape == (64, 768)
assert torch.count_nonzero(mm_inputs["pixel_values"]).item() == 0
assert mm_inputs["media_nums_per_sample"] == [1]

39
tests/data/test_moss_vl_training_configs.py Normal file
View File

@@ -0,0 +1,39 @@
# Copyright 2025 the LlamaFactory team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from pathlib import Path
import yaml
ROOT = Path(__file__).resolve().parents[2]
def test_moss_vl_training_configs_are_unpacked_and_additive():
lora = yaml.safe_load((ROOT / "examples/train_lora/mossvl_lora_sft.yaml").read_text())
full = yaml.safe_load((ROOT / "examples/train_full/mossvl_full_sft.yaml").read_text())
assert lora["template"] == full["template"] == "moss_vl"
assert lora["packing"] is full["packing"] is False
assert lora["per_device_train_batch_size"] == 2
assert full["per_device_train_batch_size"] == 1
assert full["use_reentrant_gc"] is False
assert full["gradient_checkpointing"] is True
assert full["gradient_checkpointing_kwargs"] == {"use_reentrant": False}
for config in (lora, full):
assert config["model_name_or_path"] == "OpenMOSS-Team/MOSS-VL-Instruct-0708"
assert not any("/inspire/" in str(value) or "/tmp/" in str(value) for value in config.values())
assert config["freeze_vision_tower"] is True
assert config["freeze_multi_modal_projector"] is True
assert config["freeze_language_model"] is False

24
tests/data/test_template.py
View File

@@ -19,7 +19,13 @@ import pytest
from transformers import AutoTokenizer
from llamafactory.data import get_template_and_fix_tokenizer
from llamafactory.data.template import parse_template
from llamafactory.data.template import TEMPLATES, parse_template
from llamafactory.extras.constants import (
DEFAULT_TEMPLATE,
MULTIMODAL_SUPPORTED_MODELS,
SUPPORTED_MODELS,
DownloadSource,
)
from llamafactory.extras.packages import is_transformers_version_greater_than
from llamafactory.hparams import DataArguments
@@ -91,6 +97,22 @@ def _check_template(
_check_tokenization(tokenizer, (prompt_ids, answer_ids), (prompt_str, answer_str))
def test_moss_vl_registration():
model_name = "MOSS-VL-Instruct-0708"
assert model_name in SUPPORTED_MODELS
assert SUPPORTED_MODELS[model_name][DownloadSource.DEFAULT] == "OpenMOSS-Team/MOSS-VL-Instruct-0708"
assert DEFAULT_TEMPLATE[model_name] == "moss_vl"
assert model_name in MULTIMODAL_SUPPORTED_MODELS
assert TEMPLATES["moss_vl"].mm_plugin.__class__.__name__ == "MossVLPlugin"
assert TEMPLATES["moss_vl"].mm_plugin.image_token == "<|image_pad|>"
assert TEMPLATES["moss_vl"].mm_plugin.video_token == "<|video_pad|>"
assert TEMPLATES["moss_vl"].mm_plugin.vision_bos_token == "<|vision_start|>"
assert TEMPLATES["moss_vl"].mm_plugin.vision_eos_token == "<|vision_end|>"
assert TEMPLATES["moss_vl"].mm_plugin.time_bos_token == "<|time_start|>"
assert TEMPLATES["moss_vl"].mm_plugin.time_eos_token == "<|time_end|>"
@pytest.mark.runs_on(["cpu", "mps"])
def test_encode_oneturn():
tokenizer = AutoTokenizer.from_pretrained(TINY_LLAMA3)

127
tests/model/model_utils/test_visual.py
View File

@@ -13,6 +13,7 @@
# limitations under the License.
import os
from types import SimpleNamespace
import pytest
import torch
@@ -21,7 +22,131 @@ from transformers import AutoConfig, AutoModelForImageTextToText
from llamafactory.extras.packages import is_transformers_version_greater_than
from llamafactory.hparams import FinetuningArguments, ModelArguments
from llamafactory.model.adapter import init_adapter
from llamafactory.model.adapter import _setup_freeze_tuning, _setup_full_tuning, init_adapter
from llamafactory.model.model_utils.misc import find_all_linear_modules
from llamafactory.model.model_utils.visual import COMPOSITE_MODELS, autocast_projector_dtype, patch_target_modules
class _MossVLFixture(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
self.config = SimpleNamespace(
model_type="moss_vl",
text_config=SimpleNamespace(num_hidden_layers=2),
)
self.model = torch.nn.Module()
self.model.separator_token = torch.nn.Parameter(torch.empty(4))
self.model.visual = torch.nn.Module()
self.model.visual.pos_embed = torch.nn.Embedding(4, 4)
self.model.visual.patch_embed = torch.nn.Module()
self.model.visual.patch_embed.proj = torch.nn.Linear(4, 4)
self.model.visual.blocks = torch.nn.ModuleList([self._make_block(), self._make_block()])
self.model.visual.merger = torch.nn.Module()
self.model.visual.merger.linear_fc1 = torch.nn.Linear(4, 4)
self.model.language_model = torch.nn.Module()
self.model.language_model.layers = torch.nn.ModuleList([self._make_layer(), self._make_layer()])
self.lm_head = torch.nn.Linear(4, 4)
@staticmethod
def _make_block() -> torch.nn.Module:
block = torch.nn.Module()
block.attn = torch.nn.Module()
block.attn.qkv = torch.nn.Linear(4, 4)
return block
@staticmethod
def _make_layer() -> torch.nn.Module:
layer = torch.nn.Module()
layer.self_attn = torch.nn.Module()
layer.self_attn.q_proj = torch.nn.Linear(4, 4)
return layer
@pytest.mark.parametrize("freeze_vision_tower", (False, True))
@pytest.mark.parametrize("freeze_multi_modal_projector", (False, True))
@pytest.mark.parametrize("freeze_language_model", (False, True))
def test_moss_vl_full(
freeze_vision_tower: bool,
freeze_multi_modal_projector: bool,
freeze_language_model: bool,
):
model = _MossVLFixture()
finetuning_args = FinetuningArguments(
finetuning_type="full",
freeze_vision_tower=freeze_vision_tower,
freeze_multi_modal_projector=freeze_multi_modal_projector,
freeze_language_model=freeze_language_model,
)
_setup_full_tuning(model, finetuning_args, is_trainable=True, cast_trainable_params_to_fp32=False)
for name, param in model.named_parameters():
if name.startswith("model.visual.merger") or name == "model.separator_token":
assert param.requires_grad != freeze_multi_modal_projector
elif name.startswith("model.visual"):
assert param.requires_grad != freeze_vision_tower
else:
assert param.requires_grad != freeze_language_model
@pytest.mark.parametrize("freeze_multi_modal_projector", (False, True))
def test_moss_vl_freeze(freeze_multi_modal_projector: bool):
model = _MossVLFixture()
finetuning_args = FinetuningArguments(
finetuning_type="freeze",
freeze_trainable_layers=1,
freeze_vision_tower=True,
freeze_multi_modal_projector=freeze_multi_modal_projector,
freeze_language_model=False,
)
_setup_freeze_tuning(model, finetuning_args, is_trainable=True, cast_trainable_params_to_fp32=False)
assert model.model.separator_token.requires_grad != freeze_multi_modal_projector
assert model.model.visual.merger.linear_fc1.weight.requires_grad != freeze_multi_modal_projector
assert model.model.visual.patch_embed.proj.weight.requires_grad is False
assert model.model.language_model.layers[0].self_attn.q_proj.weight.requires_grad is False
assert model.model.language_model.layers[1].self_attn.q_proj.weight.requires_grad is True
@pytest.mark.parametrize("freeze_vision_tower", (False, True))
def test_moss_vl_lora_target_all(freeze_vision_tower: bool):
model = _MossVLFixture()
finetuning_args = FinetuningArguments(
finetuning_type="lora",
lora_target="all",
freeze_vision_tower=freeze_vision_tower,
freeze_multi_modal_projector=True,
freeze_language_model=False,
)
target_modules = find_all_linear_modules(model, freeze_vision_tower)
target_modules = patch_target_modules(model, finetuning_args, target_modules)
assert any(name.startswith("model.language_model") and name.endswith("q_proj") for name in target_modules)
assert any(name.startswith("model.visual.blocks") and name.endswith("qkv") for name in target_modules) != (
freeze_vision_tower
)
assert all("patch_embed" not in name for name in target_modules)
assert all("merger" not in name for name in target_modules)
assert all("lm_head" not in name for name in target_modules)
def test_moss_vl_projector_modules():
model = _MossVLFixture()
composite_model = COMPOSITE_MODELS["moss_vl"]
assert composite_model.projector_keys == ["model.visual.merger", "model.separator_token"]
assert composite_model.get_projectors(model) == [model.model.visual.merger]
def test_moss_vl_quantized_projector_hook_skips_parameter():
model = _MossVLFixture()
model.quantization_method = "bitsandbytes"
autocast_projector_dtype(model, SimpleNamespace(compute_dtype=torch.float16))
assert len(model.model.visual.merger._forward_hooks) == 1
@pytest.mark.parametrize("freeze_vision_tower", (False, True))