diff --git a/examples/inference/mossvl.yaml b/examples/inference/mossvl.yaml
new file mode 100644
index 000000000..251747437
--- /dev/null
+++ b/examples/inference/mossvl.yaml
@@ -0,0 +1,6 @@
+### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt`
+
+model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708
+template: moss_vl
+infer_backend: huggingface # choices: [huggingface, vllm, sglang, ktransformers]
+trust_remote_code: true
diff --git a/examples/merge_lora/mossvl_lora_sft.yaml b/examples/merge_lora/mossvl_lora_sft.yaml
new file mode 100644
index 000000000..942c76445
--- /dev/null
+++ b/examples/merge_lora/mossvl_lora_sft.yaml
@@ -0,0 +1,14 @@
+### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt`
+### Note: DO NOT use quantized model or quantization_bit when merging lora adapters
+
+### model
+model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708
+adapter_name_or_path: saves/moss-vl-11b/lora/sft
+template: moss_vl
+trust_remote_code: true
+
+### export
+export_dir: saves/moss_vl_sft_merged
+export_size: 5
+export_device: cpu # choices: [cpu, auto]
+export_legacy_format: false
diff --git a/examples/train_full/mossvl_full_sft.yaml b/examples/train_full/mossvl_full_sft.yaml
new file mode 100644
index 000000000..9fd2815db
--- /dev/null
+++ b/examples/train_full/mossvl_full_sft.yaml
@@ -0,0 +1,57 @@
+### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt`
+
+### model
+model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708
+image_max_pixels: 262144
+video_max_pixels: 16384
+video_fps: 1.0
+video_maxlen: 256
+use_reentrant_gc: false
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: full
+freeze_vision_tower: true
+freeze_multi_modal_projector: true
+freeze_language_model: false
+deepspeed: examples/deepspeed/ds_z3_config.json
+
+### dataset
+dataset: mllm_demo,identity,alpaca_en_demo # video: mllm_video_demo
+template: moss_vl
+cutoff_len: 4096
+max_samples: 1000
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+packing: false
+
+### output
+output_dir: saves/moss-vl-11b/full/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+ use_reentrant: false
+learning_rate: 1.0e-5
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+
+### eval
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500
diff --git a/examples/train_lora/mossvl_lora_sft.yaml b/examples/train_lora/mossvl_lora_sft.yaml
new file mode 100644
index 000000000..99a05786f
--- /dev/null
+++ b/examples/train_lora/mossvl_lora_sft.yaml
@@ -0,0 +1,54 @@
+### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt`
+
+### model
+model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708
+image_max_pixels: 262144
+video_max_pixels: 16384
+video_fps: 1.0
+video_maxlen: 256
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_rank: 8
+lora_target: all
+freeze_vision_tower: true
+freeze_multi_modal_projector: true
+freeze_language_model: false
+
+### dataset
+dataset: mllm_demo,identity,alpaca_en_demo # video: mllm_video_demo
+template: moss_vl
+cutoff_len: 4096
+max_samples: 1000
+preprocessing_num_workers: 16
+dataloader_num_workers: 4
+packing: false
+
+### output
+output_dir: saves/moss-vl-11b/lora/sft
+logging_steps: 10
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 2
+gradient_accumulation_steps: 1
+learning_rate: 1.0e-4
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+resume_from_checkpoint: null
+
+### eval
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500
diff --git a/requirements/moss-vl.txt b/requirements/moss-vl.txt
new file mode 100644
index 000000000..e1186a6a1
--- /dev/null
+++ b/requirements/moss-vl.txt
@@ -0,0 +1,3 @@
+transformers==4.57.1
+torchcodec==0.7.0
+joblib
diff --git a/src/llamafactory/data/collator.py b/src/llamafactory/data/collator.py
index af234d99b..32a4ab61c 100644
--- a/src/llamafactory/data/collator.py
+++ b/src/llamafactory/data/collator.py
@@ -150,7 +150,9 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
if isinstance(self.model, PeftModel):
self.model = self.model.base_model.model
- if self.model is not None and hasattr(self.model, "get_rope_index"): # for qwen2vl mrope
+ if getattr(getattr(self.model, "config", None), "model_type", None) == "moss_vl":
+ self.get_rope_func = None # MOSS-VL computes its own XRoPE positions in model.forward.
+ elif self.model is not None and hasattr(self.model, "get_rope_index"): # for qwen2vl mrope
self.get_rope_func = self.model.get_rope_index # transformers < 4.52.0 or qwen2.5 omni
elif self.model is not None and hasattr(self.model, "model") and hasattr(self.model.model, "get_rope_index"):
self.get_rope_func = self.model.model.get_rope_index # transformers >= 4.52.0
@@ -322,6 +324,8 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
)
def __call__(self, features: list[dict[str, Any]]) -> dict[str, "torch.Tensor"]:
+ model_type = getattr(getattr(self.model, "config", None), "model_type", None)
+ is_moss_vl = model_type == "moss_vl"
batch_images, batch_videos, batch_audios = [], [], []
batch_imglens, batch_vidlens, batch_audlens, batch_input_ids = [], [], [], []
packing_params_list: list[dict[str, Any] | None] = []
@@ -341,7 +345,10 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
fake_input_ids = []
has_dummy_image = False
if (
- self.template.mm_plugin.image_token is not None and sum(batch_imglens) == 0 and sum(batch_vidlens) == 0
+ self.template.mm_plugin.image_token is not None
+ and sum(batch_imglens) == 0
+ and sum(batch_vidlens) == 0
+ and not is_moss_vl # MOSS-VL builds one native zero-valued dummy per text-only sample in its plugin.
): # avoid process hanging in zero3/fsdp case
fake_messages = [{"role": "user", "content": IMAGE_PLACEHOLDER}]
fake_images = [Image.new("RGB", (64, 64), (255, 255, 255))]
@@ -416,7 +423,6 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
features: dict[str, torch.Tensor] = super().__call__(features)
bsz, seq_len = features["input_ids"].shape[:2]
- model_type = getattr(self.model.config, "model_type", None) if self.model is not None else None
is_omni = model_type in [
"qwen2_5_omni_thinker",
"qwen3_omni_moe_thinker",
@@ -461,12 +467,17 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq):
):
raise ValueError(f"{self.model.config.model_type} requires 3D position ids for mrope.")
- if "cross_attention_mask" in mm_inputs: # for mllama inputs when pad_to_multiple_of is enabled
+ if (
+ "cross_attention_mask" in mm_inputs and mm_inputs["cross_attention_mask"].dtype != torch.bool
+ ): # for mllama inputs when pad_to_multiple_of is enabled
cross_attention_mask = mm_inputs.pop("cross_attention_mask")
seq_len = features["input_ids"].size(1)
orig_len = cross_attention_mask.size(1)
mm_inputs["cross_attention_mask"] = F.pad(cross_attention_mask, (0, 0, 0, 0, 0, seq_len - orig_len))
+ if is_moss_vl:
+ mm_inputs = self.template.mm_plugin.post_process_mossvl_inputs(features, mm_inputs, self.processor)
+
features.update(mm_inputs)
if "image_bound" in features: # for minicpmv inputs
diff --git a/src/llamafactory/data/mm_plugin.py b/src/llamafactory/data/mm_plugin.py
index 8c2d66c2f..10eefd3db 100644
--- a/src/llamafactory/data/mm_plugin.py
+++ b/src/llamafactory/data/mm_plugin.py
@@ -472,6 +472,354 @@ class BasePlugin(MMPluginMixin):
return self._get_mm_inputs(images, videos, audios, processor)
+@dataclass
+class MossVLPlugin(BasePlugin):
+ vision_bos_token: str = "<|vision_start|>"
+ vision_eos_token: str = "<|vision_end|>"
+ time_bos_token: str = "<|time_start|>"
+ time_eos_token: str = "<|time_end|>"
+
+ @staticmethod
+ def _split_pixel_values(
+ pixel_values: "torch.Tensor",
+ grid_thw: "torch.Tensor",
+ ) -> list["torch.Tensor"]:
+ patch_counts = [int(grid.prod().item()) for grid in grid_thw]
+ return list(torch.split(pixel_values, patch_counts))
+
+ @staticmethod
+ def _create_cross_attention_mask(
+ input_ids: Union[list[list[int]], "torch.Tensor"],
+ grid_thw: "torch.Tensor",
+ media_nums_per_sample: list[int],
+ image_token_id: int,
+ attention_mask: Optional["torch.Tensor"] = None,
+ padding_side: Literal["left", "right"] = "right",
+ ) -> "torch.Tensor":
+ r"""Create the native MOSS-VL frame-level causal cross-attention mask."""
+ if isinstance(input_ids, list):
+ max_text_len = max(len(token_ids) for token_ids in input_ids)
+ input_ids_tensor = torch.full((len(input_ids), max_text_len), -1, dtype=torch.long)
+ attention_mask_tensor = torch.zeros_like(input_ids_tensor, dtype=torch.bool)
+ for batch_index, token_ids in enumerate(input_ids):
+ seq_len = len(token_ids)
+ start = max_text_len - seq_len if padding_side == "left" else 0
+ input_ids_tensor[batch_index, start : start + seq_len] = torch.tensor(token_ids, dtype=torch.long)
+ attention_mask_tensor[batch_index, start : start + seq_len] = True
+ else:
+ input_ids_tensor = input_ids
+ attention_mask_tensor = (
+ torch.ones_like(input_ids_tensor, dtype=torch.bool)
+ if attention_mask is None
+ else attention_mask.bool()
+ )
+
+ total_frames_per_sample = []
+ media_index = 0
+ for num_media in media_nums_per_sample:
+ sample_grid = grid_thw[media_index : media_index + num_media]
+ total_frames_per_sample.append(int(sample_grid[:, 0].sum().item()))
+ media_index += num_media
+
+ max_num_frames = max(total_frames_per_sample)
+ frame_indices = torch.arange(max_num_frames, device=input_ids_tensor.device).view(1, 1, -1)
+ visible_mask = (input_ids_tensor == image_token_id).cumsum(dim=1).unsqueeze(-1) > frame_indices
+ visible_mask &= attention_mask_tensor.unsqueeze(-1)
+ valid_frames = frame_indices < torch.tensor(
+ total_frames_per_sample,
+ device=input_ids_tensor.device,
+ ).view(-1, 1, 1)
+ visible_mask &= valid_frames
+ return (~visible_mask).unsqueeze(1)
+
+ def _get_video_inputs(
+ self,
+ videos: list["VideoInput"],
+ processor: "MMProcessor",
+ return_metadata: bool,
+ ) -> dict[str, Any]:
+ video_kwargs = {"return_tensors": "pt", "return_metadata": return_metadata}
+ if getattr(processor, "video_fps", None) is not None:
+ video_kwargs["video_fps"] = processor.video_fps
+ if getattr(processor, "video_maxlen", None) is not None:
+ video_kwargs["max_frames"] = processor.video_maxlen
+
+ video_min_pixels = getattr(processor, "video_min_pixels", None)
+ video_max_pixels = getattr(processor, "video_max_pixels", None)
+ if video_min_pixels is not None and video_max_pixels is not None:
+ video_kwargs["size"] = {
+ "shortest_edge": video_min_pixels,
+ "longest_edge": video_max_pixels,
+ }
+
+ return dict(processor.video_processor(videos=videos, **video_kwargs))
+
+ def _get_media_order_from_ids(
+ self,
+ input_ids: list[int],
+ processor: "MMProcessor",
+ num_images: int,
+ num_videos: int,
+ expected_video_frames: Optional[list[int]] = None,
+ ) -> list[str]:
+ media_order = []
+ video_frame_counts = []
+ in_video = False
+ current_video_frames = 0
+ for token_id in input_ids:
+ if token_id == processor.vision_start_token_id:
+ if in_video:
+ raise ValueError(
+ "MOSS-VL encountered nested video token blocks after tokenization. "
+ "Please increase `cutoff_len` if a video placeholder was truncated."
+ )
+
+ media_order.append("video")
+ in_video = True
+ current_video_frames = 0
+ elif token_id == processor.vision_end_token_id:
+ if not in_video:
+ raise ValueError(
+ "MOSS-VL encountered a video end token without a matching start token after tokenization. "
+ "Please increase `cutoff_len` if a video placeholder was truncated."
+ )
+
+ video_frame_counts.append(current_video_frames)
+ in_video = False
+ elif token_id == processor.image_token_id:
+ if in_video:
+ current_video_frames += 1
+ else:
+ media_order.append("image")
+
+ if in_video:
+ raise ValueError(
+ "MOSS-VL encountered an incomplete video token block after tokenization. "
+ "Please increase `cutoff_len` or reduce `video_maxlen`."
+ )
+
+ if media_order.count("image") != num_images or media_order.count("video") != num_videos:
+ raise ValueError(
+ "MOSS-VL media tokens do not match the provided media after tokenization: "
+ f"order={media_order}, images={num_images}, videos={num_videos}. "
+ "Please increase `cutoff_len` if a visual placeholder was truncated."
+ )
+
+ if expected_video_frames is not None and video_frame_counts != expected_video_frames:
+ raise ValueError(
+ "MOSS-VL video frame tokens do not match the processed video after tokenization: "
+ f"tokens={video_frame_counts}, frames={expected_video_frames}. "
+ "Please increase `cutoff_len` or reduce `video_maxlen`."
+ )
+
+ return media_order
+
+ @override
+ def process_messages(
+ self,
+ messages: list[dict[str, str]],
+ images: list["ImageInput"],
+ videos: list["VideoInput"],
+ audios: list["AudioInput"],
+ processor: Optional["MMProcessor"],
+ ) -> list[dict[str, str]]:
+ self._validate_input(processor, images, videos, audios)
+ self._validate_messages(messages, images, videos, audios)
+ messages = deepcopy(messages)
+
+ video_inputs = self._get_video_inputs(videos, processor, return_metadata=True) if videos else {}
+ video_grid_thw = video_inputs.get("video_grid_thw", [])
+ video_metadata = video_inputs.get("video_metadata", [])
+ video_index = 0
+ for message in messages:
+ content = message["content"]
+ content = content.replace(IMAGE_PLACEHOLDER, self.image_token)
+ while VIDEO_PLACEHOLDER in content:
+ metadata = video_metadata[video_index]
+ if metadata.fps is None:
+ metadata.fps = 24
+
+ timestamps = processor._calculate_timestamps(
+ metadata.frames_indices,
+ metadata.total_num_frames,
+ metadata.fps,
+ metadata.duration,
+ processor.video_processor.temporal_patch_size,
+ actual_timestamps=getattr(metadata, "actual_timestamps", None),
+ )
+ num_frames = int(video_grid_thw[video_index][0].item())
+ frame_tokens = [
+ f"{self.time_bos_token}{timestamps[frame_idx]:.1f} seconds{self.time_eos_token}{self.image_token}"
+ for frame_idx in range(num_frames)
+ ]
+ video_tokens = f"{self.vision_bos_token}{''.join(frame_tokens)}{self.vision_eos_token}"
+ content = content.replace(VIDEO_PLACEHOLDER, video_tokens, 1)
+ video_index += 1
+
+ message["content"] = content
+
+ return messages
+
+ @override
+ def get_mm_inputs(
+ self,
+ images: list["ImageInput"],
+ videos: list["VideoInput"],
+ audios: list["AudioInput"],
+ imglens: list[int],
+ vidlens: list[int],
+ audlens: list[int],
+ batch_ids: list[list[int]],
+ processor: Optional["MMProcessor"],
+ ) -> dict[str, Union[list[int], "torch.Tensor"]]:
+ self._validate_input(processor, images, videos, audios)
+ if audios:
+ raise ValueError("MOSS-VL does not support audio inputs.")
+
+ if not (len(imglens) == len(vidlens) == len(batch_ids)):
+ raise ValueError("MOSS-VL batch metadata must have one entry per sample.")
+ final_pixel_values = []
+ final_grid_thw = []
+ media_nums_per_sample = []
+ image_offset = 0
+ video_offset = 0
+ for imglen, vidlen, input_ids in zip(imglens, vidlens, batch_ids):
+ sample_images = images[image_offset : image_offset + imglen]
+ sample_videos = videos[video_offset : video_offset + vidlen]
+ image_offset += imglen
+ video_offset += vidlen
+ image_chunks, image_grids = [], []
+ if sample_images:
+ regularized_images = self._regularize_images(
+ sample_images,
+ image_max_pixels=2**63 - 1,
+ image_min_pixels=1,
+ )["images"]
+ image_kwargs = {"return_tensors": "pt"}
+ if getattr(processor, "image_min_pixels", None) is not None:
+ image_kwargs["min_pixels"] = processor.image_min_pixels
+ if getattr(processor, "image_max_pixels", None) is not None:
+ image_kwargs["max_pixels"] = processor.image_max_pixels
+
+ image_inputs = processor.image_processor(images=regularized_images, **image_kwargs)
+ image_grids = list(image_inputs["image_grid_thw"])
+ image_chunks = self._split_pixel_values(image_inputs["pixel_values"], image_inputs["image_grid_thw"])
+
+ video_chunks, video_grids = [], []
+ if sample_videos:
+ video_inputs = self._get_video_inputs(sample_videos, processor, return_metadata=False)
+ video_grids = list(video_inputs["video_grid_thw"])
+ video_chunks = self._split_pixel_values(
+ video_inputs["pixel_values_videos"],
+ video_inputs["video_grid_thw"],
+ )
+
+ media_order = self._get_media_order_from_ids(
+ input_ids,
+ processor,
+ imglen,
+ vidlen,
+ expected_video_frames=[int(grid[0].item()) for grid in video_grids],
+ )
+
+ if not media_order:
+ patch_size = getattr(processor.image_processor, "patch_size", None)
+ if patch_size is None: # lightweight/test processors without the native MOSS-VL contract
+ blank_image = Image.new("RGB", (128, 128), (255, 255, 255))
+ blank_inputs = processor.image_processor(images=[blank_image], return_tensors="pt")
+ final_pixel_values.append(blank_inputs["pixel_values"])
+ final_grid_thw.append(blank_inputs["image_grid_thw"][0])
+ else:
+ temporal_patch_size = getattr(processor.image_processor, "temporal_patch_size", None) or 1
+ merge_size = getattr(processor.image_processor, "merge_size", None) or 2
+ factor = patch_size * merge_size
+ side = math.ceil(128 / factor) * factor
+ grid_thw = torch.tensor([1, side // patch_size, side // patch_size])
+ feature_dim = 3 * temporal_patch_size * patch_size * patch_size
+ final_pixel_values.append(torch.zeros((int(grid_thw.prod()), feature_dim), dtype=torch.float32))
+ final_grid_thw.append(grid_thw)
+ media_nums_per_sample.append(1)
+ continue
+
+ image_index = 0
+ video_index = 0
+ for modality in media_order:
+ if modality == "image":
+ final_pixel_values.append(image_chunks[image_index])
+ final_grid_thw.append(image_grids[image_index])
+ image_index += 1
+ else:
+ final_pixel_values.append(video_chunks[video_index])
+ final_grid_thw.append(video_grids[video_index])
+ video_index += 1
+
+ media_nums_per_sample.append(len(media_order))
+
+ if image_offset != len(images) or video_offset != len(videos):
+ raise ValueError("MOSS-VL media lengths do not consume all provided inputs.")
+
+ mm_inputs = {
+ "pixel_values": torch.cat(final_pixel_values, dim=0),
+ "grid_thw": torch.stack(final_grid_thw),
+ "media_nums_per_sample": media_nums_per_sample,
+ }
+ mm_inputs["cross_attention_mask"] = self._create_cross_attention_mask(
+ batch_ids,
+ mm_inputs["grid_thw"],
+ media_nums_per_sample,
+ processor.image_token_id,
+ padding_side=processor.tokenizer.padding_side,
+ )
+ return mm_inputs
+
+ def post_process_mossvl_inputs(
+ self,
+ features: dict[str, "torch.Tensor"],
+ mm_inputs: dict[str, Any],
+ processor: "MMProcessor",
+ ) -> dict[str, Any]:
+ r"""Create MOSS-VL batch-only inputs after the text batch has been padded."""
+ input_ids = features["input_ids"]
+ attention_mask = features["attention_mask"].bool()
+ mm_inputs["cross_attention_mask"] = self._create_cross_attention_mask(
+ input_ids,
+ mm_inputs["grid_thw"],
+ mm_inputs["media_nums_per_sample"],
+ processor.image_token_id,
+ attention_mask,
+ )
+
+ dummy_image_tokens = (input_ids == processor.image_token_id) & ~attention_mask
+ input_ids.masked_fill_(dummy_image_tokens, processor.tokenizer.pad_token_id)
+
+ labels = features.get("labels")
+ if labels is not None:
+ control_token_ids = {
+ processor.image_token_id,
+ processor.video_token_id,
+ processor.vision_start_token_id,
+ processor.vision_end_token_id,
+ processor.tokenizer.convert_tokens_to_ids(self.time_bos_token),
+ processor.tokenizer.convert_tokens_to_ids(self.time_eos_token),
+ }
+ for batch_index, token_ids in enumerate(input_ids):
+ in_vision = False
+ for token_index, token_id in enumerate(token_ids.tolist()):
+ if token_id == processor.vision_start_token_id:
+ in_vision = True
+ if in_vision or token_id in control_token_ids:
+ labels[batch_index, token_index] = IGNORE_INDEX
+ if token_id == processor.vision_end_token_id:
+ in_vision = False
+
+ # Native MOSS-VL labels_spans supervise through <|im_end|>, but not its trailing newline.
+ im_end_token_id = processor.tokenizer.convert_tokens_to_ids("<|im_end|>")
+ labels[:, 1:].masked_fill_(input_ids[:, :-1] == im_end_token_id, IGNORE_INDEX)
+
+ features.pop("position_ids", None)
+ return mm_inputs
+
+
@dataclass
class ErnieVLPlugin(BasePlugin):
@override
@@ -2911,6 +3259,7 @@ PLUGINS = {
"minicpm_v": MiniCPMVPlugin,
"minicpm_v_4_6": MiniCPMV4_6Plugin,
"mllama": MllamaPlugin,
+ "moss_vl": MossVLPlugin,
"paligemma": PaliGemmaPlugin,
"pixtral": PixtralPlugin,
"qwen2_audio": Qwen2AudioPlugin,
diff --git a/src/llamafactory/data/template.py b/src/llamafactory/data/template.py
index 492909eea..caa926abc 100644
--- a/src/llamafactory/data/template.py
+++ b/src/llamafactory/data/template.py
@@ -333,6 +333,50 @@ class Template:
return modelfile
+@dataclass
+class MossVLTemplate(Template):
+ @override
+ def _encode(
+ self,
+ tokenizer: "PreTrainedTokenizer",
+ messages: list[dict[str, str]],
+ system: Optional[str],
+ tools: Optional[str],
+ ) -> list[list[int]]:
+ system = system or self.default_system
+ encoded_messages = []
+ for i, message in enumerate(messages):
+ elements = []
+
+ if i == 0:
+ elements += self.format_prefix.apply()
+ if system or tools:
+ tool_text = self.format_tools.apply(content=tools)[0] if tools else ""
+ if tools and not system:
+ tool_text = tool_text.lstrip("\n")
+
+ elements += self.format_system.apply(content=(system + tool_text))
+
+ if message["role"] == Role.USER:
+ elements += self.format_user.apply(content=message["content"], idx=str(i // 2))
+ elif message["role"] == Role.ASSISTANT:
+ elements += self.format_assistant.apply(content=message["content"])
+ elif message["role"] == Role.OBSERVATION:
+ elements += self.format_observation.apply(content=message["content"])
+ elif message["role"] == Role.FUNCTION:
+ elements += self.format_function.apply(
+ content=message["content"],
+ thought_words=self.thought_words,
+ tool_call_words=self.tool_call_words,
+ )
+ else:
+ raise NotImplementedError("Unexpected role: {}".format(message["role"]))
+
+ encoded_messages.append(self._convert_elements_to_ids(tokenizer, elements))
+
+ return encoded_messages
+
+
@dataclass
class Llama2Template(Template):
r"""A template that fuse the system message to first user message."""
@@ -1526,6 +1570,32 @@ register_template(
)
+# copied from qwen template
+register_template(
+ name="moss_vl",
+ format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]),
+ format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]),
+ format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]),
+ format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen"),
+ format_observation=StringFormatter(
+ slots=["<|im_start|>user\n\n{{content}}\n<|im_end|>\n<|im_start|>assistant\n"]
+ ),
+ format_tools=ToolFormatter(tool_format="qwen"),
+ stop_words=["<|im_end|>"],
+ replace_eos=True,
+ mm_plugin=get_mm_plugin(
+ name="moss_vl",
+ image_token="<|image_pad|>",
+ video_token="<|video_pad|>",
+ vision_bos_token="<|vision_start|>",
+ vision_eos_token="<|vision_end|>",
+ time_bos_token="<|time_start|>",
+ time_eos_token="<|time_end|>",
+ ),
+ template_class=MossVLTemplate,
+)
+
+
# copied from vicuna template
register_template(
name="llava",
diff --git a/src/llamafactory/extras/constants.py b/src/llamafactory/extras/constants.py
index 152a5681a..99662d482 100644
--- a/src/llamafactory/extras/constants.py
+++ b/src/llamafactory/extras/constants.py
@@ -2198,6 +2198,17 @@ register_model_group(
)
+register_model_group(
+ models={
+ "MOSS-VL-Instruct-0708": {
+ DownloadSource.DEFAULT: "OpenMOSS-Team/MOSS-VL-Instruct-0708",
+ },
+ },
+ template="moss_vl",
+ multimodal=True,
+)
+
+
register_model_group(
models={
"OLMo-1B": {
diff --git a/src/llamafactory/model/model_utils/visual.py b/src/llamafactory/model/model_utils/visual.py
index d2f5bf2d5..9a5e80a98 100644
--- a/src/llamafactory/model/model_utils/visual.py
+++ b/src/llamafactory/model/model_utils/visual.py
@@ -56,7 +56,7 @@ class CompositeModel:
)
break
- if project_module is not None:
+ if isinstance(project_module, torch.nn.Module):
mm_projectors.append(project_module)
return mm_projectors
@@ -344,6 +344,15 @@ _register_composite_model(
)
+_register_composite_model(
+ model_type="moss_vl",
+ projector_keys=["model.visual.merger", "model.separator_token"],
+ vision_model_keys=["model.visual.pos_embed", "model.visual.patch_embed", "model.visual.blocks"],
+ language_model_keys=["model.language_model", "lm_head"],
+ lora_conflict_keys=["patch_embed"],
+)
+
+
_register_composite_model(
model_type="mllama",
vision_model_keys=["vision_model"],
diff --git a/src/llamafactory/model/patcher.py b/src/llamafactory/model/patcher.py
index c2e0ea21f..79c57603e 100644
--- a/src/llamafactory/model/patcher.py
+++ b/src/llamafactory/model/patcher.py
@@ -23,7 +23,7 @@ from transformers.modeling_utils import is_fsdp_enabled
from transformers.utils import is_torch_cuda_available, is_torch_npu_available
from ..extras import logging
-from ..extras.misc import infer_optim_dtype
+from ..extras.misc import check_version, infer_optim_dtype
from ..extras.packages import is_transformers_version_greater_than
from .model_utils.attention import configure_attn_implementation, print_attn_implementation
from .model_utils.checkpointing import prepare_model_for_training
@@ -418,6 +418,10 @@ def patch_config(
"pip install git+https://github.com/huggingface/transformers.git@3c2517727ce28a30f5044e01663ee204deb1cdbe"
)
+ if getattr(config, "model_type", None) == "moss_vl":
+ check_version("transformers==4.57.1", mandatory=True)
+ check_version("torchcodec==0.7.0", mandatory=True)
+
if getattr(config, "model_type", None) == "qwen3_omni_moe":
patch_qwen3_omni_moe_thinker_text_sparse_moe_block()
diff --git a/tests/data/test_mm_plugin.py b/tests/data/test_mm_plugin.py
index aabf1fbd7..73225ebaa 100644
--- a/tests/data/test_mm_plugin.py
+++ b/tests/data/test_mm_plugin.py
@@ -13,6 +13,7 @@
# limitations under the License.
import os
+from types import SimpleNamespace
from typing import TYPE_CHECKING, Any
import numpy as np
@@ -417,6 +418,24 @@ def test_qwen2_vl_plugin():
_check_plugin(**check_inputs)
+def test_moss_vl_plugin():
+ messages = [
+ {"role": "user", "content": "First , finally ."},
+ {"role": "assistant", "content": "Done."},
+ ]
+ expected_messages = [
+ {"role": "user", "content": "First <|image_pad|>, finally <|image_pad|>."},
+ {"role": "assistant", "content": "Done."},
+ ]
+ processor = SimpleNamespace(image_processor=object(), video_processor=object())
+ plugin = get_mm_plugin(name="moss_vl", image_token="<|image_pad|>", video_token="<|video_pad|>")
+
+ processed_messages = plugin.process_messages(messages, [object(), object()], [], [], processor)
+
+ assert processed_messages == expected_messages
+ assert messages[0]["content"] == "First , finally ."
+
+
@pytest.mark.runs_on(["cpu", "mps"])
@pytest.mark.skipif(not is_transformers_version_greater_than("4.57.0"), reason="Requires transformers>=4.57.0")
def test_qwen3_vl_plugin():
diff --git a/tests/data/test_moss_vl_plugin.py b/tests/data/test_moss_vl_plugin.py
new file mode 100644
index 000000000..ef9ff9ed1
--- /dev/null
+++ b/tests/data/test_moss_vl_plugin.py
@@ -0,0 +1,631 @@
+# Copyright 2025 the LlamaFactory team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+from PIL import Image
+
+from llamafactory.data.collator import MultiModalDataCollatorForSeq2Seq
+from llamafactory.data.mm_plugin import get_mm_plugin
+from llamafactory.data.processor.supervised import SupervisedDatasetProcessor
+from llamafactory.extras.constants import IGNORE_INDEX
+
+
+IMAGE_TOKEN_ID = 101
+VIDEO_TOKEN_ID = 102
+VISION_START_TOKEN_ID = 103
+VISION_END_TOKEN_ID = 104
+TIME_START_TOKEN_ID = 105
+TIME_END_TOKEN_ID = 106
+IM_END_TOKEN_ID = 107
+
+
+class _ImageProcessor:
+ def __init__(self):
+ self.calls = []
+
+ def __call__(self, images, return_tensors, **kwargs):
+ self.calls.append({"return_tensors": return_tensors, **kwargs})
+ values = []
+ for image in images:
+ marker = image.getpixel((0, 0))[0] + 1
+ values.append(torch.full((1, 3), marker, dtype=torch.float32))
+
+ return {
+ "pixel_values": torch.cat(values),
+ "image_grid_thw": torch.tensor([[1, 1, 1]] * len(images)),
+ }
+
+
+class _VideoProcessor:
+ temporal_patch_size = 1
+
+ def __init__(self):
+ self.calls = []
+
+ def __call__(self, videos, return_tensors, return_metadata, **kwargs):
+ self.calls.append(
+ {
+ "return_tensors": return_tensors,
+ "return_metadata": return_metadata,
+ **kwargs,
+ }
+ )
+ result = {
+ "pixel_values_videos": torch.cat(
+ [torch.full((2, 3), 9 + index, dtype=torch.float32) for index in range(len(videos))]
+ ),
+ "video_grid_thw": torch.tensor([[2, 1, 1]] * len(videos)),
+ }
+ if return_metadata:
+ result["video_metadata"] = [
+ SimpleNamespace(frames_indices=[0, 2], total_num_frames=2, fps=2.0, duration=2.0) for _ in videos
+ ]
+
+ return result
+
+
+class _Tokenizer:
+ pad_token_id = 0
+ padding_side = "right"
+ _token_ids = {
+ "<|time_start|>": TIME_START_TOKEN_ID,
+ "<|time_end|>": TIME_END_TOKEN_ID,
+ "<|im_end|>": IM_END_TOKEN_ID,
+ }
+
+ def convert_tokens_to_ids(self, token):
+ return self._token_ids[token]
+
+ def pad(self, features, padding, max_length, pad_to_multiple_of, return_tensors):
+ del padding, max_length, return_tensors
+ sequence_length = max(len(feature["input_ids"]) for feature in features)
+ if pad_to_multiple_of is not None:
+ sequence_length = ((sequence_length + pad_to_multiple_of - 1) // pad_to_multiple_of) * pad_to_multiple_of
+
+ padded = {"input_ids": [], "attention_mask": []}
+ for feature in features:
+ pad_length = sequence_length - len(feature["input_ids"])
+ if self.padding_side == "right":
+ padded["input_ids"].append(feature["input_ids"] + [self.pad_token_id] * pad_length)
+ padded["attention_mask"].append(feature["attention_mask"] + [0] * pad_length)
+ else:
+ padded["input_ids"].append([self.pad_token_id] * pad_length + feature["input_ids"])
+ padded["attention_mask"].append([0] * pad_length + feature["attention_mask"])
+
+ return {key: torch.tensor(value) for key, value in padded.items()}
+
+
+class _Processor:
+ image_token_id = IMAGE_TOKEN_ID
+ video_token_id = VIDEO_TOKEN_ID
+ vision_start_token_id = VISION_START_TOKEN_ID
+ vision_end_token_id = VISION_END_TOKEN_ID
+
+ def __init__(self):
+ self.image_processor = _ImageProcessor()
+ self.video_processor = _VideoProcessor()
+ self.tokenizer = _Tokenizer()
+
+ @staticmethod
+ def _calculate_timestamps(*args, **kwargs):
+ del args, kwargs
+ return [0.0, 1.0]
+
+
+def _get_plugin():
+ return get_mm_plugin(
+ name="moss_vl",
+ image_token="<|image_pad|>",
+ video_token="<|video_pad|>",
+ vision_bos_token="<|vision_start|>",
+ vision_eos_token="<|vision_end|>",
+ time_bos_token="<|time_start|>",
+ time_eos_token="<|time_end|>",
+ )
+
+
+def _video_ids(seed):
+ return [
+ VISION_START_TOKEN_ID,
+ TIME_START_TOKEN_ID,
+ seed,
+ TIME_END_TOKEN_ID,
+ IMAGE_TOKEN_ID,
+ TIME_START_TOKEN_ID,
+ seed + 1,
+ TIME_END_TOKEN_ID,
+ IMAGE_TOKEN_ID,
+ VISION_END_TOKEN_ID,
+ ]
+
+
+def _left_pad(sequences, pad_value):
+ max_len = max(map(len, sequences))
+ return torch.tensor([[pad_value] * (max_len - len(sequence)) + sequence for sequence in sequences])
+
+
+def test_moss_vl_process_messages_expands_video_frames():
+ plugin = _get_plugin()
+ processor = _Processor()
+ messages = [
+ {"role": "user", "content": "First , then