From 713b5a3f959d0c647d46f7ab76cbb8fde8155586 Mon Sep 17 00:00:00 2001 From: SSSSuperC <106602888+SSSSuperC@users.noreply.github.com> Date: Mon, 3 Aug 2026 18:18:24 +0800 Subject: [PATCH] [model] add MOSS-VL support (#10708) --- examples/inference/mossvl.yaml | 6 + examples/merge_lora/mossvl_lora_sft.yaml | 14 + examples/train_full/mossvl_full_sft.yaml | 57 ++ examples/train_lora/mossvl_lora_sft.yaml | 54 ++ requirements/moss-vl.txt | 3 + src/llamafactory/data/collator.py | 19 +- src/llamafactory/data/mm_plugin.py | 349 ++++++++++ src/llamafactory/data/template.py | 70 ++ src/llamafactory/extras/constants.py | 11 + src/llamafactory/model/model_utils/visual.py | 11 +- src/llamafactory/model/patcher.py | 6 +- tests/data/test_mm_plugin.py | 19 + tests/data/test_moss_vl_plugin.py | 631 +++++++++++++++++++ tests/data/test_moss_vl_training_configs.py | 39 ++ tests/data/test_template.py | 24 +- tests/model/model_utils/test_visual.py | 127 +++- 16 files changed, 1432 insertions(+), 8 deletions(-) create mode 100644 examples/inference/mossvl.yaml create mode 100644 examples/merge_lora/mossvl_lora_sft.yaml create mode 100644 examples/train_full/mossvl_full_sft.yaml create mode 100644 examples/train_lora/mossvl_lora_sft.yaml create mode 100644 requirements/moss-vl.txt create mode 100644 tests/data/test_moss_vl_plugin.py create mode 100644 tests/data/test_moss_vl_training_configs.py diff --git a/examples/inference/mossvl.yaml b/examples/inference/mossvl.yaml new file mode 100644 index 000000000..251747437 --- /dev/null +++ b/examples/inference/mossvl.yaml @@ -0,0 +1,6 @@ +### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt` + +model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708 +template: moss_vl +infer_backend: huggingface # choices: [huggingface, vllm, sglang, ktransformers] +trust_remote_code: true diff --git a/examples/merge_lora/mossvl_lora_sft.yaml b/examples/merge_lora/mossvl_lora_sft.yaml new file mode 100644 index 000000000..942c76445 --- /dev/null +++ b/examples/merge_lora/mossvl_lora_sft.yaml @@ -0,0 +1,14 @@ +### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt` +### Note: DO NOT use quantized model or quantization_bit when merging lora adapters + +### model +model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708 +adapter_name_or_path: saves/moss-vl-11b/lora/sft +template: moss_vl +trust_remote_code: true + +### export +export_dir: saves/moss_vl_sft_merged +export_size: 5 +export_device: cpu # choices: [cpu, auto] +export_legacy_format: false diff --git a/examples/train_full/mossvl_full_sft.yaml b/examples/train_full/mossvl_full_sft.yaml new file mode 100644 index 000000000..9fd2815db --- /dev/null +++ b/examples/train_full/mossvl_full_sft.yaml @@ -0,0 +1,57 @@ +### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt` + +### model +model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708 +image_max_pixels: 262144 +video_max_pixels: 16384 +video_fps: 1.0 +video_maxlen: 256 +use_reentrant_gc: false +trust_remote_code: true + +### method +stage: sft +do_train: true +finetuning_type: full +freeze_vision_tower: true +freeze_multi_modal_projector: true +freeze_language_model: false +deepspeed: examples/deepspeed/ds_z3_config.json + +### dataset +dataset: mllm_demo,identity,alpaca_en_demo # video: mllm_video_demo +template: moss_vl +cutoff_len: 4096 +max_samples: 1000 +preprocessing_num_workers: 16 +dataloader_num_workers: 4 +packing: false + +### output +output_dir: saves/moss-vl-11b/full/sft +logging_steps: 10 +save_steps: 500 +plot_loss: true +overwrite_output_dir: true +save_only_model: false +report_to: none # choices: [none, wandb, tensorboard, swanlab, mlflow] + +### train +per_device_train_batch_size: 1 +gradient_accumulation_steps: 1 +gradient_checkpointing: true +gradient_checkpointing_kwargs: + use_reentrant: false +learning_rate: 1.0e-5 +num_train_epochs: 3.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 +resume_from_checkpoint: null + +### eval +# val_size: 0.1 +# per_device_eval_batch_size: 1 +# eval_strategy: steps +# eval_steps: 500 diff --git a/examples/train_lora/mossvl_lora_sft.yaml b/examples/train_lora/mossvl_lora_sft.yaml new file mode 100644 index 000000000..99a05786f --- /dev/null +++ b/examples/train_lora/mossvl_lora_sft.yaml @@ -0,0 +1,54 @@ +### Install model-specific dependencies: `pip install -r requirements/moss-vl.txt` + +### model +model_name_or_path: OpenMOSS-Team/MOSS-VL-Instruct-0708 +image_max_pixels: 262144 +video_max_pixels: 16384 +video_fps: 1.0 +video_maxlen: 256 +trust_remote_code: true + +### method +stage: sft +do_train: true +finetuning_type: lora +lora_rank: 8 +lora_target: all +freeze_vision_tower: true +freeze_multi_modal_projector: true +freeze_language_model: false + +### dataset +dataset: mllm_demo,identity,alpaca_en_demo # video: mllm_video_demo +template: moss_vl +cutoff_len: 4096 +max_samples: 1000 +preprocessing_num_workers: 16 +dataloader_num_workers: 4 +packing: false + +### output +output_dir: saves/moss-vl-11b/lora/sft +logging_steps: 10 +save_steps: 500 +plot_loss: true +overwrite_output_dir: true +save_only_model: false +report_to: none # choices: [none, wandb, tensorboard, swanlab, mlflow] + +### train +per_device_train_batch_size: 2 +gradient_accumulation_steps: 1 +learning_rate: 1.0e-4 +num_train_epochs: 3.0 +lr_scheduler_type: cosine +warmup_ratio: 0.1 +bf16: true +ddp_timeout: 180000000 +resume_from_checkpoint: null + +### eval +# val_size: 0.1 +# per_device_eval_batch_size: 1 +# eval_strategy: steps +# eval_steps: 500 diff --git a/requirements/moss-vl.txt b/requirements/moss-vl.txt new file mode 100644 index 000000000..e1186a6a1 --- /dev/null +++ b/requirements/moss-vl.txt @@ -0,0 +1,3 @@ +transformers==4.57.1 +torchcodec==0.7.0 +joblib diff --git a/src/llamafactory/data/collator.py b/src/llamafactory/data/collator.py index af234d99b..32a4ab61c 100644 --- a/src/llamafactory/data/collator.py +++ b/src/llamafactory/data/collator.py @@ -150,7 +150,9 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq): if isinstance(self.model, PeftModel): self.model = self.model.base_model.model - if self.model is not None and hasattr(self.model, "get_rope_index"): # for qwen2vl mrope + if getattr(getattr(self.model, "config", None), "model_type", None) == "moss_vl": + self.get_rope_func = None # MOSS-VL computes its own XRoPE positions in model.forward. + elif self.model is not None and hasattr(self.model, "get_rope_index"): # for qwen2vl mrope self.get_rope_func = self.model.get_rope_index # transformers < 4.52.0 or qwen2.5 omni elif self.model is not None and hasattr(self.model, "model") and hasattr(self.model.model, "get_rope_index"): self.get_rope_func = self.model.model.get_rope_index # transformers >= 4.52.0 @@ -322,6 +324,8 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq): ) def __call__(self, features: list[dict[str, Any]]) -> dict[str, "torch.Tensor"]: + model_type = getattr(getattr(self.model, "config", None), "model_type", None) + is_moss_vl = model_type == "moss_vl" batch_images, batch_videos, batch_audios = [], [], [] batch_imglens, batch_vidlens, batch_audlens, batch_input_ids = [], [], [], [] packing_params_list: list[dict[str, Any] | None] = [] @@ -341,7 +345,10 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq): fake_input_ids = [] has_dummy_image = False if ( - self.template.mm_plugin.image_token is not None and sum(batch_imglens) == 0 and sum(batch_vidlens) == 0 + self.template.mm_plugin.image_token is not None + and sum(batch_imglens) == 0 + and sum(batch_vidlens) == 0 + and not is_moss_vl # MOSS-VL builds one native zero-valued dummy per text-only sample in its plugin. ): # avoid process hanging in zero3/fsdp case fake_messages = [{"role": "user", "content": IMAGE_PLACEHOLDER}] fake_images = [Image.new("RGB", (64, 64), (255, 255, 255))] @@ -416,7 +423,6 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq): features: dict[str, torch.Tensor] = super().__call__(features) bsz, seq_len = features["input_ids"].shape[:2] - model_type = getattr(self.model.config, "model_type", None) if self.model is not None else None is_omni = model_type in [ "qwen2_5_omni_thinker", "qwen3_omni_moe_thinker", @@ -461,12 +467,17 @@ class MultiModalDataCollatorForSeq2Seq(DataCollatorForSeq2Seq): ): raise ValueError(f"{self.model.config.model_type} requires 3D position ids for mrope.") - if "cross_attention_mask" in mm_inputs: # for mllama inputs when pad_to_multiple_of is enabled + if ( + "cross_attention_mask" in mm_inputs and mm_inputs["cross_attention_mask"].dtype != torch.bool + ): # for mllama inputs when pad_to_multiple_of is enabled cross_attention_mask = mm_inputs.pop("cross_attention_mask") seq_len = features["input_ids"].size(1) orig_len = cross_attention_mask.size(1) mm_inputs["cross_attention_mask"] = F.pad(cross_attention_mask, (0, 0, 0, 0, 0, seq_len - orig_len)) + if is_moss_vl: + mm_inputs = self.template.mm_plugin.post_process_mossvl_inputs(features, mm_inputs, self.processor) + features.update(mm_inputs) if "image_bound" in features: # for minicpmv inputs diff --git a/src/llamafactory/data/mm_plugin.py b/src/llamafactory/data/mm_plugin.py index 8c2d66c2f..10eefd3db 100644 --- a/src/llamafactory/data/mm_plugin.py +++ b/src/llamafactory/data/mm_plugin.py @@ -472,6 +472,354 @@ class BasePlugin(MMPluginMixin): return self._get_mm_inputs(images, videos, audios, processor) +@dataclass +class MossVLPlugin(BasePlugin): + vision_bos_token: str = "<|vision_start|>" + vision_eos_token: str = "<|vision_end|>" + time_bos_token: str = "<|time_start|>" + time_eos_token: str = "<|time_end|>" + + @staticmethod + def _split_pixel_values( + pixel_values: "torch.Tensor", + grid_thw: "torch.Tensor", + ) -> list["torch.Tensor"]: + patch_counts = [int(grid.prod().item()) for grid in grid_thw] + return list(torch.split(pixel_values, patch_counts)) + + @staticmethod + def _create_cross_attention_mask( + input_ids: Union[list[list[int]], "torch.Tensor"], + grid_thw: "torch.Tensor", + media_nums_per_sample: list[int], + image_token_id: int, + attention_mask: Optional["torch.Tensor"] = None, + padding_side: Literal["left", "right"] = "right", + ) -> "torch.Tensor": + r"""Create the native MOSS-VL frame-level causal cross-attention mask.""" + if isinstance(input_ids, list): + max_text_len = max(len(token_ids) for token_ids in input_ids) + input_ids_tensor = torch.full((len(input_ids), max_text_len), -1, dtype=torch.long) + attention_mask_tensor = torch.zeros_like(input_ids_tensor, dtype=torch.bool) + for batch_index, token_ids in enumerate(input_ids): + seq_len = len(token_ids) + start = max_text_len - seq_len if padding_side == "left" else 0 + input_ids_tensor[batch_index, start : start + seq_len] = torch.tensor(token_ids, dtype=torch.long) + attention_mask_tensor[batch_index, start : start + seq_len] = True + else: + input_ids_tensor = input_ids + attention_mask_tensor = ( + torch.ones_like(input_ids_tensor, dtype=torch.bool) + if attention_mask is None + else attention_mask.bool() + ) + + total_frames_per_sample = [] + media_index = 0 + for num_media in media_nums_per_sample: + sample_grid = grid_thw[media_index : media_index + num_media] + total_frames_per_sample.append(int(sample_grid[:, 0].sum().item())) + media_index += num_media + + max_num_frames = max(total_frames_per_sample) + frame_indices = torch.arange(max_num_frames, device=input_ids_tensor.device).view(1, 1, -1) + visible_mask = (input_ids_tensor == image_token_id).cumsum(dim=1).unsqueeze(-1) > frame_indices + visible_mask &= attention_mask_tensor.unsqueeze(-1) + valid_frames = frame_indices < torch.tensor( + total_frames_per_sample, + device=input_ids_tensor.device, + ).view(-1, 1, 1) + visible_mask &= valid_frames + return (~visible_mask).unsqueeze(1) + + def _get_video_inputs( + self, + videos: list["VideoInput"], + processor: "MMProcessor", + return_metadata: bool, + ) -> dict[str, Any]: + video_kwargs = {"return_tensors": "pt", "return_metadata": return_metadata} + if getattr(processor, "video_fps", None) is not None: + video_kwargs["video_fps"] = processor.video_fps + if getattr(processor, "video_maxlen", None) is not None: + video_kwargs["max_frames"] = processor.video_maxlen + + video_min_pixels = getattr(processor, "video_min_pixels", None) + video_max_pixels = getattr(processor, "video_max_pixels", None) + if video_min_pixels is not None and video_max_pixels is not None: + video_kwargs["size"] = { + "shortest_edge": video_min_pixels, + "longest_edge": video_max_pixels, + } + + return dict(processor.video_processor(videos=videos, **video_kwargs)) + + def _get_media_order_from_ids( + self, + input_ids: list[int], + processor: "MMProcessor", + num_images: int, + num_videos: int, + expected_video_frames: Optional[list[int]] = None, + ) -> list[str]: + media_order = [] + video_frame_counts = [] + in_video = False + current_video_frames = 0 + for token_id in input_ids: + if token_id == processor.vision_start_token_id: + if in_video: + raise ValueError( + "MOSS-VL encountered nested video token blocks after tokenization. " + "Please increase `cutoff_len` if a video placeholder was truncated." + ) + + media_order.append("video") + in_video = True + current_video_frames = 0 + elif token_id == processor.vision_end_token_id: + if not in_video: + raise ValueError( + "MOSS-VL encountered a video end token without a matching start token after tokenization. " + "Please increase `cutoff_len` if a video placeholder was truncated." + ) + + video_frame_counts.append(current_video_frames) + in_video = False + elif token_id == processor.image_token_id: + if in_video: + current_video_frames += 1 + else: + media_order.append("image") + + if in_video: + raise ValueError( + "MOSS-VL encountered an incomplete video token block after tokenization. " + "Please increase `cutoff_len` or reduce `video_maxlen`." + ) + + if media_order.count("image") != num_images or media_order.count("video") != num_videos: + raise ValueError( + "MOSS-VL media tokens do not match the provided media after tokenization: " + f"order={media_order}, images={num_images}, videos={num_videos}. " + "Please increase `cutoff_len` if a visual placeholder was truncated." + ) + + if expected_video_frames is not None and video_frame_counts != expected_video_frames: + raise ValueError( + "MOSS-VL video frame tokens do not match the processed video after tokenization: " + f"tokens={video_frame_counts}, frames={expected_video_frames}. " + "Please increase `cutoff_len` or reduce `video_maxlen`." + ) + + return media_order + + @override + def process_messages( + self, + messages: list[dict[str, str]], + images: list["ImageInput"], + videos: list["VideoInput"], + audios: list["AudioInput"], + processor: Optional["MMProcessor"], + ) -> list[dict[str, str]]: + self._validate_input(processor, images, videos, audios) + self._validate_messages(messages, images, videos, audios) + messages = deepcopy(messages) + + video_inputs = self._get_video_inputs(videos, processor, return_metadata=True) if videos else {} + video_grid_thw = video_inputs.get("video_grid_thw", []) + video_metadata = video_inputs.get("video_metadata", []) + video_index = 0 + for message in messages: + content = message["content"] + content = content.replace(IMAGE_PLACEHOLDER, self.image_token) + while VIDEO_PLACEHOLDER in content: + metadata = video_metadata[video_index] + if metadata.fps is None: + metadata.fps = 24 + + timestamps = processor._calculate_timestamps( + metadata.frames_indices, + metadata.total_num_frames, + metadata.fps, + metadata.duration, + processor.video_processor.temporal_patch_size, + actual_timestamps=getattr(metadata, "actual_timestamps", None), + ) + num_frames = int(video_grid_thw[video_index][0].item()) + frame_tokens = [ + f"{self.time_bos_token}{timestamps[frame_idx]:.1f} seconds{self.time_eos_token}{self.image_token}" + for frame_idx in range(num_frames) + ] + video_tokens = f"{self.vision_bos_token}{''.join(frame_tokens)}{self.vision_eos_token}" + content = content.replace(VIDEO_PLACEHOLDER, video_tokens, 1) + video_index += 1 + + message["content"] = content + + return messages + + @override + def get_mm_inputs( + self, + images: list["ImageInput"], + videos: list["VideoInput"], + audios: list["AudioInput"], + imglens: list[int], + vidlens: list[int], + audlens: list[int], + batch_ids: list[list[int]], + processor: Optional["MMProcessor"], + ) -> dict[str, Union[list[int], "torch.Tensor"]]: + self._validate_input(processor, images, videos, audios) + if audios: + raise ValueError("MOSS-VL does not support audio inputs.") + + if not (len(imglens) == len(vidlens) == len(batch_ids)): + raise ValueError("MOSS-VL batch metadata must have one entry per sample.") + final_pixel_values = [] + final_grid_thw = [] + media_nums_per_sample = [] + image_offset = 0 + video_offset = 0 + for imglen, vidlen, input_ids in zip(imglens, vidlens, batch_ids): + sample_images = images[image_offset : image_offset + imglen] + sample_videos = videos[video_offset : video_offset + vidlen] + image_offset += imglen + video_offset += vidlen + image_chunks, image_grids = [], [] + if sample_images: + regularized_images = self._regularize_images( + sample_images, + image_max_pixels=2**63 - 1, + image_min_pixels=1, + )["images"] + image_kwargs = {"return_tensors": "pt"} + if getattr(processor, "image_min_pixels", None) is not None: + image_kwargs["min_pixels"] = processor.image_min_pixels + if getattr(processor, "image_max_pixels", None) is not None: + image_kwargs["max_pixels"] = processor.image_max_pixels + + image_inputs = processor.image_processor(images=regularized_images, **image_kwargs) + image_grids = list(image_inputs["image_grid_thw"]) + image_chunks = self._split_pixel_values(image_inputs["pixel_values"], image_inputs["image_grid_thw"]) + + video_chunks, video_grids = [], [] + if sample_videos: + video_inputs = self._get_video_inputs(sample_videos, processor, return_metadata=False) + video_grids = list(video_inputs["video_grid_thw"]) + video_chunks = self._split_pixel_values( + video_inputs["pixel_values_videos"], + video_inputs["video_grid_thw"], + ) + + media_order = self._get_media_order_from_ids( + input_ids, + processor, + imglen, + vidlen, + expected_video_frames=[int(grid[0].item()) for grid in video_grids], + ) + + if not media_order: + patch_size = getattr(processor.image_processor, "patch_size", None) + if patch_size is None: # lightweight/test processors without the native MOSS-VL contract + blank_image = Image.new("RGB", (128, 128), (255, 255, 255)) + blank_inputs = processor.image_processor(images=[blank_image], return_tensors="pt") + final_pixel_values.append(blank_inputs["pixel_values"]) + final_grid_thw.append(blank_inputs["image_grid_thw"][0]) + else: + temporal_patch_size = getattr(processor.image_processor, "temporal_patch_size", None) or 1 + merge_size = getattr(processor.image_processor, "merge_size", None) or 2 + factor = patch_size * merge_size + side = math.ceil(128 / factor) * factor + grid_thw = torch.tensor([1, side // patch_size, side // patch_size]) + feature_dim = 3 * temporal_patch_size * patch_size * patch_size + final_pixel_values.append(torch.zeros((int(grid_thw.prod()), feature_dim), dtype=torch.float32)) + final_grid_thw.append(grid_thw) + media_nums_per_sample.append(1) + continue + + image_index = 0 + video_index = 0 + for modality in media_order: + if modality == "image": + final_pixel_values.append(image_chunks[image_index]) + final_grid_thw.append(image_grids[image_index]) + image_index += 1 + else: + final_pixel_values.append(video_chunks[video_index]) + final_grid_thw.append(video_grids[video_index]) + video_index += 1 + + media_nums_per_sample.append(len(media_order)) + + if image_offset != len(images) or video_offset != len(videos): + raise ValueError("MOSS-VL media lengths do not consume all provided inputs.") + + mm_inputs = { + "pixel_values": torch.cat(final_pixel_values, dim=0), + "grid_thw": torch.stack(final_grid_thw), + "media_nums_per_sample": media_nums_per_sample, + } + mm_inputs["cross_attention_mask"] = self._create_cross_attention_mask( + batch_ids, + mm_inputs["grid_thw"], + media_nums_per_sample, + processor.image_token_id, + padding_side=processor.tokenizer.padding_side, + ) + return mm_inputs + + def post_process_mossvl_inputs( + self, + features: dict[str, "torch.Tensor"], + mm_inputs: dict[str, Any], + processor: "MMProcessor", + ) -> dict[str, Any]: + r"""Create MOSS-VL batch-only inputs after the text batch has been padded.""" + input_ids = features["input_ids"] + attention_mask = features["attention_mask"].bool() + mm_inputs["cross_attention_mask"] = self._create_cross_attention_mask( + input_ids, + mm_inputs["grid_thw"], + mm_inputs["media_nums_per_sample"], + processor.image_token_id, + attention_mask, + ) + + dummy_image_tokens = (input_ids == processor.image_token_id) & ~attention_mask + input_ids.masked_fill_(dummy_image_tokens, processor.tokenizer.pad_token_id) + + labels = features.get("labels") + if labels is not None: + control_token_ids = { + processor.image_token_id, + processor.video_token_id, + processor.vision_start_token_id, + processor.vision_end_token_id, + processor.tokenizer.convert_tokens_to_ids(self.time_bos_token), + processor.tokenizer.convert_tokens_to_ids(self.time_eos_token), + } + for batch_index, token_ids in enumerate(input_ids): + in_vision = False + for token_index, token_id in enumerate(token_ids.tolist()): + if token_id == processor.vision_start_token_id: + in_vision = True + if in_vision or token_id in control_token_ids: + labels[batch_index, token_index] = IGNORE_INDEX + if token_id == processor.vision_end_token_id: + in_vision = False + + # Native MOSS-VL labels_spans supervise through <|im_end|>, but not its trailing newline. + im_end_token_id = processor.tokenizer.convert_tokens_to_ids("<|im_end|>") + labels[:, 1:].masked_fill_(input_ids[:, :-1] == im_end_token_id, IGNORE_INDEX) + + features.pop("position_ids", None) + return mm_inputs + + @dataclass class ErnieVLPlugin(BasePlugin): @override @@ -2911,6 +3259,7 @@ PLUGINS = { "minicpm_v": MiniCPMVPlugin, "minicpm_v_4_6": MiniCPMV4_6Plugin, "mllama": MllamaPlugin, + "moss_vl": MossVLPlugin, "paligemma": PaliGemmaPlugin, "pixtral": PixtralPlugin, "qwen2_audio": Qwen2AudioPlugin, diff --git a/src/llamafactory/data/template.py b/src/llamafactory/data/template.py index 492909eea..caa926abc 100644 --- a/src/llamafactory/data/template.py +++ b/src/llamafactory/data/template.py @@ -333,6 +333,50 @@ class Template: return modelfile +@dataclass +class MossVLTemplate(Template): + @override + def _encode( + self, + tokenizer: "PreTrainedTokenizer", + messages: list[dict[str, str]], + system: Optional[str], + tools: Optional[str], + ) -> list[list[int]]: + system = system or self.default_system + encoded_messages = [] + for i, message in enumerate(messages): + elements = [] + + if i == 0: + elements += self.format_prefix.apply() + if system or tools: + tool_text = self.format_tools.apply(content=tools)[0] if tools else "" + if tools and not system: + tool_text = tool_text.lstrip("\n") + + elements += self.format_system.apply(content=(system + tool_text)) + + if message["role"] == Role.USER: + elements += self.format_user.apply(content=message["content"], idx=str(i // 2)) + elif message["role"] == Role.ASSISTANT: + elements += self.format_assistant.apply(content=message["content"]) + elif message["role"] == Role.OBSERVATION: + elements += self.format_observation.apply(content=message["content"]) + elif message["role"] == Role.FUNCTION: + elements += self.format_function.apply( + content=message["content"], + thought_words=self.thought_words, + tool_call_words=self.tool_call_words, + ) + else: + raise NotImplementedError("Unexpected role: {}".format(message["role"])) + + encoded_messages.append(self._convert_elements_to_ids(tokenizer, elements)) + + return encoded_messages + + @dataclass class Llama2Template(Template): r"""A template that fuse the system message to first user message.""" @@ -1526,6 +1570,32 @@ register_template( ) +# copied from qwen template +register_template( + name="moss_vl", + format_user=StringFormatter(slots=["<|im_start|>user\n{{content}}<|im_end|>\n<|im_start|>assistant\n"]), + format_assistant=StringFormatter(slots=["{{content}}<|im_end|>\n"]), + format_system=StringFormatter(slots=["<|im_start|>system\n{{content}}<|im_end|>\n"]), + format_function=FunctionFormatter(slots=["{{content}}<|im_end|>\n"], tool_format="qwen"), + format_observation=StringFormatter( + slots=["<|im_start|>user\n\n{{content}}\n<|im_end|>\n<|im_start|>assistant\n"] + ), + format_tools=ToolFormatter(tool_format="qwen"), + stop_words=["<|im_end|>"], + replace_eos=True, + mm_plugin=get_mm_plugin( + name="moss_vl", + image_token="<|image_pad|>", + video_token="<|video_pad|>", + vision_bos_token="<|vision_start|>", + vision_eos_token="<|vision_end|>", + time_bos_token="<|time_start|>", + time_eos_token="<|time_end|>", + ), + template_class=MossVLTemplate, +) + + # copied from vicuna template register_template( name="llava", diff --git a/src/llamafactory/extras/constants.py b/src/llamafactory/extras/constants.py index 152a5681a..99662d482 100644 --- a/src/llamafactory/extras/constants.py +++ b/src/llamafactory/extras/constants.py @@ -2198,6 +2198,17 @@ register_model_group( ) +register_model_group( + models={ + "MOSS-VL-Instruct-0708": { + DownloadSource.DEFAULT: "OpenMOSS-Team/MOSS-VL-Instruct-0708", + }, + }, + template="moss_vl", + multimodal=True, +) + + register_model_group( models={ "OLMo-1B": { diff --git a/src/llamafactory/model/model_utils/visual.py b/src/llamafactory/model/model_utils/visual.py index d2f5bf2d5..9a5e80a98 100644 --- a/src/llamafactory/model/model_utils/visual.py +++ b/src/llamafactory/model/model_utils/visual.py @@ -56,7 +56,7 @@ class CompositeModel: ) break - if project_module is not None: + if isinstance(project_module, torch.nn.Module): mm_projectors.append(project_module) return mm_projectors @@ -344,6 +344,15 @@ _register_composite_model( ) +_register_composite_model( + model_type="moss_vl", + projector_keys=["model.visual.merger", "model.separator_token"], + vision_model_keys=["model.visual.pos_embed", "model.visual.patch_embed", "model.visual.blocks"], + language_model_keys=["model.language_model", "lm_head"], + lora_conflict_keys=["patch_embed"], +) + + _register_composite_model( model_type="mllama", vision_model_keys=["vision_model"], diff --git a/src/llamafactory/model/patcher.py b/src/llamafactory/model/patcher.py index c2e0ea21f..79c57603e 100644 --- a/src/llamafactory/model/patcher.py +++ b/src/llamafactory/model/patcher.py @@ -23,7 +23,7 @@ from transformers.modeling_utils import is_fsdp_enabled from transformers.utils import is_torch_cuda_available, is_torch_npu_available from ..extras import logging -from ..extras.misc import infer_optim_dtype +from ..extras.misc import check_version, infer_optim_dtype from ..extras.packages import is_transformers_version_greater_than from .model_utils.attention import configure_attn_implementation, print_attn_implementation from .model_utils.checkpointing import prepare_model_for_training @@ -418,6 +418,10 @@ def patch_config( "pip install git+https://github.com/huggingface/transformers.git@3c2517727ce28a30f5044e01663ee204deb1cdbe" ) + if getattr(config, "model_type", None) == "moss_vl": + check_version("transformers==4.57.1", mandatory=True) + check_version("torchcodec==0.7.0", mandatory=True) + if getattr(config, "model_type", None) == "qwen3_omni_moe": patch_qwen3_omni_moe_thinker_text_sparse_moe_block() diff --git a/tests/data/test_mm_plugin.py b/tests/data/test_mm_plugin.py index aabf1fbd7..73225ebaa 100644 --- a/tests/data/test_mm_plugin.py +++ b/tests/data/test_mm_plugin.py @@ -13,6 +13,7 @@ # limitations under the License. import os +from types import SimpleNamespace from typing import TYPE_CHECKING, Any import numpy as np @@ -417,6 +418,24 @@ def test_qwen2_vl_plugin(): _check_plugin(**check_inputs) +def test_moss_vl_plugin(): + messages = [ + {"role": "user", "content": "First , finally ."}, + {"role": "assistant", "content": "Done."}, + ] + expected_messages = [ + {"role": "user", "content": "First <|image_pad|>, finally <|image_pad|>."}, + {"role": "assistant", "content": "Done."}, + ] + processor = SimpleNamespace(image_processor=object(), video_processor=object()) + plugin = get_mm_plugin(name="moss_vl", image_token="<|image_pad|>", video_token="<|video_pad|>") + + processed_messages = plugin.process_messages(messages, [object(), object()], [], [], processor) + + assert processed_messages == expected_messages + assert messages[0]["content"] == "First , finally ." + + @pytest.mark.runs_on(["cpu", "mps"]) @pytest.mark.skipif(not is_transformers_version_greater_than("4.57.0"), reason="Requires transformers>=4.57.0") def test_qwen3_vl_plugin(): diff --git a/tests/data/test_moss_vl_plugin.py b/tests/data/test_moss_vl_plugin.py new file mode 100644 index 000000000..ef9ff9ed1 --- /dev/null +++ b/tests/data/test_moss_vl_plugin.py @@ -0,0 +1,631 @@ +# Copyright 2025 the LlamaFactory team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from types import SimpleNamespace + +import pytest +import torch +from PIL import Image + +from llamafactory.data.collator import MultiModalDataCollatorForSeq2Seq +from llamafactory.data.mm_plugin import get_mm_plugin +from llamafactory.data.processor.supervised import SupervisedDatasetProcessor +from llamafactory.extras.constants import IGNORE_INDEX + + +IMAGE_TOKEN_ID = 101 +VIDEO_TOKEN_ID = 102 +VISION_START_TOKEN_ID = 103 +VISION_END_TOKEN_ID = 104 +TIME_START_TOKEN_ID = 105 +TIME_END_TOKEN_ID = 106 +IM_END_TOKEN_ID = 107 + + +class _ImageProcessor: + def __init__(self): + self.calls = [] + + def __call__(self, images, return_tensors, **kwargs): + self.calls.append({"return_tensors": return_tensors, **kwargs}) + values = [] + for image in images: + marker = image.getpixel((0, 0))[0] + 1 + values.append(torch.full((1, 3), marker, dtype=torch.float32)) + + return { + "pixel_values": torch.cat(values), + "image_grid_thw": torch.tensor([[1, 1, 1]] * len(images)), + } + + +class _VideoProcessor: + temporal_patch_size = 1 + + def __init__(self): + self.calls = [] + + def __call__(self, videos, return_tensors, return_metadata, **kwargs): + self.calls.append( + { + "return_tensors": return_tensors, + "return_metadata": return_metadata, + **kwargs, + } + ) + result = { + "pixel_values_videos": torch.cat( + [torch.full((2, 3), 9 + index, dtype=torch.float32) for index in range(len(videos))] + ), + "video_grid_thw": torch.tensor([[2, 1, 1]] * len(videos)), + } + if return_metadata: + result["video_metadata"] = [ + SimpleNamespace(frames_indices=[0, 2], total_num_frames=2, fps=2.0, duration=2.0) for _ in videos + ] + + return result + + +class _Tokenizer: + pad_token_id = 0 + padding_side = "right" + _token_ids = { + "<|time_start|>": TIME_START_TOKEN_ID, + "<|time_end|>": TIME_END_TOKEN_ID, + "<|im_end|>": IM_END_TOKEN_ID, + } + + def convert_tokens_to_ids(self, token): + return self._token_ids[token] + + def pad(self, features, padding, max_length, pad_to_multiple_of, return_tensors): + del padding, max_length, return_tensors + sequence_length = max(len(feature["input_ids"]) for feature in features) + if pad_to_multiple_of is not None: + sequence_length = ((sequence_length + pad_to_multiple_of - 1) // pad_to_multiple_of) * pad_to_multiple_of + + padded = {"input_ids": [], "attention_mask": []} + for feature in features: + pad_length = sequence_length - len(feature["input_ids"]) + if self.padding_side == "right": + padded["input_ids"].append(feature["input_ids"] + [self.pad_token_id] * pad_length) + padded["attention_mask"].append(feature["attention_mask"] + [0] * pad_length) + else: + padded["input_ids"].append([self.pad_token_id] * pad_length + feature["input_ids"]) + padded["attention_mask"].append([0] * pad_length + feature["attention_mask"]) + + return {key: torch.tensor(value) for key, value in padded.items()} + + +class _Processor: + image_token_id = IMAGE_TOKEN_ID + video_token_id = VIDEO_TOKEN_ID + vision_start_token_id = VISION_START_TOKEN_ID + vision_end_token_id = VISION_END_TOKEN_ID + + def __init__(self): + self.image_processor = _ImageProcessor() + self.video_processor = _VideoProcessor() + self.tokenizer = _Tokenizer() + + @staticmethod + def _calculate_timestamps(*args, **kwargs): + del args, kwargs + return [0.0, 1.0] + + +def _get_plugin(): + return get_mm_plugin( + name="moss_vl", + image_token="<|image_pad|>", + video_token="<|video_pad|>", + vision_bos_token="<|vision_start|>", + vision_eos_token="<|vision_end|>", + time_bos_token="<|time_start|>", + time_eos_token="<|time_end|>", + ) + + +def _video_ids(seed): + return [ + VISION_START_TOKEN_ID, + TIME_START_TOKEN_ID, + seed, + TIME_END_TOKEN_ID, + IMAGE_TOKEN_ID, + TIME_START_TOKEN_ID, + seed + 1, + TIME_END_TOKEN_ID, + IMAGE_TOKEN_ID, + VISION_END_TOKEN_ID, + ] + + +def _left_pad(sequences, pad_value): + max_len = max(map(len, sequences)) + return torch.tensor([[pad_value] * (max_len - len(sequence)) + sequence for sequence in sequences]) + + +def test_moss_vl_process_messages_expands_video_frames(): + plugin = _get_plugin() + processor = _Processor() + messages = [ + {"role": "user", "content": "First , then