423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-08-22 13:42:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update visual.py

Browse Source 
Former-commit-id: cbeef2aaea0577fd1929e7f156a2b8601b31814e
This commit is contained in:
hoshi-hiyouga 2024-05-15 16:39:57 +08:00 committed by GitHub
parent e09d68985f
commit e80e50805c
1 changed files with 24 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
src/llmtuner/model/utils/visual.py
View File

@ -1,8 +1,8 @@
from typing import TYPE_CHECKING, Tuple from typing import TYPE_CHECKING, Tuple
import torch import torch
import transformers import transformers.models
from torch import nn from transformers.activations import ACT2FN
from ...extras.logging import get_logger from ...extras.logging import get_logger
@ -16,9 +16,23 @@ if TYPE_CHECKING:
logger = get_logger(__name__) logger = get_logger(__name__)
def configure_hidden_size(config: "PretrainedConfig") -> None: class LlavaMultiModalProjector(torch.nn.Module):
if getattr(config, "model_type", None) == "llava": def __init__(self, config: "LlavaConfig"):
setattr(config, "hidden_size", getattr(config.text_config, "hidden_size", None)) super().__init__()
self.linear_1 = torch.nn.Linear(config.vision_config.hidden_size, config.text_config.hidden_size, bias=True)
self.linear_2 = torch.nn.LayerNorm(config.text_config.hidden_size, bias=True)
self.linear_3 = torch.nn.Linear(config.text_config.hidden_size, config.text_config.hidden_size, bias=True)
self.linear_4 = torch.nn.LayerNorm(config.text_config.hidden_size, bias=True)
self.act = ACT2FN[config.projector_hidden_act]
def forward(self, image_features):
hidden_states = self.linear_1(image_features)
hidden_states = self.linear_2(hidden_states)
hidden_states = self.act(hidden_states)
hidden_states = self.linear_3(hidden_states)
hidden_states = self.linear_4(hidden_states)
return hidden_states
def autocast_projector_dtype( def autocast_projector_dtype(
@ -35,28 +49,9 @@ def autocast_projector_dtype(
mm_projector.register_forward_hook(_mm_projector_forward_post_hook) mm_projector.register_forward_hook(_mm_projector_forward_post_hook)
class LlavaMultiModalProjectorYiVL(nn.Module): def configure_visual_model(config: "PretrainedConfig") -> None:
def __init__(self, config: "LlavaConfig"): if getattr(config, "model_type", None) == "llava":
super().__init__() setattr(config, "hidden_size", getattr(config.text_config, "hidden_size", None))
self.linear_1 = nn.Linear(config.vision_config.hidden_size, config.text_config.hidden_size, bias=True)
self.linear_2 = nn.LayerNorm(config.text_config.hidden_size, bias=True)
self.linear_3 = nn.Linear(config.text_config.hidden_size, config.text_config.hidden_size, bias=True)
self.linear_4 = nn.LayerNorm(config.text_config.hidden_size, bias=True)
self.act = nn.GELU()
def forward(self, image_features): if getattr(config, "is_yi_vl_derived_model", None):
dtype_ = self.linear_1.weight.dtype transformers.models.llava.modeling_llava.LlavaMultiModalProjector = LlavaMultiModalProjector
hidden_states = self.linear_1(image_features)
hidden_states = self.linear_2(hidden_states)
hidden_states = self.act(hidden_states)
hidden_states = self.linear_3(hidden_states)
hidden_states = self.linear_4(hidden_states)
hidden_states = hidden_states.to(dtype_)
return hidden_states
def configure_visual(config: "PretrainedConfig", model_args: "ModelArguments") -> None:
logger = get_logger(__name__)
if model_args.visual_inputs and "Yi" in getattr(config.text_config, "_name_or_path", None):
transformers.models.llava.modeling_llava.LlavaMultiModalProjector = LlavaMultiModalProjectorYiVL
logger.info("Patched Multimodal Projector for Yi-VL.")