diff --git a/README.md b/README.md index 4b4e6c18..6ac73e20 100644 --- a/README.md +++ b/README.md @@ -280,7 +280,7 @@ Choose your path: | [Hunyuan](https://huggingface.co/tencent/) | 7B | hunyuan | | [Index](https://huggingface.co/IndexTeam) | 1.9B | index | | [InternLM 2-3](https://huggingface.co/internlm) | 7B/8B/20B | intern2 | -| [InternVL 2.5-3](https://huggingface.co/OpenGVLab) | 1B/2B/8B/14B/38B/78B | intern_vl | +| [InternVL 2.5-3.5](https://huggingface.co/OpenGVLab) | 1B/2B/4B/8B/14B/30B/38B/78B/241B | intern_vl | | [InternLM/Intern-S1-mini](https://huggingface.co/internlm/) | 8B | intern_s1 | | [Kimi-VL](https://huggingface.co/moonshotai) | 16B | kimi_vl | | [Llama](https://github.com/facebookresearch/llama) | 7B/13B/33B/65B | - | diff --git a/README_zh.md b/README_zh.md index 9ec81bd9..7468aa17 100644 --- a/README_zh.md +++ b/README_zh.md @@ -282,7 +282,7 @@ https://github.com/user-attachments/assets/43b700c6-a178-41db-b1f8-8190a5d3fcfc | [Hunyuan](https://huggingface.co/tencent/) | 7B | hunyuan | | [Index](https://huggingface.co/IndexTeam) | 1.9B | index | | [InternLM 2-3](https://huggingface.co/internlm) | 7B/8B/20B | intern2 | -| [InternVL 2.5-3](https://huggingface.co/OpenGVLab) | 1B/2B/8B/14B/38B/78B | intern_vl | +| [InternVL 2.5-3.5](https://huggingface.co/OpenGVLab) | 1B/2B/4B/8B/14B/30B/38B/78B/241B | intern_vl | | [InternLM/Intern-S1-mini](https://huggingface.co/internlm/) | 8B | intern_s1 | | [Kimi-VL](https://huggingface.co/moonshotai) | 16B | kimi_vl | | [Llama](https://github.com/facebookresearch/llama) | 7B/13B/33B/65B | - | diff --git a/src/llamafactory/extras/constants.py b/src/llamafactory/extras/constants.py index b7725a8b..822b8598 100644 --- a/src/llamafactory/extras/constants.py +++ b/src/llamafactory/extras/constants.py @@ -1260,6 +1260,18 @@ register_model_group( DownloadSource.DEFAULT: "OpenGVLab/InternVL3-78B-hf", DownloadSource.MODELSCOPE: "OpenGVLab/InternVL3-78B-hf", }, + "InternVL3_5-1B-hf": { + DownloadSource.DEFAULT: "llamafactory/InternVL3_5-1B-hf", + }, + "InternVL3_5-2B-hf": { + DownloadSource.DEFAULT: "llamafactory/InternVL3_5-2B-hf", + }, + "InternVL3_5-4B-hf": { + DownloadSource.DEFAULT: "llamafactory/InternVL3_5-4B-hf", + }, + "InternVL3_5-8B-hf": { + DownloadSource.DEFAULT: "llamafactory/InternVL3_5-8B-hf", + }, }, template="intern_vl", multimodal=True, diff --git a/src/llamafactory/model/model_utils/moe.py b/src/llamafactory/model/model_utils/moe.py index 95990f2f..602c71fd 100644 --- a/src/llamafactory/model/model_utils/moe.py +++ b/src/llamafactory/model/model_utils/moe.py @@ -39,6 +39,9 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None: return model_type = getattr(model.config, "model_type", None) + text_config = getattr(model.config, "text_config", None) + text_architectures = getattr(text_config, "architectures", None) + if model_type == "dbrx": from transformers.models.dbrx.modeling_dbrx import DbrxFFN @@ -102,7 +105,7 @@ def add_z3_leaf_module(model: "PreTrainedModel") -> None: _set_z3_leaf_modules(model, [Qwen2MoeSparseMoeBlock]) - if model_type == "qwen3_moe": + if model_type == "qwen3_moe" or text_architectures == "Qwen3MoeForCausalLM": # for internvl_3_5 from transformers.models.qwen3_moe.modeling_qwen3_moe import Qwen3MoeSparseMoeBlock _set_z3_leaf_modules(model, [Qwen3MoeSparseMoeBlock]) @@ -113,6 +116,8 @@ def configure_moe(config: "PretrainedConfig", model_args: "ModelArguments", is_t return model_type = getattr(config, "model_type", None) + text_config = getattr(config, "text_config", None) # for multimodal model + if model_type in [ "dbrx", "granitemoe", @@ -127,9 +132,18 @@ def configure_moe(config: "PretrainedConfig", model_args: "ModelArguments", is_t ]: setattr(config, "output_router_logits", True) + if text_config and getattr(text_config, "model_type", None) in [ + "glm4v_moe_text", # glmv4_5 + "qwen3_moe", # internvl_3_5 + ]: + setattr(text_config, "output_router_logits", True) + if model_type in ["granitemoe", "jamba", "llama4", "mixtral", "olmoe", "phimoe", "qwen2_moe", "qwen3_moe"]: setattr(config, "router_aux_loss_coef", model_args.moe_aux_loss_coef) + elif text_config and getattr(text_config, "model_type", None) in ["qwen3_moe"]: + setattr(text_config, "router_aux_loss_coef", model_args.moe_aux_loss_coef) + elif model_type == "deepseek": setattr(config, "aux_loss_alpha", model_args.moe_aux_loss_coef)