From 100e9a42c6c09f8f7849b70d60f3da445fb2024b Mon Sep 17 00:00:00 2001 From: yyj Date: Wed, 9 Sep 2026 21:12:56 +0800 Subject: [PATCH] [KT] Support Kimi K2.5/2.6 LoRA fine-tuning (#10826) --- src/llamafactory/model/loader.py | 8 +++++++- src/llamafactory/model/model_utils/attention.py | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/llamafactory/model/loader.py b/src/llamafactory/model/loader.py index 394a05059..b162c8820 100644 --- a/src/llamafactory/model/loader.py +++ b/src/llamafactory/model/loader.py @@ -125,7 +125,13 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule": def load_config(model_args: "ModelArguments") -> "PretrainedConfig": r"""Load model config.""" init_kwargs = _get_init_kwargs(model_args) - return AutoConfig.from_pretrained(model_args.model_name_or_path, **init_kwargs) + config = AutoConfig.from_pretrained(model_args.model_name_or_path, **init_kwargs) + if model_args.use_kt: + from transformers.integrations.kt_artifacts import prepare_kt_pretrained_config + + prepare_kt_pretrained_config(config) + + return config def load_model( diff --git a/src/llamafactory/model/model_utils/attention.py b/src/llamafactory/model/model_utils/attention.py index 1edf3ebcd..f3ab25df6 100644 --- a/src/llamafactory/model/model_utils/attention.py +++ b/src/llamafactory/model/model_utils/attention.py @@ -95,6 +95,10 @@ def configure_attn_implementation(config: "PretrainedConfig", model_args: "Model if getattr(config, "model_type", None) == "internlm2": # special case for custom models setattr(config, "attn_implementation", requested_attn_implementation) + elif getattr(config, "model_type", None) == "kimi_k25": + setattr(config, "_attn_implementation", requested_attn_implementation) + setattr(config.vision_config, "_attn_implementation", requested_attn_implementation) + setattr(config.text_config, "_attn_implementation", requested_attn_implementation) elif getattr(config, "model_type", None) == "kimi_vl": setattr(config.vision_config, "_attn_implementation", requested_attn_implementation) setattr(config.text_config, "_attn_implementation", requested_attn_implementation)