423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2026-09-14 11:15:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

[KT] Support Kimi K2.5/2.6 LoRA fine-tuning (#10826)

Browse Source
This commit is contained in:
yyj
2026-09-09 21:12:56 +08:00
committed by GitHub
parent 31078aa10a
commit 100e9a42c6
2 changed files with 11 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/llamafactory/model/loader.py
View File

@@ -125,7 +125,13 @@ def load_tokenizer(model_args: "ModelArguments") -> "TokenizerModule":
def load_config(model_args: "ModelArguments") -> "PretrainedConfig": def load_config(model_args: "ModelArguments") -> "PretrainedConfig":
r"""Load model config.""" r"""Load model config."""
init_kwargs = _get_init_kwargs(model_args) init_kwargs = _get_init_kwargs(model_args)
return AutoConfig.from_pretrained(model_args.model_name_or_path, **init_kwargs) config = AutoConfig.from_pretrained(model_args.model_name_or_path, **init_kwargs)
if model_args.use_kt:
from transformers.integrations.kt_artifacts import prepare_kt_pretrained_config
prepare_kt_pretrained_config(config)
return config
def load_model( def load_model(

4
src/llamafactory/model/model_utils/attention.py
View File

@@ -95,6 +95,10 @@ def configure_attn_implementation(config: "PretrainedConfig", model_args: "Model
if getattr(config, "model_type", None) == "internlm2": # special case for custom models if getattr(config, "model_type", None) == "internlm2": # special case for custom models
setattr(config, "attn_implementation", requested_attn_implementation) setattr(config, "attn_implementation", requested_attn_implementation)
elif getattr(config, "model_type", None) == "kimi_k25":
setattr(config, "_attn_implementation", requested_attn_implementation)
setattr(config.vision_config, "_attn_implementation", requested_attn_implementation)
setattr(config.text_config, "_attn_implementation", requested_attn_implementation)
elif getattr(config, "model_type", None) == "kimi_vl": elif getattr(config, "model_type", None) == "kimi_vl":
setattr(config.vision_config, "_attn_implementation", requested_attn_implementation) setattr(config.vision_config, "_attn_implementation", requested_attn_implementation)
setattr(config.text_config, "_attn_implementation", requested_attn_implementation) setattr(config.text_config, "_attn_implementation", requested_attn_implementation)