423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2025-12-16 11:50:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

support liger kernel

Browse Source 
Former-commit-id: 0f4e54abf6c5feb2329855a4047597ad5147720a
This commit is contained in:
hiyouga
2024-08-27 11:20:14 +08:00
parent 533cef8445
commit dd6c96b96d
8 changed files with 69 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/llamafactory/model/patcher.py
View File

@@ -27,6 +27,7 @@ from ..extras.misc import infer_optim_dtype
from .model_utils.attention import configure_attn_implementation, print_attn_implementation
from .model_utils.checkpointing import prepare_model_for_training
from .model_utils.embedding import resize_embedding_layer
from .model_utils.liger_kernel import configure_liger_kernel
from .model_utils.longlora import configure_longlora
from .model_utils.moe import add_z3_leaf_module, configure_moe
from .model_utils.packing import configure_packing
@@ -70,6 +71,7 @@ def patch_config(
configure_attn_implementation(config, model_args, is_trainable)
configure_rope(config, model_args, is_trainable)
configure_liger_kernel(config, model_args, is_trainable)
configure_longlora(config, model_args, is_trainable)
configure_quantization(config, tokenizer, model_args, init_kwargs)
configure_moe(config, model_args, is_trainable)