423A35C7/LLaMA-Factory
1
0
Fork 0
mirror of https://github.com/hiyouga/LLaMA-Factory.git synced 2026-06-17 04:38:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feat] support HyperParallel PT training and activation optimization (#10370)

Browse Source
This commit is contained in:
Cui-yshoho
2026-06-02 22:39:32 +08:00
committed by GitHub
parent a98a1ef101
commit 053d43c0ac
5 changed files with 326 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/llamafactory/train/tuner.py
View File

@@ -88,12 +88,19 @@ def _training_function(config: dict[str, Any]) -> None:
callbacks.append(ReporterCallback(model_args, data_args, finetuning_args, generating_args)) # add to last
if finetuning_args.stage == "sft" and finetuning_args.use_hyper_parallel:
if finetuning_args.stage in ["pt", "sft"] and finetuning_args.use_hyper_parallel:
if not is_hyper_parallel_available():
raise ImportError("hyper_parallel is not installed. Please install it with `pip install hyper_parallel`.")
from .hyper_parallel import run_sft as run_sft_hp
raise ImportError(
"hyper_parallel is not installed. Please install it with `pip install hyper_parallel`."
)
if finetuning_args.stage == "pt":
from .hyper_parallel import run_pt as run_pt_hp
run_sft_hp(model_args, data_args, training_args, finetuning_args, generating_args, callbacks)
run_pt_hp(model_args, data_args, training_args, finetuning_args, callbacks)
else:
from .hyper_parallel import run_sft as run_sft_hp
run_sft_hp(model_args, data_args, training_args, finetuning_args, generating_args, callbacks)
elif finetuning_args.stage in ["pt", "sft", "dpo"] and finetuning_args.use_mca:
if not is_mcore_adapter_available():