From 1b47415a2f584c168dcd0323f0ef28fdd5fa5e7f Mon Sep 17 00:00:00 2001 From: Chaoran Wei <77485245+wcrzlh@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:29:59 +0800 Subject: [PATCH] [train] Fix hyper parallel tail accumulation loss scaling (#10705) Co-authored-by: wcrzlh --- src/llamafactory/train/hyper_parallel/trainer.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/llamafactory/train/hyper_parallel/trainer.py b/src/llamafactory/train/hyper_parallel/trainer.py index 7f0b89070..1410073a5 100644 --- a/src/llamafactory/train/hyper_parallel/trainer.py +++ b/src/llamafactory/train/hyper_parallel/trainer.py @@ -361,7 +361,12 @@ class HyperParallelTrainer(CustomSeq2SeqTrainer): loss = loss.mean() if not getattr(self, "model_accepts_loss_kwargs", False) and getattr(self, "compute_loss_func", None) is None: - loss = loss / self.args.gradient_accumulation_steps + accumulation_steps = getattr( + self, + "current_gradient_accumulation_steps", + self.args.gradient_accumulation_steps, + ) + loss = loss / accumulation_steps self.accelerator.backward(loss)