From dd58ded73d049aa3143ee48c0c288195412594ce Mon Sep 17 00:00:00 2001 From: Darijan Gudelj Date: Mon, 5 Sep 2022 06:27:37 -0700 Subject: [PATCH] stats object counting time wrong Summary: On each call of the stats.update the object calculates current average iteration time by getting time elapsed from the time_start and then dividing it by the current number of steps. It saves the result to AverageMeter object which when queried returns the average of things saved, so the time is averaged twice which biases it towards the start value (which is often larger). Reviewed By: kjchalup Differential Revision: D39206989 fbshipit-source-id: ccab5233d7aaca1ac4fd626fb329b83c7c0d6af9 --- pytorch3d/implicitron/tools/stats.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pytorch3d/implicitron/tools/stats.py b/pytorch3d/implicitron/tools/stats.py index 8b7c0bcc..012ab54a 100644 --- a/pytorch3d/implicitron/tools/stats.py +++ b/pytorch3d/implicitron/tools/stats.py @@ -118,6 +118,7 @@ class Stats(object): self.plot_file = plot_file self.do_plot = do_plot self.hard_reset(epoch=epoch) + self._t_last_update = None @staticmethod def from_json_str(json_str): @@ -215,7 +216,6 @@ class Stats(object): self.it[stat_set] += 1 epoch = self.epoch - it = self.it[stat_set] for stat in self.log_vars: @@ -224,10 +224,11 @@ class Stats(object): if stat == "sec/it": # compute speed if time_start is None: - elapsed = 0.0 + time_per_it = 0.0 else: - elapsed = time.time() - time_start - time_per_it = float(elapsed) / float(it + 1) + now = time.time() + time_per_it = now - (self._t_last_update or time_start) + self._t_last_update = now val = time_per_it else: if stat in preds: