From ec9580a1d40b307ab98d89d28f9926a6bc04628d Mon Sep 17 00:00:00 2001 From: Jeremy Reizenstein Date: Fri, 6 May 2022 08:31:19 -0700 Subject: [PATCH] test runner for eval_demo Summary: Create a test runner for the eval_demo code. Debugging this is useful for understanding datasets. Introduces an environment variable INTERACTIVE_TESTING for ignoring tests which are not intended for use in regular test runs. Reviewed By: shapovalov Differential Revision: D35964016 fbshipit-source-id: ab0f93aff66b6cfeca942b14466cf81f7feb2224 --- pytorch3d/implicitron/dataset/dataset_zoo.py | 5 +-- pytorch3d/implicitron/eval_demo.py | 4 ++ tests/common_testing.py | 9 +++++ tests/implicitron/common_resources.py | 38 +++++++++++++++---- tests/implicitron/test_dataset_visualize.py | 13 +++++-- tests/implicitron/test_eval_demo.py | 39 ++++++++++++++++++++ 6 files changed, 93 insertions(+), 15 deletions(-) create mode 100644 tests/implicitron/test_eval_demo.py diff --git a/pytorch3d/implicitron/dataset/dataset_zoo.py b/pytorch3d/implicitron/dataset/dataset_zoo.py index d93bfb0d..73e6c8b1 100644 --- a/pytorch3d/implicitron/dataset/dataset_zoo.py +++ b/pytorch3d/implicitron/dataset/dataset_zoo.py @@ -122,6 +122,8 @@ def dataset_zoo( category, f"eval_batches_{task}.json", ) + if path_manager is not None: + batch_indices_path = path_manager.get_local_path(batch_indices_path) if not os.path.isfile(batch_indices_path): # The batch indices file does not exist. # Most probably the user has not specified the root folder. @@ -156,15 +158,12 @@ def dataset_zoo( for dataset, subsets in set_names_mapping.items(): frame_file = os.path.join(dataset_root, category, "frame_annotations.jgz") - assert os.path.isfile(frame_file) sequence_file = os.path.join( dataset_root, category, "sequence_annotations.jgz" ) - assert os.path.isfile(sequence_file) subset_lists_file = os.path.join(dataset_root, category, "set_lists.json") - assert os.path.isfile(subset_lists_file) # TODO: maybe directly in param list params = { diff --git a/pytorch3d/implicitron/eval_demo.py b/pytorch3d/implicitron/eval_demo.py index 59d27ae3..53cdb734 100644 --- a/pytorch3d/implicitron/eval_demo.py +++ b/pytorch3d/implicitron/eval_demo.py @@ -12,6 +12,7 @@ from typing import cast, Optional import lpips import torch +from iopath.common.file_io import PathManager from pytorch3d.implicitron.dataset.dataloader_zoo import dataloader_zoo from pytorch3d.implicitron.dataset.dataset_zoo import CO3D_CATEGORIES, dataset_zoo from pytorch3d.implicitron.dataset.implicitron_dataset import ( @@ -79,6 +80,7 @@ def evaluate_dbir_for_category( task: str = "singlesequence", single_sequence_id: Optional[int] = None, num_workers: int = 16, + path_manager: Optional[PathManager] = None, ): """ Evaluates new view synthesis metrics of a simple depth-based image rendering @@ -90,6 +92,7 @@ def evaluate_dbir_for_category( task: Evaluation task. Either singlesequence or multisequence. single_sequence_id: The ID of the evaluiation sequence for the singlesequence task. num_workers: The number of workers for the employed dataloaders. + path_manager: (optional) Used for interpreting paths. Returns: category_result: A dictionary of quantitative metrics. @@ -110,6 +113,7 @@ def evaluate_dbir_for_category( test_on_train=False, load_point_clouds=True, test_restrict_sequence_id=single_sequence_id, + path_manager=path_manager, ) dataloaders = dataloader_zoo( diff --git a/tests/common_testing.py b/tests/common_testing.py index 6c5c7557..a83c8c22 100644 --- a/tests/common_testing.py +++ b/tests/common_testing.py @@ -15,6 +15,15 @@ import torch from PIL import Image +def interactive_testing_requested() -> bool: + """ + Certain tests are only useful when run interactively, and so are not regularly run. + These are activated by this funciton returning True, which the user requests by + setting the environment variable `PYTORCH3D_INTERACTIVE_TESTING`. + """ + return os.environ.get("PYTORCH3D_INTERACTIVE_TESTING", False) + + def get_tests_dir() -> Path: """ Returns Path for the directory containing this file. diff --git a/tests/implicitron/common_resources.py b/tests/implicitron/common_resources.py index 2c2620c1..0ddad5cd 100644 --- a/tests/implicitron/common_resources.py +++ b/tests/implicitron/common_resources.py @@ -16,6 +16,33 @@ from zipfile import ZipFile from iopath.common.file_io import PathManager +CO3D_MANIFOLD_PATH: str = "manifold://co3d/tree/extracted" + + +def get_path_manager(silence_logs: bool = False) -> PathManager: + """ + Returns a path manager which can access manifold internally. + + Args: + silence_logs: Whether to reduce log output from iopath library. + """ + if silence_logs: + logging.getLogger("iopath.fb.manifold").setLevel(logging.CRITICAL) + logging.getLogger("iopath.common.file_io").setLevel(logging.CRITICAL) + + if os.environ.get("INSIDE_RE_WORKER", False): + raise ValueError("Cannot get to manifold from RE") + + path_manager = PathManager() + + if os.environ.get("FB_TEST", False): + from iopath.fb.manifold import ManifoldPathHandler + + path_manager.register_handler(ManifoldPathHandler()) + + return path_manager + + @contextlib.contextmanager def get_skateboard_data( avoid_manifold: bool = False, silence_logs: bool = False @@ -34,7 +61,6 @@ def get_skateboard_data( dataset_root: (str) path to dataset root. path_manager: path_manager to access it with. """ - path_manager = PathManager() if silence_logs: logging.getLogger("iopath.fb.manifold").setLevel(logging.CRITICAL) logging.getLogger("iopath.common.file_io").setLevel(logging.CRITICAL) @@ -42,7 +68,7 @@ def get_skateboard_data( if not os.environ.get("FB_TEST", False): if os.getenv("FAIR_ENV_CLUSTER", "") == "": raise unittest.SkipTest("Unknown environment. Data not available.") - yield "/checkpoint/dnovotny/datasets/co3d/download_aws_22_02_18", path_manager + yield "/checkpoint/dnovotny/datasets/co3d/download_aws_22_02_18", PathManager() elif avoid_manifold or os.environ.get("INSIDE_RE_WORKER", False): from libfb.py.parutil import get_file_path @@ -53,13 +79,9 @@ def get_skateboard_data( with tempfile.TemporaryDirectory() as dest: with ZipFile(source) as f: f.extractall(dest) - yield os.path.join(dest, "extracted"), path_manager + yield os.path.join(dest, "extracted"), PathManager() else: - from iopath.fb.manifold import ManifoldPathHandler - - path_manager.register_handler(ManifoldPathHandler()) - - yield "manifold://co3d/tree/extracted", path_manager + yield CO3D_MANIFOLD_PATH, get_path_manager() def provide_lpips_vgg(): diff --git a/tests/implicitron/test_dataset_visualize.py b/tests/implicitron/test_dataset_visualize.py index a3b3e7d3..73c048ef 100644 --- a/tests/implicitron/test_dataset_visualize.py +++ b/tests/implicitron/test_dataset_visualize.py @@ -26,11 +26,16 @@ if os.environ.get("FB_TEST", False): else: from common_resources import get_skateboard_data +if os.environ.get("FB_TEST", False): + from common_testing import interactive_testing_requested +else: + from tests.common_testing import interactive_testing_requested + class TestDatasetVisualize(unittest.TestCase): def setUp(self): - if os.environ.get("INSIDE_RE_WORKER") is not None: - raise unittest.SkipTest("Visdom not available") + if not interactive_testing_requested(): + return category = "skateboard" stack = contextlib.ExitStack() dataset_root, path_manager = stack.enter_context(get_skateboard_data()) @@ -94,8 +99,8 @@ class TestDatasetVisualize(unittest.TestCase): def test_one(self): """Test dataset visualization.""" - if os.environ.get("INSIDE_RE_WORKER") is not None: - raise unittest.SkipTest("Visdom not available") + if not interactive_testing_requested(): + return for max_frames in (16, -1): for load_dataset_point_cloud in (True, False): for dataset_key in self.datasets: diff --git a/tests/implicitron/test_eval_demo.py b/tests/implicitron/test_eval_demo.py new file mode 100644 index 00000000..46e2efd2 --- /dev/null +++ b/tests/implicitron/test_eval_demo.py @@ -0,0 +1,39 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import os +import unittest + +from pytorch3d.implicitron import eval_demo + + +if os.environ.get("FB_TEST", False): + from .common_resources import CO3D_MANIFOLD_PATH, get_path_manager +else: + from common_resources import CO3D_MANIFOLD_PATH, get_path_manager + +if os.environ.get("FB_TEST", False): + from common_testing import interactive_testing_requested +else: + from tests.common_testing import interactive_testing_requested + +""" +This test runs a single sequence eval_demo, useful for debugging datasets. +It only runs interactively. +""" + + +class TestEvalDemo(unittest.TestCase): + def test_a(self): + if not interactive_testing_requested(): + return + + os.environ["CO3D_DATASET_ROOT"] = CO3D_MANIFOLD_PATH + path_manager = get_path_manager(silence_logs=True) + + eval_demo.evaluate_dbir_for_category( + "donut", single_sequence_id=0, path_manager=path_manager + )