test runner for eval_demo

Summary:
Create a test runner for the eval_demo code.  Debugging this is useful for understanding datasets.

Introduces an environment variable INTERACTIVE_TESTING for ignoring tests which are not intended for use in regular test runs.

Reviewed By: shapovalov

Differential Revision: D35964016

fbshipit-source-id: ab0f93aff66b6cfeca942b14466cf81f7feb2224

pytorch3d/implicitron/dataset/dataset_zoo.py

@@ -122,6 +122,8 @@ def dataset_zoo(
             category,
             f"eval_batches_{task}.json",
         )
+        if path_manager is not None:
+            batch_indices_path = path_manager.get_local_path(batch_indices_path)
         if not os.path.isfile(batch_indices_path):
             # The batch indices file does not exist.
             # Most probably the user has not specified the root folder.
@@ -156,15 +158,12 @@ def dataset_zoo(
 
         for dataset, subsets in set_names_mapping.items():
             frame_file = os.path.join(dataset_root, category, "frame_annotations.jgz")
-            assert os.path.isfile(frame_file)
 
             sequence_file = os.path.join(
                 dataset_root, category, "sequence_annotations.jgz"
             )
-            assert os.path.isfile(sequence_file)
 
             subset_lists_file = os.path.join(dataset_root, category, "set_lists.json")
-            assert os.path.isfile(subset_lists_file)
 
             # TODO: maybe directly in param list
             params = {

pytorch3d/implicitron/eval_demo.py

@@ -12,6 +12,7 @@ from typing import cast, Optional
 
 import lpips
 import torch
+from iopath.common.file_io import PathManager
 from pytorch3d.implicitron.dataset.dataloader_zoo import dataloader_zoo
 from pytorch3d.implicitron.dataset.dataset_zoo import CO3D_CATEGORIES, dataset_zoo
 from pytorch3d.implicitron.dataset.implicitron_dataset import (
@@ -79,6 +80,7 @@ def evaluate_dbir_for_category(
     task: str = "singlesequence",
     single_sequence_id: Optional[int] = None,
     num_workers: int = 16,
+    path_manager: Optional[PathManager] = None,
 ):
     """
     Evaluates new view synthesis metrics of a simple depth-based image rendering
@@ -90,6 +92,7 @@ def evaluate_dbir_for_category(
         task: Evaluation task. Either singlesequence or multisequence.
         single_sequence_id: The ID of the evaluiation sequence for the singlesequence task.
         num_workers: The number of workers for the employed dataloaders.
+        path_manager: (optional) Used for interpreting paths.
 
     Returns:
         category_result: A dictionary of quantitative metrics.
@@ -110,6 +113,7 @@ def evaluate_dbir_for_category(
         test_on_train=False,
         load_point_clouds=True,
         test_restrict_sequence_id=single_sequence_id,
+        path_manager=path_manager,
     )
 
     dataloaders = dataloader_zoo(

tests/common_testing.py

@@ -15,6 +15,15 @@ import torch
 from PIL import Image
 
 
+def interactive_testing_requested() -> bool:
+    """
+    Certain tests are only useful when run interactively, and so are not regularly run.
+    These are activated by this funciton returning True, which the user requests by
+    setting the environment variable `PYTORCH3D_INTERACTIVE_TESTING`.
+    """
+    return os.environ.get("PYTORCH3D_INTERACTIVE_TESTING", False)
+
+
 def get_tests_dir() -> Path:
     """
     Returns Path for the directory containing this file.

tests/implicitron/common_resources.py

@@ -16,6 +16,33 @@ from zipfile import ZipFile
 from iopath.common.file_io import PathManager
 
 
+CO3D_MANIFOLD_PATH: str = "manifold://co3d/tree/extracted"
+
+
+def get_path_manager(silence_logs: bool = False) -> PathManager:
+    """
+    Returns a path manager which can access manifold internally.
+
+    Args:
+        silence_logs: Whether to reduce log output from iopath library.
+    """
+    if silence_logs:
+        logging.getLogger("iopath.fb.manifold").setLevel(logging.CRITICAL)
+        logging.getLogger("iopath.common.file_io").setLevel(logging.CRITICAL)
+
+    if os.environ.get("INSIDE_RE_WORKER", False):
+        raise ValueError("Cannot get to manifold from RE")
+
+    path_manager = PathManager()
+
+    if os.environ.get("FB_TEST", False):
+        from iopath.fb.manifold import ManifoldPathHandler
+
+        path_manager.register_handler(ManifoldPathHandler())
+
+    return path_manager
+
+
 @contextlib.contextmanager
 def get_skateboard_data(
     avoid_manifold: bool = False, silence_logs: bool = False
@@ -34,7 +61,6 @@ def get_skateboard_data(
         dataset_root: (str) path to dataset root.
         path_manager: path_manager to access it with.
     """
-    path_manager = PathManager()
     if silence_logs:
         logging.getLogger("iopath.fb.manifold").setLevel(logging.CRITICAL)
         logging.getLogger("iopath.common.file_io").setLevel(logging.CRITICAL)
@@ -42,7 +68,7 @@ def get_skateboard_data(
     if not os.environ.get("FB_TEST", False):
         if os.getenv("FAIR_ENV_CLUSTER", "") == "":
             raise unittest.SkipTest("Unknown environment. Data not available.")
-        yield "/checkpoint/dnovotny/datasets/co3d/download_aws_22_02_18", path_manager
+        yield "/checkpoint/dnovotny/datasets/co3d/download_aws_22_02_18", PathManager()
 
     elif avoid_manifold or os.environ.get("INSIDE_RE_WORKER", False):
         from libfb.py.parutil import get_file_path
@@ -53,13 +79,9 @@ def get_skateboard_data(
         with tempfile.TemporaryDirectory() as dest:
             with ZipFile(source) as f:
                 f.extractall(dest)
-            yield os.path.join(dest, "extracted"), path_manager
+            yield os.path.join(dest, "extracted"), PathManager()
     else:
-        from iopath.fb.manifold import ManifoldPathHandler
-
-        path_manager.register_handler(ManifoldPathHandler())
-
-        yield "manifold://co3d/tree/extracted", path_manager
+        yield CO3D_MANIFOLD_PATH, get_path_manager()
 
 
 def provide_lpips_vgg():

tests/implicitron/test_dataset_visualize.py

@@ -26,11 +26,16 @@ if os.environ.get("FB_TEST", False):
 else:
     from common_resources import get_skateboard_data
 
+if os.environ.get("FB_TEST", False):
+    from common_testing import interactive_testing_requested
+else:
+    from tests.common_testing import interactive_testing_requested
+
 
 class TestDatasetVisualize(unittest.TestCase):
     def setUp(self):
-        if os.environ.get("INSIDE_RE_WORKER") is not None:
-            raise unittest.SkipTest("Visdom not available")
+        if not interactive_testing_requested():
+            return
         category = "skateboard"
         stack = contextlib.ExitStack()
         dataset_root, path_manager = stack.enter_context(get_skateboard_data())
@@ -94,8 +99,8 @@ class TestDatasetVisualize(unittest.TestCase):
 
     def test_one(self):
         """Test dataset visualization."""
-        if os.environ.get("INSIDE_RE_WORKER") is not None:
-            raise unittest.SkipTest("Visdom not available")
+        if not interactive_testing_requested():
+            return
         for max_frames in (16, -1):
             for load_dataset_point_cloud in (True, False):
                 for dataset_key in self.datasets:

tests/implicitron/test_eval_demo.py

@@ -0,0 +1,39 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+import unittest
+
+from pytorch3d.implicitron import eval_demo
+
+
+if os.environ.get("FB_TEST", False):
+    from .common_resources import CO3D_MANIFOLD_PATH, get_path_manager
+else:
+    from common_resources import CO3D_MANIFOLD_PATH, get_path_manager
+
+if os.environ.get("FB_TEST", False):
+    from common_testing import interactive_testing_requested
+else:
+    from tests.common_testing import interactive_testing_requested
+
+"""
+This test runs a single sequence eval_demo, useful for debugging datasets.
+It only runs interactively.
+"""
+
+
+class TestEvalDemo(unittest.TestCase):
+    def test_a(self):
+        if not interactive_testing_requested():
+            return
+
+        os.environ["CO3D_DATASET_ROOT"] = CO3D_MANIFOLD_PATH
+        path_manager = get_path_manager(silence_logs=True)
+
+        eval_demo.evaluate_dbir_for_category(
+            "donut", single_sequence_id=0, path_manager=path_manager
+        )