test runner for eval_demo

Summary:
Create a test runner for the eval_demo code.  Debugging this is useful for understanding datasets.

Introduces an environment variable INTERACTIVE_TESTING for ignoring tests which are not intended for use in regular test runs.

Reviewed By: shapovalov

Differential Revision: D35964016

fbshipit-source-id: ab0f93aff66b6cfeca942b14466cf81f7feb2224

tests/common_testing.py

@@ -15,6 +15,15 @@ import torch
 from PIL import Image
 
 
+def interactive_testing_requested() -> bool:
+    """
+    Certain tests are only useful when run interactively, and so are not regularly run.
+    These are activated by this funciton returning True, which the user requests by
+    setting the environment variable `PYTORCH3D_INTERACTIVE_TESTING`.
+    """
+    return os.environ.get("PYTORCH3D_INTERACTIVE_TESTING", False)
+
+
 def get_tests_dir() -> Path:
     """
     Returns Path for the directory containing this file.

tests/implicitron/common_resources.py

@@ -16,6 +16,33 @@ from zipfile import ZipFile
 from iopath.common.file_io import PathManager
 
 
+CO3D_MANIFOLD_PATH: str = "manifold://co3d/tree/extracted"
+
+
+def get_path_manager(silence_logs: bool = False) -> PathManager:
+    """
+    Returns a path manager which can access manifold internally.
+
+    Args:
+        silence_logs: Whether to reduce log output from iopath library.
+    """
+    if silence_logs:
+        logging.getLogger("iopath.fb.manifold").setLevel(logging.CRITICAL)
+        logging.getLogger("iopath.common.file_io").setLevel(logging.CRITICAL)
+
+    if os.environ.get("INSIDE_RE_WORKER", False):
+        raise ValueError("Cannot get to manifold from RE")
+
+    path_manager = PathManager()
+
+    if os.environ.get("FB_TEST", False):
+        from iopath.fb.manifold import ManifoldPathHandler
+
+        path_manager.register_handler(ManifoldPathHandler())
+
+    return path_manager
+
+
 @contextlib.contextmanager
 def get_skateboard_data(
     avoid_manifold: bool = False, silence_logs: bool = False
@@ -34,7 +61,6 @@ def get_skateboard_data(
         dataset_root: (str) path to dataset root.
         path_manager: path_manager to access it with.
     """
-    path_manager = PathManager()
     if silence_logs:
         logging.getLogger("iopath.fb.manifold").setLevel(logging.CRITICAL)
         logging.getLogger("iopath.common.file_io").setLevel(logging.CRITICAL)
@@ -42,7 +68,7 @@ def get_skateboard_data(
     if not os.environ.get("FB_TEST", False):
         if os.getenv("FAIR_ENV_CLUSTER", "") == "":
             raise unittest.SkipTest("Unknown environment. Data not available.")
-        yield "/checkpoint/dnovotny/datasets/co3d/download_aws_22_02_18", path_manager
+        yield "/checkpoint/dnovotny/datasets/co3d/download_aws_22_02_18", PathManager()
 
     elif avoid_manifold or os.environ.get("INSIDE_RE_WORKER", False):
         from libfb.py.parutil import get_file_path
@@ -53,13 +79,9 @@ def get_skateboard_data(
         with tempfile.TemporaryDirectory() as dest:
             with ZipFile(source) as f:
                 f.extractall(dest)
-            yield os.path.join(dest, "extracted"), path_manager
+            yield os.path.join(dest, "extracted"), PathManager()
     else:
-        from iopath.fb.manifold import ManifoldPathHandler
-
-        path_manager.register_handler(ManifoldPathHandler())
-
-        yield "manifold://co3d/tree/extracted", path_manager
+        yield CO3D_MANIFOLD_PATH, get_path_manager()
 
 
 def provide_lpips_vgg():

tests/implicitron/test_dataset_visualize.py

@@ -26,11 +26,16 @@ if os.environ.get("FB_TEST", False):
 else:
     from common_resources import get_skateboard_data
 
+if os.environ.get("FB_TEST", False):
+    from common_testing import interactive_testing_requested
+else:
+    from tests.common_testing import interactive_testing_requested
+
 
 class TestDatasetVisualize(unittest.TestCase):
     def setUp(self):
-        if os.environ.get("INSIDE_RE_WORKER") is not None:
-            raise unittest.SkipTest("Visdom not available")
+        if not interactive_testing_requested():
+            return
         category = "skateboard"
         stack = contextlib.ExitStack()
         dataset_root, path_manager = stack.enter_context(get_skateboard_data())
@@ -94,8 +99,8 @@ class TestDatasetVisualize(unittest.TestCase):
 
     def test_one(self):
         """Test dataset visualization."""
-        if os.environ.get("INSIDE_RE_WORKER") is not None:
-            raise unittest.SkipTest("Visdom not available")
+        if not interactive_testing_requested():
+            return
         for max_frames in (16, -1):
             for load_dataset_point_cloud in (True, False):
                 for dataset_key in self.datasets:

tests/implicitron/test_eval_demo.py

@@ -0,0 +1,39 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import os
+import unittest
+
+from pytorch3d.implicitron import eval_demo
+
+
+if os.environ.get("FB_TEST", False):
+    from .common_resources import CO3D_MANIFOLD_PATH, get_path_manager
+else:
+    from common_resources import CO3D_MANIFOLD_PATH, get_path_manager
+
+if os.environ.get("FB_TEST", False):
+    from common_testing import interactive_testing_requested
+else:
+    from tests.common_testing import interactive_testing_requested
+
+"""
+This test runs a single sequence eval_demo, useful for debugging datasets.
+It only runs interactively.
+"""
+
+
+class TestEvalDemo(unittest.TestCase):
+    def test_a(self):
+        if not interactive_testing_requested():
+            return
+
+        os.environ["CO3D_DATASET_ROOT"] = CO3D_MANIFOLD_PATH
+        path_manager = get_path_manager(silence_logs=True)
+
+        eval_demo.evaluate_dbir_for_category(
+            "donut", single_sequence_id=0, path_manager=path_manager
+        )