pulsar interface unification.

Summary:
This diff builds on top of the `pulsar integration` diff to provide a unified interface for the existing PyTorch3D point renderer and Pulsar. For more information about the pulsar backend, see the release notes and the paper (https://arxiv.org/abs/2004.07484). For information on how to use the backend, see the point cloud rendering notebook and the examples in the folder docs/examples.

The unified interfaces are completely consistent. Switching the render backend is as easy as using `renderer = PulsarPointsRenderer(rasterizer=rasterizer).to(device)` instead of `renderer = PointsRenderer(rasterizer=rasterizer, compositor=compositor)` and adding the `gamma` parameter to the forward function. All PyTorch3D camera types are supported as far as possible; keyword arguments are properly forwarded to the camera. The `PerspectiveCamera` and `OrthographicCamera` require znear and zfar as additional parameters for the forward pass.

Reviewed By: nikhilaravi

Differential Revision: D21421443

fbshipit-source-id: 4aa0a83a419592d9a0bb5d62486a1cdea9d73ce6

tests/test_render_points.py

@@ -16,6 +16,8 @@ from PIL import Image
 from pytorch3d.renderer.cameras import (
     FoVOrthographicCameras,
     FoVPerspectiveCameras,
+    OrthographicCameras,
+    PerspectiveCameras,
     look_at_view_transform,
 )
 from pytorch3d.renderer.compositing import alpha_composite, norm_weighted_sum
@@ -25,6 +27,7 @@ from pytorch3d.renderer.points import (
     PointsRasterizationSettings,
     PointsRasterizer,
     PointsRenderer,
+    PulsarPointsRenderer,
 )
 from pytorch3d.structures.pointclouds import Pointclouds
 from pytorch3d.utils.ico_sphere import ico_sphere
@@ -72,6 +75,145 @@ class TestRenderPoints(TestCaseMixin, unittest.TestCase):
                 )
             self.assertClose(rgb, image_ref)
 
+    def test_simple_sphere_pulsar(self):
+        for device in [torch.device("cpu"), torch.device("cuda")]:
+            sphere_mesh = ico_sphere(1, device)
+            verts_padded = sphere_mesh.verts_padded()
+            # Shift vertices to check coordinate frames are correct.
+            verts_padded[..., 1] += 0.2
+            verts_padded[..., 0] += 0.2
+            pointclouds = Pointclouds(
+                points=verts_padded, features=torch.ones_like(verts_padded)
+            )
+            for azimuth in [0.0, 90.0]:
+                R, T = look_at_view_transform(2.7, 0.0, azimuth)
+                for camera_name, cameras in [
+                    ("fovperspective", FoVPerspectiveCameras(device=device, R=R, T=T)),
+                    (
+                        "fovorthographic",
+                        FoVOrthographicCameras(device=device, R=R, T=T),
+                    ),
+                    ("perspective", PerspectiveCameras(device=device, R=R, T=T)),
+                    ("orthographic", OrthographicCameras(device=device, R=R, T=T)),
+                ]:
+                    raster_settings = PointsRasterizationSettings(
+                        image_size=256, radius=5e-2, points_per_pixel=1
+                    )
+                    rasterizer = PointsRasterizer(
+                        cameras=cameras, raster_settings=raster_settings
+                    )
+                    renderer = PulsarPointsRenderer(rasterizer=rasterizer).to(device)
+                    # Load reference image
+                    filename = (
+                        "pulsar_simple_pointcloud_sphere_"
+                        f"azimuth{azimuth}_{camera_name}.png"
+                    )
+                    image_ref = load_rgb_image("test_%s" % filename, DATA_DIR)
+                    images = renderer(
+                        pointclouds, gamma=(1e-3,), znear=(1.0,), zfar=(100.0,)
+                    )
+                    rgb = images[0, ..., :3].squeeze().cpu()
+                    if DEBUG:
+                        filename = "DEBUG_%s" % filename
+                        Image.fromarray((rgb.numpy() * 255).astype(np.uint8)).save(
+                            DATA_DIR / filename
+                        )
+                    self.assertClose(rgb, image_ref, rtol=7e-3, atol=5e-3)
+
+    def test_unified_inputs_pulsar(self):
+        # Test data on different devices.
+        for device in [torch.device("cpu"), torch.device("cuda")]:
+            sphere_mesh = ico_sphere(1, device)
+            verts_padded = sphere_mesh.verts_padded()
+            pointclouds = Pointclouds(
+                points=verts_padded, features=torch.ones_like(verts_padded)
+            )
+            R, T = look_at_view_transform(2.7, 0.0, 0.0)
+            # Test the different camera types.
+            for _, cameras in [
+                ("fovperspective", FoVPerspectiveCameras(device=device, R=R, T=T)),
+                (
+                    "fovorthographic",
+                    FoVOrthographicCameras(device=device, R=R, T=T),
+                ),
+                ("perspective", PerspectiveCameras(device=device, R=R, T=T)),
+                ("orthographic", OrthographicCameras(device=device, R=R, T=T)),
+            ]:
+                # Test different ways for image size specification.
+                for image_size in (256, (256, 256)):
+                    raster_settings = PointsRasterizationSettings(
+                        image_size=image_size, radius=5e-2, points_per_pixel=1
+                    )
+                    rasterizer = PointsRasterizer(
+                        cameras=cameras, raster_settings=raster_settings
+                    )
+                    # Test that the compositor can be provided. It's value is ignored
+                    # so use a dummy.
+                    _ = PulsarPointsRenderer(rasterizer=rasterizer, compositor=1).to(
+                        device
+                    )
+                    # Constructor without compositor.
+                    _ = PulsarPointsRenderer(rasterizer=rasterizer).to(device)
+                    # Constructor with n_channels.
+                    _ = PulsarPointsRenderer(rasterizer=rasterizer, n_channels=3).to(
+                        device
+                    )
+                    # Constructor with max_num_spheres.
+                    renderer = PulsarPointsRenderer(
+                        rasterizer=rasterizer, max_num_spheres=1000
+                    ).to(device)
+                    # Test the forward function.
+                    if isinstance(cameras, (PerspectiveCameras, OrthographicCameras)):
+                        # znear and zfar is required in this case.
+                        self.assertRaises(
+                            ValueError,
+                            lambda: renderer.forward(
+                                point_clouds=pointclouds, gamma=(1e-4,)
+                            ),
+                        )
+                        renderer.forward(
+                            point_clouds=pointclouds,
+                            gamma=(1e-4,),
+                            znear=(1.0,),
+                            zfar=(2.0,),
+                        )
+                        # znear and zfar must be batched.
+                        self.assertRaises(
+                            TypeError,
+                            lambda: renderer.forward(
+                                point_clouds=pointclouds,
+                                gamma=(1e-4,),
+                                znear=1.0,
+                                zfar=(2.0,),
+                            ),
+                        )
+                        self.assertRaises(
+                            TypeError,
+                            lambda: renderer.forward(
+                                point_clouds=pointclouds,
+                                gamma=(1e-4,),
+                                znear=(1.0,),
+                                zfar=2.0,
+                            ),
+                        )
+                    else:
+                        # gamma must be batched.
+                        self.assertRaises(
+                            TypeError,
+                            lambda: renderer.forward(
+                                point_clouds=pointclouds, gamma=1e-4
+                            ),
+                        )
+                        renderer.forward(point_clouds=pointclouds, gamma=(1e-4,))
+                        # rasterizer width and height change.
+                        renderer.rasterizer.raster_settings.image_size = 0
+                        self.assertRaises(
+                            ValueError,
+                            lambda: renderer.forward(
+                                point_clouds=pointclouds, gamma=(1e-4,)
+                            ),
+                        )
+
     def test_pointcloud_with_features(self):
         device = torch.device("cuda:0")
         file_dir = Path(__file__).resolve().parent.parent / "docs/tutorials/data"