From cff4876131e79b7c2ad0c13fc875292d9dc29f8c Mon Sep 17 00:00:00 2001 From: Jeremy Reizenstein Date: Tue, 7 Dec 2021 15:02:46 -0800 Subject: [PATCH] add from_ndc to unproject_points Summary: Give unproject_points an argument letting it bypass screen space. use it to let the raysampler work for cameras defined in screen space. Reviewed By: gkioxari Differential Revision: D32596600 fbshipit-source-id: 2fe585dcd138cdbc65dd1c70e1957fd894512d3d --- pytorch3d/renderer/cameras.py | 40 ++++++++++++++++++++-- pytorch3d/renderer/implicit/raysampling.py | 2 +- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/pytorch3d/renderer/cameras.py b/pytorch3d/renderer/cameras.py index 28dc1369..9038fa48 100644 --- a/pytorch3d/renderer/cameras.py +++ b/pytorch3d/renderer/cameras.py @@ -129,6 +129,10 @@ class CamerasBase(TensorProperties): coordinates using the camera extrinsics `R` and `T`. `False` ignores `R` and `T` and unprojects to the camera view coordinates. + from_ndc: If `False` (default), assumes xy part of input is in + NDC space if self.in_ndc(), otherwise in screen space. If + `True`, assumes xy is in NDC space even if the camera + is defined in screen space. Returns new_points: unprojected points with the same shape as `xy_depth`. @@ -998,12 +1002,27 @@ class PerspectiveCameras(CamerasBase): return transform def unproject_points( - self, xy_depth: torch.Tensor, world_coordinates: bool = True, **kwargs + self, + xy_depth: torch.Tensor, + world_coordinates: bool = True, + from_ndc: bool = False, + **kwargs ) -> torch.Tensor: + """ + Args: + from_ndc: If `False` (default), assumes xy part of input is in + NDC space if self.in_ndc(), otherwise in screen space. If + `True`, assumes xy is in NDC space even if the camera + is defined in screen space. + """ if world_coordinates: to_camera_transform = self.get_full_projection_transform(**kwargs) else: to_camera_transform = self.get_projection_transform(**kwargs) + if from_ndc: + to_camera_transform = to_camera_transform.compose( + self.get_ndc_camera_transform() + ) unprojection_transform = to_camera_transform.inverse() xy_inv_depth = torch.cat( @@ -1030,6 +1049,8 @@ class PerspectiveCameras(CamerasBase): For cameras defined in screen space, we adjust the principal point computation which is defined in the image space (commonly) and scale the points to NDC space. + This transform leaves the depth unchanged. + Important: This transforms assumes PyTorch3D conventions for the input points, i.e. +X left, +Y up. """ @@ -1199,12 +1220,27 @@ class OrthographicCameras(CamerasBase): return transform def unproject_points( - self, xy_depth: torch.Tensor, world_coordinates: bool = True, **kwargs + self, + xy_depth: torch.Tensor, + world_coordinates: bool = True, + from_ndc: bool = False, + **kwargs ) -> torch.Tensor: + """ + Args: + from_ndc: If `False` (default), assumes xy part of input is in + NDC space if self.in_ndc(), otherwise in screen space. If + `True`, assumes xy is in NDC space even if the camera + is defined in screen space. + """ if world_coordinates: to_camera_transform = self.get_full_projection_transform(**kwargs) else: to_camera_transform = self.get_projection_transform(**kwargs) + if from_ndc: + to_camera_transform = to_camera_transform.compose( + self.get_ndc_camera_transform() + ) unprojection_transform = to_camera_transform.inverse() return unprojection_transform.transform_points(xy_depth) diff --git a/pytorch3d/renderer/implicit/raysampling.py b/pytorch3d/renderer/implicit/raysampling.py index f0c6f5e1..3c9e80b8 100644 --- a/pytorch3d/renderer/implicit/raysampling.py +++ b/pytorch3d/renderer/implicit/raysampling.py @@ -312,7 +312,7 @@ def _xy_to_ray_bundle( ) # unproject the points - unprojected = cameras.unproject_points(to_unproject) # pyre-ignore + unprojected = cameras.unproject_points(to_unproject, from_ndc=True) # pyre-ignore # split the two planes back rays_plane_1_world = unprojected[:, :n_rays_per_image]