Fix camera conversion between opencv and pytorch3d

Summary:
For non square image, the NDC space in pytorch3d is not square [-1, 1]. Instead, it is [-1, 1] for the smallest side, and [-u, u] for the largest side, where u > 1. This behavior is followed by the pytorch3d renderer.

See the function `get_ndc_to_screen_transform` for a example.

Without this fix, the rendering result is not correct using the converted pytorch3d-camera from a opencv-camera on non square images.

This fix also helps the `transform_points_screen` function delivers consistent results with opencv projection for the converted pytorch3d-camera.

Reviewed By: classner

Differential Revision: D31366775

fbshipit-source-id: 8858ae7b5cf5c0a4af5a2af40a1358b2fe4cf74b

pytorch3d/renderer/camera_conversions.py

@@ -28,9 +28,18 @@ def _cameras_from_opencv_projection(
     # Retype the image_size correctly and flip to width, height.
     image_size_wh = image_size.to(R).flip(dims=(1,))
 
+    # Screen to NDC conversion:
+    # For non square images, we scale the points such that smallest side
+    # has range [-1, 1] and the largest side has range [-u, u], with u > 1.
+    # This convention is consistent with the PyTorch3D renderer, as well as
+    # the transformation function `get_ndc_to_screen_transform`.
+    scale = (image_size_wh.to(R).min(dim=1, keepdim=True)[0] - 1) / 2.0
+    scale = scale.expand(-1, 2)
+    c0 = (image_size_wh - 1) / 2.0
+
     # Get the PyTorch3D focal length and principal point.
-    focal_pytorch3d = focal_length / (0.5 * image_size_wh)
+    focal_pytorch3d = focal_length / scale
-    p0_pytorch3d = -(principal_point / (0.5 * image_size_wh) - 1)
+    p0_pytorch3d = -(principal_point - c0) / scale
 
     # For R, T we flip x, y axes (opencv screen space has an opposite
     # orientation of screen axes).
@@ -45,6 +54,7 @@ def _cameras_from_opencv_projection(
         T=T_pytorch3d,
         focal_length=focal_pytorch3d,
         principal_point=p0_pytorch3d,
+        image_size=image_size,
     )
 
 
@@ -64,8 +74,13 @@ def _opencv_from_cameras_projection(
     # Retype the image_size correctly and flip to width, height.
     image_size_wh = image_size.to(R).flip(dims=(1,))
 
-    principal_point = (-p0_pytorch3d + 1.0) * (0.5 * image_size_wh)  # pyre-ignore
+    # NDC to screen conversion.
-    focal_length = focal_pytorch3d * (0.5 * image_size_wh)
+    scale = (image_size_wh.to(R).min(dim=1, keepdim=True)[0] - 1) / 2.0
+    scale = scale.expand(-1, 2)
+    c0 = (image_size_wh - 1) / 2.0
+
+    principal_point = -p0_pytorch3d * scale + c0
+    focal_length = focal_pytorch3d * scale
 
     camera_matrix = torch.zeros_like(R)
     camera_matrix[:, :2, 2] = principal_point

tests/test_camera_conversions.py

@@ -24,14 +24,6 @@ from pytorch3d.utils import (
 DATA_DIR = get_tests_dir() / "data"
 
 
-def _coords_opencv_screen_to_pytorch3d_ndc(xy_opencv, image_size):
-    """
-    Converts the OpenCV screen coordinates `xy_opencv` to PyTorch3D NDC coordinates.
-    """
-    xy_pytorch3d = -(2.0 * xy_opencv / image_size.flip(dims=(1,))[:, None] - 1.0)
-    return xy_pytorch3d
-
-
 def cv2_project_points(pts, rvec, tvec, camera_matrix):
     """
     Reproduces the `cv2.projectPoints` function from OpenCV using PyTorch.
@@ -145,18 +137,13 @@ class TestCameraConversions(TestCaseMixin, unittest.TestCase):
             R, tvec, camera_matrix, image_size
         )
 
-        # project the 3D points with converted cameras
+        # project the 3D points with converted cameras to screen space.
-        pts_proj_pytorch3d = cameras_opencv_to_pytorch3d.transform_points(pts)[..., :2]
+        pts_proj_pytorch3d_screen = cameras_opencv_to_pytorch3d.transform_points_screen(
-
+            pts
-        # convert the opencv-projected points to pytorch3d screen coords
+        )[..., :2]
-        pts_proj_opencv_in_pytorch3d_screen = _coords_opencv_screen_to_pytorch3d_ndc(
-            pts_proj_opencv, image_size
-        )
 
         # compare to the cached projected points
-        self.assertClose(
+        self.assertClose(pts_proj_opencv, pts_proj_pytorch3d_screen, atol=1e-5)
-            pts_proj_opencv_in_pytorch3d_screen, pts_proj_pytorch3d, atol=1e-5
-        )
 
         # Check the inverse.
         R_i, tvec_i, camera_matrix_i = opencv_from_cameras_projection(