Fix camera conversion between opencv and pytorch3d

Summary:
For non square image, the NDC space in pytorch3d is not square [-1, 1]. Instead, it is [-1, 1] for the smallest side, and [-u, u] for the largest side, where u > 1. This behavior is followed by the pytorch3d renderer.

See the function `get_ndc_to_screen_transform` for a example.

Without this fix, the rendering result is not correct using the converted pytorch3d-camera from a opencv-camera on non square images.

This fix also helps the `transform_points_screen` function delivers consistent results with opencv projection for the converted pytorch3d-camera.

Reviewed By: classner

Differential Revision: D31366775

fbshipit-source-id: 8858ae7b5cf5c0a4af5a2af40a1358b2fe4cf74b
This commit is contained in:
Ruilong Li 2021-10-07 10:14:23 -07:00 committed by Facebook GitHub Bot
parent 815a93ce89
commit 8fa438cbda
2 changed files with 24 additions and 22 deletions

View File

@ -28,9 +28,18 @@ def _cameras_from_opencv_projection(
# Retype the image_size correctly and flip to width, height.
image_size_wh = image_size.to(R).flip(dims=(1,))
# Screen to NDC conversion:
# For non square images, we scale the points such that smallest side
# has range [-1, 1] and the largest side has range [-u, u], with u > 1.
# This convention is consistent with the PyTorch3D renderer, as well as
# the transformation function `get_ndc_to_screen_transform`.
scale = (image_size_wh.to(R).min(dim=1, keepdim=True)[0] - 1) / 2.0
scale = scale.expand(-1, 2)
c0 = (image_size_wh - 1) / 2.0
# Get the PyTorch3D focal length and principal point.
focal_pytorch3d = focal_length / (0.5 * image_size_wh)
p0_pytorch3d = -(principal_point / (0.5 * image_size_wh) - 1)
focal_pytorch3d = focal_length / scale
p0_pytorch3d = -(principal_point - c0) / scale
# For R, T we flip x, y axes (opencv screen space has an opposite
# orientation of screen axes).
@ -45,6 +54,7 @@ def _cameras_from_opencv_projection(
T=T_pytorch3d,
focal_length=focal_pytorch3d,
principal_point=p0_pytorch3d,
image_size=image_size,
)
@ -64,8 +74,13 @@ def _opencv_from_cameras_projection(
# Retype the image_size correctly and flip to width, height.
image_size_wh = image_size.to(R).flip(dims=(1,))
principal_point = (-p0_pytorch3d + 1.0) * (0.5 * image_size_wh) # pyre-ignore
focal_length = focal_pytorch3d * (0.5 * image_size_wh)
# NDC to screen conversion.
scale = (image_size_wh.to(R).min(dim=1, keepdim=True)[0] - 1) / 2.0
scale = scale.expand(-1, 2)
c0 = (image_size_wh - 1) / 2.0
principal_point = -p0_pytorch3d * scale + c0
focal_length = focal_pytorch3d * scale
camera_matrix = torch.zeros_like(R)
camera_matrix[:, :2, 2] = principal_point

View File

@ -24,14 +24,6 @@ from pytorch3d.utils import (
DATA_DIR = get_tests_dir() / "data"
def _coords_opencv_screen_to_pytorch3d_ndc(xy_opencv, image_size):
"""
Converts the OpenCV screen coordinates `xy_opencv` to PyTorch3D NDC coordinates.
"""
xy_pytorch3d = -(2.0 * xy_opencv / image_size.flip(dims=(1,))[:, None] - 1.0)
return xy_pytorch3d
def cv2_project_points(pts, rvec, tvec, camera_matrix):
"""
Reproduces the `cv2.projectPoints` function from OpenCV using PyTorch.
@ -145,18 +137,13 @@ class TestCameraConversions(TestCaseMixin, unittest.TestCase):
R, tvec, camera_matrix, image_size
)
# project the 3D points with converted cameras
pts_proj_pytorch3d = cameras_opencv_to_pytorch3d.transform_points(pts)[..., :2]
# convert the opencv-projected points to pytorch3d screen coords
pts_proj_opencv_in_pytorch3d_screen = _coords_opencv_screen_to_pytorch3d_ndc(
pts_proj_opencv, image_size
)
# project the 3D points with converted cameras to screen space.
pts_proj_pytorch3d_screen = cameras_opencv_to_pytorch3d.transform_points_screen(
pts
)[..., :2]
# compare to the cached projected points
self.assertClose(
pts_proj_opencv_in_pytorch3d_screen, pts_proj_pytorch3d, atol=1e-5
)
self.assertClose(pts_proj_opencv, pts_proj_pytorch3d_screen, atol=1e-5)
# Check the inverse.
R_i, tvec_i, camera_matrix_i = opencv_from_cameras_projection(