mirror of
https://github.com/facebookresearch/pytorch3d.git
synced 2025-08-02 03:42:50 +08:00
Fix camera conversion between opencv and pytorch3d
Summary: For non square image, the NDC space in pytorch3d is not square [-1, 1]. Instead, it is [-1, 1] for the smallest side, and [-u, u] for the largest side, where u > 1. This behavior is followed by the pytorch3d renderer. See the function `get_ndc_to_screen_transform` for a example. Without this fix, the rendering result is not correct using the converted pytorch3d-camera from a opencv-camera on non square images. This fix also helps the `transform_points_screen` function delivers consistent results with opencv projection for the converted pytorch3d-camera. Reviewed By: classner Differential Revision: D31366775 fbshipit-source-id: 8858ae7b5cf5c0a4af5a2af40a1358b2fe4cf74b
This commit is contained in:
parent
815a93ce89
commit
8fa438cbda
@ -28,9 +28,18 @@ def _cameras_from_opencv_projection(
|
||||
# Retype the image_size correctly and flip to width, height.
|
||||
image_size_wh = image_size.to(R).flip(dims=(1,))
|
||||
|
||||
# Screen to NDC conversion:
|
||||
# For non square images, we scale the points such that smallest side
|
||||
# has range [-1, 1] and the largest side has range [-u, u], with u > 1.
|
||||
# This convention is consistent with the PyTorch3D renderer, as well as
|
||||
# the transformation function `get_ndc_to_screen_transform`.
|
||||
scale = (image_size_wh.to(R).min(dim=1, keepdim=True)[0] - 1) / 2.0
|
||||
scale = scale.expand(-1, 2)
|
||||
c0 = (image_size_wh - 1) / 2.0
|
||||
|
||||
# Get the PyTorch3D focal length and principal point.
|
||||
focal_pytorch3d = focal_length / (0.5 * image_size_wh)
|
||||
p0_pytorch3d = -(principal_point / (0.5 * image_size_wh) - 1)
|
||||
focal_pytorch3d = focal_length / scale
|
||||
p0_pytorch3d = -(principal_point - c0) / scale
|
||||
|
||||
# For R, T we flip x, y axes (opencv screen space has an opposite
|
||||
# orientation of screen axes).
|
||||
@ -45,6 +54,7 @@ def _cameras_from_opencv_projection(
|
||||
T=T_pytorch3d,
|
||||
focal_length=focal_pytorch3d,
|
||||
principal_point=p0_pytorch3d,
|
||||
image_size=image_size,
|
||||
)
|
||||
|
||||
|
||||
@ -64,8 +74,13 @@ def _opencv_from_cameras_projection(
|
||||
# Retype the image_size correctly and flip to width, height.
|
||||
image_size_wh = image_size.to(R).flip(dims=(1,))
|
||||
|
||||
principal_point = (-p0_pytorch3d + 1.0) * (0.5 * image_size_wh) # pyre-ignore
|
||||
focal_length = focal_pytorch3d * (0.5 * image_size_wh)
|
||||
# NDC to screen conversion.
|
||||
scale = (image_size_wh.to(R).min(dim=1, keepdim=True)[0] - 1) / 2.0
|
||||
scale = scale.expand(-1, 2)
|
||||
c0 = (image_size_wh - 1) / 2.0
|
||||
|
||||
principal_point = -p0_pytorch3d * scale + c0
|
||||
focal_length = focal_pytorch3d * scale
|
||||
|
||||
camera_matrix = torch.zeros_like(R)
|
||||
camera_matrix[:, :2, 2] = principal_point
|
||||
|
@ -24,14 +24,6 @@ from pytorch3d.utils import (
|
||||
DATA_DIR = get_tests_dir() / "data"
|
||||
|
||||
|
||||
def _coords_opencv_screen_to_pytorch3d_ndc(xy_opencv, image_size):
|
||||
"""
|
||||
Converts the OpenCV screen coordinates `xy_opencv` to PyTorch3D NDC coordinates.
|
||||
"""
|
||||
xy_pytorch3d = -(2.0 * xy_opencv / image_size.flip(dims=(1,))[:, None] - 1.0)
|
||||
return xy_pytorch3d
|
||||
|
||||
|
||||
def cv2_project_points(pts, rvec, tvec, camera_matrix):
|
||||
"""
|
||||
Reproduces the `cv2.projectPoints` function from OpenCV using PyTorch.
|
||||
@ -145,18 +137,13 @@ class TestCameraConversions(TestCaseMixin, unittest.TestCase):
|
||||
R, tvec, camera_matrix, image_size
|
||||
)
|
||||
|
||||
# project the 3D points with converted cameras
|
||||
pts_proj_pytorch3d = cameras_opencv_to_pytorch3d.transform_points(pts)[..., :2]
|
||||
|
||||
# convert the opencv-projected points to pytorch3d screen coords
|
||||
pts_proj_opencv_in_pytorch3d_screen = _coords_opencv_screen_to_pytorch3d_ndc(
|
||||
pts_proj_opencv, image_size
|
||||
)
|
||||
# project the 3D points with converted cameras to screen space.
|
||||
pts_proj_pytorch3d_screen = cameras_opencv_to_pytorch3d.transform_points_screen(
|
||||
pts
|
||||
)[..., :2]
|
||||
|
||||
# compare to the cached projected points
|
||||
self.assertClose(
|
||||
pts_proj_opencv_in_pytorch3d_screen, pts_proj_pytorch3d, atol=1e-5
|
||||
)
|
||||
self.assertClose(pts_proj_opencv, pts_proj_pytorch3d_screen, atol=1e-5)
|
||||
|
||||
# Check the inverse.
|
||||
R_i, tvec_i, camera_matrix_i = opencv_from_cameras_projection(
|
||||
|
Loading…
x
Reference in New Issue
Block a user