diff --git a/pytorch3d/renderer/implicit/renderer.py b/pytorch3d/renderer/implicit/renderer.py index e529be6d..9f2116bc 100644 --- a/pytorch3d/renderer/implicit/renderer.py +++ b/pytorch3d/renderer/implicit/renderer.py @@ -32,23 +32,29 @@ class ImplicitRenderer(torch.nn.Module): VOLUMETRIC_FUNCTION - The `forward` function of the renderer accepts as input the rendering cameras as well - as the `volumetric_function` `Callable`, which defines a field of opacity + The `forward` function of the renderer accepts as input the rendering cameras + as well as the `volumetric_function` `Callable`, which defines a field of opacity and feature vectors over the 3D domain of the scene. A standard `volumetric_function` has the following signature: ``` - def volumetric_function(ray_bundle: RayBundle) -> Tuple[torch.Tensor, torch.Tensor] + def volumetric_function( + ray_bundle: RayBundle, + **kwargs, + ) -> Tuple[torch.Tensor, torch.Tensor] ``` With the following arguments: `ray_bundle`: A RayBundle object containing the following variables: - `rays_origins`: A tensor of shape `(minibatch, ..., 3)` denoting + `origins`: A tensor of shape `(minibatch, ..., 3)` denoting the origins of the rendering rays. - `rays_directions`: A tensor of shape `(minibatch, ..., 3)` + `directions`: A tensor of shape `(minibatch, ..., 3)` containing the direction vectors of rendering rays. - `rays_lengths`: A tensor of shape + `lengths`: A tensor of shape `(minibatch, ..., num_points_per_ray)`containing the lengths at which the ray points are sampled. + `xys`: A tensor of shape + `(minibatch, ..., 2)` containing the + xy locations of each ray's pixel in the screen space. Calling `volumetric_function` then returns the following: `rays_densities`: A tensor of shape `(minibatch, ..., num_points_per_ray, opacity_dim)` containing @@ -57,12 +63,20 @@ class ImplicitRenderer(torch.nn.Module): `(minibatch, ..., num_points_per_ray, feature_dim)` containing the an feature vector for each ray point. + Note that, in order to increase flexibility of the API, we allow multiple + other arguments to enter the volumentric function via additional + (optional) keyword arguments `**kwargs`. + A typical use-case is passing a `CamerasBase` object as an additional + keyword argument, which can allow the volumetric function to adjust its + outputs based on the directions of the projection rays. + Example: A simple volumetric function of a 0-centered RGB sphere with a unit diameter is defined as follows: ``` def volumetric_function( ray_bundle: RayBundle, + **kwargs, ) -> Tuple[torch.Tensor, torch.Tensor]: # first convert the ray origins, directions and lengths