Tutorials textures updates and fix bug in extending meshes with uv textures

Summary:
Found a bug in extending textures with vertex uv coordinates. This was due to the padded -> list conversion of vertex uv coordinates i.e.                 The number of vertices in the mesh and in verts_uvs can differ
e.g. if a vertex is shared between 3 faces, it can
have up to 3 different uv coordinates. Therefore we cannot convert directly from padded to list using _num_verts_per_mesh

Reviewed By: bottler

Differential Revision: D23233595

fbshipit-source-id: 0c66d15baae697ead0bdc384f74c27d4c6539fc9

docs/tutorials/render_textured_meshes.ipynb

@@ -87,7 +87,7 @@
     "from pytorch3d.io import load_objs_as_meshes, load_obj\n",
     "\n",
     "# Data structures and functions for rendering\n",
-    "from pytorch3d.structures import Meshes, Textures\n",
+    "from pytorch3d.structures import Meshes\n",
     "from pytorch3d.renderer import (\n",
     "    look_at_view_transform,\n",
     "    FoVPerspectiveCameras, \n",
@@ -97,7 +97,8 @@
     "    RasterizationSettings, \n",
     "    MeshRenderer, \n",
     "    MeshRasterizer,  \n",
-    "    SoftPhongShader\n",
+    "    SoftPhongShader,\n",
+    "    TexturesUV\n",
     ")\n",
     "\n",
     "# add path for demo utils functions \n",
@@ -170,7 +171,7 @@
     "\n",
     "**Meshes** is a unique datastructure provided in PyTorch3D for working with batches of meshes of different sizes. \n",
     "\n",
-    "**Textures** is an auxillary datastructure for storing texture information about meshes. \n",
+    "**TexturesUV** is an auxillary datastructure for storing vertex uv and texture maps for meshes. \n",
     "\n",
     "**Meshes** has several class methods which are used throughout the rendering pipeline."
    ]
@@ -537,7 +538,7 @@
    "source": [
     "# We can pass arbirary keyword arguments to the rasterizer/shader via the renderer\n",
     "# so the renderer does not need to be reinitialized if any of the settings change.\n",
-    "images = renderer(meshes, cameras=cameras, lights=lights)"
+    "images = renderer(mesh, cameras=cameras, lights=lights)"
    ]
   },
   {
@@ -582,9 +583,9 @@
    "backup_notebook_id": "569222367081034"
   },
   "kernelspec": {
-   "display_name": "pytorch3d (local)",
+   "display_name": "intro_to_cv",
    "language": "python",
-   "name": "pytorch3d_local"
+   "name": "bento_kernel_intro_to_cv"
   },
   "language_info": {
    "codemirror_mode": {

pytorch3d/renderer/mesh/textures.py

@@ -599,11 +599,6 @@ class TexturesUV(TexturesBase):
             if not all(v.device == self.device for v in verts_uvs):
                 raise ValueError("verts_uvs and faces_uvs must be on the same device")
 
-            # These values may be overridden when textures is
-            # passed into the Meshes constructor. For more details
-            # refer to the __init__ of Meshes.
-            self._num_verts_per_mesh = [len(v) for v in verts_uvs]
-
         elif torch.is_tensor(verts_uvs):
             if (
                 verts_uvs.ndim != 3
@@ -621,7 +616,6 @@ class TexturesUV(TexturesBase):
             # These values may be overridden when textures is
             # passed into the Meshes constructor.
             max_V = verts_uvs.shape[1]
-            self._num_verts_per_mesh = [max_V] * self._N
         else:
             raise ValueError("Expected verts_uvs to be a tensor or list")
 
@@ -758,9 +752,11 @@ class TexturesUV(TexturesBase):
                     torch.empty((0, 2), dtype=torch.float32, device=self.device)
                 ] * self._N
             else:
-                self._verts_uvs_list = padded_to_list(
+                # The number of vertices in the mesh and in verts_uvs can differ
-                    self._verts_uvs_padded, split_size=self._num_verts_per_mesh
+                # e.g. if a vertex is shared between 3 faces, it can
-                )
+                # have up to 3 different uv coordinates. Therefore we cannot
+                # convert directly from padded to list using _num_verts_per_mesh
+                self._verts_uvs_list = list(self._verts_uvs_padded.unbind(0))
         return self._verts_uvs_list
 
     # Currently only the padded maps are used.
@@ -783,7 +779,6 @@ class TexturesUV(TexturesBase):
                 "verts_uvs_padded",
                 "faces_uvs_padded",
                 "_num_faces_per_mesh",
-                "_num_verts_per_mesh",
             ],
         )
         new_tex = TexturesUV(
@@ -791,8 +786,8 @@ class TexturesUV(TexturesBase):
             faces_uvs=new_props["faces_uvs_padded"],
             verts_uvs=new_props["verts_uvs_padded"],
         )
+
         new_tex._num_faces_per_mesh = new_props["_num_faces_per_mesh"]
-        new_tex._num_verts_per_mesh = new_props["_num_verts_per_mesh"]
         return new_tex
 
     def sample_textures(self, fragments, **kwargs) -> torch.Tensor:
@@ -860,6 +855,7 @@ class TexturesUV(TexturesBase):
         #   right-bottom pixel of input.
 
         pixel_uvs = pixel_uvs * 2.0 - 1.0
+
         texture_maps = torch.flip(texture_maps, [2])  # flip y axis of the texture map
         if texture_maps.device != pixel_uvs.device:
             texture_maps = texture_maps.to(pixel_uvs.device)

tests/test_texturing.py

@@ -588,10 +588,19 @@ class TestTexturesUV(TestCaseMixin, unittest.TestCase):
         tex_init = tex_mesh.textures
         new_tex = new_mesh.textures
 
+        new_tex_num_verts = new_mesh.num_verts_per_mesh()
         for i in range(len(tex_mesh)):
             for n in range(N):
+                tex_nv = new_tex_num_verts[i * N + n]
                 self.assertClose(
-                    tex_init.verts_uvs_list()[i], new_tex.verts_uvs_list()[i * N + n]
+                    # The original textures were initialized using
+                    # verts uvs list
+                    tex_init.verts_uvs_list()[i],
+                    # In the new textures, the verts_uvs are initialized
+                    # from padded. The verts per mesh are not used to
+                    # convert from padded to list. See TexturesUV for an
+                    # explanation.
+                    new_tex.verts_uvs_list()[i * N + n][:tex_nv, ...],
                 )
                 self.assertClose(
                     tex_init.faces_uvs_list()[i], new_tex.faces_uvs_list()[i * N + n]