mirror of
https://github.com/facebookresearch/pytorch3d.git
synced 2025-08-02 03:42:50 +08:00
Update latest version of site
This commit is contained in:
Nikhila Ravi
parent
23fddf074c
commit
4277d6a2b5
File diff suppressed because one or more lines are too long
@ -1,5 +1,5 @@
|
|||||||
/**
|
/**
|
||||||
* Copyright (c) 2017-present, Facebook, Inc.
|
* Copyright (c) Facebook, Inc. and its affiliates.
|
||||||
*
|
*
|
||||||
* This source code is licensed under the MIT license found in the
|
* This source code is licensed under the MIT license found in the
|
||||||
* LICENSE file in the root directory of this source tree.
|
* LICENSE file in the root directory of this source tree.
|
||||||
|
|||||||
Binary file not shown.
|
Before Width: | Height: | Size: 130 KiB |
BIN
docs/assets/architecture_renderer.jpg
Normal file
BIN
docs/assets/architecture_renderer.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 132 KiB |
BIN
docs/assets/texturing.jpg
Normal file
BIN
docs/assets/texturing.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 56 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 66 KiB |
BIN
docs/assets/transforms_overview.jpg
Normal file
BIN
docs/assets/transforms_overview.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 177 KiB |
BIN
docs/assets/world_camera_image.jpg
Normal file
BIN
docs/assets/world_camera_image.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 98 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 62 KiB |
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Meshes</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Meshes</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/batching">Batching</a></li><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Differentiable Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li></ul></div></div></section></div><script>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Data</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
var coll = document.getElementsByClassName('collapsible');
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
var checkActiveCategory = true;
|
var checkActiveCategory = true;
|
||||||
for (var i = 0; i < coll.length; i++) {
|
for (var i = 0; i < coll.length; i++) {
|
||||||
@ -76,4 +76,4 @@
|
|||||||
<h2><a class="anchor" aria-hidden="true" id="use-cases-for-batch-modes"></a><a href="#use-cases-for-batch-modes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Use cases for batch modes</h2>
|
<h2><a class="anchor" aria-hidden="true" id="use-cases-for-batch-modes"></a><a href="#use-cases-for-batch-modes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Use cases for batch modes</h2>
|
||||||
<p>The need for different mesh batch modes is inherent to the way pytorch operators are implemented. To fully utilize the optimized pytorch ops, the <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/structures/meshes.py">Meshes</a> data structure allows for efficient conversion between the different batch modes. This is crucial when aiming for a fast and efficient training cycle. An example of this is <a href="https://github.com/facebookresearch/meshrcnn">Mesh R-CNN</a>. Here, in the same forward pass different parts of the network assume different inputs, which are computed by converting between the different batch modes. In particular, <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/ops/vert_align.py">vert_align</a> assumes a <em>padded</em> input tensor while immediately after <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/ops/graph_conv.py">graph_conv</a> assumes a <em>packed</em> input tensor.</p>
|
<p>The need for different mesh batch modes is inherent to the way pytorch operators are implemented. To fully utilize the optimized pytorch ops, the <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/structures/meshes.py">Meshes</a> data structure allows for efficient conversion between the different batch modes. This is crucial when aiming for a fast and efficient training cycle. An example of this is <a href="https://github.com/facebookresearch/meshrcnn">Mesh R-CNN</a>. Here, in the same forward pass different parts of the network assume different inputs, which are computed by converting between the different batch modes. In particular, <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/ops/vert_align.py">vert_align</a> assumes a <em>padded</em> input tensor while immediately after <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/ops/graph_conv.py">graph_conv</a> assumes a <em>packed</em> input tensor.</p>
|
||||||
<p><img src="assets/meshrcnn.png" alt="meshrcnn" width="700" align="middle" /></p>
|
<p><img src="assets/meshrcnn.png" alt="meshrcnn" width="700" align="middle" /></p>
|
||||||
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/why_pytorch3d"><span class="arrow-prev">← </span><span class="function-name-prevnext">Why PyTorch3D</span></a><a class="docs-next button" href="/docs/meshes_io"><span>Loading from file</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#batch-modes-for-meshes">Batch modes for meshes</a></li><li><a href="#use-cases-for-batch-modes">Use cases for batch modes</a></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/datasets"><span class="arrow-prev">← </span><span>Data loaders</span></a><a class="docs-next button" href="/docs/cubify"><span>Cubify</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#batch-modes-for-meshes">Batch modes for meshes</a></li><li><a href="#use-cases-for-batch-modes">Use cases for batch modes</a></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Meshes</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Meshes</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/batching">Batching</a></li><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Differentiable Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li></ul></div></div></section></div><script>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Data</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
var coll = document.getElementsByClassName('collapsible');
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
var checkActiveCategory = true;
|
var checkActiveCategory = true;
|
||||||
for (var i = 0; i < coll.length; i++) {
|
for (var i = 0; i < coll.length; i++) {
|
||||||
@ -76,4 +76,4 @@
|
|||||||
<h2><a class="anchor" aria-hidden="true" id="use-cases-for-batch-modes"></a><a href="#use-cases-for-batch-modes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Use cases for batch modes</h2>
|
<h2><a class="anchor" aria-hidden="true" id="use-cases-for-batch-modes"></a><a href="#use-cases-for-batch-modes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Use cases for batch modes</h2>
|
||||||
<p>The need for different mesh batch modes is inherent to the way pytorch operators are implemented. To fully utilize the optimized pytorch ops, the <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/structures/meshes.py">Meshes</a> data structure allows for efficient conversion between the different batch modes. This is crucial when aiming for a fast and efficient training cycle. An example of this is <a href="https://github.com/facebookresearch/meshrcnn">Mesh R-CNN</a>. Here, in the same forward pass different parts of the network assume different inputs, which are computed by converting between the different batch modes. In particular, <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/ops/vert_align.py">vert_align</a> assumes a <em>padded</em> input tensor while immediately after <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/ops/graph_conv.py">graph_conv</a> assumes a <em>packed</em> input tensor.</p>
|
<p>The need for different mesh batch modes is inherent to the way pytorch operators are implemented. To fully utilize the optimized pytorch ops, the <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/structures/meshes.py">Meshes</a> data structure allows for efficient conversion between the different batch modes. This is crucial when aiming for a fast and efficient training cycle. An example of this is <a href="https://github.com/facebookresearch/meshrcnn">Mesh R-CNN</a>. Here, in the same forward pass different parts of the network assume different inputs, which are computed by converting between the different batch modes. In particular, <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/ops/vert_align.py">vert_align</a> assumes a <em>padded</em> input tensor while immediately after <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/ops/graph_conv.py">graph_conv</a> assumes a <em>packed</em> input tensor.</p>
|
||||||
<p><img src="assets/meshrcnn.png" alt="meshrcnn" width="700" align="middle" /></p>
|
<p><img src="assets/meshrcnn.png" alt="meshrcnn" width="700" align="middle" /></p>
|
||||||
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/why_pytorch3d"><span class="arrow-prev">← </span><span class="function-name-prevnext">Why PyTorch3D</span></a><a class="docs-next button" href="/docs/meshes_io"><span>Loading from file</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#batch-modes-for-meshes">Batch modes for meshes</a></li><li><a href="#use-cases-for-batch-modes">Use cases for batch modes</a></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/datasets"><span class="arrow-prev">← </span><span>Data loaders</span></a><a class="docs-next button" href="/docs/cubify"><span>Cubify</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#batch-modes-for-meshes">Batch modes for meshes</a></li><li><a href="#use-cases-for-batch-modes">Use cases for batch modes</a></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
115
docs/cameras.html
Normal file
115
docs/cameras.html
Normal file
@ -0,0 +1,115 @@
|
|||||||
|
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>cameras · PyTorch3D</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="# Cameras"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="cameras · PyTorch3D"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="# Cameras"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||||
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||||
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||||
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||||
|
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||||
|
|
||||||
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
|
ga('send', 'pageview');
|
||||||
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Renderer</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
|
var checkActiveCategory = true;
|
||||||
|
for (var i = 0; i < coll.length; i++) {
|
||||||
|
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||||
|
if (checkActiveCategory){
|
||||||
|
for (var j = 0; j < links.length; j++) {
|
||||||
|
if (links[j].classList.contains('navListItemActive')){
|
||||||
|
coll[i].nextElementSibling.classList.toggle('hide');
|
||||||
|
coll[i].childNodes[1].classList.toggle('rotate');
|
||||||
|
checkActiveCategory = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
coll[i].addEventListener('click', function() {
|
||||||
|
var arrow = this.childNodes[1];
|
||||||
|
arrow.classList.toggle('rotate');
|
||||||
|
var content = this.nextElementSibling;
|
||||||
|
content.classList.toggle('hide');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||||
|
createToggler('#tocToggler', 'body', 'tocActive');
|
||||||
|
|
||||||
|
var headings = document.querySelector('.toc-headings');
|
||||||
|
headings && headings.addEventListener('click', function(event) {
|
||||||
|
var el = event.target;
|
||||||
|
while(el !== headings){
|
||||||
|
if (el.tagName === 'A') {
|
||||||
|
document.body.classList.remove('tocActive');
|
||||||
|
break;
|
||||||
|
} else{
|
||||||
|
el = el.parentNode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, false);
|
||||||
|
|
||||||
|
function createToggler(togglerSelector, targetSelector, className) {
|
||||||
|
var toggler = document.querySelector(togglerSelector);
|
||||||
|
var target = document.querySelector(targetSelector);
|
||||||
|
|
||||||
|
if (!toggler) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
toggler.onclick = function(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
|
||||||
|
target.classList.toggle(className);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="cameras"></a><a href="#cameras" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Cameras</h1>
|
||||||
|
<h2><a class="anchor" aria-hidden="true" id="camera-coordinate-systems"></a><a href="#camera-coordinate-systems" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Camera Coordinate Systems</h2>
|
||||||
|
<p>When working with 3D data, there are 4 coordinate systems users need to know</p>
|
||||||
|
<ul>
|
||||||
|
<li><strong>World coordinate system</strong>
|
||||||
|
This is the system the object/scene lives - the world.</li>
|
||||||
|
<li><strong>Camera view coordinate system</strong>
|
||||||
|
This is the system that has its origin on the image plane and the <code>Z</code>-axis perpendicular to the image plane. In PyTorch3D, we assume that <code>+X</code> points left, and <code>+Y</code> points up and <code>+Z</code> points out from the image plane. The transformation from world to view happens after applying a rotation (<code>R</code>) and translation (<code>T</code>).</li>
|
||||||
|
<li><strong>NDC coordinate system</strong>
|
||||||
|
This is the normalized coordinate system that confines in a volume the renderered part of the object/scene. Also known as view volume. Under the PyTorch3D convention, <code>(+1, +1, znear)</code> is the top left near corner, and <code>(-1, -1, zfar)</code> is the bottom right far corner of the volume. The transformation from view to NDC happens after applying the camera projection matrix (<code>P</code>).</li>
|
||||||
|
<li><strong>Screen coordinate system</strong>
|
||||||
|
This is another representation of the view volume with the <code>XY</code> coordinates defined in pixel space instead of a normalized space.</li>
|
||||||
|
</ul>
|
||||||
|
<p>An illustration of the 4 coordinate systems is shown below
|
||||||
|
<img src="https://user-images.githubusercontent.com/4369065/90317960-d9b8db80-dee1-11ea-8088-39c414b1e2fa.png" alt="cameras"></p>
|
||||||
|
<h2><a class="anchor" aria-hidden="true" id="defining-cameras-in-pytorch3d"></a><a href="#defining-cameras-in-pytorch3d" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Defining Cameras in PyTorch3D</h2>
|
||||||
|
<p>Cameras in PyTorch3D transform an object/scene from world to NDC by first transforming the object/scene to view (via transforms <code>R</code> and <code>T</code>) and then projecting the 3D object/scene to NDC (via the projection matrix <code>P</code>, else known as camera matrix). Thus, the camera parameters in <code>P</code> are assumed to be in NDC space. If the user has camera parameters in screen space, which is a common use case, the parameters should transformed to NDC (see below for an example)</p>
|
||||||
|
<p>We describe the camera types in PyTorch3D and the convention for the camera parameters provided at construction time.</p>
|
||||||
|
<h3><a class="anchor" aria-hidden="true" id="camera-types"></a><a href="#camera-types" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Camera Types</h3>
|
||||||
|
<p>All cameras inherit from <code>CamerasBase</code> which is a base class for all cameras. PyTorch3D provides four different camera types. The <code>CamerasBase</code> defines methods that are common to all camera models:</p>
|
||||||
|
<ul>
|
||||||
|
<li><code>get_camera_center</code> that returns the optical center of the camera in world coordinates</li>
|
||||||
|
<li><code>get_world_to_view_transform</code> which returns a 3D transform from world coordinates to the camera view coordinates (R, T)</li>
|
||||||
|
<li><code>get_full_projection_transform</code> which composes the projection transform (P) with the world-to-view transform (R, T)</li>
|
||||||
|
<li><code>transform_points</code> which takes a set of input points in world coordinates and projects to NDC coordinates ranging from [-1, -1, znear] to [+1, +1, zfar].</li>
|
||||||
|
<li><code>transform_points_screen</code> which takes a set of input points in world coordinates and projects them to the screen coordinates ranging from [0, 0, znear] to [W-1, H-1, zfar]</li>
|
||||||
|
</ul>
|
||||||
|
<p>Users can easily customize their own cameras. For each new camera, users should implement the <code>get_projection_transform</code> routine that returns the mapping <code>P</code> from camera view coordinates to NDC coordinates.</p>
|
||||||
|
<h4><a class="anchor" aria-hidden="true" id="fovperspectivecameras-fovorthographiccameras"></a><a href="#fovperspectivecameras-fovorthographiccameras" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>FoVPerspectiveCameras, FoVOrthographicCameras</h4>
|
||||||
|
<p>These two cameras follow the OpenGL convention for perspective and orthographic cameras respectively. The user provides the near <code>znear</code> and far <code>zfar</code> field which confines the view volume in the <code>Z</code> axis. The view volume in the <code>XY</code> plane is defined by field of view angle (<code>fov</code>) in the case of <code>FoVPerspectiveCameras</code> and by <code>min_x, min_y, max_x, max_y</code> in the case of <code>FoVOrthographicCameras</code>.</p>
|
||||||
|
<h4><a class="anchor" aria-hidden="true" id="perspectivecameras-orthographiccameras"></a><a href="#perspectivecameras-orthographiccameras" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>PerspectiveCameras, OrthographicCameras</h4>
|
||||||
|
<p>These two cameras follow the Multi-View Geometry convention for cameras. The user provides the focal length (<code>fx</code>, <code>fy</code>) and the principal point (<code>px</code>, <code>py</code>). For example, <code>camera = PerspectiveCameras(focal_length=((fx, fy),), principal_point=((px, py),))</code></p>
|
||||||
|
<p>As mentioned above, the focal length and principal point are used to convert a point <code>(X, Y, Z)</code> from view coordinates to NDC coordinates, as follows</p>
|
||||||
|
<pre><code class="hljs"><span class="hljs-comment"># for perspective</span>
|
||||||
|
<span class="hljs-attr">x_ndc</span> = fx * X / Z + px
|
||||||
|
<span class="hljs-attr">y_ndc</span> = fy * Y / Z + py
|
||||||
|
<span class="hljs-attr">z_ndc</span> = <span class="hljs-number">1</span> / Z
|
||||||
|
|
||||||
|
<span class="hljs-comment"># for orthographic</span>
|
||||||
|
<span class="hljs-attr">x_ndc</span> = fx * X + px
|
||||||
|
<span class="hljs-attr">y_ndc</span> = fy * Y + py
|
||||||
|
<span class="hljs-attr">z_ndc</span> = Z
|
||||||
|
</code></pre>
|
||||||
|
<p>Commonly, users have access to the focal length (<code>fx_screen</code>, <code>fy_screen</code>) and the principal point (<code>px_screen</code>, <code>py_screen</code>) in screen space. In that case, to construct the camera the user needs to additionally provide the <code>image_size = ((image_width, image_height),)</code>. More precisely, <code>camera = PerspectiveCameras(focal_length=((fx_screen, fy_screen),), principal_point=((px_screen, py_screen),), image_size = ((image_width, image_height),))</code>. Internally, the camera parameters are converted from screen to NDC as follows:</p>
|
||||||
|
<pre><code class="hljs"><span class="hljs-attr">fx</span> = fx_screen * <span class="hljs-number">2.0</span> / image_width
|
||||||
|
<span class="hljs-attr">fy</span> = fy_screen * <span class="hljs-number">2.0</span> / image_height
|
||||||
|
|
||||||
|
<span class="hljs-attr">px</span> = - (px_screen - image_width / <span class="hljs-number">2.0</span>) * <span class="hljs-number">2.0</span> / image_width
|
||||||
|
<span class="hljs-attr">py</span> = - (py_screen - image_height / <span class="hljs-number">2.0</span>) * <span class="hljs-number">2.0</span>/ image_height
|
||||||
|
</code></pre>
|
||||||
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/renderer_getting_started"><span class="arrow-prev">← </span><span>Getting Started</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#camera-coordinate-systems">Camera Coordinate Systems</a></li><li><a href="#defining-cameras-in-pytorch3d">Defining Cameras in PyTorch3D</a><ul class="toc-headings"><li><a href="#camera-types">Camera Types</a></li></ul></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
115
docs/cameras/index.html
Normal file
115
docs/cameras/index.html
Normal file
@ -0,0 +1,115 @@
|
|||||||
|
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>cameras · PyTorch3D</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="# Cameras"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="cameras · PyTorch3D"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="# Cameras"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||||
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||||
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||||
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||||
|
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||||
|
|
||||||
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
|
ga('send', 'pageview');
|
||||||
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Renderer</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
|
var checkActiveCategory = true;
|
||||||
|
for (var i = 0; i < coll.length; i++) {
|
||||||
|
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||||
|
if (checkActiveCategory){
|
||||||
|
for (var j = 0; j < links.length; j++) {
|
||||||
|
if (links[j].classList.contains('navListItemActive')){
|
||||||
|
coll[i].nextElementSibling.classList.toggle('hide');
|
||||||
|
coll[i].childNodes[1].classList.toggle('rotate');
|
||||||
|
checkActiveCategory = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
coll[i].addEventListener('click', function() {
|
||||||
|
var arrow = this.childNodes[1];
|
||||||
|
arrow.classList.toggle('rotate');
|
||||||
|
var content = this.nextElementSibling;
|
||||||
|
content.classList.toggle('hide');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||||
|
createToggler('#tocToggler', 'body', 'tocActive');
|
||||||
|
|
||||||
|
var headings = document.querySelector('.toc-headings');
|
||||||
|
headings && headings.addEventListener('click', function(event) {
|
||||||
|
var el = event.target;
|
||||||
|
while(el !== headings){
|
||||||
|
if (el.tagName === 'A') {
|
||||||
|
document.body.classList.remove('tocActive');
|
||||||
|
break;
|
||||||
|
} else{
|
||||||
|
el = el.parentNode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, false);
|
||||||
|
|
||||||
|
function createToggler(togglerSelector, targetSelector, className) {
|
||||||
|
var toggler = document.querySelector(togglerSelector);
|
||||||
|
var target = document.querySelector(targetSelector);
|
||||||
|
|
||||||
|
if (!toggler) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
toggler.onclick = function(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
|
||||||
|
target.classList.toggle(className);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="cameras"></a><a href="#cameras" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Cameras</h1>
|
||||||
|
<h2><a class="anchor" aria-hidden="true" id="camera-coordinate-systems"></a><a href="#camera-coordinate-systems" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Camera Coordinate Systems</h2>
|
||||||
|
<p>When working with 3D data, there are 4 coordinate systems users need to know</p>
|
||||||
|
<ul>
|
||||||
|
<li><strong>World coordinate system</strong>
|
||||||
|
This is the system the object/scene lives - the world.</li>
|
||||||
|
<li><strong>Camera view coordinate system</strong>
|
||||||
|
This is the system that has its origin on the image plane and the <code>Z</code>-axis perpendicular to the image plane. In PyTorch3D, we assume that <code>+X</code> points left, and <code>+Y</code> points up and <code>+Z</code> points out from the image plane. The transformation from world to view happens after applying a rotation (<code>R</code>) and translation (<code>T</code>).</li>
|
||||||
|
<li><strong>NDC coordinate system</strong>
|
||||||
|
This is the normalized coordinate system that confines in a volume the renderered part of the object/scene. Also known as view volume. Under the PyTorch3D convention, <code>(+1, +1, znear)</code> is the top left near corner, and <code>(-1, -1, zfar)</code> is the bottom right far corner of the volume. The transformation from view to NDC happens after applying the camera projection matrix (<code>P</code>).</li>
|
||||||
|
<li><strong>Screen coordinate system</strong>
|
||||||
|
This is another representation of the view volume with the <code>XY</code> coordinates defined in pixel space instead of a normalized space.</li>
|
||||||
|
</ul>
|
||||||
|
<p>An illustration of the 4 coordinate systems is shown below
|
||||||
|
<img src="https://user-images.githubusercontent.com/4369065/90317960-d9b8db80-dee1-11ea-8088-39c414b1e2fa.png" alt="cameras"></p>
|
||||||
|
<h2><a class="anchor" aria-hidden="true" id="defining-cameras-in-pytorch3d"></a><a href="#defining-cameras-in-pytorch3d" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Defining Cameras in PyTorch3D</h2>
|
||||||
|
<p>Cameras in PyTorch3D transform an object/scene from world to NDC by first transforming the object/scene to view (via transforms <code>R</code> and <code>T</code>) and then projecting the 3D object/scene to NDC (via the projection matrix <code>P</code>, else known as camera matrix). Thus, the camera parameters in <code>P</code> are assumed to be in NDC space. If the user has camera parameters in screen space, which is a common use case, the parameters should transformed to NDC (see below for an example)</p>
|
||||||
|
<p>We describe the camera types in PyTorch3D and the convention for the camera parameters provided at construction time.</p>
|
||||||
|
<h3><a class="anchor" aria-hidden="true" id="camera-types"></a><a href="#camera-types" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Camera Types</h3>
|
||||||
|
<p>All cameras inherit from <code>CamerasBase</code> which is a base class for all cameras. PyTorch3D provides four different camera types. The <code>CamerasBase</code> defines methods that are common to all camera models:</p>
|
||||||
|
<ul>
|
||||||
|
<li><code>get_camera_center</code> that returns the optical center of the camera in world coordinates</li>
|
||||||
|
<li><code>get_world_to_view_transform</code> which returns a 3D transform from world coordinates to the camera view coordinates (R, T)</li>
|
||||||
|
<li><code>get_full_projection_transform</code> which composes the projection transform (P) with the world-to-view transform (R, T)</li>
|
||||||
|
<li><code>transform_points</code> which takes a set of input points in world coordinates and projects to NDC coordinates ranging from [-1, -1, znear] to [+1, +1, zfar].</li>
|
||||||
|
<li><code>transform_points_screen</code> which takes a set of input points in world coordinates and projects them to the screen coordinates ranging from [0, 0, znear] to [W-1, H-1, zfar]</li>
|
||||||
|
</ul>
|
||||||
|
<p>Users can easily customize their own cameras. For each new camera, users should implement the <code>get_projection_transform</code> routine that returns the mapping <code>P</code> from camera view coordinates to NDC coordinates.</p>
|
||||||
|
<h4><a class="anchor" aria-hidden="true" id="fovperspectivecameras-fovorthographiccameras"></a><a href="#fovperspectivecameras-fovorthographiccameras" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>FoVPerspectiveCameras, FoVOrthographicCameras</h4>
|
||||||
|
<p>These two cameras follow the OpenGL convention for perspective and orthographic cameras respectively. The user provides the near <code>znear</code> and far <code>zfar</code> field which confines the view volume in the <code>Z</code> axis. The view volume in the <code>XY</code> plane is defined by field of view angle (<code>fov</code>) in the case of <code>FoVPerspectiveCameras</code> and by <code>min_x, min_y, max_x, max_y</code> in the case of <code>FoVOrthographicCameras</code>.</p>
|
||||||
|
<h4><a class="anchor" aria-hidden="true" id="perspectivecameras-orthographiccameras"></a><a href="#perspectivecameras-orthographiccameras" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>PerspectiveCameras, OrthographicCameras</h4>
|
||||||
|
<p>These two cameras follow the Multi-View Geometry convention for cameras. The user provides the focal length (<code>fx</code>, <code>fy</code>) and the principal point (<code>px</code>, <code>py</code>). For example, <code>camera = PerspectiveCameras(focal_length=((fx, fy),), principal_point=((px, py),))</code></p>
|
||||||
|
<p>As mentioned above, the focal length and principal point are used to convert a point <code>(X, Y, Z)</code> from view coordinates to NDC coordinates, as follows</p>
|
||||||
|
<pre><code class="hljs"><span class="hljs-comment"># for perspective</span>
|
||||||
|
<span class="hljs-attr">x_ndc</span> = fx * X / Z + px
|
||||||
|
<span class="hljs-attr">y_ndc</span> = fy * Y / Z + py
|
||||||
|
<span class="hljs-attr">z_ndc</span> = <span class="hljs-number">1</span> / Z
|
||||||
|
|
||||||
|
<span class="hljs-comment"># for orthographic</span>
|
||||||
|
<span class="hljs-attr">x_ndc</span> = fx * X + px
|
||||||
|
<span class="hljs-attr">y_ndc</span> = fy * Y + py
|
||||||
|
<span class="hljs-attr">z_ndc</span> = Z
|
||||||
|
</code></pre>
|
||||||
|
<p>Commonly, users have access to the focal length (<code>fx_screen</code>, <code>fy_screen</code>) and the principal point (<code>px_screen</code>, <code>py_screen</code>) in screen space. In that case, to construct the camera the user needs to additionally provide the <code>image_size = ((image_width, image_height),)</code>. More precisely, <code>camera = PerspectiveCameras(focal_length=((fx_screen, fy_screen),), principal_point=((px_screen, py_screen),), image_size = ((image_width, image_height),))</code>. Internally, the camera parameters are converted from screen to NDC as follows:</p>
|
||||||
|
<pre><code class="hljs"><span class="hljs-attr">fx</span> = fx_screen * <span class="hljs-number">2.0</span> / image_width
|
||||||
|
<span class="hljs-attr">fy</span> = fy_screen * <span class="hljs-number">2.0</span> / image_height
|
||||||
|
|
||||||
|
<span class="hljs-attr">px</span> = - (px_screen - image_width / <span class="hljs-number">2.0</span>) * <span class="hljs-number">2.0</span> / image_width
|
||||||
|
<span class="hljs-attr">py</span> = - (py_screen - image_height / <span class="hljs-number">2.0</span>) * <span class="hljs-number">2.0</span>/ image_height
|
||||||
|
</code></pre>
|
||||||
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/renderer_getting_started"><span class="arrow-prev">← </span><span>Getting Started</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#camera-coordinate-systems">Camera Coordinate Systems</a></li><li><a href="#defining-cameras-in-pytorch3d">Defining Cameras in PyTorch3D</a><ul class="toc-headings"><li><a href="#camera-types">Camera Types</a></li></ul></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
69
docs/cubify.html
Normal file
69
docs/cubify.html
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>cubify · PyTorch3D</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="# Cubify"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="cubify · PyTorch3D"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="# Cubify"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||||
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||||
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||||
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||||
|
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||||
|
|
||||||
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
|
ga('send', 'pageview');
|
||||||
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Ops</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
|
var checkActiveCategory = true;
|
||||||
|
for (var i = 0; i < coll.length; i++) {
|
||||||
|
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||||
|
if (checkActiveCategory){
|
||||||
|
for (var j = 0; j < links.length; j++) {
|
||||||
|
if (links[j].classList.contains('navListItemActive')){
|
||||||
|
coll[i].nextElementSibling.classList.toggle('hide');
|
||||||
|
coll[i].childNodes[1].classList.toggle('rotate');
|
||||||
|
checkActiveCategory = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
coll[i].addEventListener('click', function() {
|
||||||
|
var arrow = this.childNodes[1];
|
||||||
|
arrow.classList.toggle('rotate');
|
||||||
|
var content = this.nextElementSibling;
|
||||||
|
content.classList.toggle('hide');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||||
|
createToggler('#tocToggler', 'body', 'tocActive');
|
||||||
|
|
||||||
|
var headings = document.querySelector('.toc-headings');
|
||||||
|
headings && headings.addEventListener('click', function(event) {
|
||||||
|
var el = event.target;
|
||||||
|
while(el !== headings){
|
||||||
|
if (el.tagName === 'A') {
|
||||||
|
document.body.classList.remove('tocActive');
|
||||||
|
break;
|
||||||
|
} else{
|
||||||
|
el = el.parentNode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, false);
|
||||||
|
|
||||||
|
function createToggler(togglerSelector, targetSelector, className) {
|
||||||
|
var toggler = document.querySelector(togglerSelector);
|
||||||
|
var target = document.querySelector(targetSelector);
|
||||||
|
|
||||||
|
if (!toggler) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
toggler.onclick = function(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
|
||||||
|
target.classList.toggle(className);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="cubify"></a><a href="#cubify" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Cubify</h1>
|
||||||
|
<p>The <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/ops/cubify.py">cubify operator</a> converts an 3D occupancy grid of shape <code>BxDxHxW</code>, where <code>B</code> is the batch size, into a mesh instantiated as a <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/structures/meshes.py">Meshes</a> data structure of <code>B</code> elements. The operator replaces every occupied voxel (if its occupancy probability is greater than a user defined threshold) with a cuboid of 12 faces and 8 vertices. Shared vertices are merged, and internal faces are removed resulting in a <strong>watertight</strong> mesh.</p>
|
||||||
|
<p>The operator provides three alignment modes {<em>topleft</em>, <em>corner</em>, <em>center</em>} which define the span of the mesh vertices with respect to the voxel grid. The alignment modes are described in the figure below for a 2D grid.</p>
|
||||||
|
<p><img src="https://user-images.githubusercontent.com/4369065/81032959-af697380-8e46-11ea-91a8-fae89597f988.png" alt="input"></p>
|
||||||
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/batching"><span class="arrow-prev">← </span><span>Batching</span></a><a class="docs-next button" href="/docs/renderer"><span>Overview</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
69
docs/cubify/index.html
Normal file
69
docs/cubify/index.html
Normal file
@ -0,0 +1,69 @@
|
|||||||
|
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>cubify · PyTorch3D</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="# Cubify"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="cubify · PyTorch3D"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="# Cubify"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||||
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||||
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||||
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||||
|
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||||
|
|
||||||
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
|
ga('send', 'pageview');
|
||||||
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Ops</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
|
var checkActiveCategory = true;
|
||||||
|
for (var i = 0; i < coll.length; i++) {
|
||||||
|
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||||
|
if (checkActiveCategory){
|
||||||
|
for (var j = 0; j < links.length; j++) {
|
||||||
|
if (links[j].classList.contains('navListItemActive')){
|
||||||
|
coll[i].nextElementSibling.classList.toggle('hide');
|
||||||
|
coll[i].childNodes[1].classList.toggle('rotate');
|
||||||
|
checkActiveCategory = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
coll[i].addEventListener('click', function() {
|
||||||
|
var arrow = this.childNodes[1];
|
||||||
|
arrow.classList.toggle('rotate');
|
||||||
|
var content = this.nextElementSibling;
|
||||||
|
content.classList.toggle('hide');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||||
|
createToggler('#tocToggler', 'body', 'tocActive');
|
||||||
|
|
||||||
|
var headings = document.querySelector('.toc-headings');
|
||||||
|
headings && headings.addEventListener('click', function(event) {
|
||||||
|
var el = event.target;
|
||||||
|
while(el !== headings){
|
||||||
|
if (el.tagName === 'A') {
|
||||||
|
document.body.classList.remove('tocActive');
|
||||||
|
break;
|
||||||
|
} else{
|
||||||
|
el = el.parentNode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, false);
|
||||||
|
|
||||||
|
function createToggler(togglerSelector, targetSelector, className) {
|
||||||
|
var toggler = document.querySelector(togglerSelector);
|
||||||
|
var target = document.querySelector(targetSelector);
|
||||||
|
|
||||||
|
if (!toggler) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
toggler.onclick = function(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
|
||||||
|
target.classList.toggle(className);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="cubify"></a><a href="#cubify" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Cubify</h1>
|
||||||
|
<p>The <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/ops/cubify.py">cubify operator</a> converts an 3D occupancy grid of shape <code>BxDxHxW</code>, where <code>B</code> is the batch size, into a mesh instantiated as a <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/structures/meshes.py">Meshes</a> data structure of <code>B</code> elements. The operator replaces every occupied voxel (if its occupancy probability is greater than a user defined threshold) with a cuboid of 12 faces and 8 vertices. Shared vertices are merged, and internal faces are removed resulting in a <strong>watertight</strong> mesh.</p>
|
||||||
|
<p>The operator provides three alignment modes {<em>topleft</em>, <em>corner</em>, <em>center</em>} which define the span of the mesh vertices with respect to the voxel grid. The alignment modes are described in the figure below for a 2D grid.</p>
|
||||||
|
<p><img src="https://user-images.githubusercontent.com/4369065/81032959-af697380-8e46-11ea-91a8-fae89597f988.png" alt="input"></p>
|
||||||
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/batching"><span class="arrow-prev">← </span><span>Batching</span></a><a class="docs-next button" href="/docs/renderer"><span>Overview</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
73
docs/datasets.html
Normal file
73
docs/datasets.html
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>datasets · PyTorch3D</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="# Data loaders for common 3D Datasets"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="datasets · PyTorch3D"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="# Data loaders for common 3D Datasets"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||||
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||||
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||||
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||||
|
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||||
|
|
||||||
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
|
ga('send', 'pageview');
|
||||||
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Data</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
|
var checkActiveCategory = true;
|
||||||
|
for (var i = 0; i < coll.length; i++) {
|
||||||
|
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||||
|
if (checkActiveCategory){
|
||||||
|
for (var j = 0; j < links.length; j++) {
|
||||||
|
if (links[j].classList.contains('navListItemActive')){
|
||||||
|
coll[i].nextElementSibling.classList.toggle('hide');
|
||||||
|
coll[i].childNodes[1].classList.toggle('rotate');
|
||||||
|
checkActiveCategory = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
coll[i].addEventListener('click', function() {
|
||||||
|
var arrow = this.childNodes[1];
|
||||||
|
arrow.classList.toggle('rotate');
|
||||||
|
var content = this.nextElementSibling;
|
||||||
|
content.classList.toggle('hide');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||||
|
createToggler('#tocToggler', 'body', 'tocActive');
|
||||||
|
|
||||||
|
var headings = document.querySelector('.toc-headings');
|
||||||
|
headings && headings.addEventListener('click', function(event) {
|
||||||
|
var el = event.target;
|
||||||
|
while(el !== headings){
|
||||||
|
if (el.tagName === 'A') {
|
||||||
|
document.body.classList.remove('tocActive');
|
||||||
|
break;
|
||||||
|
} else{
|
||||||
|
el = el.parentNode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, false);
|
||||||
|
|
||||||
|
function createToggler(togglerSelector, targetSelector, className) {
|
||||||
|
var toggler = document.querySelector(togglerSelector);
|
||||||
|
var target = document.querySelector(targetSelector);
|
||||||
|
|
||||||
|
if (!toggler) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
toggler.onclick = function(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
|
||||||
|
target.classList.toggle(className);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="data-loaders-for-common-3d-datasets"></a><a href="#data-loaders-for-common-3d-datasets" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Data loaders for common 3D Datasets</h1>
|
||||||
|
<h3><a class="anchor" aria-hidden="true" id="shapetnetcore"></a><a href="#shapetnetcore" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>ShapetNetCore</h3>
|
||||||
|
<p>ShapeNet is a dataset of 3D CAD models. ShapeNetCore is a subset of the ShapeNet dataset and can be downloaded from <a href="https://www.shapenet.org/">https://www.shapenet.org/</a>. There are two versions ShapeNetCore: v1 (55 categories) and v2 (57 categories).</p>
|
||||||
|
<p>The PyTorch3D <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/datasets/shapenet/shapenet_core.py">ShapeNetCore data loader</a> inherits from <code>torch.utils.data.Dataset</code>. It takes the path where the ShapeNetCore dataset is stored locally and loads models in the dataset. The ShapeNetCore class loads and returns models with their <code>categories</code>, <code>model_ids</code>, <code>vertices</code> and <code>faces</code>. The <code>ShapeNetCore</code> data loader also has a customized <code>render</code> function that renders models by the specified <code>model_ids (List[int])</code>, <code>categories (List[str])</code> or <code>indices (List[int])</code> with PyTorch3D's differentiable renderer.</p>
|
||||||
|
<p>The loaded dataset can be passed to <code>torch.utils.data.DataLoader</code> with PyTorch3D's customized collate_fn: <code>collate_batched_meshes</code> from the <code>pytorch3d.dataset.utils</code> module. The <code>vertices</code> and <code>faces</code> of the models are used to construct a <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/structures/meshes.py">Meshes</a> object representing the batched meshes. This <code>Meshes</code> representation can be easily used with other ops and rendering in PyTorch3D.</p>
|
||||||
|
<h3><a class="anchor" aria-hidden="true" id="r2n2"></a><a href="#r2n2" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>R2N2</h3>
|
||||||
|
<p>The R2N2 dataset contains 13 categories that are a subset of the ShapeNetCore v.1 dataset. The R2N2 dataset also contains its own 24 renderings of each object and voxelized models. The R2N2 Dataset can be downloaded following the instructions <a href="http://3d-r2n2.stanford.edu/">here</a>.</p>
|
||||||
|
<p>The PyTorch3D <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/datasets/r2n2/r2n2.py">R2N2 data loader</a> is initialized with the paths to the ShapeNet dataset, the R2N2 dataset and the splits file for R2N2. Just like <code>ShapeNetCore</code>, it can be passed to <code>torch.utils.data.DataLoader</code> with a customized collate_fn: <code>collate_batched_R2N2</code> from the <code>pytorch3d.dataset.r2n2.utils</code> module. It returns all the data that <code>ShapeNetCore</code> returns, and in addition, it returns the R2N2 renderings (24 views for each model) along with the camera calibration matrices and a voxel representation for each model. Similar to <code>ShapeNetCore</code>, it has a customized <code>render</code> function that supports rendering specified models with the PyTorch3D differentiable renderer. In addition, it supports rendering models with the same orientations as R2N2's original renderings.</p>
|
||||||
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/meshes_io"><span class="arrow-prev">← </span><span>Loading from file</span></a><a class="docs-next button" href="/docs/batching"><span>Batching</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
73
docs/datasets/index.html
Normal file
73
docs/datasets/index.html
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>datasets · PyTorch3D</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="# Data loaders for common 3D Datasets"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="datasets · PyTorch3D"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="# Data loaders for common 3D Datasets"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||||
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||||
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||||
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||||
|
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||||
|
|
||||||
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
|
ga('send', 'pageview');
|
||||||
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Data</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
|
var checkActiveCategory = true;
|
||||||
|
for (var i = 0; i < coll.length; i++) {
|
||||||
|
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||||
|
if (checkActiveCategory){
|
||||||
|
for (var j = 0; j < links.length; j++) {
|
||||||
|
if (links[j].classList.contains('navListItemActive')){
|
||||||
|
coll[i].nextElementSibling.classList.toggle('hide');
|
||||||
|
coll[i].childNodes[1].classList.toggle('rotate');
|
||||||
|
checkActiveCategory = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
coll[i].addEventListener('click', function() {
|
||||||
|
var arrow = this.childNodes[1];
|
||||||
|
arrow.classList.toggle('rotate');
|
||||||
|
var content = this.nextElementSibling;
|
||||||
|
content.classList.toggle('hide');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||||
|
createToggler('#tocToggler', 'body', 'tocActive');
|
||||||
|
|
||||||
|
var headings = document.querySelector('.toc-headings');
|
||||||
|
headings && headings.addEventListener('click', function(event) {
|
||||||
|
var el = event.target;
|
||||||
|
while(el !== headings){
|
||||||
|
if (el.tagName === 'A') {
|
||||||
|
document.body.classList.remove('tocActive');
|
||||||
|
break;
|
||||||
|
} else{
|
||||||
|
el = el.parentNode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, false);
|
||||||
|
|
||||||
|
function createToggler(togglerSelector, targetSelector, className) {
|
||||||
|
var toggler = document.querySelector(togglerSelector);
|
||||||
|
var target = document.querySelector(targetSelector);
|
||||||
|
|
||||||
|
if (!toggler) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
toggler.onclick = function(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
|
||||||
|
target.classList.toggle(className);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="data-loaders-for-common-3d-datasets"></a><a href="#data-loaders-for-common-3d-datasets" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Data loaders for common 3D Datasets</h1>
|
||||||
|
<h3><a class="anchor" aria-hidden="true" id="shapetnetcore"></a><a href="#shapetnetcore" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>ShapetNetCore</h3>
|
||||||
|
<p>ShapeNet is a dataset of 3D CAD models. ShapeNetCore is a subset of the ShapeNet dataset and can be downloaded from <a href="https://www.shapenet.org/">https://www.shapenet.org/</a>. There are two versions ShapeNetCore: v1 (55 categories) and v2 (57 categories).</p>
|
||||||
|
<p>The PyTorch3D <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/datasets/shapenet/shapenet_core.py">ShapeNetCore data loader</a> inherits from <code>torch.utils.data.Dataset</code>. It takes the path where the ShapeNetCore dataset is stored locally and loads models in the dataset. The ShapeNetCore class loads and returns models with their <code>categories</code>, <code>model_ids</code>, <code>vertices</code> and <code>faces</code>. The <code>ShapeNetCore</code> data loader also has a customized <code>render</code> function that renders models by the specified <code>model_ids (List[int])</code>, <code>categories (List[str])</code> or <code>indices (List[int])</code> with PyTorch3D's differentiable renderer.</p>
|
||||||
|
<p>The loaded dataset can be passed to <code>torch.utils.data.DataLoader</code> with PyTorch3D's customized collate_fn: <code>collate_batched_meshes</code> from the <code>pytorch3d.dataset.utils</code> module. The <code>vertices</code> and <code>faces</code> of the models are used to construct a <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/structures/meshes.py">Meshes</a> object representing the batched meshes. This <code>Meshes</code> representation can be easily used with other ops and rendering in PyTorch3D.</p>
|
||||||
|
<h3><a class="anchor" aria-hidden="true" id="r2n2"></a><a href="#r2n2" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>R2N2</h3>
|
||||||
|
<p>The R2N2 dataset contains 13 categories that are a subset of the ShapeNetCore v.1 dataset. The R2N2 dataset also contains its own 24 renderings of each object and voxelized models. The R2N2 Dataset can be downloaded following the instructions <a href="http://3d-r2n2.stanford.edu/">here</a>.</p>
|
||||||
|
<p>The PyTorch3D <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/datasets/r2n2/r2n2.py">R2N2 data loader</a> is initialized with the paths to the ShapeNet dataset, the R2N2 dataset and the splits file for R2N2. Just like <code>ShapeNetCore</code>, it can be passed to <code>torch.utils.data.DataLoader</code> with a customized collate_fn: <code>collate_batched_R2N2</code> from the <code>pytorch3d.dataset.r2n2.utils</code> module. It returns all the data that <code>ShapeNetCore</code> returns, and in addition, it returns the R2N2 renderings (24 views for each model) along with the camera calibration matrices and a voxel representation for each model. Similar to <code>ShapeNetCore</code>, it has a customized <code>render</code> function that supports rendering specified models with the PyTorch3D differentiable renderer. In addition, it supports rendering models with the same orientations as R2N2's original renderings.</p>
|
||||||
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/meshes_io"><span class="arrow-prev">← </span><span>Loading from file</span></a><a class="docs-next button" href="/docs/batching"><span>Batching</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Meshes</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Meshes</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Differentiable Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li></ul></div></div></section></div><script>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Data</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
var coll = document.getElementsByClassName('collapsible');
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
var checkActiveCategory = true;
|
var checkActiveCategory = true;
|
||||||
for (var i = 0; i < coll.length; i++) {
|
for (var i = 0; i < coll.length; i++) {
|
||||||
@ -116,4 +116,4 @@ are not triangles will be split into triangles. A Meshes object containing a
|
|||||||
single mesh can be created from this data using</p>
|
single mesh can be created from this data using</p>
|
||||||
<pre><code class="hljs"> meshes = <span class="hljs-constructor">Meshes(<span class="hljs-params">verts</span>=[<span class="hljs-params">verts</span>], <span class="hljs-params">faces</span>=[<span class="hljs-params">faces</span>])</span>
|
<pre><code class="hljs"> meshes = <span class="hljs-constructor">Meshes(<span class="hljs-params">verts</span>=[<span class="hljs-params">verts</span>], <span class="hljs-params">faces</span>=[<span class="hljs-params">faces</span>])</span>
|
||||||
</code></pre>
|
</code></pre>
|
||||||
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Jeremy Reizenstein</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/batching"><span class="arrow-prev">← </span><span>Batching</span></a><a class="docs-next button" href="/docs/renderer"><span>Overview</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#obj">OBJ</a></li><li><a href="#ply">PLY</a></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Jeremy Reizenstein</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/why_pytorch3d"><span class="arrow-prev">← </span><span class="function-name-prevnext">Why PyTorch3D</span></a><a class="docs-next button" href="/docs/datasets"><span>Data loaders</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#obj">OBJ</a></li><li><a href="#ply">PLY</a></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Meshes</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Meshes</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Differentiable Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li></ul></div></div></section></div><script>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Data</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
var coll = document.getElementsByClassName('collapsible');
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
var checkActiveCategory = true;
|
var checkActiveCategory = true;
|
||||||
for (var i = 0; i < coll.length; i++) {
|
for (var i = 0; i < coll.length; i++) {
|
||||||
@ -116,4 +116,4 @@ are not triangles will be split into triangles. A Meshes object containing a
|
|||||||
single mesh can be created from this data using</p>
|
single mesh can be created from this data using</p>
|
||||||
<pre><code class="hljs"> meshes = <span class="hljs-constructor">Meshes(<span class="hljs-params">verts</span>=[<span class="hljs-params">verts</span>], <span class="hljs-params">faces</span>=[<span class="hljs-params">faces</span>])</span>
|
<pre><code class="hljs"> meshes = <span class="hljs-constructor">Meshes(<span class="hljs-params">verts</span>=[<span class="hljs-params">verts</span>], <span class="hljs-params">faces</span>=[<span class="hljs-params">faces</span>])</span>
|
||||||
</code></pre>
|
</code></pre>
|
||||||
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Jeremy Reizenstein</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/batching"><span class="arrow-prev">← </span><span>Batching</span></a><a class="docs-next button" href="/docs/renderer"><span>Overview</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#obj">OBJ</a></li><li><a href="#ply">PLY</a></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Jeremy Reizenstein</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/why_pytorch3d"><span class="arrow-prev">← </span><span class="function-name-prevnext">Why PyTorch3D</span></a><a class="docs-next button" href="/docs/datasets"><span>Data loaders</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#obj">OBJ</a></li><li><a href="#ply">PLY</a></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Differentiable Renderer</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Meshes</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Differentiable Renderer</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li></ul></div></div></section></div><script>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Renderer</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
var coll = document.getElementsByClassName('collapsible');
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
var checkActiveCategory = true;
|
var checkActiveCategory = true;
|
||||||
for (var i = 0; i < coll.length; i++) {
|
for (var i = 0; i < coll.length; i++) {
|
||||||
@ -72,38 +72,15 @@
|
|||||||
<li>existing methods either do not support batching or assume that meshes in a batch have the same number of vertices and faces</li>
|
<li>existing methods either do not support batching or assume that meshes in a batch have the same number of vertices and faces</li>
|
||||||
<li>existing projects only provide CUDA implementations so they cannot be used without GPUs</li>
|
<li>existing projects only provide CUDA implementations so they cannot be used without GPUs</li>
|
||||||
</ul>
|
</ul>
|
||||||
<p>In order to experiment with different approaches, we wanted a modular implementation that is easy to use and extend, and supports <a href="/docs/batching">heterogeneous batching</a>.</p>
|
<p>In order to experiment with different approaches, we wanted a modular implementation that is easy to use and extend, and supports <a href="/docs/batching">heterogeneous batching</a>. Taking inspiration from existing work [<a href="#1">1</a>, <a href="#2">2</a>], we have created a new, modular, differentiable renderer with <strong>parallel implementations in PyTorch, C++ and CUDA</strong>, as well as comprehensive documentation and tests, with the aim of helping to further research in this field.</p>
|
||||||
<p>Taking inspiration from existing work [<a href="#1">1</a>, <a href="#2">2</a>], we have created a new, modular, differentiable renderer with <strong>parallel implementations in PyTorch, C++ and CUDA</strong>, as well as comprehensive documentation and tests, with the aim of helping to further research in this field.</p>
|
|
||||||
<p>Our implementation decouples the rasterization and shading steps of rendering. The core rasterization step (based on <a href="#2">[2]</a>) returns several intermediate variables and has an optimized implementation in CUDA. The rest of the pipeline is implemented purely in PyTorch, and is designed to be customized and extended. With this approach, the PyTorch3D differentiable renderer can be imported as a library.</p>
|
<p>Our implementation decouples the rasterization and shading steps of rendering. The core rasterization step (based on <a href="#2">[2]</a>) returns several intermediate variables and has an optimized implementation in CUDA. The rest of the pipeline is implemented purely in PyTorch, and is designed to be customized and extended. With this approach, the PyTorch3D differentiable renderer can be imported as a library.</p>
|
||||||
<h2><a class="anchor" aria-hidden="true" id="uget-startedu"></a><a href="#uget-startedu" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><u>Get started</u></h2>
|
<h2><a class="anchor" aria-hidden="true" id="uget-startedu"></a><a href="#uget-startedu" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><u>Get started</u></h2>
|
||||||
<p>To learn about more the implementation and start using the renderer refer to <a href="renderer_getting_started.md">/docs/renderer_getting_started</a>, which also contains the <a href="/docs/assets/architecture_overview.png">architecture overview</a> and <a href="/docs/assets/transformations_overview.png">coordinate transformation conventions</a>.</p>
|
<p>To learn about more the implementation and start using the renderer refer to <a href="/docs/renderer_getting_started">getting started with renderer</a>, which also contains the <a href="/docs/assets/architecture_overview.png">architecture overview</a> and <a href="/docs/assets/transformations_overview.png">coordinate transformation conventions</a>.</p>
|
||||||
<h2><a class="anchor" aria-hidden="true" id="ukey-featuresu"></a><a href="#ukey-featuresu" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><u>Key features</u></h2>
|
<h2><a class="anchor" aria-hidden="true" id="utech-reportu"></a><a href="#utech-reportu" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><u>Tech Report</u></h2>
|
||||||
<h3><a class="anchor" aria-hidden="true" id="1-cuda-support-for-fast-rasterization-of-large-meshes"></a><a href="#1-cuda-support-for-fast-rasterization-of-large-meshes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>1. CUDA support for fast rasterization of large meshes</h3>
|
<p>For an in depth explanation of the renderer design, key features and benchmarks please refer to the PyTorch3D Technical Report on ArXiv: <a href="https://arxiv.org/abs/2007.08501">Accelerating 3D Deep Learning with PyTorch3D</a></p>
|
||||||
<p>We implemented modular CUDA kernels for the forward and backward pass of rasterization, adaptating a traditional graphics approach known as "coarse-to-fine" rasterization.</p>
|
|
||||||
<p>First, the image is divided into a coarse grid and mesh faces are allocated to the grid cell in which they occur. This is followed by a refinement step which does pixel wise rasterization of the reduced subset of faces per grid cell. The grid cell size is a parameter which can be varied (<code>bin_size</code>).</p>
|
|
||||||
<p>We additionally introduce a parameter <code>faces_per_pixel</code> which allows users to specify the top K faces which should be returned per pixel in the image (as opposed to traditional rasterization which returns only the index of the closest face in the mesh per pixel). The top K face properties can then be aggregated using different methods (such as the sigmoid/softmax approach proposed by Li et at in SoftRasterizer <a href="#2">[2]</a>).</p>
|
|
||||||
<p>We compared PyTorch3D with SoftRasterizer to measure the effect of both these design changes on the speed of rasterization. We selected a set of meshes of different sizes from ShapeNetV1 core, and rasterized one mesh in each batch to produce images of different sizes. We report the speed of the forward and backward passes.</p>
|
|
||||||
<p><strong>Fig 1: PyTorch3D Naive vs Coarse-to-fine</strong></p>
|
|
||||||
<p>This figure shows how the coarse-to-fine strategy for rasterization results in significant speed up compared to naive rasterization for large image size and large mesh sizes.</p>
|
|
||||||
<p><img src="assets/p3d_naive_vs_coarse.png" width="1000"></p>
|
|
||||||
<p>For small mesh and image sizes, the naive approach is slightly faster. We advise that you understand the data you are using and choose the rasterization setting which suits your performance requirements. It is easy to switch between the naive and coarse-to-fine options by adjusting the <code>bin_size</code> value when initializing the <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/renderer/mesh/rasterizer.py#L26">rasterization settings</a>.</p>
|
|
||||||
<p>Setting <code>bin_size = 0</code> will enable naive rasterization. If <code>bin_size > 0</code>, the coarse-to-fine approach is used. The default is <code>bin_size = None</code> in which case we set the bin size based on <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/renderer/mesh/rasterize_meshes.py#L92">heuristics</a>.</p>
|
|
||||||
<p><strong>Fig 2: PyTorch3D Coarse-to-fine vs SoftRasterizer</strong></p>
|
|
||||||
<p>This figure shows the effect of the <em>combination</em> of coarse-to-fine rasterization and caching the faces rasterized per pixel returned from the forward pass. For large meshes and image sizes, we again observe that the PyTorch3D rasterizer is significantly faster, noting that the speed is dominated by the forward pass and the backward pass is very fast.</p>
|
|
||||||
<p>In the SoftRasterizer implementation, in both the forward and backward pass, there is a loop over every single face in the mesh for every pixel in the image. Therefore, the time for the full forward plus backward pass is ~2x the time for the forward pass. For small mesh and image sizes, the SoftRasterizer approach is slightly faster.</p>
|
|
||||||
<p><img src="assets/p3d_vs_softras.png" width="1000"></p>
|
|
||||||
<h3><a class="anchor" aria-hidden="true" id="2-support-for-heterogeneous-batches"></a><a href="#2-support-for-heterogeneous-batches" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>2. Support for Heterogeneous Batches</h3>
|
|
||||||
<p>PyTorch3D supports efficient rendering of batches of meshes where each mesh has different numbers of vertices and faces. This is done without using padded inputs.</p>
|
|
||||||
<p>We again compare with SoftRasterizer which only supports batches of homogeneous meshes and test two cases: 1) a for loop over meshes in the batch, 2) padded inputs, and compare with the native heterogeneous batching support in PyTorch3D.</p>
|
|
||||||
<p>We group meshes from ShapeNet into bins based on the number of faces in the mesh, and sample to compose a batch. We then render images of fixed size and measure the speed of the forward and backward passes.</p>
|
|
||||||
<p>We tested with a range of increasingly large meshes and bin sizes.</p>
|
|
||||||
<p><strong>Fig 3: PyTorch3D heterogeneous batching compared with SoftRasterizer</strong></p>
|
|
||||||
<p><img src="assets/fullset_batch_size_16.png" width="700"/></p>
|
|
||||||
<p>This shows that for large meshes and large bin width (i.e. more variation in mesh size in the batch) the heterogeneous batching approach in PyTorch3D is faster than either of the workarounds with SoftRasterizer.</p>
|
|
||||||
<p>(settings: batch size = 16, mesh sizes in bins ranging from 500-350k faces, image size = 64, faces per pixel = 100)</p>
|
|
||||||
<hr>
|
<hr>
|
||||||
<p><strong>NOTE: CUDA Memory usage</strong></p>
|
<p><strong>NOTE: CUDA Memory usage</strong></p>
|
||||||
<p>The SoftRasterizer forward CUDA kernel only outputs one <code>(N, H, W, 4)</code> FloatTensor compared with the PyTorch3D rasterizer forward CUDA kernel which outputs 4 tensors:</p>
|
<p>The main comparison in the Technical Report is with SoftRasterizer [<a href="#2">2</a>]. The SoftRasterizer forward CUDA kernel only outputs one <code>(N, H, W, 4)</code> FloatTensor compared with the PyTorch3D rasterizer forward CUDA kernel which outputs 4 tensors:</p>
|
||||||
<ul>
|
<ul>
|
||||||
<li><code>pix_to_face</code>, LongTensor <code>(N, H, W, K)</code></li>
|
<li><code>pix_to_face</code>, LongTensor <code>(N, H, W, K)</code></li>
|
||||||
<li><code>zbuf</code>, FloatTensor <code>(N, H, W, K)</code></li>
|
<li><code>zbuf</code>, FloatTensor <code>(N, H, W, K)</code></li>
|
||||||
@ -123,8 +100,6 @@ total_memory = memory_forward_pass + memory_backward_pass
|
|||||||
</code></pre>
|
</code></pre>
|
||||||
<p>We need 48 bytes per face per pixel of the rasterized output. In order to remain within bounds for memory usage we can vary the batch size (<strong>N</strong>), image size (<strong>H/W</strong>) and faces per pixel (<strong>K</strong>). For example, for a fixed batch size, if using a larger image size, try reducing the faces per pixel.</p>
|
<p>We need 48 bytes per face per pixel of the rasterized output. In order to remain within bounds for memory usage we can vary the batch size (<strong>N</strong>), image size (<strong>H/W</strong>) and faces per pixel (<strong>K</strong>). For example, for a fixed batch size, if using a larger image size, try reducing the faces per pixel.</p>
|
||||||
<hr>
|
<hr>
|
||||||
<h3><a class="anchor" aria-hidden="true" id="3-modular-design-for-easy-experimentation-and-extensibility"></a><a href="#3-modular-design-for-easy-experimentation-and-extensibility" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>3. Modular design for easy experimentation and extensibility.</h3>
|
|
||||||
<p>We redesigned the rendering pipeline from the ground up to be modular and extensible and challenged many of the limitations in existing libraries. Refer to <a href="renderer_getting_started.md">/docs/renderer_getting_started</a> for a detailed description of the architecture.</p>
|
|
||||||
<h3><a class="anchor" aria-hidden="true" id="references"></a><a href="#references" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>References</h3>
|
<h3><a class="anchor" aria-hidden="true" id="references"></a><a href="#references" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>References</h3>
|
||||||
<p><a id="1">[1]</a> Kato et al, 'Neural 3D Mesh Renderer', CVPR 2018</p>
|
<p><a id="1">[1]</a> Kato et al, 'Neural 3D Mesh Renderer', CVPR 2018</p>
|
||||||
<p><a id="2">[2]</a> Liu et al, 'Soft Rasterizer: A Differentiable Renderer for Image-based 3D Reasoning', ICCV 2019</p>
|
<p><a id="2">[2]</a> Liu et al, 'Soft Rasterizer: A Differentiable Renderer for Image-based 3D Reasoning', ICCV 2019</p>
|
||||||
@ -134,4 +109,4 @@ total_memory = memory_forward_pass + memory_backward_pass
|
|||||||
<p><a id="6">[6]</a> Yifan et al, 'Differentiable Surface Splatting for Point-based Geometry Processing', SIGGRAPH Asia 2019</p>
|
<p><a id="6">[6]</a> Yifan et al, 'Differentiable Surface Splatting for Point-based Geometry Processing', SIGGRAPH Asia 2019</p>
|
||||||
<p><a id="7">[7]</a> Loubet et al, 'Reparameterizing Discontinuous Integrands for Differentiable Rendering', SIGGRAPH Asia 2019</p>
|
<p><a id="7">[7]</a> Loubet et al, 'Reparameterizing Discontinuous Integrands for Differentiable Rendering', SIGGRAPH Asia 2019</p>
|
||||||
<p><a id="8">[8]</a> Chen et al, 'Learning to Predict 3D Objects with an Interpolation-based Differentiable Renderer', NeurIPS 2019</p>
|
<p><a id="8">[8]</a> Chen et al, 'Learning to Predict 3D Objects with an Interpolation-based Differentiable Renderer', NeurIPS 2019</p>
|
||||||
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Patrick Labatut</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/meshes_io"><span class="arrow-prev">← </span><span>Loading from file</span></a><a class="docs-next button" href="/docs/renderer_getting_started"><span>Getting Started</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#uget-startedu"><u>Get started</u></a></li><li><a href="#ukey-featuresu"><u>Key features</u></a><ul class="toc-headings"><li><a href="#1-cuda-support-for-fast-rasterization-of-large-meshes">1. CUDA support for fast rasterization of large meshes</a></li><li><a href="#2-support-for-heterogeneous-batches">2. Support for Heterogeneous Batches</a></li><li><a href="#3-modular-design-for-easy-experimentation-and-extensibility">3. Modular design for easy experimentation and extensibility.</a></li><li><a href="#references">References</a></li></ul></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/cubify"><span class="arrow-prev">← </span><span>Cubify</span></a><a class="docs-next button" href="/docs/renderer_getting_started"><span>Getting Started</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#uget-startedu"><u>Get started</u></a></li><li><a href="#utech-reportu"><u>Tech Report</u></a><ul class="toc-headings"><li><a href="#references">References</a></li></ul></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Differentiable Renderer</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Meshes</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Differentiable Renderer</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li></ul></div></div></section></div><script>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Renderer</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
var coll = document.getElementsByClassName('collapsible');
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
var checkActiveCategory = true;
|
var checkActiveCategory = true;
|
||||||
for (var i = 0; i < coll.length; i++) {
|
for (var i = 0; i < coll.length; i++) {
|
||||||
@ -72,38 +72,15 @@
|
|||||||
<li>existing methods either do not support batching or assume that meshes in a batch have the same number of vertices and faces</li>
|
<li>existing methods either do not support batching or assume that meshes in a batch have the same number of vertices and faces</li>
|
||||||
<li>existing projects only provide CUDA implementations so they cannot be used without GPUs</li>
|
<li>existing projects only provide CUDA implementations so they cannot be used without GPUs</li>
|
||||||
</ul>
|
</ul>
|
||||||
<p>In order to experiment with different approaches, we wanted a modular implementation that is easy to use and extend, and supports <a href="/docs/batching">heterogeneous batching</a>.</p>
|
<p>In order to experiment with different approaches, we wanted a modular implementation that is easy to use and extend, and supports <a href="/docs/batching">heterogeneous batching</a>. Taking inspiration from existing work [<a href="#1">1</a>, <a href="#2">2</a>], we have created a new, modular, differentiable renderer with <strong>parallel implementations in PyTorch, C++ and CUDA</strong>, as well as comprehensive documentation and tests, with the aim of helping to further research in this field.</p>
|
||||||
<p>Taking inspiration from existing work [<a href="#1">1</a>, <a href="#2">2</a>], we have created a new, modular, differentiable renderer with <strong>parallel implementations in PyTorch, C++ and CUDA</strong>, as well as comprehensive documentation and tests, with the aim of helping to further research in this field.</p>
|
|
||||||
<p>Our implementation decouples the rasterization and shading steps of rendering. The core rasterization step (based on <a href="#2">[2]</a>) returns several intermediate variables and has an optimized implementation in CUDA. The rest of the pipeline is implemented purely in PyTorch, and is designed to be customized and extended. With this approach, the PyTorch3D differentiable renderer can be imported as a library.</p>
|
<p>Our implementation decouples the rasterization and shading steps of rendering. The core rasterization step (based on <a href="#2">[2]</a>) returns several intermediate variables and has an optimized implementation in CUDA. The rest of the pipeline is implemented purely in PyTorch, and is designed to be customized and extended. With this approach, the PyTorch3D differentiable renderer can be imported as a library.</p>
|
||||||
<h2><a class="anchor" aria-hidden="true" id="uget-startedu"></a><a href="#uget-startedu" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><u>Get started</u></h2>
|
<h2><a class="anchor" aria-hidden="true" id="uget-startedu"></a><a href="#uget-startedu" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><u>Get started</u></h2>
|
||||||
<p>To learn about more the implementation and start using the renderer refer to <a href="renderer_getting_started.md">/docs/renderer_getting_started</a>, which also contains the <a href="/docs/assets/architecture_overview.png">architecture overview</a> and <a href="/docs/assets/transformations_overview.png">coordinate transformation conventions</a>.</p>
|
<p>To learn about more the implementation and start using the renderer refer to <a href="/docs/renderer_getting_started">getting started with renderer</a>, which also contains the <a href="/docs/assets/architecture_overview.png">architecture overview</a> and <a href="/docs/assets/transformations_overview.png">coordinate transformation conventions</a>.</p>
|
||||||
<h2><a class="anchor" aria-hidden="true" id="ukey-featuresu"></a><a href="#ukey-featuresu" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><u>Key features</u></h2>
|
<h2><a class="anchor" aria-hidden="true" id="utech-reportu"></a><a href="#utech-reportu" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a><u>Tech Report</u></h2>
|
||||||
<h3><a class="anchor" aria-hidden="true" id="1-cuda-support-for-fast-rasterization-of-large-meshes"></a><a href="#1-cuda-support-for-fast-rasterization-of-large-meshes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>1. CUDA support for fast rasterization of large meshes</h3>
|
<p>For an in depth explanation of the renderer design, key features and benchmarks please refer to the PyTorch3D Technical Report on ArXiv: <a href="https://arxiv.org/abs/2007.08501">Accelerating 3D Deep Learning with PyTorch3D</a></p>
|
||||||
<p>We implemented modular CUDA kernels for the forward and backward pass of rasterization, adaptating a traditional graphics approach known as "coarse-to-fine" rasterization.</p>
|
|
||||||
<p>First, the image is divided into a coarse grid and mesh faces are allocated to the grid cell in which they occur. This is followed by a refinement step which does pixel wise rasterization of the reduced subset of faces per grid cell. The grid cell size is a parameter which can be varied (<code>bin_size</code>).</p>
|
|
||||||
<p>We additionally introduce a parameter <code>faces_per_pixel</code> which allows users to specify the top K faces which should be returned per pixel in the image (as opposed to traditional rasterization which returns only the index of the closest face in the mesh per pixel). The top K face properties can then be aggregated using different methods (such as the sigmoid/softmax approach proposed by Li et at in SoftRasterizer <a href="#2">[2]</a>).</p>
|
|
||||||
<p>We compared PyTorch3D with SoftRasterizer to measure the effect of both these design changes on the speed of rasterization. We selected a set of meshes of different sizes from ShapeNetV1 core, and rasterized one mesh in each batch to produce images of different sizes. We report the speed of the forward and backward passes.</p>
|
|
||||||
<p><strong>Fig 1: PyTorch3D Naive vs Coarse-to-fine</strong></p>
|
|
||||||
<p>This figure shows how the coarse-to-fine strategy for rasterization results in significant speed up compared to naive rasterization for large image size and large mesh sizes.</p>
|
|
||||||
<p><img src="assets/p3d_naive_vs_coarse.png" width="1000"></p>
|
|
||||||
<p>For small mesh and image sizes, the naive approach is slightly faster. We advise that you understand the data you are using and choose the rasterization setting which suits your performance requirements. It is easy to switch between the naive and coarse-to-fine options by adjusting the <code>bin_size</code> value when initializing the <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/renderer/mesh/rasterizer.py#L26">rasterization settings</a>.</p>
|
|
||||||
<p>Setting <code>bin_size = 0</code> will enable naive rasterization. If <code>bin_size > 0</code>, the coarse-to-fine approach is used. The default is <code>bin_size = None</code> in which case we set the bin size based on <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/renderer/mesh/rasterize_meshes.py#L92">heuristics</a>.</p>
|
|
||||||
<p><strong>Fig 2: PyTorch3D Coarse-to-fine vs SoftRasterizer</strong></p>
|
|
||||||
<p>This figure shows the effect of the <em>combination</em> of coarse-to-fine rasterization and caching the faces rasterized per pixel returned from the forward pass. For large meshes and image sizes, we again observe that the PyTorch3D rasterizer is significantly faster, noting that the speed is dominated by the forward pass and the backward pass is very fast.</p>
|
|
||||||
<p>In the SoftRasterizer implementation, in both the forward and backward pass, there is a loop over every single face in the mesh for every pixel in the image. Therefore, the time for the full forward plus backward pass is ~2x the time for the forward pass. For small mesh and image sizes, the SoftRasterizer approach is slightly faster.</p>
|
|
||||||
<p><img src="assets/p3d_vs_softras.png" width="1000"></p>
|
|
||||||
<h3><a class="anchor" aria-hidden="true" id="2-support-for-heterogeneous-batches"></a><a href="#2-support-for-heterogeneous-batches" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>2. Support for Heterogeneous Batches</h3>
|
|
||||||
<p>PyTorch3D supports efficient rendering of batches of meshes where each mesh has different numbers of vertices and faces. This is done without using padded inputs.</p>
|
|
||||||
<p>We again compare with SoftRasterizer which only supports batches of homogeneous meshes and test two cases: 1) a for loop over meshes in the batch, 2) padded inputs, and compare with the native heterogeneous batching support in PyTorch3D.</p>
|
|
||||||
<p>We group meshes from ShapeNet into bins based on the number of faces in the mesh, and sample to compose a batch. We then render images of fixed size and measure the speed of the forward and backward passes.</p>
|
|
||||||
<p>We tested with a range of increasingly large meshes and bin sizes.</p>
|
|
||||||
<p><strong>Fig 3: PyTorch3D heterogeneous batching compared with SoftRasterizer</strong></p>
|
|
||||||
<p><img src="assets/fullset_batch_size_16.png" width="700"/></p>
|
|
||||||
<p>This shows that for large meshes and large bin width (i.e. more variation in mesh size in the batch) the heterogeneous batching approach in PyTorch3D is faster than either of the workarounds with SoftRasterizer.</p>
|
|
||||||
<p>(settings: batch size = 16, mesh sizes in bins ranging from 500-350k faces, image size = 64, faces per pixel = 100)</p>
|
|
||||||
<hr>
|
<hr>
|
||||||
<p><strong>NOTE: CUDA Memory usage</strong></p>
|
<p><strong>NOTE: CUDA Memory usage</strong></p>
|
||||||
<p>The SoftRasterizer forward CUDA kernel only outputs one <code>(N, H, W, 4)</code> FloatTensor compared with the PyTorch3D rasterizer forward CUDA kernel which outputs 4 tensors:</p>
|
<p>The main comparison in the Technical Report is with SoftRasterizer [<a href="#2">2</a>]. The SoftRasterizer forward CUDA kernel only outputs one <code>(N, H, W, 4)</code> FloatTensor compared with the PyTorch3D rasterizer forward CUDA kernel which outputs 4 tensors:</p>
|
||||||
<ul>
|
<ul>
|
||||||
<li><code>pix_to_face</code>, LongTensor <code>(N, H, W, K)</code></li>
|
<li><code>pix_to_face</code>, LongTensor <code>(N, H, W, K)</code></li>
|
||||||
<li><code>zbuf</code>, FloatTensor <code>(N, H, W, K)</code></li>
|
<li><code>zbuf</code>, FloatTensor <code>(N, H, W, K)</code></li>
|
||||||
@ -123,8 +100,6 @@ total_memory = memory_forward_pass + memory_backward_pass
|
|||||||
</code></pre>
|
</code></pre>
|
||||||
<p>We need 48 bytes per face per pixel of the rasterized output. In order to remain within bounds for memory usage we can vary the batch size (<strong>N</strong>), image size (<strong>H/W</strong>) and faces per pixel (<strong>K</strong>). For example, for a fixed batch size, if using a larger image size, try reducing the faces per pixel.</p>
|
<p>We need 48 bytes per face per pixel of the rasterized output. In order to remain within bounds for memory usage we can vary the batch size (<strong>N</strong>), image size (<strong>H/W</strong>) and faces per pixel (<strong>K</strong>). For example, for a fixed batch size, if using a larger image size, try reducing the faces per pixel.</p>
|
||||||
<hr>
|
<hr>
|
||||||
<h3><a class="anchor" aria-hidden="true" id="3-modular-design-for-easy-experimentation-and-extensibility"></a><a href="#3-modular-design-for-easy-experimentation-and-extensibility" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>3. Modular design for easy experimentation and extensibility.</h3>
|
|
||||||
<p>We redesigned the rendering pipeline from the ground up to be modular and extensible and challenged many of the limitations in existing libraries. Refer to <a href="renderer_getting_started.md">/docs/renderer_getting_started</a> for a detailed description of the architecture.</p>
|
|
||||||
<h3><a class="anchor" aria-hidden="true" id="references"></a><a href="#references" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>References</h3>
|
<h3><a class="anchor" aria-hidden="true" id="references"></a><a href="#references" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>References</h3>
|
||||||
<p><a id="1">[1]</a> Kato et al, 'Neural 3D Mesh Renderer', CVPR 2018</p>
|
<p><a id="1">[1]</a> Kato et al, 'Neural 3D Mesh Renderer', CVPR 2018</p>
|
||||||
<p><a id="2">[2]</a> Liu et al, 'Soft Rasterizer: A Differentiable Renderer for Image-based 3D Reasoning', ICCV 2019</p>
|
<p><a id="2">[2]</a> Liu et al, 'Soft Rasterizer: A Differentiable Renderer for Image-based 3D Reasoning', ICCV 2019</p>
|
||||||
@ -134,4 +109,4 @@ total_memory = memory_forward_pass + memory_backward_pass
|
|||||||
<p><a id="6">[6]</a> Yifan et al, 'Differentiable Surface Splatting for Point-based Geometry Processing', SIGGRAPH Asia 2019</p>
|
<p><a id="6">[6]</a> Yifan et al, 'Differentiable Surface Splatting for Point-based Geometry Processing', SIGGRAPH Asia 2019</p>
|
||||||
<p><a id="7">[7]</a> Loubet et al, 'Reparameterizing Discontinuous Integrands for Differentiable Rendering', SIGGRAPH Asia 2019</p>
|
<p><a id="7">[7]</a> Loubet et al, 'Reparameterizing Discontinuous Integrands for Differentiable Rendering', SIGGRAPH Asia 2019</p>
|
||||||
<p><a id="8">[8]</a> Chen et al, 'Learning to Predict 3D Objects with an Interpolation-based Differentiable Renderer', NeurIPS 2019</p>
|
<p><a id="8">[8]</a> Chen et al, 'Learning to Predict 3D Objects with an Interpolation-based Differentiable Renderer', NeurIPS 2019</p>
|
||||||
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Patrick Labatut</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/meshes_io"><span class="arrow-prev">← </span><span>Loading from file</span></a><a class="docs-next button" href="/docs/renderer_getting_started"><span>Getting Started</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#uget-startedu"><u>Get started</u></a></li><li><a href="#ukey-featuresu"><u>Key features</u></a><ul class="toc-headings"><li><a href="#1-cuda-support-for-fast-rasterization-of-large-meshes">1. CUDA support for fast rasterization of large meshes</a></li><li><a href="#2-support-for-heterogeneous-batches">2. Support for Heterogeneous Batches</a></li><li><a href="#3-modular-design-for-easy-experimentation-and-extensibility">3. Modular design for easy experimentation and extensibility.</a></li><li><a href="#references">References</a></li></ul></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/cubify"><span class="arrow-prev">← </span><span>Cubify</span></a><a class="docs-next button" href="/docs/renderer_getting_started"><span>Getting Started</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#uget-startedu"><u>Get started</u></a></li><li><a href="#utech-reportu"><u>Tech Report</u></a><ul class="toc-headings"><li><a href="#references">References</a></li></ul></li></ul></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -1,4 +1,4 @@
|
|||||||
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>renderer_getting_started · PyTorch3D</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="# Renderer Getting Started"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="renderer_getting_started · PyTorch3D"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="# Renderer Getting Started"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>renderer_getting_started · PyTorch3D</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="# Getting Started With Renderer"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="renderer_getting_started · PyTorch3D"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="# Getting Started With Renderer"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||||
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||||
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Differentiable Renderer</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Meshes</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Differentiable Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li></ul></div></div></section></div><script>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Renderer</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
var coll = document.getElementsByClassName('collapsible');
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
var checkActiveCategory = true;
|
var checkActiveCategory = true;
|
||||||
for (var i = 0; i < coll.length; i++) {
|
for (var i = 0; i < coll.length; i++) {
|
||||||
@ -62,10 +62,10 @@
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="renderer-getting-started"></a><a href="#renderer-getting-started" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Renderer Getting Started</h1>
|
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="getting-started-with-renderer"></a><a href="#getting-started-with-renderer" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Getting Started With Renderer</h1>
|
||||||
<h3><a class="anchor" aria-hidden="true" id="architecture-overview"></a><a href="#architecture-overview" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Architecture Overview</h3>
|
<h3><a class="anchor" aria-hidden="true" id="architecture-overview"></a><a href="#architecture-overview" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Architecture Overview</h3>
|
||||||
<p>The renderer is designed to be modular, extensible and support batching and gradients for all inputs. The following figure describes all the components of the rendering pipeline.</p>
|
<p>The renderer is designed to be modular, extensible and support batching and gradients for all inputs. The following figure describes all the components of the rendering pipeline.</p>
|
||||||
<p><img src="assets/architecture_overview.png" width="1000"></p>
|
<p><img src="assets/architecture_renderer.jpg" width="1000"></p>
|
||||||
<h5><a class="anchor" aria-hidden="true" id="fragments"></a><a href="#fragments" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Fragments</h5>
|
<h5><a class="anchor" aria-hidden="true" id="fragments"></a><a href="#fragments" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Fragments</h5>
|
||||||
<p>The <strong>rasterizer</strong> returns 4 output tensors in a named tuple.</p>
|
<p>The <strong>rasterizer</strong> returns 4 output tensors in a named tuple.</p>
|
||||||
<ul>
|
<ul>
|
||||||
@ -82,9 +82,9 @@ giving the barycentric coordinates in NDC units of the nearest faces at each pix
|
|||||||
<hr>
|
<hr>
|
||||||
<h3><a class="anchor" aria-hidden="true" id="coordinate-transformation-conventions"></a><a href="#coordinate-transformation-conventions" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Coordinate transformation conventions</h3>
|
<h3><a class="anchor" aria-hidden="true" id="coordinate-transformation-conventions"></a><a href="#coordinate-transformation-conventions" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Coordinate transformation conventions</h3>
|
||||||
<p>Rendering requires transformations between several different coordinate frames: world space, view/camera space, NDC space and screen space. At each step it is important to know where the camera is located, how the +X, +Y, +Z axes are aligned and the possible range of values. The following figure outlines the conventions used PyTorch3D.</p>
|
<p>Rendering requires transformations between several different coordinate frames: world space, view/camera space, NDC space and screen space. At each step it is important to know where the camera is located, how the +X, +Y, +Z axes are aligned and the possible range of values. The following figure outlines the conventions used PyTorch3D.</p>
|
||||||
<p><img src="assets/transformations_overview.png" width="1000"></p>
|
<p><img src="assets/transforms_overview.jpg" width="1000"></p>
|
||||||
<p>For example, given a teapot mesh, the world coordinate frame, camera coordiante frame and image are show in the figure below. Note that the world and camera coordinate frames have the +z direction pointing in to the page.</p>
|
<p>For example, given a teapot mesh, the world coordinate frame, camera coordiante frame and image are show in the figure below. Note that the world and camera coordinate frames have the +z direction pointing in to the page.</p>
|
||||||
<p><img src="assets/world_camera_image.png" width="1000"></p>
|
<p><img src="assets/world_camera_image.jpg" width="1000"></p>
|
||||||
<hr>
|
<hr>
|
||||||
<p><strong>NOTE: PyTorch3D vs OpenGL</strong></p>
|
<p><strong>NOTE: PyTorch3D vs OpenGL</strong></p>
|
||||||
<p>While we tried to emulate several aspects of OpenGL, there are differences in the coordinate frame conventions.</p>
|
<p>While we tried to emulate several aspects of OpenGL, there are differences in the coordinate frame conventions.</p>
|
||||||
@ -94,18 +94,26 @@ giving the barycentric coordinates in NDC units of the nearest faces at each pix
|
|||||||
</ul>
|
</ul>
|
||||||
<p><img align="center" src="assets/opengl_coordframes.png" width="300"></p>
|
<p><img align="center" src="assets/opengl_coordframes.png" width="300"></p>
|
||||||
<hr>
|
<hr>
|
||||||
|
<h3><a class="anchor" aria-hidden="true" id="texturing-options"></a><a href="#texturing-options" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Texturing options</h3>
|
||||||
|
<p>For mesh texturing we offer several options (in <code>pytorch3d/renderer/mesh/texturing.py</code>):</p>
|
||||||
|
<ol>
|
||||||
|
<li><strong>Vertex Textures</strong>: D dimensional textures for each vertex (for example an RGB color) which can be interpolated across the face. This can be represented as an <code>(N, V, D)</code> tensor. This is a fairly simple representation though and cannot model complex textures if the mesh faces are large.</li>
|
||||||
|
<li><strong>UV Textures</strong>: vertex UV coordinates and <strong>one</strong> texture map for the whole face. For a point on a face with given barycentric coordinates, the face color can be computed by interpolating the vertex uv coordinates and then sampling from the texture map. This representation requires two tensors (UVs: <code>(N, V, 2), Texture map:</code>(N, H, W, 3)`), and is limited to only support one texture map per mesh.</li>
|
||||||
|
<li><strong>Face Textures</strong>: In more complex cases such as ShapeNet meshes, there are multiple texture maps per mesh and some faces have texture while other do not. For these cases, a more flexible representation is a texture atlas, where each face is represented as an <code>(RxR)</code> texture map where R is the texture resolution. For a given point on the face, the texture value can be sampled from the per face texture map using the barycentric coordinates of the point. This representation requires one tensor of shape <code>(N, F, R, R, 3)</code>. This texturing method is inspired by the SoftRasterizer implementation. For more details refer to the <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/io/mtl_io.py#L123"><code>make_material_atlas</code></a> and <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/renderer/mesh/textures.py#L452"><code>sample_textures</code></a> functions.</li>
|
||||||
|
</ol>
|
||||||
|
<p><img src="assets/texturing.jpg" width="1000"></p>
|
||||||
<h3><a class="anchor" aria-hidden="true" id="a-simple-renderer"></a><a href="#a-simple-renderer" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>A simple renderer</h3>
|
<h3><a class="anchor" aria-hidden="true" id="a-simple-renderer"></a><a href="#a-simple-renderer" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>A simple renderer</h3>
|
||||||
<p>A renderer in PyTorch3D is composed of a <strong>rasterizer</strong> and a <strong>shader</strong>. Create a renderer in a few simple steps:</p>
|
<p>A renderer in PyTorch3D is composed of a <strong>rasterizer</strong> and a <strong>shader</strong>. Create a renderer in a few simple steps:</p>
|
||||||
<pre><code class="hljs"><span class="hljs-comment"># Imports</span>
|
<pre><code class="hljs"><span class="hljs-comment"># Imports</span>
|
||||||
<span class="hljs-keyword">from</span> pytorch3d.renderer import (
|
<span class="hljs-keyword">from</span> pytorch3d.renderer import (
|
||||||
OpenGLPerspectiveCameras, look_at_view_transform,
|
FoVPerspectiveCameras, look_at_view_transform,
|
||||||
RasterizationSettings, BlendParams,
|
RasterizationSettings, BlendParams,
|
||||||
MeshRenderer, MeshRasterizer, HardPhongShader
|
MeshRenderer, MeshRasterizer, HardPhongShader
|
||||||
)
|
)
|
||||||
|
|
||||||
<span class="hljs-comment"># Initialize an OpenGL perspective camera.</span>
|
<span class="hljs-comment"># Initialize an OpenGL perspective camera.</span>
|
||||||
R, T = look_at_view_transform(2.7, 10, 20)
|
R, T = look_at_view_transform(2.7, 10, 20)
|
||||||
cameras = OpenGLPerspectiveCameras(<span class="hljs-attribute">device</span>=device, <span class="hljs-attribute">R</span>=R, <span class="hljs-attribute">T</span>=T)
|
cameras = FoVPerspectiveCameras(<span class="hljs-attribute">device</span>=device, <span class="hljs-attribute">R</span>=R, <span class="hljs-attribute">T</span>=T)
|
||||||
|
|
||||||
<span class="hljs-comment"># Define the settings for rasterization and shading. Here we set the output image to be of size</span>
|
<span class="hljs-comment"># Define the settings for rasterization and shading. Here we set the output image to be of size</span>
|
||||||
<span class="hljs-comment"># 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1</span>
|
<span class="hljs-comment"># 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1</span>
|
||||||
@ -114,7 +122,6 @@ raster_settings = RasterizationSettings(
|
|||||||
<span class="hljs-attribute">image_size</span>=512,
|
<span class="hljs-attribute">image_size</span>=512,
|
||||||
<span class="hljs-attribute">blur_radius</span>=0.0,
|
<span class="hljs-attribute">blur_radius</span>=0.0,
|
||||||
<span class="hljs-attribute">faces_per_pixel</span>=1,
|
<span class="hljs-attribute">faces_per_pixel</span>=1,
|
||||||
<span class="hljs-attribute">bin_size</span>=0
|
|
||||||
)
|
)
|
||||||
|
|
||||||
<span class="hljs-comment"># Create a phong renderer by composing a rasterizer and a shader. Here we can use a predefined</span>
|
<span class="hljs-comment"># Create a phong renderer by composing a rasterizer and a shader. Here we can use a predefined</span>
|
||||||
@ -135,16 +142,15 @@ renderer = MeshRenderer(
|
|||||||
<p>We have examples of several combinations of these functions based on the texturing/shading/blending support we have currently. These are summarised in this table below. Many other combinations are possible and we plan to expand the options available for texturing, shading and blending.</p>
|
<p>We have examples of several combinations of these functions based on the texturing/shading/blending support we have currently. These are summarised in this table below. Many other combinations are possible and we plan to expand the options available for texturing, shading and blending.</p>
|
||||||
<table>
|
<table>
|
||||||
<thead>
|
<thead>
|
||||||
<tr><th>Example Shaders</th><th style="text-align:center">Vertex Textures</th><th style="text-align:center">Texture Map</th><th style="text-align:center">Flat Shading</th><th style="text-align:center">Gouraud Shading</th><th style="text-align:center">Phong Shading</th><th style="text-align:center">Hard blending</th><th style="text-align:center">Soft Blending</th></tr>
|
<tr><th>Example Shaders</th><th style="text-align:center">Vertex Textures</th><th style="text-align:center">UV Textures</th><th style="text-align:center">Textures Atlas</th><th style="text-align:center">Flat Shading</th><th style="text-align:center">Gouraud Shading</th><th style="text-align:center">Phong Shading</th><th style="text-align:center">Hard blending</th><th style="text-align:center">Soft Blending</th></tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr><td>HardPhongShader</td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td></tr>
|
<tr><td>HardPhongShader</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center"></td></tr>
|
||||||
<tr><td>SoftPhongShader</td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td></tr>
|
<tr><td>SoftPhongShader</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center">✔️</td></tr>
|
||||||
<tr><td>HardGouraudShader</td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td></tr>
|
<tr><td>HardGouraudShader</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center">✔️</td><td style="text-align:center"></td></tr>
|
||||||
<tr><td>SoftGouraudShader</td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td></tr>
|
<tr><td>SoftGouraudShader</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">✔️</td></tr>
|
||||||
<tr><td>TexturedSoftPhongShader</td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td></tr>
|
<tr><td>HardFlatShader</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">✔️</td><td style="text-align:center"></td></tr>
|
||||||
<tr><td>HardFlatShader</td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td></tr>
|
<tr><td>SoftSilhouetteShader</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">✔️</td></tr>
|
||||||
<tr><td>SoftSilhouetteShader</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td></tr>
|
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Patrick Labatut</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/renderer"><span class="arrow-prev">← </span><span>Overview</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/renderer"><span class="arrow-prev">← </span><span>Overview</span></a><a class="docs-next button" href="/docs/cameras"><span>Cameras</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -1,4 +1,4 @@
|
|||||||
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>renderer_getting_started · PyTorch3D</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="# Renderer Getting Started"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="renderer_getting_started · PyTorch3D"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="# Renderer Getting Started"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>renderer_getting_started · PyTorch3D</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="# Getting Started With Renderer"/><meta name="docsearch:language" content="en"/><meta property="og:title" content="renderer_getting_started · PyTorch3D"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="# Getting Started With Renderer"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||||
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||||
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Differentiable Renderer</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Meshes</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Differentiable Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li></ul></div></div></section></div><script>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Renderer</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
var coll = document.getElementsByClassName('collapsible');
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
var checkActiveCategory = true;
|
var checkActiveCategory = true;
|
||||||
for (var i = 0; i < coll.length; i++) {
|
for (var i = 0; i < coll.length; i++) {
|
||||||
@ -62,10 +62,10 @@
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="renderer-getting-started"></a><a href="#renderer-getting-started" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Renderer Getting Started</h1>
|
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="getting-started-with-renderer"></a><a href="#getting-started-with-renderer" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Getting Started With Renderer</h1>
|
||||||
<h3><a class="anchor" aria-hidden="true" id="architecture-overview"></a><a href="#architecture-overview" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Architecture Overview</h3>
|
<h3><a class="anchor" aria-hidden="true" id="architecture-overview"></a><a href="#architecture-overview" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Architecture Overview</h3>
|
||||||
<p>The renderer is designed to be modular, extensible and support batching and gradients for all inputs. The following figure describes all the components of the rendering pipeline.</p>
|
<p>The renderer is designed to be modular, extensible and support batching and gradients for all inputs. The following figure describes all the components of the rendering pipeline.</p>
|
||||||
<p><img src="assets/architecture_overview.png" width="1000"></p>
|
<p><img src="assets/architecture_renderer.jpg" width="1000"></p>
|
||||||
<h5><a class="anchor" aria-hidden="true" id="fragments"></a><a href="#fragments" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Fragments</h5>
|
<h5><a class="anchor" aria-hidden="true" id="fragments"></a><a href="#fragments" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Fragments</h5>
|
||||||
<p>The <strong>rasterizer</strong> returns 4 output tensors in a named tuple.</p>
|
<p>The <strong>rasterizer</strong> returns 4 output tensors in a named tuple.</p>
|
||||||
<ul>
|
<ul>
|
||||||
@ -82,9 +82,9 @@ giving the barycentric coordinates in NDC units of the nearest faces at each pix
|
|||||||
<hr>
|
<hr>
|
||||||
<h3><a class="anchor" aria-hidden="true" id="coordinate-transformation-conventions"></a><a href="#coordinate-transformation-conventions" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Coordinate transformation conventions</h3>
|
<h3><a class="anchor" aria-hidden="true" id="coordinate-transformation-conventions"></a><a href="#coordinate-transformation-conventions" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Coordinate transformation conventions</h3>
|
||||||
<p>Rendering requires transformations between several different coordinate frames: world space, view/camera space, NDC space and screen space. At each step it is important to know where the camera is located, how the +X, +Y, +Z axes are aligned and the possible range of values. The following figure outlines the conventions used PyTorch3D.</p>
|
<p>Rendering requires transformations between several different coordinate frames: world space, view/camera space, NDC space and screen space. At each step it is important to know where the camera is located, how the +X, +Y, +Z axes are aligned and the possible range of values. The following figure outlines the conventions used PyTorch3D.</p>
|
||||||
<p><img src="assets/transformations_overview.png" width="1000"></p>
|
<p><img src="assets/transforms_overview.jpg" width="1000"></p>
|
||||||
<p>For example, given a teapot mesh, the world coordinate frame, camera coordiante frame and image are show in the figure below. Note that the world and camera coordinate frames have the +z direction pointing in to the page.</p>
|
<p>For example, given a teapot mesh, the world coordinate frame, camera coordiante frame and image are show in the figure below. Note that the world and camera coordinate frames have the +z direction pointing in to the page.</p>
|
||||||
<p><img src="assets/world_camera_image.png" width="1000"></p>
|
<p><img src="assets/world_camera_image.jpg" width="1000"></p>
|
||||||
<hr>
|
<hr>
|
||||||
<p><strong>NOTE: PyTorch3D vs OpenGL</strong></p>
|
<p><strong>NOTE: PyTorch3D vs OpenGL</strong></p>
|
||||||
<p>While we tried to emulate several aspects of OpenGL, there are differences in the coordinate frame conventions.</p>
|
<p>While we tried to emulate several aspects of OpenGL, there are differences in the coordinate frame conventions.</p>
|
||||||
@ -94,18 +94,26 @@ giving the barycentric coordinates in NDC units of the nearest faces at each pix
|
|||||||
</ul>
|
</ul>
|
||||||
<p><img align="center" src="assets/opengl_coordframes.png" width="300"></p>
|
<p><img align="center" src="assets/opengl_coordframes.png" width="300"></p>
|
||||||
<hr>
|
<hr>
|
||||||
|
<h3><a class="anchor" aria-hidden="true" id="texturing-options"></a><a href="#texturing-options" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Texturing options</h3>
|
||||||
|
<p>For mesh texturing we offer several options (in <code>pytorch3d/renderer/mesh/texturing.py</code>):</p>
|
||||||
|
<ol>
|
||||||
|
<li><strong>Vertex Textures</strong>: D dimensional textures for each vertex (for example an RGB color) which can be interpolated across the face. This can be represented as an <code>(N, V, D)</code> tensor. This is a fairly simple representation though and cannot model complex textures if the mesh faces are large.</li>
|
||||||
|
<li><strong>UV Textures</strong>: vertex UV coordinates and <strong>one</strong> texture map for the whole face. For a point on a face with given barycentric coordinates, the face color can be computed by interpolating the vertex uv coordinates and then sampling from the texture map. This representation requires two tensors (UVs: <code>(N, V, 2), Texture map:</code>(N, H, W, 3)`), and is limited to only support one texture map per mesh.</li>
|
||||||
|
<li><strong>Face Textures</strong>: In more complex cases such as ShapeNet meshes, there are multiple texture maps per mesh and some faces have texture while other do not. For these cases, a more flexible representation is a texture atlas, where each face is represented as an <code>(RxR)</code> texture map where R is the texture resolution. For a given point on the face, the texture value can be sampled from the per face texture map using the barycentric coordinates of the point. This representation requires one tensor of shape <code>(N, F, R, R, 3)</code>. This texturing method is inspired by the SoftRasterizer implementation. For more details refer to the <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/io/mtl_io.py#L123"><code>make_material_atlas</code></a> and <a href="https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/renderer/mesh/textures.py#L452"><code>sample_textures</code></a> functions.</li>
|
||||||
|
</ol>
|
||||||
|
<p><img src="assets/texturing.jpg" width="1000"></p>
|
||||||
<h3><a class="anchor" aria-hidden="true" id="a-simple-renderer"></a><a href="#a-simple-renderer" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>A simple renderer</h3>
|
<h3><a class="anchor" aria-hidden="true" id="a-simple-renderer"></a><a href="#a-simple-renderer" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>A simple renderer</h3>
|
||||||
<p>A renderer in PyTorch3D is composed of a <strong>rasterizer</strong> and a <strong>shader</strong>. Create a renderer in a few simple steps:</p>
|
<p>A renderer in PyTorch3D is composed of a <strong>rasterizer</strong> and a <strong>shader</strong>. Create a renderer in a few simple steps:</p>
|
||||||
<pre><code class="hljs"><span class="hljs-comment"># Imports</span>
|
<pre><code class="hljs"><span class="hljs-comment"># Imports</span>
|
||||||
<span class="hljs-keyword">from</span> pytorch3d.renderer import (
|
<span class="hljs-keyword">from</span> pytorch3d.renderer import (
|
||||||
OpenGLPerspectiveCameras, look_at_view_transform,
|
FoVPerspectiveCameras, look_at_view_transform,
|
||||||
RasterizationSettings, BlendParams,
|
RasterizationSettings, BlendParams,
|
||||||
MeshRenderer, MeshRasterizer, HardPhongShader
|
MeshRenderer, MeshRasterizer, HardPhongShader
|
||||||
)
|
)
|
||||||
|
|
||||||
<span class="hljs-comment"># Initialize an OpenGL perspective camera.</span>
|
<span class="hljs-comment"># Initialize an OpenGL perspective camera.</span>
|
||||||
R, T = look_at_view_transform(2.7, 10, 20)
|
R, T = look_at_view_transform(2.7, 10, 20)
|
||||||
cameras = OpenGLPerspectiveCameras(<span class="hljs-attribute">device</span>=device, <span class="hljs-attribute">R</span>=R, <span class="hljs-attribute">T</span>=T)
|
cameras = FoVPerspectiveCameras(<span class="hljs-attribute">device</span>=device, <span class="hljs-attribute">R</span>=R, <span class="hljs-attribute">T</span>=T)
|
||||||
|
|
||||||
<span class="hljs-comment"># Define the settings for rasterization and shading. Here we set the output image to be of size</span>
|
<span class="hljs-comment"># Define the settings for rasterization and shading. Here we set the output image to be of size</span>
|
||||||
<span class="hljs-comment"># 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1</span>
|
<span class="hljs-comment"># 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1</span>
|
||||||
@ -114,7 +122,6 @@ raster_settings = RasterizationSettings(
|
|||||||
<span class="hljs-attribute">image_size</span>=512,
|
<span class="hljs-attribute">image_size</span>=512,
|
||||||
<span class="hljs-attribute">blur_radius</span>=0.0,
|
<span class="hljs-attribute">blur_radius</span>=0.0,
|
||||||
<span class="hljs-attribute">faces_per_pixel</span>=1,
|
<span class="hljs-attribute">faces_per_pixel</span>=1,
|
||||||
<span class="hljs-attribute">bin_size</span>=0
|
|
||||||
)
|
)
|
||||||
|
|
||||||
<span class="hljs-comment"># Create a phong renderer by composing a rasterizer and a shader. Here we can use a predefined</span>
|
<span class="hljs-comment"># Create a phong renderer by composing a rasterizer and a shader. Here we can use a predefined</span>
|
||||||
@ -135,16 +142,15 @@ renderer = MeshRenderer(
|
|||||||
<p>We have examples of several combinations of these functions based on the texturing/shading/blending support we have currently. These are summarised in this table below. Many other combinations are possible and we plan to expand the options available for texturing, shading and blending.</p>
|
<p>We have examples of several combinations of these functions based on the texturing/shading/blending support we have currently. These are summarised in this table below. Many other combinations are possible and we plan to expand the options available for texturing, shading and blending.</p>
|
||||||
<table>
|
<table>
|
||||||
<thead>
|
<thead>
|
||||||
<tr><th>Example Shaders</th><th style="text-align:center">Vertex Textures</th><th style="text-align:center">Texture Map</th><th style="text-align:center">Flat Shading</th><th style="text-align:center">Gouraud Shading</th><th style="text-align:center">Phong Shading</th><th style="text-align:center">Hard blending</th><th style="text-align:center">Soft Blending</th></tr>
|
<tr><th>Example Shaders</th><th style="text-align:center">Vertex Textures</th><th style="text-align:center">UV Textures</th><th style="text-align:center">Textures Atlas</th><th style="text-align:center">Flat Shading</th><th style="text-align:center">Gouraud Shading</th><th style="text-align:center">Phong Shading</th><th style="text-align:center">Hard blending</th><th style="text-align:center">Soft Blending</th></tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
<tr><td>HardPhongShader</td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td></tr>
|
<tr><td>HardPhongShader</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center"></td></tr>
|
||||||
<tr><td>SoftPhongShader</td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td></tr>
|
<tr><td>SoftPhongShader</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center">✔️</td></tr>
|
||||||
<tr><td>HardGouraudShader</td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td></tr>
|
<tr><td>HardGouraudShader</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center">✔️</td><td style="text-align:center"></td></tr>
|
||||||
<tr><td>SoftGouraudShader</td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td></tr>
|
<tr><td>SoftGouraudShader</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">✔️</td></tr>
|
||||||
<tr><td>TexturedSoftPhongShader</td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td></tr>
|
<tr><td>HardFlatShader</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center">✔️</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">✔️</td><td style="text-align:center"></td></tr>
|
||||||
<tr><td>HardFlatShader</td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td><td style="text-align:center"></td></tr>
|
<tr><td>SoftSilhouetteShader</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">✔️</td></tr>
|
||||||
<tr><td>SoftSilhouetteShader</td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center"></td><td style="text-align:center">:heavy_check_mark:</td></tr>
|
|
||||||
</tbody>
|
</tbody>
|
||||||
</table>
|
</table>
|
||||||
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Patrick Labatut</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/renderer"><span class="arrow-prev">← </span><span>Overview</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Nikhila Ravi</em></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/renderer"><span class="arrow-prev">← </span><span>Overview</span></a><a class="docs-next button" href="/docs/cameras"><span>Cameras</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive siteNavItemActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Introduction</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Meshes</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Differentiable Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li></ul></div></div></section></div><script>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive siteNavItemActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Introduction</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
var coll = document.getElementsByClassName('collapsible');
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
var checkActiveCategory = true;
|
var checkActiveCategory = true;
|
||||||
for (var i = 0; i < coll.length; i++) {
|
for (var i = 0; i < coll.length; i++) {
|
||||||
@ -65,4 +65,4 @@
|
|||||||
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="why-pytorch3d"></a><a href="#why-pytorch3d" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Why PyTorch3D</h1>
|
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="why-pytorch3d"></a><a href="#why-pytorch3d" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Why PyTorch3D</h1>
|
||||||
<p>Our goal with PyTorch3D is to help accelerate research at the intersection of deep learning and 3D. 3D data is more complex than 2D images and while working on projects such as <a href="https://github.com/facebookresearch/meshrcnn">Mesh R-CNN</a> and <a href="https://github.com/facebookresearch/c3dpo_nrsfm">C3DPO</a>, we encountered several challenges including 3D data representation, batching, and speed. We have developed many useful operators and abstractions for working on 3D deep learning and want to share this with the community to drive novel research in this area.</p>
|
<p>Our goal with PyTorch3D is to help accelerate research at the intersection of deep learning and 3D. 3D data is more complex than 2D images and while working on projects such as <a href="https://github.com/facebookresearch/meshrcnn">Mesh R-CNN</a> and <a href="https://github.com/facebookresearch/c3dpo_nrsfm">C3DPO</a>, we encountered several challenges including 3D data representation, batching, and speed. We have developed many useful operators and abstractions for working on 3D deep learning and want to share this with the community to drive novel research in this area.</p>
|
||||||
<p>In PyTorch3D we have included efficient 3D operators, heterogeneous batching capabilities, and a modular differentiable rendering API, to equip researchers in this field with a much needed toolkit to implement cutting-edge research with complex 3D inputs.</p>
|
<p>In PyTorch3D we have included efficient 3D operators, heterogeneous batching capabilities, and a modular differentiable rendering API, to equip researchers in this field with a much needed toolkit to implement cutting-edge research with complex 3D inputs.</p>
|
||||||
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Patrick Labatut</em></div><div class="docs-prevnext"><a class="docs-next button" href="/docs/batching"><span>Batching</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Patrick Labatut</em></div><div class="docs-prevnext"><a class="docs-next button" href="/docs/meshes_io"><span>Loading from file</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive siteNavItemActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Introduction</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Meshes</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Differentiable Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li></ul></div></div></section></div><script>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body class="sideNavVisible separateOnPageNav"><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class="siteNavGroupActive siteNavItemActive"><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span>Introduction</span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Introduction</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/docs/why_pytorch3d">Why PyTorch3D</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Data</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/meshes_io">Loading from file</a></li><li class="navListItem"><a class="navItem" href="/docs/datasets">Data loaders</a></li><li class="navListItem"><a class="navItem" href="/docs/batching">Batching</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Ops</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/cubify">Cubify</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Renderer</h3><ul class=""><li class="navListItem"><a class="navItem" href="/docs/renderer">Overview</a></li><li class="navListItem"><a class="navItem" href="/docs/renderer_getting_started">Getting Started</a></li><li class="navListItem"><a class="navItem" href="/docs/cameras">Cameras</a></li></ul></div></div></section></div><script>
|
||||||
var coll = document.getElementsByClassName('collapsible');
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
var checkActiveCategory = true;
|
var checkActiveCategory = true;
|
||||||
for (var i = 0; i < coll.length; i++) {
|
for (var i = 0; i < coll.length; i++) {
|
||||||
@ -65,4 +65,4 @@
|
|||||||
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="why-pytorch3d"></a><a href="#why-pytorch3d" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Why PyTorch3D</h1>
|
</script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"></header><article><div><span><h1><a class="anchor" aria-hidden="true" id="why-pytorch3d"></a><a href="#why-pytorch3d" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6.25c-1.09.53-2 1.84-2 3.25C6 11.31 7.55 13 9 13h4c1.45 0 3-1.69 3-3.5S14.5 6 13 6z"></path></svg></a>Why PyTorch3D</h1>
|
||||||
<p>Our goal with PyTorch3D is to help accelerate research at the intersection of deep learning and 3D. 3D data is more complex than 2D images and while working on projects such as <a href="https://github.com/facebookresearch/meshrcnn">Mesh R-CNN</a> and <a href="https://github.com/facebookresearch/c3dpo_nrsfm">C3DPO</a>, we encountered several challenges including 3D data representation, batching, and speed. We have developed many useful operators and abstractions for working on 3D deep learning and want to share this with the community to drive novel research in this area.</p>
|
<p>Our goal with PyTorch3D is to help accelerate research at the intersection of deep learning and 3D. 3D data is more complex than 2D images and while working on projects such as <a href="https://github.com/facebookresearch/meshrcnn">Mesh R-CNN</a> and <a href="https://github.com/facebookresearch/c3dpo_nrsfm">C3DPO</a>, we encountered several challenges including 3D data representation, batching, and speed. We have developed many useful operators and abstractions for working on 3D deep learning and want to share this with the community to drive novel research in this area.</p>
|
||||||
<p>In PyTorch3D we have included efficient 3D operators, heterogeneous batching capabilities, and a modular differentiable rendering API, to equip researchers in this field with a much needed toolkit to implement cutting-edge research with complex 3D inputs.</p>
|
<p>In PyTorch3D we have included efficient 3D operators, heterogeneous batching capabilities, and a modular differentiable rendering API, to equip researchers in this field with a much needed toolkit to implement cutting-edge research with complex 3D inputs.</p>
|
||||||
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Patrick Labatut</em></div><div class="docs-prevnext"><a class="docs-next button" href="/docs/batching"><span>Batching</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></article></div><div class="docLastUpdate"><em>Last updated by Patrick Labatut</em></div><div class="docs-prevnext"><a class="docs-next button" href="/docs/meshes_io"><span>Loading from file</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -12,4 +12,4 @@
|
|||||||
</span></div></h2><div><span><p>Ask questions about the documentation and project</p>
|
</span></div></h2><div><span><p>Ask questions about the documentation and project</p>
|
||||||
</span></div></div></div><div class="blockElement threeByGridBlock"><div class="blockContent"><h2><div><span><p>Stay up to date</p>
|
</span></div></div></div><div class="blockElement threeByGridBlock"><div class="blockContent"><h2><div><span><p>Stay up to date</p>
|
||||||
</span></div></h2><div><span><p>Find out what's new with this project</p>
|
</span></div></h2><div><span><p>Find out what's new with this project</p>
|
||||||
</span></div></div></div></div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></div></div></div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -12,4 +12,4 @@
|
|||||||
</span></div></h2><div><span><p>Ask questions about the documentation and project</p>
|
</span></div></h2><div><span><p>Ask questions about the documentation and project</p>
|
||||||
</span></div></div></div><div class="blockElement threeByGridBlock"><div class="blockContent"><h2><div><span><p>Stay up to date</p>
|
</span></div></div></div><div class="blockElement threeByGridBlock"><div class="blockContent"><h2><div><span><p>Stay up to date</p>
|
||||||
</span></div></h2><div><span><p>Find out what's new with this project</p>
|
</span></div></h2><div><span><p>Find out what's new with this project</p>
|
||||||
</span></div></div></div></div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></div></div></div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -12,10 +12,7 @@
|
|||||||
</span></div></h2><div><span><p>Supports optimized implementations of several common functions for 3D data</p>
|
</span></div></h2><div><span><p>Supports optimized implementations of several common functions for 3D data</p>
|
||||||
</span></div></div></div><div class="blockElement alignCenter fourByGridBlock imageAlignTop"><div class="blockImage"><img src="/img/rendering.svg"/></div><div class="blockContent"><h2><div><span><p>Differentiable Rendering</p>
|
</span></div></div></div><div class="blockElement alignCenter fourByGridBlock imageAlignTop"><div class="blockImage"><img src="/img/rendering.svg"/></div><div class="blockContent"><h2><div><span><p>Differentiable Rendering</p>
|
||||||
</span></div></h2><div><span><p>Modular differentiable rendering API with parallel implementations in PyTorch, C++ and CUDA</p>
|
</span></div></h2><div><span><p>Modular differentiable rendering API with parallel implementations in PyTorch, C++ and CUDA</p>
|
||||||
</span></div></div></div></div></div></div></div><div class="productShowcaseSection" id="quickstart" style="text-align:center"><h2>Get Started</h2><div class="container"><div class="wrapper"><ol><li><strong>Install PyTorch3D:</strong><div><span><pre><code class="hljs css language-bash">conda install pytorch torchvision -c pytorch <span class="hljs-comment"># OSX only</span>
|
</span></div></div></div></div></div></div></div><div class="productShowcaseSection" id="quickstart" style="text-align:center"><h2>Get Started</h2><div class="container"><div class="wrapper"><ol><li><strong>Install PyTorch3D </strong> (following the instructions <a href="https://github.com/facebookresearch/pytorch3d/blob/master/INSTALL.md">here</a>)</li><li><strong>Try a few 3D operators </strong>e.g. compute the chamfer loss between two meshes:<div><span><pre><code class="hljs css language-python"><span class="hljs-keyword">from</span> pytorch3d.utils <span class="hljs-keyword">import</span> ico_sphere
|
||||||
conda install pytorch3d -c pytorch3d <span class="hljs-comment"># all systems</span>
|
|
||||||
</code></pre>
|
|
||||||
</span></div></li><li><strong>Try a few 3D operators </strong>e.g. compute the chamfer loss between two meshes:<div><span><pre><code class="hljs css language-python"><span class="hljs-keyword">from</span> pytorch3d.utils <span class="hljs-keyword">import</span> ico_sphere
|
|
||||||
<span class="hljs-keyword">from</span> pytorch3d.io <span class="hljs-keyword">import</span> load_obj
|
<span class="hljs-keyword">from</span> pytorch3d.io <span class="hljs-keyword">import</span> load_obj
|
||||||
<span class="hljs-keyword">from</span> pytorch3d.structures <span class="hljs-keyword">import</span> Meshes
|
<span class="hljs-keyword">from</span> pytorch3d.structures <span class="hljs-keyword">import</span> Meshes
|
||||||
<span class="hljs-keyword">from</span> pytorch3d.ops <span class="hljs-keyword">import</span> sample_points_from_meshes
|
<span class="hljs-keyword">from</span> pytorch3d.ops <span class="hljs-keyword">import</span> sample_points_from_meshes
|
||||||
@ -31,4 +28,4 @@ sample_sphere = sample_points_from_meshes(sphere_mesh, <span class="hljs-number"
|
|||||||
sample_test = sample_points_from_meshes(test_mesh, <span class="hljs-number">5000</span>)
|
sample_test = sample_points_from_meshes(test_mesh, <span class="hljs-number">5000</span>)
|
||||||
loss_chamfer, _ = chamfer_distance(sample_sphere, sample_test)
|
loss_chamfer, _ = chamfer_distance(sample_sphere, sample_test)
|
||||||
</code></pre>
|
</code></pre>
|
||||||
</span></div></li></ol></div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></li></ol></div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -6,4 +6,4 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="mainContainer"><div class="container paddingBottom paddingTop"><div class="wrapper"><div class="showcaseSection"><div class="prose"><h1>Who is Using This?</h1><p>This project is used by many folks</p></div><div class="logos"><a href="https://www.facebook.com"><img src="/img/undraw_open_source.svg" alt="User1" title="User1"/></a></div><p>Are you using this project?</p><a href="undefined/edit/master/website/siteConfig.js" class="button">Add your company</a></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="mainContainer"><div class="container paddingBottom paddingTop"><div class="wrapper"><div class="showcaseSection"><div class="prose"><h1>Who is Using This?</h1><p>This project is used by many folks</p></div><div class="logos"><a href="https://www.facebook.com"><img src="/img/undraw_open_source.svg" alt="User1" title="User1"/></a></div><p>Are you using this project?</p><a href="undefined/edit/master/website/siteConfig.js" class="button">Add your company</a></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -6,4 +6,4 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="mainContainer"><div class="container paddingBottom paddingTop"><div class="wrapper"><div class="showcaseSection"><div class="prose"><h1>Who is Using This?</h1><p>This project is used by many folks</p></div><div class="logos"><a href="https://www.facebook.com"><img src="/img/undraw_open_source.svg" alt="User1" title="User1"/></a></div><p>Are you using this project?</p><a href="undefined/edit/master/website/siteConfig.js" class="button">Add your company</a></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="mainContainer"><div class="container paddingBottom paddingTop"><div class="wrapper"><div class="showcaseSection"><div class="prose"><h1>Who is Using This?</h1><p>This project is used by many folks</p></div><div class="logos"><a href="https://www.facebook.com"><img src="/img/undraw_open_source.svg" alt="User1" title="User1"/></a></div><p>Are you using this project?</p><a href="undefined/edit/master/website/siteConfig.js" class="button">Add your company</a></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
File diff suppressed because one or more lines are too long
@ -1,4 +1,4 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
|
|
||||||
# In[ ]:
|
# In[ ]:
|
||||||
@ -37,14 +37,19 @@
|
|||||||
|
|
||||||
# If `torch`, `torchvision` and `pytorch3d` are not installed, run the following cell:
|
# If `torch`, `torchvision` and `pytorch3d` are not installed, run the following cell:
|
||||||
|
|
||||||
# In[1]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
get_ipython().system('pip install torch torchvision')
|
get_ipython().system('pip install torch torchvision')
|
||||||
get_ipython().system("pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'")
|
import sys
|
||||||
|
import torch
|
||||||
|
if torch.__version__=='1.6.0+cu101' and sys.platform.startswith('linux'):
|
||||||
|
get_ipython().system('pip install pytorch3d')
|
||||||
|
else:
|
||||||
|
get_ipython().system("pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'")
|
||||||
|
|
||||||
|
|
||||||
# In[3]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# imports
|
# imports
|
||||||
@ -64,11 +69,16 @@ sys.path.append(os.path.abspath(''))
|
|||||||
|
|
||||||
# set for reproducibility
|
# set for reproducibility
|
||||||
torch.manual_seed(42)
|
torch.manual_seed(42)
|
||||||
|
if torch.cuda.is_available():
|
||||||
|
device = torch.device("cuda:0")
|
||||||
|
else:
|
||||||
|
device = torch.device("cpu")
|
||||||
|
print("WARNING: CPU only, this will be slow!")
|
||||||
|
|
||||||
|
|
||||||
# If using **Google Colab**, fetch the utils file for plotting the camera scene, and the ground truth camera positions:
|
# If using **Google Colab**, fetch the utils file for plotting the camera scene, and the ground truth camera positions:
|
||||||
|
|
||||||
# In[2]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
get_ipython().system('wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/camera_visualization.py')
|
get_ipython().system('wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/camera_visualization.py')
|
||||||
@ -97,16 +107,16 @@ camera_graph_file = './data/camera_graph.pth'
|
|||||||
|
|
||||||
# create the relative cameras
|
# create the relative cameras
|
||||||
cameras_relative = SfMPerspectiveCameras(
|
cameras_relative = SfMPerspectiveCameras(
|
||||||
R = R_relative.cuda(),
|
R = R_relative.to(device),
|
||||||
T = T_relative.cuda(),
|
T = T_relative.to(device),
|
||||||
device = "cuda",
|
device = device,
|
||||||
)
|
)
|
||||||
|
|
||||||
# create the absolute ground truth cameras
|
# create the absolute ground truth cameras
|
||||||
cameras_absolute_gt = SfMPerspectiveCameras(
|
cameras_absolute_gt = SfMPerspectiveCameras(
|
||||||
R = R_absolute_gt.cuda(),
|
R = R_absolute_gt.to(device),
|
||||||
T = T_absolute_gt.cuda(),
|
T = T_absolute_gt.to(device),
|
||||||
device = "cuda",
|
device = device,
|
||||||
)
|
)
|
||||||
|
|
||||||
# the number of absolute camera positions
|
# the number of absolute camera positions
|
||||||
@ -152,7 +162,7 @@ def get_relative_camera(cams, edges):
|
|||||||
SfMPerspectiveCameras(
|
SfMPerspectiveCameras(
|
||||||
R = cams.R[edges[:, i]],
|
R = cams.R[edges[:, i]],
|
||||||
T = cams.T[edges[:, i]],
|
T = cams.T[edges[:, i]],
|
||||||
device = "cuda",
|
device = device,
|
||||||
).get_world_to_view_transform()
|
).get_world_to_view_transform()
|
||||||
for i in (0, 1)
|
for i in (0, 1)
|
||||||
]
|
]
|
||||||
@ -165,7 +175,7 @@ def get_relative_camera(cams, edges):
|
|||||||
cams_relative = SfMPerspectiveCameras(
|
cams_relative = SfMPerspectiveCameras(
|
||||||
R = matrix_rel[:, :3, :3],
|
R = matrix_rel[:, :3, :3],
|
||||||
T = matrix_rel[:, 3, :3],
|
T = matrix_rel[:, 3, :3],
|
||||||
device = "cuda",
|
device = device,
|
||||||
)
|
)
|
||||||
return cams_relative
|
return cams_relative
|
||||||
|
|
||||||
@ -180,12 +190,12 @@ def get_relative_camera(cams, edges):
|
|||||||
# `R_absolute = so3_exponential_map(log_R_absolute)`
|
# `R_absolute = so3_exponential_map(log_R_absolute)`
|
||||||
#
|
#
|
||||||
|
|
||||||
# In[8]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# initialize the absolute log-rotations/translations with random entries
|
# initialize the absolute log-rotations/translations with random entries
|
||||||
log_R_absolute_init = torch.randn(N, 3).float().cuda()
|
log_R_absolute_init = torch.randn(N, 3, dtype=torch.float32, device=device)
|
||||||
T_absolute_init = torch.randn(N, 3).float().cuda()
|
T_absolute_init = torch.randn(N, 3, dtype=torch.float32, device=device)
|
||||||
|
|
||||||
# furthermore, we know that the first camera is a trivial one
|
# furthermore, we know that the first camera is a trivial one
|
||||||
# (see the description above)
|
# (see the description above)
|
||||||
@ -201,7 +211,7 @@ T_absolute.requires_grad = True
|
|||||||
# the mask the specifies which cameras are going to be optimized
|
# the mask the specifies which cameras are going to be optimized
|
||||||
# (since we know the first camera is already correct,
|
# (since we know the first camera is already correct,
|
||||||
# we only optimize over the 2nd-to-last cameras)
|
# we only optimize over the 2nd-to-last cameras)
|
||||||
camera_mask = torch.ones(N, 1).float().cuda()
|
camera_mask = torch.ones(N, 1, dtype=torch.float32, device=device)
|
||||||
camera_mask[0] = 0.
|
camera_mask[0] = 0.
|
||||||
|
|
||||||
# init the optimizer
|
# init the optimizer
|
||||||
@ -222,7 +232,7 @@ for it in range(n_iter):
|
|||||||
cameras_absolute = SfMPerspectiveCameras(
|
cameras_absolute = SfMPerspectiveCameras(
|
||||||
R = R_absolute,
|
R = R_absolute,
|
||||||
T = T_absolute * camera_mask,
|
T = T_absolute * camera_mask,
|
||||||
device = "cuda",
|
device = device,
|
||||||
)
|
)
|
||||||
|
|
||||||
# compute the relative cameras as a compositon of the absolute cameras
|
# compute the relative cameras as a compositon of the absolute cameras
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
@ -1,4 +1,4 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
|
|
||||||
# In[ ]:
|
# In[ ]:
|
||||||
@ -24,20 +24,25 @@
|
|||||||
|
|
||||||
# If `torch`, `torchvision` and `pytorch3d` are not installed, run the following cell:
|
# If `torch`, `torchvision` and `pytorch3d` are not installed, run the following cell:
|
||||||
|
|
||||||
# In[1]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
get_ipython().system('pip install torch torchvision')
|
get_ipython().system('pip install torch torchvision')
|
||||||
get_ipython().system("pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'")
|
import sys
|
||||||
|
import torch
|
||||||
|
if torch.__version__=='1.6.0+cu101' and sys.platform.startswith('linux'):
|
||||||
|
get_ipython().system('pip install pytorch3d')
|
||||||
|
else:
|
||||||
|
get_ipython().system("pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'")
|
||||||
|
|
||||||
|
|
||||||
# In[2]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import torch
|
import torch
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from tqdm import tqdm_notebook
|
from tqdm.notebook import tqdm
|
||||||
import imageio
|
import imageio
|
||||||
import torch.nn as nn
|
import torch.nn as nn
|
||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
@ -48,16 +53,16 @@ from skimage import img_as_ubyte
|
|||||||
from pytorch3d.io import load_obj
|
from pytorch3d.io import load_obj
|
||||||
|
|
||||||
# datastructures
|
# datastructures
|
||||||
from pytorch3d.structures import Meshes, Textures
|
from pytorch3d.structures import Meshes
|
||||||
|
|
||||||
# 3D transformations functions
|
# 3D transformations functions
|
||||||
from pytorch3d.transforms import Rotate, Translate
|
from pytorch3d.transforms import Rotate, Translate
|
||||||
|
|
||||||
# rendering components
|
# rendering components
|
||||||
from pytorch3d.renderer import (
|
from pytorch3d.renderer import (
|
||||||
OpenGLPerspectiveCameras, look_at_view_transform, look_at_rotation,
|
FoVPerspectiveCameras, look_at_view_transform, look_at_rotation,
|
||||||
RasterizationSettings, MeshRenderer, MeshRasterizer, BlendParams,
|
RasterizationSettings, MeshRenderer, MeshRasterizer, BlendParams,
|
||||||
SoftSilhouetteShader, HardPhongShader, PointLights
|
SoftSilhouetteShader, HardPhongShader, PointLights, TexturesVertex,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -67,19 +72,22 @@ from pytorch3d.renderer import (
|
|||||||
|
|
||||||
# If you are running this notebook locally after cloning the PyTorch3D repository, the mesh will already be available. **If using Google Colab, fetch the mesh and save it at the path `data/`**:
|
# If you are running this notebook locally after cloning the PyTorch3D repository, the mesh will already be available. **If using Google Colab, fetch the mesh and save it at the path `data/`**:
|
||||||
|
|
||||||
# In[2]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
get_ipython().system('mkdir -p data')
|
get_ipython().system('mkdir -p data')
|
||||||
get_ipython().system('wget -P data https://dl.fbaipublicfiles.com/pytorch3d/data/teapot/teapot.obj')
|
get_ipython().system('wget -P data https://dl.fbaipublicfiles.com/pytorch3d/data/teapot/teapot.obj')
|
||||||
|
|
||||||
|
|
||||||
# In[3]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# Set the cuda device
|
# Set the cuda device
|
||||||
device = torch.device("cuda:0")
|
if torch.cuda.is_available():
|
||||||
torch.cuda.set_device(device)
|
device = torch.device("cuda:0")
|
||||||
|
torch.cuda.set_device(device)
|
||||||
|
else:
|
||||||
|
device = torch.device("cpu")
|
||||||
|
|
||||||
# Load the obj and ignore the textures and materials.
|
# Load the obj and ignore the textures and materials.
|
||||||
verts, faces_idx, _ = load_obj("./data/teapot.obj")
|
verts, faces_idx, _ = load_obj("./data/teapot.obj")
|
||||||
@ -87,7 +95,7 @@ faces = faces_idx.verts_idx
|
|||||||
|
|
||||||
# Initialize each vertex to be white in color.
|
# Initialize each vertex to be white in color.
|
||||||
verts_rgb = torch.ones_like(verts)[None] # (1, V, 3)
|
verts_rgb = torch.ones_like(verts)[None] # (1, V, 3)
|
||||||
textures = Textures(verts_rgb=verts_rgb.to(device))
|
textures = TexturesVertex(verts_features=verts_rgb.to(device))
|
||||||
|
|
||||||
# Create a Meshes object for the teapot. Here we have only one mesh in the batch.
|
# Create a Meshes object for the teapot. Here we have only one mesh in the batch.
|
||||||
teapot_mesh = Meshes(
|
teapot_mesh = Meshes(
|
||||||
@ -107,11 +115,11 @@ teapot_mesh = Meshes(
|
|||||||
#
|
#
|
||||||
# For optimizing the camera position we will use a renderer which produces a **silhouette** of the object only and does not apply any **lighting** or **shading**. We will also initialize another renderer which applies full **phong shading** and use this for visualizing the outputs.
|
# For optimizing the camera position we will use a renderer which produces a **silhouette** of the object only and does not apply any **lighting** or **shading**. We will also initialize another renderer which applies full **phong shading** and use this for visualizing the outputs.
|
||||||
|
|
||||||
# In[4]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# Initialize an OpenGL perspective camera.
|
# Initialize a perspective camera.
|
||||||
cameras = OpenGLPerspectiveCameras(device=device)
|
cameras = FoVPerspectiveCameras(device=device)
|
||||||
|
|
||||||
# To blend the 100 faces we set a few parameters which control the opacity and the sharpness of
|
# To blend the 100 faces we set a few parameters which control the opacity and the sharpness of
|
||||||
# edges. Refer to blending.py for more details.
|
# edges. Refer to blending.py for more details.
|
||||||
@ -126,8 +134,6 @@ raster_settings = RasterizationSettings(
|
|||||||
image_size=256,
|
image_size=256,
|
||||||
blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma,
|
blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma,
|
||||||
faces_per_pixel=100,
|
faces_per_pixel=100,
|
||||||
bin_size = None, # this setting controls whether naive or coarse-to-fine rasterization is used
|
|
||||||
max_faces_per_bin = None # this setting is for coarse rasterization
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Create a silhouette mesh renderer by composing a rasterizer and a shader.
|
# Create a silhouette mesh renderer by composing a rasterizer and a shader.
|
||||||
@ -145,7 +151,6 @@ raster_settings = RasterizationSettings(
|
|||||||
image_size=256,
|
image_size=256,
|
||||||
blur_radius=0.0,
|
blur_radius=0.0,
|
||||||
faces_per_pixel=1,
|
faces_per_pixel=1,
|
||||||
bin_size=0
|
|
||||||
)
|
)
|
||||||
# We can add a point light in front of the object.
|
# We can add a point light in front of the object.
|
||||||
lights = PointLights(device=device, location=((2.0, 2.0, -2.0),))
|
lights = PointLights(device=device, location=((2.0, 2.0, -2.0),))
|
||||||
@ -154,7 +159,7 @@ phong_renderer = MeshRenderer(
|
|||||||
cameras=cameras,
|
cameras=cameras,
|
||||||
raster_settings=raster_settings
|
raster_settings=raster_settings
|
||||||
),
|
),
|
||||||
shader=HardPhongShader(device=device, lights=lights)
|
shader=HardPhongShader(device=device, cameras=cameras, lights=lights)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@ -166,7 +171,7 @@ phong_renderer = MeshRenderer(
|
|||||||
#
|
#
|
||||||
# We defined a camera which is positioned on the positive z axis hence sees the spout to the right.
|
# We defined a camera which is positioned on the positive z axis hence sees the spout to the right.
|
||||||
|
|
||||||
# In[5]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# Select the viewpoint using spherical angles
|
# Select the viewpoint using spherical angles
|
||||||
@ -197,7 +202,7 @@ plt.grid(False)
|
|||||||
#
|
#
|
||||||
# Here we create a simple model class and initialize a parameter for the camera position.
|
# Here we create a simple model class and initialize a parameter for the camera position.
|
||||||
|
|
||||||
# In[17]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
class Model(nn.Module):
|
class Model(nn.Module):
|
||||||
@ -234,7 +239,7 @@ class Model(nn.Module):
|
|||||||
#
|
#
|
||||||
# Now we can create an instance of the **model** above and set up an **optimizer** for the camera position parameter.
|
# Now we can create an instance of the **model** above and set up an **optimizer** for the camera position parameter.
|
||||||
|
|
||||||
# In[18]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# We will save images periodically and compose them into a GIF.
|
# We will save images periodically and compose them into a GIF.
|
||||||
@ -250,7 +255,7 @@ optimizer = torch.optim.Adam(model.parameters(), lr=0.05)
|
|||||||
|
|
||||||
# ### Visualize the starting position and the reference position
|
# ### Visualize the starting position and the reference position
|
||||||
|
|
||||||
# In[19]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
plt.figure(figsize=(10, 10))
|
plt.figure(figsize=(10, 10))
|
||||||
@ -264,17 +269,17 @@ plt.title("Starting position")
|
|||||||
plt.subplot(1, 2, 2)
|
plt.subplot(1, 2, 2)
|
||||||
plt.imshow(model.image_ref.cpu().numpy().squeeze())
|
plt.imshow(model.image_ref.cpu().numpy().squeeze())
|
||||||
plt.grid(False)
|
plt.grid(False)
|
||||||
plt.title("Reference silhouette")
|
plt.title("Reference silhouette");
|
||||||
|
|
||||||
|
|
||||||
# ## 4. Run the optimization
|
# ## 4. Run the optimization
|
||||||
#
|
#
|
||||||
# We run several iterations of the forward and backward pass and save outputs every 10 iterations. When this has finished take a look at `./teapot_optimization_demo.gif` for a cool gif of the optimization process!
|
# We run several iterations of the forward and backward pass and save outputs every 10 iterations. When this has finished take a look at `./teapot_optimization_demo.gif` for a cool gif of the optimization process!
|
||||||
|
|
||||||
# In[20]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
loop = tqdm_notebook(range(200))
|
loop = tqdm(range(200))
|
||||||
for i in loop:
|
for i in loop:
|
||||||
optimizer.zero_grad()
|
optimizer.zero_grad()
|
||||||
loss, _ = model()
|
loss, _ = model()
|
||||||
|
|||||||
541
files/dataloaders_ShapeNetCore_R2N2.ipynb
Normal file
541
files/dataloaders_ShapeNetCore_R2N2.ipynb
Normal file
@ -0,0 +1,541 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"# Dataloaders for ShapeNetCore and R2N2\n",
|
||||||
|
"This tutorial shows how to:\n",
|
||||||
|
"- Load models from ShapeNetCore and R2N2 using PyTorch3D's data loaders.\n",
|
||||||
|
"- Pass the loaded datasets to `torch.utils.data.DataLoader`.\n",
|
||||||
|
"- Render ShapeNetCore models with PyTorch3D's renderer.\n",
|
||||||
|
"- Render R2N2 models with the same orientations as the original renderings in the dataset.\n",
|
||||||
|
"- Visualize R2N2 model voxels."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## 0. Install and import modules"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"If `torch`, `torchvision` and `pytorch3d` are not installed, run the following cell:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"!pip install torch torchvision\n",
|
||||||
|
"import sys\n",
|
||||||
|
"import torch\n",
|
||||||
|
"if torch.__version__=='1.6.0+cu101' and sys.platform.startswith('linux'):\n",
|
||||||
|
" !pip install pytorch3d\n",
|
||||||
|
"else:\n",
|
||||||
|
" !pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import numpy as np\n",
|
||||||
|
"import torch\n",
|
||||||
|
"\n",
|
||||||
|
"from pytorch3d.datasets import (\n",
|
||||||
|
" R2N2,\n",
|
||||||
|
" ShapeNetCore,\n",
|
||||||
|
" collate_batched_meshes,\n",
|
||||||
|
" render_cubified_voxels,\n",
|
||||||
|
")\n",
|
||||||
|
"from pytorch3d.renderer import (\n",
|
||||||
|
" OpenGLPerspectiveCameras,\n",
|
||||||
|
" PointLights,\n",
|
||||||
|
" RasterizationSettings,\n",
|
||||||
|
" TexturesVertex,\n",
|
||||||
|
" look_at_view_transform,\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"from pytorch3d.structures import Meshes\n",
|
||||||
|
"from torch.utils.data import DataLoader\n",
|
||||||
|
"\n",
|
||||||
|
"# add path for demo utils functions \n",
|
||||||
|
"import sys\n",
|
||||||
|
"import os\n",
|
||||||
|
"sys.path.append(os.path.abspath(''))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"If using **Google Colab**, fetch the utils file for plotting image grids:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"!wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py\n",
|
||||||
|
"from plot_image_grid import image_grid"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"OR if running locally uncomment and run the following cell:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# from utils import image_grid"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## 1. Load the datasets"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"If you haven't already downloaded the ShapeNetCore dataset, first do that following the instructions here: https://www.shapenet.org/. ShapeNetCore is a subset of the ShapeNet dataset. In PyTorch3D we support both version 1 (57 categories) and version 2 (55 categories).\n",
|
||||||
|
"\n",
|
||||||
|
"Then modify `SHAPENET_PATH` below to you local path to the ShapeNetCore dataset folder. "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Setup\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" device = torch.device(\"cuda:0\")\n",
|
||||||
|
" torch.cuda.set_device(device)\n",
|
||||||
|
"else:\n",
|
||||||
|
" device = torch.device(\"cpu\")\n",
|
||||||
|
" \n",
|
||||||
|
"SHAPENET_PATH = \"\"\n",
|
||||||
|
"shapenet_dataset = ShapeNetCore(SHAPENET_PATH)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"The R2N2 dataset can be downloaded using the instructions here: http://3d-r2n2.stanford.edu/. Look at the links for `ShapeNetRendering` and `ShapeNetVox32`. The R2N2 dataset contains 13 categories that are a subset of the ShapeNetCore v.1\n",
|
||||||
|
"dataset. The R2N2 dataset also contains its own 24 renderings of each object and voxelized models.\n",
|
||||||
|
"\n",
|
||||||
|
"Then modify `R2N2_PATH` and `SPLITS_PATH` below to your local R2N2 dataset folder path and splits file path respectively. Here we will load the `train` split of R2N2 and ask the voxels of each model to be returned."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"R2N2_PATH = \"\"\n",
|
||||||
|
"SPLITS_PATH = \"None\"\n",
|
||||||
|
"r2n2_dataset = R2N2(\"train\", SHAPENET_PATH, R2N2_PATH, SPLITS_PATH, return_voxels=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"We can retrieve a model by indexing into the loaded dataset. For both ShapeNetCore and R2N2, we can examine the category this model belongs to (in the form of a synset id, equivalend to wnid described in ImageNet's API: http://image-net.org/download-API), its model id, and its vertices and faces."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"shapenet_model = shapenet_dataset[6]\n",
|
||||||
|
"print(\"This model belongs to the category \" + shapenet_model[\"synset_id\"] + \".\")\n",
|
||||||
|
"print(\"This model has model id \" + shapenet_model[\"model_id\"] + \".\")\n",
|
||||||
|
"model_verts, model_faces = shapenet_model[\"verts\"], shapenet_model[\"faces\"]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"We can use its vertices and faces to form a `Meshes` object which is a PyTorch3D datastructure for working with batched meshes."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"model_textures = TexturesVertex(verts_features=torch.ones_like(model_verts, device=device)[None])\n",
|
||||||
|
"shapenet_model_mesh = Meshes(\n",
|
||||||
|
" verts=[model_verts.to(device)], \n",
|
||||||
|
" faces=[model_faces.to(device)],\n",
|
||||||
|
" textures=model_textures\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"With R2N2, we can further examine R2N2's original renderings. For instance, if we would like to see the second and third views of the eleventh objects in the R2N2 dataset, we can do the following:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"r2n2_renderings = r2n2_dataset[10,[1,2]]\n",
|
||||||
|
"image_grid(r2n2_renderings.numpy(), rows=1, cols=2, rgb=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## 2. Use the datasets with `torch.utils.data.DataLoader`"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Training deep learning models, usually requires passing in batches of inputs. The `torch.utils.data.DataLoader` from Pytorch helps us do this. PyTorch3D provides a function `collate_batched_meshes` to group the input meshes into a single `Meshes` object which represents the batch. The `Meshes` datastructure can then be used directly by other PyTorch3D ops which might be part of the deep learning model (e.g. `graph_conv`).\n",
|
||||||
|
"\n",
|
||||||
|
"For R2N2, if all the models in the batch have the same number of views, the views, rotation matrices, translation matrices, intrinsic matrices and voxels will also be stacked into batched tensors.\n",
|
||||||
|
"\n",
|
||||||
|
"**NOTE**: All models in the `val` split of R2N2 have 24 views, but there are 8 models that split their 24 views between `train` and `test` splits, in which case `collate_batched_meshes` will only be able to join the matrices, views and voxels as lists. However, this can be avoided by laoding only one view of each model by setting `return_all_views = False`."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"batch_size = 12\n",
|
||||||
|
"r2n2_single_view = R2N2(\"train\", SHAPENET_PATH, R2N2_PATH, SPLITS_PATH, return_all_views=False, return_voxels=True)\n",
|
||||||
|
"r2n2_loader = DataLoader(r2n2_single_view, batch_size=batch_size, collate_fn=collate_batched_meshes)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Let's visualize all the views (one for each model) in the batch:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"it = iter(r2n2_loader)\n",
|
||||||
|
"r2n2_batch = next(it)\n",
|
||||||
|
"batch_renderings = r2n2_batch[\"images\"] # (N, V, H, W, 3), and in this case V is 1.\n",
|
||||||
|
"image_grid(batch_renderings.squeeze().numpy(), rows=3, cols=4, rgb=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## 3. Render ShapeNetCore models with PyTorch3D's differntiable renderer"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Both `ShapeNetCore` and `R2N2` dataloaders have customized `render` functions that support rendering models by specifying their model ids, categories or indices using PyTorch3D's differentiable renderer implementation."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Rendering settings.\n",
|
||||||
|
"R, T = look_at_view_transform(1.0, 1.0, 90)\n",
|
||||||
|
"cameras = OpenGLPerspectiveCameras(R=R, T=T, device=device)\n",
|
||||||
|
"raster_settings = RasterizationSettings(image_size=512)\n",
|
||||||
|
"lights = PointLights(location=torch.tensor([0.0, 1.0, -2.0], device=device)[None],device=device)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"First we will try to render three models by their model ids:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"images_by_model_ids = shapenet_dataset.render(\n",
|
||||||
|
" model_ids=[\n",
|
||||||
|
" \"13394ca47c89f91525a3aaf903a41c90\",\n",
|
||||||
|
" \"14755c2ee8e693aba508f621166382b0\",\n",
|
||||||
|
" \"156c4207af6d2c8f1fdc97905708b8ea\",\n",
|
||||||
|
" ],\n",
|
||||||
|
" device=device,\n",
|
||||||
|
" cameras=cameras,\n",
|
||||||
|
" raster_settings=raster_settings,\n",
|
||||||
|
" lights=lights,\n",
|
||||||
|
")\n",
|
||||||
|
"image_grid(images_by_model_ids.cpu().numpy(), rows=1, cols=3, rgb=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Suppose we would like to render the first three models in the dataset, we can render models by their indices:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"images_by_idxs = shapenet_dataset.render(\n",
|
||||||
|
" idxs=list(range(3)),\n",
|
||||||
|
" device=device,\n",
|
||||||
|
" cameras=cameras,\n",
|
||||||
|
" raster_settings=raster_settings,\n",
|
||||||
|
" lights=lights,\n",
|
||||||
|
")\n",
|
||||||
|
"image_grid(images_by_idxs.cpu().numpy(), rows=1, cols=3, rgb=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Alternatively, if we are not interested in any particular models but would like see random models from some specific categories, we can do that by specifying `categories` and `sample_nums`. For example, if we would like to render 2 models from the category \"faucet\" and 3 models from the category \"chair\", we can do the following:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"images_by_categories = shapenet_dataset.render(\n",
|
||||||
|
" categories=[\"faucet\", \"chair\"],\n",
|
||||||
|
" sample_nums=[2, 3],\n",
|
||||||
|
" device=device,\n",
|
||||||
|
" cameras=cameras,\n",
|
||||||
|
" raster_settings=raster_settings,\n",
|
||||||
|
" lights=lights,\n",
|
||||||
|
")\n",
|
||||||
|
"image_grid(images_by_categories.cpu().numpy(), rows=1, cols=5, rgb=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"If we are not interested in any particular categories and just would like to render some random models from the whole dataset, we can set the number of models to be rendered in `sample_nums` and not specify any `categories`:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"random_model_images = shapenet_dataset.render(\n",
|
||||||
|
" sample_nums=[3],\n",
|
||||||
|
" device=device,\n",
|
||||||
|
" cameras=cameras,\n",
|
||||||
|
" raster_settings=raster_settings,\n",
|
||||||
|
" lights=lights,\n",
|
||||||
|
")\n",
|
||||||
|
"image_grid(random_model_images.cpu().numpy(), rows=1, cols=5, rgb=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## 4. Render R2N2 models with the same orientations as the original renderings in the dataset"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"We can render R2N2 models the same way as we rendered ShapeNetCore models above. In addition, we can also render R2N2 models with the same orientations as the original renderings in the dataset. For this we will use R2N2's customized `render` function and a different type of PyTorch3D camera called `BlenderCamera`."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"In this example, we will render the seventh model with the same orientations as its second and third views. First we will retrieve R2N2's original renderings to compare with the result."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"original_rendering = r2n2_dataset[6,[1,2]][\"images\"]\n",
|
||||||
|
"image_grid(original_rendering.numpy(), rows=1, cols=2, rgb=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Next, we will visualize PyTorch3d's renderings:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"r2n2_oriented_images = r2n2_dataset.render(\n",
|
||||||
|
" idxs=[6],\n",
|
||||||
|
" view_idxs=[1,2],\n",
|
||||||
|
" device=device,\n",
|
||||||
|
" raster_settings=raster_settings,\n",
|
||||||
|
" lights=lights,\n",
|
||||||
|
")\n",
|
||||||
|
"image_grid(r2n2_oriented_images.cpu().numpy(), rows=1, cols=2, rgb=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"## 5. Visualize R2N2 models' voxels"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"R2N2 dataloader also returns models' voxels. We can visualize them by utilizing R2N2's `render_vox_to_mesh` function. This will cubify the voxels to a Meshes object, which will then be rendered."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"In this example we will visualize the tenth model in the dataset with the same orientation of its second and third views. First we will retrieve R2N2's original renderings to compare with the result."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"r2n2_model = r2n2_dataset[9,[1,2]]\n",
|
||||||
|
"original_rendering = r2n2_model[\"images\"]\n",
|
||||||
|
"image_grid(original_rendering.numpy(), rows=1, cols=2, rgb=True)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"Next, we will pass the voxels to `render_vox_to_mesh`:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"vox_render = render_cubified_voxels(r2n2_model[\"voxels\"], device=device)\n",
|
||||||
|
"image_grid(vox_render.cpu().numpy(), rows=1, cols=2, rgb=True)"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"anp_metadata": {
|
||||||
|
"path": "fbsource/fbcode/vision/fair/pytorch3d/docs/tutorials/Dataloaders_ShapeNetCore_R2N2.ipynb"
|
||||||
|
},
|
||||||
|
"bento_stylesheets": {
|
||||||
|
"bento/extensions/flow/main.css": true,
|
||||||
|
"bento/extensions/kernel_selector/main.css": true,
|
||||||
|
"bento/extensions/kernel_ui/main.css": true,
|
||||||
|
"bento/extensions/new_kernel/main.css": true,
|
||||||
|
"bento/extensions/system_usage/main.css": true,
|
||||||
|
"bento/extensions/theme/main.css": true
|
||||||
|
},
|
||||||
|
"disseminate_notebook_info": {
|
||||||
|
"backup_notebook_id": "669429066983805"
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "intro_to_cv",
|
||||||
|
"language": "python",
|
||||||
|
"name": "bento_kernel_intro_to_cv"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.5+"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 2
|
||||||
|
}
|
||||||
301
files/dataloaders_ShapeNetCore_R2N2.py
Normal file
301
files/dataloaders_ShapeNetCore_R2N2.py
Normal file
@ -0,0 +1,301 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# coding: utf-8
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
|
||||||
|
|
||||||
|
|
||||||
|
# # Dataloaders for ShapeNetCore and R2N2
|
||||||
|
# This tutorial shows how to:
|
||||||
|
# - Load models from ShapeNetCore and R2N2 using PyTorch3D's data loaders.
|
||||||
|
# - Pass the loaded datasets to `torch.utils.data.DataLoader`.
|
||||||
|
# - Render ShapeNetCore models with PyTorch3D's renderer.
|
||||||
|
# - Render R2N2 models with the same orientations as the original renderings in the dataset.
|
||||||
|
# - Visualize R2N2 model voxels.
|
||||||
|
|
||||||
|
# ## 0. Install and import modules
|
||||||
|
|
||||||
|
# If `torch`, `torchvision` and `pytorch3d` are not installed, run the following cell:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
get_ipython().system('pip install torch torchvision')
|
||||||
|
import sys
|
||||||
|
import torch
|
||||||
|
if torch.__version__=='1.6.0+cu101' and sys.platform.startswith('linux'):
|
||||||
|
get_ipython().system('pip install pytorch3d')
|
||||||
|
else:
|
||||||
|
get_ipython().system("pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'")
|
||||||
|
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import torch
|
||||||
|
|
||||||
|
from pytorch3d.datasets import (
|
||||||
|
R2N2,
|
||||||
|
ShapeNetCore,
|
||||||
|
collate_batched_meshes,
|
||||||
|
render_cubified_voxels,
|
||||||
|
)
|
||||||
|
from pytorch3d.renderer import (
|
||||||
|
OpenGLPerspectiveCameras,
|
||||||
|
PointLights,
|
||||||
|
RasterizationSettings,
|
||||||
|
TexturesVertex,
|
||||||
|
look_at_view_transform,
|
||||||
|
)
|
||||||
|
|
||||||
|
from pytorch3d.structures import Meshes
|
||||||
|
from torch.utils.data import DataLoader
|
||||||
|
|
||||||
|
# add path for demo utils functions
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
sys.path.append(os.path.abspath(''))
|
||||||
|
|
||||||
|
|
||||||
|
# If using **Google Colab**, fetch the utils file for plotting image grids:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
get_ipython().system('wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py')
|
||||||
|
from plot_image_grid import image_grid
|
||||||
|
|
||||||
|
|
||||||
|
# OR if running locally uncomment and run the following cell:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# from utils import image_grid
|
||||||
|
|
||||||
|
|
||||||
|
# ## 1. Load the datasets
|
||||||
|
|
||||||
|
# If you haven't already downloaded the ShapeNetCore dataset, first do that following the instructions here: https://www.shapenet.org/. ShapeNetCore is a subset of the ShapeNet dataset. In PyTorch3D we support both version 1 (57 categories) and version 2 (55 categories).
|
||||||
|
#
|
||||||
|
# Then modify `SHAPENET_PATH` below to you local path to the ShapeNetCore dataset folder.
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# Setup
|
||||||
|
if torch.cuda.is_available():
|
||||||
|
device = torch.device("cuda:0")
|
||||||
|
torch.cuda.set_device(device)
|
||||||
|
else:
|
||||||
|
device = torch.device("cpu")
|
||||||
|
|
||||||
|
SHAPENET_PATH = ""
|
||||||
|
shapenet_dataset = ShapeNetCore(SHAPENET_PATH)
|
||||||
|
|
||||||
|
|
||||||
|
# The R2N2 dataset can be downloaded using the instructions here: http://3d-r2n2.stanford.edu/. Look at the links for `ShapeNetRendering` and `ShapeNetVox32`. The R2N2 dataset contains 13 categories that are a subset of the ShapeNetCore v.1
|
||||||
|
# dataset. The R2N2 dataset also contains its own 24 renderings of each object and voxelized models.
|
||||||
|
#
|
||||||
|
# Then modify `R2N2_PATH` and `SPLITS_PATH` below to your local R2N2 dataset folder path and splits file path respectively. Here we will load the `train` split of R2N2 and ask the voxels of each model to be returned.
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
R2N2_PATH = ""
|
||||||
|
SPLITS_PATH = "None"
|
||||||
|
r2n2_dataset = R2N2("train", SHAPENET_PATH, R2N2_PATH, SPLITS_PATH, return_voxels=True)
|
||||||
|
|
||||||
|
|
||||||
|
# We can retrieve a model by indexing into the loaded dataset. For both ShapeNetCore and R2N2, we can examine the category this model belongs to (in the form of a synset id, equivalend to wnid described in ImageNet's API: http://image-net.org/download-API), its model id, and its vertices and faces.
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
shapenet_model = shapenet_dataset[6]
|
||||||
|
print("This model belongs to the category " + shapenet_model["synset_id"] + ".")
|
||||||
|
print("This model has model id " + shapenet_model["model_id"] + ".")
|
||||||
|
model_verts, model_faces = shapenet_model["verts"], shapenet_model["faces"]
|
||||||
|
|
||||||
|
|
||||||
|
# We can use its vertices and faces to form a `Meshes` object which is a PyTorch3D datastructure for working with batched meshes.
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
model_textures = TexturesVertex(verts_features=torch.ones_like(model_verts, device=device)[None])
|
||||||
|
shapenet_model_mesh = Meshes(
|
||||||
|
verts=[model_verts.to(device)],
|
||||||
|
faces=[model_faces.to(device)],
|
||||||
|
textures=model_textures
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# With R2N2, we can further examine R2N2's original renderings. For instance, if we would like to see the second and third views of the eleventh objects in the R2N2 dataset, we can do the following:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
r2n2_renderings = r2n2_dataset[10,[1,2]]
|
||||||
|
image_grid(r2n2_renderings.numpy(), rows=1, cols=2, rgb=True)
|
||||||
|
|
||||||
|
|
||||||
|
# ## 2. Use the datasets with `torch.utils.data.DataLoader`
|
||||||
|
|
||||||
|
# Training deep learning models, usually requires passing in batches of inputs. The `torch.utils.data.DataLoader` from Pytorch helps us do this. PyTorch3D provides a function `collate_batched_meshes` to group the input meshes into a single `Meshes` object which represents the batch. The `Meshes` datastructure can then be used directly by other PyTorch3D ops which might be part of the deep learning model (e.g. `graph_conv`).
|
||||||
|
#
|
||||||
|
# For R2N2, if all the models in the batch have the same number of views, the views, rotation matrices, translation matrices, intrinsic matrices and voxels will also be stacked into batched tensors.
|
||||||
|
#
|
||||||
|
# **NOTE**: All models in the `val` split of R2N2 have 24 views, but there are 8 models that split their 24 views between `train` and `test` splits, in which case `collate_batched_meshes` will only be able to join the matrices, views and voxels as lists. However, this can be avoided by laoding only one view of each model by setting `return_all_views = False`.
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
batch_size = 12
|
||||||
|
r2n2_single_view = R2N2("train", SHAPENET_PATH, R2N2_PATH, SPLITS_PATH, return_all_views=False, return_voxels=True)
|
||||||
|
r2n2_loader = DataLoader(r2n2_single_view, batch_size=batch_size, collate_fn=collate_batched_meshes)
|
||||||
|
|
||||||
|
|
||||||
|
# Let's visualize all the views (one for each model) in the batch:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
it = iter(r2n2_loader)
|
||||||
|
r2n2_batch = next(it)
|
||||||
|
batch_renderings = r2n2_batch["images"] # (N, V, H, W, 3), and in this case V is 1.
|
||||||
|
image_grid(batch_renderings.squeeze().numpy(), rows=3, cols=4, rgb=True)
|
||||||
|
|
||||||
|
|
||||||
|
# ## 3. Render ShapeNetCore models with PyTorch3D's differntiable renderer
|
||||||
|
|
||||||
|
# Both `ShapeNetCore` and `R2N2` dataloaders have customized `render` functions that support rendering models by specifying their model ids, categories or indices using PyTorch3D's differentiable renderer implementation.
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# Rendering settings.
|
||||||
|
R, T = look_at_view_transform(1.0, 1.0, 90)
|
||||||
|
cameras = OpenGLPerspectiveCameras(R=R, T=T, device=device)
|
||||||
|
raster_settings = RasterizationSettings(image_size=512)
|
||||||
|
lights = PointLights(location=torch.tensor([0.0, 1.0, -2.0], device=device)[None],device=device)
|
||||||
|
|
||||||
|
|
||||||
|
# First we will try to render three models by their model ids:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
images_by_model_ids = shapenet_dataset.render(
|
||||||
|
model_ids=[
|
||||||
|
"13394ca47c89f91525a3aaf903a41c90",
|
||||||
|
"14755c2ee8e693aba508f621166382b0",
|
||||||
|
"156c4207af6d2c8f1fdc97905708b8ea",
|
||||||
|
],
|
||||||
|
device=device,
|
||||||
|
cameras=cameras,
|
||||||
|
raster_settings=raster_settings,
|
||||||
|
lights=lights,
|
||||||
|
)
|
||||||
|
image_grid(images_by_model_ids.cpu().numpy(), rows=1, cols=3, rgb=True)
|
||||||
|
|
||||||
|
|
||||||
|
# Suppose we would like to render the first three models in the dataset, we can render models by their indices:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
images_by_idxs = shapenet_dataset.render(
|
||||||
|
idxs=list(range(3)),
|
||||||
|
device=device,
|
||||||
|
cameras=cameras,
|
||||||
|
raster_settings=raster_settings,
|
||||||
|
lights=lights,
|
||||||
|
)
|
||||||
|
image_grid(images_by_idxs.cpu().numpy(), rows=1, cols=3, rgb=True)
|
||||||
|
|
||||||
|
|
||||||
|
# Alternatively, if we are not interested in any particular models but would like see random models from some specific categories, we can do that by specifying `categories` and `sample_nums`. For example, if we would like to render 2 models from the category "faucet" and 3 models from the category "chair", we can do the following:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
images_by_categories = shapenet_dataset.render(
|
||||||
|
categories=["faucet", "chair"],
|
||||||
|
sample_nums=[2, 3],
|
||||||
|
device=device,
|
||||||
|
cameras=cameras,
|
||||||
|
raster_settings=raster_settings,
|
||||||
|
lights=lights,
|
||||||
|
)
|
||||||
|
image_grid(images_by_categories.cpu().numpy(), rows=1, cols=5, rgb=True)
|
||||||
|
|
||||||
|
|
||||||
|
# If we are not interested in any particular categories and just would like to render some random models from the whole dataset, we can set the number of models to be rendered in `sample_nums` and not specify any `categories`:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
random_model_images = shapenet_dataset.render(
|
||||||
|
sample_nums=[3],
|
||||||
|
device=device,
|
||||||
|
cameras=cameras,
|
||||||
|
raster_settings=raster_settings,
|
||||||
|
lights=lights,
|
||||||
|
)
|
||||||
|
image_grid(random_model_images.cpu().numpy(), rows=1, cols=5, rgb=True)
|
||||||
|
|
||||||
|
|
||||||
|
# ## 4. Render R2N2 models with the same orientations as the original renderings in the dataset
|
||||||
|
|
||||||
|
# We can render R2N2 models the same way as we rendered ShapeNetCore models above. In addition, we can also render R2N2 models with the same orientations as the original renderings in the dataset. For this we will use R2N2's customized `render` function and a different type of PyTorch3D camera called `BlenderCamera`.
|
||||||
|
|
||||||
|
# In this example, we will render the seventh model with the same orientations as its second and third views. First we will retrieve R2N2's original renderings to compare with the result.
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
original_rendering = r2n2_dataset[6,[1,2]]["images"]
|
||||||
|
image_grid(original_rendering.numpy(), rows=1, cols=2, rgb=True)
|
||||||
|
|
||||||
|
|
||||||
|
# Next, we will visualize PyTorch3d's renderings:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
r2n2_oriented_images = r2n2_dataset.render(
|
||||||
|
idxs=[6],
|
||||||
|
view_idxs=[1,2],
|
||||||
|
device=device,
|
||||||
|
raster_settings=raster_settings,
|
||||||
|
lights=lights,
|
||||||
|
)
|
||||||
|
image_grid(r2n2_oriented_images.cpu().numpy(), rows=1, cols=2, rgb=True)
|
||||||
|
|
||||||
|
|
||||||
|
# ## 5. Visualize R2N2 models' voxels
|
||||||
|
|
||||||
|
# R2N2 dataloader also returns models' voxels. We can visualize them by utilizing R2N2's `render_vox_to_mesh` function. This will cubify the voxels to a Meshes object, which will then be rendered.
|
||||||
|
|
||||||
|
# In this example we will visualize the tenth model in the dataset with the same orientation of its second and third views. First we will retrieve R2N2's original renderings to compare with the result.
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
r2n2_model = r2n2_dataset[9,[1,2]]
|
||||||
|
original_rendering = r2n2_model["images"]
|
||||||
|
image_grid(original_rendering.numpy(), rows=1, cols=2, rgb=True)
|
||||||
|
|
||||||
|
|
||||||
|
# Next, we will pass the voxels to `render_vox_to_mesh`:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
vox_render = render_cubified_voxels(r2n2_model["voxels"], device=device)
|
||||||
|
image_grid(vox_render.cpu().numpy(), rows=1, cols=2, rgb=True)
|
||||||
|
|
||||||
File diff suppressed because one or more lines are too long
@ -1,4 +1,4 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
|
|
||||||
# In[ ]:
|
# In[ ]:
|
||||||
@ -40,7 +40,12 @@
|
|||||||
|
|
||||||
|
|
||||||
get_ipython().system('pip install torch torchvision')
|
get_ipython().system('pip install torch torchvision')
|
||||||
get_ipython().system("pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'")
|
import sys
|
||||||
|
import torch
|
||||||
|
if torch.__version__=='1.6.0+cu101' and sys.platform.startswith('linux'):
|
||||||
|
get_ipython().system('pip install pytorch3d')
|
||||||
|
else:
|
||||||
|
get_ipython().system("pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'")
|
||||||
|
|
||||||
|
|
||||||
# In[ ]:
|
# In[ ]:
|
||||||
@ -59,7 +64,7 @@ from pytorch3d.loss import (
|
|||||||
mesh_normal_consistency,
|
mesh_normal_consistency,
|
||||||
)
|
)
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from tqdm import tqdm_notebook
|
from tqdm.notebook import tqdm
|
||||||
get_ipython().run_line_magic('matplotlib', 'notebook')
|
get_ipython().run_line_magic('matplotlib', 'notebook')
|
||||||
from mpl_toolkits.mplot3d import Axes3D
|
from mpl_toolkits.mplot3d import Axes3D
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
@ -68,14 +73,18 @@ mpl.rcParams['savefig.dpi'] = 80
|
|||||||
mpl.rcParams['figure.dpi'] = 80
|
mpl.rcParams['figure.dpi'] = 80
|
||||||
|
|
||||||
# Set the device
|
# Set the device
|
||||||
device = torch.device("cuda:0")
|
if torch.cuda.is_available():
|
||||||
|
device = torch.device("cuda:0")
|
||||||
|
else:
|
||||||
|
device = torch.device("cpu")
|
||||||
|
print("WARNING: CPU only, this will be slow!")
|
||||||
|
|
||||||
|
|
||||||
# ## 1. Load an obj file and create a Meshes object
|
# ## 1. Load an obj file and create a Meshes object
|
||||||
|
|
||||||
# Download the target 3D model of a dolphin. It will be saved locally as a file called `dolphin.obj`.
|
# Download the target 3D model of a dolphin. It will be saved locally as a file called `dolphin.obj`.
|
||||||
|
|
||||||
# In[1]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
get_ipython().system('wget https://dl.fbaipublicfiles.com/pytorch3d/data/dolphin/dolphin.obj')
|
get_ipython().system('wget https://dl.fbaipublicfiles.com/pytorch3d/data/dolphin/dolphin.obj')
|
||||||
@ -139,7 +148,7 @@ def plot_pointcloud(mesh, title=""):
|
|||||||
plt.show()
|
plt.show()
|
||||||
|
|
||||||
|
|
||||||
# In[75]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# %matplotlib notebook
|
# %matplotlib notebook
|
||||||
@ -164,7 +173,7 @@ deform_verts = torch.full(src_mesh.verts_packed().shape, 0.0, device=device, req
|
|||||||
optimizer = torch.optim.SGD([deform_verts], lr=1.0, momentum=0.9)
|
optimizer = torch.optim.SGD([deform_verts], lr=1.0, momentum=0.9)
|
||||||
|
|
||||||
|
|
||||||
# In[78]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# Number of optimization steps
|
# Number of optimization steps
|
||||||
@ -179,7 +188,7 @@ w_normal = 0.01
|
|||||||
w_laplacian = 0.1
|
w_laplacian = 0.1
|
||||||
# Plot period for the losses
|
# Plot period for the losses
|
||||||
plot_period = 250
|
plot_period = 250
|
||||||
loop = tqdm_notebook(range(Niter))
|
loop = tqdm(range(Niter))
|
||||||
|
|
||||||
chamfer_losses = []
|
chamfer_losses = []
|
||||||
laplacian_losses = []
|
laplacian_losses = []
|
||||||
@ -234,7 +243,7 @@ for i in loop:
|
|||||||
|
|
||||||
# ## 4. Visualize the loss
|
# ## 4. Visualize the loss
|
||||||
|
|
||||||
# In[79]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
fig = plt.figure(figsize=(13, 5))
|
fig = plt.figure(figsize=(13, 5))
|
||||||
@ -246,7 +255,7 @@ ax.plot(laplacian_losses, label="laplacian loss")
|
|||||||
ax.legend(fontsize="16")
|
ax.legend(fontsize="16")
|
||||||
ax.set_xlabel("Iteration", fontsize="16")
|
ax.set_xlabel("Iteration", fontsize="16")
|
||||||
ax.set_ylabel("Loss", fontsize="16")
|
ax.set_ylabel("Loss", fontsize="16")
|
||||||
ax.set_title("Loss vs iterations", fontsize="16")
|
ax.set_title("Loss vs iterations", fontsize="16");
|
||||||
|
|
||||||
|
|
||||||
# ## 5. Save the predicted mesh
|
# ## 5. Save the predicted mesh
|
||||||
|
|||||||
940
files/fit_textured_mesh.ipynb
Normal file
940
files/fit_textured_mesh.ipynb
Normal file
@ -0,0 +1,940 @@
|
|||||||
|
{
|
||||||
|
"cells": [
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "_Ip8kp4TfBLZ"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "kuXHJv44fBLe"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"# Fit a mesh via rendering\n",
|
||||||
|
"\n",
|
||||||
|
"This tutorial shows how to:\n",
|
||||||
|
"- Load a mesh and textures from an `.obj` file. \n",
|
||||||
|
"- Create a synthetic dataset by rendering a textured mesh from multiple viewpoints\n",
|
||||||
|
"- Fit a mesh to the observed synthetic images using differential silhouette rendering\n",
|
||||||
|
"- Fit a mesh and its textures using differential textured rendering"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "Bnj3THhzfBLf"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## 0. Install and Import modules"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "okLalbR_g7NS"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"If `torch`, `torchvision` and `pytorch3d` are not installed, run the following cell:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "musUWTglgxSB"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"!pip install torch torchvision\n",
|
||||||
|
"import sys\n",
|
||||||
|
"import torch\n",
|
||||||
|
"if torch.__version__=='1.6.0+cu101' and sys.platform.startswith('linux'):\n",
|
||||||
|
" !pip install pytorch3d\n",
|
||||||
|
"else:\n",
|
||||||
|
" !pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "nX99zdoffBLg"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"import os\n",
|
||||||
|
"import torch\n",
|
||||||
|
"import matplotlib.pyplot as plt\n",
|
||||||
|
"from skimage.io import imread\n",
|
||||||
|
"\n",
|
||||||
|
"from pytorch3d.utils import ico_sphere\n",
|
||||||
|
"import numpy as np\n",
|
||||||
|
"from tqdm.notebook import tqdm\n",
|
||||||
|
"\n",
|
||||||
|
"# Util function for loading meshes\n",
|
||||||
|
"from pytorch3d.io import load_objs_as_meshes, save_obj\n",
|
||||||
|
"\n",
|
||||||
|
"from pytorch3d.loss import (\n",
|
||||||
|
" chamfer_distance, \n",
|
||||||
|
" mesh_edge_loss, \n",
|
||||||
|
" mesh_laplacian_smoothing, \n",
|
||||||
|
" mesh_normal_consistency,\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"# Data structures and functions for rendering\n",
|
||||||
|
"from pytorch3d.structures import Meshes\n",
|
||||||
|
"from pytorch3d.renderer import (\n",
|
||||||
|
" look_at_view_transform,\n",
|
||||||
|
" OpenGLPerspectiveCameras, \n",
|
||||||
|
" PointLights, \n",
|
||||||
|
" DirectionalLights, \n",
|
||||||
|
" Materials, \n",
|
||||||
|
" RasterizationSettings, \n",
|
||||||
|
" MeshRenderer, \n",
|
||||||
|
" MeshRasterizer, \n",
|
||||||
|
" SoftPhongShader,\n",
|
||||||
|
" SoftSilhouetteShader,\n",
|
||||||
|
" SoftPhongShader,\n",
|
||||||
|
" TexturesVertex\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"# add path for demo utils functions \n",
|
||||||
|
"import sys\n",
|
||||||
|
"import os\n",
|
||||||
|
"sys.path.append(os.path.abspath(''))"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "Lxmehq6Zhrzv"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"If using **Google Colab**, fetch the utils file for plotting image grids:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "HZozr3Pmho-5"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"!wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py\n",
|
||||||
|
"from plot_image_grid import image_grid"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "g4B62MzYiJUM"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"OR if running **locally** uncomment and run the following cell:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "paJ4Im8ahl7O"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# from utils.plot_image_grid import image_grid"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"collapsed": true,
|
||||||
|
"id": "5jGq772XfBLk"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"### 1. Load a mesh and texture file\n",
|
||||||
|
"\n",
|
||||||
|
"Load an `.obj` file and it's associated `.mtl` file and create a **Textures** and **Meshes** object. \n",
|
||||||
|
"\n",
|
||||||
|
"**Meshes** is a unique datastructure provided in PyTorch3D for working with batches of meshes of different sizes. \n",
|
||||||
|
"\n",
|
||||||
|
"**TexturesVertex** is an auxillary datastructure for storing vertex rgb texture information about meshes. \n",
|
||||||
|
"\n",
|
||||||
|
"**Meshes** has several class methods which are used throughout the rendering pipeline."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "a8eU4zo5jd_H"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"If running this notebook using **Google Colab**, run the following cell to fetch the mesh obj and texture files and save it at the path `data/cow_mesh`:\n",
|
||||||
|
"If running locally, the data is already available at the correct path. "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "tTm0cVuOjb1W"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"!mkdir -p data/cow_mesh\n",
|
||||||
|
"!wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.obj\n",
|
||||||
|
"!wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.mtl\n",
|
||||||
|
"!wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow_texture.png"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "gi5Kd0GafBLl"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Setup\n",
|
||||||
|
"if torch.cuda.is_available():\n",
|
||||||
|
" device = torch.device(\"cuda:0\")\n",
|
||||||
|
" torch.cuda.set_device(device)\n",
|
||||||
|
"else:\n",
|
||||||
|
" device = torch.device(\"cpu\")\n",
|
||||||
|
"\n",
|
||||||
|
"# Set paths\n",
|
||||||
|
"DATA_DIR = \"./data\"\n",
|
||||||
|
"obj_filename = os.path.join(DATA_DIR, \"cow_mesh/cow.obj\")\n",
|
||||||
|
"\n",
|
||||||
|
"# Load obj file\n",
|
||||||
|
"mesh = load_objs_as_meshes([obj_filename], device=device)\n",
|
||||||
|
"\n",
|
||||||
|
"# We scale normalize and center the target mesh to fit in a sphere of radius 1 \n",
|
||||||
|
"# centered at (0,0,0). (scale, center) will be used to bring the predicted mesh \n",
|
||||||
|
"# to its original center and scale. Note that normalizing the target mesh, \n",
|
||||||
|
"# speeds up the optimization but is not necessary!\n",
|
||||||
|
"verts = mesh.verts_packed()\n",
|
||||||
|
"N = verts.shape[0]\n",
|
||||||
|
"center = verts.mean(0)\n",
|
||||||
|
"scale = max((verts - center).abs().max(0)[0])\n",
|
||||||
|
"mesh.offset_verts_(-center.expand(N, 3))\n",
|
||||||
|
"mesh.scale_verts_((1.0 / float(scale)));"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "17c4xmtyfBMH"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## 2. Dataset Creation\n",
|
||||||
|
"\n",
|
||||||
|
"We sample different camera positions that encode multiple viewpoints of the cow. We create a renderer with a shader that performs texture map interpolation. We render a synthetic dataset of images of the textured cow mesh from multiple viewpoints.\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "CDQKebNNfBMI"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# the number of different viewpoints from which we want to render the mesh.\n",
|
||||||
|
"num_views = 20\n",
|
||||||
|
"\n",
|
||||||
|
"# Get a batch of viewing angles. \n",
|
||||||
|
"elev = torch.linspace(0, 360, num_views)\n",
|
||||||
|
"azim = torch.linspace(-180, 180, num_views)\n",
|
||||||
|
"\n",
|
||||||
|
"# Place a point light in front of the object. As mentioned above, the front of \n",
|
||||||
|
"# the cow is facing the -z direction. \n",
|
||||||
|
"lights = PointLights(device=device, location=[[0.0, 0.0, -3.0]])\n",
|
||||||
|
"\n",
|
||||||
|
"# Initialize an OpenGL perspective camera that represents a batch of different \n",
|
||||||
|
"# viewing angles. All the cameras helper methods support mixed type inputs and \n",
|
||||||
|
"# broadcasting. So we can view the camera from the a distance of dist=2.7, and \n",
|
||||||
|
"# then specify elevation and azimuth angles for each viewpoint as tensors. \n",
|
||||||
|
"R, T = look_at_view_transform(dist=2.7, elev=elev, azim=azim)\n",
|
||||||
|
"cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T)\n",
|
||||||
|
"\n",
|
||||||
|
"# We arbitrarily choose one particular view that will be used to visualize \n",
|
||||||
|
"# results\n",
|
||||||
|
"camera = OpenGLPerspectiveCameras(device=device, R=R[None, 1, ...], \n",
|
||||||
|
" T=T[None, 1, ...]) \n",
|
||||||
|
"\n",
|
||||||
|
"# Define the settings for rasterization and shading. Here we set the output \n",
|
||||||
|
"# image to be of size 128X128. As we are rendering images for visualization \n",
|
||||||
|
"# purposes only we will set faces_per_pixel=1 and blur_radius=0.0. Refer to \n",
|
||||||
|
"# rasterize_meshes.py for explanations of these parameters. We also leave \n",
|
||||||
|
"# bin_size and max_faces_per_bin to their default values of None, which sets \n",
|
||||||
|
"# their values using huristics and ensures that the faster coarse-to-fine \n",
|
||||||
|
"# rasterization method is used. Refer to docs/notes/renderer.md for an \n",
|
||||||
|
"# explanation of the difference between naive and coarse-to-fine rasterization. \n",
|
||||||
|
"raster_settings = RasterizationSettings(\n",
|
||||||
|
" image_size=128, \n",
|
||||||
|
" blur_radius=0.0, \n",
|
||||||
|
" faces_per_pixel=1, \n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"# Create a phong renderer by composing a rasterizer and a shader. The textured \n",
|
||||||
|
"# phong shader will interpolate the texture uv coordinates for each vertex, \n",
|
||||||
|
"# sample from a texture image and apply the Phong lighting model\n",
|
||||||
|
"renderer = MeshRenderer(\n",
|
||||||
|
" rasterizer=MeshRasterizer(\n",
|
||||||
|
" cameras=camera, \n",
|
||||||
|
" raster_settings=raster_settings\n",
|
||||||
|
" ),\n",
|
||||||
|
" shader=SoftPhongShader(\n",
|
||||||
|
" device=device, \n",
|
||||||
|
" cameras=camera,\n",
|
||||||
|
" lights=lights\n",
|
||||||
|
" )\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"# Create a batch of meshes by repeating the cow mesh and associated textures. \n",
|
||||||
|
"# Meshes has a useful `extend` method which allows us do this very easily. \n",
|
||||||
|
"# This also extends the textures. \n",
|
||||||
|
"meshes = mesh.extend(num_views)\n",
|
||||||
|
"\n",
|
||||||
|
"# Render the cow mesh from each viewing angle\n",
|
||||||
|
"target_images = renderer(meshes, cameras=cameras, lights=lights)\n",
|
||||||
|
"\n",
|
||||||
|
"# Our multi-view cow dataset will be represented by these 2 lists of tensors,\n",
|
||||||
|
"# each of length num_views.\n",
|
||||||
|
"target_rgb = [target_images[i, ..., :3] for i in range(num_views)]\n",
|
||||||
|
"target_cameras = [OpenGLPerspectiveCameras(device=device, R=R[None, i, ...], \n",
|
||||||
|
" T=T[None, i, ...]) for i in range(num_views)]"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "TppB4PVmR1Rc"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"Visualize the dataset:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "HHE0CnbVR1Rd"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# RGB images\n",
|
||||||
|
"image_grid(target_images.cpu().numpy(), rows=4, cols=5, rgb=True)\n",
|
||||||
|
"plt.show()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "gOb4rYx65E8z"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"Later in this tutorial, we will fit a mesh to the rendered RGB images, as well as to just images of just the cow silhouette. For the latter case, we will render a dataset of silhouette images. Most shaders in PyTorch3D will output an alpha channel along with the RGB image as a 4th channel in an RGBA image. The alpha channel encodes the probability that each pixel belongs to the foreground of the object. We contruct a soft silhouette shader to render this alpha channel."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "iP_g-nwX4exM"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Rasterization settings for silhouette rendering \n",
|
||||||
|
"sigma = 1e-4\n",
|
||||||
|
"raster_settings_silhouette = RasterizationSettings(\n",
|
||||||
|
" image_size=128, \n",
|
||||||
|
" blur_radius=np.log(1. / 1e-4 - 1.)*sigma, \n",
|
||||||
|
" faces_per_pixel=50, \n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"# Silhouette renderer \n",
|
||||||
|
"renderer_silhouette = MeshRenderer(\n",
|
||||||
|
" rasterizer=MeshRasterizer(\n",
|
||||||
|
" cameras=camera, \n",
|
||||||
|
" raster_settings=raster_settings_silhouette\n",
|
||||||
|
" ),\n",
|
||||||
|
" shader=SoftSilhouetteShader()\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"# Render silhouette images. The 3rd channel of the rendering output is \n",
|
||||||
|
"# the alpha/silhouette channel\n",
|
||||||
|
"silhouette_images = renderer_silhouette(meshes, cameras=cameras, lights=lights)\n",
|
||||||
|
"target_silhouette = [silhouette_images[i, ..., 3] for i in range(num_views)]\n",
|
||||||
|
"\n",
|
||||||
|
"# Visualize silhouette images\n",
|
||||||
|
"image_grid(silhouette_images.cpu().numpy(), rows=4, cols=5, rgb=False)\n",
|
||||||
|
"plt.show()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "t3qphI1ElUb5"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## 3. Mesh prediction via silhouette rendering\n",
|
||||||
|
"In the previous section, we created a dataset of images of multiple viewpoints of a cow. In this section, we predict a mesh by observing those target images without any knowledge of the ground truth cow mesh. We assume we know the position of the cameras and lighting.\n",
|
||||||
|
"\n",
|
||||||
|
"We first define some helper functions to visualize the results of our mesh prediction:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "eeWYHROrR1Rh"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Show a visualization comparing the rendered predicted mesh to the ground truth \n",
|
||||||
|
"# mesh\n",
|
||||||
|
"def visualize_prediction(predicted_mesh, renderer=renderer_silhouette, \n",
|
||||||
|
" target_image=target_rgb[1], title='', \n",
|
||||||
|
" silhouette=False):\n",
|
||||||
|
" inds = 3 if silhouette else range(3)\n",
|
||||||
|
" predicted_images = renderer(predicted_mesh)\n",
|
||||||
|
" plt.figure(figsize=(20, 10))\n",
|
||||||
|
" plt.subplot(1, 2, 1)\n",
|
||||||
|
" plt.imshow(predicted_images[0, ..., inds].cpu().detach().numpy())\n",
|
||||||
|
"\n",
|
||||||
|
" plt.subplot(1, 2, 2)\n",
|
||||||
|
" plt.imshow(target_image.cpu().detach().numpy())\n",
|
||||||
|
" plt.title(title)\n",
|
||||||
|
" plt.grid(\"off\")\n",
|
||||||
|
" plt.axis(\"off\")\n",
|
||||||
|
"\n",
|
||||||
|
"# Plot losses as a function of optimization iteration\n",
|
||||||
|
"def plot_losses(losses):\n",
|
||||||
|
" fig = plt.figure(figsize=(13, 5))\n",
|
||||||
|
" ax = fig.gca()\n",
|
||||||
|
" for k, l in losses.items():\n",
|
||||||
|
" ax.plot(l['values'], label=k + \" loss\")\n",
|
||||||
|
" ax.legend(fontsize=\"16\")\n",
|
||||||
|
" ax.set_xlabel(\"Iteration\", fontsize=\"16\")\n",
|
||||||
|
" ax.set_ylabel(\"Loss\", fontsize=\"16\")\n",
|
||||||
|
" ax.set_title(\"Loss vs iterations\", fontsize=\"16\")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "PpsvBpuMR1Ri"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"Starting from a sphere mesh, we will learn offsets of each vertex such that the predicted mesh silhouette is more similar to the target silhouette image at each optimization step. We begin by loading our initial sphere mesh:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "i989ARH1R1Rj"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# We initialize the source shape to be a sphere of radius 1. \n",
|
||||||
|
"src_mesh = ico_sphere(4, device)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "f5xVtgLNDvC5"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"We create a new differentiable renderer for rendering the silhouette of our predicted mesh:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "sXfjzgG4DsDJ"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Rasterization settings for differentiable rendering, where the blur_radius\n",
|
||||||
|
"# initialization is based on Liu et al, 'Soft Rasterizer: A Differentiable \n",
|
||||||
|
"# Renderer for Image-based 3D Reasoning', ICCV 2019\n",
|
||||||
|
"sigma = 1e-4\n",
|
||||||
|
"raster_settings_soft = RasterizationSettings(\n",
|
||||||
|
" image_size=128, \n",
|
||||||
|
" blur_radius=np.log(1. / 1e-4 - 1.)*sigma, \n",
|
||||||
|
" faces_per_pixel=50, \n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"# Silhouette renderer \n",
|
||||||
|
"renderer_silhouette = MeshRenderer(\n",
|
||||||
|
" rasterizer=MeshRasterizer(\n",
|
||||||
|
" cameras=camera, \n",
|
||||||
|
" raster_settings=raster_settings_soft\n",
|
||||||
|
" ),\n",
|
||||||
|
" shader=SoftSilhouetteShader()\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "SGJKbCB6R1Rk"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"We initialize settings, losses, and the optimizer that will be used to iteratively fit our mesh to the target silhouettes:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "0sLrKv_MEULh"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Number of views to optimize over in each SGD iteration\n",
|
||||||
|
"num_views_per_iteration = 2\n",
|
||||||
|
"# Number of optimization steps\n",
|
||||||
|
"Niter = 2000\n",
|
||||||
|
"# Plot period for the losses\n",
|
||||||
|
"plot_period = 250\n",
|
||||||
|
"\n",
|
||||||
|
"%matplotlib inline\n",
|
||||||
|
"\n",
|
||||||
|
"# Optimize using rendered silhouette image loss, mesh edge loss, mesh normal \n",
|
||||||
|
"# consistency, and mesh laplacian smoothing\n",
|
||||||
|
"losses = {\"silhouette\": {\"weight\": 1.0, \"values\": []},\n",
|
||||||
|
" \"edge\": {\"weight\": 1.0, \"values\": []},\n",
|
||||||
|
" \"normal\": {\"weight\": 0.01, \"values\": []},\n",
|
||||||
|
" \"laplacian\": {\"weight\": 1.0, \"values\": []},\n",
|
||||||
|
" }\n",
|
||||||
|
"\n",
|
||||||
|
"# Losses to smooth / regularize the mesh shape\n",
|
||||||
|
"def update_mesh_shape_prior_losses(mesh, loss):\n",
|
||||||
|
" # and (b) the edge length of the predicted mesh\n",
|
||||||
|
" loss[\"edge\"] = mesh_edge_loss(mesh)\n",
|
||||||
|
" \n",
|
||||||
|
" # mesh normal consistency\n",
|
||||||
|
" loss[\"normal\"] = mesh_normal_consistency(mesh)\n",
|
||||||
|
" \n",
|
||||||
|
" # mesh laplacian smoothing\n",
|
||||||
|
" loss[\"laplacian\"] = mesh_laplacian_smoothing(mesh, method=\"uniform\")\n",
|
||||||
|
"\n",
|
||||||
|
"# We will learn to deform the source mesh by offsetting its vertices\n",
|
||||||
|
"# The shape of the deform parameters is equal to the total number of vertices in\n",
|
||||||
|
"# src_mesh\n",
|
||||||
|
"verts_shape = src_mesh.verts_packed().shape\n",
|
||||||
|
"deform_verts = torch.full(verts_shape, 0.0, device=device, requires_grad=True)\n",
|
||||||
|
"\n",
|
||||||
|
"# The optimizer\n",
|
||||||
|
"optimizer = torch.optim.SGD([deform_verts], lr=1.0, momentum=0.9)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "QLc9zK8lEqFS"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"We write an optimization loop to iteratively refine our predicted mesh from the sphere mesh into a mesh that matches the sillhouettes of the target images:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "gCfepfOoR1Rl"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"loop = tqdm(range(Niter))\n",
|
||||||
|
"\n",
|
||||||
|
"for i in loop:\n",
|
||||||
|
" # Initialize optimizer\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" # Deform the mesh\n",
|
||||||
|
" new_src_mesh = src_mesh.offset_verts(deform_verts)\n",
|
||||||
|
" \n",
|
||||||
|
" # Losses to smooth /regularize the mesh shape\n",
|
||||||
|
" loss = {k: torch.tensor(0.0, device=device) for k in losses}\n",
|
||||||
|
" update_mesh_shape_prior_losses(new_src_mesh, loss)\n",
|
||||||
|
" \n",
|
||||||
|
" # Compute the average silhouette loss over two random views, as the average \n",
|
||||||
|
" # squared L2 distance between the predicted silhouette and the target \n",
|
||||||
|
" # silhouette from our dataset\n",
|
||||||
|
" for j in np.random.permutation(num_views).tolist()[:num_views_per_iteration]:\n",
|
||||||
|
" images_predicted = renderer_silhouette(new_src_mesh, cameras=target_cameras[j], lights=lights)\n",
|
||||||
|
" predicted_silhouette = images_predicted[..., 3]\n",
|
||||||
|
" loss_silhouette = ((predicted_silhouette - target_silhouette[j]) ** 2).mean()\n",
|
||||||
|
" loss[\"silhouette\"] += loss_silhouette / num_views_per_iteration\n",
|
||||||
|
" \n",
|
||||||
|
" # Weighted sum of the losses\n",
|
||||||
|
" sum_loss = torch.tensor(0.0, device=device)\n",
|
||||||
|
" for k, l in loss.items():\n",
|
||||||
|
" sum_loss += l * losses[k][\"weight\"]\n",
|
||||||
|
" losses[k][\"values\"].append(l)\n",
|
||||||
|
" \n",
|
||||||
|
" # Print the losses\n",
|
||||||
|
" loop.set_description(\"total_loss = %.6f\" % sum_loss)\n",
|
||||||
|
" \n",
|
||||||
|
" # Plot mesh\n",
|
||||||
|
" if i % plot_period == 0:\n",
|
||||||
|
" visualize_prediction(new_src_mesh, title=\"iter: %d\" % i, silhouette=True,\n",
|
||||||
|
" target_image=target_silhouette[1])\n",
|
||||||
|
" \n",
|
||||||
|
" # Optimization step\n",
|
||||||
|
" sum_loss.backward()\n",
|
||||||
|
" optimizer.step()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "CX4huayKR1Rm",
|
||||||
|
"scrolled": true
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"visualize_prediction(new_src_mesh, silhouette=True, \n",
|
||||||
|
" target_image=target_silhouette[1])\n",
|
||||||
|
"plot_losses(losses)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "XJDsJQmrR1Ro"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## 3. Mesh and texture prediction via textured rendering\n",
|
||||||
|
"We can predict both the mesh and its texture if we add an additional loss based on the comparing a predicted rendered RGB image to the target image. As before, we start with a sphere mesh. We learn both translational offsets and RGB texture colors for each vertex in the sphere mesh. Since our loss is based on rendered RGB pixel values instead of just the silhouette, we use a **SoftPhongShader** instead of a **SoftSilhouetteShader**."
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "aZObyIt9R1Ro"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Rasterization settings for differentiable rendering, where the blur_radius\n",
|
||||||
|
"# initialization is based on Liu et al, 'Soft Rasterizer: A Differentiable \n",
|
||||||
|
"# Renderer for Image-based 3D Reasoning', ICCV 2019\n",
|
||||||
|
"sigma = 1e-4\n",
|
||||||
|
"raster_settings_soft = RasterizationSettings(\n",
|
||||||
|
" image_size=128, \n",
|
||||||
|
" blur_radius=np.log(1. / 1e-4 - 1.)*sigma, \n",
|
||||||
|
" faces_per_pixel=50, \n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"# Differentiable soft renderer using per vertex RGB colors for texture\n",
|
||||||
|
"renderer_textured = MeshRenderer(\n",
|
||||||
|
" rasterizer=MeshRasterizer(\n",
|
||||||
|
" cameras=camera, \n",
|
||||||
|
" raster_settings=raster_settings_soft\n",
|
||||||
|
" ),\n",
|
||||||
|
" shader=SoftPhongShader(device=device, \n",
|
||||||
|
" cameras=camera,\n",
|
||||||
|
" lights=lights)\n",
|
||||||
|
")"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "NM7gJux8GMQX"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"We initialize settings, losses, and the optimizer that will be used to iteratively fit our mesh to the target RGB images:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "BS6LAQquF3wq"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Number of views to optimize over in each SGD iteration\n",
|
||||||
|
"num_views_per_iteration = 2\n",
|
||||||
|
"# Number of optimization steps\n",
|
||||||
|
"Niter = 2000\n",
|
||||||
|
"# Plot period for the losses\n",
|
||||||
|
"plot_period = 250\n",
|
||||||
|
"\n",
|
||||||
|
"%matplotlib inline\n",
|
||||||
|
"\n",
|
||||||
|
"# Optimize using rendered RGB image loss, rendered silhouette image loss, mesh \n",
|
||||||
|
"# edge loss, mesh normal consistency, and mesh laplacian smoothing\n",
|
||||||
|
"losses = {\"rgb\": {\"weight\": 1.0, \"values\": []},\n",
|
||||||
|
" \"silhouette\": {\"weight\": 1.0, \"values\": []},\n",
|
||||||
|
" \"edge\": {\"weight\": 1.0, \"values\": []},\n",
|
||||||
|
" \"normal\": {\"weight\": 0.01, \"values\": []},\n",
|
||||||
|
" \"laplacian\": {\"weight\": 1.0, \"values\": []},\n",
|
||||||
|
" }\n",
|
||||||
|
"\n",
|
||||||
|
"# We will learn to deform the source mesh by offsetting its vertices\n",
|
||||||
|
"# The shape of the deform parameters is equal to the total number of vertices in \n",
|
||||||
|
"# src_mesh\n",
|
||||||
|
"verts_shape = src_mesh.verts_packed().shape\n",
|
||||||
|
"deform_verts = torch.full(verts_shape, 0.0, device=device, requires_grad=True)\n",
|
||||||
|
"\n",
|
||||||
|
"# We will also learn per vertex colors for our sphere mesh that define texture \n",
|
||||||
|
"# of the mesh\n",
|
||||||
|
"sphere_verts_rgb = torch.full([1, verts_shape[0], 3], 0.5, device=device, requires_grad=True)\n",
|
||||||
|
"\n",
|
||||||
|
"# The optimizer\n",
|
||||||
|
"optimizer = torch.optim.SGD([deform_verts, sphere_verts_rgb], lr=1.0, momentum=0.9)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "tzIAycuUR1Rq"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"We write an optimization loop to iteratively refine our predicted mesh and its vertex colors from the sphere mesh into a mesh that matches the target images:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "EKEH2p8-R1Rr"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"loop = tqdm(range(Niter))\n",
|
||||||
|
"\n",
|
||||||
|
"for i in loop:\n",
|
||||||
|
" # Initialize optimizer\n",
|
||||||
|
" optimizer.zero_grad()\n",
|
||||||
|
" \n",
|
||||||
|
" # Deform the mesh\n",
|
||||||
|
" new_src_mesh = src_mesh.offset_verts(deform_verts)\n",
|
||||||
|
" \n",
|
||||||
|
" # Add per vertex colors to texture the mesh\n",
|
||||||
|
" new_src_mesh.textures = TexturesVertex(verts_features=sphere_verts_rgb) \n",
|
||||||
|
" \n",
|
||||||
|
" # Losses to smooth /regularize the mesh shape\n",
|
||||||
|
" loss = {k: torch.tensor(0.0, device=device) for k in losses}\n",
|
||||||
|
" update_mesh_shape_prior_losses(new_src_mesh, loss)\n",
|
||||||
|
" \n",
|
||||||
|
" # Randomly select two views to optimize over in this iteration. Compared\n",
|
||||||
|
" # to using just one view, this helps resolve ambiguities between updating\n",
|
||||||
|
" # mesh shape vs. updating mesh texture\n",
|
||||||
|
" for j in np.random.permutation(num_views).tolist()[:num_views_per_iteration]:\n",
|
||||||
|
" images_predicted = renderer_textured(new_src_mesh, cameras=target_cameras[j], lights=lights)\n",
|
||||||
|
"\n",
|
||||||
|
" # Squared L2 distance between the predicted silhouette and the target \n",
|
||||||
|
" # silhouette from our dataset\n",
|
||||||
|
" predicted_silhouette = images_predicted[..., 3]\n",
|
||||||
|
" loss_silhouette = ((predicted_silhouette - target_silhouette[j]) ** 2).mean()\n",
|
||||||
|
" loss[\"silhouette\"] += loss_silhouette / num_views_per_iteration\n",
|
||||||
|
" \n",
|
||||||
|
" # Squared L2 distance between the predicted RGB image and the target \n",
|
||||||
|
" # image from our dataset\n",
|
||||||
|
" predicted_rgb = images_predicted[..., :3]\n",
|
||||||
|
" loss_rgb = ((predicted_rgb - target_rgb[j]) ** 2).mean()\n",
|
||||||
|
" loss[\"rgb\"] += loss_rgb / num_views_per_iteration\n",
|
||||||
|
" \n",
|
||||||
|
" # Weighted sum of the losses\n",
|
||||||
|
" sum_loss = torch.tensor(0.0, device=device)\n",
|
||||||
|
" for k, l in loss.items():\n",
|
||||||
|
" sum_loss += l * losses[k][\"weight\"]\n",
|
||||||
|
" losses[k][\"values\"].append(l)\n",
|
||||||
|
" \n",
|
||||||
|
" # Print the losses\n",
|
||||||
|
" loop.set_description(\"total_loss = %.6f\" % sum_loss)\n",
|
||||||
|
" \n",
|
||||||
|
" # Plot mesh\n",
|
||||||
|
" if i % plot_period == 0:\n",
|
||||||
|
" visualize_prediction(new_src_mesh, renderer=renderer_textured, title=\"iter: %d\" % i, silhouette=False)\n",
|
||||||
|
" \n",
|
||||||
|
" # Optimization step\n",
|
||||||
|
" sum_loss.backward()\n",
|
||||||
|
" optimizer.step()\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "2qTcHO4rR1Rs",
|
||||||
|
"scrolled": true
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"visualize_prediction(new_src_mesh, renderer=renderer_textured, silhouette=False)\n",
|
||||||
|
"plot_losses(losses)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "akBOm_xcNUms"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"Save the final predicted mesh:"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "dXoIsGyhxRyK"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## 4. Save the final predicted mesh"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"colab": {},
|
||||||
|
"colab_type": "code",
|
||||||
|
"id": "OQGhV-psKna8"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# Fetch the verts and faces of the final predicted mesh\n",
|
||||||
|
"final_verts, final_faces = new_src_mesh.get_mesh_verts_faces(0)\n",
|
||||||
|
"\n",
|
||||||
|
"# Scale normalize back to the original target size\n",
|
||||||
|
"final_verts = final_verts * scale + center\n",
|
||||||
|
"\n",
|
||||||
|
"# Store the predicted mesh using save_obj\n",
|
||||||
|
"final_obj = os.path.join('./', 'final_model.obj')\n",
|
||||||
|
"save_obj(final_obj, final_verts, final_faces)"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"colab_type": "text",
|
||||||
|
"id": "MtKYp0B6R1Ru"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## 5. Conclusion\n",
|
||||||
|
"In this tutorial, we learned how to load a textured mesh from an obj file, create a synthetic dataset by rendering the mesh from multiple viewpoints. We showed how to set up an optimization loop to fit a mesh to the observed dataset images based on a rendered silhouette loss. We then augmented this optimization loop with an additional loss based on rendered RGB images, which allowed us to predict both a mesh and its texture."
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"accelerator": "GPU",
|
||||||
|
"anp_metadata": {
|
||||||
|
"path": "fbsource/fbcode/vision/fair/pytorch3d/docs/tutorials/fit_textured_mesh.ipynb"
|
||||||
|
},
|
||||||
|
"bento_stylesheets": {
|
||||||
|
"bento/extensions/flow/main.css": true,
|
||||||
|
"bento/extensions/kernel_selector/main.css": true,
|
||||||
|
"bento/extensions/kernel_ui/main.css": true,
|
||||||
|
"bento/extensions/new_kernel/main.css": true,
|
||||||
|
"bento/extensions/system_usage/main.css": true,
|
||||||
|
"bento/extensions/theme/main.css": true
|
||||||
|
},
|
||||||
|
"colab": {
|
||||||
|
"name": "fit_textured_mesh.ipynb",
|
||||||
|
"provenance": [],
|
||||||
|
"toc_visible": true
|
||||||
|
},
|
||||||
|
"disseminate_notebook_info": {
|
||||||
|
"backup_notebook_id": "781874812352022"
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "intro_to_cv",
|
||||||
|
"language": "python",
|
||||||
|
"name": "bento_kernel_intro_to_cv"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"codemirror_mode": {
|
||||||
|
"name": "ipython",
|
||||||
|
"version": 3
|
||||||
|
},
|
||||||
|
"file_extension": ".py",
|
||||||
|
"mimetype": "text/x-python",
|
||||||
|
"name": "python",
|
||||||
|
"nbconvert_exporter": "python",
|
||||||
|
"pygments_lexer": "ipython3",
|
||||||
|
"version": "3.7.5+"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 1
|
||||||
|
}
|
||||||
580
files/fit_textured_mesh.py
Normal file
580
files/fit_textured_mesh.py
Normal file
@ -0,0 +1,580 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
|
# coding: utf-8
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
|
||||||
|
|
||||||
|
|
||||||
|
# # Fit a mesh via rendering
|
||||||
|
#
|
||||||
|
# This tutorial shows how to:
|
||||||
|
# - Load a mesh and textures from an `.obj` file.
|
||||||
|
# - Create a synthetic dataset by rendering a textured mesh from multiple viewpoints
|
||||||
|
# - Fit a mesh to the observed synthetic images using differential silhouette rendering
|
||||||
|
# - Fit a mesh and its textures using differential textured rendering
|
||||||
|
|
||||||
|
# ## 0. Install and Import modules
|
||||||
|
|
||||||
|
# If `torch`, `torchvision` and `pytorch3d` are not installed, run the following cell:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
get_ipython().system('pip install torch torchvision')
|
||||||
|
import sys
|
||||||
|
import torch
|
||||||
|
if torch.__version__=='1.6.0+cu101' and sys.platform.startswith('linux'):
|
||||||
|
get_ipython().system('pip install pytorch3d')
|
||||||
|
else:
|
||||||
|
get_ipython().system("pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'")
|
||||||
|
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
import os
|
||||||
|
import torch
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
from skimage.io import imread
|
||||||
|
|
||||||
|
from pytorch3d.utils import ico_sphere
|
||||||
|
import numpy as np
|
||||||
|
from tqdm.notebook import tqdm
|
||||||
|
|
||||||
|
# Util function for loading meshes
|
||||||
|
from pytorch3d.io import load_objs_as_meshes, save_obj
|
||||||
|
|
||||||
|
from pytorch3d.loss import (
|
||||||
|
chamfer_distance,
|
||||||
|
mesh_edge_loss,
|
||||||
|
mesh_laplacian_smoothing,
|
||||||
|
mesh_normal_consistency,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Data structures and functions for rendering
|
||||||
|
from pytorch3d.structures import Meshes
|
||||||
|
from pytorch3d.renderer import (
|
||||||
|
look_at_view_transform,
|
||||||
|
OpenGLPerspectiveCameras,
|
||||||
|
PointLights,
|
||||||
|
DirectionalLights,
|
||||||
|
Materials,
|
||||||
|
RasterizationSettings,
|
||||||
|
MeshRenderer,
|
||||||
|
MeshRasterizer,
|
||||||
|
SoftPhongShader,
|
||||||
|
SoftSilhouetteShader,
|
||||||
|
SoftPhongShader,
|
||||||
|
TexturesVertex
|
||||||
|
)
|
||||||
|
|
||||||
|
# add path for demo utils functions
|
||||||
|
import sys
|
||||||
|
import os
|
||||||
|
sys.path.append(os.path.abspath(''))
|
||||||
|
|
||||||
|
|
||||||
|
# If using **Google Colab**, fetch the utils file for plotting image grids:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
get_ipython().system('wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py')
|
||||||
|
from plot_image_grid import image_grid
|
||||||
|
|
||||||
|
|
||||||
|
# OR if running **locally** uncomment and run the following cell:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# from utils.plot_image_grid import image_grid
|
||||||
|
|
||||||
|
|
||||||
|
# ### 1. Load a mesh and texture file
|
||||||
|
#
|
||||||
|
# Load an `.obj` file and it's associated `.mtl` file and create a **Textures** and **Meshes** object.
|
||||||
|
#
|
||||||
|
# **Meshes** is a unique datastructure provided in PyTorch3D for working with batches of meshes of different sizes.
|
||||||
|
#
|
||||||
|
# **TexturesVertex** is an auxillary datastructure for storing vertex rgb texture information about meshes.
|
||||||
|
#
|
||||||
|
# **Meshes** has several class methods which are used throughout the rendering pipeline.
|
||||||
|
|
||||||
|
# If running this notebook using **Google Colab**, run the following cell to fetch the mesh obj and texture files and save it at the path `data/cow_mesh`:
|
||||||
|
# If running locally, the data is already available at the correct path.
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
get_ipython().system('mkdir -p data/cow_mesh')
|
||||||
|
get_ipython().system('wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.obj')
|
||||||
|
get_ipython().system('wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.mtl')
|
||||||
|
get_ipython().system('wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow_texture.png')
|
||||||
|
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# Setup
|
||||||
|
if torch.cuda.is_available():
|
||||||
|
device = torch.device("cuda:0")
|
||||||
|
torch.cuda.set_device(device)
|
||||||
|
else:
|
||||||
|
device = torch.device("cpu")
|
||||||
|
|
||||||
|
# Set paths
|
||||||
|
DATA_DIR = "./data"
|
||||||
|
obj_filename = os.path.join(DATA_DIR, "cow_mesh/cow.obj")
|
||||||
|
|
||||||
|
# Load obj file
|
||||||
|
mesh = load_objs_as_meshes([obj_filename], device=device)
|
||||||
|
|
||||||
|
# We scale normalize and center the target mesh to fit in a sphere of radius 1
|
||||||
|
# centered at (0,0,0). (scale, center) will be used to bring the predicted mesh
|
||||||
|
# to its original center and scale. Note that normalizing the target mesh,
|
||||||
|
# speeds up the optimization but is not necessary!
|
||||||
|
verts = mesh.verts_packed()
|
||||||
|
N = verts.shape[0]
|
||||||
|
center = verts.mean(0)
|
||||||
|
scale = max((verts - center).abs().max(0)[0])
|
||||||
|
mesh.offset_verts_(-center.expand(N, 3))
|
||||||
|
mesh.scale_verts_((1.0 / float(scale)));
|
||||||
|
|
||||||
|
|
||||||
|
# ## 2. Dataset Creation
|
||||||
|
#
|
||||||
|
# We sample different camera positions that encode multiple viewpoints of the cow. We create a renderer with a shader that performs texture map interpolation. We render a synthetic dataset of images of the textured cow mesh from multiple viewpoints.
|
||||||
|
#
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# the number of different viewpoints from which we want to render the mesh.
|
||||||
|
num_views = 20
|
||||||
|
|
||||||
|
# Get a batch of viewing angles.
|
||||||
|
elev = torch.linspace(0, 360, num_views)
|
||||||
|
azim = torch.linspace(-180, 180, num_views)
|
||||||
|
|
||||||
|
# Place a point light in front of the object. As mentioned above, the front of
|
||||||
|
# the cow is facing the -z direction.
|
||||||
|
lights = PointLights(device=device, location=[[0.0, 0.0, -3.0]])
|
||||||
|
|
||||||
|
# Initialize an OpenGL perspective camera that represents a batch of different
|
||||||
|
# viewing angles. All the cameras helper methods support mixed type inputs and
|
||||||
|
# broadcasting. So we can view the camera from the a distance of dist=2.7, and
|
||||||
|
# then specify elevation and azimuth angles for each viewpoint as tensors.
|
||||||
|
R, T = look_at_view_transform(dist=2.7, elev=elev, azim=azim)
|
||||||
|
cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T)
|
||||||
|
|
||||||
|
# We arbitrarily choose one particular view that will be used to visualize
|
||||||
|
# results
|
||||||
|
camera = OpenGLPerspectiveCameras(device=device, R=R[None, 1, ...],
|
||||||
|
T=T[None, 1, ...])
|
||||||
|
|
||||||
|
# Define the settings for rasterization and shading. Here we set the output
|
||||||
|
# image to be of size 128X128. As we are rendering images for visualization
|
||||||
|
# purposes only we will set faces_per_pixel=1 and blur_radius=0.0. Refer to
|
||||||
|
# rasterize_meshes.py for explanations of these parameters. We also leave
|
||||||
|
# bin_size and max_faces_per_bin to their default values of None, which sets
|
||||||
|
# their values using huristics and ensures that the faster coarse-to-fine
|
||||||
|
# rasterization method is used. Refer to docs/notes/renderer.md for an
|
||||||
|
# explanation of the difference between naive and coarse-to-fine rasterization.
|
||||||
|
raster_settings = RasterizationSettings(
|
||||||
|
image_size=128,
|
||||||
|
blur_radius=0.0,
|
||||||
|
faces_per_pixel=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create a phong renderer by composing a rasterizer and a shader. The textured
|
||||||
|
# phong shader will interpolate the texture uv coordinates for each vertex,
|
||||||
|
# sample from a texture image and apply the Phong lighting model
|
||||||
|
renderer = MeshRenderer(
|
||||||
|
rasterizer=MeshRasterizer(
|
||||||
|
cameras=camera,
|
||||||
|
raster_settings=raster_settings
|
||||||
|
),
|
||||||
|
shader=SoftPhongShader(
|
||||||
|
device=device,
|
||||||
|
cameras=camera,
|
||||||
|
lights=lights
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Create a batch of meshes by repeating the cow mesh and associated textures.
|
||||||
|
# Meshes has a useful `extend` method which allows us do this very easily.
|
||||||
|
# This also extends the textures.
|
||||||
|
meshes = mesh.extend(num_views)
|
||||||
|
|
||||||
|
# Render the cow mesh from each viewing angle
|
||||||
|
target_images = renderer(meshes, cameras=cameras, lights=lights)
|
||||||
|
|
||||||
|
# Our multi-view cow dataset will be represented by these 2 lists of tensors,
|
||||||
|
# each of length num_views.
|
||||||
|
target_rgb = [target_images[i, ..., :3] for i in range(num_views)]
|
||||||
|
target_cameras = [OpenGLPerspectiveCameras(device=device, R=R[None, i, ...],
|
||||||
|
T=T[None, i, ...]) for i in range(num_views)]
|
||||||
|
|
||||||
|
|
||||||
|
# Visualize the dataset:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# RGB images
|
||||||
|
image_grid(target_images.cpu().numpy(), rows=4, cols=5, rgb=True)
|
||||||
|
plt.show()
|
||||||
|
|
||||||
|
|
||||||
|
# Later in this tutorial, we will fit a mesh to the rendered RGB images, as well as to just images of just the cow silhouette. For the latter case, we will render a dataset of silhouette images. Most shaders in PyTorch3D will output an alpha channel along with the RGB image as a 4th channel in an RGBA image. The alpha channel encodes the probability that each pixel belongs to the foreground of the object. We contruct a soft silhouette shader to render this alpha channel.
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# Rasterization settings for silhouette rendering
|
||||||
|
sigma = 1e-4
|
||||||
|
raster_settings_silhouette = RasterizationSettings(
|
||||||
|
image_size=128,
|
||||||
|
blur_radius=np.log(1. / 1e-4 - 1.)*sigma,
|
||||||
|
faces_per_pixel=50,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Silhouette renderer
|
||||||
|
renderer_silhouette = MeshRenderer(
|
||||||
|
rasterizer=MeshRasterizer(
|
||||||
|
cameras=camera,
|
||||||
|
raster_settings=raster_settings_silhouette
|
||||||
|
),
|
||||||
|
shader=SoftSilhouetteShader()
|
||||||
|
)
|
||||||
|
|
||||||
|
# Render silhouette images. The 3rd channel of the rendering output is
|
||||||
|
# the alpha/silhouette channel
|
||||||
|
silhouette_images = renderer_silhouette(meshes, cameras=cameras, lights=lights)
|
||||||
|
target_silhouette = [silhouette_images[i, ..., 3] for i in range(num_views)]
|
||||||
|
|
||||||
|
# Visualize silhouette images
|
||||||
|
image_grid(silhouette_images.cpu().numpy(), rows=4, cols=5, rgb=False)
|
||||||
|
plt.show()
|
||||||
|
|
||||||
|
|
||||||
|
# ## 3. Mesh prediction via silhouette rendering
|
||||||
|
# In the previous section, we created a dataset of images of multiple viewpoints of a cow. In this section, we predict a mesh by observing those target images without any knowledge of the ground truth cow mesh. We assume we know the position of the cameras and lighting.
|
||||||
|
#
|
||||||
|
# We first define some helper functions to visualize the results of our mesh prediction:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# Show a visualization comparing the rendered predicted mesh to the ground truth
|
||||||
|
# mesh
|
||||||
|
def visualize_prediction(predicted_mesh, renderer=renderer_silhouette,
|
||||||
|
target_image=target_rgb[1], title='',
|
||||||
|
silhouette=False):
|
||||||
|
inds = 3 if silhouette else range(3)
|
||||||
|
predicted_images = renderer(predicted_mesh)
|
||||||
|
plt.figure(figsize=(20, 10))
|
||||||
|
plt.subplot(1, 2, 1)
|
||||||
|
plt.imshow(predicted_images[0, ..., inds].cpu().detach().numpy())
|
||||||
|
|
||||||
|
plt.subplot(1, 2, 2)
|
||||||
|
plt.imshow(target_image.cpu().detach().numpy())
|
||||||
|
plt.title(title)
|
||||||
|
plt.grid("off")
|
||||||
|
plt.axis("off")
|
||||||
|
|
||||||
|
# Plot losses as a function of optimization iteration
|
||||||
|
def plot_losses(losses):
|
||||||
|
fig = plt.figure(figsize=(13, 5))
|
||||||
|
ax = fig.gca()
|
||||||
|
for k, l in losses.items():
|
||||||
|
ax.plot(l['values'], label=k + " loss")
|
||||||
|
ax.legend(fontsize="16")
|
||||||
|
ax.set_xlabel("Iteration", fontsize="16")
|
||||||
|
ax.set_ylabel("Loss", fontsize="16")
|
||||||
|
ax.set_title("Loss vs iterations", fontsize="16")
|
||||||
|
|
||||||
|
|
||||||
|
# Starting from a sphere mesh, we will learn offsets of each vertex such that the predicted mesh silhouette is more similar to the target silhouette image at each optimization step. We begin by loading our initial sphere mesh:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# We initialize the source shape to be a sphere of radius 1.
|
||||||
|
src_mesh = ico_sphere(4, device)
|
||||||
|
|
||||||
|
|
||||||
|
# We create a new differentiable renderer for rendering the silhouette of our predicted mesh:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# Rasterization settings for differentiable rendering, where the blur_radius
|
||||||
|
# initialization is based on Liu et al, 'Soft Rasterizer: A Differentiable
|
||||||
|
# Renderer for Image-based 3D Reasoning', ICCV 2019
|
||||||
|
sigma = 1e-4
|
||||||
|
raster_settings_soft = RasterizationSettings(
|
||||||
|
image_size=128,
|
||||||
|
blur_radius=np.log(1. / 1e-4 - 1.)*sigma,
|
||||||
|
faces_per_pixel=50,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Silhouette renderer
|
||||||
|
renderer_silhouette = MeshRenderer(
|
||||||
|
rasterizer=MeshRasterizer(
|
||||||
|
cameras=camera,
|
||||||
|
raster_settings=raster_settings_soft
|
||||||
|
),
|
||||||
|
shader=SoftSilhouetteShader()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# We initialize settings, losses, and the optimizer that will be used to iteratively fit our mesh to the target silhouettes:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# Number of views to optimize over in each SGD iteration
|
||||||
|
num_views_per_iteration = 2
|
||||||
|
# Number of optimization steps
|
||||||
|
Niter = 2000
|
||||||
|
# Plot period for the losses
|
||||||
|
plot_period = 250
|
||||||
|
|
||||||
|
get_ipython().run_line_magic('matplotlib', 'inline')
|
||||||
|
|
||||||
|
# Optimize using rendered silhouette image loss, mesh edge loss, mesh normal
|
||||||
|
# consistency, and mesh laplacian smoothing
|
||||||
|
losses = {"silhouette": {"weight": 1.0, "values": []},
|
||||||
|
"edge": {"weight": 1.0, "values": []},
|
||||||
|
"normal": {"weight": 0.01, "values": []},
|
||||||
|
"laplacian": {"weight": 1.0, "values": []},
|
||||||
|
}
|
||||||
|
|
||||||
|
# Losses to smooth / regularize the mesh shape
|
||||||
|
def update_mesh_shape_prior_losses(mesh, loss):
|
||||||
|
# and (b) the edge length of the predicted mesh
|
||||||
|
loss["edge"] = mesh_edge_loss(mesh)
|
||||||
|
|
||||||
|
# mesh normal consistency
|
||||||
|
loss["normal"] = mesh_normal_consistency(mesh)
|
||||||
|
|
||||||
|
# mesh laplacian smoothing
|
||||||
|
loss["laplacian"] = mesh_laplacian_smoothing(mesh, method="uniform")
|
||||||
|
|
||||||
|
# We will learn to deform the source mesh by offsetting its vertices
|
||||||
|
# The shape of the deform parameters is equal to the total number of vertices in
|
||||||
|
# src_mesh
|
||||||
|
verts_shape = src_mesh.verts_packed().shape
|
||||||
|
deform_verts = torch.full(verts_shape, 0.0, device=device, requires_grad=True)
|
||||||
|
|
||||||
|
# The optimizer
|
||||||
|
optimizer = torch.optim.SGD([deform_verts], lr=1.0, momentum=0.9)
|
||||||
|
|
||||||
|
|
||||||
|
# We write an optimization loop to iteratively refine our predicted mesh from the sphere mesh into a mesh that matches the sillhouettes of the target images:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
loop = tqdm(range(Niter))
|
||||||
|
|
||||||
|
for i in loop:
|
||||||
|
# Initialize optimizer
|
||||||
|
optimizer.zero_grad()
|
||||||
|
|
||||||
|
# Deform the mesh
|
||||||
|
new_src_mesh = src_mesh.offset_verts(deform_verts)
|
||||||
|
|
||||||
|
# Losses to smooth /regularize the mesh shape
|
||||||
|
loss = {k: torch.tensor(0.0, device=device) for k in losses}
|
||||||
|
update_mesh_shape_prior_losses(new_src_mesh, loss)
|
||||||
|
|
||||||
|
# Compute the average silhouette loss over two random views, as the average
|
||||||
|
# squared L2 distance between the predicted silhouette and the target
|
||||||
|
# silhouette from our dataset
|
||||||
|
for j in np.random.permutation(num_views).tolist()[:num_views_per_iteration]:
|
||||||
|
images_predicted = renderer_silhouette(new_src_mesh, cameras=target_cameras[j], lights=lights)
|
||||||
|
predicted_silhouette = images_predicted[..., 3]
|
||||||
|
loss_silhouette = ((predicted_silhouette - target_silhouette[j]) ** 2).mean()
|
||||||
|
loss["silhouette"] += loss_silhouette / num_views_per_iteration
|
||||||
|
|
||||||
|
# Weighted sum of the losses
|
||||||
|
sum_loss = torch.tensor(0.0, device=device)
|
||||||
|
for k, l in loss.items():
|
||||||
|
sum_loss += l * losses[k]["weight"]
|
||||||
|
losses[k]["values"].append(l)
|
||||||
|
|
||||||
|
# Print the losses
|
||||||
|
loop.set_description("total_loss = %.6f" % sum_loss)
|
||||||
|
|
||||||
|
# Plot mesh
|
||||||
|
if i % plot_period == 0:
|
||||||
|
visualize_prediction(new_src_mesh, title="iter: %d" % i, silhouette=True,
|
||||||
|
target_image=target_silhouette[1])
|
||||||
|
|
||||||
|
# Optimization step
|
||||||
|
sum_loss.backward()
|
||||||
|
optimizer.step()
|
||||||
|
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
visualize_prediction(new_src_mesh, silhouette=True,
|
||||||
|
target_image=target_silhouette[1])
|
||||||
|
plot_losses(losses)
|
||||||
|
|
||||||
|
|
||||||
|
# ## 3. Mesh and texture prediction via textured rendering
|
||||||
|
# We can predict both the mesh and its texture if we add an additional loss based on the comparing a predicted rendered RGB image to the target image. As before, we start with a sphere mesh. We learn both translational offsets and RGB texture colors for each vertex in the sphere mesh. Since our loss is based on rendered RGB pixel values instead of just the silhouette, we use a **SoftPhongShader** instead of a **SoftSilhouetteShader**.
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# Rasterization settings for differentiable rendering, where the blur_radius
|
||||||
|
# initialization is based on Liu et al, 'Soft Rasterizer: A Differentiable
|
||||||
|
# Renderer for Image-based 3D Reasoning', ICCV 2019
|
||||||
|
sigma = 1e-4
|
||||||
|
raster_settings_soft = RasterizationSettings(
|
||||||
|
image_size=128,
|
||||||
|
blur_radius=np.log(1. / 1e-4 - 1.)*sigma,
|
||||||
|
faces_per_pixel=50,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Differentiable soft renderer using per vertex RGB colors for texture
|
||||||
|
renderer_textured = MeshRenderer(
|
||||||
|
rasterizer=MeshRasterizer(
|
||||||
|
cameras=camera,
|
||||||
|
raster_settings=raster_settings_soft
|
||||||
|
),
|
||||||
|
shader=SoftPhongShader(device=device,
|
||||||
|
cameras=camera,
|
||||||
|
lights=lights)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# We initialize settings, losses, and the optimizer that will be used to iteratively fit our mesh to the target RGB images:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# Number of views to optimize over in each SGD iteration
|
||||||
|
num_views_per_iteration = 2
|
||||||
|
# Number of optimization steps
|
||||||
|
Niter = 2000
|
||||||
|
# Plot period for the losses
|
||||||
|
plot_period = 250
|
||||||
|
|
||||||
|
get_ipython().run_line_magic('matplotlib', 'inline')
|
||||||
|
|
||||||
|
# Optimize using rendered RGB image loss, rendered silhouette image loss, mesh
|
||||||
|
# edge loss, mesh normal consistency, and mesh laplacian smoothing
|
||||||
|
losses = {"rgb": {"weight": 1.0, "values": []},
|
||||||
|
"silhouette": {"weight": 1.0, "values": []},
|
||||||
|
"edge": {"weight": 1.0, "values": []},
|
||||||
|
"normal": {"weight": 0.01, "values": []},
|
||||||
|
"laplacian": {"weight": 1.0, "values": []},
|
||||||
|
}
|
||||||
|
|
||||||
|
# We will learn to deform the source mesh by offsetting its vertices
|
||||||
|
# The shape of the deform parameters is equal to the total number of vertices in
|
||||||
|
# src_mesh
|
||||||
|
verts_shape = src_mesh.verts_packed().shape
|
||||||
|
deform_verts = torch.full(verts_shape, 0.0, device=device, requires_grad=True)
|
||||||
|
|
||||||
|
# We will also learn per vertex colors for our sphere mesh that define texture
|
||||||
|
# of the mesh
|
||||||
|
sphere_verts_rgb = torch.full([1, verts_shape[0], 3], 0.5, device=device, requires_grad=True)
|
||||||
|
|
||||||
|
# The optimizer
|
||||||
|
optimizer = torch.optim.SGD([deform_verts, sphere_verts_rgb], lr=1.0, momentum=0.9)
|
||||||
|
|
||||||
|
|
||||||
|
# We write an optimization loop to iteratively refine our predicted mesh and its vertex colors from the sphere mesh into a mesh that matches the target images:
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
loop = tqdm(range(Niter))
|
||||||
|
|
||||||
|
for i in loop:
|
||||||
|
# Initialize optimizer
|
||||||
|
optimizer.zero_grad()
|
||||||
|
|
||||||
|
# Deform the mesh
|
||||||
|
new_src_mesh = src_mesh.offset_verts(deform_verts)
|
||||||
|
|
||||||
|
# Add per vertex colors to texture the mesh
|
||||||
|
new_src_mesh.textures = TexturesVertex(verts_features=sphere_verts_rgb)
|
||||||
|
|
||||||
|
# Losses to smooth /regularize the mesh shape
|
||||||
|
loss = {k: torch.tensor(0.0, device=device) for k in losses}
|
||||||
|
update_mesh_shape_prior_losses(new_src_mesh, loss)
|
||||||
|
|
||||||
|
# Randomly select two views to optimize over in this iteration. Compared
|
||||||
|
# to using just one view, this helps resolve ambiguities between updating
|
||||||
|
# mesh shape vs. updating mesh texture
|
||||||
|
for j in np.random.permutation(num_views).tolist()[:num_views_per_iteration]:
|
||||||
|
images_predicted = renderer_textured(new_src_mesh, cameras=target_cameras[j], lights=lights)
|
||||||
|
|
||||||
|
# Squared L2 distance between the predicted silhouette and the target
|
||||||
|
# silhouette from our dataset
|
||||||
|
predicted_silhouette = images_predicted[..., 3]
|
||||||
|
loss_silhouette = ((predicted_silhouette - target_silhouette[j]) ** 2).mean()
|
||||||
|
loss["silhouette"] += loss_silhouette / num_views_per_iteration
|
||||||
|
|
||||||
|
# Squared L2 distance between the predicted RGB image and the target
|
||||||
|
# image from our dataset
|
||||||
|
predicted_rgb = images_predicted[..., :3]
|
||||||
|
loss_rgb = ((predicted_rgb - target_rgb[j]) ** 2).mean()
|
||||||
|
loss["rgb"] += loss_rgb / num_views_per_iteration
|
||||||
|
|
||||||
|
# Weighted sum of the losses
|
||||||
|
sum_loss = torch.tensor(0.0, device=device)
|
||||||
|
for k, l in loss.items():
|
||||||
|
sum_loss += l * losses[k]["weight"]
|
||||||
|
losses[k]["values"].append(l)
|
||||||
|
|
||||||
|
# Print the losses
|
||||||
|
loop.set_description("total_loss = %.6f" % sum_loss)
|
||||||
|
|
||||||
|
# Plot mesh
|
||||||
|
if i % plot_period == 0:
|
||||||
|
visualize_prediction(new_src_mesh, renderer=renderer_textured, title="iter: %d" % i, silhouette=False)
|
||||||
|
|
||||||
|
# Optimization step
|
||||||
|
sum_loss.backward()
|
||||||
|
optimizer.step()
|
||||||
|
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
visualize_prediction(new_src_mesh, renderer=renderer_textured, silhouette=False)
|
||||||
|
plot_losses(losses)
|
||||||
|
|
||||||
|
|
||||||
|
# Save the final predicted mesh:
|
||||||
|
|
||||||
|
# ## 4. Save the final predicted mesh
|
||||||
|
|
||||||
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
|
# Fetch the verts and faces of the final predicted mesh
|
||||||
|
final_verts, final_faces = new_src_mesh.get_mesh_verts_faces(0)
|
||||||
|
|
||||||
|
# Scale normalize back to the original target size
|
||||||
|
final_verts = final_verts * scale + center
|
||||||
|
|
||||||
|
# Store the predicted mesh using save_obj
|
||||||
|
final_obj = os.path.join('./', 'final_model.obj')
|
||||||
|
save_obj(final_obj, final_verts, final_faces)
|
||||||
|
|
||||||
|
|
||||||
|
# ## 5. Conclusion
|
||||||
|
# In this tutorial, we learned how to load a textured mesh from an obj file, create a synthetic dataset by rendering the mesh from multiple viewpoints. We showed how to set up an optimization loop to fit a mesh to the observed dataset images based on a rendered silhouette loss. We then augmented this optimization loop with an additional loss based on rendered RGB images, which allowed us to predict both a mesh and its texture.
|
||||||
File diff suppressed because one or more lines are too long
@ -1,4 +1,4 @@
|
|||||||
|
#!/usr/bin/env python
|
||||||
# coding: utf-8
|
# coding: utf-8
|
||||||
|
|
||||||
# In[ ]:
|
# In[ ]:
|
||||||
@ -20,14 +20,19 @@
|
|||||||
|
|
||||||
# If `torch`, `torchvision` and `pytorch3d` are not installed, run the following cell:
|
# If `torch`, `torchvision` and `pytorch3d` are not installed, run the following cell:
|
||||||
|
|
||||||
# In[1]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
get_ipython().system('pip install torch torchvision')
|
get_ipython().system('pip install torch torchvision')
|
||||||
get_ipython().system("pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'")
|
import sys
|
||||||
|
import torch
|
||||||
|
if torch.__version__=='1.6.0+cu101' and sys.platform.startswith('linux'):
|
||||||
|
get_ipython().system('pip install pytorch3d')
|
||||||
|
else:
|
||||||
|
get_ipython().system("pip install 'git+https://github.com/facebookresearch/pytorch3d.git@stable'")
|
||||||
|
|
||||||
|
|
||||||
# In[1]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
import os
|
import os
|
||||||
@ -36,20 +41,21 @@ import matplotlib.pyplot as plt
|
|||||||
from skimage.io import imread
|
from skimage.io import imread
|
||||||
|
|
||||||
# Util function for loading meshes
|
# Util function for loading meshes
|
||||||
from pytorch3d.io import load_objs_as_meshes
|
from pytorch3d.io import load_objs_as_meshes, load_obj
|
||||||
|
|
||||||
# Data structures and functions for rendering
|
# Data structures and functions for rendering
|
||||||
from pytorch3d.structures import Meshes, Textures
|
from pytorch3d.structures import Meshes
|
||||||
from pytorch3d.renderer import (
|
from pytorch3d.renderer import (
|
||||||
look_at_view_transform,
|
look_at_view_transform,
|
||||||
OpenGLPerspectiveCameras,
|
FoVPerspectiveCameras,
|
||||||
PointLights,
|
PointLights,
|
||||||
DirectionalLights,
|
DirectionalLights,
|
||||||
Materials,
|
Materials,
|
||||||
RasterizationSettings,
|
RasterizationSettings,
|
||||||
MeshRenderer,
|
MeshRenderer,
|
||||||
MeshRasterizer,
|
MeshRasterizer,
|
||||||
TexturedSoftPhongShader
|
SoftPhongShader,
|
||||||
|
TexturesUV
|
||||||
)
|
)
|
||||||
|
|
||||||
# add path for demo utils functions
|
# add path for demo utils functions
|
||||||
@ -60,7 +66,7 @@ sys.path.append(os.path.abspath(''))
|
|||||||
|
|
||||||
# If using **Google Colab**, fetch the utils file for plotting image grids:
|
# If using **Google Colab**, fetch the utils file for plotting image grids:
|
||||||
|
|
||||||
# In[2]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
get_ipython().system('wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py')
|
get_ipython().system('wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py')
|
||||||
@ -69,7 +75,7 @@ from plot_image_grid import image_grid
|
|||||||
|
|
||||||
# OR if running **locally** uncomment and run the following cell:
|
# OR if running **locally** uncomment and run the following cell:
|
||||||
|
|
||||||
# In[13]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# from utils import image_grid
|
# from utils import image_grid
|
||||||
@ -81,14 +87,14 @@ from plot_image_grid import image_grid
|
|||||||
#
|
#
|
||||||
# **Meshes** is a unique datastructure provided in PyTorch3D for working with batches of meshes of different sizes.
|
# **Meshes** is a unique datastructure provided in PyTorch3D for working with batches of meshes of different sizes.
|
||||||
#
|
#
|
||||||
# **Textures** is an auxillary datastructure for storing texture information about meshes.
|
# **TexturesUV** is an auxillary datastructure for storing vertex uv and texture maps for meshes.
|
||||||
#
|
#
|
||||||
# **Meshes** has several class methods which are used throughout the rendering pipeline.
|
# **Meshes** has several class methods which are used throughout the rendering pipeline.
|
||||||
|
|
||||||
# If running this notebook using **Google Colab**, run the following cell to fetch the mesh obj and texture files and save it at the path `data/cow_mesh`:
|
# If running this notebook using **Google Colab**, run the following cell to fetch the mesh obj and texture files and save it at the path `data/cow_mesh`:
|
||||||
# If running locally, the data is already available at the correct path.
|
# If running locally, the data is already available at the correct path.
|
||||||
|
|
||||||
# In[3]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
get_ipython().system('mkdir -p data/cow_mesh')
|
get_ipython().system('mkdir -p data/cow_mesh')
|
||||||
@ -97,12 +103,15 @@ get_ipython().system('wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytor
|
|||||||
get_ipython().system('wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow_texture.png')
|
get_ipython().system('wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow_texture.png')
|
||||||
|
|
||||||
|
|
||||||
# In[2]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# Setup
|
# Setup
|
||||||
device = torch.device("cuda:0")
|
if torch.cuda.is_available():
|
||||||
torch.cuda.set_device(device)
|
device = torch.device("cuda:0")
|
||||||
|
torch.cuda.set_device(device)
|
||||||
|
else:
|
||||||
|
device = torch.device("cpu")
|
||||||
|
|
||||||
# Set paths
|
# Set paths
|
||||||
DATA_DIR = "./data"
|
DATA_DIR = "./data"
|
||||||
@ -115,7 +124,7 @@ texture_image=mesh.textures.maps_padded()
|
|||||||
|
|
||||||
# #### Let's visualize the texture map
|
# #### Let's visualize the texture map
|
||||||
|
|
||||||
# In[3]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
plt.figure(figsize=(7,7))
|
plt.figure(figsize=(7,7))
|
||||||
@ -130,14 +139,14 @@ plt.axis('off');
|
|||||||
#
|
#
|
||||||
# In this example we will first create a **renderer** which uses a **perspective camera**, a **point light** and applies **phong shading**. Then we learn how to vary different components using the modular API.
|
# In this example we will first create a **renderer** which uses a **perspective camera**, a **point light** and applies **phong shading**. Then we learn how to vary different components using the modular API.
|
||||||
|
|
||||||
# In[4]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# Initialize an OpenGL perspective camera.
|
# Initialize a camera.
|
||||||
# With world coordinates +Y up, +X left and +Z in, the front of the cow is facing the -Z direction.
|
# With world coordinates +Y up, +X left and +Z in, the front of the cow is facing the -Z direction.
|
||||||
# So we move the camera by 180 in the azimuth direction so it is facing the front of the cow.
|
# So we move the camera by 180 in the azimuth direction so it is facing the front of the cow.
|
||||||
R, T = look_at_view_transform(2.7, 0, 180)
|
R, T = look_at_view_transform(2.7, 0, 180)
|
||||||
cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T)
|
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
|
||||||
|
|
||||||
# Define the settings for rasterization and shading. Here we set the output image to be of size
|
# Define the settings for rasterization and shading. Here we set the output image to be of size
|
||||||
# 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1
|
# 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1
|
||||||
@ -149,8 +158,6 @@ raster_settings = RasterizationSettings(
|
|||||||
image_size=512,
|
image_size=512,
|
||||||
blur_radius=0.0,
|
blur_radius=0.0,
|
||||||
faces_per_pixel=1,
|
faces_per_pixel=1,
|
||||||
bin_size = None, # this setting controls whether naive or coarse-to-fine rasterization is used
|
|
||||||
max_faces_per_bin = None # this setting is for coarse rasterization
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# Place a point light in front of the object. As mentioned above, the front of the cow is facing the
|
# Place a point light in front of the object. As mentioned above, the front of the cow is facing the
|
||||||
@ -165,7 +172,7 @@ renderer = MeshRenderer(
|
|||||||
cameras=cameras,
|
cameras=cameras,
|
||||||
raster_settings=raster_settings
|
raster_settings=raster_settings
|
||||||
),
|
),
|
||||||
shader=TexturedSoftPhongShader(
|
shader=SoftPhongShader(
|
||||||
device=device,
|
device=device,
|
||||||
cameras=cameras,
|
cameras=cameras,
|
||||||
lights=lights
|
lights=lights
|
||||||
@ -177,7 +184,7 @@ renderer = MeshRenderer(
|
|||||||
|
|
||||||
# The light is in front of the object so it is bright and the image has specular highlights.
|
# The light is in front of the object so it is bright and the image has specular highlights.
|
||||||
|
|
||||||
# In[5]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
images = renderer(mesh)
|
images = renderer(mesh)
|
||||||
@ -195,7 +202,7 @@ plt.axis("off");
|
|||||||
#
|
#
|
||||||
# The image is now dark as there is only ambient lighting, and there are no specular highlights.
|
# The image is now dark as there is only ambient lighting, and there are no specular highlights.
|
||||||
|
|
||||||
# In[6]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# Now move the light so it is on the +Z axis which will be behind the cow.
|
# Now move the light so it is on the +Z axis which will be behind the cow.
|
||||||
@ -203,7 +210,7 @@ lights.location = torch.tensor([0.0, 0.0, +1.0], device=device)[None]
|
|||||||
images = renderer(mesh, lights=lights)
|
images = renderer(mesh, lights=lights)
|
||||||
|
|
||||||
|
|
||||||
# In[7]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
plt.figure(figsize=(10, 10))
|
plt.figure(figsize=(10, 10))
|
||||||
@ -220,12 +227,12 @@ plt.axis("off");
|
|||||||
# - change the **position** of the point light
|
# - change the **position** of the point light
|
||||||
# - change the **material reflectance** properties of the mesh
|
# - change the **material reflectance** properties of the mesh
|
||||||
|
|
||||||
# In[8]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# Rotate the object by increasing the elevation and azimuth angles
|
# Rotate the object by increasing the elevation and azimuth angles
|
||||||
R, T = look_at_view_transform(dist=2.7, elev=10, azim=-150)
|
R, T = look_at_view_transform(dist=2.7, elev=10, azim=-150)
|
||||||
cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T)
|
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
|
||||||
|
|
||||||
# Move the light location so the light is shining on the cow's face.
|
# Move the light location so the light is shining on the cow's face.
|
||||||
lights.location = torch.tensor([[2.0, 2.0, -2.0]], device=device)
|
lights.location = torch.tensor([[2.0, 2.0, -2.0]], device=device)
|
||||||
@ -241,7 +248,7 @@ materials = Materials(
|
|||||||
images = renderer(mesh, lights=lights, materials=materials, cameras=cameras)
|
images = renderer(mesh, lights=lights, materials=materials, cameras=cameras)
|
||||||
|
|
||||||
|
|
||||||
# In[9]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
plt.figure(figsize=(10, 10))
|
plt.figure(figsize=(10, 10))
|
||||||
@ -256,7 +263,7 @@ plt.axis("off");
|
|||||||
# The renderer and associated components can take batched inputs and **render a batch of output images in one forward pass**. We will now use this feature to render the mesh from many different viewpoints.
|
# The renderer and associated components can take batched inputs and **render a batch of output images in one forward pass**. We will now use this feature to render the mesh from many different viewpoints.
|
||||||
#
|
#
|
||||||
|
|
||||||
# In[10]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# Set batch size - this is the number of different viewpoints from which we want to render the mesh.
|
# Set batch size - this is the number of different viewpoints from which we want to render the mesh.
|
||||||
@ -275,13 +282,13 @@ azim = torch.linspace(-180, 180, batch_size)
|
|||||||
# view the camera from the same distance and specify dist=2.7 as a float,
|
# view the camera from the same distance and specify dist=2.7 as a float,
|
||||||
# and then specify elevation and azimuth angles for each viewpoint as tensors.
|
# and then specify elevation and azimuth angles for each viewpoint as tensors.
|
||||||
R, T = look_at_view_transform(dist=2.7, elev=elev, azim=azim)
|
R, T = look_at_view_transform(dist=2.7, elev=elev, azim=azim)
|
||||||
cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T)
|
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
|
||||||
|
|
||||||
# Move the light back in front of the cow which is facing the -z direction.
|
# Move the light back in front of the cow which is facing the -z direction.
|
||||||
lights.location = torch.tensor([[0.0, 0.0, -3.0]], device=device)
|
lights.location = torch.tensor([[0.0, 0.0, -3.0]], device=device)
|
||||||
|
|
||||||
|
|
||||||
# In[11]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
# We can pass arbirary keyword arguments to the rasterizer/shader via the renderer
|
# We can pass arbirary keyword arguments to the rasterizer/shader via the renderer
|
||||||
@ -289,7 +296,7 @@ lights.location = torch.tensor([[0.0, 0.0, -3.0]], device=device)
|
|||||||
images = renderer(meshes, cameras=cameras, lights=lights)
|
images = renderer(meshes, cameras=cameras, lights=lights)
|
||||||
|
|
||||||
|
|
||||||
# In[14]:
|
# In[ ]:
|
||||||
|
|
||||||
|
|
||||||
image_grid(images.cpu().numpy(), rows=4, cols=5, rgb=True)
|
image_grid(images.cpu().numpy(), rows=4, cols=5, rgb=True)
|
||||||
|
|||||||
@ -12,4 +12,4 @@
|
|||||||
</span></div></h2><div><span><p>Ask questions about the documentation and project</p>
|
</span></div></h2><div><span><p>Ask questions about the documentation and project</p>
|
||||||
</span></div></div></div><div class="blockElement threeByGridBlock"><div class="blockContent"><h2><div><span><p>Stay up to date</p>
|
</span></div></div></div><div class="blockElement threeByGridBlock"><div class="blockContent"><h2><div><span><p>Stay up to date</p>
|
||||||
</span></div></h2><div><span><p>Find out what's new with this project</p>
|
</span></div></h2><div><span><p>Find out what's new with this project</p>
|
||||||
</span></div></div></div></div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></div></div></div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -12,4 +12,4 @@
|
|||||||
</span></div></h2><div><span><p>Ask questions about the documentation and project</p>
|
</span></div></h2><div><span><p>Ask questions about the documentation and project</p>
|
||||||
</span></div></div></div><div class="blockElement threeByGridBlock"><div class="blockContent"><h2><div><span><p>Stay up to date</p>
|
</span></div></div></div><div class="blockElement threeByGridBlock"><div class="blockContent"><h2><div><span><p>Stay up to date</p>
|
||||||
</span></div></h2><div><span><p>Find out what's new with this project</p>
|
</span></div></h2><div><span><p>Find out what's new with this project</p>
|
||||||
</span></div></div></div></div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></div></div></div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -12,10 +12,7 @@
|
|||||||
</span></div></h2><div><span><p>Supports optimized implementations of several common functions for 3D data</p>
|
</span></div></h2><div><span><p>Supports optimized implementations of several common functions for 3D data</p>
|
||||||
</span></div></div></div><div class="blockElement alignCenter fourByGridBlock imageAlignTop"><div class="blockImage"><img src="/img/rendering.svg"/></div><div class="blockContent"><h2><div><span><p>Differentiable Rendering</p>
|
</span></div></div></div><div class="blockElement alignCenter fourByGridBlock imageAlignTop"><div class="blockImage"><img src="/img/rendering.svg"/></div><div class="blockContent"><h2><div><span><p>Differentiable Rendering</p>
|
||||||
</span></div></h2><div><span><p>Modular differentiable rendering API with parallel implementations in PyTorch, C++ and CUDA</p>
|
</span></div></h2><div><span><p>Modular differentiable rendering API with parallel implementations in PyTorch, C++ and CUDA</p>
|
||||||
</span></div></div></div></div></div></div></div><div class="productShowcaseSection" id="quickstart" style="text-align:center"><h2>Get Started</h2><div class="container"><div class="wrapper"><ol><li><strong>Install PyTorch3D:</strong><div><span><pre><code class="hljs css language-bash">conda install pytorch torchvision -c pytorch <span class="hljs-comment"># OSX only</span>
|
</span></div></div></div></div></div></div></div><div class="productShowcaseSection" id="quickstart" style="text-align:center"><h2>Get Started</h2><div class="container"><div class="wrapper"><ol><li><strong>Install PyTorch3D </strong> (following the instructions <a href="https://github.com/facebookresearch/pytorch3d/blob/master/INSTALL.md">here</a>)</li><li><strong>Try a few 3D operators </strong>e.g. compute the chamfer loss between two meshes:<div><span><pre><code class="hljs css language-python"><span class="hljs-keyword">from</span> pytorch3d.utils <span class="hljs-keyword">import</span> ico_sphere
|
||||||
conda install pytorch3d -c pytorch3d <span class="hljs-comment"># all systems</span>
|
|
||||||
</code></pre>
|
|
||||||
</span></div></li><li><strong>Try a few 3D operators </strong>e.g. compute the chamfer loss between two meshes:<div><span><pre><code class="hljs css language-python"><span class="hljs-keyword">from</span> pytorch3d.utils <span class="hljs-keyword">import</span> ico_sphere
|
|
||||||
<span class="hljs-keyword">from</span> pytorch3d.io <span class="hljs-keyword">import</span> load_obj
|
<span class="hljs-keyword">from</span> pytorch3d.io <span class="hljs-keyword">import</span> load_obj
|
||||||
<span class="hljs-keyword">from</span> pytorch3d.structures <span class="hljs-keyword">import</span> Meshes
|
<span class="hljs-keyword">from</span> pytorch3d.structures <span class="hljs-keyword">import</span> Meshes
|
||||||
<span class="hljs-keyword">from</span> pytorch3d.ops <span class="hljs-keyword">import</span> sample_points_from_meshes
|
<span class="hljs-keyword">from</span> pytorch3d.ops <span class="hljs-keyword">import</span> sample_points_from_meshes
|
||||||
@ -31,4 +28,4 @@ sample_sphere = sample_points_from_meshes(sphere_mesh, <span class="hljs-number"
|
|||||||
sample_test = sample_points_from_meshes(test_mesh, <span class="hljs-number">5000</span>)
|
sample_test = sample_points_from_meshes(test_mesh, <span class="hljs-number">5000</span>)
|
||||||
loss_chamfer, _ = chamfer_distance(sample_sphere, sample_test)
|
loss_chamfer, _ = chamfer_distance(sample_sphere, sample_test)
|
||||||
</code></pre>
|
</code></pre>
|
||||||
</span></div></li></ol></div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div></li></ol></div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -1,5 +1,5 @@
|
|||||||
/**
|
/**
|
||||||
* Copyright (c) 2017-present, Facebook, Inc.
|
* Copyright (c) Facebook, Inc. and its affiliates.
|
||||||
*
|
*
|
||||||
* This source code is licensed under the MIT license found in the
|
* This source code is licensed under the MIT license found in the
|
||||||
* LICENSE file in the root directory of this source tree.
|
* LICENSE file in the root directory of this source tree.
|
||||||
@ -7,19 +7,19 @@
|
|||||||
// Turn off ESLint for this file because it's sent down to users as-is.
|
// Turn off ESLint for this file because it's sent down to users as-is.
|
||||||
|
|
||||||
/* eslint-disable */
|
/* eslint-disable */
|
||||||
window.addEventListener('load', function() {
|
window.addEventListener('load', function () {
|
||||||
// add event listener for all tab
|
// add event listener for all tab
|
||||||
document.querySelectorAll('.nav-link').forEach(function(el) {
|
document.querySelectorAll('.nav-link').forEach(function (el) {
|
||||||
el.addEventListener('click', function(e) {
|
el.addEventListener('click', function (e) {
|
||||||
var groupId = e.target.getAttribute('data-group');
|
var groupId = e.target.getAttribute('data-group');
|
||||||
document
|
document
|
||||||
.querySelectorAll('.nav-link[data-group='.concat(groupId, ']'))
|
.querySelectorAll('.nav-link[data-group='.concat(groupId, ']'))
|
||||||
.forEach(function(el) {
|
.forEach(function (el) {
|
||||||
el.classList.remove('active');
|
el.classList.remove('active');
|
||||||
});
|
});
|
||||||
document
|
document
|
||||||
.querySelectorAll('.tab-pane[data-group='.concat(groupId, ']'))
|
.querySelectorAll('.tab-pane[data-group='.concat(groupId, ']'))
|
||||||
.forEach(function(el) {
|
.forEach(function (el) {
|
||||||
el.classList.remove('active');
|
el.classList.remove('active');
|
||||||
});
|
});
|
||||||
e.target.classList.add('active');
|
e.target.classList.add('active');
|
||||||
|
|||||||
@ -1,5 +1,5 @@
|
|||||||
/**
|
/**
|
||||||
* Copyright (c) 2017-present, Facebook, Inc.
|
* Copyright (c) Facebook, Inc. and its affiliates.
|
||||||
*
|
*
|
||||||
* This source code is licensed under the MIT license found in the
|
* This source code is licensed under the MIT license found in the
|
||||||
* LICENSE file in the root directory of this source tree.
|
* LICENSE file in the root directory of this source tree.
|
||||||
@ -21,7 +21,7 @@
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
timer = setTimeout(function() {
|
timer = setTimeout(function () {
|
||||||
timer = null;
|
timer = null;
|
||||||
var activeNavFound = false;
|
var activeNavFound = false;
|
||||||
var headings = findHeadings(); // toc nav anchors
|
var headings = findHeadings(); // toc nav anchors
|
||||||
@ -75,7 +75,7 @@
|
|||||||
|
|
||||||
document.addEventListener('scroll', onScroll);
|
document.addEventListener('scroll', onScroll);
|
||||||
document.addEventListener('resize', onScroll);
|
document.addEventListener('resize', onScroll);
|
||||||
document.addEventListener('DOMContentLoaded', function() {
|
document.addEventListener('DOMContentLoaded', function () {
|
||||||
// Cache the headings once the page has fully loaded.
|
// Cache the headings once the page has fully loaded.
|
||||||
headingsCache = findHeadings();
|
headingsCache = findHeadings();
|
||||||
onScroll();
|
onScroll();
|
||||||
|
|||||||
@ -1 +1 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:mobile="http://www.google.com/schemas/sitemap-mobile/1.0" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"><url><loc>https://pytorch3d.org/help</loc><changefreq>weekly</changefreq><priority>0.5</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/en/help"/></url><url><loc>https://pytorch3d.org/index</loc><changefreq>weekly</changefreq><priority>0.5</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/en/index"/></url><url><loc>https://pytorch3d.org/users</loc><changefreq>weekly</changefreq><priority>0.5</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/en/users"/></url><url><loc>https://pytorch3d.org/docs/batching</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/batching"/></url><url><loc>https://pytorch3d.org/docs/meshes_io</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/meshes_io"/></url><url><loc>https://pytorch3d.org/docs/renderer_getting_started</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/renderer_getting_started"/></url><url><loc>https://pytorch3d.org/docs/renderer</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/renderer"/></url><url><loc>https://pytorch3d.org/docs/why_pytorch3d</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/why_pytorch3d"/></url></urlset>
|
<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:mobile="http://www.google.com/schemas/sitemap-mobile/1.0" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"><url><loc>https://pytorch3d.org/help</loc><changefreq>weekly</changefreq><priority>0.5</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/en/help"/></url><url><loc>https://pytorch3d.org/index</loc><changefreq>weekly</changefreq><priority>0.5</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/en/index"/></url><url><loc>https://pytorch3d.org/users</loc><changefreq>weekly</changefreq><priority>0.5</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/en/users"/></url><url><loc>https://pytorch3d.org/docs/batching</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/batching"/></url><url><loc>https://pytorch3d.org/docs/cameras</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/cameras"/></url><url><loc>https://pytorch3d.org/docs/cubify</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/cubify"/></url><url><loc>https://pytorch3d.org/docs/datasets</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/datasets"/></url><url><loc>https://pytorch3d.org/docs/meshes_io</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/meshes_io"/></url><url><loc>https://pytorch3d.org/docs/renderer_getting_started</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/renderer_getting_started"/></url><url><loc>https://pytorch3d.org/docs/renderer</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/renderer"/></url><url><loc>https://pytorch3d.org/docs/why_pytorch3d</loc><changefreq>hourly</changefreq><priority>1.0</priority><xhtml:link rel="alternate" hreflang="en" href="https://pytorch3d.org/docs/why_pytorch3d"/></url></urlset>
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
614
tutorials/dataloaders_ShapeNetCore_R2N2.html
Normal file
614
tutorials/dataloaders_ShapeNetCore_R2N2.html
Normal file
@ -0,0 +1,614 @@
|
|||||||
|
<!DOCTYPE html><html lang=""><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>PyTorch3D · A library for deep learning with 3D data</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="A library for deep learning with 3D data"/><meta property="og:title" content="PyTorch3D · A library for deep learning with 3D data"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="A library for deep learning with 3D data"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||||
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||||
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||||
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||||
|
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||||
|
|
||||||
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
|
ga('send', 'pageview');
|
||||||
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="container docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span></span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Tutorials</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/">Overview</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">3D operators</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/deform_source_mesh_to_target_mesh">Fit Mesh</a></li><li class="navListItem"><a class="navItem" href="/tutorials/bundle_adjustment">Bundle Adjustment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Rendering</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/render_textured_meshes">Render Textured Meshes</a></li><li class="navListItem"><a class="navItem" href="/tutorials/fit_textured_mesh">Fit a mesh with texture via rendering</a></li><li class="navListItem"><a class="navItem" href="/tutorials/camera_position_optimization_with_differentiable_rendering">Camera Position Optimization</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Dataloaders</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/tutorials/dataloaders_ShapeNetCore_R2N2">Data loaders for ShapeNetCore and R2N2</a></li></ul></div></div></section></div><script>
|
||||||
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
|
var checkActiveCategory = true;
|
||||||
|
for (var i = 0; i < coll.length; i++) {
|
||||||
|
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||||
|
if (checkActiveCategory){
|
||||||
|
for (var j = 0; j < links.length; j++) {
|
||||||
|
if (links[j].classList.contains('navListItemActive')){
|
||||||
|
coll[i].nextElementSibling.classList.toggle('hide');
|
||||||
|
coll[i].childNodes[1].classList.toggle('rotate');
|
||||||
|
checkActiveCategory = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
coll[i].addEventListener('click', function() {
|
||||||
|
var arrow = this.childNodes[1];
|
||||||
|
arrow.classList.toggle('rotate');
|
||||||
|
var content = this.nextElementSibling;
|
||||||
|
content.classList.toggle('hide');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||||
|
createToggler('#tocToggler', 'body', 'tocActive');
|
||||||
|
|
||||||
|
var headings = document.querySelector('.toc-headings');
|
||||||
|
headings && headings.addEventListener('click', function(event) {
|
||||||
|
var el = event.target;
|
||||||
|
while(el !== headings){
|
||||||
|
if (el.tagName === 'A') {
|
||||||
|
document.body.classList.remove('tocActive');
|
||||||
|
break;
|
||||||
|
} else{
|
||||||
|
el = el.parentNode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, false);
|
||||||
|
|
||||||
|
function createToggler(togglerSelector, targetSelector, className) {
|
||||||
|
var toggler = document.querySelector(togglerSelector);
|
||||||
|
var target = document.querySelector(targetSelector);
|
||||||
|
|
||||||
|
if (!toggler) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
toggler.onclick = function(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
|
||||||
|
target.classList.toggle(className);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script></nav></div><div class="container mainContainer"><div class="wrapper"><div class="tutorialButtonsWrapper"><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="https://colab.research.google.com/github/facebookresearch/pytorch3d/blob/stable/docs/tutorials/dataloaders_ShapeNetCore_R2N2.ipynb" target="_blank"><img class="colabButton" align="left" src="/img/colab_icon.png"/>Run in Google Colab</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/dataloaders_ShapeNetCore_R2N2.ipynb" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Jupyter Notebook</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/dataloaders_ShapeNetCore_R2N2.py" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Source Code</a></div></div><div class="tutorialBody">
|
||||||
|
<script
|
||||||
|
src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.10/require.min.js">
|
||||||
|
</script>
|
||||||
|
<script
|
||||||
|
src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js">
|
||||||
|
</script>
|
||||||
|
<div class="notebook">
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h1 id="Dataloaders-for-ShapeNetCore-and-R2N2">Dataloaders for ShapeNetCore and R2N2<a class="anchor-link" href="#Dataloaders-for-ShapeNetCore-and-R2N2">¶</a></h1><p>This tutorial shows how to:</p>
|
||||||
|
<ul>
|
||||||
|
<li>Load models from ShapeNetCore and R2N2 using PyTorch3D's data loaders.</li>
|
||||||
|
<li>Pass the loaded datasets to <code>torch.utils.data.DataLoader</code>.</li>
|
||||||
|
<li>Render ShapeNetCore models with PyTorch3D's renderer.</li>
|
||||||
|
<li>Render R2N2 models with the same orientations as the original renderings in the dataset.</li>
|
||||||
|
<li>Visualize R2N2 model voxels.</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="0.-Install-and-import-modules">0. Install and import modules<a class="anchor-link" href="#0.-Install-and-import-modules">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If <code>torch</code>, <code>torchvision</code> and <code>pytorch3d</code> are not installed, run the following cell:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>pip install torch torchvision
|
||||||
|
<span class="kn">import</span> <span class="nn">sys</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">torch</span>
|
||||||
|
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">__version__</span><span class="o">==</span><span class="s1">'1.6.0+cu101'</span> <span class="ow">and</span> <span class="n">sys</span><span class="o">.</span><span class="n">platform</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'linux'</span><span class="p">):</span>
|
||||||
|
<span class="o">!</span>pip install pytorch3d
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="o">!</span>pip install <span class="s1">'git+https://github.com/facebookresearch/pytorch3d.git@stable'</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">torch</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.datasets</span> <span class="k">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">R2N2</span><span class="p">,</span>
|
||||||
|
<span class="n">ShapeNetCore</span><span class="p">,</span>
|
||||||
|
<span class="n">collate_batched_meshes</span><span class="p">,</span>
|
||||||
|
<span class="n">render_cubified_voxels</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.renderer</span> <span class="k">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">OpenGLPerspectiveCameras</span><span class="p">,</span>
|
||||||
|
<span class="n">PointLights</span><span class="p">,</span>
|
||||||
|
<span class="n">RasterizationSettings</span><span class="p">,</span>
|
||||||
|
<span class="n">TexturesVertex</span><span class="p">,</span>
|
||||||
|
<span class="n">look_at_view_transform</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.structures</span> <span class="k">import</span> <span class="n">Meshes</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">torch.utils.data</span> <span class="k">import</span> <span class="n">DataLoader</span>
|
||||||
|
|
||||||
|
<span class="c1"># add path for demo utils functions </span>
|
||||||
|
<span class="kn">import</span> <span class="nn">sys</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">os</span>
|
||||||
|
<span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="s1">''</span><span class="p">))</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If using <strong>Google Colab</strong>, fetch the utils file for plotting image grids:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py
|
||||||
|
<span class="kn">from</span> <span class="nn">plot_image_grid</span> <span class="k">import</span> <span class="n">image_grid</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>OR if running locally uncomment and run the following cell:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># from utils import image_grid</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="1.-Load-the-datasets">1. Load the datasets<a class="anchor-link" href="#1.-Load-the-datasets">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If you haven't already downloaded the ShapeNetCore dataset, first do that following the instructions here: <a href="https://www.shapenet.org/">https://www.shapenet.org/</a>. ShapeNetCore is a subset of the ShapeNet dataset. In PyTorch3D we support both version 1 (57 categories) and version 2 (55 categories).</p>
|
||||||
|
<p>Then modify <code>SHAPENET_PATH</code> below to you local path to the ShapeNetCore dataset folder.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Setup</span>
|
||||||
|
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
|
||||||
|
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cuda:0"</span><span class="p">)</span>
|
||||||
|
<span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">set_device</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cpu"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">SHAPENET_PATH</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">shapenet_dataset</span> <span class="o">=</span> <span class="n">ShapeNetCore</span><span class="p">(</span><span class="n">SHAPENET_PATH</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>The R2N2 dataset can be downloaded using the instructions here: <a href="http://3d-r2n2.stanford.edu/">http://3d-r2n2.stanford.edu/</a>. Look at the links for <code>ShapeNetRendering</code> and <code>ShapeNetVox32</code>. The R2N2 dataset contains 13 categories that are a subset of the ShapeNetCore v.1
|
||||||
|
dataset. The R2N2 dataset also contains its own 24 renderings of each object and voxelized models.</p>
|
||||||
|
<p>Then modify <code>R2N2_PATH</code> and <code>SPLITS_PATH</code> below to your local R2N2 dataset folder path and splits file path respectively. Here we will load the <code>train</code> split of R2N2 and ask the voxels of each model to be returned.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">R2N2_PATH</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">SPLITS_PATH</span> <span class="o">=</span> <span class="s2">"None"</span>
|
||||||
|
<span class="n">r2n2_dataset</span> <span class="o">=</span> <span class="n">R2N2</span><span class="p">(</span><span class="s2">"train"</span><span class="p">,</span> <span class="n">SHAPENET_PATH</span><span class="p">,</span> <span class="n">R2N2_PATH</span><span class="p">,</span> <span class="n">SPLITS_PATH</span><span class="p">,</span> <span class="n">return_voxels</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We can retrieve a model by indexing into the loaded dataset. For both ShapeNetCore and R2N2, we can examine the category this model belongs to (in the form of a synset id, equivalend to wnid described in ImageNet's API: <a href="http://image-net.org/download-API">http://image-net.org/download-API</a>), its model id, and its vertices and faces.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">shapenet_model</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="p">[</span><span class="mi">6</span><span class="p">]</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"This model belongs to the category "</span> <span class="o">+</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"synset_id"</span><span class="p">]</span> <span class="o">+</span> <span class="s2">"."</span><span class="p">)</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"This model has model id "</span> <span class="o">+</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"model_id"</span><span class="p">]</span> <span class="o">+</span> <span class="s2">"."</span><span class="p">)</span>
|
||||||
|
<span class="n">model_verts</span><span class="p">,</span> <span class="n">model_faces</span> <span class="o">=</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"verts"</span><span class="p">],</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"faces"</span><span class="p">]</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We can use its vertices and faces to form a <code>Meshes</code> object which is a PyTorch3D datastructure for working with batched meshes.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">model_textures</span> <span class="o">=</span> <span class="n">TexturesVertex</span><span class="p">(</span><span class="n">verts_features</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">ones_like</span><span class="p">(</span><span class="n">model_verts</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)[</span><span class="kc">None</span><span class="p">])</span>
|
||||||
|
<span class="n">shapenet_model_mesh</span> <span class="o">=</span> <span class="n">Meshes</span><span class="p">(</span>
|
||||||
|
<span class="n">verts</span><span class="o">=</span><span class="p">[</span><span class="n">model_verts</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)],</span>
|
||||||
|
<span class="n">faces</span><span class="o">=</span><span class="p">[</span><span class="n">model_faces</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)],</span>
|
||||||
|
<span class="n">textures</span><span class="o">=</span><span class="n">model_textures</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>With R2N2, we can further examine R2N2's original renderings. For instance, if we would like to see the second and third views of the eleventh objects in the R2N2 dataset, we can do the following:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">r2n2_renderings</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="p">[</span><span class="mi">10</span><span class="p">,[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">]]</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">r2n2_renderings</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="2.-Use-the-datasets-with-torch.utils.data.DataLoader">2. Use the datasets with <code>torch.utils.data.DataLoader</code><a class="anchor-link" href="#2.-Use-the-datasets-with-torch.utils.data.DataLoader">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Training deep learning models, usually requires passing in batches of inputs. The <code>torch.utils.data.DataLoader</code> from Pytorch helps us do this. PyTorch3D provides a function <code>collate_batched_meshes</code> to group the input meshes into a single <code>Meshes</code> object which represents the batch. The <code>Meshes</code> datastructure can then be used directly by other PyTorch3D ops which might be part of the deep learning model (e.g. <code>graph_conv</code>).</p>
|
||||||
|
<p>For R2N2, if all the models in the batch have the same number of views, the views, rotation matrices, translation matrices, intrinsic matrices and voxels will also be stacked into batched tensors.</p>
|
||||||
|
<p><strong>NOTE</strong>: All models in the <code>val</code> split of R2N2 have 24 views, but there are 8 models that split their 24 views between <code>train</code> and <code>test</code> splits, in which case <code>collate_batched_meshes</code> will only be able to join the matrices, views and voxels as lists. However, this can be avoided by laoding only one view of each model by setting <code>return_all_views = False</code>.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">batch_size</span> <span class="o">=</span> <span class="mi">12</span>
|
||||||
|
<span class="n">r2n2_single_view</span> <span class="o">=</span> <span class="n">R2N2</span><span class="p">(</span><span class="s2">"train"</span><span class="p">,</span> <span class="n">SHAPENET_PATH</span><span class="p">,</span> <span class="n">R2N2_PATH</span><span class="p">,</span> <span class="n">SPLITS_PATH</span><span class="p">,</span> <span class="n">return_all_views</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">return_voxels</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
<span class="n">r2n2_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span><span class="n">r2n2_single_view</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">collate_fn</span><span class="o">=</span><span class="n">collate_batched_meshes</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Let's visualize all the views (one for each model) in the batch:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">it</span> <span class="o">=</span> <span class="nb">iter</span><span class="p">(</span><span class="n">r2n2_loader</span><span class="p">)</span>
|
||||||
|
<span class="n">r2n2_batch</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="n">it</span><span class="p">)</span>
|
||||||
|
<span class="n">batch_renderings</span> <span class="o">=</span> <span class="n">r2n2_batch</span><span class="p">[</span><span class="s2">"images"</span><span class="p">]</span> <span class="c1"># (N, V, H, W, 3), and in this case V is 1.</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">batch_renderings</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="3.-Render-ShapeNetCore-models-with-PyTorch3D's-differntiable-renderer">3. Render ShapeNetCore models with PyTorch3D's differntiable renderer<a class="anchor-link" href="#3.-Render-ShapeNetCore-models-with-PyTorch3D's-differntiable-renderer">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Both <code>ShapeNetCore</code> and <code>R2N2</code> dataloaders have customized <code>render</code> functions that support rendering models by specifying their model ids, categories or indices using PyTorch3D's differentiable renderer implementation.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rendering settings.</span>
|
||||||
|
<span class="n">R</span><span class="p">,</span> <span class="n">T</span> <span class="o">=</span> <span class="n">look_at_view_transform</span><span class="p">(</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="mi">90</span><span class="p">)</span>
|
||||||
|
<span class="n">cameras</span> <span class="o">=</span> <span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">,</span> <span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
<span class="n">raster_settings</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span><span class="n">image_size</span><span class="o">=</span><span class="mi">512</span><span class="p">)</span>
|
||||||
|
<span class="n">lights</span> <span class="o">=</span> <span class="n">PointLights</span><span class="p">(</span><span class="n">location</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="o">-</span><span class="mf">2.0</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)[</span><span class="kc">None</span><span class="p">],</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>First we will try to render three models by their model ids:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">images_by_model_ids</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||||
|
<span class="n">model_ids</span><span class="o">=</span><span class="p">[</span>
|
||||||
|
<span class="s2">"13394ca47c89f91525a3aaf903a41c90"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"14755c2ee8e693aba508f621166382b0"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"156c4207af6d2c8f1fdc97905708b8ea"</span><span class="p">,</span>
|
||||||
|
<span class="p">],</span>
|
||||||
|
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">images_by_model_ids</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Suppose we would like to render the first three models in the dataset, we can render models by their indices:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">images_by_idxs</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||||
|
<span class="n">idxs</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">)),</span>
|
||||||
|
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">images_by_idxs</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Alternatively, if we are not interested in any particular models but would like see random models from some specific categories, we can do that by specifying <code>categories</code> and <code>sample_nums</code>. For example, if we would like to render 2 models from the category "faucet" and 3 models from the category "chair", we can do the following:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">images_by_categories</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||||
|
<span class="n">categories</span><span class="o">=</span><span class="p">[</span><span class="s2">"faucet"</span><span class="p">,</span> <span class="s2">"chair"</span><span class="p">],</span>
|
||||||
|
<span class="n">sample_nums</span><span class="o">=</span><span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span>
|
||||||
|
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">images_by_categories</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If we are not interested in any particular categories and just would like to render some random models from the whole dataset, we can set the number of models to be rendered in <code>sample_nums</code> and not specify any <code>categories</code>:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">random_model_images</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||||
|
<span class="n">sample_nums</span><span class="o">=</span><span class="p">[</span><span class="mi">3</span><span class="p">],</span>
|
||||||
|
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">random_model_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="4.-Render-R2N2-models-with-the-same-orientations-as-the-original-renderings-in-the-dataset">4. Render R2N2 models with the same orientations as the original renderings in the dataset<a class="anchor-link" href="#4.-Render-R2N2-models-with-the-same-orientations-as-the-original-renderings-in-the-dataset">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We can render R2N2 models the same way as we rendered ShapeNetCore models above. In addition, we can also render R2N2 models with the same orientations as the original renderings in the dataset. For this we will use R2N2's customized <code>render</code> function and a different type of PyTorch3D camera called <code>BlenderCamera</code>.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>In this example, we will render the seventh model with the same orientations as its second and third views. First we will retrieve R2N2's original renderings to compare with the result.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">original_rendering</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="p">[</span><span class="mi">6</span><span class="p">,[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">]][</span><span class="s2">"images"</span><span class="p">]</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">original_rendering</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Next, we will visualize PyTorch3d's renderings:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">r2n2_oriented_images</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||||
|
<span class="n">idxs</span><span class="o">=</span><span class="p">[</span><span class="mi">6</span><span class="p">],</span>
|
||||||
|
<span class="n">view_idxs</span><span class="o">=</span><span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">],</span>
|
||||||
|
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">r2n2_oriented_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="5.-Visualize-R2N2-models'-voxels">5. Visualize R2N2 models' voxels<a class="anchor-link" href="#5.-Visualize-R2N2-models'-voxels">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>R2N2 dataloader also returns models' voxels. We can visualize them by utilizing R2N2's <code>render_vox_to_mesh</code> function. This will cubify the voxels to a Meshes object, which will then be rendered.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>In this example we will visualize the tenth model in the dataset with the same orientation of its second and third views. First we will retrieve R2N2's original renderings to compare with the result.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">r2n2_model</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="p">[</span><span class="mi">9</span><span class="p">,[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">]]</span>
|
||||||
|
<span class="n">original_rendering</span> <span class="o">=</span> <span class="n">r2n2_model</span><span class="p">[</span><span class="s2">"images"</span><span class="p">]</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">original_rendering</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Next, we will pass the voxels to <code>render_vox_to_mesh</code>:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">vox_render</span> <span class="o">=</span> <span class="n">render_cubified_voxels</span><span class="p">(</span><span class="n">r2n2_model</span><span class="p">[</span><span class="s2">"voxels"</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">vox_render</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
614
tutorials/dataloaders_ShapeNetCore_R2N2/index.html
Normal file
614
tutorials/dataloaders_ShapeNetCore_R2N2/index.html
Normal file
@ -0,0 +1,614 @@
|
|||||||
|
<!DOCTYPE html><html lang=""><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>PyTorch3D · A library for deep learning with 3D data</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="A library for deep learning with 3D data"/><meta property="og:title" content="PyTorch3D · A library for deep learning with 3D data"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="A library for deep learning with 3D data"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||||
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||||
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||||
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||||
|
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||||
|
|
||||||
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
|
ga('send', 'pageview');
|
||||||
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="container docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span></span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Tutorials</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/">Overview</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">3D operators</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/deform_source_mesh_to_target_mesh">Fit Mesh</a></li><li class="navListItem"><a class="navItem" href="/tutorials/bundle_adjustment">Bundle Adjustment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Rendering</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/render_textured_meshes">Render Textured Meshes</a></li><li class="navListItem"><a class="navItem" href="/tutorials/fit_textured_mesh">Fit a mesh with texture via rendering</a></li><li class="navListItem"><a class="navItem" href="/tutorials/camera_position_optimization_with_differentiable_rendering">Camera Position Optimization</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Dataloaders</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/tutorials/dataloaders_ShapeNetCore_R2N2">Data loaders for ShapeNetCore and R2N2</a></li></ul></div></div></section></div><script>
|
||||||
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
|
var checkActiveCategory = true;
|
||||||
|
for (var i = 0; i < coll.length; i++) {
|
||||||
|
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||||
|
if (checkActiveCategory){
|
||||||
|
for (var j = 0; j < links.length; j++) {
|
||||||
|
if (links[j].classList.contains('navListItemActive')){
|
||||||
|
coll[i].nextElementSibling.classList.toggle('hide');
|
||||||
|
coll[i].childNodes[1].classList.toggle('rotate');
|
||||||
|
checkActiveCategory = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
coll[i].addEventListener('click', function() {
|
||||||
|
var arrow = this.childNodes[1];
|
||||||
|
arrow.classList.toggle('rotate');
|
||||||
|
var content = this.nextElementSibling;
|
||||||
|
content.classList.toggle('hide');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||||
|
createToggler('#tocToggler', 'body', 'tocActive');
|
||||||
|
|
||||||
|
var headings = document.querySelector('.toc-headings');
|
||||||
|
headings && headings.addEventListener('click', function(event) {
|
||||||
|
var el = event.target;
|
||||||
|
while(el !== headings){
|
||||||
|
if (el.tagName === 'A') {
|
||||||
|
document.body.classList.remove('tocActive');
|
||||||
|
break;
|
||||||
|
} else{
|
||||||
|
el = el.parentNode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, false);
|
||||||
|
|
||||||
|
function createToggler(togglerSelector, targetSelector, className) {
|
||||||
|
var toggler = document.querySelector(togglerSelector);
|
||||||
|
var target = document.querySelector(targetSelector);
|
||||||
|
|
||||||
|
if (!toggler) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
toggler.onclick = function(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
|
||||||
|
target.classList.toggle(className);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script></nav></div><div class="container mainContainer"><div class="wrapper"><div class="tutorialButtonsWrapper"><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="https://colab.research.google.com/github/facebookresearch/pytorch3d/blob/stable/docs/tutorials/dataloaders_ShapeNetCore_R2N2.ipynb" target="_blank"><img class="colabButton" align="left" src="/img/colab_icon.png"/>Run in Google Colab</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/dataloaders_ShapeNetCore_R2N2.ipynb" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Jupyter Notebook</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/dataloaders_ShapeNetCore_R2N2.py" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Source Code</a></div></div><div class="tutorialBody">
|
||||||
|
<script
|
||||||
|
src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.10/require.min.js">
|
||||||
|
</script>
|
||||||
|
<script
|
||||||
|
src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js">
|
||||||
|
</script>
|
||||||
|
<div class="notebook">
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h1 id="Dataloaders-for-ShapeNetCore-and-R2N2">Dataloaders for ShapeNetCore and R2N2<a class="anchor-link" href="#Dataloaders-for-ShapeNetCore-and-R2N2">¶</a></h1><p>This tutorial shows how to:</p>
|
||||||
|
<ul>
|
||||||
|
<li>Load models from ShapeNetCore and R2N2 using PyTorch3D's data loaders.</li>
|
||||||
|
<li>Pass the loaded datasets to <code>torch.utils.data.DataLoader</code>.</li>
|
||||||
|
<li>Render ShapeNetCore models with PyTorch3D's renderer.</li>
|
||||||
|
<li>Render R2N2 models with the same orientations as the original renderings in the dataset.</li>
|
||||||
|
<li>Visualize R2N2 model voxels.</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="0.-Install-and-import-modules">0. Install and import modules<a class="anchor-link" href="#0.-Install-and-import-modules">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If <code>torch</code>, <code>torchvision</code> and <code>pytorch3d</code> are not installed, run the following cell:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>pip install torch torchvision
|
||||||
|
<span class="kn">import</span> <span class="nn">sys</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">torch</span>
|
||||||
|
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">__version__</span><span class="o">==</span><span class="s1">'1.6.0+cu101'</span> <span class="ow">and</span> <span class="n">sys</span><span class="o">.</span><span class="n">platform</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'linux'</span><span class="p">):</span>
|
||||||
|
<span class="o">!</span>pip install pytorch3d
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="o">!</span>pip install <span class="s1">'git+https://github.com/facebookresearch/pytorch3d.git@stable'</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">torch</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.datasets</span> <span class="k">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">R2N2</span><span class="p">,</span>
|
||||||
|
<span class="n">ShapeNetCore</span><span class="p">,</span>
|
||||||
|
<span class="n">collate_batched_meshes</span><span class="p">,</span>
|
||||||
|
<span class="n">render_cubified_voxels</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.renderer</span> <span class="k">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">OpenGLPerspectiveCameras</span><span class="p">,</span>
|
||||||
|
<span class="n">PointLights</span><span class="p">,</span>
|
||||||
|
<span class="n">RasterizationSettings</span><span class="p">,</span>
|
||||||
|
<span class="n">TexturesVertex</span><span class="p">,</span>
|
||||||
|
<span class="n">look_at_view_transform</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.structures</span> <span class="k">import</span> <span class="n">Meshes</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">torch.utils.data</span> <span class="k">import</span> <span class="n">DataLoader</span>
|
||||||
|
|
||||||
|
<span class="c1"># add path for demo utils functions </span>
|
||||||
|
<span class="kn">import</span> <span class="nn">sys</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">os</span>
|
||||||
|
<span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="s1">''</span><span class="p">))</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If using <strong>Google Colab</strong>, fetch the utils file for plotting image grids:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py
|
||||||
|
<span class="kn">from</span> <span class="nn">plot_image_grid</span> <span class="k">import</span> <span class="n">image_grid</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>OR if running locally uncomment and run the following cell:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># from utils import image_grid</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="1.-Load-the-datasets">1. Load the datasets<a class="anchor-link" href="#1.-Load-the-datasets">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If you haven't already downloaded the ShapeNetCore dataset, first do that following the instructions here: <a href="https://www.shapenet.org/">https://www.shapenet.org/</a>. ShapeNetCore is a subset of the ShapeNet dataset. In PyTorch3D we support both version 1 (57 categories) and version 2 (55 categories).</p>
|
||||||
|
<p>Then modify <code>SHAPENET_PATH</code> below to you local path to the ShapeNetCore dataset folder.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Setup</span>
|
||||||
|
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
|
||||||
|
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cuda:0"</span><span class="p">)</span>
|
||||||
|
<span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">set_device</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cpu"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="n">SHAPENET_PATH</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">shapenet_dataset</span> <span class="o">=</span> <span class="n">ShapeNetCore</span><span class="p">(</span><span class="n">SHAPENET_PATH</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>The R2N2 dataset can be downloaded using the instructions here: <a href="http://3d-r2n2.stanford.edu/">http://3d-r2n2.stanford.edu/</a>. Look at the links for <code>ShapeNetRendering</code> and <code>ShapeNetVox32</code>. The R2N2 dataset contains 13 categories that are a subset of the ShapeNetCore v.1
|
||||||
|
dataset. The R2N2 dataset also contains its own 24 renderings of each object and voxelized models.</p>
|
||||||
|
<p>Then modify <code>R2N2_PATH</code> and <code>SPLITS_PATH</code> below to your local R2N2 dataset folder path and splits file path respectively. Here we will load the <code>train</code> split of R2N2 and ask the voxels of each model to be returned.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">R2N2_PATH</span> <span class="o">=</span> <span class="s2">""</span>
|
||||||
|
<span class="n">SPLITS_PATH</span> <span class="o">=</span> <span class="s2">"None"</span>
|
||||||
|
<span class="n">r2n2_dataset</span> <span class="o">=</span> <span class="n">R2N2</span><span class="p">(</span><span class="s2">"train"</span><span class="p">,</span> <span class="n">SHAPENET_PATH</span><span class="p">,</span> <span class="n">R2N2_PATH</span><span class="p">,</span> <span class="n">SPLITS_PATH</span><span class="p">,</span> <span class="n">return_voxels</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We can retrieve a model by indexing into the loaded dataset. For both ShapeNetCore and R2N2, we can examine the category this model belongs to (in the form of a synset id, equivalend to wnid described in ImageNet's API: <a href="http://image-net.org/download-API">http://image-net.org/download-API</a>), its model id, and its vertices and faces.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">shapenet_model</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="p">[</span><span class="mi">6</span><span class="p">]</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"This model belongs to the category "</span> <span class="o">+</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"synset_id"</span><span class="p">]</span> <span class="o">+</span> <span class="s2">"."</span><span class="p">)</span>
|
||||||
|
<span class="nb">print</span><span class="p">(</span><span class="s2">"This model has model id "</span> <span class="o">+</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"model_id"</span><span class="p">]</span> <span class="o">+</span> <span class="s2">"."</span><span class="p">)</span>
|
||||||
|
<span class="n">model_verts</span><span class="p">,</span> <span class="n">model_faces</span> <span class="o">=</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"verts"</span><span class="p">],</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"faces"</span><span class="p">]</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We can use its vertices and faces to form a <code>Meshes</code> object which is a PyTorch3D datastructure for working with batched meshes.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">model_textures</span> <span class="o">=</span> <span class="n">TexturesVertex</span><span class="p">(</span><span class="n">verts_features</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">ones_like</span><span class="p">(</span><span class="n">model_verts</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)[</span><span class="kc">None</span><span class="p">])</span>
|
||||||
|
<span class="n">shapenet_model_mesh</span> <span class="o">=</span> <span class="n">Meshes</span><span class="p">(</span>
|
||||||
|
<span class="n">verts</span><span class="o">=</span><span class="p">[</span><span class="n">model_verts</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)],</span>
|
||||||
|
<span class="n">faces</span><span class="o">=</span><span class="p">[</span><span class="n">model_faces</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)],</span>
|
||||||
|
<span class="n">textures</span><span class="o">=</span><span class="n">model_textures</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>With R2N2, we can further examine R2N2's original renderings. For instance, if we would like to see the second and third views of the eleventh objects in the R2N2 dataset, we can do the following:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">r2n2_renderings</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="p">[</span><span class="mi">10</span><span class="p">,[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">]]</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">r2n2_renderings</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="2.-Use-the-datasets-with-torch.utils.data.DataLoader">2. Use the datasets with <code>torch.utils.data.DataLoader</code><a class="anchor-link" href="#2.-Use-the-datasets-with-torch.utils.data.DataLoader">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Training deep learning models, usually requires passing in batches of inputs. The <code>torch.utils.data.DataLoader</code> from Pytorch helps us do this. PyTorch3D provides a function <code>collate_batched_meshes</code> to group the input meshes into a single <code>Meshes</code> object which represents the batch. The <code>Meshes</code> datastructure can then be used directly by other PyTorch3D ops which might be part of the deep learning model (e.g. <code>graph_conv</code>).</p>
|
||||||
|
<p>For R2N2, if all the models in the batch have the same number of views, the views, rotation matrices, translation matrices, intrinsic matrices and voxels will also be stacked into batched tensors.</p>
|
||||||
|
<p><strong>NOTE</strong>: All models in the <code>val</code> split of R2N2 have 24 views, but there are 8 models that split their 24 views between <code>train</code> and <code>test</code> splits, in which case <code>collate_batched_meshes</code> will only be able to join the matrices, views and voxels as lists. However, this can be avoided by laoding only one view of each model by setting <code>return_all_views = False</code>.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">batch_size</span> <span class="o">=</span> <span class="mi">12</span>
|
||||||
|
<span class="n">r2n2_single_view</span> <span class="o">=</span> <span class="n">R2N2</span><span class="p">(</span><span class="s2">"train"</span><span class="p">,</span> <span class="n">SHAPENET_PATH</span><span class="p">,</span> <span class="n">R2N2_PATH</span><span class="p">,</span> <span class="n">SPLITS_PATH</span><span class="p">,</span> <span class="n">return_all_views</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">return_voxels</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
<span class="n">r2n2_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span><span class="n">r2n2_single_view</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">collate_fn</span><span class="o">=</span><span class="n">collate_batched_meshes</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Let's visualize all the views (one for each model) in the batch:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">it</span> <span class="o">=</span> <span class="nb">iter</span><span class="p">(</span><span class="n">r2n2_loader</span><span class="p">)</span>
|
||||||
|
<span class="n">r2n2_batch</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="n">it</span><span class="p">)</span>
|
||||||
|
<span class="n">batch_renderings</span> <span class="o">=</span> <span class="n">r2n2_batch</span><span class="p">[</span><span class="s2">"images"</span><span class="p">]</span> <span class="c1"># (N, V, H, W, 3), and in this case V is 1.</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">batch_renderings</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="3.-Render-ShapeNetCore-models-with-PyTorch3D's-differntiable-renderer">3. Render ShapeNetCore models with PyTorch3D's differntiable renderer<a class="anchor-link" href="#3.-Render-ShapeNetCore-models-with-PyTorch3D's-differntiable-renderer">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Both <code>ShapeNetCore</code> and <code>R2N2</code> dataloaders have customized <code>render</code> functions that support rendering models by specifying their model ids, categories or indices using PyTorch3D's differentiable renderer implementation.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rendering settings.</span>
|
||||||
|
<span class="n">R</span><span class="p">,</span> <span class="n">T</span> <span class="o">=</span> <span class="n">look_at_view_transform</span><span class="p">(</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="mi">90</span><span class="p">)</span>
|
||||||
|
<span class="n">cameras</span> <span class="o">=</span> <span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">,</span> <span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
<span class="n">raster_settings</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span><span class="n">image_size</span><span class="o">=</span><span class="mi">512</span><span class="p">)</span>
|
||||||
|
<span class="n">lights</span> <span class="o">=</span> <span class="n">PointLights</span><span class="p">(</span><span class="n">location</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="o">-</span><span class="mf">2.0</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)[</span><span class="kc">None</span><span class="p">],</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>First we will try to render three models by their model ids:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">images_by_model_ids</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||||
|
<span class="n">model_ids</span><span class="o">=</span><span class="p">[</span>
|
||||||
|
<span class="s2">"13394ca47c89f91525a3aaf903a41c90"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"14755c2ee8e693aba508f621166382b0"</span><span class="p">,</span>
|
||||||
|
<span class="s2">"156c4207af6d2c8f1fdc97905708b8ea"</span><span class="p">,</span>
|
||||||
|
<span class="p">],</span>
|
||||||
|
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">images_by_model_ids</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Suppose we would like to render the first three models in the dataset, we can render models by their indices:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">images_by_idxs</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||||
|
<span class="n">idxs</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">)),</span>
|
||||||
|
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">images_by_idxs</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Alternatively, if we are not interested in any particular models but would like see random models from some specific categories, we can do that by specifying <code>categories</code> and <code>sample_nums</code>. For example, if we would like to render 2 models from the category "faucet" and 3 models from the category "chair", we can do the following:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">images_by_categories</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||||
|
<span class="n">categories</span><span class="o">=</span><span class="p">[</span><span class="s2">"faucet"</span><span class="p">,</span> <span class="s2">"chair"</span><span class="p">],</span>
|
||||||
|
<span class="n">sample_nums</span><span class="o">=</span><span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span>
|
||||||
|
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">images_by_categories</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If we are not interested in any particular categories and just would like to render some random models from the whole dataset, we can set the number of models to be rendered in <code>sample_nums</code> and not specify any <code>categories</code>:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">random_model_images</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||||
|
<span class="n">sample_nums</span><span class="o">=</span><span class="p">[</span><span class="mi">3</span><span class="p">],</span>
|
||||||
|
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">random_model_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="4.-Render-R2N2-models-with-the-same-orientations-as-the-original-renderings-in-the-dataset">4. Render R2N2 models with the same orientations as the original renderings in the dataset<a class="anchor-link" href="#4.-Render-R2N2-models-with-the-same-orientations-as-the-original-renderings-in-the-dataset">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We can render R2N2 models the same way as we rendered ShapeNetCore models above. In addition, we can also render R2N2 models with the same orientations as the original renderings in the dataset. For this we will use R2N2's customized <code>render</code> function and a different type of PyTorch3D camera called <code>BlenderCamera</code>.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>In this example, we will render the seventh model with the same orientations as its second and third views. First we will retrieve R2N2's original renderings to compare with the result.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">original_rendering</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="p">[</span><span class="mi">6</span><span class="p">,[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">]][</span><span class="s2">"images"</span><span class="p">]</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">original_rendering</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Next, we will visualize PyTorch3d's renderings:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">r2n2_oriented_images</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||||
|
<span class="n">idxs</span><span class="o">=</span><span class="p">[</span><span class="mi">6</span><span class="p">],</span>
|
||||||
|
<span class="n">view_idxs</span><span class="o">=</span><span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">],</span>
|
||||||
|
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">r2n2_oriented_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="5.-Visualize-R2N2-models'-voxels">5. Visualize R2N2 models' voxels<a class="anchor-link" href="#5.-Visualize-R2N2-models'-voxels">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>R2N2 dataloader also returns models' voxels. We can visualize them by utilizing R2N2's <code>render_vox_to_mesh</code> function. This will cubify the voxels to a Meshes object, which will then be rendered.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>In this example we will visualize the tenth model in the dataset with the same orientation of its second and third views. First we will retrieve R2N2's original renderings to compare with the result.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">r2n2_model</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="p">[</span><span class="mi">9</span><span class="p">,[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">]]</span>
|
||||||
|
<span class="n">original_rendering</span> <span class="o">=</span> <span class="n">r2n2_model</span><span class="p">[</span><span class="s2">"images"</span><span class="p">]</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">original_rendering</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Next, we will pass the voxels to <code>render_vox_to_mesh</code>:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">vox_render</span> <span class="o">=</span> <span class="n">render_cubified_voxels</span><span class="p">(</span><span class="n">r2n2_model</span><span class="p">[</span><span class="s2">"voxels"</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">vox_render</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
849
tutorials/fit_textured_mesh.html
Normal file
849
tutorials/fit_textured_mesh.html
Normal file
@ -0,0 +1,849 @@
|
|||||||
|
<!DOCTYPE html><html lang=""><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>PyTorch3D · A library for deep learning with 3D data</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="A library for deep learning with 3D data"/><meta property="og:title" content="PyTorch3D · A library for deep learning with 3D data"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="A library for deep learning with 3D data"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||||
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||||
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||||
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||||
|
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||||
|
|
||||||
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
|
ga('send', 'pageview');
|
||||||
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="container docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span></span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Tutorials</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/">Overview</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">3D operators</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/deform_source_mesh_to_target_mesh">Fit Mesh</a></li><li class="navListItem"><a class="navItem" href="/tutorials/bundle_adjustment">Bundle Adjustment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Rendering</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/render_textured_meshes">Render Textured Meshes</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/tutorials/fit_textured_mesh">Fit a mesh with texture via rendering</a></li><li class="navListItem"><a class="navItem" href="/tutorials/camera_position_optimization_with_differentiable_rendering">Camera Position Optimization</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Dataloaders</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/dataloaders_ShapeNetCore_R2N2">Data loaders for ShapeNetCore and R2N2</a></li></ul></div></div></section></div><script>
|
||||||
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
|
var checkActiveCategory = true;
|
||||||
|
for (var i = 0; i < coll.length; i++) {
|
||||||
|
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||||
|
if (checkActiveCategory){
|
||||||
|
for (var j = 0; j < links.length; j++) {
|
||||||
|
if (links[j].classList.contains('navListItemActive')){
|
||||||
|
coll[i].nextElementSibling.classList.toggle('hide');
|
||||||
|
coll[i].childNodes[1].classList.toggle('rotate');
|
||||||
|
checkActiveCategory = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
coll[i].addEventListener('click', function() {
|
||||||
|
var arrow = this.childNodes[1];
|
||||||
|
arrow.classList.toggle('rotate');
|
||||||
|
var content = this.nextElementSibling;
|
||||||
|
content.classList.toggle('hide');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||||
|
createToggler('#tocToggler', 'body', 'tocActive');
|
||||||
|
|
||||||
|
var headings = document.querySelector('.toc-headings');
|
||||||
|
headings && headings.addEventListener('click', function(event) {
|
||||||
|
var el = event.target;
|
||||||
|
while(el !== headings){
|
||||||
|
if (el.tagName === 'A') {
|
||||||
|
document.body.classList.remove('tocActive');
|
||||||
|
break;
|
||||||
|
} else{
|
||||||
|
el = el.parentNode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, false);
|
||||||
|
|
||||||
|
function createToggler(togglerSelector, targetSelector, className) {
|
||||||
|
var toggler = document.querySelector(togglerSelector);
|
||||||
|
var target = document.querySelector(targetSelector);
|
||||||
|
|
||||||
|
if (!toggler) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
toggler.onclick = function(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
|
||||||
|
target.classList.toggle(className);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script></nav></div><div class="container mainContainer"><div class="wrapper"><div class="tutorialButtonsWrapper"><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="https://colab.research.google.com/github/facebookresearch/pytorch3d/blob/stable/docs/tutorials/fit_textured_mesh.ipynb" target="_blank"><img class="colabButton" align="left" src="/img/colab_icon.png"/>Run in Google Colab</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/fit_textured_mesh.ipynb" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Jupyter Notebook</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/fit_textured_mesh.py" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Source Code</a></div></div><div class="tutorialBody">
|
||||||
|
<script
|
||||||
|
src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.10/require.min.js">
|
||||||
|
</script>
|
||||||
|
<script
|
||||||
|
src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js">
|
||||||
|
</script>
|
||||||
|
<div class="notebook">
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h1 id="Fit-a-mesh-via-rendering">Fit a mesh via rendering<a class="anchor-link" href="#Fit-a-mesh-via-rendering">¶</a></h1><p>This tutorial shows how to:</p>
|
||||||
|
<ul>
|
||||||
|
<li>Load a mesh and textures from an <code>.obj</code> file. </li>
|
||||||
|
<li>Create a synthetic dataset by rendering a textured mesh from multiple viewpoints</li>
|
||||||
|
<li>Fit a mesh to the observed synthetic images using differential silhouette rendering</li>
|
||||||
|
<li>Fit a mesh and its textures using differential textured rendering</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="0.-Install-and-Import-modules">0. Install and Import modules<a class="anchor-link" href="#0.-Install-and-Import-modules">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If <code>torch</code>, <code>torchvision</code> and <code>pytorch3d</code> are not installed, run the following cell:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>pip install torch torchvision
|
||||||
|
<span class="kn">import</span> <span class="nn">sys</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">torch</span>
|
||||||
|
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">__version__</span><span class="o">==</span><span class="s1">'1.6.0+cu101'</span> <span class="ow">and</span> <span class="n">sys</span><span class="o">.</span><span class="n">platform</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'linux'</span><span class="p">):</span>
|
||||||
|
<span class="o">!</span>pip install pytorch3d
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="o">!</span>pip install <span class="s1">'git+https://github.com/facebookresearch/pytorch3d.git@stable'</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="kn">import</span> <span class="nn">os</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">torch</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">skimage.io</span> <span class="k">import</span> <span class="n">imread</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.utils</span> <span class="k">import</span> <span class="n">ico_sphere</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">tqdm.notebook</span> <span class="k">import</span> <span class="n">tqdm</span>
|
||||||
|
|
||||||
|
<span class="c1"># Util function for loading meshes</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.io</span> <span class="k">import</span> <span class="n">load_objs_as_meshes</span><span class="p">,</span> <span class="n">save_obj</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.loss</span> <span class="k">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">chamfer_distance</span><span class="p">,</span>
|
||||||
|
<span class="n">mesh_edge_loss</span><span class="p">,</span>
|
||||||
|
<span class="n">mesh_laplacian_smoothing</span><span class="p">,</span>
|
||||||
|
<span class="n">mesh_normal_consistency</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Data structures and functions for rendering</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.structures</span> <span class="k">import</span> <span class="n">Meshes</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.renderer</span> <span class="k">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">look_at_view_transform</span><span class="p">,</span>
|
||||||
|
<span class="n">OpenGLPerspectiveCameras</span><span class="p">,</span>
|
||||||
|
<span class="n">PointLights</span><span class="p">,</span>
|
||||||
|
<span class="n">DirectionalLights</span><span class="p">,</span>
|
||||||
|
<span class="n">Materials</span><span class="p">,</span>
|
||||||
|
<span class="n">RasterizationSettings</span><span class="p">,</span>
|
||||||
|
<span class="n">MeshRenderer</span><span class="p">,</span>
|
||||||
|
<span class="n">MeshRasterizer</span><span class="p">,</span>
|
||||||
|
<span class="n">SoftPhongShader</span><span class="p">,</span>
|
||||||
|
<span class="n">SoftSilhouetteShader</span><span class="p">,</span>
|
||||||
|
<span class="n">SoftPhongShader</span><span class="p">,</span>
|
||||||
|
<span class="n">TexturesVertex</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># add path for demo utils functions </span>
|
||||||
|
<span class="kn">import</span> <span class="nn">sys</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">os</span>
|
||||||
|
<span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="s1">''</span><span class="p">))</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If using <strong>Google Colab</strong>, fetch the utils file for plotting image grids:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py
|
||||||
|
<span class="kn">from</span> <span class="nn">plot_image_grid</span> <span class="k">import</span> <span class="n">image_grid</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>OR if running <strong>locally</strong> uncomment and run the following cell:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># from utils.plot_image_grid import image_grid</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h3 id="1.-Load-a-mesh-and-texture-file">1. Load a mesh and texture file<a class="anchor-link" href="#1.-Load-a-mesh-and-texture-file">¶</a></h3><p>Load an <code>.obj</code> file and it's associated <code>.mtl</code> file and create a <strong>Textures</strong> and <strong>Meshes</strong> object.</p>
|
||||||
|
<p><strong>Meshes</strong> is a unique datastructure provided in PyTorch3D for working with batches of meshes of different sizes.</p>
|
||||||
|
<p><strong>TexturesVertex</strong> is an auxillary datastructure for storing vertex rgb texture information about meshes.</p>
|
||||||
|
<p><strong>Meshes</strong> has several class methods which are used throughout the rendering pipeline.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If running this notebook using <strong>Google Colab</strong>, run the following cell to fetch the mesh obj and texture files and save it at the path <code>data/cow_mesh</code>:
|
||||||
|
If running locally, the data is already available at the correct path.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>mkdir -p data/cow_mesh
|
||||||
|
<span class="o">!</span>wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.obj
|
||||||
|
<span class="o">!</span>wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.mtl
|
||||||
|
<span class="o">!</span>wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow_texture.png
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Setup</span>
|
||||||
|
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
|
||||||
|
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cuda:0"</span><span class="p">)</span>
|
||||||
|
<span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">set_device</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cpu"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Set paths</span>
|
||||||
|
<span class="n">DATA_DIR</span> <span class="o">=</span> <span class="s2">"./data"</span>
|
||||||
|
<span class="n">obj_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">DATA_DIR</span><span class="p">,</span> <span class="s2">"cow_mesh/cow.obj"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Load obj file</span>
|
||||||
|
<span class="n">mesh</span> <span class="o">=</span> <span class="n">load_objs_as_meshes</span><span class="p">([</span><span class="n">obj_filename</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># We scale normalize and center the target mesh to fit in a sphere of radius 1 </span>
|
||||||
|
<span class="c1"># centered at (0,0,0). (scale, center) will be used to bring the predicted mesh </span>
|
||||||
|
<span class="c1"># to its original center and scale. Note that normalizing the target mesh, </span>
|
||||||
|
<span class="c1"># speeds up the optimization but is not necessary!</span>
|
||||||
|
<span class="n">verts</span> <span class="o">=</span> <span class="n">mesh</span><span class="o">.</span><span class="n">verts_packed</span><span class="p">()</span>
|
||||||
|
<span class="n">N</span> <span class="o">=</span> <span class="n">verts</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">center</span> <span class="o">=</span> <span class="n">verts</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||||
|
<span class="n">scale</span> <span class="o">=</span> <span class="nb">max</span><span class="p">((</span><span class="n">verts</span> <span class="o">-</span> <span class="n">center</span><span class="p">)</span><span class="o">.</span><span class="n">abs</span><span class="p">()</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="mi">0</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
|
||||||
|
<span class="n">mesh</span><span class="o">.</span><span class="n">offset_verts_</span><span class="p">(</span><span class="o">-</span><span class="n">center</span><span class="o">.</span><span class="n">expand</span><span class="p">(</span><span class="n">N</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
|
||||||
|
<span class="n">mesh</span><span class="o">.</span><span class="n">scale_verts_</span><span class="p">((</span><span class="mf">1.0</span> <span class="o">/</span> <span class="nb">float</span><span class="p">(</span><span class="n">scale</span><span class="p">)));</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="2.-Dataset-Creation">2. Dataset Creation<a class="anchor-link" href="#2.-Dataset-Creation">¶</a></h2><p>We sample different camera positions that encode multiple viewpoints of the cow. We create a renderer with a shader that performs texture map interpolation. We render a synthetic dataset of images of the textured cow mesh from multiple viewpoints.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># the number of different viewpoints from which we want to render the mesh.</span>
|
||||||
|
<span class="n">num_views</span> <span class="o">=</span> <span class="mi">20</span>
|
||||||
|
|
||||||
|
<span class="c1"># Get a batch of viewing angles. </span>
|
||||||
|
<span class="n">elev</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">360</span><span class="p">,</span> <span class="n">num_views</span><span class="p">)</span>
|
||||||
|
<span class="n">azim</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">180</span><span class="p">,</span> <span class="mi">180</span><span class="p">,</span> <span class="n">num_views</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Place a point light in front of the object. As mentioned above, the front of </span>
|
||||||
|
<span class="c1"># the cow is facing the -z direction. </span>
|
||||||
|
<span class="n">lights</span> <span class="o">=</span> <span class="n">PointLights</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">location</span><span class="o">=</span><span class="p">[[</span><span class="mf">0.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="o">-</span><span class="mf">3.0</span><span class="p">]])</span>
|
||||||
|
|
||||||
|
<span class="c1"># Initialize an OpenGL perspective camera that represents a batch of different </span>
|
||||||
|
<span class="c1"># viewing angles. All the cameras helper methods support mixed type inputs and </span>
|
||||||
|
<span class="c1"># broadcasting. So we can view the camera from the a distance of dist=2.7, and </span>
|
||||||
|
<span class="c1"># then specify elevation and azimuth angles for each viewpoint as tensors. </span>
|
||||||
|
<span class="n">R</span><span class="p">,</span> <span class="n">T</span> <span class="o">=</span> <span class="n">look_at_view_transform</span><span class="p">(</span><span class="n">dist</span><span class="o">=</span><span class="mf">2.7</span><span class="p">,</span> <span class="n">elev</span><span class="o">=</span><span class="n">elev</span><span class="p">,</span> <span class="n">azim</span><span class="o">=</span><span class="n">azim</span><span class="p">)</span>
|
||||||
|
<span class="n">cameras</span> <span class="o">=</span> <span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">,</span> <span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># We arbitrarily choose one particular view that will be used to visualize </span>
|
||||||
|
<span class="c1"># results</span>
|
||||||
|
<span class="n">camera</span> <span class="o">=</span> <span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">...</span><span class="p">],</span>
|
||||||
|
<span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">...</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="c1"># Define the settings for rasterization and shading. Here we set the output </span>
|
||||||
|
<span class="c1"># image to be of size 128X128. As we are rendering images for visualization </span>
|
||||||
|
<span class="c1"># purposes only we will set faces_per_pixel=1 and blur_radius=0.0. Refer to </span>
|
||||||
|
<span class="c1"># rasterize_meshes.py for explanations of these parameters. We also leave </span>
|
||||||
|
<span class="c1"># bin_size and max_faces_per_bin to their default values of None, which sets </span>
|
||||||
|
<span class="c1"># their values using huristics and ensures that the faster coarse-to-fine </span>
|
||||||
|
<span class="c1"># rasterization method is used. Refer to docs/notes/renderer.md for an </span>
|
||||||
|
<span class="c1"># explanation of the difference between naive and coarse-to-fine rasterization. </span>
|
||||||
|
<span class="n">raster_settings</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||||
|
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||||
|
<span class="n">blur_radius</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
|
||||||
|
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Create a phong renderer by composing a rasterizer and a shader. The textured </span>
|
||||||
|
<span class="c1"># phong shader will interpolate the texture uv coordinates for each vertex, </span>
|
||||||
|
<span class="c1"># sample from a texture image and apply the Phong lighting model</span>
|
||||||
|
<span class="n">renderer</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||||
|
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">shader</span><span class="o">=</span><span class="n">SoftPhongShader</span><span class="p">(</span>
|
||||||
|
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Create a batch of meshes by repeating the cow mesh and associated textures. </span>
|
||||||
|
<span class="c1"># Meshes has a useful `extend` method which allows us do this very easily. </span>
|
||||||
|
<span class="c1"># This also extends the textures. </span>
|
||||||
|
<span class="n">meshes</span> <span class="o">=</span> <span class="n">mesh</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">num_views</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Render the cow mesh from each viewing angle</span>
|
||||||
|
<span class="n">target_images</span> <span class="o">=</span> <span class="n">renderer</span><span class="p">(</span><span class="n">meshes</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Our multi-view cow dataset will be represented by these 2 lists of tensors,</span>
|
||||||
|
<span class="c1"># each of length num_views.</span>
|
||||||
|
<span class="n">target_rgb</span> <span class="o">=</span> <span class="p">[</span><span class="n">target_images</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="p">:</span><span class="mi">3</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_views</span><span class="p">)]</span>
|
||||||
|
<span class="n">target_cameras</span> <span class="o">=</span> <span class="p">[</span><span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">],</span>
|
||||||
|
<span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">])</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_views</span><span class="p">)]</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Visualize the dataset:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># RGB images</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">target_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Later in this tutorial, we will fit a mesh to the rendered RGB images, as well as to just images of just the cow silhouette. For the latter case, we will render a dataset of silhouette images. Most shaders in PyTorch3D will output an alpha channel along with the RGB image as a 4th channel in an RGBA image. The alpha channel encodes the probability that each pixel belongs to the foreground of the object. We contruct a soft silhouette shader to render this alpha channel.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rasterization settings for silhouette rendering </span>
|
||||||
|
<span class="n">sigma</span> <span class="o">=</span> <span class="mf">1e-4</span>
|
||||||
|
<span class="n">raster_settings_silhouette</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||||
|
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||||
|
<span class="n">blur_radius</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">1.</span> <span class="o">/</span> <span class="mf">1e-4</span> <span class="o">-</span> <span class="mf">1.</span><span class="p">)</span><span class="o">*</span><span class="n">sigma</span><span class="p">,</span>
|
||||||
|
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Silhouette renderer </span>
|
||||||
|
<span class="n">renderer_silhouette</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||||
|
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings_silhouette</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">shader</span><span class="o">=</span><span class="n">SoftSilhouetteShader</span><span class="p">()</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Render silhouette images. The 3rd channel of the rendering output is </span>
|
||||||
|
<span class="c1"># the alpha/silhouette channel</span>
|
||||||
|
<span class="n">silhouette_images</span> <span class="o">=</span> <span class="n">renderer_silhouette</span><span class="p">(</span><span class="n">meshes</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||||
|
<span class="n">target_silhouette</span> <span class="o">=</span> <span class="p">[</span><span class="n">silhouette_images</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_views</span><span class="p">)]</span>
|
||||||
|
|
||||||
|
<span class="c1"># Visualize silhouette images</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">silhouette_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="3.-Mesh-prediction-via-silhouette-rendering">3. Mesh prediction via silhouette rendering<a class="anchor-link" href="#3.-Mesh-prediction-via-silhouette-rendering">¶</a></h2><p>In the previous section, we created a dataset of images of multiple viewpoints of a cow. In this section, we predict a mesh by observing those target images without any knowledge of the ground truth cow mesh. We assume we know the position of the cameras and lighting.</p>
|
||||||
|
<p>We first define some helper functions to visualize the results of our mesh prediction:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Show a visualization comparing the rendered predicted mesh to the ground truth </span>
|
||||||
|
<span class="c1"># mesh</span>
|
||||||
|
<span class="k">def</span> <span class="nf">visualize_prediction</span><span class="p">(</span><span class="n">predicted_mesh</span><span class="p">,</span> <span class="n">renderer</span><span class="o">=</span><span class="n">renderer_silhouette</span><span class="p">,</span>
|
||||||
|
<span class="n">target_image</span><span class="o">=</span><span class="n">target_rgb</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">title</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span>
|
||||||
|
<span class="n">silhouette</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||||
|
<span class="n">inds</span> <span class="o">=</span> <span class="mi">3</span> <span class="k">if</span> <span class="n">silhouette</span> <span class="k">else</span> <span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
|
||||||
|
<span class="n">predicted_images</span> <span class="o">=</span> <span class="n">renderer</span><span class="p">(</span><span class="n">predicted_mesh</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">predicted_images</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="n">inds</span><span class="p">]</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
|
||||||
|
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">target_image</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">grid</span><span class="p">(</span><span class="s2">"off"</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">axis</span><span class="p">(</span><span class="s2">"off"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Plot losses as a function of optimization iteration</span>
|
||||||
|
<span class="k">def</span> <span class="nf">plot_losses</span><span class="p">(</span><span class="n">losses</span><span class="p">):</span>
|
||||||
|
<span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">13</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span>
|
||||||
|
<span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">gca</span><span class="p">()</span>
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">losses</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">l</span><span class="p">[</span><span class="s1">'values'</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="n">k</span> <span class="o">+</span> <span class="s2">" loss"</span><span class="p">)</span>
|
||||||
|
<span class="n">ax</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||||
|
<span class="n">ax</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s2">"Iteration"</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||||
|
<span class="n">ax</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s2">"Loss"</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||||
|
<span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s2">"Loss vs iterations"</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Starting from a sphere mesh, we will learn offsets of each vertex such that the predicted mesh silhouette is more similar to the target silhouette image at each optimization step. We begin by loading our initial sphere mesh:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># We initialize the source shape to be a sphere of radius 1. </span>
|
||||||
|
<span class="n">src_mesh</span> <span class="o">=</span> <span class="n">ico_sphere</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="n">device</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We create a new differentiable renderer for rendering the silhouette of our predicted mesh:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rasterization settings for differentiable rendering, where the blur_radius</span>
|
||||||
|
<span class="c1"># initialization is based on Liu et al, 'Soft Rasterizer: A Differentiable </span>
|
||||||
|
<span class="c1"># Renderer for Image-based 3D Reasoning', ICCV 2019</span>
|
||||||
|
<span class="n">sigma</span> <span class="o">=</span> <span class="mf">1e-4</span>
|
||||||
|
<span class="n">raster_settings_soft</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||||
|
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||||
|
<span class="n">blur_radius</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">1.</span> <span class="o">/</span> <span class="mf">1e-4</span> <span class="o">-</span> <span class="mf">1.</span><span class="p">)</span><span class="o">*</span><span class="n">sigma</span><span class="p">,</span>
|
||||||
|
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Silhouette renderer </span>
|
||||||
|
<span class="n">renderer_silhouette</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||||
|
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings_soft</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">shader</span><span class="o">=</span><span class="n">SoftSilhouetteShader</span><span class="p">()</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We initialize settings, losses, and the optimizer that will be used to iteratively fit our mesh to the target silhouettes:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Number of views to optimize over in each SGD iteration</span>
|
||||||
|
<span class="n">num_views_per_iteration</span> <span class="o">=</span> <span class="mi">2</span>
|
||||||
|
<span class="c1"># Number of optimization steps</span>
|
||||||
|
<span class="n">Niter</span> <span class="o">=</span> <span class="mi">2000</span>
|
||||||
|
<span class="c1"># Plot period for the losses</span>
|
||||||
|
<span class="n">plot_period</span> <span class="o">=</span> <span class="mi">250</span>
|
||||||
|
|
||||||
|
<span class="o">%</span><span class="k">matplotlib</span> inline
|
||||||
|
|
||||||
|
<span class="c1"># Optimize using rendered silhouette image loss, mesh edge loss, mesh normal </span>
|
||||||
|
<span class="c1"># consistency, and mesh laplacian smoothing</span>
|
||||||
|
<span class="n">losses</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"silhouette"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"edge"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"normal"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">0.01</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"laplacian"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># Losses to smooth / regularize the mesh shape</span>
|
||||||
|
<span class="k">def</span> <span class="nf">update_mesh_shape_prior_losses</span><span class="p">(</span><span class="n">mesh</span><span class="p">,</span> <span class="n">loss</span><span class="p">):</span>
|
||||||
|
<span class="c1"># and (b) the edge length of the predicted mesh</span>
|
||||||
|
<span class="n">loss</span><span class="p">[</span><span class="s2">"edge"</span><span class="p">]</span> <span class="o">=</span> <span class="n">mesh_edge_loss</span><span class="p">(</span><span class="n">mesh</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># mesh normal consistency</span>
|
||||||
|
<span class="n">loss</span><span class="p">[</span><span class="s2">"normal"</span><span class="p">]</span> <span class="o">=</span> <span class="n">mesh_normal_consistency</span><span class="p">(</span><span class="n">mesh</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># mesh laplacian smoothing</span>
|
||||||
|
<span class="n">loss</span><span class="p">[</span><span class="s2">"laplacian"</span><span class="p">]</span> <span class="o">=</span> <span class="n">mesh_laplacian_smoothing</span><span class="p">(</span><span class="n">mesh</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s2">"uniform"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># We will learn to deform the source mesh by offsetting its vertices</span>
|
||||||
|
<span class="c1"># The shape of the deform parameters is equal to the total number of vertices in</span>
|
||||||
|
<span class="c1"># src_mesh</span>
|
||||||
|
<span class="n">verts_shape</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">verts_packed</span><span class="p">()</span><span class="o">.</span><span class="n">shape</span>
|
||||||
|
<span class="n">deform_verts</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">(</span><span class="n">verts_shape</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># The optimizer</span>
|
||||||
|
<span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">([</span><span class="n">deform_verts</span><span class="p">],</span> <span class="n">lr</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We write an optimization loop to iteratively refine our predicted mesh from the sphere mesh into a mesh that matches the sillhouettes of the target images:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">loop</span> <span class="o">=</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">Niter</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">loop</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Initialize optimizer</span>
|
||||||
|
<span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># Deform the mesh</span>
|
||||||
|
<span class="n">new_src_mesh</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">offset_verts</span><span class="p">(</span><span class="n">deform_verts</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Losses to smooth /regularize the mesh shape</span>
|
||||||
|
<span class="n">loss</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">losses</span><span class="p">}</span>
|
||||||
|
<span class="n">update_mesh_shape_prior_losses</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">loss</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Compute the average silhouette loss over two random views, as the average </span>
|
||||||
|
<span class="c1"># squared L2 distance between the predicted silhouette and the target </span>
|
||||||
|
<span class="c1"># silhouette from our dataset</span>
|
||||||
|
<span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">permutation</span><span class="p">(</span><span class="n">num_views</span><span class="p">)</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[:</span><span class="n">num_views_per_iteration</span><span class="p">]:</span>
|
||||||
|
<span class="n">images_predicted</span> <span class="o">=</span> <span class="n">renderer_silhouette</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">target_cameras</span><span class="p">[</span><span class="n">j</span><span class="p">],</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||||
|
<span class="n">predicted_silhouette</span> <span class="o">=</span> <span class="n">images_predicted</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
|
||||||
|
<span class="n">loss_silhouette</span> <span class="o">=</span> <span class="p">((</span><span class="n">predicted_silhouette</span> <span class="o">-</span> <span class="n">target_silhouette</span><span class="p">[</span><span class="n">j</span><span class="p">])</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||||||
|
<span class="n">loss</span><span class="p">[</span><span class="s2">"silhouette"</span><span class="p">]</span> <span class="o">+=</span> <span class="n">loss_silhouette</span> <span class="o">/</span> <span class="n">num_views_per_iteration</span>
|
||||||
|
|
||||||
|
<span class="c1"># Weighted sum of the losses</span>
|
||||||
|
<span class="n">sum_loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">loss</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">sum_loss</span> <span class="o">+=</span> <span class="n">l</span> <span class="o">*</span> <span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"weight"</span><span class="p">]</span>
|
||||||
|
<span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"values"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Print the losses</span>
|
||||||
|
<span class="n">loop</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">"total_loss = </span><span class="si">%.6f</span><span class="s2">"</span> <span class="o">%</span> <span class="n">sum_loss</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Plot mesh</span>
|
||||||
|
<span class="k">if</span> <span class="n">i</span> <span class="o">%</span> <span class="n">plot_period</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="s2">"iter: </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">i</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="n">target_image</span><span class="o">=</span><span class="n">target_silhouette</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="c1"># Optimization step</span>
|
||||||
|
<span class="n">sum_loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
|
||||||
|
<span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="n">target_image</span><span class="o">=</span><span class="n">target_silhouette</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||||||
|
<span class="n">plot_losses</span><span class="p">(</span><span class="n">losses</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="3.-Mesh-and-texture-prediction-via-textured-rendering">3. Mesh and texture prediction via textured rendering<a class="anchor-link" href="#3.-Mesh-and-texture-prediction-via-textured-rendering">¶</a></h2><p>We can predict both the mesh and its texture if we add an additional loss based on the comparing a predicted rendered RGB image to the target image. As before, we start with a sphere mesh. We learn both translational offsets and RGB texture colors for each vertex in the sphere mesh. Since our loss is based on rendered RGB pixel values instead of just the silhouette, we use a <strong>SoftPhongShader</strong> instead of a <strong>SoftSilhouetteShader</strong>.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rasterization settings for differentiable rendering, where the blur_radius</span>
|
||||||
|
<span class="c1"># initialization is based on Liu et al, 'Soft Rasterizer: A Differentiable </span>
|
||||||
|
<span class="c1"># Renderer for Image-based 3D Reasoning', ICCV 2019</span>
|
||||||
|
<span class="n">sigma</span> <span class="o">=</span> <span class="mf">1e-4</span>
|
||||||
|
<span class="n">raster_settings_soft</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||||
|
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||||
|
<span class="n">blur_radius</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">1.</span> <span class="o">/</span> <span class="mf">1e-4</span> <span class="o">-</span> <span class="mf">1.</span><span class="p">)</span><span class="o">*</span><span class="n">sigma</span><span class="p">,</span>
|
||||||
|
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Differentiable soft renderer using per vertex RGB colors for texture</span>
|
||||||
|
<span class="n">renderer_textured</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||||
|
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings_soft</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">shader</span><span class="o">=</span><span class="n">SoftPhongShader</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We initialize settings, losses, and the optimizer that will be used to iteratively fit our mesh to the target RGB images:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Number of views to optimize over in each SGD iteration</span>
|
||||||
|
<span class="n">num_views_per_iteration</span> <span class="o">=</span> <span class="mi">2</span>
|
||||||
|
<span class="c1"># Number of optimization steps</span>
|
||||||
|
<span class="n">Niter</span> <span class="o">=</span> <span class="mi">2000</span>
|
||||||
|
<span class="c1"># Plot period for the losses</span>
|
||||||
|
<span class="n">plot_period</span> <span class="o">=</span> <span class="mi">250</span>
|
||||||
|
|
||||||
|
<span class="o">%</span><span class="k">matplotlib</span> inline
|
||||||
|
|
||||||
|
<span class="c1"># Optimize using rendered RGB image loss, rendered silhouette image loss, mesh </span>
|
||||||
|
<span class="c1"># edge loss, mesh normal consistency, and mesh laplacian smoothing</span>
|
||||||
|
<span class="n">losses</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"rgb"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"silhouette"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"edge"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"normal"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">0.01</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"laplacian"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># We will learn to deform the source mesh by offsetting its vertices</span>
|
||||||
|
<span class="c1"># The shape of the deform parameters is equal to the total number of vertices in </span>
|
||||||
|
<span class="c1"># src_mesh</span>
|
||||||
|
<span class="n">verts_shape</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">verts_packed</span><span class="p">()</span><span class="o">.</span><span class="n">shape</span>
|
||||||
|
<span class="n">deform_verts</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">(</span><span class="n">verts_shape</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># We will also learn per vertex colors for our sphere mesh that define texture </span>
|
||||||
|
<span class="c1"># of the mesh</span>
|
||||||
|
<span class="n">sphere_verts_rgb</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="n">verts_shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">3</span><span class="p">],</span> <span class="mf">0.5</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># The optimizer</span>
|
||||||
|
<span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">([</span><span class="n">deform_verts</span><span class="p">,</span> <span class="n">sphere_verts_rgb</span><span class="p">],</span> <span class="n">lr</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We write an optimization loop to iteratively refine our predicted mesh and its vertex colors from the sphere mesh into a mesh that matches the target images:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">loop</span> <span class="o">=</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">Niter</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">loop</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Initialize optimizer</span>
|
||||||
|
<span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># Deform the mesh</span>
|
||||||
|
<span class="n">new_src_mesh</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">offset_verts</span><span class="p">(</span><span class="n">deform_verts</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Add per vertex colors to texture the mesh</span>
|
||||||
|
<span class="n">new_src_mesh</span><span class="o">.</span><span class="n">textures</span> <span class="o">=</span> <span class="n">TexturesVertex</span><span class="p">(</span><span class="n">verts_features</span><span class="o">=</span><span class="n">sphere_verts_rgb</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Losses to smooth /regularize the mesh shape</span>
|
||||||
|
<span class="n">loss</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">losses</span><span class="p">}</span>
|
||||||
|
<span class="n">update_mesh_shape_prior_losses</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">loss</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Randomly select two views to optimize over in this iteration. Compared</span>
|
||||||
|
<span class="c1"># to using just one view, this helps resolve ambiguities between updating</span>
|
||||||
|
<span class="c1"># mesh shape vs. updating mesh texture</span>
|
||||||
|
<span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">permutation</span><span class="p">(</span><span class="n">num_views</span><span class="p">)</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[:</span><span class="n">num_views_per_iteration</span><span class="p">]:</span>
|
||||||
|
<span class="n">images_predicted</span> <span class="o">=</span> <span class="n">renderer_textured</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">target_cameras</span><span class="p">[</span><span class="n">j</span><span class="p">],</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Squared L2 distance between the predicted silhouette and the target </span>
|
||||||
|
<span class="c1"># silhouette from our dataset</span>
|
||||||
|
<span class="n">predicted_silhouette</span> <span class="o">=</span> <span class="n">images_predicted</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
|
||||||
|
<span class="n">loss_silhouette</span> <span class="o">=</span> <span class="p">((</span><span class="n">predicted_silhouette</span> <span class="o">-</span> <span class="n">target_silhouette</span><span class="p">[</span><span class="n">j</span><span class="p">])</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||||||
|
<span class="n">loss</span><span class="p">[</span><span class="s2">"silhouette"</span><span class="p">]</span> <span class="o">+=</span> <span class="n">loss_silhouette</span> <span class="o">/</span> <span class="n">num_views_per_iteration</span>
|
||||||
|
|
||||||
|
<span class="c1"># Squared L2 distance between the predicted RGB image and the target </span>
|
||||||
|
<span class="c1"># image from our dataset</span>
|
||||||
|
<span class="n">predicted_rgb</span> <span class="o">=</span> <span class="n">images_predicted</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="p">:</span><span class="mi">3</span><span class="p">]</span>
|
||||||
|
<span class="n">loss_rgb</span> <span class="o">=</span> <span class="p">((</span><span class="n">predicted_rgb</span> <span class="o">-</span> <span class="n">target_rgb</span><span class="p">[</span><span class="n">j</span><span class="p">])</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||||||
|
<span class="n">loss</span><span class="p">[</span><span class="s2">"rgb"</span><span class="p">]</span> <span class="o">+=</span> <span class="n">loss_rgb</span> <span class="o">/</span> <span class="n">num_views_per_iteration</span>
|
||||||
|
|
||||||
|
<span class="c1"># Weighted sum of the losses</span>
|
||||||
|
<span class="n">sum_loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">loss</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">sum_loss</span> <span class="o">+=</span> <span class="n">l</span> <span class="o">*</span> <span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"weight"</span><span class="p">]</span>
|
||||||
|
<span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"values"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Print the losses</span>
|
||||||
|
<span class="n">loop</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">"total_loss = </span><span class="si">%.6f</span><span class="s2">"</span> <span class="o">%</span> <span class="n">sum_loss</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Plot mesh</span>
|
||||||
|
<span class="k">if</span> <span class="n">i</span> <span class="o">%</span> <span class="n">plot_period</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">renderer</span><span class="o">=</span><span class="n">renderer_textured</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="s2">"iter: </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">i</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Optimization step</span>
|
||||||
|
<span class="n">sum_loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
|
||||||
|
<span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">renderer</span><span class="o">=</span><span class="n">renderer_textured</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||||
|
<span class="n">plot_losses</span><span class="p">(</span><span class="n">losses</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Save the final predicted mesh:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="4.-Save-the-final-predicted-mesh">4. Save the final predicted mesh<a class="anchor-link" href="#4.-Save-the-final-predicted-mesh">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Fetch the verts and faces of the final predicted mesh</span>
|
||||||
|
<span class="n">final_verts</span><span class="p">,</span> <span class="n">final_faces</span> <span class="o">=</span> <span class="n">new_src_mesh</span><span class="o">.</span><span class="n">get_mesh_verts_faces</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Scale normalize back to the original target size</span>
|
||||||
|
<span class="n">final_verts</span> <span class="o">=</span> <span class="n">final_verts</span> <span class="o">*</span> <span class="n">scale</span> <span class="o">+</span> <span class="n">center</span>
|
||||||
|
|
||||||
|
<span class="c1"># Store the predicted mesh using save_obj</span>
|
||||||
|
<span class="n">final_obj</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s1">'./'</span><span class="p">,</span> <span class="s1">'final_model.obj'</span><span class="p">)</span>
|
||||||
|
<span class="n">save_obj</span><span class="p">(</span><span class="n">final_obj</span><span class="p">,</span> <span class="n">final_verts</span><span class="p">,</span> <span class="n">final_faces</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="5.-Conclusion">5. Conclusion<a class="anchor-link" href="#5.-Conclusion">¶</a></h2><p>In this tutorial, we learned how to load a textured mesh from an obj file, create a synthetic dataset by rendering the mesh from multiple viewpoints. We showed how to set up an optimization loop to fit a mesh to the observed dataset images based on a rendered silhouette loss. We then augmented this optimization loop with an additional loss based on rendered RGB images, which allowed us to predict both a mesh and its texture.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
849
tutorials/fit_textured_mesh/index.html
Normal file
849
tutorials/fit_textured_mesh/index.html
Normal file
@ -0,0 +1,849 @@
|
|||||||
|
<!DOCTYPE html><html lang=""><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>PyTorch3D · A library for deep learning with 3D data</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="A library for deep learning with 3D data"/><meta property="og:title" content="PyTorch3D · A library for deep learning with 3D data"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="A library for deep learning with 3D data"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||||
|
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||||
|
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||||
|
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||||
|
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||||
|
|
||||||
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
|
ga('send', 'pageview');
|
||||||
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="container docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span></span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Tutorials</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/">Overview</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">3D operators</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/deform_source_mesh_to_target_mesh">Fit Mesh</a></li><li class="navListItem"><a class="navItem" href="/tutorials/bundle_adjustment">Bundle Adjustment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Rendering</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/render_textured_meshes">Render Textured Meshes</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/tutorials/fit_textured_mesh">Fit a mesh with texture via rendering</a></li><li class="navListItem"><a class="navItem" href="/tutorials/camera_position_optimization_with_differentiable_rendering">Camera Position Optimization</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Dataloaders</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/dataloaders_ShapeNetCore_R2N2">Data loaders for ShapeNetCore and R2N2</a></li></ul></div></div></section></div><script>
|
||||||
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
|
var checkActiveCategory = true;
|
||||||
|
for (var i = 0; i < coll.length; i++) {
|
||||||
|
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||||
|
if (checkActiveCategory){
|
||||||
|
for (var j = 0; j < links.length; j++) {
|
||||||
|
if (links[j].classList.contains('navListItemActive')){
|
||||||
|
coll[i].nextElementSibling.classList.toggle('hide');
|
||||||
|
coll[i].childNodes[1].classList.toggle('rotate');
|
||||||
|
checkActiveCategory = false;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
coll[i].addEventListener('click', function() {
|
||||||
|
var arrow = this.childNodes[1];
|
||||||
|
arrow.classList.toggle('rotate');
|
||||||
|
var content = this.nextElementSibling;
|
||||||
|
content.classList.toggle('hide');
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
document.addEventListener('DOMContentLoaded', function() {
|
||||||
|
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||||
|
createToggler('#tocToggler', 'body', 'tocActive');
|
||||||
|
|
||||||
|
var headings = document.querySelector('.toc-headings');
|
||||||
|
headings && headings.addEventListener('click', function(event) {
|
||||||
|
var el = event.target;
|
||||||
|
while(el !== headings){
|
||||||
|
if (el.tagName === 'A') {
|
||||||
|
document.body.classList.remove('tocActive');
|
||||||
|
break;
|
||||||
|
} else{
|
||||||
|
el = el.parentNode;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, false);
|
||||||
|
|
||||||
|
function createToggler(togglerSelector, targetSelector, className) {
|
||||||
|
var toggler = document.querySelector(togglerSelector);
|
||||||
|
var target = document.querySelector(targetSelector);
|
||||||
|
|
||||||
|
if (!toggler) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
toggler.onclick = function(event) {
|
||||||
|
event.preventDefault();
|
||||||
|
|
||||||
|
target.classList.toggle(className);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
});
|
||||||
|
</script></nav></div><div class="container mainContainer"><div class="wrapper"><div class="tutorialButtonsWrapper"><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="https://colab.research.google.com/github/facebookresearch/pytorch3d/blob/stable/docs/tutorials/fit_textured_mesh.ipynb" target="_blank"><img class="colabButton" align="left" src="/img/colab_icon.png"/>Run in Google Colab</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/fit_textured_mesh.ipynb" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Jupyter Notebook</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/fit_textured_mesh.py" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Source Code</a></div></div><div class="tutorialBody">
|
||||||
|
<script
|
||||||
|
src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.10/require.min.js">
|
||||||
|
</script>
|
||||||
|
<script
|
||||||
|
src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js">
|
||||||
|
</script>
|
||||||
|
<div class="notebook">
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h1 id="Fit-a-mesh-via-rendering">Fit a mesh via rendering<a class="anchor-link" href="#Fit-a-mesh-via-rendering">¶</a></h1><p>This tutorial shows how to:</p>
|
||||||
|
<ul>
|
||||||
|
<li>Load a mesh and textures from an <code>.obj</code> file. </li>
|
||||||
|
<li>Create a synthetic dataset by rendering a textured mesh from multiple viewpoints</li>
|
||||||
|
<li>Fit a mesh to the observed synthetic images using differential silhouette rendering</li>
|
||||||
|
<li>Fit a mesh and its textures using differential textured rendering</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="0.-Install-and-Import-modules">0. Install and Import modules<a class="anchor-link" href="#0.-Install-and-Import-modules">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If <code>torch</code>, <code>torchvision</code> and <code>pytorch3d</code> are not installed, run the following cell:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>pip install torch torchvision
|
||||||
|
<span class="kn">import</span> <span class="nn">sys</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">torch</span>
|
||||||
|
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">__version__</span><span class="o">==</span><span class="s1">'1.6.0+cu101'</span> <span class="ow">and</span> <span class="n">sys</span><span class="o">.</span><span class="n">platform</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'linux'</span><span class="p">):</span>
|
||||||
|
<span class="o">!</span>pip install pytorch3d
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="o">!</span>pip install <span class="s1">'git+https://github.com/facebookresearch/pytorch3d.git@stable'</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="kn">import</span> <span class="nn">os</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">torch</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">skimage.io</span> <span class="k">import</span> <span class="n">imread</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.utils</span> <span class="k">import</span> <span class="n">ico_sphere</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">tqdm.notebook</span> <span class="k">import</span> <span class="n">tqdm</span>
|
||||||
|
|
||||||
|
<span class="c1"># Util function for loading meshes</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.io</span> <span class="k">import</span> <span class="n">load_objs_as_meshes</span><span class="p">,</span> <span class="n">save_obj</span>
|
||||||
|
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.loss</span> <span class="k">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">chamfer_distance</span><span class="p">,</span>
|
||||||
|
<span class="n">mesh_edge_loss</span><span class="p">,</span>
|
||||||
|
<span class="n">mesh_laplacian_smoothing</span><span class="p">,</span>
|
||||||
|
<span class="n">mesh_normal_consistency</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Data structures and functions for rendering</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.structures</span> <span class="k">import</span> <span class="n">Meshes</span>
|
||||||
|
<span class="kn">from</span> <span class="nn">pytorch3d.renderer</span> <span class="k">import</span> <span class="p">(</span>
|
||||||
|
<span class="n">look_at_view_transform</span><span class="p">,</span>
|
||||||
|
<span class="n">OpenGLPerspectiveCameras</span><span class="p">,</span>
|
||||||
|
<span class="n">PointLights</span><span class="p">,</span>
|
||||||
|
<span class="n">DirectionalLights</span><span class="p">,</span>
|
||||||
|
<span class="n">Materials</span><span class="p">,</span>
|
||||||
|
<span class="n">RasterizationSettings</span><span class="p">,</span>
|
||||||
|
<span class="n">MeshRenderer</span><span class="p">,</span>
|
||||||
|
<span class="n">MeshRasterizer</span><span class="p">,</span>
|
||||||
|
<span class="n">SoftPhongShader</span><span class="p">,</span>
|
||||||
|
<span class="n">SoftSilhouetteShader</span><span class="p">,</span>
|
||||||
|
<span class="n">SoftPhongShader</span><span class="p">,</span>
|
||||||
|
<span class="n">TexturesVertex</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># add path for demo utils functions </span>
|
||||||
|
<span class="kn">import</span> <span class="nn">sys</span>
|
||||||
|
<span class="kn">import</span> <span class="nn">os</span>
|
||||||
|
<span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="s1">''</span><span class="p">))</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If using <strong>Google Colab</strong>, fetch the utils file for plotting image grids:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py
|
||||||
|
<span class="kn">from</span> <span class="nn">plot_image_grid</span> <span class="k">import</span> <span class="n">image_grid</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>OR if running <strong>locally</strong> uncomment and run the following cell:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># from utils.plot_image_grid import image_grid</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h3 id="1.-Load-a-mesh-and-texture-file">1. Load a mesh and texture file<a class="anchor-link" href="#1.-Load-a-mesh-and-texture-file">¶</a></h3><p>Load an <code>.obj</code> file and it's associated <code>.mtl</code> file and create a <strong>Textures</strong> and <strong>Meshes</strong> object.</p>
|
||||||
|
<p><strong>Meshes</strong> is a unique datastructure provided in PyTorch3D for working with batches of meshes of different sizes.</p>
|
||||||
|
<p><strong>TexturesVertex</strong> is an auxillary datastructure for storing vertex rgb texture information about meshes.</p>
|
||||||
|
<p><strong>Meshes</strong> has several class methods which are used throughout the rendering pipeline.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>If running this notebook using <strong>Google Colab</strong>, run the following cell to fetch the mesh obj and texture files and save it at the path <code>data/cow_mesh</code>:
|
||||||
|
If running locally, the data is already available at the correct path.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>mkdir -p data/cow_mesh
|
||||||
|
<span class="o">!</span>wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.obj
|
||||||
|
<span class="o">!</span>wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.mtl
|
||||||
|
<span class="o">!</span>wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow_texture.png
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Setup</span>
|
||||||
|
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
|
||||||
|
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cuda:0"</span><span class="p">)</span>
|
||||||
|
<span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">set_device</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
<span class="k">else</span><span class="p">:</span>
|
||||||
|
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cpu"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Set paths</span>
|
||||||
|
<span class="n">DATA_DIR</span> <span class="o">=</span> <span class="s2">"./data"</span>
|
||||||
|
<span class="n">obj_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">DATA_DIR</span><span class="p">,</span> <span class="s2">"cow_mesh/cow.obj"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Load obj file</span>
|
||||||
|
<span class="n">mesh</span> <span class="o">=</span> <span class="n">load_objs_as_meshes</span><span class="p">([</span><span class="n">obj_filename</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># We scale normalize and center the target mesh to fit in a sphere of radius 1 </span>
|
||||||
|
<span class="c1"># centered at (0,0,0). (scale, center) will be used to bring the predicted mesh </span>
|
||||||
|
<span class="c1"># to its original center and scale. Note that normalizing the target mesh, </span>
|
||||||
|
<span class="c1"># speeds up the optimization but is not necessary!</span>
|
||||||
|
<span class="n">verts</span> <span class="o">=</span> <span class="n">mesh</span><span class="o">.</span><span class="n">verts_packed</span><span class="p">()</span>
|
||||||
|
<span class="n">N</span> <span class="o">=</span> <span class="n">verts</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||||
|
<span class="n">center</span> <span class="o">=</span> <span class="n">verts</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||||
|
<span class="n">scale</span> <span class="o">=</span> <span class="nb">max</span><span class="p">((</span><span class="n">verts</span> <span class="o">-</span> <span class="n">center</span><span class="p">)</span><span class="o">.</span><span class="n">abs</span><span class="p">()</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="mi">0</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
|
||||||
|
<span class="n">mesh</span><span class="o">.</span><span class="n">offset_verts_</span><span class="p">(</span><span class="o">-</span><span class="n">center</span><span class="o">.</span><span class="n">expand</span><span class="p">(</span><span class="n">N</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
|
||||||
|
<span class="n">mesh</span><span class="o">.</span><span class="n">scale_verts_</span><span class="p">((</span><span class="mf">1.0</span> <span class="o">/</span> <span class="nb">float</span><span class="p">(</span><span class="n">scale</span><span class="p">)));</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="2.-Dataset-Creation">2. Dataset Creation<a class="anchor-link" href="#2.-Dataset-Creation">¶</a></h2><p>We sample different camera positions that encode multiple viewpoints of the cow. We create a renderer with a shader that performs texture map interpolation. We render a synthetic dataset of images of the textured cow mesh from multiple viewpoints.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># the number of different viewpoints from which we want to render the mesh.</span>
|
||||||
|
<span class="n">num_views</span> <span class="o">=</span> <span class="mi">20</span>
|
||||||
|
|
||||||
|
<span class="c1"># Get a batch of viewing angles. </span>
|
||||||
|
<span class="n">elev</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">360</span><span class="p">,</span> <span class="n">num_views</span><span class="p">)</span>
|
||||||
|
<span class="n">azim</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">180</span><span class="p">,</span> <span class="mi">180</span><span class="p">,</span> <span class="n">num_views</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Place a point light in front of the object. As mentioned above, the front of </span>
|
||||||
|
<span class="c1"># the cow is facing the -z direction. </span>
|
||||||
|
<span class="n">lights</span> <span class="o">=</span> <span class="n">PointLights</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">location</span><span class="o">=</span><span class="p">[[</span><span class="mf">0.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="o">-</span><span class="mf">3.0</span><span class="p">]])</span>
|
||||||
|
|
||||||
|
<span class="c1"># Initialize an OpenGL perspective camera that represents a batch of different </span>
|
||||||
|
<span class="c1"># viewing angles. All the cameras helper methods support mixed type inputs and </span>
|
||||||
|
<span class="c1"># broadcasting. So we can view the camera from the a distance of dist=2.7, and </span>
|
||||||
|
<span class="c1"># then specify elevation and azimuth angles for each viewpoint as tensors. </span>
|
||||||
|
<span class="n">R</span><span class="p">,</span> <span class="n">T</span> <span class="o">=</span> <span class="n">look_at_view_transform</span><span class="p">(</span><span class="n">dist</span><span class="o">=</span><span class="mf">2.7</span><span class="p">,</span> <span class="n">elev</span><span class="o">=</span><span class="n">elev</span><span class="p">,</span> <span class="n">azim</span><span class="o">=</span><span class="n">azim</span><span class="p">)</span>
|
||||||
|
<span class="n">cameras</span> <span class="o">=</span> <span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">,</span> <span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># We arbitrarily choose one particular view that will be used to visualize </span>
|
||||||
|
<span class="c1"># results</span>
|
||||||
|
<span class="n">camera</span> <span class="o">=</span> <span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">...</span><span class="p">],</span>
|
||||||
|
<span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">...</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="c1"># Define the settings for rasterization and shading. Here we set the output </span>
|
||||||
|
<span class="c1"># image to be of size 128X128. As we are rendering images for visualization </span>
|
||||||
|
<span class="c1"># purposes only we will set faces_per_pixel=1 and blur_radius=0.0. Refer to </span>
|
||||||
|
<span class="c1"># rasterize_meshes.py for explanations of these parameters. We also leave </span>
|
||||||
|
<span class="c1"># bin_size and max_faces_per_bin to their default values of None, which sets </span>
|
||||||
|
<span class="c1"># their values using huristics and ensures that the faster coarse-to-fine </span>
|
||||||
|
<span class="c1"># rasterization method is used. Refer to docs/notes/renderer.md for an </span>
|
||||||
|
<span class="c1"># explanation of the difference between naive and coarse-to-fine rasterization. </span>
|
||||||
|
<span class="n">raster_settings</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||||
|
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||||
|
<span class="n">blur_radius</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
|
||||||
|
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Create a phong renderer by composing a rasterizer and a shader. The textured </span>
|
||||||
|
<span class="c1"># phong shader will interpolate the texture uv coordinates for each vertex, </span>
|
||||||
|
<span class="c1"># sample from a texture image and apply the Phong lighting model</span>
|
||||||
|
<span class="n">renderer</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||||
|
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">shader</span><span class="o">=</span><span class="n">SoftPhongShader</span><span class="p">(</span>
|
||||||
|
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Create a batch of meshes by repeating the cow mesh and associated textures. </span>
|
||||||
|
<span class="c1"># Meshes has a useful `extend` method which allows us do this very easily. </span>
|
||||||
|
<span class="c1"># This also extends the textures. </span>
|
||||||
|
<span class="n">meshes</span> <span class="o">=</span> <span class="n">mesh</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">num_views</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Render the cow mesh from each viewing angle</span>
|
||||||
|
<span class="n">target_images</span> <span class="o">=</span> <span class="n">renderer</span><span class="p">(</span><span class="n">meshes</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Our multi-view cow dataset will be represented by these 2 lists of tensors,</span>
|
||||||
|
<span class="c1"># each of length num_views.</span>
|
||||||
|
<span class="n">target_rgb</span> <span class="o">=</span> <span class="p">[</span><span class="n">target_images</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="p">:</span><span class="mi">3</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_views</span><span class="p">)]</span>
|
||||||
|
<span class="n">target_cameras</span> <span class="o">=</span> <span class="p">[</span><span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">],</span>
|
||||||
|
<span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">])</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_views</span><span class="p">)]</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Visualize the dataset:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># RGB images</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">target_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Later in this tutorial, we will fit a mesh to the rendered RGB images, as well as to just images of just the cow silhouette. For the latter case, we will render a dataset of silhouette images. Most shaders in PyTorch3D will output an alpha channel along with the RGB image as a 4th channel in an RGBA image. The alpha channel encodes the probability that each pixel belongs to the foreground of the object. We contruct a soft silhouette shader to render this alpha channel.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rasterization settings for silhouette rendering </span>
|
||||||
|
<span class="n">sigma</span> <span class="o">=</span> <span class="mf">1e-4</span>
|
||||||
|
<span class="n">raster_settings_silhouette</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||||
|
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||||
|
<span class="n">blur_radius</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">1.</span> <span class="o">/</span> <span class="mf">1e-4</span> <span class="o">-</span> <span class="mf">1.</span><span class="p">)</span><span class="o">*</span><span class="n">sigma</span><span class="p">,</span>
|
||||||
|
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Silhouette renderer </span>
|
||||||
|
<span class="n">renderer_silhouette</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||||
|
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings_silhouette</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">shader</span><span class="o">=</span><span class="n">SoftSilhouetteShader</span><span class="p">()</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Render silhouette images. The 3rd channel of the rendering output is </span>
|
||||||
|
<span class="c1"># the alpha/silhouette channel</span>
|
||||||
|
<span class="n">silhouette_images</span> <span class="o">=</span> <span class="n">renderer_silhouette</span><span class="p">(</span><span class="n">meshes</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||||
|
<span class="n">target_silhouette</span> <span class="o">=</span> <span class="p">[</span><span class="n">silhouette_images</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_views</span><span class="p">)]</span>
|
||||||
|
|
||||||
|
<span class="c1"># Visualize silhouette images</span>
|
||||||
|
<span class="n">image_grid</span><span class="p">(</span><span class="n">silhouette_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="3.-Mesh-prediction-via-silhouette-rendering">3. Mesh prediction via silhouette rendering<a class="anchor-link" href="#3.-Mesh-prediction-via-silhouette-rendering">¶</a></h2><p>In the previous section, we created a dataset of images of multiple viewpoints of a cow. In this section, we predict a mesh by observing those target images without any knowledge of the ground truth cow mesh. We assume we know the position of the cameras and lighting.</p>
|
||||||
|
<p>We first define some helper functions to visualize the results of our mesh prediction:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Show a visualization comparing the rendered predicted mesh to the ground truth </span>
|
||||||
|
<span class="c1"># mesh</span>
|
||||||
|
<span class="k">def</span> <span class="nf">visualize_prediction</span><span class="p">(</span><span class="n">predicted_mesh</span><span class="p">,</span> <span class="n">renderer</span><span class="o">=</span><span class="n">renderer_silhouette</span><span class="p">,</span>
|
||||||
|
<span class="n">target_image</span><span class="o">=</span><span class="n">target_rgb</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">title</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span>
|
||||||
|
<span class="n">silhouette</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||||
|
<span class="n">inds</span> <span class="o">=</span> <span class="mi">3</span> <span class="k">if</span> <span class="n">silhouette</span> <span class="k">else</span> <span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
|
||||||
|
<span class="n">predicted_images</span> <span class="o">=</span> <span class="n">renderer</span><span class="p">(</span><span class="n">predicted_mesh</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">predicted_images</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="n">inds</span><span class="p">]</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
|
||||||
|
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">target_image</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">grid</span><span class="p">(</span><span class="s2">"off"</span><span class="p">)</span>
|
||||||
|
<span class="n">plt</span><span class="o">.</span><span class="n">axis</span><span class="p">(</span><span class="s2">"off"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Plot losses as a function of optimization iteration</span>
|
||||||
|
<span class="k">def</span> <span class="nf">plot_losses</span><span class="p">(</span><span class="n">losses</span><span class="p">):</span>
|
||||||
|
<span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">13</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span>
|
||||||
|
<span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">gca</span><span class="p">()</span>
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">losses</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">l</span><span class="p">[</span><span class="s1">'values'</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="n">k</span> <span class="o">+</span> <span class="s2">" loss"</span><span class="p">)</span>
|
||||||
|
<span class="n">ax</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||||
|
<span class="n">ax</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s2">"Iteration"</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||||
|
<span class="n">ax</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s2">"Loss"</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||||
|
<span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s2">"Loss vs iterations"</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Starting from a sphere mesh, we will learn offsets of each vertex such that the predicted mesh silhouette is more similar to the target silhouette image at each optimization step. We begin by loading our initial sphere mesh:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># We initialize the source shape to be a sphere of radius 1. </span>
|
||||||
|
<span class="n">src_mesh</span> <span class="o">=</span> <span class="n">ico_sphere</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="n">device</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We create a new differentiable renderer for rendering the silhouette of our predicted mesh:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rasterization settings for differentiable rendering, where the blur_radius</span>
|
||||||
|
<span class="c1"># initialization is based on Liu et al, 'Soft Rasterizer: A Differentiable </span>
|
||||||
|
<span class="c1"># Renderer for Image-based 3D Reasoning', ICCV 2019</span>
|
||||||
|
<span class="n">sigma</span> <span class="o">=</span> <span class="mf">1e-4</span>
|
||||||
|
<span class="n">raster_settings_soft</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||||
|
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||||
|
<span class="n">blur_radius</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">1.</span> <span class="o">/</span> <span class="mf">1e-4</span> <span class="o">-</span> <span class="mf">1.</span><span class="p">)</span><span class="o">*</span><span class="n">sigma</span><span class="p">,</span>
|
||||||
|
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Silhouette renderer </span>
|
||||||
|
<span class="n">renderer_silhouette</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||||
|
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings_soft</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">shader</span><span class="o">=</span><span class="n">SoftSilhouetteShader</span><span class="p">()</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We initialize settings, losses, and the optimizer that will be used to iteratively fit our mesh to the target silhouettes:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Number of views to optimize over in each SGD iteration</span>
|
||||||
|
<span class="n">num_views_per_iteration</span> <span class="o">=</span> <span class="mi">2</span>
|
||||||
|
<span class="c1"># Number of optimization steps</span>
|
||||||
|
<span class="n">Niter</span> <span class="o">=</span> <span class="mi">2000</span>
|
||||||
|
<span class="c1"># Plot period for the losses</span>
|
||||||
|
<span class="n">plot_period</span> <span class="o">=</span> <span class="mi">250</span>
|
||||||
|
|
||||||
|
<span class="o">%</span><span class="k">matplotlib</span> inline
|
||||||
|
|
||||||
|
<span class="c1"># Optimize using rendered silhouette image loss, mesh edge loss, mesh normal </span>
|
||||||
|
<span class="c1"># consistency, and mesh laplacian smoothing</span>
|
||||||
|
<span class="n">losses</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"silhouette"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"edge"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"normal"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">0.01</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"laplacian"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># Losses to smooth / regularize the mesh shape</span>
|
||||||
|
<span class="k">def</span> <span class="nf">update_mesh_shape_prior_losses</span><span class="p">(</span><span class="n">mesh</span><span class="p">,</span> <span class="n">loss</span><span class="p">):</span>
|
||||||
|
<span class="c1"># and (b) the edge length of the predicted mesh</span>
|
||||||
|
<span class="n">loss</span><span class="p">[</span><span class="s2">"edge"</span><span class="p">]</span> <span class="o">=</span> <span class="n">mesh_edge_loss</span><span class="p">(</span><span class="n">mesh</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># mesh normal consistency</span>
|
||||||
|
<span class="n">loss</span><span class="p">[</span><span class="s2">"normal"</span><span class="p">]</span> <span class="o">=</span> <span class="n">mesh_normal_consistency</span><span class="p">(</span><span class="n">mesh</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># mesh laplacian smoothing</span>
|
||||||
|
<span class="n">loss</span><span class="p">[</span><span class="s2">"laplacian"</span><span class="p">]</span> <span class="o">=</span> <span class="n">mesh_laplacian_smoothing</span><span class="p">(</span><span class="n">mesh</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s2">"uniform"</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># We will learn to deform the source mesh by offsetting its vertices</span>
|
||||||
|
<span class="c1"># The shape of the deform parameters is equal to the total number of vertices in</span>
|
||||||
|
<span class="c1"># src_mesh</span>
|
||||||
|
<span class="n">verts_shape</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">verts_packed</span><span class="p">()</span><span class="o">.</span><span class="n">shape</span>
|
||||||
|
<span class="n">deform_verts</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">(</span><span class="n">verts_shape</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># The optimizer</span>
|
||||||
|
<span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">([</span><span class="n">deform_verts</span><span class="p">],</span> <span class="n">lr</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We write an optimization loop to iteratively refine our predicted mesh from the sphere mesh into a mesh that matches the sillhouettes of the target images:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">loop</span> <span class="o">=</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">Niter</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">loop</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Initialize optimizer</span>
|
||||||
|
<span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># Deform the mesh</span>
|
||||||
|
<span class="n">new_src_mesh</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">offset_verts</span><span class="p">(</span><span class="n">deform_verts</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Losses to smooth /regularize the mesh shape</span>
|
||||||
|
<span class="n">loss</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">losses</span><span class="p">}</span>
|
||||||
|
<span class="n">update_mesh_shape_prior_losses</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">loss</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Compute the average silhouette loss over two random views, as the average </span>
|
||||||
|
<span class="c1"># squared L2 distance between the predicted silhouette and the target </span>
|
||||||
|
<span class="c1"># silhouette from our dataset</span>
|
||||||
|
<span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">permutation</span><span class="p">(</span><span class="n">num_views</span><span class="p">)</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[:</span><span class="n">num_views_per_iteration</span><span class="p">]:</span>
|
||||||
|
<span class="n">images_predicted</span> <span class="o">=</span> <span class="n">renderer_silhouette</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">target_cameras</span><span class="p">[</span><span class="n">j</span><span class="p">],</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||||
|
<span class="n">predicted_silhouette</span> <span class="o">=</span> <span class="n">images_predicted</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
|
||||||
|
<span class="n">loss_silhouette</span> <span class="o">=</span> <span class="p">((</span><span class="n">predicted_silhouette</span> <span class="o">-</span> <span class="n">target_silhouette</span><span class="p">[</span><span class="n">j</span><span class="p">])</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||||||
|
<span class="n">loss</span><span class="p">[</span><span class="s2">"silhouette"</span><span class="p">]</span> <span class="o">+=</span> <span class="n">loss_silhouette</span> <span class="o">/</span> <span class="n">num_views_per_iteration</span>
|
||||||
|
|
||||||
|
<span class="c1"># Weighted sum of the losses</span>
|
||||||
|
<span class="n">sum_loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">loss</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">sum_loss</span> <span class="o">+=</span> <span class="n">l</span> <span class="o">*</span> <span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"weight"</span><span class="p">]</span>
|
||||||
|
<span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"values"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Print the losses</span>
|
||||||
|
<span class="n">loop</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">"total_loss = </span><span class="si">%.6f</span><span class="s2">"</span> <span class="o">%</span> <span class="n">sum_loss</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Plot mesh</span>
|
||||||
|
<span class="k">if</span> <span class="n">i</span> <span class="o">%</span> <span class="n">plot_period</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="s2">"iter: </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">i</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="n">target_image</span><span class="o">=</span><span class="n">target_silhouette</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||||||
|
|
||||||
|
<span class="c1"># Optimization step</span>
|
||||||
|
<span class="n">sum_loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
|
||||||
|
<span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||||
|
<span class="n">target_image</span><span class="o">=</span><span class="n">target_silhouette</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||||||
|
<span class="n">plot_losses</span><span class="p">(</span><span class="n">losses</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="3.-Mesh-and-texture-prediction-via-textured-rendering">3. Mesh and texture prediction via textured rendering<a class="anchor-link" href="#3.-Mesh-and-texture-prediction-via-textured-rendering">¶</a></h2><p>We can predict both the mesh and its texture if we add an additional loss based on the comparing a predicted rendered RGB image to the target image. As before, we start with a sphere mesh. We learn both translational offsets and RGB texture colors for each vertex in the sphere mesh. Since our loss is based on rendered RGB pixel values instead of just the silhouette, we use a <strong>SoftPhongShader</strong> instead of a <strong>SoftSilhouetteShader</strong>.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rasterization settings for differentiable rendering, where the blur_radius</span>
|
||||||
|
<span class="c1"># initialization is based on Liu et al, 'Soft Rasterizer: A Differentiable </span>
|
||||||
|
<span class="c1"># Renderer for Image-based 3D Reasoning', ICCV 2019</span>
|
||||||
|
<span class="n">sigma</span> <span class="o">=</span> <span class="mf">1e-4</span>
|
||||||
|
<span class="n">raster_settings_soft</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||||
|
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||||
|
<span class="n">blur_radius</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">1.</span> <span class="o">/</span> <span class="mf">1e-4</span> <span class="o">-</span> <span class="mf">1.</span><span class="p">)</span><span class="o">*</span><span class="n">sigma</span><span class="p">,</span>
|
||||||
|
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Differentiable soft renderer using per vertex RGB colors for texture</span>
|
||||||
|
<span class="n">renderer_textured</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||||
|
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||||
|
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings_soft</span>
|
||||||
|
<span class="p">),</span>
|
||||||
|
<span class="n">shader</span><span class="o">=</span><span class="n">SoftPhongShader</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||||
|
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||||
|
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||||
|
<span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We initialize settings, losses, and the optimizer that will be used to iteratively fit our mesh to the target RGB images:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Number of views to optimize over in each SGD iteration</span>
|
||||||
|
<span class="n">num_views_per_iteration</span> <span class="o">=</span> <span class="mi">2</span>
|
||||||
|
<span class="c1"># Number of optimization steps</span>
|
||||||
|
<span class="n">Niter</span> <span class="o">=</span> <span class="mi">2000</span>
|
||||||
|
<span class="c1"># Plot period for the losses</span>
|
||||||
|
<span class="n">plot_period</span> <span class="o">=</span> <span class="mi">250</span>
|
||||||
|
|
||||||
|
<span class="o">%</span><span class="k">matplotlib</span> inline
|
||||||
|
|
||||||
|
<span class="c1"># Optimize using rendered RGB image loss, rendered silhouette image loss, mesh </span>
|
||||||
|
<span class="c1"># edge loss, mesh normal consistency, and mesh laplacian smoothing</span>
|
||||||
|
<span class="n">losses</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"rgb"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"silhouette"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"edge"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"normal"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">0.01</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="s2">"laplacian"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||||
|
<span class="p">}</span>
|
||||||
|
|
||||||
|
<span class="c1"># We will learn to deform the source mesh by offsetting its vertices</span>
|
||||||
|
<span class="c1"># The shape of the deform parameters is equal to the total number of vertices in </span>
|
||||||
|
<span class="c1"># src_mesh</span>
|
||||||
|
<span class="n">verts_shape</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">verts_packed</span><span class="p">()</span><span class="o">.</span><span class="n">shape</span>
|
||||||
|
<span class="n">deform_verts</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">(</span><span class="n">verts_shape</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># We will also learn per vertex colors for our sphere mesh that define texture </span>
|
||||||
|
<span class="c1"># of the mesh</span>
|
||||||
|
<span class="n">sphere_verts_rgb</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="n">verts_shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">3</span><span class="p">],</span> <span class="mf">0.5</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># The optimizer</span>
|
||||||
|
<span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">([</span><span class="n">deform_verts</span><span class="p">,</span> <span class="n">sphere_verts_rgb</span><span class="p">],</span> <span class="n">lr</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>We write an optimization loop to iteratively refine our predicted mesh and its vertex colors from the sphere mesh into a mesh that matches the target images:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">loop</span> <span class="o">=</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">Niter</span><span class="p">))</span>
|
||||||
|
|
||||||
|
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">loop</span><span class="p">:</span>
|
||||||
|
<span class="c1"># Initialize optimizer</span>
|
||||||
|
<span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
|
||||||
|
|
||||||
|
<span class="c1"># Deform the mesh</span>
|
||||||
|
<span class="n">new_src_mesh</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">offset_verts</span><span class="p">(</span><span class="n">deform_verts</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Add per vertex colors to texture the mesh</span>
|
||||||
|
<span class="n">new_src_mesh</span><span class="o">.</span><span class="n">textures</span> <span class="o">=</span> <span class="n">TexturesVertex</span><span class="p">(</span><span class="n">verts_features</span><span class="o">=</span><span class="n">sphere_verts_rgb</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Losses to smooth /regularize the mesh shape</span>
|
||||||
|
<span class="n">loss</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">losses</span><span class="p">}</span>
|
||||||
|
<span class="n">update_mesh_shape_prior_losses</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">loss</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Randomly select two views to optimize over in this iteration. Compared</span>
|
||||||
|
<span class="c1"># to using just one view, this helps resolve ambiguities between updating</span>
|
||||||
|
<span class="c1"># mesh shape vs. updating mesh texture</span>
|
||||||
|
<span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">permutation</span><span class="p">(</span><span class="n">num_views</span><span class="p">)</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[:</span><span class="n">num_views_per_iteration</span><span class="p">]:</span>
|
||||||
|
<span class="n">images_predicted</span> <span class="o">=</span> <span class="n">renderer_textured</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">target_cameras</span><span class="p">[</span><span class="n">j</span><span class="p">],</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Squared L2 distance between the predicted silhouette and the target </span>
|
||||||
|
<span class="c1"># silhouette from our dataset</span>
|
||||||
|
<span class="n">predicted_silhouette</span> <span class="o">=</span> <span class="n">images_predicted</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
|
||||||
|
<span class="n">loss_silhouette</span> <span class="o">=</span> <span class="p">((</span><span class="n">predicted_silhouette</span> <span class="o">-</span> <span class="n">target_silhouette</span><span class="p">[</span><span class="n">j</span><span class="p">])</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||||||
|
<span class="n">loss</span><span class="p">[</span><span class="s2">"silhouette"</span><span class="p">]</span> <span class="o">+=</span> <span class="n">loss_silhouette</span> <span class="o">/</span> <span class="n">num_views_per_iteration</span>
|
||||||
|
|
||||||
|
<span class="c1"># Squared L2 distance between the predicted RGB image and the target </span>
|
||||||
|
<span class="c1"># image from our dataset</span>
|
||||||
|
<span class="n">predicted_rgb</span> <span class="o">=</span> <span class="n">images_predicted</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="p">:</span><span class="mi">3</span><span class="p">]</span>
|
||||||
|
<span class="n">loss_rgb</span> <span class="o">=</span> <span class="p">((</span><span class="n">predicted_rgb</span> <span class="o">-</span> <span class="n">target_rgb</span><span class="p">[</span><span class="n">j</span><span class="p">])</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||||||
|
<span class="n">loss</span><span class="p">[</span><span class="s2">"rgb"</span><span class="p">]</span> <span class="o">+=</span> <span class="n">loss_rgb</span> <span class="o">/</span> <span class="n">num_views_per_iteration</span>
|
||||||
|
|
||||||
|
<span class="c1"># Weighted sum of the losses</span>
|
||||||
|
<span class="n">sum_loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||||
|
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">loss</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||||
|
<span class="n">sum_loss</span> <span class="o">+=</span> <span class="n">l</span> <span class="o">*</span> <span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"weight"</span><span class="p">]</span>
|
||||||
|
<span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"values"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Print the losses</span>
|
||||||
|
<span class="n">loop</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">"total_loss = </span><span class="si">%.6f</span><span class="s2">"</span> <span class="o">%</span> <span class="n">sum_loss</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Plot mesh</span>
|
||||||
|
<span class="k">if</span> <span class="n">i</span> <span class="o">%</span> <span class="n">plot_period</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||||
|
<span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">renderer</span><span class="o">=</span><span class="n">renderer_textured</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="s2">"iter: </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">i</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Optimization step</span>
|
||||||
|
<span class="n">sum_loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
|
||||||
|
<span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">renderer</span><span class="o">=</span><span class="n">renderer_textured</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||||
|
<span class="n">plot_losses</span><span class="p">(</span><span class="n">losses</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<p>Save the final predicted mesh:</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="4.-Save-the-final-predicted-mesh">4. Save the final predicted mesh<a class="anchor-link" href="#4.-Save-the-final-predicted-mesh">¶</a></h2>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing code_cell rendered">
|
||||||
|
<div class="input">
|
||||||
|
<div class="prompt input_prompt">In [ ]:</div>
|
||||||
|
<div class="inner_cell">
|
||||||
|
<div class="input_area">
|
||||||
|
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Fetch the verts and faces of the final predicted mesh</span>
|
||||||
|
<span class="n">final_verts</span><span class="p">,</span> <span class="n">final_faces</span> <span class="o">=</span> <span class="n">new_src_mesh</span><span class="o">.</span><span class="n">get_mesh_verts_faces</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||||
|
|
||||||
|
<span class="c1"># Scale normalize back to the original target size</span>
|
||||||
|
<span class="n">final_verts</span> <span class="o">=</span> <span class="n">final_verts</span> <span class="o">*</span> <span class="n">scale</span> <span class="o">+</span> <span class="n">center</span>
|
||||||
|
|
||||||
|
<span class="c1"># Store the predicted mesh using save_obj</span>
|
||||||
|
<span class="n">final_obj</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s1">'./'</span><span class="p">,</span> <span class="s1">'final_model.obj'</span><span class="p">)</span>
|
||||||
|
<span class="n">save_obj</span><span class="p">(</span><span class="n">final_obj</span><span class="p">,</span> <span class="n">final_verts</span><span class="p">,</span> <span class="n">final_faces</span><span class="p">)</span>
|
||||||
|
</pre></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||||
|
</div><div class="inner_cell">
|
||||||
|
<div class="text_cell_render border-box-sizing rendered_html">
|
||||||
|
<h2 id="5.-Conclusion">5. Conclusion<a class="anchor-link" href="#5.-Conclusion">¶</a></h2><p>In this tutorial, we learned how to load a textured mesh from an obj file, create a synthetic dataset by rendering the mesh from multiple viewpoints. We showed how to set up an optimization loop to fit a mesh to the observed dataset images based on a rendered silhouette loss. We then augmented this optimization loop with an additional loss based on rendered RGB images, which allowed us to predict both a mesh and its texture.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="container docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span></span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Tutorials</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/tutorials/">Overview</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">3D operators</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/deform_source_mesh_to_target_mesh">Fit Mesh</a></li><li class="navListItem"><a class="navItem" href="/tutorials/bundle_adjustment">Bundle Adjustment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Rendering</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/render_textured_meshes">Render Textured Meshes</a></li><li class="navListItem"><a class="navItem" href="/tutorials/camera_position_optimization_with_differentiable_rendering">Camera Position Optimization</a></li></ul></div></div></section></div><script>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="container docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span></span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Tutorials</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/tutorials/">Overview</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">3D operators</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/deform_source_mesh_to_target_mesh">Fit Mesh</a></li><li class="navListItem"><a class="navItem" href="/tutorials/bundle_adjustment">Bundle Adjustment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Rendering</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/render_textured_meshes">Render Textured Meshes</a></li><li class="navListItem"><a class="navItem" href="/tutorials/fit_textured_mesh">Fit a mesh with texture via rendering</a></li><li class="navListItem"><a class="navItem" href="/tutorials/camera_position_optimization_with_differentiable_rendering">Camera Position Optimization</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Dataloaders</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/dataloaders_ShapeNetCore_R2N2">Data loaders for ShapeNetCore and R2N2</a></li></ul></div></div></section></div><script>
|
||||||
var coll = document.getElementsByClassName('collapsible');
|
var coll = document.getElementsByClassName('collapsible');
|
||||||
var checkActiveCategory = true;
|
var checkActiveCategory = true;
|
||||||
for (var i = 0; i < coll.length; i++) {
|
for (var i = 0; i < coll.length; i++) {
|
||||||
@ -65,4 +65,4 @@
|
|||||||
</script></nav></div><div class="container mainContainer documentContainer postContainer"><div class="wrapper"><div class="post"><header class="postHeader"><h1 class="postHeaderTitle">Welcome to the PyTorch3D Tutorials</h1></header><p>Here you can learn about the structure and applications of Pytorch3D from examples which are in the form of ipython notebooks.</p><h3> Run interactively </h3><p>At the top of each example you can find a button named <strong>"Run in Google Colab"</strong> which will open the notebook in <a href="https://colab.research.google.com/notebooks/intro.ipynb"> Google Colaboratory </a> where you can run the code directly in the browser with access to GPU support - it looks like this:</p><div class="tutorialButtonsWrapper"><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" target="_blank"><img class="colabButton" align="left" src="/img/colab_icon.png"/>Run in Google Colab</a></div></div><p> You can modify the code and experiment with varying different settings. Remember to install pytorch, torchvision, fvcore and pytorch3d in the first cell of the colab notebook by running: </p><div><span><pre><code class="hljs css language-bash">!pip install torch torchvision
|
</script></nav></div><div class="container mainContainer documentContainer postContainer"><div class="wrapper"><div class="post"><header class="postHeader"><h1 class="postHeaderTitle">Welcome to the PyTorch3D Tutorials</h1></header><p>Here you can learn about the structure and applications of Pytorch3D from examples which are in the form of ipython notebooks.</p><h3> Run interactively </h3><p>At the top of each example you can find a button named <strong>"Run in Google Colab"</strong> which will open the notebook in <a href="https://colab.research.google.com/notebooks/intro.ipynb"> Google Colaboratory </a> where you can run the code directly in the browser with access to GPU support - it looks like this:</p><div class="tutorialButtonsWrapper"><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" target="_blank"><img class="colabButton" align="left" src="/img/colab_icon.png"/>Run in Google Colab</a></div></div><p> You can modify the code and experiment with varying different settings. Remember to install pytorch, torchvision, fvcore and pytorch3d in the first cell of the colab notebook by running: </p><div><span><pre><code class="hljs css language-bash">!pip install torch torchvision
|
||||||
!pip install <span class="hljs-string">'git+https://github.com/facebookresearch/pytorch3d.git@stable'</span>
|
!pip install <span class="hljs-string">'git+https://github.com/facebookresearch/pytorch3d.git@stable'</span>
|
||||||
</code></pre>
|
</code></pre>
|
||||||
</span></div>This installs the latest stable version of PyTorch3D from github.<h3> Run locally </h3><p> There is also a button to download the notebook and source code to run it locally. </p></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</span></div>This installs the latest stable version of PyTorch3D from github.<h3> Run locally </h3><p> There is also a button to download the notebook and source code to run it locally. </p></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@ -6,4 +6,4 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="mainContainer"><div class="container paddingBottom paddingTop"><div class="wrapper"><div class="showcaseSection"><div class="prose"><h1>Who is Using This?</h1><p>This project is used by many folks</p></div><div class="logos"><a href="https://www.facebook.com"><img src="/img/undraw_open_source.svg" alt="User1" title="User1"/></a></div><p>Are you using this project?</p><a href="undefined/edit/master/website/siteConfig.js" class="button">Add your company</a></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="mainContainer"><div class="container paddingBottom paddingTop"><div class="wrapper"><div class="showcaseSection"><div class="prose"><h1>Who is Using This?</h1><p>This project is used by many folks</p></div><div class="logos"><a href="https://www.facebook.com"><img src="/img/undraw_open_source.svg" alt="User1" title="User1"/></a></div><p>Are you using this project?</p><a href="undefined/edit/master/website/siteConfig.js" class="button">Add your company</a></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
@ -6,4 +6,4 @@
|
|||||||
|
|
||||||
ga('create', 'UA-157376881-1', 'auto');
|
ga('create', 'UA-157376881-1', 'auto');
|
||||||
ga('send', 'pageview');
|
ga('send', 'pageview');
|
||||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="mainContainer"><div class="container paddingBottom paddingTop"><div class="wrapper"><div class="showcaseSection"><div class="prose"><h1>Who is Using This?</h1><p>This project is used by many folks</p></div><div class="logos"><a href="https://www.facebook.com"><img src="/img/undraw_open_source.svg" alt="User1" title="User1"/></a></div><p>Are you using this project?</p><a href="undefined/edit/master/website/siteConfig.js" class="button">Add your company</a></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="mainContainer"><div class="container paddingBottom paddingTop"><div class="wrapper"><div class="showcaseSection"><div class="prose"><h1>Who is Using This?</h1><p>This project is used by many folks</p></div><div class="logos"><a href="https://www.facebook.com"><img src="/img/undraw_open_source.svg" alt="User1" title="User1"/></a></div><p>Are you using this project?</p><a href="undefined/edit/master/website/siteConfig.js" class="button">Add your company</a></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||||
Loading…
x
Reference in New Issue
Block a user