mirror of
https://github.com/facebookresearch/pytorch3d.git
synced 2026-02-06 22:12:16 +08:00
Update latest version of site
This commit is contained in:
Nikhila Ravi
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
614
tutorials/dataloaders_ShapeNetCore_R2N2.html
Normal file
614
tutorials/dataloaders_ShapeNetCore_R2N2.html
Normal file
@@ -0,0 +1,614 @@
|
||||
<!DOCTYPE html><html lang=""><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>PyTorch3D · A library for deep learning with 3D data</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="A library for deep learning with 3D data"/><meta property="og:title" content="PyTorch3D · A library for deep learning with 3D data"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="A library for deep learning with 3D data"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-157376881-1', 'auto');
|
||||
ga('send', 'pageview');
|
||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="container docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span></span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Tutorials</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/">Overview</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">3D operators</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/deform_source_mesh_to_target_mesh">Fit Mesh</a></li><li class="navListItem"><a class="navItem" href="/tutorials/bundle_adjustment">Bundle Adjustment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Rendering</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/render_textured_meshes">Render Textured Meshes</a></li><li class="navListItem"><a class="navItem" href="/tutorials/fit_textured_mesh">Fit a mesh with texture via rendering</a></li><li class="navListItem"><a class="navItem" href="/tutorials/camera_position_optimization_with_differentiable_rendering">Camera Position Optimization</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Dataloaders</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/tutorials/dataloaders_ShapeNetCore_R2N2">Data loaders for ShapeNetCore and R2N2</a></li></ul></div></div></section></div><script>
|
||||
var coll = document.getElementsByClassName('collapsible');
|
||||
var checkActiveCategory = true;
|
||||
for (var i = 0; i < coll.length; i++) {
|
||||
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||
if (checkActiveCategory){
|
||||
for (var j = 0; j < links.length; j++) {
|
||||
if (links[j].classList.contains('navListItemActive')){
|
||||
coll[i].nextElementSibling.classList.toggle('hide');
|
||||
coll[i].childNodes[1].classList.toggle('rotate');
|
||||
checkActiveCategory = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
coll[i].addEventListener('click', function() {
|
||||
var arrow = this.childNodes[1];
|
||||
arrow.classList.toggle('rotate');
|
||||
var content = this.nextElementSibling;
|
||||
content.classList.toggle('hide');
|
||||
});
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||
createToggler('#tocToggler', 'body', 'tocActive');
|
||||
|
||||
var headings = document.querySelector('.toc-headings');
|
||||
headings && headings.addEventListener('click', function(event) {
|
||||
var el = event.target;
|
||||
while(el !== headings){
|
||||
if (el.tagName === 'A') {
|
||||
document.body.classList.remove('tocActive');
|
||||
break;
|
||||
} else{
|
||||
el = el.parentNode;
|
||||
}
|
||||
}
|
||||
}, false);
|
||||
|
||||
function createToggler(togglerSelector, targetSelector, className) {
|
||||
var toggler = document.querySelector(togglerSelector);
|
||||
var target = document.querySelector(targetSelector);
|
||||
|
||||
if (!toggler) {
|
||||
return;
|
||||
}
|
||||
|
||||
toggler.onclick = function(event) {
|
||||
event.preventDefault();
|
||||
|
||||
target.classList.toggle(className);
|
||||
};
|
||||
}
|
||||
});
|
||||
</script></nav></div><div class="container mainContainer"><div class="wrapper"><div class="tutorialButtonsWrapper"><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="https://colab.research.google.com/github/facebookresearch/pytorch3d/blob/stable/docs/tutorials/dataloaders_ShapeNetCore_R2N2.ipynb" target="_blank"><img class="colabButton" align="left" src="/img/colab_icon.png"/>Run in Google Colab</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/dataloaders_ShapeNetCore_R2N2.ipynb" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Jupyter Notebook</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/dataloaders_ShapeNetCore_R2N2.py" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Source Code</a></div></div><div class="tutorialBody">
|
||||
<script
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.10/require.min.js">
|
||||
</script>
|
||||
<script
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js">
|
||||
</script>
|
||||
<div class="notebook">
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h1 id="Dataloaders-for-ShapeNetCore-and-R2N2">Dataloaders for ShapeNetCore and R2N2<a class="anchor-link" href="#Dataloaders-for-ShapeNetCore-and-R2N2">¶</a></h1><p>This tutorial shows how to:</p>
|
||||
<ul>
|
||||
<li>Load models from ShapeNetCore and R2N2 using PyTorch3D's data loaders.</li>
|
||||
<li>Pass the loaded datasets to <code>torch.utils.data.DataLoader</code>.</li>
|
||||
<li>Render ShapeNetCore models with PyTorch3D's renderer.</li>
|
||||
<li>Render R2N2 models with the same orientations as the original renderings in the dataset.</li>
|
||||
<li>Visualize R2N2 model voxels.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="0.-Install-and-import-modules">0. Install and import modules<a class="anchor-link" href="#0.-Install-and-import-modules">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If <code>torch</code>, <code>torchvision</code> and <code>pytorch3d</code> are not installed, run the following cell:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>pip install torch torchvision
|
||||
<span class="kn">import</span> <span class="nn">sys</span>
|
||||
<span class="kn">import</span> <span class="nn">torch</span>
|
||||
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">__version__</span><span class="o">==</span><span class="s1">'1.6.0+cu101'</span> <span class="ow">and</span> <span class="n">sys</span><span class="o">.</span><span class="n">platform</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'linux'</span><span class="p">):</span>
|
||||
<span class="o">!</span>pip install pytorch3d
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="o">!</span>pip install <span class="s1">'git+https://github.com/facebookresearch/pytorch3d.git@stable'</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||||
<span class="kn">import</span> <span class="nn">torch</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.datasets</span> <span class="k">import</span> <span class="p">(</span>
|
||||
<span class="n">R2N2</span><span class="p">,</span>
|
||||
<span class="n">ShapeNetCore</span><span class="p">,</span>
|
||||
<span class="n">collate_batched_meshes</span><span class="p">,</span>
|
||||
<span class="n">render_cubified_voxels</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.renderer</span> <span class="k">import</span> <span class="p">(</span>
|
||||
<span class="n">OpenGLPerspectiveCameras</span><span class="p">,</span>
|
||||
<span class="n">PointLights</span><span class="p">,</span>
|
||||
<span class="n">RasterizationSettings</span><span class="p">,</span>
|
||||
<span class="n">TexturesVertex</span><span class="p">,</span>
|
||||
<span class="n">look_at_view_transform</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.structures</span> <span class="k">import</span> <span class="n">Meshes</span>
|
||||
<span class="kn">from</span> <span class="nn">torch.utils.data</span> <span class="k">import</span> <span class="n">DataLoader</span>
|
||||
|
||||
<span class="c1"># add path for demo utils functions </span>
|
||||
<span class="kn">import</span> <span class="nn">sys</span>
|
||||
<span class="kn">import</span> <span class="nn">os</span>
|
||||
<span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="s1">''</span><span class="p">))</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If using <strong>Google Colab</strong>, fetch the utils file for plotting image grids:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py
|
||||
<span class="kn">from</span> <span class="nn">plot_image_grid</span> <span class="k">import</span> <span class="n">image_grid</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>OR if running locally uncomment and run the following cell:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># from utils import image_grid</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="1.-Load-the-datasets">1. Load the datasets<a class="anchor-link" href="#1.-Load-the-datasets">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If you haven't already downloaded the ShapeNetCore dataset, first do that following the instructions here: <a href="https://www.shapenet.org/">https://www.shapenet.org/</a>. ShapeNetCore is a subset of the ShapeNet dataset. In PyTorch3D we support both version 1 (57 categories) and version 2 (55 categories).</p>
|
||||
<p>Then modify <code>SHAPENET_PATH</code> below to you local path to the ShapeNetCore dataset folder.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Setup</span>
|
||||
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
|
||||
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cuda:0"</span><span class="p">)</span>
|
||||
<span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">set_device</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cpu"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">SHAPENET_PATH</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="n">shapenet_dataset</span> <span class="o">=</span> <span class="n">ShapeNetCore</span><span class="p">(</span><span class="n">SHAPENET_PATH</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>The R2N2 dataset can be downloaded using the instructions here: <a href="http://3d-r2n2.stanford.edu/">http://3d-r2n2.stanford.edu/</a>. Look at the links for <code>ShapeNetRendering</code> and <code>ShapeNetVox32</code>. The R2N2 dataset contains 13 categories that are a subset of the ShapeNetCore v.1
|
||||
dataset. The R2N2 dataset also contains its own 24 renderings of each object and voxelized models.</p>
|
||||
<p>Then modify <code>R2N2_PATH</code> and <code>SPLITS_PATH</code> below to your local R2N2 dataset folder path and splits file path respectively. Here we will load the <code>train</code> split of R2N2 and ask the voxels of each model to be returned.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">R2N2_PATH</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="n">SPLITS_PATH</span> <span class="o">=</span> <span class="s2">"None"</span>
|
||||
<span class="n">r2n2_dataset</span> <span class="o">=</span> <span class="n">R2N2</span><span class="p">(</span><span class="s2">"train"</span><span class="p">,</span> <span class="n">SHAPENET_PATH</span><span class="p">,</span> <span class="n">R2N2_PATH</span><span class="p">,</span> <span class="n">SPLITS_PATH</span><span class="p">,</span> <span class="n">return_voxels</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We can retrieve a model by indexing into the loaded dataset. For both ShapeNetCore and R2N2, we can examine the category this model belongs to (in the form of a synset id, equivalend to wnid described in ImageNet's API: <a href="http://image-net.org/download-API">http://image-net.org/download-API</a>), its model id, and its vertices and faces.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">shapenet_model</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="p">[</span><span class="mi">6</span><span class="p">]</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s2">"This model belongs to the category "</span> <span class="o">+</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"synset_id"</span><span class="p">]</span> <span class="o">+</span> <span class="s2">"."</span><span class="p">)</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s2">"This model has model id "</span> <span class="o">+</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"model_id"</span><span class="p">]</span> <span class="o">+</span> <span class="s2">"."</span><span class="p">)</span>
|
||||
<span class="n">model_verts</span><span class="p">,</span> <span class="n">model_faces</span> <span class="o">=</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"verts"</span><span class="p">],</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"faces"</span><span class="p">]</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We can use its vertices and faces to form a <code>Meshes</code> object which is a PyTorch3D datastructure for working with batched meshes.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">model_textures</span> <span class="o">=</span> <span class="n">TexturesVertex</span><span class="p">(</span><span class="n">verts_features</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">ones_like</span><span class="p">(</span><span class="n">model_verts</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)[</span><span class="kc">None</span><span class="p">])</span>
|
||||
<span class="n">shapenet_model_mesh</span> <span class="o">=</span> <span class="n">Meshes</span><span class="p">(</span>
|
||||
<span class="n">verts</span><span class="o">=</span><span class="p">[</span><span class="n">model_verts</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)],</span>
|
||||
<span class="n">faces</span><span class="o">=</span><span class="p">[</span><span class="n">model_faces</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)],</span>
|
||||
<span class="n">textures</span><span class="o">=</span><span class="n">model_textures</span>
|
||||
<span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>With R2N2, we can further examine R2N2's original renderings. For instance, if we would like to see the second and third views of the eleventh objects in the R2N2 dataset, we can do the following:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">r2n2_renderings</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="p">[</span><span class="mi">10</span><span class="p">,[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">]]</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">r2n2_renderings</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="2.-Use-the-datasets-with-torch.utils.data.DataLoader">2. Use the datasets with <code>torch.utils.data.DataLoader</code><a class="anchor-link" href="#2.-Use-the-datasets-with-torch.utils.data.DataLoader">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Training deep learning models, usually requires passing in batches of inputs. The <code>torch.utils.data.DataLoader</code> from Pytorch helps us do this. PyTorch3D provides a function <code>collate_batched_meshes</code> to group the input meshes into a single <code>Meshes</code> object which represents the batch. The <code>Meshes</code> datastructure can then be used directly by other PyTorch3D ops which might be part of the deep learning model (e.g. <code>graph_conv</code>).</p>
|
||||
<p>For R2N2, if all the models in the batch have the same number of views, the views, rotation matrices, translation matrices, intrinsic matrices and voxels will also be stacked into batched tensors.</p>
|
||||
<p><strong>NOTE</strong>: All models in the <code>val</code> split of R2N2 have 24 views, but there are 8 models that split their 24 views between <code>train</code> and <code>test</code> splits, in which case <code>collate_batched_meshes</code> will only be able to join the matrices, views and voxels as lists. However, this can be avoided by laoding only one view of each model by setting <code>return_all_views = False</code>.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">batch_size</span> <span class="o">=</span> <span class="mi">12</span>
|
||||
<span class="n">r2n2_single_view</span> <span class="o">=</span> <span class="n">R2N2</span><span class="p">(</span><span class="s2">"train"</span><span class="p">,</span> <span class="n">SHAPENET_PATH</span><span class="p">,</span> <span class="n">R2N2_PATH</span><span class="p">,</span> <span class="n">SPLITS_PATH</span><span class="p">,</span> <span class="n">return_all_views</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">return_voxels</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">r2n2_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span><span class="n">r2n2_single_view</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">collate_fn</span><span class="o">=</span><span class="n">collate_batched_meshes</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Let's visualize all the views (one for each model) in the batch:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">it</span> <span class="o">=</span> <span class="nb">iter</span><span class="p">(</span><span class="n">r2n2_loader</span><span class="p">)</span>
|
||||
<span class="n">r2n2_batch</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="n">it</span><span class="p">)</span>
|
||||
<span class="n">batch_renderings</span> <span class="o">=</span> <span class="n">r2n2_batch</span><span class="p">[</span><span class="s2">"images"</span><span class="p">]</span> <span class="c1"># (N, V, H, W, 3), and in this case V is 1.</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">batch_renderings</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="3.-Render-ShapeNetCore-models-with-PyTorch3D's-differntiable-renderer">3. Render ShapeNetCore models with PyTorch3D's differntiable renderer<a class="anchor-link" href="#3.-Render-ShapeNetCore-models-with-PyTorch3D's-differntiable-renderer">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Both <code>ShapeNetCore</code> and <code>R2N2</code> dataloaders have customized <code>render</code> functions that support rendering models by specifying their model ids, categories or indices using PyTorch3D's differentiable renderer implementation.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rendering settings.</span>
|
||||
<span class="n">R</span><span class="p">,</span> <span class="n">T</span> <span class="o">=</span> <span class="n">look_at_view_transform</span><span class="p">(</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="mi">90</span><span class="p">)</span>
|
||||
<span class="n">cameras</span> <span class="o">=</span> <span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">,</span> <span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="n">raster_settings</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span><span class="n">image_size</span><span class="o">=</span><span class="mi">512</span><span class="p">)</span>
|
||||
<span class="n">lights</span> <span class="o">=</span> <span class="n">PointLights</span><span class="p">(</span><span class="n">location</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="o">-</span><span class="mf">2.0</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)[</span><span class="kc">None</span><span class="p">],</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>First we will try to render three models by their model ids:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">images_by_model_ids</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||
<span class="n">model_ids</span><span class="o">=</span><span class="p">[</span>
|
||||
<span class="s2">"13394ca47c89f91525a3aaf903a41c90"</span><span class="p">,</span>
|
||||
<span class="s2">"14755c2ee8e693aba508f621166382b0"</span><span class="p">,</span>
|
||||
<span class="s2">"156c4207af6d2c8f1fdc97905708b8ea"</span><span class="p">,</span>
|
||||
<span class="p">],</span>
|
||||
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">images_by_model_ids</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Suppose we would like to render the first three models in the dataset, we can render models by their indices:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">images_by_idxs</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||
<span class="n">idxs</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">)),</span>
|
||||
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">images_by_idxs</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Alternatively, if we are not interested in any particular models but would like see random models from some specific categories, we can do that by specifying <code>categories</code> and <code>sample_nums</code>. For example, if we would like to render 2 models from the category "faucet" and 3 models from the category "chair", we can do the following:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">images_by_categories</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||
<span class="n">categories</span><span class="o">=</span><span class="p">[</span><span class="s2">"faucet"</span><span class="p">,</span> <span class="s2">"chair"</span><span class="p">],</span>
|
||||
<span class="n">sample_nums</span><span class="o">=</span><span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span>
|
||||
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">images_by_categories</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If we are not interested in any particular categories and just would like to render some random models from the whole dataset, we can set the number of models to be rendered in <code>sample_nums</code> and not specify any <code>categories</code>:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">random_model_images</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||
<span class="n">sample_nums</span><span class="o">=</span><span class="p">[</span><span class="mi">3</span><span class="p">],</span>
|
||||
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">random_model_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="4.-Render-R2N2-models-with-the-same-orientations-as-the-original-renderings-in-the-dataset">4. Render R2N2 models with the same orientations as the original renderings in the dataset<a class="anchor-link" href="#4.-Render-R2N2-models-with-the-same-orientations-as-the-original-renderings-in-the-dataset">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We can render R2N2 models the same way as we rendered ShapeNetCore models above. In addition, we can also render R2N2 models with the same orientations as the original renderings in the dataset. For this we will use R2N2's customized <code>render</code> function and a different type of PyTorch3D camera called <code>BlenderCamera</code>.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>In this example, we will render the seventh model with the same orientations as its second and third views. First we will retrieve R2N2's original renderings to compare with the result.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">original_rendering</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="p">[</span><span class="mi">6</span><span class="p">,[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">]][</span><span class="s2">"images"</span><span class="p">]</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">original_rendering</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Next, we will visualize PyTorch3d's renderings:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">r2n2_oriented_images</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||
<span class="n">idxs</span><span class="o">=</span><span class="p">[</span><span class="mi">6</span><span class="p">],</span>
|
||||
<span class="n">view_idxs</span><span class="o">=</span><span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">],</span>
|
||||
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">r2n2_oriented_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="5.-Visualize-R2N2-models'-voxels">5. Visualize R2N2 models' voxels<a class="anchor-link" href="#5.-Visualize-R2N2-models'-voxels">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>R2N2 dataloader also returns models' voxels. We can visualize them by utilizing R2N2's <code>render_vox_to_mesh</code> function. This will cubify the voxels to a Meshes object, which will then be rendered.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>In this example we will visualize the tenth model in the dataset with the same orientation of its second and third views. First we will retrieve R2N2's original renderings to compare with the result.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">r2n2_model</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="p">[</span><span class="mi">9</span><span class="p">,[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">]]</span>
|
||||
<span class="n">original_rendering</span> <span class="o">=</span> <span class="n">r2n2_model</span><span class="p">[</span><span class="s2">"images"</span><span class="p">]</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">original_rendering</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Next, we will pass the voxels to <code>render_vox_to_mesh</code>:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">vox_render</span> <span class="o">=</span> <span class="n">render_cubified_voxels</span><span class="p">(</span><span class="n">r2n2_model</span><span class="p">[</span><span class="s2">"voxels"</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">vox_render</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||
614
tutorials/dataloaders_ShapeNetCore_R2N2/index.html
Normal file
614
tutorials/dataloaders_ShapeNetCore_R2N2/index.html
Normal file
@@ -0,0 +1,614 @@
|
||||
<!DOCTYPE html><html lang=""><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>PyTorch3D · A library for deep learning with 3D data</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="A library for deep learning with 3D data"/><meta property="og:title" content="PyTorch3D · A library for deep learning with 3D data"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="A library for deep learning with 3D data"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-157376881-1', 'auto');
|
||||
ga('send', 'pageview');
|
||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="container docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span></span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Tutorials</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/">Overview</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">3D operators</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/deform_source_mesh_to_target_mesh">Fit Mesh</a></li><li class="navListItem"><a class="navItem" href="/tutorials/bundle_adjustment">Bundle Adjustment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Rendering</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/render_textured_meshes">Render Textured Meshes</a></li><li class="navListItem"><a class="navItem" href="/tutorials/fit_textured_mesh">Fit a mesh with texture via rendering</a></li><li class="navListItem"><a class="navItem" href="/tutorials/camera_position_optimization_with_differentiable_rendering">Camera Position Optimization</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Dataloaders</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/tutorials/dataloaders_ShapeNetCore_R2N2">Data loaders for ShapeNetCore and R2N2</a></li></ul></div></div></section></div><script>
|
||||
var coll = document.getElementsByClassName('collapsible');
|
||||
var checkActiveCategory = true;
|
||||
for (var i = 0; i < coll.length; i++) {
|
||||
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||
if (checkActiveCategory){
|
||||
for (var j = 0; j < links.length; j++) {
|
||||
if (links[j].classList.contains('navListItemActive')){
|
||||
coll[i].nextElementSibling.classList.toggle('hide');
|
||||
coll[i].childNodes[1].classList.toggle('rotate');
|
||||
checkActiveCategory = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
coll[i].addEventListener('click', function() {
|
||||
var arrow = this.childNodes[1];
|
||||
arrow.classList.toggle('rotate');
|
||||
var content = this.nextElementSibling;
|
||||
content.classList.toggle('hide');
|
||||
});
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||
createToggler('#tocToggler', 'body', 'tocActive');
|
||||
|
||||
var headings = document.querySelector('.toc-headings');
|
||||
headings && headings.addEventListener('click', function(event) {
|
||||
var el = event.target;
|
||||
while(el !== headings){
|
||||
if (el.tagName === 'A') {
|
||||
document.body.classList.remove('tocActive');
|
||||
break;
|
||||
} else{
|
||||
el = el.parentNode;
|
||||
}
|
||||
}
|
||||
}, false);
|
||||
|
||||
function createToggler(togglerSelector, targetSelector, className) {
|
||||
var toggler = document.querySelector(togglerSelector);
|
||||
var target = document.querySelector(targetSelector);
|
||||
|
||||
if (!toggler) {
|
||||
return;
|
||||
}
|
||||
|
||||
toggler.onclick = function(event) {
|
||||
event.preventDefault();
|
||||
|
||||
target.classList.toggle(className);
|
||||
};
|
||||
}
|
||||
});
|
||||
</script></nav></div><div class="container mainContainer"><div class="wrapper"><div class="tutorialButtonsWrapper"><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="https://colab.research.google.com/github/facebookresearch/pytorch3d/blob/stable/docs/tutorials/dataloaders_ShapeNetCore_R2N2.ipynb" target="_blank"><img class="colabButton" align="left" src="/img/colab_icon.png"/>Run in Google Colab</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/dataloaders_ShapeNetCore_R2N2.ipynb" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Jupyter Notebook</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/dataloaders_ShapeNetCore_R2N2.py" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Source Code</a></div></div><div class="tutorialBody">
|
||||
<script
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.10/require.min.js">
|
||||
</script>
|
||||
<script
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js">
|
||||
</script>
|
||||
<div class="notebook">
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h1 id="Dataloaders-for-ShapeNetCore-and-R2N2">Dataloaders for ShapeNetCore and R2N2<a class="anchor-link" href="#Dataloaders-for-ShapeNetCore-and-R2N2">¶</a></h1><p>This tutorial shows how to:</p>
|
||||
<ul>
|
||||
<li>Load models from ShapeNetCore and R2N2 using PyTorch3D's data loaders.</li>
|
||||
<li>Pass the loaded datasets to <code>torch.utils.data.DataLoader</code>.</li>
|
||||
<li>Render ShapeNetCore models with PyTorch3D's renderer.</li>
|
||||
<li>Render R2N2 models with the same orientations as the original renderings in the dataset.</li>
|
||||
<li>Visualize R2N2 model voxels.</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="0.-Install-and-import-modules">0. Install and import modules<a class="anchor-link" href="#0.-Install-and-import-modules">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If <code>torch</code>, <code>torchvision</code> and <code>pytorch3d</code> are not installed, run the following cell:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>pip install torch torchvision
|
||||
<span class="kn">import</span> <span class="nn">sys</span>
|
||||
<span class="kn">import</span> <span class="nn">torch</span>
|
||||
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">__version__</span><span class="o">==</span><span class="s1">'1.6.0+cu101'</span> <span class="ow">and</span> <span class="n">sys</span><span class="o">.</span><span class="n">platform</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'linux'</span><span class="p">):</span>
|
||||
<span class="o">!</span>pip install pytorch3d
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="o">!</span>pip install <span class="s1">'git+https://github.com/facebookresearch/pytorch3d.git@stable'</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||||
<span class="kn">import</span> <span class="nn">torch</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.datasets</span> <span class="k">import</span> <span class="p">(</span>
|
||||
<span class="n">R2N2</span><span class="p">,</span>
|
||||
<span class="n">ShapeNetCore</span><span class="p">,</span>
|
||||
<span class="n">collate_batched_meshes</span><span class="p">,</span>
|
||||
<span class="n">render_cubified_voxels</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.renderer</span> <span class="k">import</span> <span class="p">(</span>
|
||||
<span class="n">OpenGLPerspectiveCameras</span><span class="p">,</span>
|
||||
<span class="n">PointLights</span><span class="p">,</span>
|
||||
<span class="n">RasterizationSettings</span><span class="p">,</span>
|
||||
<span class="n">TexturesVertex</span><span class="p">,</span>
|
||||
<span class="n">look_at_view_transform</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.structures</span> <span class="k">import</span> <span class="n">Meshes</span>
|
||||
<span class="kn">from</span> <span class="nn">torch.utils.data</span> <span class="k">import</span> <span class="n">DataLoader</span>
|
||||
|
||||
<span class="c1"># add path for demo utils functions </span>
|
||||
<span class="kn">import</span> <span class="nn">sys</span>
|
||||
<span class="kn">import</span> <span class="nn">os</span>
|
||||
<span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="s1">''</span><span class="p">))</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If using <strong>Google Colab</strong>, fetch the utils file for plotting image grids:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py
|
||||
<span class="kn">from</span> <span class="nn">plot_image_grid</span> <span class="k">import</span> <span class="n">image_grid</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>OR if running locally uncomment and run the following cell:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># from utils import image_grid</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="1.-Load-the-datasets">1. Load the datasets<a class="anchor-link" href="#1.-Load-the-datasets">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If you haven't already downloaded the ShapeNetCore dataset, first do that following the instructions here: <a href="https://www.shapenet.org/">https://www.shapenet.org/</a>. ShapeNetCore is a subset of the ShapeNet dataset. In PyTorch3D we support both version 1 (57 categories) and version 2 (55 categories).</p>
|
||||
<p>Then modify <code>SHAPENET_PATH</code> below to you local path to the ShapeNetCore dataset folder.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Setup</span>
|
||||
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
|
||||
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cuda:0"</span><span class="p">)</span>
|
||||
<span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">set_device</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cpu"</span><span class="p">)</span>
|
||||
|
||||
<span class="n">SHAPENET_PATH</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="n">shapenet_dataset</span> <span class="o">=</span> <span class="n">ShapeNetCore</span><span class="p">(</span><span class="n">SHAPENET_PATH</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>The R2N2 dataset can be downloaded using the instructions here: <a href="http://3d-r2n2.stanford.edu/">http://3d-r2n2.stanford.edu/</a>. Look at the links for <code>ShapeNetRendering</code> and <code>ShapeNetVox32</code>. The R2N2 dataset contains 13 categories that are a subset of the ShapeNetCore v.1
|
||||
dataset. The R2N2 dataset also contains its own 24 renderings of each object and voxelized models.</p>
|
||||
<p>Then modify <code>R2N2_PATH</code> and <code>SPLITS_PATH</code> below to your local R2N2 dataset folder path and splits file path respectively. Here we will load the <code>train</code> split of R2N2 and ask the voxels of each model to be returned.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">R2N2_PATH</span> <span class="o">=</span> <span class="s2">""</span>
|
||||
<span class="n">SPLITS_PATH</span> <span class="o">=</span> <span class="s2">"None"</span>
|
||||
<span class="n">r2n2_dataset</span> <span class="o">=</span> <span class="n">R2N2</span><span class="p">(</span><span class="s2">"train"</span><span class="p">,</span> <span class="n">SHAPENET_PATH</span><span class="p">,</span> <span class="n">R2N2_PATH</span><span class="p">,</span> <span class="n">SPLITS_PATH</span><span class="p">,</span> <span class="n">return_voxels</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We can retrieve a model by indexing into the loaded dataset. For both ShapeNetCore and R2N2, we can examine the category this model belongs to (in the form of a synset id, equivalend to wnid described in ImageNet's API: <a href="http://image-net.org/download-API">http://image-net.org/download-API</a>), its model id, and its vertices and faces.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">shapenet_model</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="p">[</span><span class="mi">6</span><span class="p">]</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s2">"This model belongs to the category "</span> <span class="o">+</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"synset_id"</span><span class="p">]</span> <span class="o">+</span> <span class="s2">"."</span><span class="p">)</span>
|
||||
<span class="nb">print</span><span class="p">(</span><span class="s2">"This model has model id "</span> <span class="o">+</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"model_id"</span><span class="p">]</span> <span class="o">+</span> <span class="s2">"."</span><span class="p">)</span>
|
||||
<span class="n">model_verts</span><span class="p">,</span> <span class="n">model_faces</span> <span class="o">=</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"verts"</span><span class="p">],</span> <span class="n">shapenet_model</span><span class="p">[</span><span class="s2">"faces"</span><span class="p">]</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We can use its vertices and faces to form a <code>Meshes</code> object which is a PyTorch3D datastructure for working with batched meshes.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">model_textures</span> <span class="o">=</span> <span class="n">TexturesVertex</span><span class="p">(</span><span class="n">verts_features</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">ones_like</span><span class="p">(</span><span class="n">model_verts</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)[</span><span class="kc">None</span><span class="p">])</span>
|
||||
<span class="n">shapenet_model_mesh</span> <span class="o">=</span> <span class="n">Meshes</span><span class="p">(</span>
|
||||
<span class="n">verts</span><span class="o">=</span><span class="p">[</span><span class="n">model_verts</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)],</span>
|
||||
<span class="n">faces</span><span class="o">=</span><span class="p">[</span><span class="n">model_faces</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="n">device</span><span class="p">)],</span>
|
||||
<span class="n">textures</span><span class="o">=</span><span class="n">model_textures</span>
|
||||
<span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>With R2N2, we can further examine R2N2's original renderings. For instance, if we would like to see the second and third views of the eleventh objects in the R2N2 dataset, we can do the following:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">r2n2_renderings</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="p">[</span><span class="mi">10</span><span class="p">,[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">]]</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">r2n2_renderings</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="2.-Use-the-datasets-with-torch.utils.data.DataLoader">2. Use the datasets with <code>torch.utils.data.DataLoader</code><a class="anchor-link" href="#2.-Use-the-datasets-with-torch.utils.data.DataLoader">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Training deep learning models, usually requires passing in batches of inputs. The <code>torch.utils.data.DataLoader</code> from Pytorch helps us do this. PyTorch3D provides a function <code>collate_batched_meshes</code> to group the input meshes into a single <code>Meshes</code> object which represents the batch. The <code>Meshes</code> datastructure can then be used directly by other PyTorch3D ops which might be part of the deep learning model (e.g. <code>graph_conv</code>).</p>
|
||||
<p>For R2N2, if all the models in the batch have the same number of views, the views, rotation matrices, translation matrices, intrinsic matrices and voxels will also be stacked into batched tensors.</p>
|
||||
<p><strong>NOTE</strong>: All models in the <code>val</code> split of R2N2 have 24 views, but there are 8 models that split their 24 views between <code>train</code> and <code>test</code> splits, in which case <code>collate_batched_meshes</code> will only be able to join the matrices, views and voxels as lists. However, this can be avoided by laoding only one view of each model by setting <code>return_all_views = False</code>.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">batch_size</span> <span class="o">=</span> <span class="mi">12</span>
|
||||
<span class="n">r2n2_single_view</span> <span class="o">=</span> <span class="n">R2N2</span><span class="p">(</span><span class="s2">"train"</span><span class="p">,</span> <span class="n">SHAPENET_PATH</span><span class="p">,</span> <span class="n">R2N2_PATH</span><span class="p">,</span> <span class="n">SPLITS_PATH</span><span class="p">,</span> <span class="n">return_all_views</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">return_voxels</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">r2n2_loader</span> <span class="o">=</span> <span class="n">DataLoader</span><span class="p">(</span><span class="n">r2n2_single_view</span><span class="p">,</span> <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span> <span class="n">collate_fn</span><span class="o">=</span><span class="n">collate_batched_meshes</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Let's visualize all the views (one for each model) in the batch:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">it</span> <span class="o">=</span> <span class="nb">iter</span><span class="p">(</span><span class="n">r2n2_loader</span><span class="p">)</span>
|
||||
<span class="n">r2n2_batch</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span><span class="n">it</span><span class="p">)</span>
|
||||
<span class="n">batch_renderings</span> <span class="o">=</span> <span class="n">r2n2_batch</span><span class="p">[</span><span class="s2">"images"</span><span class="p">]</span> <span class="c1"># (N, V, H, W, 3), and in this case V is 1.</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">batch_renderings</span><span class="o">.</span><span class="n">squeeze</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="3.-Render-ShapeNetCore-models-with-PyTorch3D's-differntiable-renderer">3. Render ShapeNetCore models with PyTorch3D's differntiable renderer<a class="anchor-link" href="#3.-Render-ShapeNetCore-models-with-PyTorch3D's-differntiable-renderer">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Both <code>ShapeNetCore</code> and <code>R2N2</code> dataloaders have customized <code>render</code> functions that support rendering models by specifying their model ids, categories or indices using PyTorch3D's differentiable renderer implementation.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rendering settings.</span>
|
||||
<span class="n">R</span><span class="p">,</span> <span class="n">T</span> <span class="o">=</span> <span class="n">look_at_view_transform</span><span class="p">(</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="mi">90</span><span class="p">)</span>
|
||||
<span class="n">cameras</span> <span class="o">=</span> <span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">,</span> <span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="n">raster_settings</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span><span class="n">image_size</span><span class="o">=</span><span class="mi">512</span><span class="p">)</span>
|
||||
<span class="n">lights</span> <span class="o">=</span> <span class="n">PointLights</span><span class="p">(</span><span class="n">location</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">([</span><span class="mf">0.0</span><span class="p">,</span> <span class="mf">1.0</span><span class="p">,</span> <span class="o">-</span><span class="mf">2.0</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)[</span><span class="kc">None</span><span class="p">],</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>First we will try to render three models by their model ids:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">images_by_model_ids</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||
<span class="n">model_ids</span><span class="o">=</span><span class="p">[</span>
|
||||
<span class="s2">"13394ca47c89f91525a3aaf903a41c90"</span><span class="p">,</span>
|
||||
<span class="s2">"14755c2ee8e693aba508f621166382b0"</span><span class="p">,</span>
|
||||
<span class="s2">"156c4207af6d2c8f1fdc97905708b8ea"</span><span class="p">,</span>
|
||||
<span class="p">],</span>
|
||||
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">images_by_model_ids</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Suppose we would like to render the first three models in the dataset, we can render models by their indices:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">images_by_idxs</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||
<span class="n">idxs</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">)),</span>
|
||||
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">images_by_idxs</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Alternatively, if we are not interested in any particular models but would like see random models from some specific categories, we can do that by specifying <code>categories</code> and <code>sample_nums</code>. For example, if we would like to render 2 models from the category "faucet" and 3 models from the category "chair", we can do the following:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">images_by_categories</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||
<span class="n">categories</span><span class="o">=</span><span class="p">[</span><span class="s2">"faucet"</span><span class="p">,</span> <span class="s2">"chair"</span><span class="p">],</span>
|
||||
<span class="n">sample_nums</span><span class="o">=</span><span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span>
|
||||
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">images_by_categories</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If we are not interested in any particular categories and just would like to render some random models from the whole dataset, we can set the number of models to be rendered in <code>sample_nums</code> and not specify any <code>categories</code>:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">random_model_images</span> <span class="o">=</span> <span class="n">shapenet_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||
<span class="n">sample_nums</span><span class="o">=</span><span class="p">[</span><span class="mi">3</span><span class="p">],</span>
|
||||
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">random_model_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="4.-Render-R2N2-models-with-the-same-orientations-as-the-original-renderings-in-the-dataset">4. Render R2N2 models with the same orientations as the original renderings in the dataset<a class="anchor-link" href="#4.-Render-R2N2-models-with-the-same-orientations-as-the-original-renderings-in-the-dataset">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We can render R2N2 models the same way as we rendered ShapeNetCore models above. In addition, we can also render R2N2 models with the same orientations as the original renderings in the dataset. For this we will use R2N2's customized <code>render</code> function and a different type of PyTorch3D camera called <code>BlenderCamera</code>.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>In this example, we will render the seventh model with the same orientations as its second and third views. First we will retrieve R2N2's original renderings to compare with the result.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">original_rendering</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="p">[</span><span class="mi">6</span><span class="p">,[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">]][</span><span class="s2">"images"</span><span class="p">]</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">original_rendering</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Next, we will visualize PyTorch3d's renderings:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">r2n2_oriented_images</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="o">.</span><span class="n">render</span><span class="p">(</span>
|
||||
<span class="n">idxs</span><span class="o">=</span><span class="p">[</span><span class="mi">6</span><span class="p">],</span>
|
||||
<span class="n">view_idxs</span><span class="o">=</span><span class="p">[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">],</span>
|
||||
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">r2n2_oriented_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="5.-Visualize-R2N2-models'-voxels">5. Visualize R2N2 models' voxels<a class="anchor-link" href="#5.-Visualize-R2N2-models'-voxels">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>R2N2 dataloader also returns models' voxels. We can visualize them by utilizing R2N2's <code>render_vox_to_mesh</code> function. This will cubify the voxels to a Meshes object, which will then be rendered.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>In this example we will visualize the tenth model in the dataset with the same orientation of its second and third views. First we will retrieve R2N2's original renderings to compare with the result.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">r2n2_model</span> <span class="o">=</span> <span class="n">r2n2_dataset</span><span class="p">[</span><span class="mi">9</span><span class="p">,[</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">]]</span>
|
||||
<span class="n">original_rendering</span> <span class="o">=</span> <span class="n">r2n2_model</span><span class="p">[</span><span class="s2">"images"</span><span class="p">]</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">original_rendering</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Next, we will pass the voxels to <code>render_vox_to_mesh</code>:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">vox_render</span> <span class="o">=</span> <span class="n">render_cubified_voxels</span><span class="p">(</span><span class="n">r2n2_model</span><span class="p">[</span><span class="s2">"voxels"</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">vox_render</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
849
tutorials/fit_textured_mesh.html
Normal file
849
tutorials/fit_textured_mesh.html
Normal file
@@ -0,0 +1,849 @@
|
||||
<!DOCTYPE html><html lang=""><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>PyTorch3D · A library for deep learning with 3D data</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="A library for deep learning with 3D data"/><meta property="og:title" content="PyTorch3D · A library for deep learning with 3D data"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="A library for deep learning with 3D data"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-157376881-1', 'auto');
|
||||
ga('send', 'pageview');
|
||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="container docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span></span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Tutorials</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/">Overview</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">3D operators</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/deform_source_mesh_to_target_mesh">Fit Mesh</a></li><li class="navListItem"><a class="navItem" href="/tutorials/bundle_adjustment">Bundle Adjustment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Rendering</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/render_textured_meshes">Render Textured Meshes</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/tutorials/fit_textured_mesh">Fit a mesh with texture via rendering</a></li><li class="navListItem"><a class="navItem" href="/tutorials/camera_position_optimization_with_differentiable_rendering">Camera Position Optimization</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Dataloaders</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/dataloaders_ShapeNetCore_R2N2">Data loaders for ShapeNetCore and R2N2</a></li></ul></div></div></section></div><script>
|
||||
var coll = document.getElementsByClassName('collapsible');
|
||||
var checkActiveCategory = true;
|
||||
for (var i = 0; i < coll.length; i++) {
|
||||
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||
if (checkActiveCategory){
|
||||
for (var j = 0; j < links.length; j++) {
|
||||
if (links[j].classList.contains('navListItemActive')){
|
||||
coll[i].nextElementSibling.classList.toggle('hide');
|
||||
coll[i].childNodes[1].classList.toggle('rotate');
|
||||
checkActiveCategory = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
coll[i].addEventListener('click', function() {
|
||||
var arrow = this.childNodes[1];
|
||||
arrow.classList.toggle('rotate');
|
||||
var content = this.nextElementSibling;
|
||||
content.classList.toggle('hide');
|
||||
});
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||
createToggler('#tocToggler', 'body', 'tocActive');
|
||||
|
||||
var headings = document.querySelector('.toc-headings');
|
||||
headings && headings.addEventListener('click', function(event) {
|
||||
var el = event.target;
|
||||
while(el !== headings){
|
||||
if (el.tagName === 'A') {
|
||||
document.body.classList.remove('tocActive');
|
||||
break;
|
||||
} else{
|
||||
el = el.parentNode;
|
||||
}
|
||||
}
|
||||
}, false);
|
||||
|
||||
function createToggler(togglerSelector, targetSelector, className) {
|
||||
var toggler = document.querySelector(togglerSelector);
|
||||
var target = document.querySelector(targetSelector);
|
||||
|
||||
if (!toggler) {
|
||||
return;
|
||||
}
|
||||
|
||||
toggler.onclick = function(event) {
|
||||
event.preventDefault();
|
||||
|
||||
target.classList.toggle(className);
|
||||
};
|
||||
}
|
||||
});
|
||||
</script></nav></div><div class="container mainContainer"><div class="wrapper"><div class="tutorialButtonsWrapper"><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="https://colab.research.google.com/github/facebookresearch/pytorch3d/blob/stable/docs/tutorials/fit_textured_mesh.ipynb" target="_blank"><img class="colabButton" align="left" src="/img/colab_icon.png"/>Run in Google Colab</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/fit_textured_mesh.ipynb" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Jupyter Notebook</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/fit_textured_mesh.py" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Source Code</a></div></div><div class="tutorialBody">
|
||||
<script
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.10/require.min.js">
|
||||
</script>
|
||||
<script
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js">
|
||||
</script>
|
||||
<div class="notebook">
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h1 id="Fit-a-mesh-via-rendering">Fit a mesh via rendering<a class="anchor-link" href="#Fit-a-mesh-via-rendering">¶</a></h1><p>This tutorial shows how to:</p>
|
||||
<ul>
|
||||
<li>Load a mesh and textures from an <code>.obj</code> file. </li>
|
||||
<li>Create a synthetic dataset by rendering a textured mesh from multiple viewpoints</li>
|
||||
<li>Fit a mesh to the observed synthetic images using differential silhouette rendering</li>
|
||||
<li>Fit a mesh and its textures using differential textured rendering</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="0.-Install-and-Import-modules">0. Install and Import modules<a class="anchor-link" href="#0.-Install-and-Import-modules">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If <code>torch</code>, <code>torchvision</code> and <code>pytorch3d</code> are not installed, run the following cell:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>pip install torch torchvision
|
||||
<span class="kn">import</span> <span class="nn">sys</span>
|
||||
<span class="kn">import</span> <span class="nn">torch</span>
|
||||
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">__version__</span><span class="o">==</span><span class="s1">'1.6.0+cu101'</span> <span class="ow">and</span> <span class="n">sys</span><span class="o">.</span><span class="n">platform</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'linux'</span><span class="p">):</span>
|
||||
<span class="o">!</span>pip install pytorch3d
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="o">!</span>pip install <span class="s1">'git+https://github.com/facebookresearch/pytorch3d.git@stable'</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="kn">import</span> <span class="nn">os</span>
|
||||
<span class="kn">import</span> <span class="nn">torch</span>
|
||||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||||
<span class="kn">from</span> <span class="nn">skimage.io</span> <span class="k">import</span> <span class="n">imread</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.utils</span> <span class="k">import</span> <span class="n">ico_sphere</span>
|
||||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||||
<span class="kn">from</span> <span class="nn">tqdm.notebook</span> <span class="k">import</span> <span class="n">tqdm</span>
|
||||
|
||||
<span class="c1"># Util function for loading meshes</span>
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.io</span> <span class="k">import</span> <span class="n">load_objs_as_meshes</span><span class="p">,</span> <span class="n">save_obj</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.loss</span> <span class="k">import</span> <span class="p">(</span>
|
||||
<span class="n">chamfer_distance</span><span class="p">,</span>
|
||||
<span class="n">mesh_edge_loss</span><span class="p">,</span>
|
||||
<span class="n">mesh_laplacian_smoothing</span><span class="p">,</span>
|
||||
<span class="n">mesh_normal_consistency</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Data structures and functions for rendering</span>
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.structures</span> <span class="k">import</span> <span class="n">Meshes</span>
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.renderer</span> <span class="k">import</span> <span class="p">(</span>
|
||||
<span class="n">look_at_view_transform</span><span class="p">,</span>
|
||||
<span class="n">OpenGLPerspectiveCameras</span><span class="p">,</span>
|
||||
<span class="n">PointLights</span><span class="p">,</span>
|
||||
<span class="n">DirectionalLights</span><span class="p">,</span>
|
||||
<span class="n">Materials</span><span class="p">,</span>
|
||||
<span class="n">RasterizationSettings</span><span class="p">,</span>
|
||||
<span class="n">MeshRenderer</span><span class="p">,</span>
|
||||
<span class="n">MeshRasterizer</span><span class="p">,</span>
|
||||
<span class="n">SoftPhongShader</span><span class="p">,</span>
|
||||
<span class="n">SoftSilhouetteShader</span><span class="p">,</span>
|
||||
<span class="n">SoftPhongShader</span><span class="p">,</span>
|
||||
<span class="n">TexturesVertex</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># add path for demo utils functions </span>
|
||||
<span class="kn">import</span> <span class="nn">sys</span>
|
||||
<span class="kn">import</span> <span class="nn">os</span>
|
||||
<span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="s1">''</span><span class="p">))</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If using <strong>Google Colab</strong>, fetch the utils file for plotting image grids:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py
|
||||
<span class="kn">from</span> <span class="nn">plot_image_grid</span> <span class="k">import</span> <span class="n">image_grid</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>OR if running <strong>locally</strong> uncomment and run the following cell:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># from utils.plot_image_grid import image_grid</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h3 id="1.-Load-a-mesh-and-texture-file">1. Load a mesh and texture file<a class="anchor-link" href="#1.-Load-a-mesh-and-texture-file">¶</a></h3><p>Load an <code>.obj</code> file and it's associated <code>.mtl</code> file and create a <strong>Textures</strong> and <strong>Meshes</strong> object.</p>
|
||||
<p><strong>Meshes</strong> is a unique datastructure provided in PyTorch3D for working with batches of meshes of different sizes.</p>
|
||||
<p><strong>TexturesVertex</strong> is an auxillary datastructure for storing vertex rgb texture information about meshes.</p>
|
||||
<p><strong>Meshes</strong> has several class methods which are used throughout the rendering pipeline.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If running this notebook using <strong>Google Colab</strong>, run the following cell to fetch the mesh obj and texture files and save it at the path <code>data/cow_mesh</code>:
|
||||
If running locally, the data is already available at the correct path.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>mkdir -p data/cow_mesh
|
||||
<span class="o">!</span>wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.obj
|
||||
<span class="o">!</span>wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.mtl
|
||||
<span class="o">!</span>wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow_texture.png
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Setup</span>
|
||||
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
|
||||
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cuda:0"</span><span class="p">)</span>
|
||||
<span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">set_device</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cpu"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Set paths</span>
|
||||
<span class="n">DATA_DIR</span> <span class="o">=</span> <span class="s2">"./data"</span>
|
||||
<span class="n">obj_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">DATA_DIR</span><span class="p">,</span> <span class="s2">"cow_mesh/cow.obj"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Load obj file</span>
|
||||
<span class="n">mesh</span> <span class="o">=</span> <span class="n">load_objs_as_meshes</span><span class="p">([</span><span class="n">obj_filename</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># We scale normalize and center the target mesh to fit in a sphere of radius 1 </span>
|
||||
<span class="c1"># centered at (0,0,0). (scale, center) will be used to bring the predicted mesh </span>
|
||||
<span class="c1"># to its original center and scale. Note that normalizing the target mesh, </span>
|
||||
<span class="c1"># speeds up the optimization but is not necessary!</span>
|
||||
<span class="n">verts</span> <span class="o">=</span> <span class="n">mesh</span><span class="o">.</span><span class="n">verts_packed</span><span class="p">()</span>
|
||||
<span class="n">N</span> <span class="o">=</span> <span class="n">verts</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="n">center</span> <span class="o">=</span> <span class="n">verts</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">scale</span> <span class="o">=</span> <span class="nb">max</span><span class="p">((</span><span class="n">verts</span> <span class="o">-</span> <span class="n">center</span><span class="p">)</span><span class="o">.</span><span class="n">abs</span><span class="p">()</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="mi">0</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
|
||||
<span class="n">mesh</span><span class="o">.</span><span class="n">offset_verts_</span><span class="p">(</span><span class="o">-</span><span class="n">center</span><span class="o">.</span><span class="n">expand</span><span class="p">(</span><span class="n">N</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
|
||||
<span class="n">mesh</span><span class="o">.</span><span class="n">scale_verts_</span><span class="p">((</span><span class="mf">1.0</span> <span class="o">/</span> <span class="nb">float</span><span class="p">(</span><span class="n">scale</span><span class="p">)));</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="2.-Dataset-Creation">2. Dataset Creation<a class="anchor-link" href="#2.-Dataset-Creation">¶</a></h2><p>We sample different camera positions that encode multiple viewpoints of the cow. We create a renderer with a shader that performs texture map interpolation. We render a synthetic dataset of images of the textured cow mesh from multiple viewpoints.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># the number of different viewpoints from which we want to render the mesh.</span>
|
||||
<span class="n">num_views</span> <span class="o">=</span> <span class="mi">20</span>
|
||||
|
||||
<span class="c1"># Get a batch of viewing angles. </span>
|
||||
<span class="n">elev</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">360</span><span class="p">,</span> <span class="n">num_views</span><span class="p">)</span>
|
||||
<span class="n">azim</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">180</span><span class="p">,</span> <span class="mi">180</span><span class="p">,</span> <span class="n">num_views</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Place a point light in front of the object. As mentioned above, the front of </span>
|
||||
<span class="c1"># the cow is facing the -z direction. </span>
|
||||
<span class="n">lights</span> <span class="o">=</span> <span class="n">PointLights</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">location</span><span class="o">=</span><span class="p">[[</span><span class="mf">0.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="o">-</span><span class="mf">3.0</span><span class="p">]])</span>
|
||||
|
||||
<span class="c1"># Initialize an OpenGL perspective camera that represents a batch of different </span>
|
||||
<span class="c1"># viewing angles. All the cameras helper methods support mixed type inputs and </span>
|
||||
<span class="c1"># broadcasting. So we can view the camera from the a distance of dist=2.7, and </span>
|
||||
<span class="c1"># then specify elevation and azimuth angles for each viewpoint as tensors. </span>
|
||||
<span class="n">R</span><span class="p">,</span> <span class="n">T</span> <span class="o">=</span> <span class="n">look_at_view_transform</span><span class="p">(</span><span class="n">dist</span><span class="o">=</span><span class="mf">2.7</span><span class="p">,</span> <span class="n">elev</span><span class="o">=</span><span class="n">elev</span><span class="p">,</span> <span class="n">azim</span><span class="o">=</span><span class="n">azim</span><span class="p">)</span>
|
||||
<span class="n">cameras</span> <span class="o">=</span> <span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">,</span> <span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># We arbitrarily choose one particular view that will be used to visualize </span>
|
||||
<span class="c1"># results</span>
|
||||
<span class="n">camera</span> <span class="o">=</span> <span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">...</span><span class="p">],</span>
|
||||
<span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">...</span><span class="p">])</span>
|
||||
|
||||
<span class="c1"># Define the settings for rasterization and shading. Here we set the output </span>
|
||||
<span class="c1"># image to be of size 128X128. As we are rendering images for visualization </span>
|
||||
<span class="c1"># purposes only we will set faces_per_pixel=1 and blur_radius=0.0. Refer to </span>
|
||||
<span class="c1"># rasterize_meshes.py for explanations of these parameters. We also leave </span>
|
||||
<span class="c1"># bin_size and max_faces_per_bin to their default values of None, which sets </span>
|
||||
<span class="c1"># their values using huristics and ensures that the faster coarse-to-fine </span>
|
||||
<span class="c1"># rasterization method is used. Refer to docs/notes/renderer.md for an </span>
|
||||
<span class="c1"># explanation of the difference between naive and coarse-to-fine rasterization. </span>
|
||||
<span class="n">raster_settings</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||
<span class="n">blur_radius</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
|
||||
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Create a phong renderer by composing a rasterizer and a shader. The textured </span>
|
||||
<span class="c1"># phong shader will interpolate the texture uv coordinates for each vertex, </span>
|
||||
<span class="c1"># sample from a texture image and apply the Phong lighting model</span>
|
||||
<span class="n">renderer</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">shader</span><span class="o">=</span><span class="n">SoftPhongShader</span><span class="p">(</span>
|
||||
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Create a batch of meshes by repeating the cow mesh and associated textures. </span>
|
||||
<span class="c1"># Meshes has a useful `extend` method which allows us do this very easily. </span>
|
||||
<span class="c1"># This also extends the textures. </span>
|
||||
<span class="n">meshes</span> <span class="o">=</span> <span class="n">mesh</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">num_views</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Render the cow mesh from each viewing angle</span>
|
||||
<span class="n">target_images</span> <span class="o">=</span> <span class="n">renderer</span><span class="p">(</span><span class="n">meshes</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Our multi-view cow dataset will be represented by these 2 lists of tensors,</span>
|
||||
<span class="c1"># each of length num_views.</span>
|
||||
<span class="n">target_rgb</span> <span class="o">=</span> <span class="p">[</span><span class="n">target_images</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="p">:</span><span class="mi">3</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_views</span><span class="p">)]</span>
|
||||
<span class="n">target_cameras</span> <span class="o">=</span> <span class="p">[</span><span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">],</span>
|
||||
<span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">])</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_views</span><span class="p">)]</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Visualize the dataset:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># RGB images</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">target_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Later in this tutorial, we will fit a mesh to the rendered RGB images, as well as to just images of just the cow silhouette. For the latter case, we will render a dataset of silhouette images. Most shaders in PyTorch3D will output an alpha channel along with the RGB image as a 4th channel in an RGBA image. The alpha channel encodes the probability that each pixel belongs to the foreground of the object. We contruct a soft silhouette shader to render this alpha channel.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rasterization settings for silhouette rendering </span>
|
||||
<span class="n">sigma</span> <span class="o">=</span> <span class="mf">1e-4</span>
|
||||
<span class="n">raster_settings_silhouette</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||
<span class="n">blur_radius</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">1.</span> <span class="o">/</span> <span class="mf">1e-4</span> <span class="o">-</span> <span class="mf">1.</span><span class="p">)</span><span class="o">*</span><span class="n">sigma</span><span class="p">,</span>
|
||||
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Silhouette renderer </span>
|
||||
<span class="n">renderer_silhouette</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings_silhouette</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">shader</span><span class="o">=</span><span class="n">SoftSilhouetteShader</span><span class="p">()</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Render silhouette images. The 3rd channel of the rendering output is </span>
|
||||
<span class="c1"># the alpha/silhouette channel</span>
|
||||
<span class="n">silhouette_images</span> <span class="o">=</span> <span class="n">renderer_silhouette</span><span class="p">(</span><span class="n">meshes</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||
<span class="n">target_silhouette</span> <span class="o">=</span> <span class="p">[</span><span class="n">silhouette_images</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_views</span><span class="p">)]</span>
|
||||
|
||||
<span class="c1"># Visualize silhouette images</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">silhouette_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="3.-Mesh-prediction-via-silhouette-rendering">3. Mesh prediction via silhouette rendering<a class="anchor-link" href="#3.-Mesh-prediction-via-silhouette-rendering">¶</a></h2><p>In the previous section, we created a dataset of images of multiple viewpoints of a cow. In this section, we predict a mesh by observing those target images without any knowledge of the ground truth cow mesh. We assume we know the position of the cameras and lighting.</p>
|
||||
<p>We first define some helper functions to visualize the results of our mesh prediction:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Show a visualization comparing the rendered predicted mesh to the ground truth </span>
|
||||
<span class="c1"># mesh</span>
|
||||
<span class="k">def</span> <span class="nf">visualize_prediction</span><span class="p">(</span><span class="n">predicted_mesh</span><span class="p">,</span> <span class="n">renderer</span><span class="o">=</span><span class="n">renderer_silhouette</span><span class="p">,</span>
|
||||
<span class="n">target_image</span><span class="o">=</span><span class="n">target_rgb</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">title</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span>
|
||||
<span class="n">silhouette</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="n">inds</span> <span class="o">=</span> <span class="mi">3</span> <span class="k">if</span> <span class="n">silhouette</span> <span class="k">else</span> <span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
|
||||
<span class="n">predicted_images</span> <span class="o">=</span> <span class="n">renderer</span><span class="p">(</span><span class="n">predicted_mesh</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">predicted_images</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="n">inds</span><span class="p">]</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
|
||||
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">target_image</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">grid</span><span class="p">(</span><span class="s2">"off"</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">axis</span><span class="p">(</span><span class="s2">"off"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Plot losses as a function of optimization iteration</span>
|
||||
<span class="k">def</span> <span class="nf">plot_losses</span><span class="p">(</span><span class="n">losses</span><span class="p">):</span>
|
||||
<span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">13</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span>
|
||||
<span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">gca</span><span class="p">()</span>
|
||||
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">losses</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">l</span><span class="p">[</span><span class="s1">'values'</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="n">k</span> <span class="o">+</span> <span class="s2">" loss"</span><span class="p">)</span>
|
||||
<span class="n">ax</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||
<span class="n">ax</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s2">"Iteration"</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||
<span class="n">ax</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s2">"Loss"</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||
<span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s2">"Loss vs iterations"</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Starting from a sphere mesh, we will learn offsets of each vertex such that the predicted mesh silhouette is more similar to the target silhouette image at each optimization step. We begin by loading our initial sphere mesh:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># We initialize the source shape to be a sphere of radius 1. </span>
|
||||
<span class="n">src_mesh</span> <span class="o">=</span> <span class="n">ico_sphere</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="n">device</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We create a new differentiable renderer for rendering the silhouette of our predicted mesh:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rasterization settings for differentiable rendering, where the blur_radius</span>
|
||||
<span class="c1"># initialization is based on Liu et al, 'Soft Rasterizer: A Differentiable </span>
|
||||
<span class="c1"># Renderer for Image-based 3D Reasoning', ICCV 2019</span>
|
||||
<span class="n">sigma</span> <span class="o">=</span> <span class="mf">1e-4</span>
|
||||
<span class="n">raster_settings_soft</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||
<span class="n">blur_radius</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">1.</span> <span class="o">/</span> <span class="mf">1e-4</span> <span class="o">-</span> <span class="mf">1.</span><span class="p">)</span><span class="o">*</span><span class="n">sigma</span><span class="p">,</span>
|
||||
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Silhouette renderer </span>
|
||||
<span class="n">renderer_silhouette</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings_soft</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">shader</span><span class="o">=</span><span class="n">SoftSilhouetteShader</span><span class="p">()</span>
|
||||
<span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We initialize settings, losses, and the optimizer that will be used to iteratively fit our mesh to the target silhouettes:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Number of views to optimize over in each SGD iteration</span>
|
||||
<span class="n">num_views_per_iteration</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="c1"># Number of optimization steps</span>
|
||||
<span class="n">Niter</span> <span class="o">=</span> <span class="mi">2000</span>
|
||||
<span class="c1"># Plot period for the losses</span>
|
||||
<span class="n">plot_period</span> <span class="o">=</span> <span class="mi">250</span>
|
||||
|
||||
<span class="o">%</span><span class="k">matplotlib</span> inline
|
||||
|
||||
<span class="c1"># Optimize using rendered silhouette image loss, mesh edge loss, mesh normal </span>
|
||||
<span class="c1"># consistency, and mesh laplacian smoothing</span>
|
||||
<span class="n">losses</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"silhouette"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"edge"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"normal"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">0.01</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"laplacian"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># Losses to smooth / regularize the mesh shape</span>
|
||||
<span class="k">def</span> <span class="nf">update_mesh_shape_prior_losses</span><span class="p">(</span><span class="n">mesh</span><span class="p">,</span> <span class="n">loss</span><span class="p">):</span>
|
||||
<span class="c1"># and (b) the edge length of the predicted mesh</span>
|
||||
<span class="n">loss</span><span class="p">[</span><span class="s2">"edge"</span><span class="p">]</span> <span class="o">=</span> <span class="n">mesh_edge_loss</span><span class="p">(</span><span class="n">mesh</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># mesh normal consistency</span>
|
||||
<span class="n">loss</span><span class="p">[</span><span class="s2">"normal"</span><span class="p">]</span> <span class="o">=</span> <span class="n">mesh_normal_consistency</span><span class="p">(</span><span class="n">mesh</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># mesh laplacian smoothing</span>
|
||||
<span class="n">loss</span><span class="p">[</span><span class="s2">"laplacian"</span><span class="p">]</span> <span class="o">=</span> <span class="n">mesh_laplacian_smoothing</span><span class="p">(</span><span class="n">mesh</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s2">"uniform"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># We will learn to deform the source mesh by offsetting its vertices</span>
|
||||
<span class="c1"># The shape of the deform parameters is equal to the total number of vertices in</span>
|
||||
<span class="c1"># src_mesh</span>
|
||||
<span class="n">verts_shape</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">verts_packed</span><span class="p">()</span><span class="o">.</span><span class="n">shape</span>
|
||||
<span class="n">deform_verts</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">(</span><span class="n">verts_shape</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># The optimizer</span>
|
||||
<span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">([</span><span class="n">deform_verts</span><span class="p">],</span> <span class="n">lr</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We write an optimization loop to iteratively refine our predicted mesh from the sphere mesh into a mesh that matches the sillhouettes of the target images:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">loop</span> <span class="o">=</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">Niter</span><span class="p">))</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">loop</span><span class="p">:</span>
|
||||
<span class="c1"># Initialize optimizer</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
|
||||
|
||||
<span class="c1"># Deform the mesh</span>
|
||||
<span class="n">new_src_mesh</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">offset_verts</span><span class="p">(</span><span class="n">deform_verts</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Losses to smooth /regularize the mesh shape</span>
|
||||
<span class="n">loss</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">losses</span><span class="p">}</span>
|
||||
<span class="n">update_mesh_shape_prior_losses</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">loss</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Compute the average silhouette loss over two random views, as the average </span>
|
||||
<span class="c1"># squared L2 distance between the predicted silhouette and the target </span>
|
||||
<span class="c1"># silhouette from our dataset</span>
|
||||
<span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">permutation</span><span class="p">(</span><span class="n">num_views</span><span class="p">)</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[:</span><span class="n">num_views_per_iteration</span><span class="p">]:</span>
|
||||
<span class="n">images_predicted</span> <span class="o">=</span> <span class="n">renderer_silhouette</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">target_cameras</span><span class="p">[</span><span class="n">j</span><span class="p">],</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||
<span class="n">predicted_silhouette</span> <span class="o">=</span> <span class="n">images_predicted</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
|
||||
<span class="n">loss_silhouette</span> <span class="o">=</span> <span class="p">((</span><span class="n">predicted_silhouette</span> <span class="o">-</span> <span class="n">target_silhouette</span><span class="p">[</span><span class="n">j</span><span class="p">])</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||||
<span class="n">loss</span><span class="p">[</span><span class="s2">"silhouette"</span><span class="p">]</span> <span class="o">+=</span> <span class="n">loss_silhouette</span> <span class="o">/</span> <span class="n">num_views_per_iteration</span>
|
||||
|
||||
<span class="c1"># Weighted sum of the losses</span>
|
||||
<span class="n">sum_loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">loss</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">sum_loss</span> <span class="o">+=</span> <span class="n">l</span> <span class="o">*</span> <span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"weight"</span><span class="p">]</span>
|
||||
<span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"values"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Print the losses</span>
|
||||
<span class="n">loop</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">"total_loss = </span><span class="si">%.6f</span><span class="s2">"</span> <span class="o">%</span> <span class="n">sum_loss</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Plot mesh</span>
|
||||
<span class="k">if</span> <span class="n">i</span> <span class="o">%</span> <span class="n">plot_period</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="s2">"iter: </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">i</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">target_image</span><span class="o">=</span><span class="n">target_silhouette</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||||
|
||||
<span class="c1"># Optimization step</span>
|
||||
<span class="n">sum_loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">target_image</span><span class="o">=</span><span class="n">target_silhouette</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||||
<span class="n">plot_losses</span><span class="p">(</span><span class="n">losses</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="3.-Mesh-and-texture-prediction-via-textured-rendering">3. Mesh and texture prediction via textured rendering<a class="anchor-link" href="#3.-Mesh-and-texture-prediction-via-textured-rendering">¶</a></h2><p>We can predict both the mesh and its texture if we add an additional loss based on the comparing a predicted rendered RGB image to the target image. As before, we start with a sphere mesh. We learn both translational offsets and RGB texture colors for each vertex in the sphere mesh. Since our loss is based on rendered RGB pixel values instead of just the silhouette, we use a <strong>SoftPhongShader</strong> instead of a <strong>SoftSilhouetteShader</strong>.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rasterization settings for differentiable rendering, where the blur_radius</span>
|
||||
<span class="c1"># initialization is based on Liu et al, 'Soft Rasterizer: A Differentiable </span>
|
||||
<span class="c1"># Renderer for Image-based 3D Reasoning', ICCV 2019</span>
|
||||
<span class="n">sigma</span> <span class="o">=</span> <span class="mf">1e-4</span>
|
||||
<span class="n">raster_settings_soft</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||
<span class="n">blur_radius</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">1.</span> <span class="o">/</span> <span class="mf">1e-4</span> <span class="o">-</span> <span class="mf">1.</span><span class="p">)</span><span class="o">*</span><span class="n">sigma</span><span class="p">,</span>
|
||||
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Differentiable soft renderer using per vertex RGB colors for texture</span>
|
||||
<span class="n">renderer_textured</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings_soft</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">shader</span><span class="o">=</span><span class="n">SoftPhongShader</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We initialize settings, losses, and the optimizer that will be used to iteratively fit our mesh to the target RGB images:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Number of views to optimize over in each SGD iteration</span>
|
||||
<span class="n">num_views_per_iteration</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="c1"># Number of optimization steps</span>
|
||||
<span class="n">Niter</span> <span class="o">=</span> <span class="mi">2000</span>
|
||||
<span class="c1"># Plot period for the losses</span>
|
||||
<span class="n">plot_period</span> <span class="o">=</span> <span class="mi">250</span>
|
||||
|
||||
<span class="o">%</span><span class="k">matplotlib</span> inline
|
||||
|
||||
<span class="c1"># Optimize using rendered RGB image loss, rendered silhouette image loss, mesh </span>
|
||||
<span class="c1"># edge loss, mesh normal consistency, and mesh laplacian smoothing</span>
|
||||
<span class="n">losses</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"rgb"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"silhouette"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"edge"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"normal"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">0.01</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"laplacian"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># We will learn to deform the source mesh by offsetting its vertices</span>
|
||||
<span class="c1"># The shape of the deform parameters is equal to the total number of vertices in </span>
|
||||
<span class="c1"># src_mesh</span>
|
||||
<span class="n">verts_shape</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">verts_packed</span><span class="p">()</span><span class="o">.</span><span class="n">shape</span>
|
||||
<span class="n">deform_verts</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">(</span><span class="n">verts_shape</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># We will also learn per vertex colors for our sphere mesh that define texture </span>
|
||||
<span class="c1"># of the mesh</span>
|
||||
<span class="n">sphere_verts_rgb</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="n">verts_shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">3</span><span class="p">],</span> <span class="mf">0.5</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># The optimizer</span>
|
||||
<span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">([</span><span class="n">deform_verts</span><span class="p">,</span> <span class="n">sphere_verts_rgb</span><span class="p">],</span> <span class="n">lr</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We write an optimization loop to iteratively refine our predicted mesh and its vertex colors from the sphere mesh into a mesh that matches the target images:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">loop</span> <span class="o">=</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">Niter</span><span class="p">))</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">loop</span><span class="p">:</span>
|
||||
<span class="c1"># Initialize optimizer</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
|
||||
|
||||
<span class="c1"># Deform the mesh</span>
|
||||
<span class="n">new_src_mesh</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">offset_verts</span><span class="p">(</span><span class="n">deform_verts</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Add per vertex colors to texture the mesh</span>
|
||||
<span class="n">new_src_mesh</span><span class="o">.</span><span class="n">textures</span> <span class="o">=</span> <span class="n">TexturesVertex</span><span class="p">(</span><span class="n">verts_features</span><span class="o">=</span><span class="n">sphere_verts_rgb</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Losses to smooth /regularize the mesh shape</span>
|
||||
<span class="n">loss</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">losses</span><span class="p">}</span>
|
||||
<span class="n">update_mesh_shape_prior_losses</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">loss</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Randomly select two views to optimize over in this iteration. Compared</span>
|
||||
<span class="c1"># to using just one view, this helps resolve ambiguities between updating</span>
|
||||
<span class="c1"># mesh shape vs. updating mesh texture</span>
|
||||
<span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">permutation</span><span class="p">(</span><span class="n">num_views</span><span class="p">)</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[:</span><span class="n">num_views_per_iteration</span><span class="p">]:</span>
|
||||
<span class="n">images_predicted</span> <span class="o">=</span> <span class="n">renderer_textured</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">target_cameras</span><span class="p">[</span><span class="n">j</span><span class="p">],</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Squared L2 distance between the predicted silhouette and the target </span>
|
||||
<span class="c1"># silhouette from our dataset</span>
|
||||
<span class="n">predicted_silhouette</span> <span class="o">=</span> <span class="n">images_predicted</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
|
||||
<span class="n">loss_silhouette</span> <span class="o">=</span> <span class="p">((</span><span class="n">predicted_silhouette</span> <span class="o">-</span> <span class="n">target_silhouette</span><span class="p">[</span><span class="n">j</span><span class="p">])</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||||
<span class="n">loss</span><span class="p">[</span><span class="s2">"silhouette"</span><span class="p">]</span> <span class="o">+=</span> <span class="n">loss_silhouette</span> <span class="o">/</span> <span class="n">num_views_per_iteration</span>
|
||||
|
||||
<span class="c1"># Squared L2 distance between the predicted RGB image and the target </span>
|
||||
<span class="c1"># image from our dataset</span>
|
||||
<span class="n">predicted_rgb</span> <span class="o">=</span> <span class="n">images_predicted</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="p">:</span><span class="mi">3</span><span class="p">]</span>
|
||||
<span class="n">loss_rgb</span> <span class="o">=</span> <span class="p">((</span><span class="n">predicted_rgb</span> <span class="o">-</span> <span class="n">target_rgb</span><span class="p">[</span><span class="n">j</span><span class="p">])</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||||
<span class="n">loss</span><span class="p">[</span><span class="s2">"rgb"</span><span class="p">]</span> <span class="o">+=</span> <span class="n">loss_rgb</span> <span class="o">/</span> <span class="n">num_views_per_iteration</span>
|
||||
|
||||
<span class="c1"># Weighted sum of the losses</span>
|
||||
<span class="n">sum_loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">loss</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">sum_loss</span> <span class="o">+=</span> <span class="n">l</span> <span class="o">*</span> <span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"weight"</span><span class="p">]</span>
|
||||
<span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"values"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Print the losses</span>
|
||||
<span class="n">loop</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">"total_loss = </span><span class="si">%.6f</span><span class="s2">"</span> <span class="o">%</span> <span class="n">sum_loss</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Plot mesh</span>
|
||||
<span class="k">if</span> <span class="n">i</span> <span class="o">%</span> <span class="n">plot_period</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">renderer</span><span class="o">=</span><span class="n">renderer_textured</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="s2">"iter: </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">i</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Optimization step</span>
|
||||
<span class="n">sum_loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">renderer</span><span class="o">=</span><span class="n">renderer_textured</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
<span class="n">plot_losses</span><span class="p">(</span><span class="n">losses</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Save the final predicted mesh:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="4.-Save-the-final-predicted-mesh">4. Save the final predicted mesh<a class="anchor-link" href="#4.-Save-the-final-predicted-mesh">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Fetch the verts and faces of the final predicted mesh</span>
|
||||
<span class="n">final_verts</span><span class="p">,</span> <span class="n">final_faces</span> <span class="o">=</span> <span class="n">new_src_mesh</span><span class="o">.</span><span class="n">get_mesh_verts_faces</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Scale normalize back to the original target size</span>
|
||||
<span class="n">final_verts</span> <span class="o">=</span> <span class="n">final_verts</span> <span class="o">*</span> <span class="n">scale</span> <span class="o">+</span> <span class="n">center</span>
|
||||
|
||||
<span class="c1"># Store the predicted mesh using save_obj</span>
|
||||
<span class="n">final_obj</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s1">'./'</span><span class="p">,</span> <span class="s1">'final_model.obj'</span><span class="p">)</span>
|
||||
<span class="n">save_obj</span><span class="p">(</span><span class="n">final_obj</span><span class="p">,</span> <span class="n">final_verts</span><span class="p">,</span> <span class="n">final_faces</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="5.-Conclusion">5. Conclusion<a class="anchor-link" href="#5.-Conclusion">¶</a></h2><p>In this tutorial, we learned how to load a textured mesh from an obj file, create a synthetic dataset by rendering the mesh from multiple viewpoints. We showed how to set up an optimization loop to fit a mesh to the observed dataset images based on a rendered silhouette loss. We then augmented this optimization loop with an additional loss based on rendered RGB images, which allowed us to predict both a mesh and its texture.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||
849
tutorials/fit_textured_mesh/index.html
Normal file
849
tutorials/fit_textured_mesh/index.html
Normal file
@@ -0,0 +1,849 @@
|
||||
<!DOCTYPE html><html lang=""><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>PyTorch3D · A library for deep learning with 3D data</title><meta name="viewport" content="width=device-width"/><meta name="generator" content="Docusaurus"/><meta name="description" content="A library for deep learning with 3D data"/><meta property="og:title" content="PyTorch3D · A library for deep learning with 3D data"/><meta property="og:type" content="website"/><meta property="og:url" content="https://pytorch3d.org/"/><meta property="og:description" content="A library for deep learning with 3D data"/><meta property="og:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><meta name="twitter:card" content="summary"/><meta name="twitter:image" content="https://pytorch3d.org/img/pytorch3dlogoicon.svg"/><link rel="shortcut icon" href="/img/pytorch3dfavicon.png"/><link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/default.min.css"/><script>
|
||||
(function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
|
||||
(i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
|
||||
m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
|
||||
})(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
|
||||
|
||||
ga('create', 'UA-157376881-1', 'auto');
|
||||
ga('send', 'pageview');
|
||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="container docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span></span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Tutorials</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/">Overview</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">3D operators</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/deform_source_mesh_to_target_mesh">Fit Mesh</a></li><li class="navListItem"><a class="navItem" href="/tutorials/bundle_adjustment">Bundle Adjustment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Rendering</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/render_textured_meshes">Render Textured Meshes</a></li><li class="navListItem navListItemActive"><a class="navItem" href="/tutorials/fit_textured_mesh">Fit a mesh with texture via rendering</a></li><li class="navListItem"><a class="navItem" href="/tutorials/camera_position_optimization_with_differentiable_rendering">Camera Position Optimization</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Dataloaders</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/dataloaders_ShapeNetCore_R2N2">Data loaders for ShapeNetCore and R2N2</a></li></ul></div></div></section></div><script>
|
||||
var coll = document.getElementsByClassName('collapsible');
|
||||
var checkActiveCategory = true;
|
||||
for (var i = 0; i < coll.length; i++) {
|
||||
var links = coll[i].nextElementSibling.getElementsByTagName('*');
|
||||
if (checkActiveCategory){
|
||||
for (var j = 0; j < links.length; j++) {
|
||||
if (links[j].classList.contains('navListItemActive')){
|
||||
coll[i].nextElementSibling.classList.toggle('hide');
|
||||
coll[i].childNodes[1].classList.toggle('rotate');
|
||||
checkActiveCategory = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
coll[i].addEventListener('click', function() {
|
||||
var arrow = this.childNodes[1];
|
||||
arrow.classList.toggle('rotate');
|
||||
var content = this.nextElementSibling;
|
||||
content.classList.toggle('hide');
|
||||
});
|
||||
}
|
||||
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
createToggler('#navToggler', '#docsNav', 'docsSliderActive');
|
||||
createToggler('#tocToggler', 'body', 'tocActive');
|
||||
|
||||
var headings = document.querySelector('.toc-headings');
|
||||
headings && headings.addEventListener('click', function(event) {
|
||||
var el = event.target;
|
||||
while(el !== headings){
|
||||
if (el.tagName === 'A') {
|
||||
document.body.classList.remove('tocActive');
|
||||
break;
|
||||
} else{
|
||||
el = el.parentNode;
|
||||
}
|
||||
}
|
||||
}, false);
|
||||
|
||||
function createToggler(togglerSelector, targetSelector, className) {
|
||||
var toggler = document.querySelector(togglerSelector);
|
||||
var target = document.querySelector(targetSelector);
|
||||
|
||||
if (!toggler) {
|
||||
return;
|
||||
}
|
||||
|
||||
toggler.onclick = function(event) {
|
||||
event.preventDefault();
|
||||
|
||||
target.classList.toggle(className);
|
||||
};
|
||||
}
|
||||
});
|
||||
</script></nav></div><div class="container mainContainer"><div class="wrapper"><div class="tutorialButtonsWrapper"><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="https://colab.research.google.com/github/facebookresearch/pytorch3d/blob/stable/docs/tutorials/fit_textured_mesh.ipynb" target="_blank"><img class="colabButton" align="left" src="/img/colab_icon.png"/>Run in Google Colab</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/fit_textured_mesh.ipynb" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Jupyter Notebook</a></div><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" download="" href="/files/fit_textured_mesh.py" target="_blank"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="file-download" class="svg-inline--fa fa-file-download fa-w-12" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 384 512"><path fill="currentColor" d="M224 136V0H24C10.7 0 0 10.7 0 24v464c0 13.3 10.7 24 24 24h336c13.3 0 24-10.7 24-24V160H248c-13.2 0-24-10.8-24-24zm76.45 211.36l-96.42 95.7c-6.65 6.61-17.39 6.61-24.04 0l-96.42-95.7C73.42 337.29 80.54 320 94.82 320H160v-80c0-8.84 7.16-16 16-16h32c8.84 0 16 7.16 16 16v80h65.18c14.28 0 21.4 17.29 11.27 27.36zM377 105L279.1 7c-4.5-4.5-10.6-7-17-7H256v128h128v-6.1c0-6.3-2.5-12.4-7-16.9z"></path></svg>Download Tutorial Source Code</a></div></div><div class="tutorialBody">
|
||||
<script
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.1.10/require.min.js">
|
||||
</script>
|
||||
<script
|
||||
src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.0.3/jquery.min.js">
|
||||
</script>
|
||||
<div class="notebook">
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h1 id="Fit-a-mesh-via-rendering">Fit a mesh via rendering<a class="anchor-link" href="#Fit-a-mesh-via-rendering">¶</a></h1><p>This tutorial shows how to:</p>
|
||||
<ul>
|
||||
<li>Load a mesh and textures from an <code>.obj</code> file. </li>
|
||||
<li>Create a synthetic dataset by rendering a textured mesh from multiple viewpoints</li>
|
||||
<li>Fit a mesh to the observed synthetic images using differential silhouette rendering</li>
|
||||
<li>Fit a mesh and its textures using differential textured rendering</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="0.-Install-and-Import-modules">0. Install and Import modules<a class="anchor-link" href="#0.-Install-and-Import-modules">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If <code>torch</code>, <code>torchvision</code> and <code>pytorch3d</code> are not installed, run the following cell:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>pip install torch torchvision
|
||||
<span class="kn">import</span> <span class="nn">sys</span>
|
||||
<span class="kn">import</span> <span class="nn">torch</span>
|
||||
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">__version__</span><span class="o">==</span><span class="s1">'1.6.0+cu101'</span> <span class="ow">and</span> <span class="n">sys</span><span class="o">.</span><span class="n">platform</span><span class="o">.</span><span class="n">startswith</span><span class="p">(</span><span class="s1">'linux'</span><span class="p">):</span>
|
||||
<span class="o">!</span>pip install pytorch3d
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="o">!</span>pip install <span class="s1">'git+https://github.com/facebookresearch/pytorch3d.git@stable'</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="kn">import</span> <span class="nn">os</span>
|
||||
<span class="kn">import</span> <span class="nn">torch</span>
|
||||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||||
<span class="kn">from</span> <span class="nn">skimage.io</span> <span class="k">import</span> <span class="n">imread</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.utils</span> <span class="k">import</span> <span class="n">ico_sphere</span>
|
||||
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||||
<span class="kn">from</span> <span class="nn">tqdm.notebook</span> <span class="k">import</span> <span class="n">tqdm</span>
|
||||
|
||||
<span class="c1"># Util function for loading meshes</span>
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.io</span> <span class="k">import</span> <span class="n">load_objs_as_meshes</span><span class="p">,</span> <span class="n">save_obj</span>
|
||||
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.loss</span> <span class="k">import</span> <span class="p">(</span>
|
||||
<span class="n">chamfer_distance</span><span class="p">,</span>
|
||||
<span class="n">mesh_edge_loss</span><span class="p">,</span>
|
||||
<span class="n">mesh_laplacian_smoothing</span><span class="p">,</span>
|
||||
<span class="n">mesh_normal_consistency</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Data structures and functions for rendering</span>
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.structures</span> <span class="k">import</span> <span class="n">Meshes</span>
|
||||
<span class="kn">from</span> <span class="nn">pytorch3d.renderer</span> <span class="k">import</span> <span class="p">(</span>
|
||||
<span class="n">look_at_view_transform</span><span class="p">,</span>
|
||||
<span class="n">OpenGLPerspectiveCameras</span><span class="p">,</span>
|
||||
<span class="n">PointLights</span><span class="p">,</span>
|
||||
<span class="n">DirectionalLights</span><span class="p">,</span>
|
||||
<span class="n">Materials</span><span class="p">,</span>
|
||||
<span class="n">RasterizationSettings</span><span class="p">,</span>
|
||||
<span class="n">MeshRenderer</span><span class="p">,</span>
|
||||
<span class="n">MeshRasterizer</span><span class="p">,</span>
|
||||
<span class="n">SoftPhongShader</span><span class="p">,</span>
|
||||
<span class="n">SoftSilhouetteShader</span><span class="p">,</span>
|
||||
<span class="n">SoftPhongShader</span><span class="p">,</span>
|
||||
<span class="n">TexturesVertex</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># add path for demo utils functions </span>
|
||||
<span class="kn">import</span> <span class="nn">sys</span>
|
||||
<span class="kn">import</span> <span class="nn">os</span>
|
||||
<span class="n">sys</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">abspath</span><span class="p">(</span><span class="s1">''</span><span class="p">))</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If using <strong>Google Colab</strong>, fetch the utils file for plotting image grids:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>wget https://raw.githubusercontent.com/facebookresearch/pytorch3d/master/docs/tutorials/utils/plot_image_grid.py
|
||||
<span class="kn">from</span> <span class="nn">plot_image_grid</span> <span class="k">import</span> <span class="n">image_grid</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>OR if running <strong>locally</strong> uncomment and run the following cell:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># from utils.plot_image_grid import image_grid</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h3 id="1.-Load-a-mesh-and-texture-file">1. Load a mesh and texture file<a class="anchor-link" href="#1.-Load-a-mesh-and-texture-file">¶</a></h3><p>Load an <code>.obj</code> file and it's associated <code>.mtl</code> file and create a <strong>Textures</strong> and <strong>Meshes</strong> object.</p>
|
||||
<p><strong>Meshes</strong> is a unique datastructure provided in PyTorch3D for working with batches of meshes of different sizes.</p>
|
||||
<p><strong>TexturesVertex</strong> is an auxillary datastructure for storing vertex rgb texture information about meshes.</p>
|
||||
<p><strong>Meshes</strong> has several class methods which are used throughout the rendering pipeline.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>If running this notebook using <strong>Google Colab</strong>, run the following cell to fetch the mesh obj and texture files and save it at the path <code>data/cow_mesh</code>:
|
||||
If running locally, the data is already available at the correct path.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="o">!</span>mkdir -p data/cow_mesh
|
||||
<span class="o">!</span>wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.obj
|
||||
<span class="o">!</span>wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow.mtl
|
||||
<span class="o">!</span>wget -P data/cow_mesh https://dl.fbaipublicfiles.com/pytorch3d/data/cow_mesh/cow_texture.png
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Setup</span>
|
||||
<span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
|
||||
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cuda:0"</span><span class="p">)</span>
|
||||
<span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">set_device</span><span class="p">(</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">device</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">device</span><span class="p">(</span><span class="s2">"cpu"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Set paths</span>
|
||||
<span class="n">DATA_DIR</span> <span class="o">=</span> <span class="s2">"./data"</span>
|
||||
<span class="n">obj_filename</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="n">DATA_DIR</span><span class="p">,</span> <span class="s2">"cow_mesh/cow.obj"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Load obj file</span>
|
||||
<span class="n">mesh</span> <span class="o">=</span> <span class="n">load_objs_as_meshes</span><span class="p">([</span><span class="n">obj_filename</span><span class="p">],</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># We scale normalize and center the target mesh to fit in a sphere of radius 1 </span>
|
||||
<span class="c1"># centered at (0,0,0). (scale, center) will be used to bring the predicted mesh </span>
|
||||
<span class="c1"># to its original center and scale. Note that normalizing the target mesh, </span>
|
||||
<span class="c1"># speeds up the optimization but is not necessary!</span>
|
||||
<span class="n">verts</span> <span class="o">=</span> <span class="n">mesh</span><span class="o">.</span><span class="n">verts_packed</span><span class="p">()</span>
|
||||
<span class="n">N</span> <span class="o">=</span> <span class="n">verts</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="n">center</span> <span class="o">=</span> <span class="n">verts</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||
<span class="n">scale</span> <span class="o">=</span> <span class="nb">max</span><span class="p">((</span><span class="n">verts</span> <span class="o">-</span> <span class="n">center</span><span class="p">)</span><span class="o">.</span><span class="n">abs</span><span class="p">()</span><span class="o">.</span><span class="n">max</span><span class="p">(</span><span class="mi">0</span><span class="p">)[</span><span class="mi">0</span><span class="p">])</span>
|
||||
<span class="n">mesh</span><span class="o">.</span><span class="n">offset_verts_</span><span class="p">(</span><span class="o">-</span><span class="n">center</span><span class="o">.</span><span class="n">expand</span><span class="p">(</span><span class="n">N</span><span class="p">,</span> <span class="mi">3</span><span class="p">))</span>
|
||||
<span class="n">mesh</span><span class="o">.</span><span class="n">scale_verts_</span><span class="p">((</span><span class="mf">1.0</span> <span class="o">/</span> <span class="nb">float</span><span class="p">(</span><span class="n">scale</span><span class="p">)));</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="2.-Dataset-Creation">2. Dataset Creation<a class="anchor-link" href="#2.-Dataset-Creation">¶</a></h2><p>We sample different camera positions that encode multiple viewpoints of the cow. We create a renderer with a shader that performs texture map interpolation. We render a synthetic dataset of images of the textured cow mesh from multiple viewpoints.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># the number of different viewpoints from which we want to render the mesh.</span>
|
||||
<span class="n">num_views</span> <span class="o">=</span> <span class="mi">20</span>
|
||||
|
||||
<span class="c1"># Get a batch of viewing angles. </span>
|
||||
<span class="n">elev</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mi">360</span><span class="p">,</span> <span class="n">num_views</span><span class="p">)</span>
|
||||
<span class="n">azim</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">180</span><span class="p">,</span> <span class="mi">180</span><span class="p">,</span> <span class="n">num_views</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Place a point light in front of the object. As mentioned above, the front of </span>
|
||||
<span class="c1"># the cow is facing the -z direction. </span>
|
||||
<span class="n">lights</span> <span class="o">=</span> <span class="n">PointLights</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">location</span><span class="o">=</span><span class="p">[[</span><span class="mf">0.0</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="o">-</span><span class="mf">3.0</span><span class="p">]])</span>
|
||||
|
||||
<span class="c1"># Initialize an OpenGL perspective camera that represents a batch of different </span>
|
||||
<span class="c1"># viewing angles. All the cameras helper methods support mixed type inputs and </span>
|
||||
<span class="c1"># broadcasting. So we can view the camera from the a distance of dist=2.7, and </span>
|
||||
<span class="c1"># then specify elevation and azimuth angles for each viewpoint as tensors. </span>
|
||||
<span class="n">R</span><span class="p">,</span> <span class="n">T</span> <span class="o">=</span> <span class="n">look_at_view_transform</span><span class="p">(</span><span class="n">dist</span><span class="o">=</span><span class="mf">2.7</span><span class="p">,</span> <span class="n">elev</span><span class="o">=</span><span class="n">elev</span><span class="p">,</span> <span class="n">azim</span><span class="o">=</span><span class="n">azim</span><span class="p">)</span>
|
||||
<span class="n">cameras</span> <span class="o">=</span> <span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">,</span> <span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># We arbitrarily choose one particular view that will be used to visualize </span>
|
||||
<span class="c1"># results</span>
|
||||
<span class="n">camera</span> <span class="o">=</span> <span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">...</span><span class="p">],</span>
|
||||
<span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="o">...</span><span class="p">])</span>
|
||||
|
||||
<span class="c1"># Define the settings for rasterization and shading. Here we set the output </span>
|
||||
<span class="c1"># image to be of size 128X128. As we are rendering images for visualization </span>
|
||||
<span class="c1"># purposes only we will set faces_per_pixel=1 and blur_radius=0.0. Refer to </span>
|
||||
<span class="c1"># rasterize_meshes.py for explanations of these parameters. We also leave </span>
|
||||
<span class="c1"># bin_size and max_faces_per_bin to their default values of None, which sets </span>
|
||||
<span class="c1"># their values using huristics and ensures that the faster coarse-to-fine </span>
|
||||
<span class="c1"># rasterization method is used. Refer to docs/notes/renderer.md for an </span>
|
||||
<span class="c1"># explanation of the difference between naive and coarse-to-fine rasterization. </span>
|
||||
<span class="n">raster_settings</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||
<span class="n">blur_radius</span><span class="o">=</span><span class="mf">0.0</span><span class="p">,</span>
|
||||
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Create a phong renderer by composing a rasterizer and a shader. The textured </span>
|
||||
<span class="c1"># phong shader will interpolate the texture uv coordinates for each vertex, </span>
|
||||
<span class="c1"># sample from a texture image and apply the Phong lighting model</span>
|
||||
<span class="n">renderer</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">shader</span><span class="o">=</span><span class="n">SoftPhongShader</span><span class="p">(</span>
|
||||
<span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span>
|
||||
<span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Create a batch of meshes by repeating the cow mesh and associated textures. </span>
|
||||
<span class="c1"># Meshes has a useful `extend` method which allows us do this very easily. </span>
|
||||
<span class="c1"># This also extends the textures. </span>
|
||||
<span class="n">meshes</span> <span class="o">=</span> <span class="n">mesh</span><span class="o">.</span><span class="n">extend</span><span class="p">(</span><span class="n">num_views</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Render the cow mesh from each viewing angle</span>
|
||||
<span class="n">target_images</span> <span class="o">=</span> <span class="n">renderer</span><span class="p">(</span><span class="n">meshes</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Our multi-view cow dataset will be represented by these 2 lists of tensors,</span>
|
||||
<span class="c1"># each of length num_views.</span>
|
||||
<span class="n">target_rgb</span> <span class="o">=</span> <span class="p">[</span><span class="n">target_images</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="p">:</span><span class="mi">3</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_views</span><span class="p">)]</span>
|
||||
<span class="n">target_cameras</span> <span class="o">=</span> <span class="p">[</span><span class="n">OpenGLPerspectiveCameras</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">R</span><span class="o">=</span><span class="n">R</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">],</span>
|
||||
<span class="n">T</span><span class="o">=</span><span class="n">T</span><span class="p">[</span><span class="kc">None</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">])</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_views</span><span class="p">)]</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Visualize the dataset:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># RGB images</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">target_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Later in this tutorial, we will fit a mesh to the rendered RGB images, as well as to just images of just the cow silhouette. For the latter case, we will render a dataset of silhouette images. Most shaders in PyTorch3D will output an alpha channel along with the RGB image as a 4th channel in an RGBA image. The alpha channel encodes the probability that each pixel belongs to the foreground of the object. We contruct a soft silhouette shader to render this alpha channel.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rasterization settings for silhouette rendering </span>
|
||||
<span class="n">sigma</span> <span class="o">=</span> <span class="mf">1e-4</span>
|
||||
<span class="n">raster_settings_silhouette</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||
<span class="n">blur_radius</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">1.</span> <span class="o">/</span> <span class="mf">1e-4</span> <span class="o">-</span> <span class="mf">1.</span><span class="p">)</span><span class="o">*</span><span class="n">sigma</span><span class="p">,</span>
|
||||
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Silhouette renderer </span>
|
||||
<span class="n">renderer_silhouette</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings_silhouette</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">shader</span><span class="o">=</span><span class="n">SoftSilhouetteShader</span><span class="p">()</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Render silhouette images. The 3rd channel of the rendering output is </span>
|
||||
<span class="c1"># the alpha/silhouette channel</span>
|
||||
<span class="n">silhouette_images</span> <span class="o">=</span> <span class="n">renderer_silhouette</span><span class="p">(</span><span class="n">meshes</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">cameras</span><span class="p">,</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||
<span class="n">target_silhouette</span> <span class="o">=</span> <span class="p">[</span><span class="n">silhouette_images</span><span class="p">[</span><span class="n">i</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">num_views</span><span class="p">)]</span>
|
||||
|
||||
<span class="c1"># Visualize silhouette images</span>
|
||||
<span class="n">image_grid</span><span class="p">(</span><span class="n">silhouette_images</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">(),</span> <span class="n">rows</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">cols</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">rgb</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="3.-Mesh-prediction-via-silhouette-rendering">3. Mesh prediction via silhouette rendering<a class="anchor-link" href="#3.-Mesh-prediction-via-silhouette-rendering">¶</a></h2><p>In the previous section, we created a dataset of images of multiple viewpoints of a cow. In this section, we predict a mesh by observing those target images without any knowledge of the ground truth cow mesh. We assume we know the position of the cameras and lighting.</p>
|
||||
<p>We first define some helper functions to visualize the results of our mesh prediction:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Show a visualization comparing the rendered predicted mesh to the ground truth </span>
|
||||
<span class="c1"># mesh</span>
|
||||
<span class="k">def</span> <span class="nf">visualize_prediction</span><span class="p">(</span><span class="n">predicted_mesh</span><span class="p">,</span> <span class="n">renderer</span><span class="o">=</span><span class="n">renderer_silhouette</span><span class="p">,</span>
|
||||
<span class="n">target_image</span><span class="o">=</span><span class="n">target_rgb</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">title</span><span class="o">=</span><span class="s1">''</span><span class="p">,</span>
|
||||
<span class="n">silhouette</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
|
||||
<span class="n">inds</span> <span class="o">=</span> <span class="mi">3</span> <span class="k">if</span> <span class="n">silhouette</span> <span class="k">else</span> <span class="nb">range</span><span class="p">(</span><span class="mi">3</span><span class="p">)</span>
|
||||
<span class="n">predicted_images</span> <span class="o">=</span> <span class="n">renderer</span><span class="p">(</span><span class="n">predicted_mesh</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">predicted_images</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="o">...</span><span class="p">,</span> <span class="n">inds</span><span class="p">]</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
|
||||
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">subplot</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">imshow</span><span class="p">(</span><span class="n">target_image</span><span class="o">.</span><span class="n">cpu</span><span class="p">()</span><span class="o">.</span><span class="n">detach</span><span class="p">()</span><span class="o">.</span><span class="n">numpy</span><span class="p">())</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">title</span><span class="p">(</span><span class="n">title</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">grid</span><span class="p">(</span><span class="s2">"off"</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">axis</span><span class="p">(</span><span class="s2">"off"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Plot losses as a function of optimization iteration</span>
|
||||
<span class="k">def</span> <span class="nf">plot_losses</span><span class="p">(</span><span class="n">losses</span><span class="p">):</span>
|
||||
<span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">13</span><span class="p">,</span> <span class="mi">5</span><span class="p">))</span>
|
||||
<span class="n">ax</span> <span class="o">=</span> <span class="n">fig</span><span class="o">.</span><span class="n">gca</span><span class="p">()</span>
|
||||
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">losses</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">ax</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">l</span><span class="p">[</span><span class="s1">'values'</span><span class="p">],</span> <span class="n">label</span><span class="o">=</span><span class="n">k</span> <span class="o">+</span> <span class="s2">" loss"</span><span class="p">)</span>
|
||||
<span class="n">ax</span><span class="o">.</span><span class="n">legend</span><span class="p">(</span><span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||
<span class="n">ax</span><span class="o">.</span><span class="n">set_xlabel</span><span class="p">(</span><span class="s2">"Iteration"</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||
<span class="n">ax</span><span class="o">.</span><span class="n">set_ylabel</span><span class="p">(</span><span class="s2">"Loss"</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||
<span class="n">ax</span><span class="o">.</span><span class="n">set_title</span><span class="p">(</span><span class="s2">"Loss vs iterations"</span><span class="p">,</span> <span class="n">fontsize</span><span class="o">=</span><span class="s2">"16"</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Starting from a sphere mesh, we will learn offsets of each vertex such that the predicted mesh silhouette is more similar to the target silhouette image at each optimization step. We begin by loading our initial sphere mesh:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># We initialize the source shape to be a sphere of radius 1. </span>
|
||||
<span class="n">src_mesh</span> <span class="o">=</span> <span class="n">ico_sphere</span><span class="p">(</span><span class="mi">4</span><span class="p">,</span> <span class="n">device</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We create a new differentiable renderer for rendering the silhouette of our predicted mesh:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rasterization settings for differentiable rendering, where the blur_radius</span>
|
||||
<span class="c1"># initialization is based on Liu et al, 'Soft Rasterizer: A Differentiable </span>
|
||||
<span class="c1"># Renderer for Image-based 3D Reasoning', ICCV 2019</span>
|
||||
<span class="n">sigma</span> <span class="o">=</span> <span class="mf">1e-4</span>
|
||||
<span class="n">raster_settings_soft</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||
<span class="n">blur_radius</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">1.</span> <span class="o">/</span> <span class="mf">1e-4</span> <span class="o">-</span> <span class="mf">1.</span><span class="p">)</span><span class="o">*</span><span class="n">sigma</span><span class="p">,</span>
|
||||
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Silhouette renderer </span>
|
||||
<span class="n">renderer_silhouette</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings_soft</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">shader</span><span class="o">=</span><span class="n">SoftSilhouetteShader</span><span class="p">()</span>
|
||||
<span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We initialize settings, losses, and the optimizer that will be used to iteratively fit our mesh to the target silhouettes:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Number of views to optimize over in each SGD iteration</span>
|
||||
<span class="n">num_views_per_iteration</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="c1"># Number of optimization steps</span>
|
||||
<span class="n">Niter</span> <span class="o">=</span> <span class="mi">2000</span>
|
||||
<span class="c1"># Plot period for the losses</span>
|
||||
<span class="n">plot_period</span> <span class="o">=</span> <span class="mi">250</span>
|
||||
|
||||
<span class="o">%</span><span class="k">matplotlib</span> inline
|
||||
|
||||
<span class="c1"># Optimize using rendered silhouette image loss, mesh edge loss, mesh normal </span>
|
||||
<span class="c1"># consistency, and mesh laplacian smoothing</span>
|
||||
<span class="n">losses</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"silhouette"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"edge"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"normal"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">0.01</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"laplacian"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># Losses to smooth / regularize the mesh shape</span>
|
||||
<span class="k">def</span> <span class="nf">update_mesh_shape_prior_losses</span><span class="p">(</span><span class="n">mesh</span><span class="p">,</span> <span class="n">loss</span><span class="p">):</span>
|
||||
<span class="c1"># and (b) the edge length of the predicted mesh</span>
|
||||
<span class="n">loss</span><span class="p">[</span><span class="s2">"edge"</span><span class="p">]</span> <span class="o">=</span> <span class="n">mesh_edge_loss</span><span class="p">(</span><span class="n">mesh</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># mesh normal consistency</span>
|
||||
<span class="n">loss</span><span class="p">[</span><span class="s2">"normal"</span><span class="p">]</span> <span class="o">=</span> <span class="n">mesh_normal_consistency</span><span class="p">(</span><span class="n">mesh</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># mesh laplacian smoothing</span>
|
||||
<span class="n">loss</span><span class="p">[</span><span class="s2">"laplacian"</span><span class="p">]</span> <span class="o">=</span> <span class="n">mesh_laplacian_smoothing</span><span class="p">(</span><span class="n">mesh</span><span class="p">,</span> <span class="n">method</span><span class="o">=</span><span class="s2">"uniform"</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># We will learn to deform the source mesh by offsetting its vertices</span>
|
||||
<span class="c1"># The shape of the deform parameters is equal to the total number of vertices in</span>
|
||||
<span class="c1"># src_mesh</span>
|
||||
<span class="n">verts_shape</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">verts_packed</span><span class="p">()</span><span class="o">.</span><span class="n">shape</span>
|
||||
<span class="n">deform_verts</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">(</span><span class="n">verts_shape</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># The optimizer</span>
|
||||
<span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">([</span><span class="n">deform_verts</span><span class="p">],</span> <span class="n">lr</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We write an optimization loop to iteratively refine our predicted mesh from the sphere mesh into a mesh that matches the sillhouettes of the target images:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">loop</span> <span class="o">=</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">Niter</span><span class="p">))</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">loop</span><span class="p">:</span>
|
||||
<span class="c1"># Initialize optimizer</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
|
||||
|
||||
<span class="c1"># Deform the mesh</span>
|
||||
<span class="n">new_src_mesh</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">offset_verts</span><span class="p">(</span><span class="n">deform_verts</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Losses to smooth /regularize the mesh shape</span>
|
||||
<span class="n">loss</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">losses</span><span class="p">}</span>
|
||||
<span class="n">update_mesh_shape_prior_losses</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">loss</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Compute the average silhouette loss over two random views, as the average </span>
|
||||
<span class="c1"># squared L2 distance between the predicted silhouette and the target </span>
|
||||
<span class="c1"># silhouette from our dataset</span>
|
||||
<span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">permutation</span><span class="p">(</span><span class="n">num_views</span><span class="p">)</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[:</span><span class="n">num_views_per_iteration</span><span class="p">]:</span>
|
||||
<span class="n">images_predicted</span> <span class="o">=</span> <span class="n">renderer_silhouette</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">target_cameras</span><span class="p">[</span><span class="n">j</span><span class="p">],</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||
<span class="n">predicted_silhouette</span> <span class="o">=</span> <span class="n">images_predicted</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
|
||||
<span class="n">loss_silhouette</span> <span class="o">=</span> <span class="p">((</span><span class="n">predicted_silhouette</span> <span class="o">-</span> <span class="n">target_silhouette</span><span class="p">[</span><span class="n">j</span><span class="p">])</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||||
<span class="n">loss</span><span class="p">[</span><span class="s2">"silhouette"</span><span class="p">]</span> <span class="o">+=</span> <span class="n">loss_silhouette</span> <span class="o">/</span> <span class="n">num_views_per_iteration</span>
|
||||
|
||||
<span class="c1"># Weighted sum of the losses</span>
|
||||
<span class="n">sum_loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">loss</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">sum_loss</span> <span class="o">+=</span> <span class="n">l</span> <span class="o">*</span> <span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"weight"</span><span class="p">]</span>
|
||||
<span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"values"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Print the losses</span>
|
||||
<span class="n">loop</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">"total_loss = </span><span class="si">%.6f</span><span class="s2">"</span> <span class="o">%</span> <span class="n">sum_loss</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Plot mesh</span>
|
||||
<span class="k">if</span> <span class="n">i</span> <span class="o">%</span> <span class="n">plot_period</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="s2">"iter: </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">i</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">target_image</span><span class="o">=</span><span class="n">target_silhouette</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||||
|
||||
<span class="c1"># Optimization step</span>
|
||||
<span class="n">sum_loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||
<span class="n">target_image</span><span class="o">=</span><span class="n">target_silhouette</span><span class="p">[</span><span class="mi">1</span><span class="p">])</span>
|
||||
<span class="n">plot_losses</span><span class="p">(</span><span class="n">losses</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="3.-Mesh-and-texture-prediction-via-textured-rendering">3. Mesh and texture prediction via textured rendering<a class="anchor-link" href="#3.-Mesh-and-texture-prediction-via-textured-rendering">¶</a></h2><p>We can predict both the mesh and its texture if we add an additional loss based on the comparing a predicted rendered RGB image to the target image. As before, we start with a sphere mesh. We learn both translational offsets and RGB texture colors for each vertex in the sphere mesh. Since our loss is based on rendered RGB pixel values instead of just the silhouette, we use a <strong>SoftPhongShader</strong> instead of a <strong>SoftSilhouetteShader</strong>.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Rasterization settings for differentiable rendering, where the blur_radius</span>
|
||||
<span class="c1"># initialization is based on Liu et al, 'Soft Rasterizer: A Differentiable </span>
|
||||
<span class="c1"># Renderer for Image-based 3D Reasoning', ICCV 2019</span>
|
||||
<span class="n">sigma</span> <span class="o">=</span> <span class="mf">1e-4</span>
|
||||
<span class="n">raster_settings_soft</span> <span class="o">=</span> <span class="n">RasterizationSettings</span><span class="p">(</span>
|
||||
<span class="n">image_size</span><span class="o">=</span><span class="mi">128</span><span class="p">,</span>
|
||||
<span class="n">blur_radius</span><span class="o">=</span><span class="n">np</span><span class="o">.</span><span class="n">log</span><span class="p">(</span><span class="mf">1.</span> <span class="o">/</span> <span class="mf">1e-4</span> <span class="o">-</span> <span class="mf">1.</span><span class="p">)</span><span class="o">*</span><span class="n">sigma</span><span class="p">,</span>
|
||||
<span class="n">faces_per_pixel</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># Differentiable soft renderer using per vertex RGB colors for texture</span>
|
||||
<span class="n">renderer_textured</span> <span class="o">=</span> <span class="n">MeshRenderer</span><span class="p">(</span>
|
||||
<span class="n">rasterizer</span><span class="o">=</span><span class="n">MeshRasterizer</span><span class="p">(</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||
<span class="n">raster_settings</span><span class="o">=</span><span class="n">raster_settings_soft</span>
|
||||
<span class="p">),</span>
|
||||
<span class="n">shader</span><span class="o">=</span><span class="n">SoftPhongShader</span><span class="p">(</span><span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span>
|
||||
<span class="n">cameras</span><span class="o">=</span><span class="n">camera</span><span class="p">,</span>
|
||||
<span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||
<span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We initialize settings, losses, and the optimizer that will be used to iteratively fit our mesh to the target RGB images:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Number of views to optimize over in each SGD iteration</span>
|
||||
<span class="n">num_views_per_iteration</span> <span class="o">=</span> <span class="mi">2</span>
|
||||
<span class="c1"># Number of optimization steps</span>
|
||||
<span class="n">Niter</span> <span class="o">=</span> <span class="mi">2000</span>
|
||||
<span class="c1"># Plot period for the losses</span>
|
||||
<span class="n">plot_period</span> <span class="o">=</span> <span class="mi">250</span>
|
||||
|
||||
<span class="o">%</span><span class="k">matplotlib</span> inline
|
||||
|
||||
<span class="c1"># Optimize using rendered RGB image loss, rendered silhouette image loss, mesh </span>
|
||||
<span class="c1"># edge loss, mesh normal consistency, and mesh laplacian smoothing</span>
|
||||
<span class="n">losses</span> <span class="o">=</span> <span class="p">{</span><span class="s2">"rgb"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"silhouette"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"edge"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"normal"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">0.01</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="s2">"laplacian"</span><span class="p">:</span> <span class="p">{</span><span class="s2">"weight"</span><span class="p">:</span> <span class="mf">1.0</span><span class="p">,</span> <span class="s2">"values"</span><span class="p">:</span> <span class="p">[]},</span>
|
||||
<span class="p">}</span>
|
||||
|
||||
<span class="c1"># We will learn to deform the source mesh by offsetting its vertices</span>
|
||||
<span class="c1"># The shape of the deform parameters is equal to the total number of vertices in </span>
|
||||
<span class="c1"># src_mesh</span>
|
||||
<span class="n">verts_shape</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">verts_packed</span><span class="p">()</span><span class="o">.</span><span class="n">shape</span>
|
||||
<span class="n">deform_verts</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">(</span><span class="n">verts_shape</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># We will also learn per vertex colors for our sphere mesh that define texture </span>
|
||||
<span class="c1"># of the mesh</span>
|
||||
<span class="n">sphere_verts_rgb</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">full</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="n">verts_shape</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="mi">3</span><span class="p">],</span> <span class="mf">0.5</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># The optimizer</span>
|
||||
<span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">([</span><span class="n">deform_verts</span><span class="p">,</span> <span class="n">sphere_verts_rgb</span><span class="p">],</span> <span class="n">lr</span><span class="o">=</span><span class="mf">1.0</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>We write an optimization loop to iteratively refine our predicted mesh and its vertex colors from the sphere mesh into a mesh that matches the target images:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">loop</span> <span class="o">=</span> <span class="n">tqdm</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">Niter</span><span class="p">))</span>
|
||||
|
||||
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">loop</span><span class="p">:</span>
|
||||
<span class="c1"># Initialize optimizer</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
|
||||
|
||||
<span class="c1"># Deform the mesh</span>
|
||||
<span class="n">new_src_mesh</span> <span class="o">=</span> <span class="n">src_mesh</span><span class="o">.</span><span class="n">offset_verts</span><span class="p">(</span><span class="n">deform_verts</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Add per vertex colors to texture the mesh</span>
|
||||
<span class="n">new_src_mesh</span><span class="o">.</span><span class="n">textures</span> <span class="o">=</span> <span class="n">TexturesVertex</span><span class="p">(</span><span class="n">verts_features</span><span class="o">=</span><span class="n">sphere_verts_rgb</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Losses to smooth /regularize the mesh shape</span>
|
||||
<span class="n">loss</span> <span class="o">=</span> <span class="p">{</span><span class="n">k</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span> <span class="k">for</span> <span class="n">k</span> <span class="ow">in</span> <span class="n">losses</span><span class="p">}</span>
|
||||
<span class="n">update_mesh_shape_prior_losses</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">loss</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Randomly select two views to optimize over in this iteration. Compared</span>
|
||||
<span class="c1"># to using just one view, this helps resolve ambiguities between updating</span>
|
||||
<span class="c1"># mesh shape vs. updating mesh texture</span>
|
||||
<span class="k">for</span> <span class="n">j</span> <span class="ow">in</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">permutation</span><span class="p">(</span><span class="n">num_views</span><span class="p">)</span><span class="o">.</span><span class="n">tolist</span><span class="p">()[:</span><span class="n">num_views_per_iteration</span><span class="p">]:</span>
|
||||
<span class="n">images_predicted</span> <span class="o">=</span> <span class="n">renderer_textured</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">cameras</span><span class="o">=</span><span class="n">target_cameras</span><span class="p">[</span><span class="n">j</span><span class="p">],</span> <span class="n">lights</span><span class="o">=</span><span class="n">lights</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Squared L2 distance between the predicted silhouette and the target </span>
|
||||
<span class="c1"># silhouette from our dataset</span>
|
||||
<span class="n">predicted_silhouette</span> <span class="o">=</span> <span class="n">images_predicted</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
|
||||
<span class="n">loss_silhouette</span> <span class="o">=</span> <span class="p">((</span><span class="n">predicted_silhouette</span> <span class="o">-</span> <span class="n">target_silhouette</span><span class="p">[</span><span class="n">j</span><span class="p">])</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||||
<span class="n">loss</span><span class="p">[</span><span class="s2">"silhouette"</span><span class="p">]</span> <span class="o">+=</span> <span class="n">loss_silhouette</span> <span class="o">/</span> <span class="n">num_views_per_iteration</span>
|
||||
|
||||
<span class="c1"># Squared L2 distance between the predicted RGB image and the target </span>
|
||||
<span class="c1"># image from our dataset</span>
|
||||
<span class="n">predicted_rgb</span> <span class="o">=</span> <span class="n">images_predicted</span><span class="p">[</span><span class="o">...</span><span class="p">,</span> <span class="p">:</span><span class="mi">3</span><span class="p">]</span>
|
||||
<span class="n">loss_rgb</span> <span class="o">=</span> <span class="p">((</span><span class="n">predicted_rgb</span> <span class="o">-</span> <span class="n">target_rgb</span><span class="p">[</span><span class="n">j</span><span class="p">])</span> <span class="o">**</span> <span class="mi">2</span><span class="p">)</span><span class="o">.</span><span class="n">mean</span><span class="p">()</span>
|
||||
<span class="n">loss</span><span class="p">[</span><span class="s2">"rgb"</span><span class="p">]</span> <span class="o">+=</span> <span class="n">loss_rgb</span> <span class="o">/</span> <span class="n">num_views_per_iteration</span>
|
||||
|
||||
<span class="c1"># Weighted sum of the losses</span>
|
||||
<span class="n">sum_loss</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">tensor</span><span class="p">(</span><span class="mf">0.0</span><span class="p">,</span> <span class="n">device</span><span class="o">=</span><span class="n">device</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">l</span> <span class="ow">in</span> <span class="n">loss</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">sum_loss</span> <span class="o">+=</span> <span class="n">l</span> <span class="o">*</span> <span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"weight"</span><span class="p">]</span>
|
||||
<span class="n">losses</span><span class="p">[</span><span class="n">k</span><span class="p">][</span><span class="s2">"values"</span><span class="p">]</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">l</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Print the losses</span>
|
||||
<span class="n">loop</span><span class="o">.</span><span class="n">set_description</span><span class="p">(</span><span class="s2">"total_loss = </span><span class="si">%.6f</span><span class="s2">"</span> <span class="o">%</span> <span class="n">sum_loss</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Plot mesh</span>
|
||||
<span class="k">if</span> <span class="n">i</span> <span class="o">%</span> <span class="n">plot_period</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">renderer</span><span class="o">=</span><span class="n">renderer_textured</span><span class="p">,</span> <span class="n">title</span><span class="o">=</span><span class="s2">"iter: </span><span class="si">%d</span><span class="s2">"</span> <span class="o">%</span> <span class="n">i</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Optimization step</span>
|
||||
<span class="n">sum_loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="n">visualize_prediction</span><span class="p">(</span><span class="n">new_src_mesh</span><span class="p">,</span> <span class="n">renderer</span><span class="o">=</span><span class="n">renderer_textured</span><span class="p">,</span> <span class="n">silhouette</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
<span class="n">plot_losses</span><span class="p">(</span><span class="n">losses</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<p>Save the final predicted mesh:</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="4.-Save-the-final-predicted-mesh">4. Save the final predicted mesh<a class="anchor-link" href="#4.-Save-the-final-predicted-mesh">¶</a></h2>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing code_cell rendered">
|
||||
<div class="input">
|
||||
<div class="prompt input_prompt">In [ ]:</div>
|
||||
<div class="inner_cell">
|
||||
<div class="input_area">
|
||||
<div class="highlight hl-ipython3"><pre><span></span><span class="c1"># Fetch the verts and faces of the final predicted mesh</span>
|
||||
<span class="n">final_verts</span><span class="p">,</span> <span class="n">final_faces</span> <span class="o">=</span> <span class="n">new_src_mesh</span><span class="o">.</span><span class="n">get_mesh_verts_faces</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Scale normalize back to the original target size</span>
|
||||
<span class="n">final_verts</span> <span class="o">=</span> <span class="n">final_verts</span> <span class="o">*</span> <span class="n">scale</span> <span class="o">+</span> <span class="n">center</span>
|
||||
|
||||
<span class="c1"># Store the predicted mesh using save_obj</span>
|
||||
<span class="n">final_obj</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="s1">'./'</span><span class="p">,</span> <span class="s1">'final_model.obj'</span><span class="p">)</span>
|
||||
<span class="n">save_obj</span><span class="p">(</span><span class="n">final_obj</span><span class="p">,</span> <span class="n">final_verts</span><span class="p">,</span> <span class="n">final_faces</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell border-box-sizing text_cell rendered"><div class="prompt input_prompt">
|
||||
</div><div class="inner_cell">
|
||||
<div class="text_cell_render border-box-sizing rendered_html">
|
||||
<h2 id="5.-Conclusion">5. Conclusion<a class="anchor-link" href="#5.-Conclusion">¶</a></h2><p>In this tutorial, we learned how to load a textured mesh from an obj file, create a synthetic dataset by rendering the mesh from multiple viewpoints. We showed how to set up an optimization loop to fit a mesh to the observed dataset images based on a rendered silhouette loss. We then augmented this optimization loop with an additional loss based on rendered RGB images, which allowed us to predict both a mesh and its texture.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
ga('create', 'UA-157376881-1', 'auto');
|
||||
ga('send', 'pageview');
|
||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="container docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span></span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Tutorials</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/tutorials/">Overview</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">3D operators</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/deform_source_mesh_to_target_mesh">Fit Mesh</a></li><li class="navListItem"><a class="navItem" href="/tutorials/bundle_adjustment">Bundle Adjustment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Rendering</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/render_textured_meshes">Render Textured Meshes</a></li><li class="navListItem"><a class="navItem" href="/tutorials/camera_position_optimization_with_differentiable_rendering">Camera Position Optimization</a></li></ul></div></div></section></div><script>
|
||||
</script><script type="text/javascript" src="https://buttons.github.io/buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head><body><div class="fixedHeaderContainer"><div class="headerWrapper wrapper"><header><a href="/"><img class="logo" src="/img/pytorch3dfavicon.png" alt="PyTorch3D"/><h2 class="headerTitleWithLogo">PyTorch3D</h2></a><div class="navigationWrapper navigationSlider"><nav class="slidingNav"><ul class="nav-site nav-site-internal"><li class=""><a href="/docs/why_pytorch3d" target="_self">Docs</a></li><li class=""><a href="/tutorials" target="_self">Tutorials</a></li><li class=""><a href="https://pytorch3d.readthedocs.io/" target="_self">API</a></li><li class=""><a href="https://github.com/facebookresearch/pytorch3d" target="_self">GitHub</a></li></ul></nav></div></header></div></div><div class="navPusher"><div class="docMainWrapper wrapper"><div class="container docsNavContainer" id="docsNav"><nav class="toc"><div class="toggleNav"><section class="navWrapper wrapper"><div class="navBreadcrumb wrapper"><div class="navToggle" id="navToggler"><div class="hamburger-menu"><div class="line1"></div><div class="line2"></div><div class="line3"></div></div></div><h2><i>›</i><span></span></h2><div class="tocToggler" id="tocToggler"><i class="icon-toc"></i></div></div><div class="navGroups"><div class="navGroup"><h3 class="navGroupCategoryTitle">Tutorials</h3><ul class=""><li class="navListItem navListItemActive"><a class="navItem" href="/tutorials/">Overview</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">3D operators</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/deform_source_mesh_to_target_mesh">Fit Mesh</a></li><li class="navListItem"><a class="navItem" href="/tutorials/bundle_adjustment">Bundle Adjustment</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Rendering</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/render_textured_meshes">Render Textured Meshes</a></li><li class="navListItem"><a class="navItem" href="/tutorials/fit_textured_mesh">Fit a mesh with texture via rendering</a></li><li class="navListItem"><a class="navItem" href="/tutorials/camera_position_optimization_with_differentiable_rendering">Camera Position Optimization</a></li></ul></div><div class="navGroup"><h3 class="navGroupCategoryTitle">Dataloaders</h3><ul class=""><li class="navListItem"><a class="navItem" href="/tutorials/dataloaders_ShapeNetCore_R2N2">Data loaders for ShapeNetCore and R2N2</a></li></ul></div></div></section></div><script>
|
||||
var coll = document.getElementsByClassName('collapsible');
|
||||
var checkActiveCategory = true;
|
||||
for (var i = 0; i < coll.length; i++) {
|
||||
@@ -65,4 +65,4 @@
|
||||
</script></nav></div><div class="container mainContainer documentContainer postContainer"><div class="wrapper"><div class="post"><header class="postHeader"><h1 class="postHeaderTitle">Welcome to the PyTorch3D Tutorials</h1></header><p>Here you can learn about the structure and applications of Pytorch3D from examples which are in the form of ipython notebooks.</p><h3> Run interactively </h3><p>At the top of each example you can find a button named <strong>"Run in Google Colab"</strong> which will open the notebook in <a href="https://colab.research.google.com/notebooks/intro.ipynb"> Google Colaboratory </a> where you can run the code directly in the browser with access to GPU support - it looks like this:</p><div class="tutorialButtonsWrapper"><div class="tutorialButtonWrapper buttonWrapper"><a class="tutorialButton button" target="_blank"><img class="colabButton" align="left" src="/img/colab_icon.png"/>Run in Google Colab</a></div></div><p> You can modify the code and experiment with varying different settings. Remember to install pytorch, torchvision, fvcore and pytorch3d in the first cell of the colab notebook by running: </p><div><span><pre><code class="hljs css language-bash">!pip install torch torchvision
|
||||
!pip install <span class="hljs-string">'git+https://github.com/facebookresearch/pytorch3d.git@stable'</span>
|
||||
</code></pre>
|
||||
</span></div>This installs the latest stable version of PyTorch3D from github.<h3> Run locally </h3><p> There is also a button to download the notebook and source code to run it locally. </p></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc</section></footer></div></body></html>
|
||||
</span></div>This installs the latest stable version of PyTorch3D from github.<h3> Run locally </h3><p> There is also a button to download the notebook and source code to run it locally. </p></div></div></div></div><footer class="nav-footer" id="footer"><section class="sitemap"><div class="footerSection"><div class="social"><a class="github-button" href="https://github.com/facebookresearch/pytorch3d" data-count-href="https://github.com/facebookresearch/pytorch3d/stargazers" data-show-count="true" data-count-aria-label="# stargazers on GitHub" aria-label="Star PyTorch3D on GitHub">pytorch3d</a></div></div></section><a href="https://opensource.facebook.com/" target="_blank" rel="noreferrer noopener" class="fbOpenSource"><img src="/img/oss_logo.png" alt="Facebook Open Source" width="170" height="45"/></a><section class="copyright">Copyright © 2020 Facebook Inc<br/>Legal:<a href="https://opensource.facebook.com/legal/privacy/" target="_blank" rel="noreferrer noopener">Privacy</a><a href="https://opensource.facebook.com/legal/terms/" target="_blank" rel="noreferrer noopener">Terms</a></section></footer></div></body></html>
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user