diff --git a/.circleci/config.in.yml b/.circleci/config.in.yml index 44b85008..0a0626de 100644 --- a/.circleci/config.in.yml +++ b/.circleci/config.in.yml @@ -248,14 +248,19 @@ workflows: cu_version: cpu name: macos_wheel_py36_cpu python_version: '3.6' - pytorch_version: '1.6.0' + pytorch_version: '1.7.1' - binary_macos_wheel: cu_version: cpu name: macos_wheel_py37_cpu python_version: '3.7' - pytorch_version: '1.6.0' + pytorch_version: '1.7.1' - binary_macos_wheel: cu_version: cpu name: macos_wheel_py38_cpu python_version: '3.8' - pytorch_version: '1.6.0' + pytorch_version: '1.7.1' + - binary_macos_wheel: + cu_version: cpu + name: macos_wheel_py39_cpu + python_version: '3.9' + pytorch_version: '1.7.1' diff --git a/.circleci/config.yml b/.circleci/config.yml index 4bfe692e..a6adc429 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -549,24 +549,6 @@ workflows: name: linux_conda_py39_cu110_pyt171 python_version: '3.9' pytorch_version: 1.7.1 - - binary_linux_wheel: - context: DOCKERHUB_TOKEN - cu_version: cu101 - name: linux_wheel_py36_cu101_pyt160 - python_version: '3.6' - pytorch_version: 1.6.0 - - binary_linux_wheel: - context: DOCKERHUB_TOKEN - cu_version: cu101 - name: linux_wheel_py37_cu101_pyt160 - python_version: '3.7' - pytorch_version: 1.6.0 - - binary_linux_wheel: - context: DOCKERHUB_TOKEN - cu_version: cu101 - name: linux_wheel_py38_cu101_pyt160 - python_version: '3.8' - pytorch_version: 1.6.0 - binary_linux_conda_cuda: name: testrun_conda_cuda_py36_cu101_pyt14 context: DOCKERHUB_TOKEN @@ -589,14 +571,19 @@ workflows: cu_version: cpu name: macos_wheel_py36_cpu python_version: '3.6' - pytorch_version: '1.6.0' + pytorch_version: '1.7.1' - binary_macos_wheel: cu_version: cpu name: macos_wheel_py37_cpu python_version: '3.7' - pytorch_version: '1.6.0' + pytorch_version: '1.7.1' - binary_macos_wheel: cu_version: cpu name: macos_wheel_py38_cpu python_version: '3.8' - pytorch_version: '1.6.0' + pytorch_version: '1.7.1' + - binary_macos_wheel: + cu_version: cpu + name: macos_wheel_py39_cpu + python_version: '3.9' + pytorch_version: '1.7.1' diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index fbf5f51a..a8f867cf 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -46,18 +46,6 @@ def workflows(prefix="", filter_branch=None, upload=False, indentation=6): upload=upload, filter_branch=filter_branch, ) - for btype in ["wheel"]: - for python_version in ["3.6", "3.7", "3.8"]: - for cu_version in ["cu101"]: - w += workflow_pair( - btype=btype, - python_version=python_version, - pytorch_version="1.6.0", - cu_version=cu_version, - prefix=prefix, - upload=upload, - filter_branch=filter_branch, - ) return indent(indentation, w) diff --git a/INSTALL.md b/INSTALL.md index 138c990a..6cc1c7f3 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -9,7 +9,7 @@ The core library is written in PyTorch. Several components have underlying imple - Linux or macOS or Windows - Python 3.6, 3.7 or 3.8 -- PyTorch 1.4, 1.5.0, 1.5.1, 1.6.0, or 1.7.0. +- PyTorch 1.4, 1.5.0, 1.5.1, 1.6.0, 1.7.0, or 1.7.1. - torchvision that matches the PyTorch installation. You can install them together as explained at pytorch.org to make sure of this. - gcc & g++ ≥ 4.9 - [fvcore](https://github.com/facebookresearch/fvcore) @@ -21,8 +21,8 @@ The runtime dependencies can be installed by running: ``` conda create -n pytorch3d python=3.8 conda activate pytorch3d -conda install -c pytorch pytorch=1.7.0 torchvision cudatoolkit=10.2 -conda install -c conda-forge fvcore iopath +conda install -c pytorch pytorch=1.7.1 torchvision cudatoolkit=10.2 +conda install -c conda-forge -c fvcore -c iopath fvcore iopath ``` For the CUB build time dependency, if you are using conda, you can continue with @@ -77,12 +77,31 @@ Or, to install a nightly (non-official, alpha) build: # Anaconda Cloud conda install pytorch3d -c pytorch3d-nightly ``` -### 2. Install from PyPI, on Linux and Mac -This works with pytorch 1.6.0 only. +### 2. Install from PyPI, on Mac only. +This works with pytorch 1.7.1 only. The build is CPU only. ``` pip install pytorch3d ``` -On Linux this has support for CUDA 10.1. On Mac this is CPU-only. + +### 3. Install wheels for Linux +We have prebuilt wheels with CUDA for Linux for PyTorch 1.7.0 and 1.7.1, for each of the CUDA versions that they support. +These are installed in a special way. +For example, to install for Python 3.6, PyTorch 1.7.0 and CUDA 10.1 +``` +pip install pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py36_cu101_pyt170/download.html +``` + +In general, from inside IPython, or in Google Colab or a jupyter notebook, you can install with +``` +import sys +import torch +version_str="".join([ + f"py3{sys.version_info.minor}_cu", + torch.version.cuda.replace(".",""), + f"_pyt{torch.__version__[0:5:2]}" +]) +!pip install pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/{version_str}/download.html +``` ## Building / installing from source. CUDA support will be included if CUDA is available in pytorch or if the environment variable diff --git a/dev/run_tutorials.sh b/dev/run_tutorials.sh index 26395426..f52cd5cd 100644 --- a/dev/run_tutorials.sh +++ b/dev/run_tutorials.sh @@ -19,7 +19,7 @@ conda init bash source ~/.bashrc conda create -y -n myenv python=3.8 matplotlib ipython ipywidgets nbconvert conda activate myenv -conda install -y -c conda-forge fvcore iopath +conda install -y -c conda-forge -c fvcore -c iopath fvcore iopath conda install -y -c pytorch pytorch=1.6.0 cudatoolkit=10.1 torchvision conda install -y -c pytorch3d-nightly pytorch3d pip install plotly scikit-image diff --git a/packaging/linux_wheels/README.md b/packaging/linux_wheels/README.md new file mode 100644 index 00000000..da61400a --- /dev/null +++ b/packaging/linux_wheels/README.md @@ -0,0 +1,29 @@ +## Building Linux pip Packages + +1. Make sure this directory is on a filesystem which docker can +use - e.g. not NFS. If you are using a local hard drive there is +nothing to do here. + +2. You may want to `docker pull pytorch/conda-cuda:latest`. + +3. Run `bash go.sh` in this directory. This takes ages +and writes packages to `inside/output`. + +4. You can upload the packages to s3, along with basic html files +which enable them to be used, with `bash after.sh`. + + +In particular, if you are in a jupyter/colab notebook you can +then install using these wheels with the following series of +commands. + +``` +import sys +import torch +version_str="".join([ + f"py3{sys.version_info.minor}_cu", + torch.version.cuda.replace(".",""), + f"_pyt{torch.__version__[0:5:2]}" +]) +!pip install pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/{version_str}/download.html +``` diff --git a/packaging/linux_wheels/after.sh b/packaging/linux_wheels/after.sh new file mode 100644 index 00000000..51de777e --- /dev/null +++ b/packaging/linux_wheels/after.sh @@ -0,0 +1,5 @@ +#!/usr/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. +set -ex +sudo chown -R "$USER" output +python publish.py diff --git a/packaging/linux_wheels/go.sh b/packaging/linux_wheels/go.sh new file mode 100644 index 00000000..28cdbf3b --- /dev/null +++ b/packaging/linux_wheels/go.sh @@ -0,0 +1,3 @@ +#!/usr/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. +sudo docker run --rm -v "$PWD/../../:/inside" pytorch/conda-cuda bash inside/packaging/linux_wheels/inside.sh diff --git a/packaging/linux_wheels/inside.sh b/packaging/linux_wheels/inside.sh new file mode 100644 index 00000000..d4105753 --- /dev/null +++ b/packaging/linux_wheels/inside.sh @@ -0,0 +1,102 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. +set -ex + +conda init bash +# shellcheck source=/dev/null +source ~/.bashrc + +cd /inside +VERSION=$(python -c "exec(open('pytorch3d/__init__.py').read()); print(__version__)") + +export BUILD_VERSION=$VERSION +export FORCE_CUDA=1 + +wget --no-verbose https://github.com/NVIDIA/cub/archive/1.10.0.tar.gz +tar xzf 1.10.0.tar.gz +CUB_HOME=$(realpath ./cub-1.10.0) +export CUB_HOME +echo "CUB_HOME is now $CUB_HOME" + + +PYTHON_VERSIONS="3.6 3.7 3.8 3.9" +# the keys are pytorch versions +declare -A CONDA_CUDA_VERSIONS=( +# ["1.4.0"]="cu101" +# ["1.5.0"]="cu101 cu102" +# ["1.5.1"]="cu101 cu102" +# ["1.6.0"]="cu101 cu102" + ["1.7.0"]="cu101 cu102 cu110" + ["1.7.1"]="cu101 cu102 cu110" +) + + + +for python_version in $PYTHON_VERSIONS +do + for pytorch_version in "${!CONDA_CUDA_VERSIONS[@]}" + do + if [[ "3.6 3.7 3.8" != *$python_version* ]] && [[ "1.4.0 1.5.0 1.5.1 1.6.0 1.7.0" == *$pytorch_version* ]] + then + #python 3.9 and later not supported by pytorch 1.7.0 and before + continue + fi + + if [[ "3.9" == "$python_version" ]] + then + extra_channel="-c conda-forge" + else + extra_channel="" + fi + + for cu_version in ${CONDA_CUDA_VERSIONS[$pytorch_version]} + do + case "$cu_version" in + cu110) + export CUDA_HOME=/usr/local/cuda-11.0/ + export CUDA_TAG=11.0 + export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_50,code=compute_50" + ;; + cu102) + export CUDA_HOME=/usr/local/cuda-10.2/ + export CUDA_TAG=10.2 + export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50" + ;; + cu101) + export CUDA_HOME=/usr/local/cuda-10.1/ + export CUDA_TAG=10.1 + export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50" + ;; + *) + echo "Unrecognized cu_version=$cu_version" + exit 1 + ;; + esac + tag=py"${python_version//./}"_"${cu_version}"_pyt"${pytorch_version//./}" + + outdir="/inside/packaging/linux_wheels/output/$tag" + if [[ -d "$outdir" ]] + then + continue + fi + + conda create -y -n "$tag" "python=$python_version" + conda activate "$tag" + conda install -y -c pytorch $extra_channel "pytorch=$pytorch_version" "cudatoolkit=$CUDA_TAG" torchvision + pip install fvcore iopath + echo "python version" "$python_version" "pytorch version" "$pytorch_version" "cuda version" "$cu_version" "tag" "$tag" + + rm -rf dist + + python setup.py clean + python setup.py bdist_wheel + + rm -rf "$outdir" + mkdir -p "$outdir" + cp dist/*whl "$outdir" + + conda deactivate + done + done +done +echo "DONE" diff --git a/packaging/linux_wheels/publish.py b/packaging/linux_wheels/publish.py new file mode 100644 index 00000000..b2a09193 --- /dev/null +++ b/packaging/linux_wheels/publish.py @@ -0,0 +1,76 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. +import os +import subprocess +from pathlib import Path +from typing import List + + +dest = "s3://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/" + +output = Path("output") + + +def fs3cmd(args, allow_failure: bool = False) -> List[str]: + """ + This function returns the args for subprocess to mimic the bash command + fs3cmd available in the fairusers_aws module on the FAIR cluster. + """ + os.environ["FAIR_CLUSTER_NAME"] = os.environ["FAIR_ENV_CLUSTER"].lower() + cmd_args = ["/public/apps/fairusers_aws/bin/fs3cmd"] + args + return cmd_args + + +def fs3_exists(path) -> bool: + """ + Returns True if the path exists inside dest on S3. + In fact, will also return True if there is a file which has the given + path as a prefix, but we are careful about this. + """ + out = subprocess.check_output(fs3cmd(["ls", path])) + return len(out) != 0 + + +def get_html_wrappers() -> None: + for directory in sorted(output.iterdir()): + output_wrapper = directory / "download.html" + assert not output_wrapper.exists() + dest_wrapper = dest + directory.name + "/download.html" + if fs3_exists(dest_wrapper): + subprocess.check_call(fs3cmd(["get", dest_wrapper, str(output_wrapper)])) + + +def write_html_wrappers() -> None: + html = """ + $
+ """ + + for directory in sorted(output.iterdir()): + files = list(directory.glob("*.whl")) + assert len(files) == 1, files + [wheel] = files + + this_html = html.replace("$", wheel.name) + output_wrapper = directory / "download.html" + if output_wrapper.exists(): + contents = output_wrapper.read_text() + if this_html not in contents: + with open(output_wrapper, "a") as f: + f.write(this_html) + else: + output_wrapper.write_text(this_html) + + +def to_aws() -> None: + for directory in output.iterdir(): + for file in directory.iterdir(): + print(file) + subprocess.check_call( + fs3cmd(["put", str(file), dest + str(file.relative_to(output))]) + ) + + +if __name__ == "__main__": + # Uncomment this for subsequent releases. + # get_html_wrappers() + write_html_wrappers() + to_aws() diff --git a/setup.py b/setup.py index 84d2665b..28c4081d 100755 --- a/setup.py +++ b/setup.py @@ -132,8 +132,10 @@ setup( url="https://github.com/facebookresearch/pytorch3d", description="PyTorch3D is FAIR's library of reusable components " "for deep Learning with 3D data.", - packages=find_packages(exclude=("configs", "tests", "tests.*")), - install_requires=["torchvision>=0.4", "fvcore", "iopath"], + packages=find_packages( + exclude=("configs", "tests", "tests.*", "docs.*", "projects.*") + ), + install_requires=["fvcore", "iopath"], extras_require={ "all": ["matplotlib", "tqdm>4.29.0", "imageio", "ipywidgets"], "dev": ["flake8", "isort", "black==19.3b0"],