mirror of
https://github.com/facebookresearch/pytorch3d.git
synced 2025-08-01 03:12:49 +08:00
Build wheels for s3
Summary: For Linux, instead of uploading wheels to PyPI which will only work with one particular version of PyTorch and CUDA, from the next release we will store a range of built wheels on S3. Reviewed By: nikhilaravi Differential Revision: D26209398 fbshipit-source-id: 945a6907b78807e1eedb25007f87f90bbf59f80e
This commit is contained in:
parent
3463f418b8
commit
e0753f0b0d
@ -248,14 +248,19 @@ workflows:
|
||||
cu_version: cpu
|
||||
name: macos_wheel_py36_cpu
|
||||
python_version: '3.6'
|
||||
pytorch_version: '1.6.0'
|
||||
pytorch_version: '1.7.1'
|
||||
- binary_macos_wheel:
|
||||
cu_version: cpu
|
||||
name: macos_wheel_py37_cpu
|
||||
python_version: '3.7'
|
||||
pytorch_version: '1.6.0'
|
||||
pytorch_version: '1.7.1'
|
||||
- binary_macos_wheel:
|
||||
cu_version: cpu
|
||||
name: macos_wheel_py38_cpu
|
||||
python_version: '3.8'
|
||||
pytorch_version: '1.6.0'
|
||||
pytorch_version: '1.7.1'
|
||||
- binary_macos_wheel:
|
||||
cu_version: cpu
|
||||
name: macos_wheel_py39_cpu
|
||||
python_version: '3.9'
|
||||
pytorch_version: '1.7.1'
|
||||
|
@ -549,24 +549,6 @@ workflows:
|
||||
name: linux_conda_py39_cu110_pyt171
|
||||
python_version: '3.9'
|
||||
pytorch_version: 1.7.1
|
||||
- binary_linux_wheel:
|
||||
context: DOCKERHUB_TOKEN
|
||||
cu_version: cu101
|
||||
name: linux_wheel_py36_cu101_pyt160
|
||||
python_version: '3.6'
|
||||
pytorch_version: 1.6.0
|
||||
- binary_linux_wheel:
|
||||
context: DOCKERHUB_TOKEN
|
||||
cu_version: cu101
|
||||
name: linux_wheel_py37_cu101_pyt160
|
||||
python_version: '3.7'
|
||||
pytorch_version: 1.6.0
|
||||
- binary_linux_wheel:
|
||||
context: DOCKERHUB_TOKEN
|
||||
cu_version: cu101
|
||||
name: linux_wheel_py38_cu101_pyt160
|
||||
python_version: '3.8'
|
||||
pytorch_version: 1.6.0
|
||||
- binary_linux_conda_cuda:
|
||||
name: testrun_conda_cuda_py36_cu101_pyt14
|
||||
context: DOCKERHUB_TOKEN
|
||||
@ -589,14 +571,19 @@ workflows:
|
||||
cu_version: cpu
|
||||
name: macos_wheel_py36_cpu
|
||||
python_version: '3.6'
|
||||
pytorch_version: '1.6.0'
|
||||
pytorch_version: '1.7.1'
|
||||
- binary_macos_wheel:
|
||||
cu_version: cpu
|
||||
name: macos_wheel_py37_cpu
|
||||
python_version: '3.7'
|
||||
pytorch_version: '1.6.0'
|
||||
pytorch_version: '1.7.1'
|
||||
- binary_macos_wheel:
|
||||
cu_version: cpu
|
||||
name: macos_wheel_py38_cpu
|
||||
python_version: '3.8'
|
||||
pytorch_version: '1.6.0'
|
||||
pytorch_version: '1.7.1'
|
||||
- binary_macos_wheel:
|
||||
cu_version: cpu
|
||||
name: macos_wheel_py39_cpu
|
||||
python_version: '3.9'
|
||||
pytorch_version: '1.7.1'
|
||||
|
@ -46,18 +46,6 @@ def workflows(prefix="", filter_branch=None, upload=False, indentation=6):
|
||||
upload=upload,
|
||||
filter_branch=filter_branch,
|
||||
)
|
||||
for btype in ["wheel"]:
|
||||
for python_version in ["3.6", "3.7", "3.8"]:
|
||||
for cu_version in ["cu101"]:
|
||||
w += workflow_pair(
|
||||
btype=btype,
|
||||
python_version=python_version,
|
||||
pytorch_version="1.6.0",
|
||||
cu_version=cu_version,
|
||||
prefix=prefix,
|
||||
upload=upload,
|
||||
filter_branch=filter_branch,
|
||||
)
|
||||
|
||||
return indent(indentation, w)
|
||||
|
||||
|
31
INSTALL.md
31
INSTALL.md
@ -9,7 +9,7 @@ The core library is written in PyTorch. Several components have underlying imple
|
||||
|
||||
- Linux or macOS or Windows
|
||||
- Python 3.6, 3.7 or 3.8
|
||||
- PyTorch 1.4, 1.5.0, 1.5.1, 1.6.0, or 1.7.0.
|
||||
- PyTorch 1.4, 1.5.0, 1.5.1, 1.6.0, 1.7.0, or 1.7.1.
|
||||
- torchvision that matches the PyTorch installation. You can install them together as explained at pytorch.org to make sure of this.
|
||||
- gcc & g++ ≥ 4.9
|
||||
- [fvcore](https://github.com/facebookresearch/fvcore)
|
||||
@ -21,8 +21,8 @@ The runtime dependencies can be installed by running:
|
||||
```
|
||||
conda create -n pytorch3d python=3.8
|
||||
conda activate pytorch3d
|
||||
conda install -c pytorch pytorch=1.7.0 torchvision cudatoolkit=10.2
|
||||
conda install -c conda-forge fvcore iopath
|
||||
conda install -c pytorch pytorch=1.7.1 torchvision cudatoolkit=10.2
|
||||
conda install -c conda-forge -c fvcore -c iopath fvcore iopath
|
||||
```
|
||||
|
||||
For the CUB build time dependency, if you are using conda, you can continue with
|
||||
@ -77,12 +77,31 @@ Or, to install a nightly (non-official, alpha) build:
|
||||
# Anaconda Cloud
|
||||
conda install pytorch3d -c pytorch3d-nightly
|
||||
```
|
||||
### 2. Install from PyPI, on Linux and Mac
|
||||
This works with pytorch 1.6.0 only.
|
||||
### 2. Install from PyPI, on Mac only.
|
||||
This works with pytorch 1.7.1 only. The build is CPU only.
|
||||
```
|
||||
pip install pytorch3d
|
||||
```
|
||||
On Linux this has support for CUDA 10.1. On Mac this is CPU-only.
|
||||
|
||||
### 3. Install wheels for Linux
|
||||
We have prebuilt wheels with CUDA for Linux for PyTorch 1.7.0 and 1.7.1, for each of the CUDA versions that they support.
|
||||
These are installed in a special way.
|
||||
For example, to install for Python 3.6, PyTorch 1.7.0 and CUDA 10.1
|
||||
```
|
||||
pip install pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py36_cu101_pyt170/download.html
|
||||
```
|
||||
|
||||
In general, from inside IPython, or in Google Colab or a jupyter notebook, you can install with
|
||||
```
|
||||
import sys
|
||||
import torch
|
||||
version_str="".join([
|
||||
f"py3{sys.version_info.minor}_cu",
|
||||
torch.version.cuda.replace(".",""),
|
||||
f"_pyt{torch.__version__[0:5:2]}"
|
||||
])
|
||||
!pip install pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/{version_str}/download.html
|
||||
```
|
||||
|
||||
## Building / installing from source.
|
||||
CUDA support will be included if CUDA is available in pytorch or if the environment variable
|
||||
|
@ -19,7 +19,7 @@ conda init bash
|
||||
source ~/.bashrc
|
||||
conda create -y -n myenv python=3.8 matplotlib ipython ipywidgets nbconvert
|
||||
conda activate myenv
|
||||
conda install -y -c conda-forge fvcore iopath
|
||||
conda install -y -c conda-forge -c fvcore -c iopath fvcore iopath
|
||||
conda install -y -c pytorch pytorch=1.6.0 cudatoolkit=10.1 torchvision
|
||||
conda install -y -c pytorch3d-nightly pytorch3d
|
||||
pip install plotly scikit-image
|
||||
|
29
packaging/linux_wheels/README.md
Normal file
29
packaging/linux_wheels/README.md
Normal file
@ -0,0 +1,29 @@
|
||||
## Building Linux pip Packages
|
||||
|
||||
1. Make sure this directory is on a filesystem which docker can
|
||||
use - e.g. not NFS. If you are using a local hard drive there is
|
||||
nothing to do here.
|
||||
|
||||
2. You may want to `docker pull pytorch/conda-cuda:latest`.
|
||||
|
||||
3. Run `bash go.sh` in this directory. This takes ages
|
||||
and writes packages to `inside/output`.
|
||||
|
||||
4. You can upload the packages to s3, along with basic html files
|
||||
which enable them to be used, with `bash after.sh`.
|
||||
|
||||
|
||||
In particular, if you are in a jupyter/colab notebook you can
|
||||
then install using these wheels with the following series of
|
||||
commands.
|
||||
|
||||
```
|
||||
import sys
|
||||
import torch
|
||||
version_str="".join([
|
||||
f"py3{sys.version_info.minor}_cu",
|
||||
torch.version.cuda.replace(".",""),
|
||||
f"_pyt{torch.__version__[0:5:2]}"
|
||||
])
|
||||
!pip install pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/{version_str}/download.html
|
||||
```
|
5
packaging/linux_wheels/after.sh
Normal file
5
packaging/linux_wheels/after.sh
Normal file
@ -0,0 +1,5 @@
|
||||
#!/usr/bin/bash
|
||||
# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
|
||||
set -ex
|
||||
sudo chown -R "$USER" output
|
||||
python publish.py
|
3
packaging/linux_wheels/go.sh
Normal file
3
packaging/linux_wheels/go.sh
Normal file
@ -0,0 +1,3 @@
|
||||
#!/usr/bin/bash
|
||||
# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
|
||||
sudo docker run --rm -v "$PWD/../../:/inside" pytorch/conda-cuda bash inside/packaging/linux_wheels/inside.sh
|
102
packaging/linux_wheels/inside.sh
Normal file
102
packaging/linux_wheels/inside.sh
Normal file
@ -0,0 +1,102 @@
|
||||
#!/bin/bash
|
||||
# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
|
||||
set -ex
|
||||
|
||||
conda init bash
|
||||
# shellcheck source=/dev/null
|
||||
source ~/.bashrc
|
||||
|
||||
cd /inside
|
||||
VERSION=$(python -c "exec(open('pytorch3d/__init__.py').read()); print(__version__)")
|
||||
|
||||
export BUILD_VERSION=$VERSION
|
||||
export FORCE_CUDA=1
|
||||
|
||||
wget --no-verbose https://github.com/NVIDIA/cub/archive/1.10.0.tar.gz
|
||||
tar xzf 1.10.0.tar.gz
|
||||
CUB_HOME=$(realpath ./cub-1.10.0)
|
||||
export CUB_HOME
|
||||
echo "CUB_HOME is now $CUB_HOME"
|
||||
|
||||
|
||||
PYTHON_VERSIONS="3.6 3.7 3.8 3.9"
|
||||
# the keys are pytorch versions
|
||||
declare -A CONDA_CUDA_VERSIONS=(
|
||||
# ["1.4.0"]="cu101"
|
||||
# ["1.5.0"]="cu101 cu102"
|
||||
# ["1.5.1"]="cu101 cu102"
|
||||
# ["1.6.0"]="cu101 cu102"
|
||||
["1.7.0"]="cu101 cu102 cu110"
|
||||
["1.7.1"]="cu101 cu102 cu110"
|
||||
)
|
||||
|
||||
|
||||
|
||||
for python_version in $PYTHON_VERSIONS
|
||||
do
|
||||
for pytorch_version in "${!CONDA_CUDA_VERSIONS[@]}"
|
||||
do
|
||||
if [[ "3.6 3.7 3.8" != *$python_version* ]] && [[ "1.4.0 1.5.0 1.5.1 1.6.0 1.7.0" == *$pytorch_version* ]]
|
||||
then
|
||||
#python 3.9 and later not supported by pytorch 1.7.0 and before
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "3.9" == "$python_version" ]]
|
||||
then
|
||||
extra_channel="-c conda-forge"
|
||||
else
|
||||
extra_channel=""
|
||||
fi
|
||||
|
||||
for cu_version in ${CONDA_CUDA_VERSIONS[$pytorch_version]}
|
||||
do
|
||||
case "$cu_version" in
|
||||
cu110)
|
||||
export CUDA_HOME=/usr/local/cuda-11.0/
|
||||
export CUDA_TAG=11.0
|
||||
export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_50,code=compute_50"
|
||||
;;
|
||||
cu102)
|
||||
export CUDA_HOME=/usr/local/cuda-10.2/
|
||||
export CUDA_TAG=10.2
|
||||
export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50"
|
||||
;;
|
||||
cu101)
|
||||
export CUDA_HOME=/usr/local/cuda-10.1/
|
||||
export CUDA_TAG=10.1
|
||||
export NVCC_FLAGS="-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_50,code=sm_50 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_70,code=sm_70 -gencode=arch=compute_75,code=sm_75 -gencode=arch=compute_50,code=compute_50"
|
||||
;;
|
||||
*)
|
||||
echo "Unrecognized cu_version=$cu_version"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
tag=py"${python_version//./}"_"${cu_version}"_pyt"${pytorch_version//./}"
|
||||
|
||||
outdir="/inside/packaging/linux_wheels/output/$tag"
|
||||
if [[ -d "$outdir" ]]
|
||||
then
|
||||
continue
|
||||
fi
|
||||
|
||||
conda create -y -n "$tag" "python=$python_version"
|
||||
conda activate "$tag"
|
||||
conda install -y -c pytorch $extra_channel "pytorch=$pytorch_version" "cudatoolkit=$CUDA_TAG" torchvision
|
||||
pip install fvcore iopath
|
||||
echo "python version" "$python_version" "pytorch version" "$pytorch_version" "cuda version" "$cu_version" "tag" "$tag"
|
||||
|
||||
rm -rf dist
|
||||
|
||||
python setup.py clean
|
||||
python setup.py bdist_wheel
|
||||
|
||||
rm -rf "$outdir"
|
||||
mkdir -p "$outdir"
|
||||
cp dist/*whl "$outdir"
|
||||
|
||||
conda deactivate
|
||||
done
|
||||
done
|
||||
done
|
||||
echo "DONE"
|
76
packaging/linux_wheels/publish.py
Normal file
76
packaging/linux_wheels/publish.py
Normal file
@ -0,0 +1,76 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates. All rights reserved.
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
|
||||
dest = "s3://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/"
|
||||
|
||||
output = Path("output")
|
||||
|
||||
|
||||
def fs3cmd(args, allow_failure: bool = False) -> List[str]:
|
||||
"""
|
||||
This function returns the args for subprocess to mimic the bash command
|
||||
fs3cmd available in the fairusers_aws module on the FAIR cluster.
|
||||
"""
|
||||
os.environ["FAIR_CLUSTER_NAME"] = os.environ["FAIR_ENV_CLUSTER"].lower()
|
||||
cmd_args = ["/public/apps/fairusers_aws/bin/fs3cmd"] + args
|
||||
return cmd_args
|
||||
|
||||
|
||||
def fs3_exists(path) -> bool:
|
||||
"""
|
||||
Returns True if the path exists inside dest on S3.
|
||||
In fact, will also return True if there is a file which has the given
|
||||
path as a prefix, but we are careful about this.
|
||||
"""
|
||||
out = subprocess.check_output(fs3cmd(["ls", path]))
|
||||
return len(out) != 0
|
||||
|
||||
|
||||
def get_html_wrappers() -> None:
|
||||
for directory in sorted(output.iterdir()):
|
||||
output_wrapper = directory / "download.html"
|
||||
assert not output_wrapper.exists()
|
||||
dest_wrapper = dest + directory.name + "/download.html"
|
||||
if fs3_exists(dest_wrapper):
|
||||
subprocess.check_call(fs3cmd(["get", dest_wrapper, str(output_wrapper)]))
|
||||
|
||||
|
||||
def write_html_wrappers() -> None:
|
||||
html = """
|
||||
<a href="$">$</a><br>
|
||||
"""
|
||||
|
||||
for directory in sorted(output.iterdir()):
|
||||
files = list(directory.glob("*.whl"))
|
||||
assert len(files) == 1, files
|
||||
[wheel] = files
|
||||
|
||||
this_html = html.replace("$", wheel.name)
|
||||
output_wrapper = directory / "download.html"
|
||||
if output_wrapper.exists():
|
||||
contents = output_wrapper.read_text()
|
||||
if this_html not in contents:
|
||||
with open(output_wrapper, "a") as f:
|
||||
f.write(this_html)
|
||||
else:
|
||||
output_wrapper.write_text(this_html)
|
||||
|
||||
|
||||
def to_aws() -> None:
|
||||
for directory in output.iterdir():
|
||||
for file in directory.iterdir():
|
||||
print(file)
|
||||
subprocess.check_call(
|
||||
fs3cmd(["put", str(file), dest + str(file.relative_to(output))])
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Uncomment this for subsequent releases.
|
||||
# get_html_wrappers()
|
||||
write_html_wrappers()
|
||||
to_aws()
|
6
setup.py
6
setup.py
@ -132,8 +132,10 @@ setup(
|
||||
url="https://github.com/facebookresearch/pytorch3d",
|
||||
description="PyTorch3D is FAIR's library of reusable components "
|
||||
"for deep Learning with 3D data.",
|
||||
packages=find_packages(exclude=("configs", "tests", "tests.*")),
|
||||
install_requires=["torchvision>=0.4", "fvcore", "iopath"],
|
||||
packages=find_packages(
|
||||
exclude=("configs", "tests", "tests.*", "docs.*", "projects.*")
|
||||
),
|
||||
install_requires=["fvcore", "iopath"],
|
||||
extras_require={
|
||||
"all": ["matplotlib", "tqdm>4.29.0", "imageio", "ipywidgets"],
|
||||
"dev": ["flake8", "isort", "black==19.3b0"],
|
||||
|
Loading…
x
Reference in New Issue
Block a user