2017-08-08 16:36:23 -07:00
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
2015-09-28 19:56:37 -07:00
# pylint: disable=invalid-name, exec-used
""" Setup mxnet package. """
2015-08-05 22:51:03 -06:00
from __future__ import absolute_import
2016-09-08 19:10:58 -07:00
import os
import sys
2019-05-25 07:36:30 +09:00
from setuptools import find_packages # This must precede distutils
2018-08-24 18:19:56 +02:00
2016-09-07 22:49:04 -07:00
# need to use distutils.core for correct placement of cython dll
2017-05-23 09:04:37 -07:00
kwargs = { }
2016-11-20 22:23:15 -08:00
if " --inplace " in sys . argv :
from distutils . core import setup
from distutils . extension import Extension
else :
from setuptools import setup
from setuptools . extension import Extension
2020-04-22 23:53:12 -07:00
kwargs = { ' install_requires ' : [ ' numpy>=1.17 ' , ' requests>=2.20.0,<3 ' , ' graphviz<0.9.0,>=0.8.1 ' ] , ' zip_safe ' : False }
2017-05-23 09:04:37 -07:00
2017-05-23 11:44:38 -07:00
with_cython = False
2017-05-23 09:04:37 -07:00
if ' --with-cython ' in sys . argv :
with_cython = True
sys . argv . remove ( ' --with-cython ' )
2015-08-05 22:51:03 -06:00
2015-09-28 19:56:37 -07:00
# We can not import `mxnet.info.py` in setup.py directly since mxnet/__init__.py
# Will be invoked which introduces dependences
CURRENT_DIR = os . path . dirname ( __file__ )
libinfo_py = os . path . join ( CURRENT_DIR , ' mxnet/libinfo.py ' )
libinfo = { ' __file__ ' : libinfo_py }
exec ( compile ( open ( libinfo_py , " rb " ) . read ( ) , libinfo_py , ' exec ' ) , libinfo , libinfo )
LIB_PATH = libinfo [ ' find_lib_path ' ] ( )
__version__ = libinfo [ ' __version__ ' ]
2015-08-05 22:51:03 -06:00
2017-10-16 21:42:45 -07:00
sys . path . insert ( 0 , CURRENT_DIR )
# Try to generate auto-complete code
try :
from mxnet . base import _generate_op_module_signature
from mxnet . ndarray . register import _generate_ndarray_function_code
from mxnet . symbol . register import _generate_symbol_function_code
_generate_op_module_signature ( ' mxnet ' , ' symbol ' , _generate_symbol_function_code )
_generate_op_module_signature ( ' mxnet ' , ' ndarray ' , _generate_ndarray_function_code )
except : # pylint: disable=bare-except
pass
2016-09-18 00:34:46 +08:00
2016-09-07 16:08:55 -07:00
def config_cython ( ) :
2017-01-18 18:51:32 -08:00
""" Try to configure cython and return cython configuration """
2017-05-23 09:04:37 -07:00
if not with_cython :
return [ ]
2017-02-24 09:21:33 -08:00
# pylint: disable=unreachable
2017-01-18 18:51:32 -08:00
if os . name == ' nt ' :
print ( " WARNING: Cython is not supported on Windows, will compile without cython module " )
return [ ]
2016-09-07 16:08:55 -07:00
try :
from Cython . Build import cythonize
2020-02-15 01:31:53 +00:00
subdir = " _cy3 "
2016-09-07 16:08:55 -07:00
ret = [ ]
path = " mxnet/cython "
2016-09-18 00:34:46 +08:00
if os . name == ' nt ' :
2016-12-23 16:43:01 +08:00
library_dirs = [ ' mxnet ' , ' ../build/Release ' , ' ../build ' ]
2016-09-18 00:34:46 +08:00
libraries = [ ' libmxnet ' ]
else :
2019-05-25 07:36:30 +09:00
library_dirs = [ os . path . dirname ( p ) for p in LIB_PATH ]
libraries = [ ' mxnet ' ]
# Default paths to libmxnet.so relative to the shared library file generated by cython.
# These precede LD_LIBRARY_PATH.
extra_link_args = [ " -Wl,-rpath=$ORIGIN/..:$ORIGIN/../../../lib:$ORIGIN/../../../build " ]
2016-09-07 16:08:55 -07:00
for fn in os . listdir ( path ) :
if not fn . endswith ( " .pyx " ) :
continue
ret . append ( Extension (
2019-05-25 07:36:30 +09:00
" mxnet. %s . %s " % ( subdir , fn [ : - 4 ] ) ,
2016-09-07 16:08:55 -07:00
[ " mxnet/cython/ %s " % fn ] ,
2018-08-11 20:49:32 +08:00
include_dirs = [ " ../include/ " , " ../3rdparty/tvm/nnvm/include " ] ,
2016-09-18 00:34:46 +08:00
library_dirs = library_dirs ,
libraries = libraries ,
2019-05-25 07:36:30 +09:00
extra_link_args = extra_link_args ,
2016-09-07 16:08:55 -07:00
language = " c++ " ) )
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
2020-02-25 04:47:23 +08:00
path = " mxnet/_ffi/_cython "
for fn in os . listdir ( path ) :
if not fn . endswith ( " .pyx " ) :
continue
ret . append ( Extension (
" mxnet._ffi. %s . %s " % ( subdir , fn [ : - 4 ] ) ,
[ " mxnet/_ffi/_cython/ %s " % fn ] ,
include_dirs = [ " ../include/ " , " ../3rdparty/tvm/nnvm/include " ] ,
library_dirs = library_dirs ,
libraries = libraries ,
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
extra_compile_args = [ " -std=c++17 " ] ,
2020-02-25 04:47:23 +08:00
extra_link_args = extra_link_args ,
language = " c++ " ) )
2019-05-25 07:36:30 +09:00
# If `force=True` is not used and you cythonize the modules for python2 and python3
# successively, you need to delete `mxnet/cython/ndarray.cpp` after the first cythonize.
return cythonize ( ret , force = True )
2016-09-07 22:49:04 -07:00
except ImportError :
2016-09-07 16:08:55 -07:00
print ( " WARNING: Cython is not installed, will compile without cython module " )
return [ ]
2017-10-16 21:42:45 -07:00
2015-08-05 22:51:03 -06:00
setup ( name = ' mxnet ' ,
2015-09-28 19:56:37 -07:00
version = __version__ ,
description = open ( os . path . join ( CURRENT_DIR , ' README.md ' ) ) . read ( ) ,
2017-07-23 16:23:50 -07:00
packages = find_packages ( ) ,
2015-08-05 22:51:03 -06:00
data_files = [ ( ' mxnet ' , [ LIB_PATH [ 0 ] ] ) ] ,
2018-01-03 22:42:07 -08:00
url = ' https://github.com/apache/incubator-mxnet ' ,
2017-05-23 09:04:37 -07:00
ext_modules = config_cython ( ) ,
2018-07-20 05:00:38 +02:00
classifiers = [
# https://pypi.org/pypi?%3Aaction=list_classifiers
' Development Status :: 5 - Production/Stable ' ,
' Intended Audience :: Developers ' ,
' Intended Audience :: Education ' ,
' Intended Audience :: Science/Research ' ,
' License :: OSI Approved :: Apache Software License ' ,
' Programming Language :: C++ ' ,
' Programming Language :: Cython ' ,
' Programming Language :: Other ' , # R, Scala
' Programming Language :: Perl ' ,
' Programming Language :: Python ' ,
' Programming Language :: Python :: 2.7 ' ,
' Programming Language :: Python :: 3.4 ' ,
' Programming Language :: Python :: 3.5 ' ,
' Programming Language :: Python :: 3.6 ' ,
' Programming Language :: Python :: Implementation :: CPython ' ,
' Topic :: Scientific/Engineering ' ,
' Topic :: Scientific/Engineering :: Artificial Intelligence ' ,
' Topic :: Scientific/Engineering :: Mathematics ' ,
' Topic :: Software Development ' ,
' Topic :: Software Development :: Libraries ' ,
' Topic :: Software Development :: Libraries :: Python Modules ' ,
] ,
2017-05-23 09:04:37 -07:00
* * kwargs )
