2018-03-09 23:33:10 +01:00
#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# build and install are separated so changes to build don't invalidate
# the whole docker cache for the image
set -ex
2022-03-08 15:00:20 -08:00
# compute capabilities for CI instances supported by CUDA 10.x (i.e. p3, g4)
CI_CMAKE_CUDA10_ARCH = "5.2 7.5"
# compute capabilities for CI instances supported by CUDA >= 11.1 (i.e. p3, g4, g5)
CI_CMAKE_CUDA_ARCH = "5.2 7.5 8.6"
2018-06-22 01:31:52 +02:00
2022-05-16 08:25:38 -07:00
# On newer nvidia cuda containers, these environment variables
# are prefixed with NV_, so provide compatibility
if [ ! -z " $NV_CUDNN_VERSION " ] ; then
if [ -z " $CUDNN_VERSION " ] ; then
export CUDNN_VERSION = $NV_CUDNN_VERSION
fi
fi
2018-03-09 23:33:10 +01:00
clean_repo( ) {
set -ex
git clean -xfd
git submodule foreach --recursive git clean -xfd
git reset --hard
git submodule foreach --recursive git reset --hard
git submodule update --init --recursive
}
2019-01-08 11:31:35 -08:00
scala_prepare( ) {
# Clean up maven logs
export MAVEN_OPTS = "-Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn"
}
2019-05-25 07:36:30 +09:00
check_cython( ) {
set -ex
2020-02-15 01:31:53 +00:00
local is_cython_used = $( python3 <<EOF
2019-05-25 07:36:30 +09:00
import sys
import mxnet as mx
2020-02-15 01:31:53 +00:00
cython_ndarraybase = 'mxnet._cy3.ndarray'
2019-05-25 07:36:30 +09:00
print(mx.nd._internal.NDArrayBase.__module__ == cython_ndarraybase)
EOF
)
if [ " ${ is_cython_used } " != "True" ] ; then
echo "ERROR: cython is not used."
return 1
else
echo "NOTE: cython is used."
return 0
2019-07-11 14:23:20 -07:00
fi
2019-05-25 07:36:30 +09:00
}
2018-06-15 02:32:01 +02:00
build_wheel( ) {
2018-03-09 23:33:10 +01:00
set -ex
pushd .
2018-06-08 03:12:30 +02:00
2018-06-15 02:32:01 +02:00
PYTHON_DIR = ${ 1 :- /work/mxnet/python }
BUILD_DIR = ${ 2 :- /work/build }
2018-06-08 03:12:30 +02:00
2018-06-15 02:32:01 +02:00
# build
2018-06-08 03:12:30 +02:00
2018-06-15 02:32:01 +02:00
export MXNET_LIBRARY_PATH = ${ BUILD_DIR } /libmxnet.so
2018-03-09 23:33:10 +01:00
2018-06-15 02:32:01 +02:00
cd ${ PYTHON_DIR }
2020-02-03 09:58:01 -08:00
python3 setup.py bdist_wheel
2018-03-09 23:33:10 +01:00
2018-06-15 02:32:01 +02:00
# repackage
2018-04-10 13:22:54 +00:00
# Fix pathing issues in the wheel. We need to move libmxnet.so from the data folder to the
# mxnet folder, then repackage the wheel.
2018-03-09 23:33:10 +01:00
WHEEL = ` readlink -f dist/*.whl`
TMPDIR = ` mktemp -d`
2018-06-15 02:32:01 +02:00
unzip -d ${ TMPDIR } ${ WHEEL }
rm ${ WHEEL }
cd ${ TMPDIR }
2020-08-09 13:33:03 -07:00
mv *.data/data/mxnet/libmxnet.so mxnet
2018-06-15 02:32:01 +02:00
zip -r ${ WHEEL } .
cp ${ WHEEL } ${ BUILD_DIR }
rm -rf ${ TMPDIR }
popd
}
2019-09-16 22:11:19 +02:00
gather_licenses( ) {
mkdir -p licenses
cp tools/dependencies/LICENSE.binary.dependencies licenses/
cp NOTICE licenses/
cp LICENSE licenses/
}
# Compiles the dynamic mxnet library
# Parameters:
2020-10-06 09:56:16 -07:00
# $1 -> mxnet_variant: the mxnet variant to build, e.g. cpu, native, cu101, cu102, etc.
2019-09-16 22:11:19 +02:00
build_dynamic_libmxnet( ) {
set -ex
2019-10-08 16:23:51 -07:00
2019-09-16 22:11:19 +02:00
local mxnet_variant = ${ 1 : ? "This function requires a mxnet variant as the first argument" }
# relevant licenses will be placed in the licenses directory
gather_licenses
2020-04-15 22:59:33 -07:00
cd /work/build
2020-10-05 11:34:20 -07:00
source /opt/rh/devtoolset-8/enable
2020-09-19 18:42:34 -07:00
# Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.
export CXXFLAGS = "-fabi-version=11 -fabi-compat-version=7"
2019-09-16 22:11:19 +02:00
if [ [ ${ mxnet_variant } = "cpu" ] ] ; then
2021-03-12 20:44:24 +01:00
cmake -DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= ON \
2020-04-15 22:59:33 -07:00
-DUSE_CUDA= OFF \
-G Ninja /work/mxnet
2020-03-09 13:41:46 -07:00
elif [ [ ${ mxnet_variant } = "native" ] ] ; then
2021-03-12 20:44:24 +01:00
cmake -DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= OFF \
2020-04-15 22:59:33 -07:00
-DUSE_CUDA= OFF \
-G Ninja /work/mxnet
2019-09-16 22:11:19 +02:00
elif [ [ ${ mxnet_variant } = ~ cu[ 0-9] +$ ] ] ; then
2021-03-12 20:44:24 +01:00
cmake -DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= ON \
2020-04-15 22:59:33 -07:00
-DUSE_DIST_KVSTORE= ON \
-DUSE_CUDA= ON \
-G Ninja /work/mxnet
2019-09-16 22:11:19 +02:00
else
echo " Error: Unrecognized mxnet variant ' ${ mxnet_variant } ' "
2020-03-09 13:41:46 -07:00
exit 1
2019-09-16 22:11:19 +02:00
fi
2020-04-15 22:59:33 -07:00
ninja
2019-09-16 22:11:19 +02:00
}
2018-06-15 02:32:01 +02:00
build_jetson( ) {
set -ex
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
cd /work/build
cmake \
-DCMAKE_TOOLCHAIN_FILE= ${ CMAKE_TOOLCHAIN_FILE } \
-DUSE_CUDA= ON \
-DMXNET_CUDA_ARCH= "5.2" \
-DUSE_OPENCV= OFF \
-DUSE_OPENMP= ON \
-DUSE_LAPACK= OFF \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
-DCMAKE_BUILD_TYPE= Release \
-G Ninja /work/mxnet
ninja
build_wheel
2018-03-09 23:33:10 +01:00
}
2018-07-19 17:18:04 +02:00
#
# ARM builds
#
2018-03-22 17:04:15 +01:00
build_armv6( ) {
set -ex
cd /work/build
2018-04-11 14:31:18 +02:00
# We do not need OpenMP, since most armv6 systems have only 1 core
2018-03-22 17:04:15 +01:00
cmake \
2018-06-15 02:32:01 +02:00
-DCMAKE_TOOLCHAIN_FILE= ${ CMAKE_TOOLCHAIN_FILE } \
2018-03-22 17:04:15 +01:00
-DUSE_CUDA= OFF \
-DUSE_OPENCV= OFF \
2018-04-11 14:31:18 +02:00
-DUSE_OPENMP= OFF \
2018-04-06 14:57:03 +02:00
-DCMAKE_BUILD_TYPE= Release \
2018-03-22 17:04:15 +01:00
-DUSE_LAPACK= OFF \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2018-04-06 14:57:03 +02:00
-DBUILD_CPP_EXAMPLES= OFF \
2018-03-22 17:04:15 +01:00
-G Ninja /work/mxnet
2018-06-15 02:32:01 +02:00
2020-02-19 21:22:57 -08:00
ninja
2018-06-15 02:32:01 +02:00
build_wheel
2018-03-22 17:04:15 +01:00
}
2018-03-09 23:33:10 +01:00
build_armv7( ) {
set -ex
cd /work/build
2018-06-15 02:32:01 +02:00
2018-06-08 03:12:30 +02:00
cmake \
2018-06-15 02:32:01 +02:00
-DCMAKE_TOOLCHAIN_FILE= ${ CMAKE_TOOLCHAIN_FILE } \
-DUSE_CUDA= OFF \
-DUSE_OPENCV= OFF \
-DUSE_OPENMP= ON \
-DCMAKE_BUILD_TYPE= Release \
-DUSE_LAPACK= OFF \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2018-06-15 02:32:01 +02:00
-DBUILD_CPP_EXAMPLES= OFF \
2018-03-09 23:33:10 +01:00
-G Ninja /work/mxnet
2018-06-15 02:32:01 +02:00
2020-02-19 21:22:57 -08:00
ninja
2018-06-15 02:32:01 +02:00
build_wheel
2018-03-09 23:33:10 +01:00
}
2018-07-19 17:18:04 +02:00
build_armv8( ) {
2020-02-28 11:59:22 -08:00
cd /work/build
2018-06-08 03:12:30 +02:00
cmake \
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
-DCMAKE_TOOLCHAIN_FILE= ${ CMAKE_TOOLCHAIN_FILE } \
-DUSE_CUDA= OFF \
-DUSE_OPENCV= OFF \
2018-08-22 15:24:19 +02:00
-DUSE_OPENMP= ON \
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
-DUSE_LAPACK= OFF \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
-DCMAKE_BUILD_TYPE= Release \
2018-03-09 23:33:10 +01:00
-G Ninja /work/mxnet
2020-02-19 21:22:57 -08:00
ninja
2018-07-02 19:35:22 +02:00
build_wheel
2018-03-09 23:33:10 +01:00
}
2018-07-19 17:18:04 +02:00
#
# ANDROID builds
#
2018-06-15 19:33:33 +02:00
build_android_armv7( ) {
2018-03-09 23:33:10 +01:00
set -ex
cd /work/build
2020-06-19 14:46:27 -07:00
# ANDROID_ABI and ANDROID_STL are options of the CMAKE_TOOLCHAIN_FILE
# provided by Android NDK
2018-06-08 03:12:30 +02:00
cmake \
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
-DCMAKE_TOOLCHAIN_FILE= ${ CMAKE_TOOLCHAIN_FILE } \
-DANDROID_ABI= "armeabi-v7a" \
-DANDROID_STL= "c++_shared" \
-DUSE_CUDA= OFF \
-DUSE_LAPACK= OFF \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
-DUSE_OPENCV= OFF \
-DUSE_OPENMP= OFF \
2018-03-09 23:33:10 +01:00
-G Ninja /work/mxnet
2020-02-19 21:22:57 -08:00
ninja
2018-03-09 23:33:10 +01:00
}
2018-07-19 17:18:04 +02:00
build_android_armv8( ) {
2018-06-15 19:33:33 +02:00
set -ex
cd /work/build
2020-06-19 14:46:27 -07:00
# ANDROID_ABI and ANDROID_STL are options of the CMAKE_TOOLCHAIN_FILE
# provided by Android NDK
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
cmake \
-DCMAKE_TOOLCHAIN_FILE= ${ CMAKE_TOOLCHAIN_FILE } \
-DANDROID_ABI= "arm64-v8a" \
-DANDROID_STL= "c++_shared" \
-DUSE_CUDA= OFF \
-DUSE_LAPACK= OFF \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
-DUSE_OPENCV= OFF \
-DUSE_OPENMP= OFF \
-DUSE_SIGNAL_HANDLER= ON \
2018-06-15 19:33:33 +02:00
-G Ninja /work/mxnet
2020-02-19 21:22:57 -08:00
ninja
2018-06-15 19:33:33 +02:00
}
2018-03-09 23:33:10 +01:00
build_centos7_cpu( ) {
2020-02-28 11:59:22 -08:00
set -ex
cd /work/build
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
source /opt/rh/devtoolset-7/enable
2020-09-19 18:42:34 -07:00
# Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.
export CXXFLAGS = "-fabi-version=11 -fabi-compat-version=7"
2020-02-28 11:59:22 -08:00
cmake \
-DCMAKE_BUILD_TYPE= "RelWithDebInfo" \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= OFF \
2020-02-28 11:59:22 -08:00
-DUSE_DIST_KVSTORE= ON \
-DUSE_CUDA= OFF \
2020-11-13 23:13:06 -08:00
-DBUILD_EXTENSION_PATH= /work/mxnet/example/extensions/lib_external_ops \
2020-11-17 12:09:42 -08:00
-DUSE_INT64_TENSOR_SIZE= OFF \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2020-02-28 11:59:22 -08:00
-G Ninja /work/mxnet
ninja
}
2021-03-24 15:15:32 +01:00
build_centos7_onednn( ) {
2018-03-26 19:05:47 +08:00
set -ex
2020-02-28 11:59:22 -08:00
cd /work/build
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
source /opt/rh/devtoolset-7/enable
2020-09-19 18:42:34 -07:00
# Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.
export CXXFLAGS = "-fabi-version=11 -fabi-compat-version=7"
2021-03-12 20:44:24 +01:00
cmake -DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= ON \
2020-02-28 11:59:22 -08:00
-DUSE_CUDA= OFF \
2020-11-17 12:09:42 -08:00
-DUSE_INT64_TENSOR_SIZE= OFF \
2020-02-28 11:59:22 -08:00
-G Ninja /work/mxnet
ninja
2018-03-26 19:05:47 +08:00
}
2018-03-09 23:33:10 +01:00
build_centos7_gpu( ) {
set -ex
2020-02-28 11:59:22 -08:00
cd /work/build
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
source /opt/rh/devtoolset-7/enable
2020-09-19 18:42:34 -07:00
# Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.
export CXXFLAGS = "-fabi-version=11 -fabi-compat-version=7"
2020-02-28 11:59:22 -08:00
cmake \
-DCMAKE_BUILD_TYPE= "RelWithDebInfo" \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= ON \
2020-02-28 11:59:22 -08:00
-DUSE_CUDA= ON \
2022-03-08 15:00:20 -08:00
-DMXNET_CUDA_ARCH= " $CI_CMAKE_CUDA10_ARCH " \
2020-11-17 12:09:42 -08:00
-DUSE_DIST_KVSTORE= ON \
2020-11-13 23:13:06 -08:00
-DBUILD_EXTENSION_PATH= /work/mxnet/example/extensions/lib_external_ops \
2020-11-17 12:09:42 -08:00
-DUSE_INT64_TENSOR_SIZE= OFF \
2020-02-28 11:59:22 -08:00
-G Ninja /work/mxnet
ninja
2018-03-09 23:33:10 +01:00
}
2018-06-08 03:12:30 +02:00
build_ubuntu_cpu( ) {
build_ubuntu_cpu_openblas
}
2018-03-09 23:33:10 +01:00
build_ubuntu_cpu_openblas( ) {
2020-02-28 11:59:22 -08:00
set -ex
cd /work/build
2020-12-22 17:33:36 -08:00
CXXFLAGS = "-Wno-error=strict-overflow" CC = gcc-7 CXX = g++-7 cmake \
2020-02-28 11:59:22 -08:00
-DCMAKE_BUILD_TYPE= "RelWithDebInfo" \
2020-04-04 01:51:56 +00:00
-DENABLE_TESTCOVERAGE= ON \
2020-02-28 11:59:22 -08:00
-DUSE_TVM_OP= ON \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= OFF \
2020-02-28 11:59:22 -08:00
-DUSE_CUDA= OFF \
-DUSE_DIST_KVSTORE= ON \
-DBUILD_CYTHON_MODULES= ON \
2020-11-13 23:13:06 -08:00
-DBUILD_EXTENSION_PATH= /work/mxnet/example/extensions/lib_external_ops \
2020-02-28 11:59:22 -08:00
-G Ninja /work/mxnet
2021-10-05 18:08:58 -07:00
ninja -j$(( $( nproc) / 2 ))
2020-02-28 11:59:22 -08:00
}
2018-12-12 03:57:09 +08:00
build_ubuntu_cpu_mkl( ) {
set -ex
2020-03-04 08:55:01 -08:00
cd /work/build
2020-12-22 17:33:36 -08:00
CC = gcc-7 CXX = g++-7 cmake \
2020-03-04 08:55:01 -08:00
-DCMAKE_BUILD_TYPE= "RelWithDebInfo" \
2020-07-15 00:57:38 +00:00
-DENABLE_TESTCOVERAGE= OFF \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= OFF \
2020-03-04 08:55:01 -08:00
-DUSE_CUDA= OFF \
-DUSE_TVM_OP= ON \
[PERFORMANCE] [master] Layer normalization code from Marian for CPU (#19602)
* Layer normalization code from Marian
* Remove MKL version of LayerNorm.
Experiment with OMP_NUM_THREADS=4, times in s, c5.12xlarge
|batchxchanne| New code | MKL |
| 1x 32 | 0.0000288| 0.0000278|
| 128x 32 | 0.0000308| 0.0000311|
| 2560x 32 | 0.0000712| 0.0000672|
| 4096x 32 | 0.0000946| 0.0000910|
| 8192x 32 | 0.0001597| 0.0001523|
|16384x 32 | 0.0002905| 0.0002619|
| 1x 64 | 0.0000264| 0.0000256|
| 128x 64 | 0.0000339| 0.0000330|
| 2560x 64 | 0.0000829| 0.0000972|
| 4096x 64 | 0.0001137| 0.0001356|
| 8192x 64 | 0.0002027| 0.0002435|
|16384x 64 | 0.0003715| 0.0004639|
| 1x 128 | 0.0000262| 0.0000263|
| 128x 128 | 0.0000325| 0.0000389|
| 2560x 128 | 0.0001074| 0.0001580|
| 4096x 128 | 0.0001505| 0.0002336|
| 8192x 128 | 0.0002861| 0.0004481|
|16384x 128 | 0.0005648| 0.0008613|
| 1x 256 | 0.0000273| 0.0000276|
| 128x 256 | 0.0000390| 0.0000431|
| 2560x 256 | 0.0001533| 0.0002811|
| 4096x 256 | 0.0002258| 0.0004300|
| 8192x 256 | 0.0004300| 0.0008464|
|16384x 256 | 0.0010436| 0.0017613|
| 1x 512 | 0.0000256| 0.0000302|
| 128x 512 | 0.0000408| 0.0000551|
| 2560x 512 | 0.0002444| 0.0005225|
| 4096x 512 | 0.0003828| 0.0008147|
| 8192x 512 | 0.0008832| 0.0017192|
|16384x 512 | 0.0058463| 0.0074497|
| 1x 768 | 0.0000252| 0.0000308|
| 128x 768 | 0.0000450| 0.0000676|
| 2560x 768 | 0.0003440| 0.0007719|
| 4096x 768 | 0.0005890| 0.0013346|
| 8192x 768 | 0.0014946| 0.0026145|
|16384x 768 | 0.0089495| 0.0113557|
| 1x 1024 | 0.0000285| 0.0000308|
| 128x 1024 | 0.0000487| 0.0000786|
| 2560x 1024 | 0.0004614| 0.0010190|
| 4096x 1024 | 0.0008083| 0.0017376|
| 8192x 1024 | 0.0059020| 0.0075588|
|16384x 1024 | 0.0116553| 0.0146855|
Benchmark program
```python
import mxnet as mx
import time
def time_procedure(shape, count):
data = mx.nd.random_uniform(shape=shape, low=-1.0, high = 1.0)
factors = mx.nd.random_uniform(shape=(shape[-1],))
mx.nd.waitall()
begin = time.time()
for i in range(0, count):
out = mx.nd.LayerNorm(data, factors, factors)
mx.nd.waitall()
return (time.time() - begin) / count
count = 200
for channel in [32, 64, 128, 256, 512, 768, 1024]:
for batch in [1, 128, 2560, 4096, 8192, 16384]:
s = (batch, channel)
timing = time_procedure(s, count)
print("{:5d}x{:5d} | {:.7f}".format(s[0], s[1], timing))
```
* Enable pragma omp simd on MSVC
* Fix MSVC error C3016: 'j': index variable in OpenMP 'for' statement must have signed integral type
* Try to make MSVC happy since it doesn't have ssize_t
* Revert "Remove MKL version of LayerNorm."
This reverts commit 740c4726c3068ac30b3809cd6280fa7e91af8c52.
* Restore MKL layer normalization code, but it isn't called yet
* Pull division out of the hot loop
* Option to use MKL version requested by @samskalicky
* Add -DUSE_MKL_LAYERNORM=ON to ubuntu MKL CPU test
Co-authored-by: Kenneth Heafield <kheafiel@amazon.com>
2021-01-04 08:57:05 +00:00
-DUSE_MKL_LAYERNORM= ON \
2020-03-04 08:55:01 -08:00
-DUSE_BLAS= MKL \
2020-11-13 23:13:06 -08:00
-DBUILD_EXTENSION_PATH= /work/mxnet/example/extensions/lib_external_ops \
2020-03-04 08:55:01 -08:00
-GNinja /work/mxnet
ninja
2018-12-12 03:57:09 +08:00
}
2018-08-03 00:37:12 +02:00
build_ubuntu_cpu_cmake_debug( ) {
set -ex
cd /work/build
2020-12-22 17:33:36 -08:00
CC = gcc-7 CXX = g++-7 cmake \
2020-04-04 01:51:56 +00:00
-DCMAKE_BUILD_TYPE= Debug \
-DENABLE_TESTCOVERAGE= ON \
2018-08-03 00:37:12 +02:00
-DUSE_CUDA= OFF \
2019-09-05 14:39:38 +08:00
-DUSE_TVM_OP= ON \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2018-08-03 00:37:12 +02:00
-DUSE_OPENMP= OFF \
-DUSE_OPENCV= ON \
2019-01-08 14:01:44 +01:00
-DUSE_SIGNAL_HANDLER= ON \
2018-08-03 00:37:12 +02:00
-G Ninja \
/work/mxnet
2020-02-19 21:22:57 -08:00
ninja
2018-08-03 00:37:12 +02:00
}
2019-10-19 08:19:36 -07:00
build_ubuntu_cpu_cmake_no_tvm_op( ) {
set -ex
cd /work/build
2020-12-22 17:33:36 -08:00
CC = gcc-7 CXX = g++-7 cmake \
2019-10-19 08:19:36 -07:00
-DUSE_CUDA= OFF \
-DUSE_TVM_OP= OFF \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2019-10-19 08:19:36 -07:00
-DUSE_OPENMP= OFF \
-DUSE_OPENCV= ON \
-DUSE_SIGNAL_HANDLER= ON \
-DCMAKE_BUILD_TYPE= Release \
2020-11-13 23:13:06 -08:00
-DBUILD_EXTENSION_PATH= /work/mxnet/example/extensions/lib_external_ops \
2019-10-19 08:19:36 -07:00
-G Ninja \
/work/mxnet
2020-02-19 21:22:57 -08:00
ninja
2019-10-19 08:19:36 -07:00
}
2018-09-19 07:06:33 -07:00
build_ubuntu_cpu_cmake_asan( ) {
set -ex
cd /work/build
cmake \
-DUSE_CUDA= OFF \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= OFF \
2018-09-19 07:06:33 -07:00
-DUSE_OPENMP= OFF \
-DUSE_OPENCV= OFF \
-DCMAKE_BUILD_TYPE= Debug \
-DUSE_GPERFTOOLS= OFF \
-DUSE_JEMALLOC= OFF \
-DUSE_ASAN= ON \
/work/mxnet
make -j $( nproc) mxnet
}
2020-03-03 21:01:59 -08:00
build_ubuntu_cpu_gcc8_werror( ) {
set -ex
cd /work/build
2020-12-03 19:58:14 -07:00
CC = gcc-8 CXX = g++-8 cmake \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2020-03-03 21:01:59 -08:00
-DUSE_CUDA= OFF \
-DCMAKE_BUILD_TYPE= "RelWithDebInfo" \
-GNinja /work/mxnet
ninja
}
2020-03-17 21:36:50 -07:00
build_ubuntu_cpu_clang10_werror( ) {
set -ex
cd /work/build
CXX = clang++-10 CC = clang-10 cmake \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2020-03-17 21:36:50 -07:00
-DUSE_CUDA= OFF \
-DCMAKE_BUILD_TYPE= "RelWithDebInfo" \
-GNinja /work/mxnet
ninja
}
build_ubuntu_gpu_clang10_werror( ) {
set -ex
cd /work/build
# Disable cpp package as OpWrapperGenerator.py dlopens libmxnet.so,
# requiring presence of cuda driver libraries that are missing on CI host
export LD_LIBRARY_PATH = ${ LD_LIBRARY_PATH } :/usr/local/cuda-10.1/targets/x86_64-linux/lib/stubs
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
# Workaround https://github.com/thrust/thrust/issues/1072
# Can be deleted on Cuda 11
export CXXFLAGS = "-I/usr/local/thrust"
2020-03-17 21:36:50 -07:00
CXX = clang++-10 CC = clang-10 cmake \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2020-03-17 21:36:50 -07:00
-DUSE_CUDA= ON \
2020-12-03 19:58:14 -07:00
-DUSE_NVML= OFF \
2020-03-17 21:36:50 -07:00
-DMXNET_CUDA_ARCH= " $CI_CMAKE_CUDA_ARCH " \
-DCMAKE_BUILD_TYPE= "RelWithDebInfo" \
-GNinja /work/mxnet
ninja
}
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
build_ubuntu_cpu_clang6( ) {
2018-03-09 23:33:10 +01:00
set -ex
2020-02-28 11:59:22 -08:00
cd /work/build
2020-12-04 11:00:18 -08:00
export OpenBLAS_HOME = /usr/local/openblas-clang/
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
CXX = clang++-6.0 CC = clang-6.0 cmake \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= OFF \
2020-02-28 11:59:22 -08:00
-DUSE_CUDA= OFF \
-DUSE_OPENMP= OFF \
-DUSE_DIST_KVSTORE= ON \
-G Ninja /work/mxnet
ninja
2018-03-09 23:33:10 +01:00
}
2020-03-17 21:36:50 -07:00
build_ubuntu_cpu_clang100( ) {
2018-03-09 23:33:10 +01:00
set -ex
2020-02-28 11:59:22 -08:00
cd /work/build
2020-12-04 11:00:18 -08:00
export OpenBLAS_HOME = /usr/local/openblas-clang/
2020-03-17 21:36:50 -07:00
CXX = clang++-10 CC = clang-10 cmake \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= OFF \
2020-02-28 11:59:22 -08:00
-DUSE_CUDA= OFF \
-DUSE_OPENMP= ON \
-DUSE_DIST_KVSTORE= ON \
-G Ninja /work/mxnet
ninja
2018-03-09 23:33:10 +01:00
}
2018-08-27 13:18:13 +02:00
build_ubuntu_cpu_clang_tidy( ) {
set -ex
cd /work/build
2020-12-04 11:00:18 -08:00
export OpenBLAS_HOME = /usr/local/openblas-clang/
2020-03-17 21:36:50 -07:00
# TODO(leezu) USE_OPENMP=OFF 3rdparty/dmlc-core/CMakeLists.txt:79 broken?
2020-07-29 20:31:19 +00:00
CXX = clang++-10 CC = clang-10 cmake \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= OFF \
2020-02-28 11:59:22 -08:00
-DUSE_CUDA= OFF \
2020-03-17 21:36:50 -07:00
-DUSE_OPENMP= OFF \
2020-02-28 11:59:22 -08:00
-DCMAKE_BUILD_TYPE= Debug \
-DUSE_DIST_KVSTORE= ON \
2020-07-29 20:31:19 +00:00
-DCMAKE_CXX_CLANG_TIDY= clang-tidy-10 \
2020-02-28 11:59:22 -08:00
-G Ninja /work/mxnet
2020-02-19 21:22:57 -08:00
ninja
2018-08-27 13:18:13 +02:00
}
2021-03-24 15:15:32 +01:00
build_ubuntu_cpu_clang6_onednn( ) {
2018-03-14 22:38:45 +08:00
set -ex
2020-02-28 11:59:22 -08:00
cd /work/build
2020-12-04 11:00:18 -08:00
export OpenBLAS_HOME = /usr/local/openblas-clang/
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
CXX = clang++-6.0 CC = clang-6.0 cmake \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= ON \
2020-02-28 11:59:22 -08:00
-DUSE_CUDA= OFF \
-DUSE_OPENMP= OFF \
-G Ninja /work/mxnet
ninja
2018-03-14 22:38:45 +08:00
}
2021-03-24 15:15:32 +01:00
build_ubuntu_cpu_clang100_onednn( ) {
2018-03-14 22:38:45 +08:00
set -ex
2020-02-28 11:59:22 -08:00
cd /work/build
2020-12-04 11:00:18 -08:00
export OpenBLAS_HOME = /usr/local/openblas-clang/
2020-03-17 21:36:50 -07:00
CXX = clang++-10 CC = clang-10 cmake \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= ON \
2020-02-28 11:59:22 -08:00
-DUSE_CUDA= OFF \
-G Ninja /work/mxnet
ninja
2018-03-14 22:38:45 +08:00
}
2021-03-24 15:15:32 +01:00
build_ubuntu_cpu_onednn( ) {
2018-12-12 03:57:09 +08:00
set -ex
2020-02-28 11:59:22 -08:00
cd /work/build
2020-12-22 17:33:36 -08:00
CC = gcc-7 CXX = g++-7 cmake \
2020-02-28 11:59:22 -08:00
-DCMAKE_BUILD_TYPE= "RelWithDebInfo" \
2020-04-04 01:51:56 +00:00
-DENABLE_TESTCOVERAGE= ON \
-DUSE_TVM_OP= ON \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= ON \
2020-04-04 01:51:56 +00:00
-DUSE_CUDA= OFF \
2020-11-13 23:13:06 -08:00
-DBUILD_EXTENSION_PATH= /work/mxnet/example/extensions/lib_external_ops \
2020-04-04 01:51:56 +00:00
-G Ninja /work/mxnet
2020-02-28 11:59:22 -08:00
ninja
}
2018-12-12 03:57:09 +08:00
2021-03-24 15:15:32 +01:00
build_ubuntu_cpu_onednn_mkl( ) {
2020-02-28 11:59:22 -08:00
set -ex
2020-03-04 08:55:01 -08:00
cd /work/build
2020-12-22 17:33:36 -08:00
CC = gcc-7 CXX = g++-7 cmake \
2020-03-04 08:55:01 -08:00
-DCMAKE_BUILD_TYPE= "RelWithDebInfo" \
2020-07-15 00:57:38 +00:00
-DENABLE_TESTCOVERAGE= OFF \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= ON \
2020-03-04 08:55:01 -08:00
-DUSE_CUDA= OFF \
-DUSE_TVM_OP= ON \
-DUSE_BLAS= MKL \
2020-11-13 23:13:06 -08:00
-DBUILD_EXTENSION_PATH= /work/mxnet/example/extensions/lib_external_ops \
2020-03-04 08:55:01 -08:00
-GNinja /work/mxnet
ninja
2018-12-12 03:57:09 +08:00
}
2018-08-10 02:38:04 -07:00
build_ubuntu_gpu_tensorrt( ) {
set -ex
2020-12-22 17:33:36 -08:00
export CC = gcc-7
export CXX = g++-7
2020-08-03 17:15:02 -07:00
export ONNX_NAMESPACE = onnx
2022-05-16 08:25:38 -07:00
export PYBIN = $( which python3)
PYVERFULL = $( $PYBIN -V | awk '{print $2}' )
export PYVER = ${ PYVERFULL %.* }
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
2018-08-10 02:38:04 -07:00
# Build ONNX
pushd .
echo "Installing ONNX."
cd 3rdparty/onnx-tensorrt/third_party/onnx
rm -rf build
mkdir -p build
cd build
2022-05-16 08:25:38 -07:00
cmake -DPYTHON_EXECUTABLE= $PYBIN -DCMAKE_CXX_FLAGS= -I/usr/include/python${ PYVER } -DBUILD_SHARED_LIBS= ON ..
2020-08-03 17:15:02 -07:00
make -j$( nproc)
2018-08-10 02:38:04 -07:00
export LIBRARY_PATH = ` pwd ` :` pwd ` /onnx/:$LIBRARY_PATH
export CPLUS_INCLUDE_PATH = ` pwd ` :$CPLUS_INCLUDE_PATH
2020-08-03 17:15:02 -07:00
export CXXFLAGS = -I` pwd `
2018-08-10 02:38:04 -07:00
popd
# Build ONNX-TensorRT
2020-07-23 18:09:10 +00:00
export LD_LIBRARY_PATH = ${ LD_LIBRARY_PATH } :/usr/local/lib
2022-05-16 08:25:38 -07:00
export CPLUS_INCLUDE_PATH = ${ CPLUS_INCLUDE_PATH } :/usr/local/cuda/targets/x86_64-linux/include/
2018-08-10 02:38:04 -07:00
pushd .
cd 3rdparty/onnx-tensorrt/
mkdir -p build
cd build
2022-05-16 08:25:38 -07:00
cmake -DPYTHON_EXECUTABLE= $PYBIN -DONNX_NAMESPACE= $ONNX_NAMESPACE ..
2018-08-10 02:38:04 -07:00
make -j$( nproc)
export LIBRARY_PATH = ` pwd ` :$LIBRARY_PATH
popd
mkdir -p /work/mxnet/lib/
cp 3rdparty/onnx-tensorrt/third_party/onnx/build/*.so /work/mxnet/lib/
2020-08-03 17:15:02 -07:00
cp -L 3rdparty/onnx-tensorrt/build/libnvonnxparser.so /work/mxnet/lib/
2018-08-10 02:38:04 -07:00
2019-01-15 22:14:46 -08:00
cd /work/build
cmake -DUSE_CUDA= 1 \
-DUSE_CUDNN= 1 \
-DUSE_OPENCV= 1 \
-DUSE_TENSORRT= 1 \
2022-05-16 08:25:38 -07:00
-DUSE_INT64_TENSOR_SIZE= 1 \
2019-01-15 22:14:46 -08:00
-DUSE_OPENMP= 0 \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= 0 \
2020-12-03 19:58:14 -07:00
-DUSE_NVML= OFF \
2019-12-30 09:37:43 +00:00
-DMXNET_CUDA_ARCH= " $CI_CMAKE_CUDA_ARCH " \
2019-01-15 22:14:46 -08:00
-G Ninja \
/work/mxnet
2020-02-19 21:22:57 -08:00
ninja
2018-08-10 02:38:04 -07:00
}
2021-03-24 15:15:32 +01:00
build_ubuntu_gpu_onednn( ) {
2018-03-09 23:33:10 +01:00
set -ex
2020-02-28 11:59:22 -08:00
cd /work/build
2020-12-22 17:33:36 -08:00
CC = gcc-7 CXX = g++-7 cmake \
2020-03-17 21:36:50 -07:00
-DCMAKE_BUILD_TYPE= "RelWithDebInfo" \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2020-02-28 11:59:22 -08:00
-DUSE_CUDA= ON \
2020-12-03 19:58:14 -07:00
-DUSE_NVML= OFF \
2020-02-28 11:59:22 -08:00
-DMXNET_CUDA_ARCH= " $CI_CMAKE_CUDA_ARCH " \
2020-11-13 23:13:06 -08:00
-DBUILD_EXTENSION_PATH= /work/mxnet/example/extensions/lib_external_ops \
2020-02-28 11:59:22 -08:00
-G Ninja /work/mxnet
ninja
2018-03-09 23:33:10 +01:00
}
2021-03-24 15:15:32 +01:00
build_ubuntu_gpu_onednn_nocudnn( ) {
2018-07-12 16:40:24 -07:00
set -ex
2020-02-28 11:59:22 -08:00
cd /work/build
2020-12-22 17:33:36 -08:00
CC = gcc-7 CXX = g++-7 cmake \
2020-03-17 21:36:50 -07:00
-DCMAKE_BUILD_TYPE= "RelWithDebInfo" \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2020-02-28 11:59:22 -08:00
-DUSE_CUDA= ON \
2020-12-03 19:58:14 -07:00
-DUSE_NVML= OFF \
2020-02-28 11:59:22 -08:00
-DMXNET_CUDA_ARCH= " $CI_CMAKE_CUDA_ARCH " \
-DUSE_CUDNN= OFF \
2020-11-13 23:13:06 -08:00
-DBUILD_EXTENSION_PATH= /work/mxnet/example/extensions/lib_external_ops \
2020-02-28 11:59:22 -08:00
-G Ninja /work/mxnet
ninja
2018-07-12 16:40:24 -07:00
}
2020-12-03 19:58:14 -07:00
build_ubuntu_gpu( ) {
2020-02-28 11:59:22 -08:00
set -ex
cd /work/build
2021-05-24 13:44:39 -07:00
# Work around to link libcuda to libmxnet
# should be removed after https://github.com/apache/incubator-mxnet/issues/17858 is resolved.
ln -s -f /usr/local/cuda/targets/x86_64-linux/lib/stubs/libcuda.so libcuda.so.1
export LIBRARY_PATH = ${ LIBRARY_PATH } :/work/build
export LD_LIBRARY_PATH = ${ LD_LIBRARY_PATH } :/work/build
2020-12-22 17:33:36 -08:00
CC = gcc-7 CXX = g++-7 cmake \
2020-03-17 21:36:50 -07:00
-DCMAKE_BUILD_TYPE= "RelWithDebInfo" \
2020-02-28 11:59:22 -08:00
-DUSE_CUDA= ON \
2020-12-03 19:58:14 -07:00
-DUSE_NVML= OFF \
2020-02-28 11:59:22 -08:00
-DMXNET_CUDA_ARCH= " $CI_CMAKE_CUDA_ARCH " \
-DUSE_CUDNN= ON \
2021-05-24 13:44:39 -07:00
-DUSE_CPP_PACKAGE= ON \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= OFF \
2020-04-06 12:49:41 -07:00
-DUSE_DIST_KVSTORE= ON \
2020-02-28 11:59:22 -08:00
-DBUILD_CYTHON_MODULES= ON \
2020-11-13 23:13:06 -08:00
-DBUILD_EXTENSION_PATH= /work/mxnet/example/extensions/lib_external_ops \
2020-02-28 11:59:22 -08:00
-G Ninja /work/mxnet
2021-10-05 18:08:58 -07:00
ninja -j$(( $( nproc) / 2 ))
2020-02-28 11:59:22 -08:00
}
2020-12-03 19:58:14 -07:00
build_ubuntu_gpu_debug( ) {
2020-05-06 10:45:34 -07:00
set -ex
cd /work/build
2020-12-22 17:33:36 -08:00
CC = gcc-7 CXX = g++-7 cmake \
2020-05-06 10:45:34 -07:00
-DCMAKE_BUILD_TYPE= Debug \
-DUSE_CUDA= ON \
2020-12-03 19:58:14 -07:00
-DUSE_NVML= OFF \
2020-05-06 10:45:34 -07:00
-DMXNET_CUDA_ARCH= " $CI_CMAKE_CUDA_ARCH " \
-DUSE_CUDNN= ON \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= OFF \
2020-05-06 10:45:34 -07:00
-DUSE_DIST_KVSTORE= ON \
-DBUILD_CYTHON_MODULES= ON \
-G Ninja /work/mxnet
ninja
}
2019-04-23 14:47:10 -07:00
build_ubuntu_cpu_large_tensor( ) {
set -ex
cd /work/build
2020-12-22 17:33:36 -08:00
CC = gcc-7 CXX = g++-7 cmake \
2019-04-23 14:47:10 -07:00
-DUSE_SIGNAL_HANDLER= ON \
-DUSE_CUDA= OFF \
-DUSE_CUDNN= OFF \
2021-03-12 20:44:24 +01:00
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= ON \
2019-04-23 14:47:10 -07:00
-G Ninja \
/work/mxnet
2020-02-19 21:22:57 -08:00
ninja
2019-04-23 14:47:10 -07:00
}
build_ubuntu_gpu_large_tensor( ) {
set -ex
cd /work/build
2020-12-22 17:33:36 -08:00
CC = gcc-7 CXX = g++-7 cmake \
2019-04-23 14:47:10 -07:00
-DUSE_SIGNAL_HANDLER= ON \
-DUSE_CUDA= ON \
-DUSE_CUDNN= ON \
2021-03-12 20:44:24 +01:00
-DUSE_NVML= OFF \
-DUSE_BLAS= Open \
2021-03-15 17:32:37 +01:00
-DUSE_ONEDNN= ON \
2019-04-23 14:47:10 -07:00
-DUSE_DIST_KVSTORE= ON \
-DCMAKE_BUILD_TYPE= Release \
2019-12-30 09:37:43 +00:00
-DMXNET_CUDA_ARCH= " $CI_CMAKE_CUDA_ARCH " \
2019-04-23 14:47:10 -07:00
-G Ninja \
/work/mxnet
2020-02-19 21:22:57 -08:00
ninja
2019-04-23 14:47:10 -07:00
}
2018-03-09 23:33:10 +01:00
# Testing
sanity_check( ) {
2020-07-25 02:48:30 +00:00
set -ex
2021-10-10 00:02:42 +02:00
sanity_clang
2020-07-25 02:48:30 +00:00
sanity_license
2021-11-21 04:05:47 +01:00
sanity_cmakelint
2021-10-30 16:54:39 +02:00
sanity_tutorial
sanity_python_prospector
2020-07-25 02:48:30 +00:00
sanity_cpp
}
2021-11-21 04:05:47 +01:00
sanity_cmakelint( ) {
set -exu
git ls-files -z -- bootstrap '*.cmake' '*.cmake.in' '*CMakeLists.txt' | grep -E -z -v '^(3rdparty)|cmake/Modules/|cmake/upstream/' | xargs -0 cmakelint --config= .cmakelintrc --quiet
}
2021-10-30 16:54:39 +02:00
sanity_tutorial( ) {
set -ex
export DMLC_LOG_STACK_TRACE_DEPTH = 100
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -n 4 tests/tutorials/test_sanity_tutorials.py
}
2020-07-25 02:48:30 +00:00
sanity_license( ) {
2018-03-09 23:33:10 +01:00
set -ex
tools/license_header.py check
2020-07-25 02:48:30 +00:00
}
2020-07-28 22:11:20 +00:00
sanity_cpp( ) {
2020-07-25 02:48:30 +00:00
set -ex
2021-05-24 13:44:39 -07:00
3rdparty/dmlc-core/scripts/lint.py mxnet cpp include src plugin cpp-package tests --exclude_path src/operator/contrib/ctc_include include/onednn
2020-07-25 02:48:30 +00:00
}
2021-10-30 16:54:39 +02:00
sanity_python_prospector( ) {
set -e
set +x
# Run Prospector
python3 -m prospector --profile prospector.yaml | tee prospector-output.txt
error_cnt = $( awk '/Messages Found:/{print $NF}' prospector-output.txt)
if [ $error_cnt -ne 0 ] ; then
echo 'Please fix the above Prospector warnings.'
rm -rf prospector-output.txt
exit 1
fi
rm -rf prospector-output.txt
}
2021-10-10 00:02:42 +02:00
sanity_clang( ) {
2021-10-18 12:44:32 +02:00
set -e
set +x
2021-10-10 00:02:42 +02:00
# .github/workgflows/greetings.yml passes BASE_SHA, GITHUB_RUN_ID, GITHUB_BASE_REF for pull requests.
BASE_SHA = " ${ GITHUB_PR_BASE_SHA } "
GITHUB_RUN_ID = " ${ GITHUB_PR_RUN_ID } "
GITHUB_BASE_REF = " ${ GITHUB_PR_BASE_REF } "
if [ " ${ BASE_SHA } " = = "" ] ; then
BASE_SHA = ` git show-ref --hash refs/remotes/origin/master`
if [ " ${ GITHUB_RUN_ID } " = = "" ] || [ " ${ GITHUB_BASE_REF } " = = "" ] ; then
GITHUB_RUN_ID = ` ( git log --pretty= format:'%h' -n 1) `
GITHUB_BASE_REF = "master"
fi
fi
git remote add " ${ GITHUB_RUN_ID } " https://github.com/apache/incubator-mxnet.git
git fetch " ${ GITHUB_RUN_ID } " " $GITHUB_BASE_REF "
tools/lint/clang_format_ci.sh " ${ BASE_SHA } "
GIT_DIFFERENCE = $( git diff)
if [ [ -z $GIT_DIFFERENCE ] ] ; then
git remote remove " ${ GITHUB_RUN_ID } " # temporary remote is removed
2021-10-21 22:27:13 +02:00
return
2021-10-10 00:02:42 +02:00
fi
2021-10-18 12:44:32 +02:00
2021-10-10 00:02:42 +02:00
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
2021-10-18 12:44:32 +02:00
echo "| Clang-format failures found! Run: "
echo " | tools/lint/clang_format_ci.sh ${ BASE_SHA } "
2021-10-10 00:02:42 +02:00
echo "| to fix this error. "
2021-10-18 12:44:32 +02:00
echo "| For more info, see: https://mxnet.apache.org/versions/master/community/clang_format_guide"
2021-10-10 00:02:42 +02:00
echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
2021-10-18 12:44:32 +02:00
2021-10-10 00:02:42 +02:00
echo " $GIT_DIFFERENCE "
git remote remove " ${ GITHUB_RUN_ID } " # temporary remote is removed
exit 1
}
2019-05-23 12:48:44 +02:00
# Tests libmxnet
# Parameters:
# $1 -> mxnet_variant: The variant of the libmxnet.so library
cd_unittest_ubuntu( ) {
set -ex
2021-10-19 22:21:04 -07:00
source /opt/rh/rh-python38/enable
2019-05-23 12:48:44 +02:00
export PYTHONPATH = ./python/
2021-03-16 20:16:53 +01:00
export MXNET_ONEDNN_DEBUG = 0 # Ignored if not present
2019-05-23 12:48:44 +02:00
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE = 0
2019-10-24 04:47:01 -05:00
export MXNET_SUBGRAPH_VERBOSE = 0
2019-09-12 23:00:46 +02:00
export MXNET_ENABLE_CYTHON = 0
export CD_JOB = 1 # signal this is a CD run so any unecessary tests can be skipped
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2019-05-23 12:48:44 +02:00
local mxnet_variant = ${ 1 : ? "This function requires a mxnet variant as the first argument" }
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -n 4 --durations= 50 --verbose tests/python/unittest
2020-05-04 16:44:27 -07:00
pytest -m 'serial' --durations= 50 --verbose tests/python/unittest
2019-05-23 12:48:44 +02:00
2023-01-04 04:09:23 -08:00
# https://github.com/apache/mxnet/issues/11801
2019-05-23 12:48:44 +02:00
# if [[ ${mxnet_variant} = "cpu" ]] || [[ ${mxnet_variant} = "mkl" ]]; then
# integrationtest_ubuntu_cpu_dist_kvstore
# fi
if [ [ ${ mxnet_variant } = cu* ] ] ; then
2020-05-16 19:04:44 -07:00
MXNET_GPU_MEM_POOL_TYPE = Unpooled \
MXNET_ENGINE_TYPE = NaiveEngine \
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations= 50 --verbose tests/python/gpu
2020-05-16 19:04:44 -07:00
MXNET_GPU_MEM_POOL_TYPE = Unpooled \
2020-11-19 15:58:32 -08:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations= 50 --verbose tests/python/gpu
2020-05-04 16:44:27 -07:00
pytest -m 'serial' --durations= 50 --verbose tests/python/gpu
2020-11-19 15:58:32 -08:00
pytest --durations= 50 --verbose tests/python/gpu/test_amp_init.py
2019-05-23 12:48:44 +02:00
2020-04-30 16:02:55 -07:00
# TODO(szha): fix and reenable the hanging issue. tracked in #18098
# integrationtest_ubuntu_gpu_dist_kvstore
2020-06-11 09:17:44 -07:00
# TODO(eric-haibin-lin): fix and reenable
# integrationtest_ubuntu_gpu_byteps
2019-05-23 12:48:44 +02:00
fi
2020-04-22 23:53:12 -07:00
if [ [ ${ mxnet_variant } = *mkl ] ] ; then
2021-10-13 22:48:10 +02:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -n 4 --durations= 50 --verbose tests/python/dnnl
2019-05-23 12:48:44 +02:00
fi
}
2021-07-16 14:30:24 -07:00
unittest_ubuntu_python3_cpu_onnx( ) {
set -ex
export PYTHONPATH = ./python/
export MXNET_SUBGRAPH_VERBOSE = 0
export DMLC_LOG_STACK_TRACE_DEPTH = 10
pytest --cov-report xml:onnx_unittest.xml --verbose tests/python/onnx/test_operators.py
pytest --cov-report xml:onnx_unittest.xml --cov-append --verbose tests/python/onnx/test_models.py
}
2018-03-09 23:33:10 +01:00
unittest_ubuntu_python3_cpu( ) {
set -ex
2018-04-01 16:19:45 -07:00
export PYTHONPATH = ./python/
2021-03-16 20:16:53 +01:00
export MXNET_ONEDNN_DEBUG = 0 # Ignored if not present
2018-03-09 23:33:10 +01:00
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE = 0
2019-10-24 04:47:01 -05:00
export MXNET_SUBGRAPH_VERBOSE = 0
2019-05-25 07:36:30 +09:00
export MXNET_ENABLE_CYTHON = 0
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations= 50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
2020-05-16 19:04:44 -07:00
MXNET_ENGINE_TYPE = NaiveEngine \
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations= 50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
2020-05-04 16:44:27 -07:00
pytest -m 'serial' --durations= 50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
2018-03-09 23:33:10 +01:00
}
2021-03-24 15:15:32 +01:00
unittest_ubuntu_python3_cpu_onednn( ) {
2018-04-24 10:48:01 -07:00
set -ex
2018-06-14 12:58:33 +08:00
export PYTHONPATH = ./python/
2021-03-16 20:16:53 +01:00
export MXNET_ONEDNN_DEBUG = 0 # Ignored if not present
2018-04-24 10:48:01 -07:00
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE = 0
2019-10-24 04:47:01 -05:00
export MXNET_SUBGRAPH_VERBOSE = 0
2019-05-25 07:36:30 +09:00
export MXNET_ENABLE_CYTHON = 0
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'not test_operator' -n 4 --durations= 50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
MXNET_ENGINE_TYPE = NaiveEngine \
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations= 50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
pytest -m 'serial' --durations= 50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
2021-10-13 22:48:10 +02:00
pytest --durations= 50 --cov-report xml:tests_mkl.xml --verbose tests/python/dnnl
2018-04-24 10:48:01 -07:00
}
2021-09-09 18:17:58 -07:00
unittest_array_api_standardization( ) {
set -ex
python3 -m pip install -e /work/mxnet/python --user
cd ..
git clone https://github.com/data-apis/array-api-tests.git
pushd /work/array-api-tests
2021-10-04 08:49:55 -07:00
git checkout c1dba80a196a03f880d2e0a998a272fb3867b720
2021-09-09 18:17:58 -07:00
export ARRAY_API_TESTS_MODULE = mxnet.numpy pytest
2022-03-17 16:15:53 -07:00
export MXNET_ENABLE_CYTHON = 1
2021-09-09 18:17:58 -07:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2021-11-20 18:58:56 -08:00
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose array_api_tests/test_creation_functions.py
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose array_api_tests/test_indexing.py
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose array_api_tests/test_elementwise_functions.py
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose array_api_tests/test_constants.py
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose array_api_tests/test_broadcasting.py
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose \
2021-09-09 18:17:58 -07:00
array_api_tests/test_type_promotion.py::test_elementwise_function_two_arg_bool_type_promotion
2021-11-20 18:58:56 -08:00
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose \
array_api_tests/test_type_promotion.py::test_elementwise_function_two_arg_promoted_type_promotion
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose \
array_api_tests/test_type_promotion.py::test_elementwise_function_one_arg_bool
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose \
array_api_tests/test_type_promotion.py::test_elementwise_function_one_arg_type_promotion
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose \
array_api_tests/test_type_promotion.py::test_operator_one_arg_type_promotion
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose \
array_api_tests/test_type_promotion.py::test_operator_two_arg_bool_promotion
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose \
array_api_tests/test_type_promotion.py::test_operator_two_arg_promoted_promotion
python3 -m pytest --reruns 3 --durations= 50 --cov-report xml:tests_api.xml --verbose \
array_api_tests/test_type_promotion.py::test_operator_inplace_two_arg_promoted_promotion
2021-09-09 18:17:58 -07:00
popd
}
2018-03-09 23:33:10 +01:00
unittest_ubuntu_python3_gpu( ) {
set -ex
2018-04-01 16:19:45 -07:00
export PYTHONPATH = ./python/
2021-03-16 20:16:53 +01:00
export MXNET_ONEDNN_DEBUG = 0 # Ignored if not present
2018-03-09 23:33:10 +01:00
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE = 0
2019-10-24 04:47:01 -05:00
export MXNET_SUBGRAPH_VERBOSE = 0
2019-04-03 02:49:36 +02:00
export CUDNN_VERSION = ${ CUDNN_VERSION :- 7 .0.3 }
2019-05-25 07:36:30 +09:00
export MXNET_ENABLE_CYTHON = 0
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2020-05-16 19:04:44 -07:00
MXNET_GPU_MEM_POOL_TYPE = Unpooled \
2020-11-19 15:58:32 -08:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations= 50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
2020-05-16 19:04:44 -07:00
MXNET_GPU_MEM_POOL_TYPE = Unpooled \
MXNET_ENGINE_TYPE = NaiveEngine \
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
2020-05-04 16:44:27 -07:00
pytest -m 'serial' --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
2020-11-19 15:58:32 -08:00
pytest --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu/test_amp_init.py
2019-05-25 07:36:30 +09:00
}
unittest_ubuntu_python3_gpu_cython( ) {
set -ex
export PYTHONPATH = ./python/
2021-03-16 20:16:53 +01:00
export MXNET_ONEDNN_DEBUG = 1 # Ignored if not present
2019-05-25 07:36:30 +09:00
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE = 0
2019-10-24 04:47:01 -05:00
export MXNET_SUBGRAPH_VERBOSE = 0
2019-05-25 07:36:30 +09:00
export CUDNN_VERSION = ${ CUDNN_VERSION :- 7 .0.3 }
export MXNET_ENABLE_CYTHON = 1
export MXNET_ENFORCE_CYTHON = 1
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2020-02-15 01:31:53 +00:00
check_cython
2020-05-16 19:04:44 -07:00
MXNET_GPU_MEM_POOL_TYPE = Unpooled \
2020-11-19 15:58:32 -08:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations= 50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
2020-05-16 19:04:44 -07:00
MXNET_GPU_MEM_POOL_TYPE = Unpooled \
MXNET_ENGINE_TYPE = NaiveEngine \
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
2020-05-04 16:44:27 -07:00
pytest -m 'serial' --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
2020-11-19 15:58:32 -08:00
pytest --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu/test_amp_init.py
2018-03-09 23:33:10 +01:00
}
2018-07-12 16:40:24 -07:00
unittest_ubuntu_python3_gpu_nocudnn( ) {
set -ex
export PYTHONPATH = ./python/
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE = 0
2019-10-24 04:47:01 -05:00
export MXNET_SUBGRAPH_VERBOSE = 0
2018-07-12 16:40:24 -07:00
export CUDNN_OFF_TEST_ONLY = true
2019-05-25 07:36:30 +09:00
export MXNET_ENABLE_CYTHON = 0
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2020-05-16 19:04:44 -07:00
MXNET_GPU_MEM_POOL_TYPE = Unpooled \
2020-11-19 15:58:32 -08:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations= 50 --cov-report xml:tests_gpu.xml --verbose tests/python/gpu
2020-05-16 19:04:44 -07:00
MXNET_GPU_MEM_POOL_TYPE = Unpooled \
MXNET_ENGINE_TYPE = NaiveEngine \
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
2020-05-04 16:44:27 -07:00
pytest -m 'serial' --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
2020-11-19 15:58:32 -08:00
pytest --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu/test_amp_init.py
2018-07-12 16:40:24 -07:00
}
2019-01-08 14:01:44 +01:00
unittest_cpp( ) {
2018-03-09 23:33:10 +01:00
set -ex
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2018-03-09 23:33:10 +01:00
build/tests/mxnet_unit_tests
}
unittest_centos7_cpu( ) {
set -ex
2021-10-19 22:21:04 -07:00
source /opt/rh/rh-python38/enable
2018-03-09 23:33:10 +01:00
cd /work/mxnet
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) python -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations= 50 --cov-report xml:tests_unittest.xml --verbose tests/python/unittest
2020-05-16 19:04:44 -07:00
MXNET_ENGINE_TYPE = NaiveEngine \
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) python -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations= 50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
2020-05-04 16:44:27 -07:00
python -m pytest -m 'serial' --durations= 50 --cov-report xml:tests_unittest.xml --cov-append --verbose tests/python/unittest
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) python -m pytest -n 4 --durations= 50 --cov-report xml:tests_train.xml --verbose tests/python/train
2018-03-09 23:33:10 +01:00
}
unittest_centos7_gpu( ) {
set -ex
2021-10-19 22:21:04 -07:00
source /opt/rh/rh-python38/enable
2018-03-09 23:33:10 +01:00
cd /work/mxnet
2019-04-03 02:49:36 +02:00
export CUDNN_VERSION = ${ CUDNN_VERSION :- 7 .0.3 }
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2020-05-16 19:04:44 -07:00
MXNET_GPU_MEM_POOL_TYPE = Unpooled \
2020-11-19 15:58:32 -08:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'not test_operator and not test_amp_init.py' -n 4 --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
2020-05-16 19:04:44 -07:00
MXNET_GPU_MEM_POOL_TYPE = Unpooled \
MXNET_ENGINE_TYPE = NaiveEngine \
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -m 'not serial' -k 'test_operator' -n 4 --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
2020-05-04 16:44:27 -07:00
pytest -m 'serial' --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu
2020-11-19 15:58:32 -08:00
pytest --durations= 50 --cov-report xml:tests_gpu.xml --cov-append --verbose tests/python/gpu/test_amp_init.py
2018-03-09 23:33:10 +01:00
}
2021-05-24 13:44:39 -07:00
integrationtest_ubuntu_cpp_package_gpu( ) {
set -ex
export DMLC_LOG_STACK_TRACE_DEPTH = 10
cpp-package/tests/ci_test.sh
}
2021-11-29 06:22:04 -08:00
test_python3_data_interchange_gpu( ) {
set -ex
python3 -m pip install torch = = 1.10.0+cu113 torchvision = = 0.11.1+cu113 torchaudio = = 0.10.0+cu113 \
-f https://download.pytorch.org/whl/cu113/torch_stable.html
MXNET_ENGINE_TYPE = ThreadedEngineAsync \
python3 -m pytest --durations= 50 tests/python/array-api/test_data_interchange.py
}
2018-04-01 16:19:45 -07:00
integrationtest_ubuntu_cpu_onnx( ) {
[MXNET-34] Onnx Module to import onnx models into mxnet (#9963)
* Onnx Module to import onnx models into mxnet
* Change package name to onnx from serde.
* Remove onnx install time dependency
* Remove Renamer class
* Add apache license to files.
* Refactor test files to tests/python folder.
* Removed export folder.
* Refactor Attribute COnverter logic
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Changing the translation and utils file.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* - Fixed Pylint issues
- Added Sigmoid operator.
- Add onnx, protobuf as CI pipeline dependencies.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Add UTs for reduce ops.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* pylint - newline, whitespace.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Added operators:
- AvgPool
- ArgMax
- ArgMin
- Abs
Minor changes in logic for import_onnx
* Added operators:
- Ceil
- Cast
- Constant
* - Added Pad operator support.
- Minor changes for comments
* RandomUniform,Normal,Sub,Mul,Div,Tanh,Relu,Reciprocal,Sqrt operators
added.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* lint fix
* Add protobuf-compile to CI bash script. Add MatMul and Pow operator.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Max,Min,Sum,Reduce operators.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* BatchNorm,SpatialBN, Split
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Slice,Transpose and Squeeze Operators.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Onnx tests in CI integration tests.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Addressing Marco's comments
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Floor, LeakyRelu, Elu, PRelu, Softmax, Exp, Log operator.
* Added operators:
- Convolution
- Deconvolution
Refactored convert_operator
* lint fix
* Rebase fix
* Added Maxpool operator
* Adding FullyConnected operator
* Adding operator- GlobalPooling - max and avg
Minor lint fixes.
* Adding operator - Gemm
* Change test Path, LRN and Dropout operator.
* Add asserts for the super_res example.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Fixing conv test failures.
Removed redundant code
* Update Jenkins job.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Nits: Removing commented out code
* Rebase after Docker PR
* Fetch test files by version number. Verify the high resolution example.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Fix method arguments for Python3.5+
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Remove logging configuration from test files.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Verify result image in example by hash
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Remove fetching test files by ETag.
Will add it as a separate PR as per review comments.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
2018-03-14 13:42:59 -07:00
set -ex
export PYTHONPATH = ./python/
2020-11-24 13:52:02 -08:00
export MXNET_SUBGRAPH_VERBOSE = 0
2021-02-15 15:04:06 -08:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2020-04-22 23:53:12 -07:00
python3 tests/python/unittest/onnx/backend_test.py
2021-02-15 15:04:06 -08:00
#OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/mxnet_export_test.py
#OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_models.py
#OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_node.py
#OMP_NUM_THREADS=$(expr $(nproc) / 4) pytest -n 4 tests/python/unittest/onnx/test_onnxruntime.py
[MXNET-34] Onnx Module to import onnx models into mxnet (#9963)
* Onnx Module to import onnx models into mxnet
* Change package name to onnx from serde.
* Remove onnx install time dependency
* Remove Renamer class
* Add apache license to files.
* Refactor test files to tests/python folder.
* Removed export folder.
* Refactor Attribute COnverter logic
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Changing the translation and utils file.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* - Fixed Pylint issues
- Added Sigmoid operator.
- Add onnx, protobuf as CI pipeline dependencies.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Add UTs for reduce ops.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* pylint - newline, whitespace.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Added operators:
- AvgPool
- ArgMax
- ArgMin
- Abs
Minor changes in logic for import_onnx
* Added operators:
- Ceil
- Cast
- Constant
* - Added Pad operator support.
- Minor changes for comments
* RandomUniform,Normal,Sub,Mul,Div,Tanh,Relu,Reciprocal,Sqrt operators
added.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* lint fix
* Add protobuf-compile to CI bash script. Add MatMul and Pow operator.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Max,Min,Sum,Reduce operators.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* BatchNorm,SpatialBN, Split
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Slice,Transpose and Squeeze Operators.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Onnx tests in CI integration tests.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Addressing Marco's comments
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Floor, LeakyRelu, Elu, PRelu, Softmax, Exp, Log operator.
* Added operators:
- Convolution
- Deconvolution
Refactored convert_operator
* lint fix
* Rebase fix
* Added Maxpool operator
* Adding FullyConnected operator
* Adding operator- GlobalPooling - max and avg
Minor lint fixes.
* Adding operator - Gemm
* Change test Path, LRN and Dropout operator.
* Add asserts for the super_res example.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Fixing conv test failures.
Removed redundant code
* Update Jenkins job.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Nits: Removing commented out code
* Rebase after Docker PR
* Fetch test files by version number. Verify the high resolution example.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Fix method arguments for Python3.5+
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Remove logging configuration from test files.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Verify result image in example by hash
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
* Remove fetching test files by ETag.
Will add it as a separate PR as per review comments.
Signed-off-by: Acharya <aanirud@8c85904b0bf4.ant.amazon.com>
2018-03-14 13:42:59 -07:00
}
2018-07-13 16:54:14 -07:00
integrationtest_ubuntu_cpu_dist_kvstore( ) {
set -ex
2019-05-23 14:52:58 +02:00
pushd .
2018-07-13 16:54:14 -07:00
export PYTHONPATH = ./python/
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE = 0
2019-10-24 04:47:01 -05:00
export MXNET_SUBGRAPH_VERBOSE = 0
2018-07-13 16:54:14 -07:00
export MXNET_USE_OPERATOR_TUNING = 0
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2018-07-13 16:54:14 -07:00
cd tests/nightly/
2020-02-16 11:38:29 -08:00
python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type= gluon_step_cpu
python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type= gluon_sparse_step_cpu
python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type= invalid_cpu
python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type= gluon_type_cpu
python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py
python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --no-multiprecision
2021-03-11 06:45:53 +08:00
python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type= compressed_cpu_1bit
python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type= compressed_cpu_1bit --no-multiprecision
python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type= compressed_cpu_2bit
python3 ../../tools/launch.py -n 7 --launcher local python3 dist_sync_kvstore.py --type= compressed_cpu_2bit --no-multiprecision
2020-02-16 11:38:29 -08:00
python3 ../../tools/launch.py -n 3 --launcher local python3 test_server_profiling.py
2019-05-23 14:52:58 +02:00
popd
2018-07-13 16:54:14 -07:00
}
2018-04-11 10:20:56 -07:00
integrationtest_ubuntu_gpu_dist_kvstore( ) {
set -ex
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2019-05-23 14:52:58 +02:00
pushd .
2020-04-14 13:41:40 -07:00
cd /work/mxnet/python
pip3 install -e .
pip3 install --no-cache-dir horovod
cd /work/mxnet/tests/nightly
2020-04-06 12:49:41 -07:00
./test_distributed_training-gpu.sh
2019-05-23 14:52:58 +02:00
popd
2018-04-11 10:20:56 -07:00
}
2018-03-09 23:33:10 +01:00
2020-06-04 14:20:52 -07:00
integrationtest_ubuntu_gpu_byteps( ) {
set -ex
pushd .
export PYTHONPATH = $PWD /python/
export BYTEPS_WITHOUT_PYTORCH = 1
export BYTEPS_WITHOUT_TENSORFLOW = 1
pip3 install byteps = = 0.2.3 --user
git clone -b v0.2.3 https://github.com/bytedance/byteps ~/byteps
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE = 0
export MXNET_SUBGRAPH_VERBOSE = 0
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2020-06-04 14:20:52 -07:00
cd tests/nightly/
export NVIDIA_VISIBLE_DEVICES = 0
export DMLC_WORKER_ID = 0 # your worker id
export DMLC_NUM_WORKER = 1 # one worker
export DMLC_ROLE = worker
# the following value does not matter for non-distributed jobs
export DMLC_NUM_SERVER = 1
export DMLC_PS_ROOT_URI = 0.0.0.127
export DMLC_PS_ROOT_PORT = 1234
python3 ~/byteps/launcher/launch.py python3 dist_device_sync_kvstore_byteps.py
popd
}
2018-03-09 23:33:10 +01:00
test_ubuntu_cpu_python3( ) {
set -ex
pushd .
export MXNET_LIBRARY_PATH = /work/build/libmxnet.so
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2018-03-09 23:33:10 +01:00
VENV = mxnet_py3_venv
virtualenv -p ` which python3` $VENV
source $VENV /bin/activate
cd /work/mxnet/python
pip3 install -e .
cd /work/mxnet
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) python3 -m pytest -m 'not serial' -k 'not test_operator' -n 4 --durations= 50 --verbose tests/python/unittest
2020-05-16 19:04:44 -07:00
MXNET_ENGINE_TYPE = NaiveEngine \
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) python3 -m pytest -m 'not serial' -k 'test_operator' -n 4 --durations= 50 --verbose tests/python/unittest
2020-05-04 16:44:27 -07:00
python3 -m pytest -m 'serial' --durations= 50 --verbose tests/python/unittest
2018-03-09 23:33:10 +01:00
popd
}
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
# QEMU based ARM tests
unittest_ubuntu_python3_arm( ) {
set -ex
export PYTHONPATH = ./python/
2021-03-16 20:16:53 +01:00
export MXNET_ONEDNN_DEBUG = 0 # Ignored if not present
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE = 0
export MXNET_SUBGRAPH_VERBOSE = 0
export MXNET_ENABLE_CYTHON = 0
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2020-05-04 16:44:27 -07:00
python3 -m pytest -n 2 --verbose tests/python/unittest/test_engine.py
Switch to C++17 and modernize toolchain + CI (#17984)
As per #17968, require C++17 compatible compiler. For cuda code, use C++14 mode introduced in Cuda 9. C++17 support for Cuda will be available in Cuda 11.
Switching to C++17 requires modernizing the toolchain, which exposed a number of technical debt issues in the codebase. All blocking issues are fixed as part of this PR. See the full list below.
This PR contains the following specific changes:
Switch CI pipeline to use gcc7 on Ubuntu and CentOS
Switch CD pipeline to CentOS 7 with https://www.softwarecollections.org/en/scls/rhscl/devtoolset-7/ This enables us to build with gcc7 C++17 compiler while keeping a relatively old glibc requirement for distribution.
Simplify ARM Edge builds
Switch to standard Ubuntu / Debian cross-compilation toolchain for ARMv7, ARMv8
Switch to https://toolchains.bootlin.com/ toolchain for ARMv6 (the Debian ARMv6 toolchain is for ARMv4 + ARMv5 + ARMv6, but we wish to only target ARMv6 and make use of ARMv6 features)
Remove reliance on dockcross for cross compilation.
Simplify Jetson build
Use standard Ubuntu / Debian cross-compilation toolchain for ARMv8
Upgrade to Cuda 10 and Jetpack 4.3
Simplify build setup
Simplify QEMU ARM virtualization test setup on CI
Remove complex "Virtual Machine in Docker" logic and run a QEMU based Docker container instead based on arm32v7/ubuntu
Fix out of bounds vector accesses in
SoftmaxGradOpType
MKLDNNFCBackward
Fix use of non-standard rand_r function (which is not available on anymore on newer Android toolchains and shouldn't be use in any case).
Fix reproducibility of RNN with Dropout
Fix reproducibility of DGL Graph Sampling Operators
Update tests for Android Edge build to NDK19. The previously used standalone toolchain is obsolete.
Those Dockerfiles that required refactoring as part of the effort were refactored based on the following consideration
Maximize the use of system dependencies provided by the distribution instead of manually installing dependencies from source or from third party vendors. This reduces the complexity of the installation process and essentially pins the dependency versions, increasing CI stability. Further, Dockerfile build speed is improved. To facilitate this, use recent distribution versions. We still ensure backwards compatibility via CentOS7 based build and test stages
Minimize the number of layers in the Dockerfile. Don't have 5 different script files executed, each calling apt-get update and install, but just execute once. Speeds up the build and reduces image size. Keep each Dockerfile simple and tailored to a purpose, instead of running 20 scripts to install dependencies for every thinkable scenario, which is unmaintainable.
Some more small changes:
Remove outdated references to Cuda 7 and Cuda 8 in various files.
Remove C++03 support in mshadow
Disable broken tests
NumpyBooleanAssignForwardCPU #17990
test_init.test_rsp_const_init #17988
quantized_elemwise_mul #18034
List of squashed commits
* cpp standard
* Remove leftover files of Cuda 7 and Cuda 8 support
* thrust 1.9.8 for clang10
* compiler warnings
* Disable broken test_init.test_rsp_const_init
* Disable tests invoking NumpyBooleanAssignForwardCPU
* Fix out of bounds access in SoftmaxGradOpType
* Use CentOS 7 for staticbuilds
CentOS 7 fullfills the requirements for PEP 599 manylinux-2014 and provides a
C++17 toolchain.
* Fix MKLDNNFCBackward
* Update edge toolchain
* Support platforms without rand_r
* Cleanup random.h
* Greatly simplify qemu setup
* Remove unused functions in Jenkins_steps.groovy
* Skip quantized_elemwise_mul due QuantizedElemwiseMulOpShape bug
* Fix R package installation
https://github.com/apache/incubator-mxnet/issues/18042
* Fix centos ccache
* Fix GPU Makefile staticbuild on CentOS7
* CentOS7 NCCL
* CentOS7 staticbuild fix link with libculibos
2020-04-14 10:29:29 -07:00
}
2018-06-20 19:17:45 -07:00
# Functions that run the nightly Tests:
#Runs Apache RAT Check on MXNet Source for License Headers
2020-07-23 18:09:10 +00:00
test_rat_check( ) {
2018-06-29 19:07:07 +02:00
set -e
2020-12-23 15:18:24 -05:00
set -o pipefail
2018-06-29 19:07:07 +02:00
pushd .
2018-07-19 17:18:04 +02:00
2020-07-23 18:09:10 +00:00
cd /usr/local/src/apache-rat-0.13
2018-06-29 19:07:07 +02:00
# Use shell number 5 to duplicate the log output. It get sprinted and stored in $OUTPUT at the same time https://stackoverflow.com/a/12451419
exec 5>& 1
2021-01-15 17:49:18 -05:00
OUTPUT = $( java -jar apache-rat-0.13.jar -E /work/mxnet/rat-excludes -d /work/mxnet| tee >( cat - >& 5) )
2018-06-29 19:07:07 +02:00
ERROR_MESSAGE = "Printing headers for text files without a valid license header"
echo "-------Process The Output-------"
if [ [ $OUTPUT = ~ $ERROR_MESSAGE ] ] ; then
echo "ERROR: RAT Check detected files with unknown licenses. Please fix and run test again!" ;
exit 1
else
echo "SUCCESS: There are no files with an Unknown License." ;
fi
popd
2018-06-20 19:17:45 -07:00
}
#Single Node KVStore Test
nightly_test_KVStore_singleNode( ) {
set -ex
export PYTHONPATH = ./python/
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2020-02-03 09:58:01 -08:00
tests/nightly/test_kvstore.py
2018-06-20 19:17:45 -07:00
}
2019-04-23 14:47:10 -07:00
#Test Large Tensor Size
nightly_test_large_tensor( ) {
set -ex
export PYTHONPATH = ./python/
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2021-02-03 12:00:09 -08:00
pytest -s --exitfirst --verbose --timeout= 7200 tests/nightly/test_np_large_array.py
2020-01-21 11:02:55 -08:00
}
2018-07-31 02:50:13 -07:00
#Tests Model backwards compatibility on MXNet
nightly_model_backwards_compat_test( ) {
set -ex
export PYTHONPATH = /work/mxnet/python/
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2018-07-31 02:50:13 -07:00
./tests/nightly/model_backwards_compatibility_check/model_backward_compat_checker.sh
}
#Backfills S3 bucket with models trained on earlier versions of mxnet
nightly_model_backwards_compat_train( ) {
set -ex
export PYTHONPATH = ./python/
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2018-07-31 02:50:13 -07:00
./tests/nightly/model_backwards_compatibility_check/train_mxnet_legacy_models.sh
}
2018-11-07 15:16:40 -08:00
nightly_tutorial_test_ubuntu_python3_gpu( ) {
set -ex
cd /work/mxnet/docs
2018-12-18 09:02:02 -08:00
export BUILD_VER = tutorial
2018-11-07 15:16:40 -08:00
export MXNET_DOCS_BUILD_MXNET = 0
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2018-11-07 15:16:40 -08:00
make html
export MXNET_STORAGE_FALLBACK_LOG_VERBOSE = 0
2019-10-24 04:47:01 -05:00
export MXNET_SUBGRAPH_VERBOSE = 0
2018-11-07 15:16:40 -08:00
export PYTHONPATH = /work/mxnet/python/
export MXNET_TUTORIAL_TEST_KERNEL = python3
cd /work/mxnet/tests/tutorials
2020-04-22 23:53:12 -07:00
pytest --durations= 50 --cov-report xml:tests_tutorials.xml --capture= no test_tutorials.py
2018-11-07 15:16:40 -08:00
}
2019-04-23 20:33:03 -07:00
nightly_estimator( ) {
2019-04-03 15:12:56 -07:00
set -ex
2020-12-08 11:13:09 -05:00
export DMLC_LOG_STACK_TRACE_DEPTH = 100
2019-04-03 15:12:56 -07:00
cd /work/mxnet/tests/nightly/estimator
export PYTHONPATH = /work/mxnet/python/
2020-04-22 23:53:12 -07:00
pytest test_estimator_cnn.py
pytest test_sentiment_rnn.py
2019-04-03 14:28:08 -07:00
}
2019-09-19 22:17:04 -07:00
# For testing PRs
2018-03-09 23:33:10 +01:00
deploy_docs( ) {
set -ex
pushd .
2018-04-01 16:19:45 -07:00
2019-07-11 14:23:20 -07:00
export CC = "ccache gcc"
export CXX = "ccache g++"
2018-09-29 02:48:43 +08:00
2019-09-19 22:17:04 -07:00
build_python_docs
popd
}
build_docs_setup( ) {
build_folder = "docs/_build"
mxnetlib_folder = "/work/mxnet/lib"
mkdir -p $build_folder
mkdir -p $mxnetlib_folder
}
build_jekyll_docs( ) {
set -ex
pushd .
build_docs_setup
pushd docs/static_site
make clean
make html
2019-07-11 14:23:20 -07:00
popd
2019-09-19 22:17:04 -07:00
GZIP = -9 tar zcvf jekyll-artifacts.tgz -C docs/static_site/build html
mv jekyll-artifacts.tgz docs/_build/
popd
}
build_python_docs( ) {
2020-09-26 22:28:29 -07:00
set -ex
pushd .
2019-09-19 22:17:04 -07:00
2020-09-26 22:28:29 -07:00
build_docs_setup
2019-09-19 22:17:04 -07:00
2020-09-26 22:28:29 -07:00
pushd docs/python_docs
python3 -m pip install -r requirements
python3 -m pip install themes/mx-theme
python3 -m pip install -e /work/mxnet/python --user
2020-08-18 16:57:35 +00:00
2020-09-26 22:28:29 -07:00
export PATH = /home/jenkins_slave/.local/bin:$PATH
2019-09-19 22:17:04 -07:00
2020-09-26 22:28:29 -07:00
pushd python
2021-08-17 09:26:14 -07:00
cp tutorials/getting-started/crash-course/prepare_dataset.py .
2020-09-26 22:28:29 -07:00
make clean
2021-08-17 09:26:14 -07:00
make html EVAL = 1
2019-09-19 22:17:04 -07:00
2020-09-26 22:28:29 -07:00
GZIP = -9 tar zcvf python-artifacts.tgz -C build/_build/html .
popd
2019-09-19 22:17:04 -07:00
2020-09-26 22:28:29 -07:00
mv python/python-artifacts.tgz /work/mxnet/docs/_build/
popd
2019-09-19 22:17:04 -07:00
2020-09-26 22:28:29 -07:00
popd
2019-09-19 22:17:04 -07:00
}
build_c_docs( ) {
set -ex
pushd .
build_docs_setup
doc_path = "docs/cpp_docs"
pushd $doc_path
make clean
make html
doc_artifact = "c-artifacts.tgz"
GZIP = -9 tar zcvf $doc_artifact -C build/html/html .
popd
mv $doc_path /$doc_artifact docs/_build/
popd
}
build_docs( ) {
pushd docs/_build
tar -xzf jekyll-artifacts.tgz
2020-09-26 22:28:29 -07:00
python_doc_folder = 'html/api/python/docs'
2021-05-24 13:44:39 -07:00
api_folder = 'html/api'
2020-09-26 22:28:29 -07:00
2019-09-19 22:17:04 -07:00
# Python has it's own landing page/site so we don't put it in /docs/api
2020-09-26 22:28:29 -07:00
mkdir -p $python_doc_folder && tar -xzf python-artifacts.tgz --directory $python_doc_folder
2021-05-24 13:44:39 -07:00
mkdir -p $api_folder /cpp/docs/api && tar -xzf c-artifacts.tgz --directory $api_folder /cpp/docs/api
2020-09-26 22:28:29 -07:00
2021-09-13 22:29:34 -07:00
# check if .asf.yaml file exists
if [ ! -f "html/.asf.yaml" ] ; then
echo "html/.asf.yaml file does not exist. Exiting 1"
exit 1
fi
# check if .htaccess file exists
2020-09-23 22:08:32 -07:00
if [ ! -f "html/.htaccess" ] ; then
echo "html/.htaccess file does not exist. Exiting 1"
exit 1
fi
# get the version
version = $( grep "RewriteRule" html/.htaccess | grep -E "versions\/[0-9]" | sed -nre 's/^[^0-9]*(([0-9]+\.)*[0-9]+).*/\1/p' )
# count how many versions are found
lines = $( echo " $version " | wc -l)
# check if multiple versions are found
if [ " $lines " != "1" ] ; then
echo " multiple versions detected: $lines . Exiting 1 "
exit 1
fi
# check if no version is found
if [ " $version " = = "" ] ; then
echo "no version found. Exiting 1"
exit 1
fi
# print the one and only default mxnet version
2020-09-26 22:28:29 -07:00
echo " detected version is $version "
2020-09-23 22:08:32 -07:00
# check if the artifacts for this version exist
if [ -d " html/versions/ $version /api " ] ; then
2020-09-26 22:28:29 -07:00
echo " html/versions/ $version /api directory exists "
2020-09-23 22:08:32 -07:00
else
2020-09-26 22:28:29 -07:00
echo " html/versions/ $version /api directory does not exist! Exiting 1 "
2020-09-23 22:08:32 -07:00
exit 1
2020-09-26 22:28:29 -07:00
fi
# copy the full site for this version to versions folder
2020-09-27 15:48:55 -07:00
mkdir -p html/versions/master
2022-06-28 08:53:03 -07:00
for f in 404.html api assets community ecosystem features trusted_by feed.xml get_started index.html; do
2020-09-27 15:48:55 -07:00
cp -r html/$f html/versions/master/
2020-09-26 22:28:29 -07:00
done
# clean up temp files
find html -type f -name '.DS_Store' -delete
# archive artifact
2019-09-19 22:17:04 -07:00
GZIP = -9 tar -zcvf full_website.tgz -C html .
popd
}
2019-10-10 16:33:34 -07:00
build_docs_beta( ) {
2019-09-19 22:17:04 -07:00
pushd docs/_build
tar -xzf jekyll-artifacts.tgz
2020-09-26 22:28:29 -07:00
python_doc_folder = " html/versions/ $BRANCH /api/python/docs "
2021-05-24 13:44:39 -07:00
cpp_doc_folder = " html/versions/ $BRANCH /api/cpp/docs "
2020-09-26 22:28:29 -07:00
mkdir -p $python_doc_folder && tar -xzf python-artifacts.tgz --directory $python_doc_folder
2021-05-24 13:44:39 -07:00
mkdir -p $cpp_doc_folder && tar -xzf c-artifacts.tgz --directory $cpp_doc_folder
2019-10-10 16:33:34 -07:00
GZIP = -9 tar -zcvf beta_website.tgz -C html .
2019-09-19 22:17:04 -07:00
popd
}
2021-03-11 15:36:05 -08:00
push_docs( ) {
folder_name = $1
set -ex
export PATH = ~/.local/bin:$PATH
pushd docs/_build
tar -xzf full_website.tgz --strip-components 1
# check if folder_name already exists in versions
pushd versions
if [ -d " $folder_name " ] ; then
echo " Folder $folder_name already exists in versions. Please double check the FOLDER_NAME variable in Jenkens pipeline "
exit 1
fi
mv master $folder_name
popd
zip -r9 versions.zip versions/.
# Upload versions folder
aws s3 cp versions.zip s3://mxnet-website-static-artifacts --acl public-read
# Backup versions folder with the latest version name
backup_file = " versions_backup_upto_ $folder_name .zip "
aws s3 cp s3://mxnet-website-static-artifacts/versions.zip s3://mxnet-website-static-artifacts/$backup_file --acl public-read
popd
}
2019-09-19 22:17:04 -07:00
create_repo( ) {
repo_folder = $1
mxnet_url = $2
git clone $mxnet_url $repo_folder --recursive
echo "Adding MXNet upstream repo..."
cd $repo_folder
2023-01-04 04:09:23 -08:00
git remote add upstream https://github.com/apache/mxnet
2019-09-19 22:17:04 -07:00
cd ..
}
refresh_branches( ) {
repo_folder = $1
cd $repo_folder
git fetch
git fetch upstream
cd ..
}
checkout( ) {
repo_folder = $1
cd $repo_folder
# Overriding configs later will cause a conflict here, so stashing...
git stash
# Fails to checkout if not available locally, so try upstream
git checkout " $repo_folder " || git branch $repo_folder " upstream/ $repo_folder " && git checkout " $repo_folder " || exit 1
if [ $tag = = 'master' ] ; then
git pull
# master gets warnings as errors for Sphinx builds
OPTS = "-W"
else
OPTS =
fi
git submodule update --init --recursive
cd ..
2018-09-29 02:48:43 +08:00
}
2019-05-23 12:48:44 +02:00
build_static_libmxnet( ) {
set -ex
pushd .
2020-10-05 11:34:20 -07:00
source /opt/rh/devtoolset-8/enable
2021-10-19 22:21:04 -07:00
source /opt/rh/rh-python38/enable
2020-09-19 18:42:34 -07:00
# Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.
export CXXFLAGS = "-fabi-version=11 -fabi-compat-version=7"
2019-05-23 12:48:44 +02:00
local mxnet_variant = ${ 1 : ? "This function requires a python command as the first argument" }
2019-12-09 20:23:47 -06:00
source tools/staticbuild/build.sh ${ mxnet_variant }
2019-05-23 12:48:44 +02:00
popd
}
2020-06-16 16:52:46 -07:00
# Tests CD PyPI packaging in CI
ci_package_pypi( ) {
set -ex
2021-11-22 07:41:25 +01:00
# copies oneDNN header files to 3rdparty/onednn/include/oneapi/dnnl/ as in CD
2021-03-15 17:32:37 +01:00
mkdir -p 3rdparty/onednn/include/oneapi/dnnl
cp include/onednn/oneapi/dnnl/dnnl_version.h 3rdparty/onednn/include/oneapi/dnnl/.
cp include/onednn/oneapi/dnnl/dnnl_config.h 3rdparty/onednn/include/oneapi/dnnl/.
2020-06-16 16:52:46 -07:00
local mxnet_variant = ${ 1 : ? "This function requires a python command as the first argument" }
cd_package_pypi ${ mxnet_variant }
cd_integration_test_pypi
}
2019-10-11 00:20:40 +02:00
# Packages libmxnet into wheel file
cd_package_pypi( ) {
set -ex
pushd .
2020-10-05 11:34:20 -07:00
source /opt/rh/devtoolset-8/enable
2021-10-19 22:21:04 -07:00
source /opt/rh/rh-python38/enable
2020-09-19 18:42:34 -07:00
# Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.
export CXXFLAGS = "-fabi-version=11 -fabi-compat-version=7"
2019-10-11 00:20:40 +02:00
local mxnet_variant = ${ 1 : ? "This function requires a python command as the first argument" }
./cd/python/pypi/pypi_package.sh ${ mxnet_variant }
popd
}
2019-10-24 04:47:01 -05:00
# Sanity checks wheel file
2019-10-11 00:20:40 +02:00
cd_integration_test_pypi( ) {
set -ex
2021-10-19 22:21:04 -07:00
source /opt/rh/rh-python38/enable
2020-04-21 17:34:56 -07:00
2019-10-11 00:20:40 +02:00
# install mxnet wheel package
2020-04-22 23:53:12 -07:00
pip3 install --user ./wheel_build/dist/*.whl
2019-10-11 00:20:40 +02:00
# execute tests
2020-06-13 18:30:29 -07:00
# TODO: Add tests (18549)
2019-10-11 00:20:40 +02:00
}
# Publishes wheel to PyPI
cd_pypi_publish( ) {
set -ex
pip3 install --user twine
2020-04-22 23:53:12 -07:00
python3 ./cd/python/pypi/pypi_publish.py ` readlink -f wheel_build/dist/*.whl`
2019-10-11 00:20:40 +02:00
}
2019-12-25 13:18:32 -08:00
cd_s3_publish( ) {
set -ex
filepath = $( readlink -f wheel_build/dist/*.whl)
2020-01-14 11:33:45 +08:00
filename = $( basename $filepath )
2019-12-25 13:18:32 -08:00
variant = $( echo $filename | cut -d'-' -f1 | cut -d'_' -f2 -s)
2020-01-14 11:33:45 +08:00
if [ -z " ${ variant } " ] ; then
variant = "cpu"
fi
2021-02-24 11:48:37 -08:00
export PATH = /usr/local/bin:$PATH
2020-01-29 01:23:16 +08:00
aws s3 cp ${ filepath } s3://apache-mxnet/dist/python/${ variant } /${ filename } --grants read = uri = http://acs.amazonaws.com/groups/global/AllUsers full = id = 43f628fab72838a4f0b929d7f1993b14411f4b0294b011261bc6bd3e950a6822
2019-12-25 13:18:32 -08:00
}
2020-02-16 02:54:08 +08:00
build_static_python_cpu( ) {
2019-01-23 14:11:23 -08:00
set -ex
pushd .
2020-02-16 02:54:08 +08:00
export mxnet_variant = cpu
2020-10-05 11:34:20 -07:00
source /opt/rh/devtoolset-8/enable
2021-10-19 22:21:04 -07:00
source /opt/rh/rh-python38/enable
2020-09-19 18:42:34 -07:00
# Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.
export CXXFLAGS = "-fabi-version=11 -fabi-compat-version=7"
2019-01-23 14:11:23 -08:00
./ci/publish/python/build.sh
popd
}
2020-10-05 11:34:20 -07:00
build_static_python_cu102( ) {
2019-04-19 00:25:50 +08:00
set -ex
pushd .
2020-10-05 11:34:20 -07:00
export mxnet_variant = cu102
source /opt/rh/devtoolset-8/enable
2021-10-19 22:21:04 -07:00
source /opt/rh/rh-python38/enable
2020-09-19 18:42:34 -07:00
# Opt in to newer GCC C++ ABI. devtoolset defaults to ABI Version 2.
export CXXFLAGS = "-fabi-version=11 -fabi-compat-version=7"
2019-04-19 00:25:50 +08:00
./ci/publish/python/build.sh
popd
}
2020-05-04 16:44:27 -07:00
# artifact repository unit tests
2019-05-25 11:33:39 +02:00
test_artifact_repository( ) {
set -ex
pushd .
2019-05-25 11:50:45 +02:00
cd cd/utils/
2020-07-15 00:57:38 +00:00
OMP_NUM_THREADS = $( expr $( nproc) / 4) pytest -n 4 test_artifact_repository.py
2019-05-25 11:33:39 +02:00
popd
}
2018-03-09 23:33:10 +01:00
##############################################################
# MAIN
#
# Run function passed as argument
set +x
if [ $# -gt 0 ]
then
$@
else
cat<<EOF
$0: Execute a function by passing it as an argument to the script:
Possible commands:
EOF
declare -F | cut -d' ' -f3
echo
fi
