2018-11-13 14:18:45 +01:00
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# -*- coding: utf-8 -*-
""" Tool to ease working with the build system and reproducing test results """
2018-12-14 00:20:18 +01:00
import argparse
2018-11-13 14:18:45 +01:00
import os
import sys
from subprocess import check_call
import shlex
from ci . util import retry , remember_cwd
from typing import List
from collections import OrderedDict
import logging
import yaml
2018-12-14 00:20:18 +01:00
import shutil
2019-05-09 06:47:55 -04:00
2018-12-14 00:20:18 +01:00
DEFAULT_PYENV = os . environ . get ( ' DEFAULT_PYENV ' , ' py3_venv ' )
DEFAULT_PYTHON = os . environ . get ( ' DEFAULT_PYTHON ' , ' python3 ' )
DEFAULT_CMAKE_OPTIONS = os . environ . get ( ' DEFAULT_CMAKE_OPTIONS ' , ' cmake_options.yml ' )
2018-11-13 14:18:45 +01:00
2019-05-09 06:47:55 -04:00
2018-11-13 14:18:45 +01:00
class Confirm ( object ) :
def __init__ ( self , cmds ) :
self . cmds = cmds
def __call__ ( self ) :
resp = input ( " This will run the following command(s) ' {} ' are you sure? yes / no: " . format ( self . cmds ) )
while True :
if resp . lower ( ) == ' yes ' :
handle_commands ( self . cmds )
return
elif resp . lower ( ) == ' no ' :
return
else :
resp = input ( " Please answer yes or no: " )
2019-05-09 06:47:55 -04:00
2018-11-13 14:18:45 +01:00
class CMake ( object ) :
2018-12-14 00:20:18 +01:00
def __init__ ( self , cmake_options_yaml = DEFAULT_CMAKE_OPTIONS , cmake_options_yaml_default = ' cmake/cmake_options.yml ' ) :
2018-12-03 15:17:09 +01:00
if os . path . exists ( cmake_options_yaml ) :
self . cmake_options_yaml = cmake_options_yaml
else :
self . cmake_options_yaml = cmake_options_yaml_default
logging . info ( ' Using {} for CMake configuration ' . format ( self . cmake_options_yaml ) )
2018-11-13 14:18:45 +01:00
self . cmake_options = None
self . read_config ( )
def cmake_command ( self ) - > str :
"""
:return: Cmake command to run given the options
"""
2020-02-12 15:51:20 -08:00
cmd_lst = [ ' cmake ' , ' -C ' , ' config.cmake ' ]
2018-11-13 14:18:45 +01:00
cmd_lst . extend ( self . _cmdlineflags ( ) )
return cmd_lst
def __call__ ( self , build_dir = ' build ' , generator = ' Ninja ' , build_cmd = ' ninja ' ) :
logging . info ( " CMake / {} build in directory {} " . format (
generator , os . path . abspath ( build_dir ) ) )
cmd_lst = self . cmake_command ( )
os . makedirs ( build_dir , exist_ok = True )
with remember_cwd ( ) :
os . chdir ( build_dir )
cmd_lst . extend ( [ ' -G {} ' . format ( generator ) , ' .. ' ] )
logging . info ( ' Executing: {} ' . format ( ' \t \n ' . join ( cmd_lst ) ) )
check_call ( cmd_lst )
logging . info ( ' Now building ' )
check_call ( shlex . split ( build_cmd ) )
2019-05-09 06:47:55 -04:00
2018-12-14 00:20:18 +01:00
def create_virtualenv ( venv_exe , pyexe , venv ) - > None :
logging . info ( " Creating virtualenv in %s with python %s " , venv , pyexe )
if not ( venv_exe and pyexe and venv ) :
logging . warn ( " Skipping creation of virtualenv " )
return
check_call ( [ venv_exe , ' -p ' , pyexe , venv ] )
2019-05-09 06:47:55 -04:00
2018-12-14 00:20:18 +01:00
def create_virtualenv_default ( ) :
create_virtualenv ( ' virtualenv ' , DEFAULT_PYTHON , DEFAULT_PYENV )
logging . info ( " You can use the virtualenv by executing ' source %s /bin/activate ' " , DEFAULT_PYENV )
2018-11-13 14:18:45 +01:00
2019-05-09 06:47:55 -04:00
def provision_virtualenv ( venv_path = DEFAULT_PYENV ) :
pip = os . path . join ( venv_path , ' bin ' , ' pip ' )
if os . path . exists ( pip ) :
# Install MXNet python bindigs
check_call ( [ pip , ' install ' , ' --upgrade ' , ' --force-reinstall ' , ' -e ' , ' python ' ] )
# Install test dependencies
check_call ( [ pip , ' install ' , ' --upgrade ' , ' --force-reinstall ' , ' -r ' , os . path . join ( ' tests ' ,
' requirements.txt ' ) ] )
else :
logging . warn ( " Can ' t find pip: ' %s ' not found " , pip )
2018-11-13 14:18:45 +01:00
COMMANDS = OrderedDict ( [
2019-01-11 01:09:22 +01:00
( ' [Local] BUILD CMake/Ninja (using cmake_options.yaml (cp cmake/cmake_options.yml .) and edit) ( {} virtualenv in " {} " ) ' . format ( DEFAULT_PYTHON , DEFAULT_PYENV ) ,
2018-12-14 00:20:18 +01:00
[
CMake ( ) ,
create_virtualenv_default ,
2019-05-09 06:47:55 -04:00
provision_virtualenv ,
2018-12-14 00:20:18 +01:00
] ) ,
2019-01-11 01:09:22 +01:00
( ' [Local] Python Unit tests ' ,
" ./py3_venv/bin/nosetests -v tests/python/unittest/ "
) ,
2019-10-17 17:43:07 -07:00
( ' [Docker] Build the MXNet binary - outputs to " lib/ " ' ,
" ci/build.py --platform ubuntu_cpu_lite /work/runtime_functions.sh build_ubuntu_cpu_docs " ) ,
( ' [Docker] Build the Jekyll website - outputs to " docs/static_site/build/html/ " ' ,
" ci/build.py --platform ubuntu_cpu_jekyll /work/runtime_functions.sh build_jekyll_docs " ) ,
( ' [Docker] Build the Python API docs - outputs to " docs/python_docs/python/build/_build/html/ " ' ,
" ci/build.py --platform ubuntu_cpu_python /work/runtime_functions.sh build_python_docs " ) ,
( ' [Docker] Build the C++ API docs - outputs to " docs/cpp_docs/build/html/html/ " ' ,
" ci/build.py --platform ubuntu_cpu_c /work/runtime_functions.sh build_c_docs " ) ,
( ' [Docker] Build the Clojure API docs - outputs to " contrib/clojure-package/target/doc " ' ,
" ci/build.py --platform ubuntu_cpu_scala /work/runtime_functions.sh build_clojure_docs " ) ,
( ' [Docker] Build the Java API docs - outputs to " docs/scala-package/build/docs/java " ' ,
" ci/build.py --platform ubuntu_cpu_scala /work/runtime_functions.sh build_java_docs " ) ,
( ' [Docker] Build the Julia API docs - outputs to " julia/docs/site/ " ' ,
[v1.9.x] Refactor dockerfiles in CI, migrate some ubuntu docker containers to use docker-compose. Update CI to use Cuda 11.7 (#21126)
* Remove deprecated dockerfiles.
* Update documentation to use different image.
* Install Scala in centos7 CD container and build tools.
* Update static scala build to use CD container, change julia container.
* Removed deprecated Jenkins pipeline files, remove old disabled build steps.
* Add new base Dockerfile for docker-compose.
* Migrate ubuntu cuda containers to docker-compose.
* Build python from source on ubuntu for portability.
* Remove old dockerfiles, upgrade nightly gpu image to cuda 11.7.
* Remove Cuda versions from runtime function names to simplify.
* Update Jenkins pipelines to use newer Cuda containers.
* Install LLVM before TVM.
* Fix ubuntu TVM install script (was failing but returning true.)
* Move cmake install into unified script.
* Move cmake install for ubuntu into centralized script.
* Update cudnn version passed to builds.
* Consolidate installation of packages for efficiency.
* Remove unused containers from docker-compose config.
* Fix pylint.
* Set LD_LIBRARY_PATH on ubuntu_gpu images to find libcuda.so.
* Set CUB_IGNORE_DEPRECATED_CPP_DIALECT to prevent build failures with gcc-4.8 + Cuda 11.7.
* Install sqlite headers/library before building python on ubuntu.
* Revert "Remove unused containers from docker-compose config."
This reverts commit 5de82dfe8f092f589fe18c010ac42b07faf21985.
* Revert "Set CUB_IGNORE_DEPRECATED_CPP_DIALECT to prevent build failures with gcc-4.8 + Cuda 11.7."
This reverts commit e6496607c0673965c2153ae7167e1785eac26bfb.
* Allow building CUB with c++11 to prevent failures on newer cuda versions.
* Set variable only on gpu make builds.
* Use docker-compose to also build ubuntu_cpu image.
* We no longer need to enable python3.8 on aarch64 since we are building from source now.
* Add Cuda 11.1 and 11.3 centos7 images which is used by CD testing phase.
* Don't install python-opencv, we are installing the module via pip instead.
* Change Makefile to set CUB_IGNORE_DEPRECATED_CPP_DIALECT when using Cuda, not only for < 11.0.
* Don't pin down h5py (old versions do not work on aarch64.)
* Conditionally install different versions of h5py dependending on architecture.
* Fix value for platform_machine.
* Don't install h5py on aarch64 at all.
* Set USE_LAPATH_PATH to correct path on ubuntu 18.04.
* Rearrange dockerfiles to build more efficiently when small changes occur. Split python install into 2 steps: building python and install requirements.
* Since we are not using multi-stage builds, do not specify target to ensure docker cache works as expected.
* When building docker-compose based containers, pull the latest version for caching before building.
* When pulling docker-compose images, pass quiet option to squell CI logs.
* When pulling docker-compose images, pass quiet option to squell CI logs.
* Clean up docker cache build code.
2022-08-22 13:58:58 -07:00
" ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_julia_docs " ) ,
2019-10-17 17:43:07 -07:00
( ' [Docker] Build the R API docs - outputs to " R-package/build/mxnet-r-reference-manual.pdf " ' ,
" ci/build.py --platform ubuntu_cpu_r /work/runtime_functions.sh build_r_docs " ) ,
( ' [Docker] Build the Scala API docs - outputs to " scala-package/docs/build/docs/scala " ' ,
" ci/build.py --platform ubuntu_cpu_scala /work/runtime_functions.sh build_scala_docs " ) ,
2019-02-14 19:26:59 +01:00
( ' [Docker] sanity_check. Check for linting and code formatting and licenses. ' ,
[
" ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh sanity_check " ,
2019-04-15 19:34:58 -07:00
" ci/build.py --platform ubuntu_rat /work/runtime_functions.sh nightly_test_rat_check " ,
2019-02-14 19:26:59 +01:00
] ) ,
2018-11-13 14:18:45 +01:00
( ' [Docker] Python3 CPU unittests ' ,
[
" ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_openblas " ,
" ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_python3_cpu " ,
] ) ,
( ' [Docker] Python3 GPU unittests ' ,
[
2019-04-05 13:49:29 -07:00
" ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu " ,
2018-11-13 14:18:45 +01:00
" ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_python3_gpu " ,
] ) ,
2018-12-03 15:17:09 +01:00
( ' [Docker] Python3 GPU+MKLDNN unittests ' ,
[
2019-04-05 13:49:29 -07:00
" ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh build_ubuntu_gpu_cmake_mkldnn " ,
2018-12-03 15:17:09 +01:00
" ci/build.py --nvidiadocker --platform ubuntu_gpu /work/runtime_functions.sh unittest_ubuntu_python3_gpu " ,
] ) ,
2018-11-28 13:14:48 +01:00
( ' [Docker] Python3 CPU Intel MKLDNN unittests ' ,
[
" ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh build_ubuntu_cpu_mkldnn " ,
" ci/build.py --platform ubuntu_cpu /work/runtime_functions.sh unittest_ubuntu_python3_cpu " ,
] ) ,
( ' [Docker] Python3 ARMv7 unittests (QEMU) ' ,
[
" ci/build.py -p armv7 " ,
2020-05-26 20:26:18 +03:00
" ci/build.py -p test.armv7 /work/runtime_functions.sh unittest_ubuntu_python3_armv7 "
2018-11-28 13:14:48 +01:00
] ) ,
2018-11-13 14:18:45 +01:00
( ' Clean (RESET HARD) repository (Warning! erases local changes / DATA LOSS) ' ,
Confirm ( " ci/docker/runtime_functions.sh clean_repo " ) )
] )
def clip ( x , mini , maxi ) :
return min ( max ( x , mini ) , maxi )
@retry ( ( ValueError , RuntimeError ) , 3 , delay_s = 0 )
def show_menu ( items : List [ str ] , header = None ) - > int :
2018-12-14 00:20:18 +01:00
print ( ' \n -- MXNet dev menu -- \n ' )
2018-11-13 14:18:45 +01:00
def hr ( ) :
print ( ' ' . join ( [ ' - ' ] * 30 ) )
if header :
print ( header )
hr ( )
for i , x in enumerate ( items , 1 ) :
print ( ' {} . {} ' . format ( i , x ) )
hr ( )
choice = int ( input ( ' Choose option> ' ) ) - 1
if choice < 0 or choice > = len ( items ) :
raise RuntimeError ( ' Choice must be between {} and {} ' . format ( 1 , len ( items ) ) )
return choice
def handle_commands ( cmds ) - > None :
def handle_command ( cmd ) :
logging . info ( " Executing command: %s " , cmd )
check_call ( shlex . split ( cmd ) )
if type ( cmds ) is list :
for cmd in cmds :
handle_commands ( cmd )
elif type ( cmds ) is str :
handle_command ( cmds )
elif callable ( cmds ) :
cmds ( )
else :
raise RuntimeError ( " handle_commands(cmds): argument should be str or List[str] but is %s " , type ( cmds ) )
2018-12-14 00:20:18 +01:00
def use_menu_ui ( args ) - > None :
2018-11-13 14:18:45 +01:00
command_list = list ( COMMANDS . keys ( ) )
2019-01-11 01:09:22 +01:00
if hasattr ( args , ' choice ' ) and args . choice and args . choice [ 0 ] . isdigit ( ) :
choice = int ( args . choice [ 0 ] ) - 1
else :
choice = show_menu ( command_list , ' Available actions ' )
2018-11-13 14:18:45 +01:00
handle_commands ( COMMANDS [ command_list [ choice ] ] )
2018-12-14 00:20:18 +01:00
def build ( args ) - > None :
""" Build using CMake """
venv_exe = shutil . which ( ' virtualenv ' )
pyexe = shutil . which ( args . pyexe )
if not venv_exe :
2019-01-11 01:09:22 +01:00
logging . warn ( " virtualenv wasn ' t found in path, it ' s recommended to install virtualenv to manage python environments " )
2018-12-14 00:20:18 +01:00
if not pyexe :
logging . warn ( " Python executable %s not found in path " , args . pyexe )
if args . cmake_options :
cmake = CMake ( args . cmake_options )
else :
cmake = CMake ( )
cmake ( )
2019-05-09 06:47:55 -04:00
create_virtualenv_default ( )
provision_virtualenv ( )
2018-12-14 00:20:18 +01:00
def main ( ) :
logging . getLogger ( ) . setLevel ( logging . INFO )
parser = argparse . ArgumentParser ( description = """ Utility for compiling and testing MXNet easily """ )
parser . set_defaults ( command = ' use_menu_ui ' )
subparsers = parser . add_subparsers ( help = ' sub-command help ' )
build_parser = subparsers . add_parser ( ' build ' , help = ' build with the specified flags from file ' )
build_parser . add_argument ( ' cmake_options ' , nargs = ' ? ' ,
help = ' File containing CMake options in YAML ' )
build_parser . add_argument ( ' -v ' , ' --venv ' ,
type = str ,
default = DEFAULT_PYENV ,
help = ' virtualenv dir ' )
build_parser . add_argument ( ' -p ' , ' --pyexe ' ,
type = str ,
default = DEFAULT_PYTHON ,
help = ' python executable ' )
build_parser . set_defaults ( command = ' build ' )
2019-01-11 01:09:22 +01:00
menu_parser = subparsers . add_parser ( ' menu ' , help = ' jump to menu option # ' )
menu_parser . set_defaults ( command = ' use_menu_ui ' )
menu_parser . add_argument ( ' choice ' , nargs = 1 )
2018-12-14 00:20:18 +01:00
args = parser . parse_args ( )
globals ( ) [ args . command ] ( args )
2018-11-13 14:18:45 +01:00
return 0
if __name__ == ' __main__ ' :
sys . exit ( main ( ) )
