#!/usr/bin/env bash # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. # Check Params programname=$0 function usage { echo "usage: $programname [version] [pip_tag] [path]" echo " [version] Mxnet Version to build" echo " [pip_tag] Pip Tag to use" echo " [path] Path to MXNet repository (to run tests)" echo " " exit 1 } if [ $# -le 2 ] || [ $# -ge 4 ] then usage exit 1 fi # Two params provided echo "Building Docker Images for Apache MXNet (Incubating) v$1" mxnet_version="${1}" pip_tag="${2}" test_dir="${3}" # Remove the logs directory if it already exists else it may fail due to old logs. LOGDIR=~/temp/docker_logs if [ -d "${LOGDIR}" ]; then rm -rf ${LOGDIR} fi # Create ~/temp if it does not exist mkdir -p ~/temp mkdir ${LOGDIR} # Functions docker_test_image_cpu(){ image_tag="${1}" python_version="${2}" echo "Running tests on mxnet/python:${image_tag}" docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" } docker_test_image_gpu(){ image_tag="${1}" python_version="${2}" echo "Running tests on mxnet/python:${1}" nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}" } # if both $MXNET_DOCKERHUB_PASSWORD and $MXNET_DOCKERHUB_USERNAME environment variables are set, docker will automatically login # if env variables are not set, login will be interactive. docker_account_login(){ if [[ -z $MXNET_DOCKERHUB_PASSWORD ]] || [[ -z $MXNET_DOCKERHUB_USERNAME ]]; then docker login else echo $MXNET_DOCKERHUB_PASSWORD | docker login -u $MXNET_DOCKERHUB_USERNAME --password-stdin fi } docker_account_logout(){ docker logout } docker_push_image(){ image_tag="${1}" docker push mxnet/python:${image_tag} } docker_generate_image_cpu(){ image_tag="${1}" dockerfile="${2}" python_version="${3}" echo "Building docker image mxnet/python:${image_tag}" docker build --build-arg version=${pip_tag} -t mxnet/python:${image_tag} -f ${dockerfile} . docker_test_image_cpu ${image_tag} ${python_version} } docker_tag_image_cpu(){ original_tag="${1}" image_tag="${2}" python_version="${3}" docker tag mxnet/python:${original_tag} mxnet/python:${image_tag} docker_test_image_cpu ${image_tag} ${python_version} } docker_generate_image_gpu(){ image_tag="${1}" dockerfile="${2}" python_version="${3}" echo "Building docker image mxnet/python:${1}" docker build --build-arg version=${pip_tag} -t mxnet/python:${image_tag} -f ${dockerfile} . docker_test_image_gpu ${image_tag} ${python_version} } docker_tag_image_gpu(){ original_tag="${1}" image_tag="${2}" python_version="${3}" docker tag mxnet/python:${original_tag} mxnet/python:${image_tag} docker_test_image_gpu ${image_tag} ${python_version} } check_errors(){ egrep -i "not found|error|returned a non-zero code|fail" ${LOGDIR}/docker* if [ $? -eq 0 ]; then echo "ERROR: One of the build/test commands failed. Refer to the filename above to see which image tag caused it." exit 1 else echo "Success: No errors found" fi } # Build and Test dockerfiles - CPU docker_generate_image_cpu "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" "python" > ${LOGDIR}/docker_cpu.out 2>&1 & docker_generate_image_cpu "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" "python" > ${LOGDIR}/docker_cpu_mkl.out 2>&1 & #Build and Test dockerfiles - GPU docker_generate_image_gpu "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" "python" > ${LOGDIR}/docker_gpu_cu92.out 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl" "python" > ${LOGDIR}/docker_gpu_cu92_mkl.out 2>&1 echo "Waiting for MXNet Python2 Docker Images to Build" wait # Build and Test Python3 dockerfiles - CPU docker_generate_image_cpu "${mxnet_version}_cpu_py3" "Dockerfile.mxnet.python3.cpu" "python3" > ${LOGDIR}/docker_cpu_py3.out 2>&1 & docker_generate_image_cpu "${mxnet_version}_cpu_mkl_py3" "Dockerfile.mxnet.python3.cpu.mkl" "python3" > ${LOGDIR}/docker_cpu_mkl_py3.out 2>&1 & #Build and Test Python3 dockerfiles - GPU docker_generate_image_gpu "${mxnet_version}_gpu_cu92_py3" "Dockerfile.mxnet.python3.gpu.cu92" "python3" > ${LOGDIR}/docker_gpu_cu92_py3.out 2>&1 & docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu92.mkl" "python3" > ${LOGDIR}/docker_gpu_cu92_mkl_py3.out 2>&1 echo "Waiting for MXNet Python3 Docker Images to Build" wait echo "Re-Tag 6 images with version-free names (for Benchmarking) - only after previous builds complete. " docker_tag_image_cpu "${mxnet_version}_cpu" "latest" "python" > ${LOGDIR}/docker_latest.out 2>&1 & docker_tag_image_cpu "${mxnet_version}_cpu_mkl" "latest_cpu_mkl_py2" "python" > ${LOGDIR}/docker_latest_cpu_mkl_py2.out 2>&1 & docker_tag_image_cpu "${mxnet_version}_cpu_mkl_py3" "latest_cpu_mkl_py3" "python3" > ${LOGDIR}/docker_latest_cpu_mkl_py3.out 2>&1 & wait # Parse all the docker logfiles to make sure there is no error. Fail script if error is found. check_errors # Push dockerfiles echo "All images were successfully built. Now login to dockerhub and push images" docker_account_login # Python2 docker_push_image "${mxnet_version}_cpu" docker_push_image "${mxnet_version}_cpu_mkl" docker_push_image "${mxnet_version}_gpu_cu92" docker_push_image "${mxnet_version}_gpu_cu92_mkl" # Python3 docker_push_image "${mxnet_version}_cpu_py3" docker_push_image "${mxnet_version}_cpu_mkl_py3" docker_push_image "${mxnet_version}_gpu_cu92_py3" docker_push_image "${mxnet_version}_gpu_cu92_mkl_py3" docker_push_image "latest" docker_push_image "gpu" docker_push_image "latest_cpu_mkl_py2" docker_push_image "latest_cpu_mkl_py3" docker_push_image "latest_gpu_mkl_py2" docker_push_image "latest_gpu_mkl_py3" docker_account_logout echo "Successfully Built, Tested and Pushed all Images to Dockerhub. Link: https://hub.docker.com/r/mxnet/python/tags/" #Delete the log directory since everything succeeded: rm -rf ${LOGDIR}