2020-04-15 08:10:22 -07:00
|
|
|
#!/usr/bin/env bash
|
|
|
|
|
|
2020-05-05 10:47:49 -07:00
|
|
|
# Push caller's shell options (quietly)
|
|
|
|
|
{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null
|
2020-04-15 08:10:22 -07:00
|
|
|
|
2020-05-05 10:47:49 -07:00
|
|
|
set -eo pipefail
|
|
|
|
|
if [ -z "${TRAVIS_PULL_REQUEST-}" ] || [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi
|
2020-04-15 08:10:22 -07:00
|
|
|
|
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
|
2022-04-13 18:11:30 +01:00
|
|
|
WORKSPACE_DIR="${ROOT_DIR}/.."
|
2020-04-15 08:10:22 -07:00
|
|
|
|
2020-04-21 09:53:08 -07:00
|
|
|
suppress_output() {
|
|
|
|
|
"${WORKSPACE_DIR}"/ci/suppress_output "$@"
|
|
|
|
|
}
|
|
|
|
|
|
2023-06-30 19:30:16 +01:00
|
|
|
compile_pip_dependencies() {
|
|
|
|
|
# Compile boundaries
|
2023-08-08 14:03:21 +02:00
|
|
|
TARGET="${1-requirements_compiled.txt}"
|
2023-06-30 19:30:16 +01:00
|
|
|
|
|
|
|
|
if [[ "${HOSTTYPE}" == "aarch64" || "${HOSTTYPE}" = "arm64" ]]; then
|
|
|
|
|
# Resolution currently does not work on aarch64 as some pinned packages
|
|
|
|
|
# are not available. Once they are reasonably upgraded we should be able
|
|
|
|
|
# to enable this here.p
|
|
|
|
|
echo "Skipping for aarch64"
|
|
|
|
|
return 0
|
|
|
|
|
fi
|
|
|
|
|
|
2025-02-19 11:42:14 -08:00
|
|
|
(
|
|
|
|
|
# shellcheck disable=SC2262
|
|
|
|
|
alias pip="python -m pip"
|
|
|
|
|
|
|
|
|
|
cd "${WORKSPACE_DIR}"
|
|
|
|
|
|
|
|
|
|
echo "Target file: $TARGET"
|
2025-04-09 07:26:24 -07:00
|
|
|
pip install "pip-tools==7.4.1" "wheel==0.45.1"
|
2025-02-19 11:42:14 -08:00
|
|
|
|
|
|
|
|
# Required packages to lookup e.g. dragonfly-opt
|
|
|
|
|
HAS_TORCH=0
|
|
|
|
|
python -c "import torch" 2>/dev/null && HAS_TORCH=1
|
|
|
|
|
pip install --no-cache-dir numpy torch
|
|
|
|
|
|
|
|
|
|
pip-compile --verbose --resolver=backtracking \
|
|
|
|
|
--pip-args --no-deps --strip-extras --no-header \
|
|
|
|
|
--unsafe-package ray \
|
|
|
|
|
--unsafe-package pip \
|
|
|
|
|
--unsafe-package setuptools \
|
|
|
|
|
-o "python/$TARGET" \
|
|
|
|
|
python/requirements.txt \
|
|
|
|
|
python/requirements/lint-requirements.txt \
|
|
|
|
|
python/requirements/test-requirements.txt \
|
|
|
|
|
python/requirements/cloud-requirements.txt \
|
|
|
|
|
python/requirements/docker/ray-docker-requirements.txt \
|
|
|
|
|
python/requirements/ml/core-requirements.txt \
|
|
|
|
|
python/requirements/ml/data-requirements.txt \
|
|
|
|
|
python/requirements/ml/data-test-requirements.txt \
|
|
|
|
|
python/requirements/ml/dl-cpu-requirements.txt \
|
|
|
|
|
python/requirements/ml/rllib-requirements.txt \
|
|
|
|
|
python/requirements/ml/rllib-test-requirements.txt \
|
|
|
|
|
python/requirements/ml/train-requirements.txt \
|
|
|
|
|
python/requirements/ml/train-test-requirements.txt \
|
|
|
|
|
python/requirements/ml/tune-requirements.txt \
|
|
|
|
|
python/requirements/ml/tune-test-requirements.txt \
|
|
|
|
|
python/requirements/security-requirements.txt
|
|
|
|
|
|
|
|
|
|
# Delete local installation
|
|
|
|
|
sed -i "/@ file/d" "python/$TARGET"
|
|
|
|
|
|
|
|
|
|
# Remove +cpu and +pt20cpu suffixes e.g. for torch dependencies
|
|
|
|
|
# This is needed because we specify the requirements as torch==version, but
|
|
|
|
|
# the resolver adds the device-specific version tag. If this is not removed,
|
|
|
|
|
# pip install will complain about irresolvable constraints.
|
|
|
|
|
sed -i -E 's/==([\.0-9]+)\+[^\b]*cpu/==\1/g' "python/$TARGET"
|
|
|
|
|
|
|
|
|
|
cat "python/$TARGET"
|
|
|
|
|
|
|
|
|
|
if [[ "$HAS_TORCH" == "0" ]]; then
|
|
|
|
|
pip uninstall -y torch
|
|
|
|
|
fi
|
|
|
|
|
)
|
2023-06-30 19:30:16 +01:00
|
|
|
}
|
|
|
|
|
|
2020-04-29 21:19:02 -07:00
|
|
|
test_cpp() {
|
2025-10-19 17:15:08 -07:00
|
|
|
if [[ "${OSTYPE}" == darwin* ]]; then
|
|
|
|
|
echo "use macos_ci.sh to run cpp tests"
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
2021-09-17 12:13:56 +08:00
|
|
|
# C++ worker example need _GLIBCXX_USE_CXX11_ABI flag, but if we put the flag into .bazelrc, the linux ci can't pass.
|
|
|
|
|
# So only set the flag in c++ worker example. More details: https://github.com/ray-project/ray/pull/18273
|
|
|
|
|
echo build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=0" >> ~/.bazelrc
|
2020-08-28 13:53:36 +08:00
|
|
|
bazel build --config=ci //cpp:all
|
2025-08-05 07:40:43 -07:00
|
|
|
bazel run --config=ci //cpp:gen_ray_cpp_pkg
|
2024-06-13 14:52:21 -07:00
|
|
|
|
|
|
|
|
BAZEL_EXPORT_OPTIONS=($(./ci/run/bazel_export_options))
|
|
|
|
|
bazel test --config=ci "${BAZEL_EXPORT_OPTIONS[@]}" --test_strategy=exclusive //cpp:all --build_tests_only
|
2021-06-11 11:49:13 +08:00
|
|
|
# run cluster mode test with external cluster
|
2025-10-19 17:15:08 -07:00
|
|
|
bazel test --config=ci //cpp:cluster_mode_test --test_arg=--external_cluster=true \
|
2024-12-02 13:43:01 -08:00
|
|
|
--test_arg=--ray_redis_password="1234" --test_arg=--ray_redis_username="default"
|
2025-10-19 17:15:08 -07:00
|
|
|
bazel test --config=ci --test_output=all //cpp:test_python_call_cpp
|
2021-02-01 19:24:33 +08:00
|
|
|
|
2024-05-14 16:51:50 -07:00
|
|
|
# run the cpp example, currently does not work on mac
|
2025-10-19 17:15:08 -07:00
|
|
|
rm -rf ray-template
|
|
|
|
|
ray cpp --generate-bazel-project-template-to ray-template
|
|
|
|
|
(
|
|
|
|
|
cd ray-template
|
|
|
|
|
bash run.sh
|
|
|
|
|
)
|
2020-04-29 21:19:02 -07:00
|
|
|
}
|
|
|
|
|
|
2025-05-21 18:37:02 -07:00
|
|
|
test_macos_wheels() {
|
2023-06-20 10:22:55 -07:00
|
|
|
local TEST_WHEEL_RESULT=0
|
2020-04-29 21:19:02 -07:00
|
|
|
|
2025-05-21 10:32:32 -07:00
|
|
|
"${WORKSPACE_DIR}"/ci/build/test-macos-wheels.sh || TEST_WHEEL_RESULT=$?
|
2020-04-29 21:19:02 -07:00
|
|
|
|
2023-06-20 10:22:55 -07:00
|
|
|
if [[ "${TEST_WHEEL_RESULT}" != 0 ]]; then
|
2020-04-29 21:19:02 -07:00
|
|
|
cat -- /tmp/ray/session_latest/logs/* || true
|
|
|
|
|
sleep 60 # Explicitly sleep 60 seconds for logs to go through
|
|
|
|
|
fi
|
|
|
|
|
|
2023-06-20 10:22:55 -07:00
|
|
|
return "${TEST_WHEEL_RESULT}"
|
2020-04-29 21:19:02 -07:00
|
|
|
}
|
|
|
|
|
|
2025-09-09 23:52:13 -07:00
|
|
|
_install_npm_project() {
|
|
|
|
|
if [[ "${OSTYPE}" == msys ]]; then
|
2020-05-05 10:47:49 -07:00
|
|
|
# Not Windows-compatible: https://github.com/npm/cli/issues/558#issuecomment-584673763
|
|
|
|
|
{ echo "WARNING: Skipping NPM due to module incompatibilities with Windows"; } 2> /dev/null
|
|
|
|
|
else
|
2022-11-07 07:21:03 -08:00
|
|
|
npm ci
|
2020-05-05 10:47:49 -07:00
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
build_dashboard_front_end() {
|
2025-11-25 23:03:58 -08:00
|
|
|
if [[ "${OSTYPE}" == msys ]]; then
|
2020-05-05 10:47:49 -07:00
|
|
|
{ echo "WARNING: Skipping dashboard due to NPM incompatibilities with Windows"; } 2> /dev/null
|
2025-11-25 23:03:58 -08:00
|
|
|
elif [[ "${NO_DASHBOARD-}" == "1" ]]; then
|
2023-05-02 18:44:41 -07:00
|
|
|
echo "Skipping dashboard build"
|
2020-05-05 10:47:49 -07:00
|
|
|
else
|
|
|
|
|
(
|
2021-09-15 11:17:15 -05:00
|
|
|
cd ray/dashboard/client
|
2021-01-18 00:44:24 -08:00
|
|
|
|
2021-06-14 11:30:55 -07:00
|
|
|
# skip nvm activation on buildkite linux instances.
|
2025-09-09 23:52:13 -07:00
|
|
|
if [[ -z "${BUILDKITE-}" || "${OSTYPE}" != linux* ]]; then
|
|
|
|
|
if [[ -d "${HOME}/.nvm" ]]; then
|
|
|
|
|
set +x # suppress set -x since it'll get very noisy here
|
|
|
|
|
. "${HOME}/.nvm/nvm.sh"
|
|
|
|
|
NODE_VERSION="14"
|
|
|
|
|
nvm install $NODE_VERSION
|
|
|
|
|
nvm use --silent $NODE_VERSION
|
|
|
|
|
fi
|
2021-01-18 00:44:24 -08:00
|
|
|
fi
|
2025-09-09 23:52:13 -07:00
|
|
|
_install_npm_project
|
2022-11-07 07:21:03 -08:00
|
|
|
npm run build
|
2020-05-05 10:47:49 -07:00
|
|
|
)
|
|
|
|
|
fi
|
|
|
|
|
}
|
|
|
|
|
|
2020-04-29 21:19:02 -07:00
|
|
|
build_sphinx_docs() {
|
2023-06-20 10:22:27 -07:00
|
|
|
install_ray
|
|
|
|
|
|
2020-04-29 21:19:02 -07:00
|
|
|
(
|
|
|
|
|
cd "${WORKSPACE_DIR}"/doc
|
2025-11-25 23:03:58 -08:00
|
|
|
if [[ "${OSTYPE}" == msys ]]; then
|
2020-04-29 21:19:02 -07:00
|
|
|
echo "WARNING: Documentation not built on Windows due to currently-unresolved issues"
|
|
|
|
|
else
|
2024-02-02 10:05:00 -08:00
|
|
|
make html
|
2022-09-12 20:22:12 -07:00
|
|
|
pip install datasets==2.0.0
|
2020-04-29 21:19:02 -07:00
|
|
|
fi
|
|
|
|
|
)
|
|
|
|
|
}
|
|
|
|
|
|
2022-03-29 01:08:53 -07:00
|
|
|
check_sphinx_links() {
|
|
|
|
|
(
|
|
|
|
|
cd "${WORKSPACE_DIR}"/doc
|
2025-11-25 23:03:58 -08:00
|
|
|
if [[ "${OSTYPE}" == msys ]]; then
|
2022-03-29 01:08:53 -07:00
|
|
|
echo "WARNING: Documentation not built on Windows due to currently-unresolved issues"
|
|
|
|
|
else
|
2024-02-02 10:05:00 -08:00
|
|
|
make linkcheck
|
2022-03-29 01:08:53 -07:00
|
|
|
fi
|
|
|
|
|
)
|
|
|
|
|
}
|
|
|
|
|
|
2020-07-16 16:28:37 -07:00
|
|
|
_bazel_build_before_install() {
|
|
|
|
|
# NOTE: Do not add build flags here. Use .bazelrc and --config instead.
|
2021-08-05 17:58:19 -07:00
|
|
|
|
2025-05-19 14:46:02 -07:00
|
|
|
if [[ -z "${RAY_DEBUG_BUILD:-}" ]]; then
|
2025-08-03 15:59:10 -07:00
|
|
|
bazel run //:gen_ray_pkg
|
2025-05-19 14:46:02 -07:00
|
|
|
elif [[ "${RAY_DEBUG_BUILD}" == "asan" ]]; then
|
|
|
|
|
echo "No need to build anything before install"
|
|
|
|
|
elif [[ "${RAY_DEBUG_BUILD}" == "debug" ]]; then
|
2025-08-03 15:59:10 -07:00
|
|
|
bazel run --config debug //:gen_ray_pkg
|
2021-08-17 10:21:41 -07:00
|
|
|
else
|
|
|
|
|
echo "Invalid config given"
|
|
|
|
|
exit 1
|
2021-08-05 17:58:19 -07:00
|
|
|
fi
|
2020-07-14 14:51:51 -07:00
|
|
|
}
|
2020-07-02 21:06:31 -07:00
|
|
|
|
2020-07-14 14:51:51 -07:00
|
|
|
install_ray() {
|
|
|
|
|
# TODO(mehrdadn): This function should be unified with the one in python/build-wheel-windows.sh.
|
|
|
|
|
(
|
2020-05-05 10:47:49 -07:00
|
|
|
cd "${WORKSPACE_DIR}"/python
|
|
|
|
|
build_dashboard_front_end
|
2025-06-25 14:16:28 -07:00
|
|
|
|
|
|
|
|
# This is required so that pip does not pick up a cython version that is
|
|
|
|
|
# too high that can break CI, especially on MacOS.
|
|
|
|
|
pip install -q cython==3.0.12
|
|
|
|
|
|
|
|
|
|
pip install -v -e . -c requirements_compiled.txt
|
2020-05-05 10:47:49 -07:00
|
|
|
)
|
2022-06-14 20:24:02 +03:00
|
|
|
(
|
|
|
|
|
# For runtime_env tests, wheels are needed
|
|
|
|
|
cd "${WORKSPACE_DIR}"
|
2025-06-25 14:16:28 -07:00
|
|
|
pip wheel -e python -w .whl
|
2022-06-14 20:24:02 +03:00
|
|
|
)
|
2020-04-29 21:19:02 -07:00
|
|
|
}
|
|
|
|
|
|
2025-09-09 19:01:31 -07:00
|
|
|
_validate_macos_wheels_commit_str() {
|
|
|
|
|
if [[ -n "${BUILDKITE_COMMIT}" ]]; then
|
|
|
|
|
EXPECTED_COMMIT="${BUILDKITE_COMMIT:-}"
|
2021-10-06 13:48:24 +01:00
|
|
|
else
|
2025-09-09 19:01:31 -07:00
|
|
|
EXPECTED_COMMIT="$(git rev-parse HEAD)"
|
2021-10-06 13:48:24 +01:00
|
|
|
fi
|
|
|
|
|
|
2025-09-09 19:01:31 -07:00
|
|
|
if [[ -z "$EXPECTED_COMMIT" ]]; then
|
|
|
|
|
echo "Could not validate expected wheel commits: BUILDKITE_COMMIT is empty." >&2
|
|
|
|
|
exit 1
|
2021-10-06 13:48:24 +01:00
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
for whl in .whl/*.whl; do
|
2025-09-09 19:01:31 -07:00
|
|
|
basename="${whl##*/}"
|
2021-10-06 13:48:24 +01:00
|
|
|
|
|
|
|
|
if [[ "$basename" =~ "_cpp" ]]; then
|
|
|
|
|
# cpp wheels cannot be checked this way
|
|
|
|
|
echo "Skipping CPP wheel ${basename} for wheel commit validation."
|
|
|
|
|
continue
|
|
|
|
|
fi
|
|
|
|
|
|
2023-10-23 16:34:09 -07:00
|
|
|
WHL_COMMIT=$(unzip -p "$whl" "*ray/_version.py" | grep "^commit" | awk -F'"' '{print $2}')
|
2021-10-06 13:48:24 +01:00
|
|
|
|
|
|
|
|
if [ "${WHL_COMMIT}" != "${EXPECTED_COMMIT}" ]; then
|
2023-03-20 09:54:39 -07:00
|
|
|
echo "Wheel ${basename} has incorrect commit: (${WHL_COMMIT}) is not expected commit (${EXPECTED_COMMIT}). Aborting."
|
2021-10-06 13:48:24 +01:00
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
echo "Wheel ${basename} has the correct commit: ${WHL_COMMIT}"
|
|
|
|
|
done
|
|
|
|
|
|
|
|
|
|
echo "All wheels passed the sanity check and have the correct wheel commit set."
|
|
|
|
|
}
|
|
|
|
|
|
2025-09-09 19:01:31 -07:00
|
|
|
build_macos_wheels_and_jars() {
|
|
|
|
|
if [[ "${OSTYPE}" != darwin* ]]; then
|
|
|
|
|
echo "Not on macOS"
|
|
|
|
|
exit 1
|
|
|
|
|
fi
|
|
|
|
|
|
2023-06-20 10:22:27 -07:00
|
|
|
_bazel_build_before_install
|
|
|
|
|
|
2021-10-06 13:48:24 +01:00
|
|
|
# Create wheel output directory and empty contents
|
|
|
|
|
# If buildkite runners are re-used, wheels from previous builds might be here, so we delete them.
|
2025-09-09 19:01:31 -07:00
|
|
|
rm -rf .whl
|
2021-10-06 13:48:24 +01:00
|
|
|
mkdir -p .whl
|
2023-05-21 21:59:03 +08:00
|
|
|
|
2025-09-09 19:01:31 -07:00
|
|
|
# This command should be kept in sync with ray/python/README-building-wheels.md.
|
|
|
|
|
"${WORKSPACE_DIR}"/python/build-wheel-macos.sh
|
|
|
|
|
|
|
|
|
|
mkdir -p /tmp/artifacts
|
|
|
|
|
rm -rf /tmp/artifacts/.whl
|
|
|
|
|
cp -r .whl /tmp/artifacts/.whl
|
|
|
|
|
chmod 755 /tmp/artifacts/.whl
|
|
|
|
|
chmod 644 /tmp/artifacts/.whl/*
|
2021-10-06 13:48:24 +01:00
|
|
|
|
2025-09-09 19:01:31 -07:00
|
|
|
_validate_macos_wheels_commit_str
|
2020-04-29 21:19:02 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
configure_system() {
|
|
|
|
|
git config --global advice.detachedHead false
|
|
|
|
|
git config --global core.askpass ""
|
|
|
|
|
git config --global credential.helper ""
|
|
|
|
|
git config --global credential.modalprompt false
|
2021-12-21 20:16:34 -08:00
|
|
|
|
|
|
|
|
# Requests library need root certificates.
|
2024-05-15 13:35:03 -07:00
|
|
|
if [[ "${OSTYPE}" == "msys" ]]; then
|
2021-12-21 20:16:34 -08:00
|
|
|
certutil -generateSSTFromWU roots.sst && certutil -addstore -f root roots.sst && rm roots.sst
|
|
|
|
|
fi
|
2020-04-29 21:19:02 -07:00
|
|
|
}
|
|
|
|
|
|
2020-04-16 15:23:00 -07:00
|
|
|
# Initializes the environment for the current job. Performs the following tasks:
|
2020-05-05 10:47:49 -07:00
|
|
|
# - Calls 'exit 0' in this job step and all subsequent steps to quickly exit if provided a list of
|
|
|
|
|
# job names and none of them has been triggered.
|
2020-04-16 15:23:00 -07:00
|
|
|
# - Sets variables to indicate the job names that have been triggered.
|
|
|
|
|
# Note: Please avoid exporting these variables. Instead, source any callees that need to use them.
|
2020-05-05 10:47:49 -07:00
|
|
|
# This helps reduce implicit coupling of callees to their parents, as they will be unable to run
|
|
|
|
|
# when not sourced, (especially with set -u).
|
2020-04-16 15:23:00 -07:00
|
|
|
# - Installs dependencies for the current job.
|
|
|
|
|
# - Exports any environment variables necessary to run the build.
|
|
|
|
|
# Usage: init [JOB_NAMES]
|
|
|
|
|
# - JOB_NAMES (optional): Comma-separated list of job names to trigger on.
|
|
|
|
|
init() {
|
2020-04-29 21:19:02 -07:00
|
|
|
configure_system
|
2020-04-15 08:10:22 -07:00
|
|
|
|
2023-08-07 12:16:56 -07:00
|
|
|
"${ROOT_DIR}/env/install-dependencies.sh"
|
2020-04-15 08:10:22 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
build() {
|
2023-06-20 10:22:27 -07:00
|
|
|
# Build and install ray into the system.
|
|
|
|
|
# For building the wheel, see build_wheels_and_jars.
|
|
|
|
|
_bazel_build_before_install
|
|
|
|
|
install_ray
|
2020-04-29 21:19:02 -07:00
|
|
|
}
|
2020-04-15 08:10:22 -07:00
|
|
|
|
2025-11-25 23:03:58 -08:00
|
|
|
"$@"
|
2020-04-15 08:10:22 -07:00
|
|
|
|
2020-05-05 10:47:49 -07:00
|
|
|
# Pop caller's shell options (quietly)
|
|
|
|
|
{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null
|
