2020-07-28 18:10:23 -07:00
|
|
|
import errno
|
2020-07-21 13:35:29 -07:00
|
|
|
import logging
|
2017-05-27 21:35:48 -07:00
|
|
|
import os
|
2022-08-16 08:51:14 -07:00
|
|
|
import pathlib
|
2018-11-05 05:53:55 +08:00
|
|
|
import re
|
2023-05-26 10:19:31 -07:00
|
|
|
import shlex
|
2017-05-27 21:35:48 -07:00
|
|
|
import shutil
|
2016-10-31 17:08:03 -07:00
|
|
|
import subprocess
|
2017-07-31 21:04:15 -07:00
|
|
|
import sys
|
2023-05-26 10:19:31 -07:00
|
|
|
import warnings
|
2022-06-17 13:40:32 -07:00
|
|
|
from enum import Enum
|
|
|
|
|
from itertools import chain
|
2016-02-07 15:50:02 -08:00
|
|
|
|
2023-03-09 09:28:02 -08:00
|
|
|
# Workaround for setuptools_scm (used on macos) adding junk files
|
|
|
|
|
# https://stackoverflow.com/a/61274968/8162137
|
|
|
|
|
try:
|
|
|
|
|
import setuptools_scm.integration
|
|
|
|
|
|
|
|
|
|
setuptools_scm.integration.find_files = lambda _: []
|
|
|
|
|
except ImportError:
|
|
|
|
|
pass
|
|
|
|
|
|
2020-07-21 13:35:29 -07:00
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
2026-02-05 14:38:44 -08:00
|
|
|
SUPPORTED_PYTHONS = [(3, 10), (3, 11), (3, 12), (3, 13), (3, 14)]
|
2021-12-09 23:25:22 +00:00
|
|
|
# When the bazel version is updated, make sure to update it
|
|
|
|
|
# in WORKSPACE file as well.
|
2022-05-26 17:09:40 -07:00
|
|
|
|
2020-07-16 09:26:47 -07:00
|
|
|
ROOT_DIR = os.path.dirname(__file__)
|
2025-09-25 09:37:00 -07:00
|
|
|
BUILD_CORE = os.getenv("RAY_BUILD_CORE", "1") == "1"
|
|
|
|
|
BUILD_JAVA = os.getenv("RAY_INSTALL_JAVA", "0") == "1"
|
2025-08-18 11:28:28 -07:00
|
|
|
BUILD_CPP = os.getenv("RAY_DISABLE_EXTRA_CPP") != "1"
|
add gen_redis_pkg bazel target (#56527)
<!-- Thank you for your contribution! Please review
https://github.com/ray-project/ray/blob/master/CONTRIBUTING.rst before
opening a pull request. -->
<!-- Please add a reviewer to the assignee section when you create a PR.
If you don't have the access to it, we will shortly find a reviewer and
assign them to your PR. -->
## Why are these changes needed?
Separate building ray and building redis-cli, redis-server. The latter
two are only needed for tests, in
`src/ray/gcs/gcs_client/tests/BUILD.bazel` and
`src/ray/gcs/store_client/tests/BUILD.bazel`. Replaces #55731. Follow on
for #55975, #56036 which provide binary versions of redis components
rather than building from source. Once complete, conda-forge can use
this to avoid building redis.
<!-- Please give a short summary of the change and the problem this
solves. -->
## Related issue number
<!-- For example: "Closes #1234" -->
## Checks
- [x] I've signed off every commit(by using the -s flag, i.e., `git
commit -s`) in this PR.
- [ ] I've run `scripts/format.sh` to lint the changes in this PR.
- [ ] I've included any doc changes needed for
https://docs.ray.io/en/master/.
- [ ] I've added any new APIs to the API Reference. For example, if I
added a
method in Tune, I've added it in `doc/source/tune/api/` under the
corresponding `.rst` file.
- [ ] I've made sure the tests are passing. Note that there might be a
few flaky tests, see the recent failures at https://flakey-tests.ray.io/
- Testing Strategy
- [ ] Unit tests
- [ ] Release tests
- [ ] This PR is not tested :(
---------
Signed-off-by: mattip <matti.picus@gmail.com>
Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com>
2025-09-23 19:32:33 +03:00
|
|
|
BUILD_REDIS = os.getenv("RAY_BUILD_REDIS", "1") == "1"
|
2021-10-13 03:01:58 +08:00
|
|
|
SKIP_BAZEL_BUILD = os.getenv("SKIP_BAZEL_BUILD") == "1"
|
2023-05-26 10:19:31 -07:00
|
|
|
BAZEL_ARGS = os.getenv("BAZEL_ARGS")
|
2021-12-03 02:39:36 +02:00
|
|
|
BAZEL_LIMIT_CPUS = os.getenv("BAZEL_LIMIT_CPUS")
|
2020-07-16 09:26:47 -07:00
|
|
|
|
2020-07-28 18:10:23 -07:00
|
|
|
THIRDPARTY_SUBDIR = os.path.join("ray", "thirdparty_files")
|
2023-07-10 19:23:51 -07:00
|
|
|
RUNTIME_ENV_AGENT_THIRDPARTY_SUBDIR = os.path.join(
|
|
|
|
|
"ray", "_private", "runtime_env", "agent", "thirdparty_files"
|
|
|
|
|
)
|
2025-09-22 16:23:54 -07:00
|
|
|
DEPS_ONLY_VERSION = "100.0.0.dev0"
|
2021-09-08 10:37:17 +01:00
|
|
|
# In automated builds, we do a few adjustments before building. For instance,
|
|
|
|
|
# the bazel environment is set up slightly differently, and symlinks are
|
2025-02-13 03:23:41 -08:00
|
|
|
# replaced with junctions in Windows. This variable is set in our conda-forge
|
2021-09-08 10:37:17 +01:00
|
|
|
# feedstock.
|
2025-02-13 03:23:41 -08:00
|
|
|
is_conda_forge_build = bool(int(os.environ.get("IS_AUTOMATED_BUILD", "0")))
|
2021-09-08 10:37:17 +01:00
|
|
|
|
2020-04-21 09:53:08 -07:00
|
|
|
exe_suffix = ".exe" if sys.platform == "win32" else ""
|
|
|
|
|
|
|
|
|
|
# .pyd is the extension Python requires on Windows for shared libraries.
|
|
|
|
|
# https://docs.python.org/3/faq/windows.html#is-a-pyd-file-the-same-as-a-dll
|
|
|
|
|
pyd_suffix = ".pyd" if sys.platform == "win32" else ".so"
|
|
|
|
|
|
2021-07-16 13:01:48 +08:00
|
|
|
|
|
|
|
|
def find_version(*filepath):
|
|
|
|
|
# Extract version information from filepath
|
|
|
|
|
with open(os.path.join(ROOT_DIR, *filepath)) as fp:
|
2023-10-23 16:34:09 -07:00
|
|
|
version_match = re.search(r"^version = ['\"]([^'\"]*)['\"]", fp.read(), re.M)
|
2021-07-16 13:01:48 +08:00
|
|
|
if version_match:
|
|
|
|
|
return version_match.group(1)
|
|
|
|
|
raise RuntimeError("Unable to find version string.")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class SetupType(Enum):
|
|
|
|
|
RAY = 1
|
|
|
|
|
RAY_CPP = 2
|
|
|
|
|
|
|
|
|
|
|
2021-08-05 17:58:19 -07:00
|
|
|
class BuildType(Enum):
|
|
|
|
|
DEFAULT = 1
|
|
|
|
|
DEBUG = 2
|
2021-08-17 10:21:41 -07:00
|
|
|
ASAN = 3
|
2021-11-02 22:29:51 -07:00
|
|
|
TSAN = 4
|
2025-08-29 16:39:13 -07:00
|
|
|
DEPS_ONLY = 5
|
2021-08-05 17:58:19 -07:00
|
|
|
|
|
|
|
|
|
2021-07-16 13:01:48 +08:00
|
|
|
class SetupSpec:
|
2021-08-05 17:58:19 -07:00
|
|
|
def __init__(
|
|
|
|
|
self, type: SetupType, name: str, description: str, build_type: BuildType
|
|
|
|
|
):
|
2021-07-16 13:01:48 +08:00
|
|
|
self.type: SetupType = type
|
|
|
|
|
self.name: str = name
|
2023-10-23 16:34:09 -07:00
|
|
|
version = find_version("ray", "_version.py")
|
2021-08-05 17:58:19 -07:00
|
|
|
# add .dbg suffix if debug mode is on.
|
2021-08-17 10:21:41 -07:00
|
|
|
if build_type == BuildType.DEBUG:
|
|
|
|
|
self.version: str = f"{version}+dbg"
|
|
|
|
|
elif build_type == BuildType.ASAN:
|
|
|
|
|
self.version: str = f"{version}+asan"
|
2021-11-02 22:29:51 -07:00
|
|
|
elif build_type == BuildType.TSAN:
|
|
|
|
|
self.version: str = f"{version}+tsan"
|
2025-08-29 16:39:13 -07:00
|
|
|
elif build_type == BuildType.DEPS_ONLY:
|
|
|
|
|
self.version: str = DEPS_ONLY_VERSION
|
2021-08-17 10:21:41 -07:00
|
|
|
else:
|
|
|
|
|
self.version = version
|
2021-07-16 13:01:48 +08:00
|
|
|
self.description: str = description
|
2021-08-05 17:58:19 -07:00
|
|
|
self.build_type: BuildType = build_type
|
2021-07-16 13:01:48 +08:00
|
|
|
self.files_to_include: list = []
|
|
|
|
|
self.install_requires: list = []
|
|
|
|
|
self.extras: dict = {}
|
|
|
|
|
|
|
|
|
|
def get_packages(self):
|
2025-08-29 16:39:13 -07:00
|
|
|
if self.type == SetupType.RAY and self.build_type != BuildType.DEPS_ONLY:
|
2023-03-09 09:28:02 -08:00
|
|
|
return setuptools.find_packages(exclude=("tests", "*.tests", "*.tests.*"))
|
2021-07-16 13:01:48 +08:00
|
|
|
else:
|
|
|
|
|
return []
|
|
|
|
|
|
|
|
|
|
|
2021-08-17 10:21:41 -07:00
|
|
|
build_type = os.getenv("RAY_DEBUG_BUILD")
|
|
|
|
|
if build_type == "debug":
|
|
|
|
|
BUILD_TYPE = BuildType.DEBUG
|
|
|
|
|
elif build_type == "asan":
|
|
|
|
|
BUILD_TYPE = BuildType.ASAN
|
2021-11-02 22:29:51 -07:00
|
|
|
elif build_type == "tsan":
|
|
|
|
|
BUILD_TYPE = BuildType.TSAN
|
2025-08-29 16:39:13 -07:00
|
|
|
elif build_type == "deps-only":
|
|
|
|
|
BUILD_TYPE = BuildType.DEPS_ONLY
|
2021-08-17 10:21:41 -07:00
|
|
|
else:
|
|
|
|
|
BUILD_TYPE = BuildType.DEFAULT
|
2021-08-05 17:58:19 -07:00
|
|
|
|
2021-07-16 13:01:48 +08:00
|
|
|
if os.getenv("RAY_INSTALL_CPP") == "1":
|
|
|
|
|
# "ray-cpp" wheel package.
|
2021-09-08 04:45:04 +08:00
|
|
|
setup_spec = SetupSpec(
|
|
|
|
|
SetupType.RAY_CPP,
|
|
|
|
|
"ray-cpp",
|
|
|
|
|
"A subpackage of Ray which provides the Ray C++ API.",
|
|
|
|
|
BUILD_TYPE,
|
|
|
|
|
)
|
2021-07-16 13:01:48 +08:00
|
|
|
else:
|
|
|
|
|
# "ray" primary wheel package.
|
|
|
|
|
setup_spec = SetupSpec(
|
|
|
|
|
SetupType.RAY,
|
|
|
|
|
"ray",
|
|
|
|
|
"Ray provides a simple, "
|
2021-08-05 17:58:19 -07:00
|
|
|
"universal API for building distributed applications.",
|
|
|
|
|
BUILD_TYPE,
|
|
|
|
|
)
|
2021-07-16 13:01:48 +08:00
|
|
|
|
2021-06-10 10:20:30 -07:00
|
|
|
# Ideally, we could include these files by putting them in a
|
|
|
|
|
# MANIFEST.in or using the package_data argument to setup, but the
|
|
|
|
|
# MANIFEST.in gets applied at the very beginning when setup.py runs
|
|
|
|
|
# before these files have been created, so we have to move the files
|
|
|
|
|
# manually.
|
|
|
|
|
|
2019-04-02 22:17:33 -07:00
|
|
|
# NOTE: The lists below must be kept in sync with ray/BUILD.bazel.
|
2017-08-07 21:17:28 -07:00
|
|
|
ray_files = [
|
2020-04-21 09:53:08 -07:00
|
|
|
"ray/_raylet" + pyd_suffix,
|
|
|
|
|
"ray/core/src/ray/gcs/gcs_server" + exe_suffix,
|
|
|
|
|
"ray/core/src/ray/raylet/raylet" + exe_suffix,
|
2017-08-07 21:17:28 -07:00
|
|
|
]
|
2017-07-31 21:04:15 -07:00
|
|
|
|
2023-09-05 12:47:53 -07:00
|
|
|
if sys.platform == "linux":
|
|
|
|
|
ray_files.append("ray/core/libjemalloc.so")
|
|
|
|
|
|
2020-11-30 13:53:09 +08:00
|
|
|
if BUILD_JAVA or os.path.exists(os.path.join(ROOT_DIR, "ray/jars/ray_dist.jar")):
|
2020-01-10 11:41:00 +08:00
|
|
|
ray_files.append("ray/jars/ray_dist.jar")
|
|
|
|
|
|
2021-07-16 13:01:48 +08:00
|
|
|
if setup_spec.type == SetupType.RAY_CPP:
|
2021-10-18 16:03:49 +08:00
|
|
|
setup_spec.files_to_include += ["ray/cpp/default_worker" + exe_suffix]
|
2021-07-08 14:42:26 +08:00
|
|
|
# C++ API library and project template files.
|
2021-07-16 13:01:48 +08:00
|
|
|
setup_spec.files_to_include += [
|
2021-07-08 14:42:26 +08:00
|
|
|
os.path.join(dirpath, filename)
|
|
|
|
|
for dirpath, dirnames, filenames in os.walk("ray/cpp")
|
|
|
|
|
for filename in filenames
|
|
|
|
|
]
|
|
|
|
|
|
2019-07-08 22:41:37 +08:00
|
|
|
# These are the directories where automatically generated Python protobuf
|
2018-05-29 16:25:54 -07:00
|
|
|
# bindings are created.
|
|
|
|
|
generated_python_directories = [
|
2019-07-08 22:41:37 +08:00
|
|
|
"ray/core/generated",
|
2021-09-16 11:08:23 -07:00
|
|
|
"ray/serve/generated",
|
2018-05-29 16:25:54 -07:00
|
|
|
]
|
|
|
|
|
|
2020-11-24 17:41:58 -06:00
|
|
|
# Autoscaler files.
|
|
|
|
|
ray_files += [
|
2020-10-27 18:27:18 -04:00
|
|
|
"ray/autoscaler/aws/defaults.yaml",
|
2021-11-09 11:48:55 -08:00
|
|
|
"ray/autoscaler/aws/cloudwatch/prometheus.yml",
|
|
|
|
|
"ray/autoscaler/aws/cloudwatch/ray_prometheus_waiter.sh",
|
2020-10-27 18:27:18 -04:00
|
|
|
"ray/autoscaler/azure/defaults.yaml",
|
2023-10-05 12:26:14 +08:00
|
|
|
"ray/autoscaler/spark/defaults.yaml",
|
2025-07-11 18:43:56 -07:00
|
|
|
"ray/autoscaler/_private/readonly/defaults.yaml",
|
2021-07-10 14:55:00 -04:00
|
|
|
"ray/autoscaler/_private/_azure/azure-vm-template.json",
|
|
|
|
|
"ray/autoscaler/_private/_azure/azure-config-template.json",
|
2020-10-27 18:27:18 -04:00
|
|
|
"ray/autoscaler/gcp/defaults.yaml",
|
|
|
|
|
"ray/autoscaler/local/defaults.yaml",
|
2023-09-29 00:23:59 +08:00
|
|
|
"ray/autoscaler/vsphere/defaults.yaml",
|
2020-10-27 18:27:18 -04:00
|
|
|
"ray/autoscaler/ray-schema.json",
|
2018-05-31 09:00:03 -07:00
|
|
|
]
|
2018-03-04 23:35:58 -08:00
|
|
|
|
2020-11-24 17:41:58 -06:00
|
|
|
# Dashboard files.
|
|
|
|
|
ray_files += [
|
2021-09-15 11:17:15 -05:00
|
|
|
os.path.join(dirpath, filename)
|
|
|
|
|
for dirpath, dirnames, filenames in os.walk("ray/dashboard/client/build")
|
|
|
|
|
for filename in filenames
|
2019-09-23 08:50:40 -07:00
|
|
|
]
|
|
|
|
|
|
2022-09-20 14:59:03 -07:00
|
|
|
# Dashboard metrics files.
|
|
|
|
|
ray_files += [
|
|
|
|
|
os.path.join(dirpath, filename)
|
|
|
|
|
for dirpath, dirnames, filenames in os.walk("ray/dashboard/modules/metrics/export")
|
|
|
|
|
for filename in filenames
|
|
|
|
|
]
|
2023-03-22 22:49:07 -07:00
|
|
|
ray_files += [
|
|
|
|
|
os.path.join(dirpath, filename)
|
|
|
|
|
for dirpath, dirnames, filenames in os.walk(
|
|
|
|
|
"ray/dashboard/modules/metrics/dashboards"
|
|
|
|
|
)
|
|
|
|
|
for filename in filenames
|
|
|
|
|
if filename.endswith(".json")
|
|
|
|
|
]
|
2022-09-20 14:59:03 -07:00
|
|
|
|
2022-08-16 08:51:14 -07:00
|
|
|
# html templates for notebook integration
|
2022-07-12 11:23:57 -07:00
|
|
|
ray_files += [
|
2022-08-16 08:51:14 -07:00
|
|
|
p.as_posix() for p in pathlib.Path("ray/widgets/templates/").glob("*.html.j2")
|
2022-07-12 11:23:57 -07:00
|
|
|
]
|
|
|
|
|
|
2020-07-30 11:22:56 -07:00
|
|
|
# If you're adding dependencies for ray extras, please
|
2021-01-22 16:29:05 -08:00
|
|
|
# also update the matching section of requirements/requirements.txt
|
2020-07-30 11:22:56 -07:00
|
|
|
# in this directory
|
2021-07-16 13:01:48 +08:00
|
|
|
if setup_spec.type == SetupType.RAY:
|
2023-04-14 13:47:09 -07:00
|
|
|
pandas_dep = "pandas >= 1.3"
|
|
|
|
|
numpy_dep = "numpy >= 1.20"
|
2024-10-31 14:49:08 -07:00
|
|
|
pyarrow_deps = [
|
2024-11-22 21:15:40 -08:00
|
|
|
"pyarrow >= 9.0.0",
|
2024-10-31 14:49:08 -07:00
|
|
|
]
|
2025-12-08 12:23:21 -08:00
|
|
|
pydantic_dep = "pydantic!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*,!=2.11.*,<3"
|
2021-07-16 13:01:48 +08:00
|
|
|
setup_spec.extras = {
|
2024-12-17 21:55:41 -08:00
|
|
|
"cgraph": [
|
2024-08-14 13:31:13 -07:00
|
|
|
"cupy-cuda12x; sys_platform != 'darwin'",
|
|
|
|
|
],
|
|
|
|
|
"client": [
|
|
|
|
|
# The Ray client needs a specific range of gRPC to work:
|
|
|
|
|
# Tracking issues: https://github.com/grpc/grpc/issues/33714
|
2025-01-09 04:01:33 +01:00
|
|
|
"grpcio != 1.56.0; sys_platform == 'darwin'",
|
|
|
|
|
"grpcio",
|
2024-08-14 13:31:13 -07:00
|
|
|
],
|
2021-09-29 17:29:31 -07:00
|
|
|
"data": [
|
2022-08-05 20:44:12 -06:00
|
|
|
numpy_dep,
|
|
|
|
|
pandas_dep,
|
2024-10-31 14:49:08 -07:00
|
|
|
*pyarrow_deps,
|
2021-09-29 17:29:31 -07:00
|
|
|
"fsspec",
|
|
|
|
|
],
|
2021-07-16 13:01:48 +08:00
|
|
|
"default": [
|
2022-08-03 14:24:51 -07:00
|
|
|
# If adding dependencies necessary to launch the dashboard api server,
|
2024-08-01 20:40:45 -07:00
|
|
|
# please add it to python/ray/dashboard/optional_deps.py as well.
|
2026-01-16 20:47:42 -08:00
|
|
|
"aiohttp >= 3.13.3",
|
2021-09-05 22:03:32 -07:00
|
|
|
"aiohttp_cors",
|
|
|
|
|
"colorful",
|
2026-01-04 12:27:19 -08:00
|
|
|
"py-spy >= 0.2.0; python_version < '3.12'",
|
|
|
|
|
"py-spy >= 0.4.0; python_version >= '3.12'",
|
2021-09-05 22:03:32 -07:00
|
|
|
"requests",
|
2026-01-04 12:27:19 -08:00
|
|
|
"grpcio >= 1.42.0",
|
2021-09-05 22:03:32 -07:00
|
|
|
"opencensus",
|
2025-07-09 17:16:29 -07:00
|
|
|
"opentelemetry-sdk >= 1.30.0",
|
2025-05-27 14:20:43 -07:00
|
|
|
"opentelemetry-exporter-prometheus",
|
2025-05-28 10:03:40 -07:00
|
|
|
"opentelemetry-proto",
|
2025-02-13 18:24:06 -08:00
|
|
|
pydantic_dep,
|
2022-12-15 13:04:01 -08:00
|
|
|
"prometheus_client >= 0.7.1",
|
2021-11-18 08:00:30 -08:00
|
|
|
"smart_open",
|
2023-12-14 09:37:06 -08:00
|
|
|
"virtualenv >=20.0.24, !=20.21.1", # For pip runtime env.
|
2022-01-14 16:00:51 -08:00
|
|
|
],
|
2024-08-14 13:31:13 -07:00
|
|
|
"observability": [
|
2024-12-06 06:51:04 -07:00
|
|
|
"memray; sys_platform != 'win32'",
|
2023-06-08 12:58:06 -07:00
|
|
|
],
|
2023-08-23 08:55:15 -07:00
|
|
|
"serve": [
|
2023-09-13 20:29:39 -07:00
|
|
|
"uvicorn[standard]",
|
2023-08-23 08:55:15 -07:00
|
|
|
"requests",
|
|
|
|
|
"starlette",
|
2024-02-12 14:19:12 -06:00
|
|
|
"fastapi",
|
2023-08-23 08:55:15 -07:00
|
|
|
"watchfiles",
|
|
|
|
|
],
|
2024-11-01 16:44:35 -07:00
|
|
|
"tune": [
|
|
|
|
|
"pandas",
|
2025-11-06 15:40:31 -08:00
|
|
|
# TODO: Remove pydantic dependency from tune once tune doesn't import train
|
|
|
|
|
pydantic_dep,
|
2024-11-01 16:44:35 -07:00
|
|
|
"tensorboardX>=1.9",
|
|
|
|
|
"requests",
|
|
|
|
|
*pyarrow_deps,
|
|
|
|
|
"fsspec",
|
|
|
|
|
],
|
2021-07-16 13:01:48 +08:00
|
|
|
}
|
2021-09-10 16:56:51 +08:00
|
|
|
|
2024-12-17 21:55:41 -08:00
|
|
|
# Both "adag" and "cgraph" are for Compiled Graphs.
|
2024-12-12 13:10:20 -08:00
|
|
|
# "adag" is deprecated and will be removed in the future.
|
2024-12-17 21:55:41 -08:00
|
|
|
setup_spec.extras["adag"] = list(setup_spec.extras["cgraph"])
|
2024-12-12 13:10:20 -08:00
|
|
|
|
2021-10-21 13:47:29 -07:00
|
|
|
# Ray Serve depends on the Ray dashboard components.
|
|
|
|
|
setup_spec.extras["serve"] = list(
|
|
|
|
|
set(setup_spec.extras["serve"] + setup_spec.extras["default"])
|
2022-01-29 18:41:57 -08:00
|
|
|
)
|
2021-10-21 13:47:29 -07:00
|
|
|
|
2023-08-22 16:23:07 +00:00
|
|
|
# Ensure gRPC library exists for Ray Serve gRPC support.
|
|
|
|
|
setup_spec.extras["serve-grpc"] = list(
|
|
|
|
|
set(
|
|
|
|
|
setup_spec.extras["serve"]
|
|
|
|
|
+ [
|
2026-01-04 12:27:19 -08:00
|
|
|
"grpcio >= 1.42.0",
|
2024-09-23 15:07:44 -07:00
|
|
|
"pyOpenSSL",
|
2023-08-22 16:23:07 +00:00
|
|
|
]
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
|
2025-08-14 10:21:53 +05:30
|
|
|
# This is required for supporting the asynchronous inference, allowing the ray serve applications to
|
|
|
|
|
# allow asynchronously execute their code, via the use of celery task processor.
|
|
|
|
|
setup_spec.extras["serve-async-inference"] = list(
|
|
|
|
|
set(
|
|
|
|
|
setup_spec.extras["serve"]
|
|
|
|
|
+ [
|
|
|
|
|
"celery",
|
2026-02-14 03:11:09 +05:30
|
|
|
"taskiq",
|
2025-08-14 10:21:53 +05:30
|
|
|
]
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
|
2026-01-17 14:13:21 -08:00
|
|
|
setup_spec.extras["cpp"] = ["ray-cpp==" + setup_spec.version]
|
2021-09-10 16:56:51 +08:00
|
|
|
|
2021-07-16 13:01:48 +08:00
|
|
|
setup_spec.extras["rllib"] = setup_spec.extras["tune"] + [
|
|
|
|
|
"dm_tree",
|
2025-12-18 04:26:24 -08:00
|
|
|
"gymnasium==1.2.2",
|
2021-07-16 13:01:48 +08:00
|
|
|
"lz4",
|
2025-12-29 22:24:39 -08:00
|
|
|
"ormsgpack>=1.7.0",
|
2021-07-16 13:01:48 +08:00
|
|
|
"pyyaml",
|
|
|
|
|
"scipy",
|
2021-05-24 13:15:41 -07:00
|
|
|
]
|
2020-01-19 01:49:33 -08:00
|
|
|
|
2025-02-13 18:24:06 -08:00
|
|
|
setup_spec.extras["train"] = setup_spec.extras["tune"] + [pydantic_dep]
|
2022-06-08 21:34:18 -07:00
|
|
|
|
2022-05-12 09:25:52 -07:00
|
|
|
# Ray AI Runtime should encompass Data, Tune, and Serve.
|
|
|
|
|
setup_spec.extras["air"] = list(
|
|
|
|
|
set(
|
|
|
|
|
setup_spec.extras["tune"]
|
|
|
|
|
+ setup_spec.extras["data"]
|
2022-06-08 21:34:18 -07:00
|
|
|
+ setup_spec.extras["train"]
|
2022-05-12 09:25:52 -07:00
|
|
|
+ setup_spec.extras["serve"]
|
|
|
|
|
)
|
|
|
|
|
)
|
|
|
|
|
|
2025-01-15 17:52:02 -08:00
|
|
|
# NOTE: While we keep ray[all] for compatibility, you probably
|
|
|
|
|
# shouldn't use it because it contains too many dependencies
|
|
|
|
|
# and no deployment needs all of them. Instead you should list
|
|
|
|
|
# the extras you actually need, see
|
|
|
|
|
# https://docs.ray.io/en/latest/ray-overview/installation.html#from-wheels
|
|
|
|
|
#
|
2024-08-01 20:40:45 -07:00
|
|
|
# "all" will not include "cpp" anymore. It is a big depedendency
|
|
|
|
|
# that most people do not need.
|
|
|
|
|
#
|
|
|
|
|
# Instead, when cpp is supported, we add a "all-cpp".
|
2021-07-16 13:01:48 +08:00
|
|
|
setup_spec.extras["all"] = list(
|
2024-08-01 20:40:45 -07:00
|
|
|
set(
|
|
|
|
|
chain.from_iterable([v for k, v in setup_spec.extras.items() if k != "cpp"])
|
|
|
|
|
)
|
2022-01-29 18:41:57 -08:00
|
|
|
)
|
2026-01-17 14:13:21 -08:00
|
|
|
setup_spec.extras["all-cpp"] = list(
|
|
|
|
|
set(setup_spec.extras["all"] + setup_spec.extras["cpp"])
|
|
|
|
|
)
|
2019-11-27 17:05:36 -08:00
|
|
|
|
2025-02-08 00:49:22 -08:00
|
|
|
# "llm" is not included in all, by design. vllm's dependency set is very
|
|
|
|
|
# large and specific, will likely run into dependency conflicts with other
|
|
|
|
|
# ML libraries. As a result, it is an "extra-extra" that is not part
|
|
|
|
|
# ray[all].
|
|
|
|
|
#
|
|
|
|
|
# ray[llm] depends on ray[data].
|
|
|
|
|
#
|
|
|
|
|
# Keep this in sync with python/requirements/llm/llm-requirements.txt
|
|
|
|
|
#
|
2025-02-18 15:00:17 -08:00
|
|
|
setup_spec.extras["llm"] = list(
|
|
|
|
|
set(
|
|
|
|
|
[
|
2026-03-25 08:45:54 -07:00
|
|
|
"vllm[audio]>=0.18.0",
|
2025-10-15 09:55:15 -07:00
|
|
|
"nixl>=0.6.1",
|
2025-02-18 15:00:17 -08:00
|
|
|
"jsonref>=1.1.0",
|
2025-02-25 18:01:37 -08:00
|
|
|
"jsonschema",
|
2025-03-07 17:21:24 -08:00
|
|
|
"ninja",
|
2025-02-25 14:47:18 -08:00
|
|
|
# async-timeout is a backport of asyncio.timeout for python < 3.11
|
|
|
|
|
"async-timeout; python_version < '3.11'",
|
2025-03-11 14:28:26 -07:00
|
|
|
"typer",
|
2025-10-24 23:01:58 +05:30
|
|
|
"meson",
|
|
|
|
|
"pybind11",
|
2025-07-16 08:34:01 -07:00
|
|
|
"hf_transfer",
|
2025-02-18 15:00:17 -08:00
|
|
|
]
|
|
|
|
|
+ setup_spec.extras["data"]
|
|
|
|
|
+ setup_spec.extras["serve"]
|
|
|
|
|
)
|
|
|
|
|
)
|
2025-02-08 00:49:22 -08:00
|
|
|
|
2020-07-30 11:22:56 -07:00
|
|
|
# These are the main dependencies for users of ray. This list
|
|
|
|
|
# should be carefully curated. If you change it, please reflect
|
2021-01-22 16:29:05 -08:00
|
|
|
# the change in the matching section of requirements/requirements.txt
|
2022-09-26 18:22:38 -07:00
|
|
|
#
|
|
|
|
|
# NOTE: if you add any unbounded dependency, please also update
|
|
|
|
|
# install-core-prerelease-dependencies.sh so we can test
|
|
|
|
|
# new releases candidates.
|
2021-07-16 13:01:48 +08:00
|
|
|
if setup_spec.type == SetupType.RAY:
|
|
|
|
|
setup_spec.install_requires = [
|
2025-11-18 07:36:01 -08:00
|
|
|
"click>=7.0",
|
2021-07-16 13:01:48 +08:00
|
|
|
"filelock",
|
2021-10-28 13:04:22 -07:00
|
|
|
"jsonschema",
|
2021-07-16 13:01:48 +08:00
|
|
|
"msgpack >= 1.0.0, < 2.0.0",
|
2025-12-08 12:23:21 -08:00
|
|
|
"packaging>=24.2",
|
2025-08-12 01:59:59 +08:00
|
|
|
"protobuf>=3.20.3",
|
2021-07-16 13:01:48 +08:00
|
|
|
"pyyaml",
|
2022-02-23 08:32:02 +09:00
|
|
|
"requests",
|
2021-07-16 13:01:48 +08:00
|
|
|
]
|
2020-07-30 11:22:56 -07:00
|
|
|
|
2017-07-31 21:04:15 -07:00
|
|
|
|
2020-07-16 09:26:47 -07:00
|
|
|
def is_native_windows_or_msys():
|
|
|
|
|
"""Check to see if we are running on native Windows,
|
|
|
|
|
but NOT WSL (which is seen as Linux)."""
|
|
|
|
|
return sys.platform == "msys" or sys.platform == "win32"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def is_invalid_windows_platform():
|
|
|
|
|
# 'GCC' check is how you detect MinGW:
|
|
|
|
|
# https://github.com/msys2/MINGW-packages/blob/abd06ca92d876b9db05dd65f27d71c4ebe2673a9/mingw-w64-python2/0410-MINGW-build-extensions-with-GCC.patch#L53
|
|
|
|
|
platform = sys.platform
|
|
|
|
|
ver = sys.version
|
|
|
|
|
return platform == "msys" or (platform == "win32" and ver and "GCC" in ver)
|
|
|
|
|
|
|
|
|
|
|
2025-07-09 15:40:03 -07:00
|
|
|
def _find_bazel_bin():
|
|
|
|
|
candidates = []
|
|
|
|
|
|
|
|
|
|
# User specified bazel location.
|
|
|
|
|
bazel_path = os.getenv("BAZEL_PATH")
|
|
|
|
|
if bazel_path:
|
|
|
|
|
candidates.append(bazel_path)
|
|
|
|
|
|
|
|
|
|
# Default bazel locations; prefers bazelisk.
|
|
|
|
|
candidates.extend(["bazelisk", "bazel"])
|
|
|
|
|
|
2020-07-21 13:35:29 -07:00
|
|
|
if sys.platform == "win32":
|
|
|
|
|
mingw_dir = os.getenv("MINGW_DIR")
|
|
|
|
|
if mingw_dir:
|
2025-07-09 15:40:03 -07:00
|
|
|
candidates.append(os.path.join(mingw_dir, "bin", "bazel.exe"))
|
2020-07-21 13:35:29 -07:00
|
|
|
else:
|
2025-07-09 15:40:03 -07:00
|
|
|
home_dir = os.path.expanduser("~")
|
|
|
|
|
candidates.append(os.path.join(home_dir, "bin", "bazel"))
|
|
|
|
|
|
|
|
|
|
for bazel in candidates:
|
|
|
|
|
bazel_bin = shutil.which(bazel)
|
|
|
|
|
if bazel_bin:
|
|
|
|
|
return bazel_bin
|
|
|
|
|
|
|
|
|
|
raise RuntimeError("Cannot find bazel in PATH")
|
2020-07-21 13:35:29 -07:00
|
|
|
|
|
|
|
|
|
2021-09-08 10:37:17 +01:00
|
|
|
def patch_isdir():
|
|
|
|
|
"""
|
|
|
|
|
Python on Windows is having hard times at telling if a symlink is
|
|
|
|
|
a directory - it can "guess" wrong at times, which bites when
|
|
|
|
|
finding packages. Replace with a fixed version which unwraps links first.
|
|
|
|
|
"""
|
|
|
|
|
orig_isdir = os.path.isdir
|
|
|
|
|
|
|
|
|
|
def fixed_isdir(path):
|
|
|
|
|
while os.path.islink(path):
|
|
|
|
|
try:
|
|
|
|
|
link = os.readlink(path)
|
|
|
|
|
except OSError:
|
|
|
|
|
break
|
|
|
|
|
path = os.path.abspath(os.path.join(os.path.dirname(path), link))
|
|
|
|
|
return orig_isdir(path)
|
|
|
|
|
|
|
|
|
|
os.path.isdir = fixed_isdir
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def replace_symlinks_with_junctions():
|
|
|
|
|
"""
|
|
|
|
|
Per default Windows requires admin access to create symlinks, while
|
|
|
|
|
junctions (which behave similarly) can be created by users.
|
|
|
|
|
|
|
|
|
|
This function replaces symlinks (which might be broken when checked
|
|
|
|
|
out without admin rights) with junctions so Ray can be built both
|
|
|
|
|
with and without admin access.
|
|
|
|
|
"""
|
|
|
|
|
assert is_native_windows_or_msys()
|
|
|
|
|
|
|
|
|
|
# Update this list if new symlinks are introduced to the source tree
|
|
|
|
|
_LINKS = {
|
|
|
|
|
r"ray\rllib": "../../rllib",
|
|
|
|
|
}
|
|
|
|
|
root_dir = os.path.dirname(__file__)
|
|
|
|
|
for link, default in _LINKS.items():
|
|
|
|
|
path = os.path.join(root_dir, link)
|
|
|
|
|
try:
|
|
|
|
|
out = subprocess.check_output(
|
|
|
|
|
"DIR /A:LD /B", shell=True, cwd=os.path.dirname(path)
|
2022-01-29 18:41:57 -08:00
|
|
|
)
|
2021-09-08 10:37:17 +01:00
|
|
|
except subprocess.CalledProcessError:
|
|
|
|
|
out = b""
|
|
|
|
|
if os.path.basename(path) in out.decode("utf8").splitlines():
|
|
|
|
|
logger.info(f"'{link}' is already converted to junction point")
|
|
|
|
|
else:
|
|
|
|
|
logger.info(f"Converting '{link}' to junction point...")
|
|
|
|
|
if os.path.isfile(path):
|
|
|
|
|
with open(path) as inp:
|
|
|
|
|
target = inp.read()
|
|
|
|
|
os.unlink(path)
|
|
|
|
|
elif os.path.isdir(path):
|
|
|
|
|
target = default
|
|
|
|
|
try:
|
|
|
|
|
# unlink() works on links as well as on regular files,
|
|
|
|
|
# and links to directories are considered directories now
|
|
|
|
|
os.unlink(path)
|
|
|
|
|
except OSError as err:
|
|
|
|
|
# On Windows attempt to unlink a regular directory results
|
|
|
|
|
# in a PermissionError with errno set to errno.EACCES.
|
|
|
|
|
if err.errno != errno.EACCES:
|
|
|
|
|
raise
|
|
|
|
|
# For regular directories deletion is done with rmdir call.
|
|
|
|
|
os.rmdir(path)
|
|
|
|
|
else:
|
|
|
|
|
raise ValueError(f"Unexpected type of entry: '{path}'")
|
|
|
|
|
target = os.path.abspath(os.path.join(os.path.dirname(path), target))
|
|
|
|
|
logger.info("Setting {} -> {}".format(link, target))
|
|
|
|
|
subprocess.check_call(
|
2022-01-20 12:38:56 -08:00
|
|
|
f'MKLINK /J "{os.path.basename(link)}" "{target}"',
|
2021-09-08 10:37:17 +01:00
|
|
|
shell=True,
|
|
|
|
|
cwd=os.path.dirname(path),
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
2025-02-13 03:23:41 -08:00
|
|
|
if is_conda_forge_build and is_native_windows_or_msys():
|
2021-09-08 10:37:17 +01:00
|
|
|
# Automated replacements should only happen in automatic build
|
|
|
|
|
# contexts for now
|
|
|
|
|
patch_isdir()
|
|
|
|
|
replace_symlinks_with_junctions()
|
|
|
|
|
|
|
|
|
|
|
add gen_redis_pkg bazel target (#56527)
<!-- Thank you for your contribution! Please review
https://github.com/ray-project/ray/blob/master/CONTRIBUTING.rst before
opening a pull request. -->
<!-- Please add a reviewer to the assignee section when you create a PR.
If you don't have the access to it, we will shortly find a reviewer and
assign them to your PR. -->
## Why are these changes needed?
Separate building ray and building redis-cli, redis-server. The latter
two are only needed for tests, in
`src/ray/gcs/gcs_client/tests/BUILD.bazel` and
`src/ray/gcs/store_client/tests/BUILD.bazel`. Replaces #55731. Follow on
for #55975, #56036 which provide binary versions of redis components
rather than building from source. Once complete, conda-forge can use
this to avoid building redis.
<!-- Please give a short summary of the change and the problem this
solves. -->
## Related issue number
<!-- For example: "Closes #1234" -->
## Checks
- [x] I've signed off every commit(by using the -s flag, i.e., `git
commit -s`) in this PR.
- [ ] I've run `scripts/format.sh` to lint the changes in this PR.
- [ ] I've included any doc changes needed for
https://docs.ray.io/en/master/.
- [ ] I've added any new APIs to the API Reference. For example, if I
added a
method in Tune, I've added it in `doc/source/tune/api/` under the
corresponding `.rst` file.
- [ ] I've made sure the tests are passing. Note that there might be a
few flaky tests, see the recent failures at https://flakey-tests.ray.io/
- Testing Strategy
- [ ] Unit tests
- [ ] Release tests
- [ ] This PR is not tested :(
---------
Signed-off-by: mattip <matti.picus@gmail.com>
Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com>
2025-09-23 19:32:33 +03:00
|
|
|
def build(build_python, build_java, build_cpp, build_redis):
|
2020-07-16 09:26:47 -07:00
|
|
|
if tuple(sys.version_info[:2]) not in SUPPORTED_PYTHONS:
|
|
|
|
|
msg = (
|
|
|
|
|
"Detected Python version {}, which is not supported. "
|
|
|
|
|
"Only Python {} are supported."
|
|
|
|
|
).format(
|
|
|
|
|
".".join(map(str, sys.version_info[:2])),
|
|
|
|
|
", ".join(".".join(map(str, v)) for v in SUPPORTED_PYTHONS),
|
|
|
|
|
)
|
|
|
|
|
raise RuntimeError(msg)
|
|
|
|
|
|
|
|
|
|
if is_invalid_windows_platform():
|
|
|
|
|
msg = (
|
|
|
|
|
"Please use official native CPython on Windows,"
|
|
|
|
|
" not Cygwin/MSYS/MSYS2/MinGW/etc.\n"
|
|
|
|
|
+ "Detected: {}\n at: {!r}".format(sys.version, sys.executable)
|
2022-01-29 18:41:57 -08:00
|
|
|
)
|
2020-07-16 09:26:47 -07:00
|
|
|
raise OSError(msg)
|
|
|
|
|
|
2025-09-27 15:56:16 -07:00
|
|
|
# Vendor thirdparty packages.
|
|
|
|
|
#
|
|
|
|
|
# TODO(ray-core, ray-ci): the version of these vendored packages should be
|
|
|
|
|
# pinned, so that the build is reproducible.
|
2025-01-24 13:55:32 +11:00
|
|
|
if not os.getenv("SKIP_THIRDPARTY_INSTALL_CONDA_FORGE"):
|
2025-06-10 00:00:13 -07:00
|
|
|
pip_packages = ["psutil", "colorama"]
|
2020-07-16 09:26:47 -07:00
|
|
|
subprocess.check_call(
|
|
|
|
|
[
|
|
|
|
|
sys.executable,
|
|
|
|
|
"-m",
|
|
|
|
|
"pip",
|
|
|
|
|
"install",
|
|
|
|
|
"-q",
|
2020-07-28 18:10:23 -07:00
|
|
|
"--target=" + os.path.join(ROOT_DIR, THIRDPARTY_SUBDIR),
|
2020-07-16 09:26:47 -07:00
|
|
|
]
|
|
|
|
|
+ pip_packages,
|
|
|
|
|
env=dict(os.environ, CC="gcc"),
|
|
|
|
|
)
|
|
|
|
|
|
2025-01-24 13:55:32 +11:00
|
|
|
# runtime env agent dependenceis
|
|
|
|
|
runtime_env_agent_pip_packages = ["aiohttp"]
|
|
|
|
|
subprocess.check_call(
|
|
|
|
|
[
|
|
|
|
|
sys.executable,
|
|
|
|
|
"-m",
|
|
|
|
|
"pip",
|
|
|
|
|
"install",
|
|
|
|
|
"-q",
|
|
|
|
|
"--target="
|
|
|
|
|
+ os.path.join(ROOT_DIR, RUNTIME_ENV_AGENT_THIRDPARTY_SUBDIR),
|
|
|
|
|
]
|
|
|
|
|
+ runtime_env_agent_pip_packages
|
|
|
|
|
)
|
2023-07-10 19:23:51 -07:00
|
|
|
|
2025-09-27 15:56:16 -07:00
|
|
|
bazel_targets = []
|
|
|
|
|
if build_python:
|
|
|
|
|
bazel_targets.append("//:gen_ray_pkg")
|
|
|
|
|
if build_cpp:
|
|
|
|
|
bazel_targets.append("//cpp:gen_ray_cpp_pkg")
|
|
|
|
|
if build_java:
|
|
|
|
|
bazel_targets.append("//java:gen_ray_java_pkg")
|
|
|
|
|
if build_redis:
|
|
|
|
|
bazel_targets.append("//:gen_redis_pkg")
|
|
|
|
|
|
|
|
|
|
if not bazel_targets:
|
|
|
|
|
return
|
|
|
|
|
|
|
|
|
|
bazel_env = os.environ.copy()
|
|
|
|
|
bazel_env["PYTHON3_BIN_PATH"] = sys.executable
|
|
|
|
|
|
|
|
|
|
if is_native_windows_or_msys():
|
|
|
|
|
SHELL = bazel_env.get("SHELL")
|
|
|
|
|
if SHELL:
|
|
|
|
|
bazel_env.setdefault("BAZEL_SH", os.path.normpath(SHELL))
|
|
|
|
|
BAZEL_SH = bazel_env.get("BAZEL_SH", "")
|
|
|
|
|
SYSTEMROOT = os.getenv("SystemRoot")
|
|
|
|
|
wsl_bash = os.path.join(SYSTEMROOT, "System32", "bash.exe")
|
|
|
|
|
if (not BAZEL_SH) and SYSTEMROOT and os.path.isfile(wsl_bash):
|
|
|
|
|
msg = (
|
|
|
|
|
"You appear to have Bash from WSL,"
|
|
|
|
|
" which Bazel may invoke unexpectedly. "
|
|
|
|
|
"To avoid potential problems,"
|
|
|
|
|
" please explicitly set the {name!r}"
|
|
|
|
|
" environment variable for Bazel."
|
|
|
|
|
).format(name="BAZEL_SH")
|
|
|
|
|
raise RuntimeError(msg)
|
|
|
|
|
|
2021-09-08 10:37:17 +01:00
|
|
|
bazel_flags = ["--verbose_failures"]
|
2023-05-26 10:19:31 -07:00
|
|
|
if BAZEL_ARGS:
|
|
|
|
|
bazel_flags.extend(shlex.split(BAZEL_ARGS))
|
|
|
|
|
|
2021-12-03 02:39:36 +02:00
|
|
|
if BAZEL_LIMIT_CPUS:
|
|
|
|
|
n = int(BAZEL_LIMIT_CPUS) # the value must be an int
|
|
|
|
|
bazel_flags.append(f"--local_cpu_resources={n}")
|
2023-05-26 10:19:31 -07:00
|
|
|
warnings.warn(
|
|
|
|
|
"Setting BAZEL_LIMIT_CPUS is deprecated and will be removed in a future"
|
|
|
|
|
" version. Please use BAZEL_ARGS instead.",
|
|
|
|
|
FutureWarning,
|
|
|
|
|
)
|
2021-09-08 10:37:17 +01:00
|
|
|
|
2025-02-13 03:23:41 -08:00
|
|
|
if is_conda_forge_build:
|
2024-01-25 23:18:05 +02:00
|
|
|
src_dir = os.environ.get("SRC_DIR", False) or os.getcwd()
|
|
|
|
|
src_dir = os.path.abspath(src_dir)
|
|
|
|
|
if is_native_windows_or_msys():
|
|
|
|
|
drive = os.path.splitdrive(src_dir)[0] + "\\"
|
|
|
|
|
root_dir = os.path.join(drive, "bazel-root")
|
|
|
|
|
out_dir = os.path.join(drive, "b-o")
|
|
|
|
|
bazel_flags.append("--enable_runfiles=false")
|
|
|
|
|
else:
|
|
|
|
|
root_dir = os.path.join(src_dir, "..", "bazel-root")
|
|
|
|
|
out_dir = os.path.join(src_dir, "..", "b-o")
|
2021-09-08 10:37:17 +01:00
|
|
|
|
|
|
|
|
for d in (root_dir, out_dir):
|
|
|
|
|
if not os.path.exists(d):
|
|
|
|
|
os.makedirs(d)
|
|
|
|
|
|
|
|
|
|
bazel_precmd_flags = [
|
|
|
|
|
"--output_user_root=" + root_dir,
|
|
|
|
|
"--output_base=" + out_dir,
|
|
|
|
|
]
|
2024-01-25 23:18:05 +02:00
|
|
|
else:
|
|
|
|
|
bazel_precmd_flags = []
|
2025-06-30 08:13:05 +05:30
|
|
|
if sys.platform == "win32":
|
|
|
|
|
bazel_precmd_flags = ["--output_user_root=C:/tmp"]
|
2021-09-08 10:37:17 +01:00
|
|
|
|
2021-08-05 17:58:19 -07:00
|
|
|
if setup_spec.build_type == BuildType.DEBUG:
|
2025-02-13 03:23:41 -08:00
|
|
|
bazel_flags.append("--config=debug")
|
2021-08-17 10:21:41 -07:00
|
|
|
if setup_spec.build_type == BuildType.ASAN:
|
2025-02-13 03:23:41 -08:00
|
|
|
bazel_flags.append("--config=asan-build")
|
2021-11-02 22:29:51 -07:00
|
|
|
if setup_spec.build_type == BuildType.TSAN:
|
2025-02-13 03:23:41 -08:00
|
|
|
bazel_flags.append("--config=tsan")
|
2021-08-05 17:58:19 -07:00
|
|
|
|
2025-07-09 15:40:03 -07:00
|
|
|
bazel_bin = _find_bazel_bin()
|
2025-08-01 14:09:22 -07:00
|
|
|
# Build all things first.
|
2025-07-09 15:40:03 -07:00
|
|
|
subprocess.check_call(
|
|
|
|
|
[bazel_bin]
|
|
|
|
|
+ bazel_precmd_flags
|
|
|
|
|
+ ["build"]
|
|
|
|
|
+ bazel_flags
|
|
|
|
|
+ ["--"]
|
|
|
|
|
+ bazel_targets,
|
2020-07-20 12:47:27 -07:00
|
|
|
env=bazel_env,
|
|
|
|
|
)
|
2025-08-01 14:09:22 -07:00
|
|
|
# Then run the actions.
|
|
|
|
|
for action in bazel_targets:
|
|
|
|
|
subprocess.check_call(
|
|
|
|
|
[bazel_bin] + bazel_precmd_flags + ["run"] + bazel_flags + [action],
|
|
|
|
|
env=bazel_env,
|
|
|
|
|
)
|
2020-07-16 09:26:47 -07:00
|
|
|
|
|
|
|
|
|
2024-07-09 15:18:48 -07:00
|
|
|
def _walk_thirdparty_dir(directory):
|
2020-07-16 09:26:47 -07:00
|
|
|
file_list = []
|
2023-06-23 10:47:10 -07:00
|
|
|
for root, dirs, filenames in os.walk(directory):
|
2024-07-09 15:18:48 -07:00
|
|
|
# Exclude generated bytecode cache directories and tests directories
|
|
|
|
|
# from vendored packages.
|
|
|
|
|
for exclude_dir in ["__pycache__", "tests"]:
|
|
|
|
|
if exclude_dir in dirs:
|
|
|
|
|
dirs.remove(exclude_dir)
|
2020-07-16 09:26:47 -07:00
|
|
|
for name in filenames:
|
|
|
|
|
file_list.append(os.path.join(root, name))
|
|
|
|
|
return file_list
|
|
|
|
|
|
|
|
|
|
|
2021-06-10 10:20:30 -07:00
|
|
|
def copy_file(target_dir, filename, rootdir):
|
2020-07-16 09:26:47 -07:00
|
|
|
# TODO(rkn): This feels very brittle. It may not handle all cases. See
|
|
|
|
|
# https://github.com/apache/arrow/blob/master/python/setup.py for an
|
|
|
|
|
# example.
|
2024-07-09 15:18:48 -07:00
|
|
|
# File names can be absolute paths, e.g. from _walk_thirdparty_dir().
|
2021-06-10 10:20:30 -07:00
|
|
|
source = os.path.relpath(filename, rootdir)
|
|
|
|
|
destination = os.path.join(target_dir, source)
|
2020-07-16 09:26:47 -07:00
|
|
|
# Create the target directory if it doesn't already exist.
|
|
|
|
|
os.makedirs(os.path.dirname(destination), exist_ok=True)
|
|
|
|
|
if not os.path.exists(destination):
|
|
|
|
|
if sys.platform == "win32":
|
|
|
|
|
# Does not preserve file mode (needed to avoid read-only bit)
|
|
|
|
|
shutil.copyfile(source, destination, follow_symlinks=True)
|
|
|
|
|
else:
|
|
|
|
|
# Preserves file mode (needed to copy executable bit)
|
|
|
|
|
shutil.copy(source, destination, follow_symlinks=True)
|
2021-06-10 10:20:30 -07:00
|
|
|
return 1
|
|
|
|
|
return 0
|
2017-05-27 21:35:48 -07:00
|
|
|
|
2017-03-21 12:57:54 -07:00
|
|
|
|
2020-07-16 09:26:47 -07:00
|
|
|
def pip_run(build_ext):
|
2025-08-29 16:39:13 -07:00
|
|
|
if SKIP_BAZEL_BUILD or setup_spec.build_type == BuildType.DEPS_ONLY:
|
add gen_redis_pkg bazel target (#56527)
<!-- Thank you for your contribution! Please review
https://github.com/ray-project/ray/blob/master/CONTRIBUTING.rst before
opening a pull request. -->
<!-- Please add a reviewer to the assignee section when you create a PR.
If you don't have the access to it, we will shortly find a reviewer and
assign them to your PR. -->
## Why are these changes needed?
Separate building ray and building redis-cli, redis-server. The latter
two are only needed for tests, in
`src/ray/gcs/gcs_client/tests/BUILD.bazel` and
`src/ray/gcs/store_client/tests/BUILD.bazel`. Replaces #55731. Follow on
for #55975, #56036 which provide binary versions of redis components
rather than building from source. Once complete, conda-forge can use
this to avoid building redis.
<!-- Please give a short summary of the change and the problem this
solves. -->
## Related issue number
<!-- For example: "Closes #1234" -->
## Checks
- [x] I've signed off every commit(by using the -s flag, i.e., `git
commit -s`) in this PR.
- [ ] I've run `scripts/format.sh` to lint the changes in this PR.
- [ ] I've included any doc changes needed for
https://docs.ray.io/en/master/.
- [ ] I've added any new APIs to the API Reference. For example, if I
added a
method in Tune, I've added it in `doc/source/tune/api/` under the
corresponding `.rst` file.
- [ ] I've made sure the tests are passing. Note that there might be a
few flaky tests, see the recent failures at https://flakey-tests.ray.io/
- Testing Strategy
- [ ] Unit tests
- [ ] Release tests
- [ ] This PR is not tested :(
---------
Signed-off-by: mattip <matti.picus@gmail.com>
Co-authored-by: Lonnie Liu <95255098+aslonnie@users.noreply.github.com>
2025-09-23 19:32:33 +03:00
|
|
|
build(False, False, False, False)
|
2021-10-13 03:01:58 +08:00
|
|
|
else:
|
2025-09-25 09:37:00 -07:00
|
|
|
build(BUILD_CORE, BUILD_JAVA, BUILD_CPP, BUILD_REDIS)
|
2020-07-16 09:26:47 -07:00
|
|
|
|
2021-07-16 13:01:48 +08:00
|
|
|
if setup_spec.type == SetupType.RAY:
|
2025-08-29 16:39:13 -07:00
|
|
|
if setup_spec.build_type == BuildType.DEPS_ONLY:
|
|
|
|
|
setup_spec.files_to_include = []
|
|
|
|
|
return
|
2021-07-16 13:01:48 +08:00
|
|
|
setup_spec.files_to_include += ray_files
|
2020-07-16 09:26:47 -07:00
|
|
|
|
2021-07-16 13:01:48 +08:00
|
|
|
thirdparty_dir = os.path.join(ROOT_DIR, THIRDPARTY_SUBDIR)
|
2024-07-09 15:18:48 -07:00
|
|
|
setup_spec.files_to_include += _walk_thirdparty_dir(thirdparty_dir)
|
2020-07-16 09:26:47 -07:00
|
|
|
|
2023-08-21 11:43:10 -07:00
|
|
|
runtime_env_agent_thirdparty_dir = os.path.join(
|
|
|
|
|
ROOT_DIR, RUNTIME_ENV_AGENT_THIRDPARTY_SUBDIR
|
|
|
|
|
)
|
2024-07-09 15:18:48 -07:00
|
|
|
setup_spec.files_to_include += _walk_thirdparty_dir(
|
|
|
|
|
runtime_env_agent_thirdparty_dir
|
|
|
|
|
)
|
2023-08-21 11:43:10 -07:00
|
|
|
|
2021-07-16 13:01:48 +08:00
|
|
|
# Copy over the autogenerated protobuf Python bindings.
|
|
|
|
|
for directory in generated_python_directories:
|
|
|
|
|
for filename in os.listdir(directory):
|
|
|
|
|
if filename[-3:] == ".py":
|
|
|
|
|
setup_spec.files_to_include.append(
|
|
|
|
|
os.path.join(directory, filename)
|
2022-01-29 18:41:57 -08:00
|
|
|
)
|
2020-07-16 09:26:47 -07:00
|
|
|
|
2021-06-10 10:20:30 -07:00
|
|
|
copied_files = 0
|
2021-07-16 13:01:48 +08:00
|
|
|
for filename in setup_spec.files_to_include:
|
2021-06-10 10:20:30 -07:00
|
|
|
copied_files += copy_file(build_ext.build_lib, filename, ROOT_DIR)
|
|
|
|
|
print("# of files copied to {}: {}".format(build_ext.build_lib, copied_files))
|
2020-07-16 09:26:47 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
import setuptools
|
|
|
|
|
import setuptools.command.build_ext
|
|
|
|
|
|
2026-01-13 19:29:22 -08:00
|
|
|
# bdist_wheel location varies: setuptools>=70.1 has it built-in,
|
|
|
|
|
# older versions require the wheel package
|
|
|
|
|
try:
|
|
|
|
|
from setuptools.command.bdist_wheel import bdist_wheel
|
|
|
|
|
except ImportError:
|
|
|
|
|
from wheel.bdist_wheel import bdist_wheel
|
|
|
|
|
|
2020-07-16 09:26:47 -07:00
|
|
|
class build_ext(setuptools.command.build_ext.build_ext):
|
|
|
|
|
def run(self):
|
|
|
|
|
return pip_run(self)
|
|
|
|
|
|
|
|
|
|
class BinaryDistribution(setuptools.Distribution):
|
|
|
|
|
def has_ext_modules(self):
|
|
|
|
|
return True
|
|
|
|
|
|
2026-01-13 19:29:22 -08:00
|
|
|
class RayCppBdistWheel(bdist_wheel):
|
2026-01-14 12:40:05 -08:00
|
|
|
"""Build a Python-agnostic wheel for ray-cpp.
|
2026-01-13 19:29:22 -08:00
|
|
|
|
2026-01-14 12:40:05 -08:00
|
|
|
The wheel contains platform-specific C++ binaries, so we keep a platform
|
|
|
|
|
tag (e.g., manylinux2014_x86_64) but force the Python/ABI tags to py3-none.
|
2026-01-13 19:29:22 -08:00
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
def finalize_options(self):
|
|
|
|
|
super().finalize_options()
|
2026-01-14 12:40:05 -08:00
|
|
|
# Wheel contains C++ binaries, so force a real platform tag, not "any".
|
2026-01-13 19:29:22 -08:00
|
|
|
self.root_is_pure = False
|
|
|
|
|
|
|
|
|
|
def get_tag(self):
|
|
|
|
|
_, _, platform_tag = super().get_tag()
|
|
|
|
|
return "py3", "none", platform_tag
|
|
|
|
|
|
2025-06-26 12:28:59 -07:00
|
|
|
# Ensure no remaining lib files.
|
|
|
|
|
build_dir = os.path.join(ROOT_DIR, "build")
|
|
|
|
|
if os.path.isdir(build_dir):
|
|
|
|
|
shutil.rmtree(build_dir)
|
|
|
|
|
|
2025-11-30 12:34:41 -08:00
|
|
|
with open(
|
|
|
|
|
os.path.join(ROOT_DIR, os.path.pardir, "README.rst"), "r", encoding="utf-8"
|
|
|
|
|
) as f:
|
|
|
|
|
long_readme = f.read()
|
|
|
|
|
|
|
|
|
|
with open(os.path.join(ROOT_DIR, "LICENSE.txt"), "r", encoding="utf-8") as f:
|
|
|
|
|
license_text = f.read().strip()
|
|
|
|
|
if "\n" in license_text:
|
|
|
|
|
# If the license text has multiple lines, add an ending endline.
|
|
|
|
|
license_text += "\n"
|
|
|
|
|
|
2026-01-13 19:29:22 -08:00
|
|
|
# Build cmdclass dict. Use RayCppBdistWheel for ray-cpp to produce
|
|
|
|
|
# Python-agnostic wheels. See RayCppBdistWheel docstring for details.
|
|
|
|
|
cmdclass = {"build_ext": build_ext}
|
|
|
|
|
if setup_spec.type == SetupType.RAY_CPP:
|
|
|
|
|
cmdclass["bdist_wheel"] = RayCppBdistWheel
|
|
|
|
|
|
2025-06-26 12:28:59 -07:00
|
|
|
setuptools.setup(
|
|
|
|
|
name=setup_spec.name,
|
|
|
|
|
version=setup_spec.version,
|
|
|
|
|
author="Ray Team",
|
|
|
|
|
author_email="ray-dev@googlegroups.com",
|
|
|
|
|
description=(setup_spec.description),
|
2025-11-30 12:34:41 -08:00
|
|
|
long_description=long_readme,
|
2025-06-26 12:28:59 -07:00
|
|
|
url="https://github.com/ray-project/ray",
|
|
|
|
|
keywords=(
|
|
|
|
|
"ray distributed parallel machine-learning hyperparameter-tuning"
|
|
|
|
|
"reinforcement-learning deep-learning serving python"
|
|
|
|
|
),
|
2026-01-04 12:27:19 -08:00
|
|
|
python_requires=">=3.10",
|
2025-06-26 12:28:59 -07:00
|
|
|
classifiers=[
|
|
|
|
|
"Programming Language :: Python :: 3.10",
|
|
|
|
|
"Programming Language :: Python :: 3.11",
|
|
|
|
|
"Programming Language :: Python :: 3.12",
|
2025-08-05 05:06:23 +02:00
|
|
|
"Programming Language :: Python :: 3.13",
|
2026-02-05 14:38:44 -08:00
|
|
|
"Programming Language :: Python :: 3.14",
|
2025-03-13 20:12:23 -07:00
|
|
|
],
|
2025-06-26 12:28:59 -07:00
|
|
|
packages=setup_spec.get_packages(),
|
2026-01-13 19:29:22 -08:00
|
|
|
cmdclass=cmdclass,
|
2025-11-19 12:23:27 -08:00
|
|
|
distclass=( # Avoid building extensions for deps-only builds.
|
|
|
|
|
BinaryDistribution if setup_spec.build_type != BuildType.DEPS_ONLY else None
|
|
|
|
|
),
|
2025-06-26 12:28:59 -07:00
|
|
|
install_requires=setup_spec.install_requires,
|
|
|
|
|
setup_requires=["cython >= 3.0.12", "pip", "wheel"],
|
|
|
|
|
extras_require=setup_spec.extras,
|
|
|
|
|
entry_points={
|
|
|
|
|
"console_scripts": [
|
|
|
|
|
"ray=ray.scripts.scripts:main",
|
|
|
|
|
"tune=ray.tune.cli.scripts:cli",
|
|
|
|
|
"serve=ray.serve.scripts:cli",
|
|
|
|
|
]
|
|
|
|
|
},
|
|
|
|
|
package_data={
|
|
|
|
|
"ray": [
|
|
|
|
|
"includes/*.pxd",
|
|
|
|
|
"*.pxd",
|
|
|
|
|
"llm/_internal/serve/config_generator/base_configs/templates/*.yaml",
|
|
|
|
|
],
|
|
|
|
|
},
|
|
|
|
|
include_package_data=True,
|
|
|
|
|
exclude_package_data={
|
|
|
|
|
# Empty string means "any package".
|
|
|
|
|
# Therefore, exclude BUILD from every package:
|
2025-09-09 22:55:40 -07:00
|
|
|
"": ["BUILD", "BUILD.bazel"],
|
2025-06-26 12:28:59 -07:00
|
|
|
},
|
|
|
|
|
zip_safe=False,
|
2025-11-30 12:34:41 -08:00
|
|
|
license=license_text,
|
2025-06-26 12:28:59 -07:00
|
|
|
)
|
