From 3593c964de198646a2a284c24fc53cccd21afd7e Mon Sep 17 00:00:00 2001 From: rasmith Date: Mon, 31 Aug 2026 11:09:46 -0500 Subject: [PATCH] [ROCm] Add TheRock preview docker updates, Keep Python 3.12 and Ubuntu 22.04 (#49925) Signed-off-by: Randall Smith Co-authored-by: Andreas Karatzas --- docker/Dockerfile.rock | 992 ++++++++++++++++++++++++++++++++++++ docker/Dockerfile.rock_base | 360 +++++++++++++ pyproject.toml | 1 + requirements/build/rock.txt | 19 + requirements/build/rocm.txt | 12 +- requirements/test/rocm.in | 1 + requirements/test/rocm.txt | 6 +- 7 files changed, 1384 insertions(+), 7 deletions(-) create mode 100644 docker/Dockerfile.rock create mode 100644 docker/Dockerfile.rock_base create mode 100644 requirements/build/rock.txt diff --git a/docker/Dockerfile.rock b/docker/Dockerfile.rock new file mode 100644 index 0000000000..c3ce785c67 --- /dev/null +++ b/docker/Dockerfile.rock @@ -0,0 +1,992 @@ +# default base image +ARG REMOTE_VLLM="0" +ARG VLLM_REPO="https://github.com/vllm-project/vllm.git" +ARG VLLM_BRANCH="main" +ARG COMMON_WORKDIR=/app +ARG BASE_IMAGE=rocm/vllm-dev:base +ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base +ARG ROCM_TRITON_KERNELS_COMMIT=0f380657dbf3ee86eb57558ff71df24f03b5d4e7 +ARG INSTALL_LMCACHE=false +# NIC backend for MoRI RDMA support. +# By default (all), drivers and userspace libraries for all supported NIC types +# (ainic and bnxt) are installed; MoRI selects the appropriate one at runtime. +# To install drivers for a single NIC type only, set NIC_BACKEND explicitly: +# --build-arg NIC_BACKEND=ainic # AMD AINIC (Pensando) only +# --build-arg NIC_BACKEND=bnxt # Broadcom Thor-2 only +# --build-arg NIC_BACKEND=none # Install nothing. +ARG NIC_BACKEND=all +# AMD AINIC apt repo settings +# Users can specify a custom version compatible with their host drivers. +# The default version has been tested with ioinic-dkms=25.11.1.001 +ARG AINIC_VERSION=1.117.3-hydra +ARG UBUNTU_CODENAME=jammy + +# Sccache configuration. Release builds use this today; CI can opt in when a +# shared S3-compatible cache backend is available. +ARG USE_SCCACHE +ARG SCCACHE_DOWNLOAD_URL +ARG SCCACHE_ENDPOINT +ARG SCCACHE_BUCKET_NAME=vllm-build-sccache +ARG SCCACHE_REGION_NAME=us-west-2 +ARG SCCACHE_S3_NO_CREDENTIALS=0 + +FROM ${BASE_IMAGE} AS base + +ARG ARG_PYTORCH_ROCM_ARCH +ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}} + +# Install build dependencies and utilities +RUN apt-get update -q -y && apt-get install -q -y \ + sqlite3 libsqlite3-dev libfmt-dev libmsgpack-dev libsuitesparse-dev \ + apt-transport-https ca-certificates wget curl \ + libnuma-dev ccache mold +RUN --mount=type=cache,target=/root/.cache/pip \ + python3 -m pip install --upgrade pip +# Note: mold is installed but not set as the system default linker because +# some packages use JIT compilation at runtime with flags mold does not support. +# Build stages opt in via LDFLAGS="-fuse-ld=mold". +# Remove sccache only if not using sccache (it exists in base image from Dockerfile.rocm_base) +ARG USE_SCCACHE +RUN if [ "$USE_SCCACHE" != "1" ]; then \ + apt-get purge -y sccache || true; \ + python3 -m pip uninstall -y sccache || true; \ + rm -f "$(which sccache)" || true; \ + fi + +# Install UV — download first, then run, so a curl failure is not masked by the pipe +RUN curl -LsSf --retry 3 --retry-delay 5 https://astral.sh/uv/install.sh -o /tmp/uv-install.sh \ + && env UV_INSTALL_DIR="/usr/local/bin" sh /tmp/uv-install.sh \ + && rm -f /tmp/uv-install.sh \ + && uv --version + +# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out +# Reference: https://github.com/astral-sh/uv/pull/1694 +ENV UV_HTTP_TIMEOUT=500 +ENV UV_INDEX_STRATEGY="unsafe-best-match" +# Use copy mode to avoid hardlink failures with Docker cache mounts +ENV UV_LINK_MODE=copy +# ccache directory - persisted across layer rebuilds via cache mounts. +ENV CCACHE_DIR=/root/.cache/ccache +ENV CCACHE_COMPILERCHECK=content +# Empty by default so build steps fall back to $(nproc); CI can override. +ARG max_jobs +ENV MAX_JOBS=${max_jobs} + +# Install sccache if USE_SCCACHE is enabled (for release builds) +ARG USE_SCCACHE +ARG SCCACHE_DOWNLOAD_URL +ARG SCCACHE_ENDPOINT +ARG SCCACHE_BUCKET_NAME +ARG SCCACHE_REGION_NAME +ARG SCCACHE_S3_NO_CREDENTIALS +RUN if [ "$USE_SCCACHE" = "1" ]; then \ + if command -v sccache >/dev/null 2>&1; then \ + echo "sccache already installed, skipping installation"; \ + sccache --version; \ + else \ + echo "Installing sccache..." \ + && SCCACHE_ARCH="x86_64" \ + && SCCACHE_VERSION="v0.8.1" \ + && SCCACHE_DL_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \ + && curl -L -o /tmp/sccache.tar.gz ${SCCACHE_DL_URL} \ + && tar -xzf /tmp/sccache.tar.gz -C /tmp \ + && mv /tmp/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \ + && chmod +x /usr/bin/sccache \ + && rm -rf /tmp/sccache.tar.gz /tmp/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl \ + && sccache --version; \ + fi; \ + fi + +# Set sccache environment variables only when USE_SCCACHE=1 +# This prevents S3 config from leaking into images when sccache is not used +ARG USE_SCCACHE +ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}} +ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}} +ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}} +ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0} + +ARG COMMON_WORKDIR +WORKDIR ${COMMON_WORKDIR} + + +# ----------------------- +# vLLM fetch stages +FROM base AS fetch_vllm_0 +ONBUILD COPY ./ vllm/ +FROM base AS fetch_vllm_1 +ARG VLLM_REPO +ARG VLLM_BRANCH +ONBUILD RUN git clone ${VLLM_REPO} \ + && cd vllm \ + && git fetch -v --prune -- origin ${VLLM_BRANCH} \ + && git checkout FETCH_HEAD \ + && if [ ${VLLM_REPO} != "https://github.com/vllm-project/vllm.git" ] ; then \ + git remote add upstream "https://github.com/vllm-project/vllm.git" \ + && git fetch upstream ; fi +FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm + +# Resolve the device-independent source version once for every downstream +# artifact. setuptools-scm reads Git metadata for local/remote sources and the +# archive metadata generated for canonical CI contexts. +FROM fetch_vllm AS vllm-version +ARG COMMON_WORKDIR +RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ + cd ${COMMON_WORKDIR}/vllm \ + && uv pip install --system -r requirements/build/rust.txt \ + && python3 -c \ + "from setuptools_scm import get_version; print(get_version())" \ + > /vllm-version.txt + +# ----------------------- +# Rust toolchain inputs +# +# Keep these separate from Rust source inputs so Rust source-only changes do not +# invalidate the long-lived ci_base image that carries the toolchain. +FROM scratch AS rust_toolchain_input_0 +COPY rust-toolchain.toml /rust-toolchain-input/vllm/rust-toolchain.toml + +FROM base AS rust_toolchain_input_1 +ARG VLLM_REPO +ARG VLLM_BRANCH +RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-toolchain-input/vllm \ + && cd /rust-toolchain-input/vllm \ + && git fetch -v --prune -- origin ${VLLM_BRANCH} \ + && git sparse-checkout init --no-cone \ + && git sparse-checkout set \ + rust-toolchain.toml \ + && git checkout FETCH_HEAD + +FROM rust_toolchain_input_${REMOTE_VLLM} AS rust-toolchain-input + +# ----------------------- +# Rust build inputs +# +# REMOTE_VLLM=0: use only the local files that can affect Rust artifacts. +# REMOTE_VLLM=1: fetch only those same files from the requested remote ref. +FROM scratch AS rust_input_0 +COPY requirements/build/rust.txt /rust-input/vllm/requirements/build/rust.txt +COPY rust/Cargo.lock rust/Cargo.toml /rust-input/vllm/rust/ +COPY rust/proto /rust-input/vllm/rust/proto +COPY rust/src /rust-input/vllm/rust/src +COPY rust-toolchain.toml /rust-input/vllm/rust-toolchain.toml +COPY tools/build_rust.py /rust-input/vllm/tools/build_rust.py +COPY build_rust.sh /rust-input/vllm/build_rust.sh + +FROM base AS rust_input_1 +ARG VLLM_REPO +ARG VLLM_BRANCH +RUN git clone --no-checkout --filter=blob:none ${VLLM_REPO} /rust-input/vllm \ + && cd /rust-input/vllm \ + && git fetch -v --prune -- origin ${VLLM_BRANCH} \ + && git sparse-checkout init --no-cone \ + && git sparse-checkout set \ + requirements/build/rust.txt \ + rust/Cargo.lock \ + rust/Cargo.toml \ + rust/proto/** \ + rust/src/** \ + rust-toolchain.toml \ + tools/build_rust.py \ + build_rust.sh \ + && git checkout FETCH_HEAD + +FROM rust_input_${REMOTE_VLLM} AS rust-input + +# ----------------------- +# Rust toolchain +FROM base AS rust-toolchain + +ENV CARGO_HOME=/root/.cargo +ENV RUSTUP_HOME=/root/.rustup +ENV PATH=${CARGO_HOME}/bin:${PATH} + +RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \ + ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=rust-toolchain-input /rust-toolchain-input/vllm/rust-toolchain.toml /tmp/rust-toolchain.toml +RUN TOOLCHAIN="$(grep '^channel' /tmp/rust-toolchain.toml | sed 's/.*= *"\(.*\)"/\1/')" \ + && if ! command -v rustup >/dev/null 2>&1; then \ + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ + | sh -s -- -y --default-toolchain none; \ + fi \ + && rustup toolchain install "${TOOLCHAIN}" \ + && rustup default "${TOOLCHAIN}" \ + && rustc --version \ + && cargo --version + +# ----------------------- +# Rust build stage +# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages +# don't need the rust toolchain. +FROM rust-toolchain AS rust-build +ARG COMMON_WORKDIR +ARG USE_SCCACHE + +# Cap cargo parallelism to avoid exhausting the AMD CI host's open-file limit +# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise). +ENV CARGO_BUILD_JOBS=4 +ENV CARGO_NET_RETRY=10 +ENV RUSTUP_MAX_RETRIES=10 + +WORKDIR ${COMMON_WORKDIR}/vllm + +COPY --from=rust-input /rust-input/vllm/requirements/build/rust.txt requirements/build/rust.txt + +# BuildKit can run this stage in parallel with ROCm native builds. Keep Rust on +# a separate local sccache daemon while sharing the same remote cache backend. +ENV SCCACHE_SERVER_PORT=4227 + +RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ + uv pip install --system -r requirements/build/rust.txt + +COPY --from=rust-input /rust-input/vllm/rust rust +COPY --from=rust-input /rust-input/vllm/rust-toolchain.toml rust-toolchain.toml +COPY --from=rust-input /rust-input/vllm/tools/build_rust.py tools/build_rust.py +COPY --from=rust-input /rust-input/vllm/build_rust.sh build_rust.sh +COPY --from=vllm-version /vllm-version.txt /tmp/vllm-version.txt + +# Build the release binary. Cargo's registry/git caches can be written by +# concurrent BuildKit jobs on shared workers, so lock those cache mounts while +# keeping the cache benefit. Do not cache target/, because stale target metadata +# can outlive source updates across BuildKit cache reuse. +RUN --mount=type=cache,id=vllm-rocm-cargo-registry,target=/root/.cargo/registry,sharing=locked \ + --mount=type=cache,id=vllm-rocm-cargo-git,target=/root/.cargo/git,sharing=locked \ + if [ "$USE_SCCACHE" = "1" ]; then \ + export RUSTC_WRAPPER=sccache \ + && sccache --show-stats; \ + fi \ + && export SETUPTOOLS_SCM_PRETEND_VERSION="$(cat /tmp/vllm-version.txt)" \ + && bash build_rust.sh \ + && test -x vllm/vllm-rs \ + && if [ "$USE_SCCACHE" = "1" ]; then \ + sccache --show-stats; \ + fi + +# ----------------------- +# Shared vLLM build dependencies +# +# Install requirements once, before bringing source-specific inputs into either +# consumer. Native compilation and wheel packaging then share this expensive +# layer while retaining their independent source cache boundaries. +FROM base AS build_vllm_dependencies +ARG COMMON_WORKDIR +ENV VLLM_TARGET_DEVICE=rocm +WORKDIR ${COMMON_WORKDIR}/vllm + +COPY --from=fetch_vllm \ + ${COMMON_WORKDIR}/vllm/requirements/common.txt \ + ${COMMON_WORKDIR}/vllm/requirements/rocm.txt \ + requirements/ + +# Use constraints file to protect triton, pybind11, and torch dependencies. +RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ + uv pip install --system -c /etc/rocm-constraints.txt -r requirements/rocm.txt + +# Fetch only the pinned ROCm Triton kernels package. Keeping this before the +# vLLM source COPYs makes the download reusable when native sources change. +FROM base AS rocm-triton-kernels +ARG ROCM_TRITON_KERNELS_COMMIT +ADD https://codeload.github.com/ROCm/triton/tar.gz/${ROCM_TRITON_KERNELS_COMMIT} \ + /tmp/rocm-triton.tar.gz +RUN mkdir -p /opt/rocm-triton-kernels \ + && tar --extract --gzip --file /tmp/rocm-triton.tar.gz \ + --directory /opt/rocm-triton-kernels --strip-components=4 \ + --no-same-owner \ + "triton-${ROCM_TRITON_KERNELS_COMMIT}/python/triton_kernels/triton_kernels" \ + && test -f /opt/rocm-triton-kernels/__init__.py \ + && rm /tmp/rocm-triton.tar.gz + +# ----------------------- +# vLLM native build stages +# +# csrc-build intentionally copies only files that affect ROCm native extension +# compilation. That keeps unrelated CI/test/docs edits from invalidating the +# expensive HIP/C++ build layer. +FROM build_vllm_dependencies AS csrc-build +ARG ROCM_TRITON_KERNELS_COMMIT +COPY --from=rocm-triton-kernels \ + /opt/rocm-triton-kernels /opt/rocm-triton-kernels +ENV TRITON_KERNELS_SRC_DIR=/opt/rocm-triton-kernels + +# pyproject.toml is bind-mounted so it is not persisted in the csrc-build image, +# but it still participates in the BuildKit cache key for this RUN step. +COPY setup.py CMakeLists.txt ./ +COPY tools/build_rust.py tools/build_rust.py +COPY cmake cmake/ +COPY csrc csrc/ +COPY vllm/envs.py vllm/envs.py +COPY vllm/__init__.py vllm/__init__.py + +ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build" + +RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \ + --mount=type=cache,id=vllm-rocm-ccache,target=/root/.cache/ccache \ + grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \ + cmake/external_projects/triton_kernels.cmake \ + && export CCACHE_BASEDIR="$PWD" \ + && echo "=== ccache stats before ROCm native build ===" \ + && (ccache --show-stats || true) \ + && (ccache --zero-stats || true) \ + && EFFECTIVE_MAX_JOBS="${MAX_JOBS:-$(nproc)}" \ + && echo "Building ROCm native extension wheel with MAX_JOBS=${EFFECTIVE_MAX_JOBS}" \ + && PYTHONDONTWRITEBYTECODE=1 LDFLAGS="-fuse-ld=mold" MAX_JOBS="${EFFECTIVE_MAX_JOBS}" python3 setup.py bdist_wheel --dist-dir=dist \ + && test -d dist \ + && ls dist/*.whl >/dev/null \ + && echo "=== ccache stats after ROCm native build ===" \ + && (ccache --show-stats || true) + +# Build the full vLLM ROCm wheel by reusing the native extension wheel from +# csrc-build. This stage still rebuilds for Python/package changes, but skips +# the expensive dependency installation and HIP/C++ compilation when their +# respective inputs are unchanged. +FROM build_vllm_dependencies AS build_vllm +COPY --from=fetch_vllm ${COMMON_WORKDIR}/vllm ${COMMON_WORKDIR}/vllm +COPY --from=csrc-build ${COMMON_WORKDIR}/vllm/dist /precompiled-wheels +COPY --from=vllm-version /vllm-version.txt /tmp/vllm-version.txt + +# Drop the pre-built Rust artifacts into the source tree. setup.py detects +# them and ships them as-is, skipping the local Rust build. +COPY --from=rust-build ${COMMON_WORKDIR}/vllm/vllm/vllm-rs ${COMMON_WORKDIR}/vllm/vllm/vllm-rs +COPY --from=rust-build ${COMMON_WORKDIR}/vllm/vllm/_rust_*.so ${COMMON_WORKDIR}/vllm/vllm/ + +RUN cd "${COMMON_WORKDIR}/vllm" \ + && export SETUPTOOLS_SCM_PRETEND_VERSION="$(cat /tmp/vllm-version.txt)" \ + && export VLLM_USE_PRECOMPILED=1 \ + && export VLLM_PRECOMPILED_WHEEL_LOCATION="$(ls /precompiled-wheels/*.whl)" \ + && export VLLM_DOCKER_BUILD_CONTEXT=1 \ + && echo "Packaging vLLM ROCm wheel using precompiled extensions from ${VLLM_PRECOMPILED_WHEEL_LOCATION}" \ + && PYTHONDONTWRITEBYTECODE=1 python3 setup.py bdist_wheel --dist-dir=dist \ + && test -d dist \ + && ls dist/*.whl >/dev/null +FROM scratch AS export_vllm +ARG COMMON_WORKDIR +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/dist/*.whl / +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/scripts/benchmark_helion_kernels.py /scripts/benchmark_helion_kernels.py +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/ +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_base /docker/Dockerfile.rocm_base +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/ci-rocm.hcl /docker/ci-rocm.hcl +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/docker-bake.hcl /docker/docker-bake.hcl +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/docker-bake-rocm.hcl /docker/docker-bake-rocm.hcl +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/pyproject.toml /pyproject.toml +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/rust /rust +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/rust-toolchain.toml /rust-toolchain.toml +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/vllm/v1 /vllm_v1 + +# NIXL/UCX build stages +FROM base AS build_nixl +ARG NIXL_BRANCH="231d56753047c989062a5cb2ac703a1ad761c7d2" +ARG NIXL_REPO="https://github.com/ai-dynamo/nixl.git" +ARG UCX_BRANCH="96e58a16039f6d7d213bc967b8069238742c5194" +ARG UCX_REPO="https://github.com/openucx/ucx.git" +ENV UCX_HOME=/usr/local/ucx +ENV NIXL_HOME=/usr/local/nixl + +# NIXL build system dependencies and RDMA support +RUN apt-get -y update && apt-get -y install autoconf libtool pkg-config \ + libgrpc-dev \ + libgrpc++-dev \ + libprotobuf-dev \ + protobuf-compiler-grpc \ + libcpprest-dev \ + libaio-dev \ + librdmacm1 \ + librdmacm-dev \ + libibverbs1 \ + libibverbs-dev \ + ibverbs-utils \ + rdmacm-utils \ + ibverbs-providers \ + && rm -rf /var/lib/apt/lists/* + +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system meson meson-python pyyaml types-PyYAML \ + auditwheel build patchelf pytest tomlkit "setuptools>=80.9.0" + +RUN --mount=type=cache,target=/root/.cache/ccache \ + cd /usr/local/src && \ + git clone ${UCX_REPO} && \ + cd ucx && \ + git checkout ${UCX_BRANCH} && \ + ./autogen.sh && \ + mkdir build && cd build && \ + CC="ccache gcc" CXX="ccache g++" \ + ../configure \ + --prefix=/usr/local/ucx \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-devel-headers \ + --with-rocm=${ROCM_PATH} \ + --with-verbs \ + --with-dm \ + --enable-mt && \ + make -j$(nproc) && \ + make install + +ENV PATH=/usr/local/ucx/bin:$PATH +ENV LD_LIBRARY_PATH=${UCX_HOME}/lib:${LD_LIBRARY_PATH} + +RUN --mount=type=cache,target=/root/.cache/ccache \ + git clone ${NIXL_REPO} /opt/nixl && \ + cd /opt/nixl && \ + git checkout ${NIXL_BRANCH} && \ + CC="ccache gcc" CXX="ccache g++" \ + meson setup build --prefix=${NIXL_HOME} \ + -Ducx_path=${UCX_HOME} \ + -Dwheel_variant=rocm \ + -Dbuild_tests=false \ + -Dbuild_examples=false && \ + cd build && \ + ninja -j$(nproc) && \ + ninja install && \ + echo "${NIXL_HOME}/lib/$(uname -m)-linux-gnu" \ + > /etc/ld.so.conf.d/nixl.conf && \ + echo "${NIXL_HOME}/lib/$(uname -m)-linux-gnu/plugins" \ + >> /etc/ld.so.conf.d/nixl.conf && \ + ldconfig + +# Generate the ROCm NIXL wheel. Upstream's generic wheel helper detects CUDA, +# so configure the ROCm wheel variant directly through Meson. +# Exclude libcore and libpull from auditwheel: transitive dependencies +# that are not shipped in the wheel and vary across base images. +RUN cd /opt/nixl && \ + ./contrib/tomlutil.py --wheel-name nixl-rocm pyproject.toml && \ + CC="ccache gcc" CXX="ccache g++" \ + uv build --wheel --no-build-isolation --out-dir /tmp/nixl_wheels \ + --python ${PYTHON_VERSION} \ + -Csetup-args=-Ducx_path=${UCX_HOME} \ + -Csetup-args=-Dwheel_variant=rocm \ + -Csetup-args=-Dbuild_tests=false \ + -Csetup-args=-Dbuild_examples=false && \ + mkdir -p /tmp/nixl_wheels/repaired /app/install && \ + auditwheel repair \ + --exclude 'libamdhip64*' \ + --exclude 'libcore*' \ + --exclude 'libpull*' \ + /tmp/nixl_wheels/nixl_rocm*.whl \ + --plat manylinux_2_34_$(uname -m) \ + --wheel-dir /tmp/nixl_wheels/repaired && \ + ./contrib/wheel_add_ucx_plugins.py \ + --ucx-plugins-dir ${UCX_HOME}/lib/ucx \ + --nixl-plugins-dir ${NIXL_HOME}/lib/$(uname -m)-linux-gnu/plugins \ + /tmp/nixl_wheels/repaired/*.whl && \ + cp /tmp/nixl_wheels/repaired/*.whl /app/install + +# Fetch and verify pinned LMCache source. +FROM base AS lmcache_source +ARG LMCACHE_REPO="https://github.com/LMCache/LMCache.git" +ARG LMCACHE_REF="140819c9d57a975dbc5678a6459a218e544cb58b" +ARG LMCACHE_VERSION="0.5.3" + +# Remove NVIDIA-only dependencies and accept the release image's NumPy version. +RUN printf '%s\n' "${LMCACHE_REF}" | grep -Eq '^[0-9a-f]{40}$' \ + && git clone --depth=1 --branch "v${LMCACHE_VERSION}" \ + "${LMCACHE_REPO}" /app/lmcache \ + && test "$(git -C /app/lmcache rev-parse HEAD)" = "${LMCACHE_REF}" \ + && test "$(grep -Ec '^(cufile-python|nvtx)([<=>[:space:]]|$)' \ + /app/lmcache/requirements/common.txt)" -eq 2 \ + && test "$(grep -Fxc 'numpy<=2.2.6' \ + /app/lmcache/requirements/common.txt)" -eq 1 \ + && sed -i -E \ + -e '/^(cufile-python|nvtx)([<=>[:space:]]|$)/d' \ + -e 's/^numpy<=2\.2\.6$/numpy<=2.3.5/' \ + /app/lmcache/requirements/common.txt \ + && rm -rf /app/lmcache/.git + +FROM base AS build_lmcache +COPY --from=lmcache_source \ + /app/lmcache/requirements/build.txt /tmp/lmcache-build-requirements.txt +RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ + uv pip install --system -r /tmp/lmcache-build-requirements.txt + +ARG LMCACHE_VERSION="0.5.3" +ARG LMCACHE_ROCM_ARCH="gfx942;gfx950" +ARG USE_SCCACHE + +COPY --from=lmcache_source /app/lmcache /app/lmcache +WORKDIR /app/lmcache + +RUN --mount=type=cache,id=vllm-rocm-ccache,target=/root/.cache/ccache \ + set -eu; \ + printf '%s\n' "${LMCACHE_ROCM_ARCH}" \ + | grep -Eq '^gfx[0-9a-z]+(;gfx[0-9a-z]+)*$'; \ + offload_flags="$(printf '%s' "${LMCACHE_ROCM_ARCH}" \ + | sed 's/;/ --offload-arch=/g; s/^/--offload-arch=/')"; \ + if [ "${USE_SCCACHE:-0}" = "1" ]; then \ + export HIP_CLANG_PATH=/opt/sccache-wrappers; \ + fi; \ + mkdir -p /app/install; \ + SETUPTOOLS_SCM_PRETEND_VERSION="${LMCACHE_VERSION}" \ + PYTORCH_ROCM_ARCH="${LMCACHE_ROCM_ARCH}" \ + HIPCC_COMPILE_FLAGS_APPEND="${offload_flags}" \ + BUILD_WITH_HIP=1 CXX=hipcc \ + python3 setup.py bdist_wheel --dist-dir=/app/install; \ + test "$(find /app/install -maxdepth 1 -name '*.whl' | wc -l)" -eq 1 + +RUN --mount=type=tmpfs,target=/tmp/lmcache-wheel \ + set -eu; \ + python3 -m zipfile -e /app/install/*.whl /tmp/lmcache-wheel; \ + c_ops="$(find /tmp/lmcache-wheel -type f -name 'c_ops*.so')"; \ + test "$(find /tmp/lmcache-wheel -type f -name 'c_ops*.so' | wc -l)" -eq 1; \ + expected="$(printf '%s' "${LMCACHE_ROCM_ARCH}" | tr ';' '\n' | sort -u)"; \ + actual="$(llvm-objdump --offloading "${c_ops}" \ + | grep -oE 'gfx[0-9a-z]+' | sort -u)"; \ + if [ "${actual}" != "${expected}" ]; then \ + printf 'LMCache code objects mismatch\nexpected:\n%s\nactual:\n%s\n' \ + "${expected}" "${actual}" >&2; \ + exit 1; \ + fi + +# DeepEP build stage. ROCShmem is provided by TheRock SDK +# ROCSHMEM_DIR set to the SDK install path so DeepEP finds the headers, etc. +FROM base AS build_deepep +ENV ROCSHMEM_DIR=${ROCM_PATH} +ARG DEEPEP_BRANCH="0f63d3e9" +ARG DEEPEP_REPO="https://github.com/ROCm/DeepEP.git" +ARG DEEPEP_NIC="cx7" + +# Build DeepEP wheel. DeepEP looks for rocshmem at ROCSHMEM_DIR. +# DeepEP only supports gfx942 and gfx950, so avoid gfx90a in the default list. +# We pass --rocm-gfx942-fp8-fnuz-max 224 to align the DeepEP dispatch FP8 +# quantization bound with vLLM's own clamp +# (vllm/model_executor/layers/quantization/utils/fp8_utils.py): 224 for E4M3FNUZ +RUN --mount=type=cache,target=/root/.cache/ccache \ + export PYTORCH_ROCM_ARCH="gfx942;gfx950" \ + && git clone ${DEEPEP_REPO} \ + && cd DeepEP \ + && git checkout ${DEEPEP_BRANCH} \ + && LDFLAGS="-fuse-ld=mold" MAX_JOBS="${MAX_JOBS:-$(nproc)}" python3 setup.py --variant rocm --rocm-explicit-ctx --nic ${DEEPEP_NIC} --rocm-gfx942-fp8-fnuz-max 224 bdist_wheel --dist-dir=/app/deep_install + +# MoRI runtime dependencies live in Dockerfile.rocm so NIC backend changes do +# not force users to rebuild the long-lived Dockerfile.rocm_base image. +FROM base AS mori_base +ARG NIC_BACKEND +ARG AINIC_VERSION +ARG UBUNTU_CODENAME +RUN /bin/bash -lc 'set -euo pipefail; \ + \ + install_ainic() { \ + apt-get update && apt-get install -y --no-install-recommends ca-certificates curl gnupg apt-transport-https; \ + rm -rf /var/lib/apt/lists/*; \ + mkdir -p /etc/apt/keyrings; \ + curl -fsSL https://repo.radeon.com/rocm/rocm.gpg.key | gpg --dearmor > /etc/apt/keyrings/amdainic.gpg; \ + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/amdainic.gpg] https://repo.radeon.com/amdainic/pensando/ubuntu/${AINIC_VERSION} ${UBUNTU_CODENAME} main" \ + > /etc/apt/sources.list.d/amdainic.list; \ + apt-get update && apt-get install -y --no-install-recommends \ + libionic-dev \ + ionic-common \ + ; \ + rm -rf /var/lib/apt/lists/*; \ + }; \ + \ + # NOTE: requires FW 235.2.86.0 and kernel drivers on the host: \ + # bnxt-en-dkms=1.10.3.235.2.86.0 bnxt-re-dkms=235.2.86.0 (from packages.broadcom.com PPA) \ + install_bnxt() { \ + install -m 0755 -d /etc/apt/keyrings; \ + curl -fsSL https://packages.broadcom.com/artifactory/api/security/keypair/PackagesKey/public \ + -o /etc/apt/keyrings/broadcom-nic.asc; \ + chmod a+r /etc/apt/keyrings/broadcom-nic.asc; \ + echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/broadcom-nic.asc] https://packages.broadcom.com/artifactory/ethernet-nic-debian-public jammy main" \ + > /etc/apt/sources.list.d/broadcom-nic.list; \ + apt-get update && apt-get install -y --no-install-recommends \ + bnxt-rocelib=235.2.86.0 \ + ; \ + cp -a /usr/local/lib/x86_64-linux-gnu/libbnxt_re* /usr/local/lib/; \ + ldconfig; \ + rm -rf /var/lib/apt/lists/*; \ + }; \ + \ + echo "[MORI] Install MoRI proxy deps"; \ + pip install --quiet --ignore-installed blinker && \ + pip install --quiet quart msgpack aiohttp pyzmq; \ + echo "[MORI] NIC_BACKEND=${NIC_BACKEND}"; \ + \ + # NIC backend deps — mori auto-detects NIC at runtime (MORI_DEVICE_NIC env var override). \ + # Only vendor packages are installed here for dlopen; no compile-time flags needed. \ + case "${NIC_BACKEND}" in \ + none) ;; \ + all) install_ainic; install_bnxt ;; \ + ainic) install_ainic ;; \ + bnxt) install_bnxt ;; \ + *) echo "ERROR: unknown NIC_BACKEND=${NIC_BACKEND}. Use one of: none, ainic, bnxt, all"; exit 2 ;; \ + esac' + +# ----------------------- +# vLLM wheel release build stage (for building distributable wheels) +# This stage pins dependencies to custom ROCm wheel versions and handles version detection +FROM fetch_vllm AS build_vllm_wheel_release + +ARG COMMON_WORKDIR +ARG ROCM_TRITON_KERNELS_COMMIT +COPY --from=vllm-version /vllm-version.txt /tmp/vllm-version.txt +COPY --from=rocm-triton-kernels \ + /opt/rocm-triton-kernels /opt/rocm-triton-kernels +ENV TRITON_KERNELS_SRC_DIR=/opt/rocm-triton-kernels + +# Drop the pre-built Rust artifacts into the source tree. setup.py detects +# them and ships them as-is, skipping the local Rust build. +COPY --from=rust-build ${COMMON_WORKDIR}/vllm/vllm/vllm-rs ${COMMON_WORKDIR}/vllm/vllm/vllm-rs +COPY --from=rust-build ${COMMON_WORKDIR}/vllm/vllm/_rust_*.so ${COMMON_WORKDIR}/vllm/vllm/ + +# Create /install directory for custom wheels +RUN mkdir -p /install + +# Copy custom ROCm wheels from docker/context if they exist +# COPY ensures Docker cache is invalidated when wheels change +# .keep file ensures directory always exists for COPY to work +COPY docker/context/base-wheels/ /tmp/base-wheels/ +# This is how we know if we are building for a wheel release or not. +# If there are not wheels found there, we are not building for a wheel release. +# So we exit with an error. To skip this stage. +RUN if [ -n "$(ls /tmp/base-wheels/*.whl 2>/dev/null)" ]; then \ + echo "Found custom wheels - copying to /install"; \ + cp /tmp/base-wheels/*.whl /install/ && \ + echo "Copied custom wheels:"; \ + ls -lh /install/; \ + else \ + echo "ERROR: No custom wheels found in docker/context/base-wheels/"; \ + echo "Wheel releases require pre-built ROCm wheels."; \ + exit 1; \ + fi + +# GIT_REPO_CHECK: Verify repo is clean and tags are available (for release builds) +# This matches CUDA's Dockerfile behavior for proper version detection via setuptools_scm +ARG GIT_REPO_CHECK=0 +RUN if [ "$GIT_REPO_CHECK" != "0" ]; then \ + echo "Running repository checks..."; \ + cd vllm && bash tools/check_repo.sh; \ + fi + +# Install the packaging helpers before modifying requirements/rocm.txt. +RUN --mount=type=cache,target=/root/.cache/uv \ + grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \ + vllm/cmake/external_projects/triton_kernels.cmake \ + && cd vllm \ + && uv pip install --system setuptools_scm regex + +# Fail if git-based package dependencies are found in requirements files +# (uv doesn't handle git+ URLs well, and packages should be distributed on PyPI) +# Extra notes: pip install is able to handle git+ URLs, but uv doesn't. +RUN echo "Checking for git-based packages in requirements files..." \ + && echo "Checking common.txt for git-based packages:" \ + && if grep -q 'git+' ${COMMON_WORKDIR}/vllm/requirements/common.txt; then \ + echo "ERROR: Git-based packages found in common.txt:"; \ + grep 'git+' ${COMMON_WORKDIR}/vllm/requirements/common.txt; \ + echo "Please publish these packages to PyPI instead of using git dependencies."; \ + exit 1; \ + else \ + echo " ✓ No git-based packages found in common.txt"; \ + fi \ + && echo "Checking rocm.txt for git-based packages:" \ + && if grep -q 'git+' ${COMMON_WORKDIR}/vllm/requirements/rocm.txt; then \ + echo "ERROR: Git-based packages found in rocm.txt:"; \ + grep 'git+' ${COMMON_WORKDIR}/vllm/requirements/rocm.txt; \ + echo "Please publish these packages to PyPI instead of using git dependencies."; \ + exit 1; \ + else \ + echo " ✓ No git-based packages found in rocm.txt"; \ + fi \ + && echo "All requirements files are clean - no git-based packages found" + +# Pin vLLM dependencies to exact versions of custom ROCm wheels +# This ensures 'pip install vllm' automatically installs correct torch/triton/torchvision/amdsmi +COPY tools/vllm-rocm/pin_rocm_dependencies.py /tmp/pin_rocm_dependencies.py +RUN echo "Pinning vLLM dependencies to custom wheel versions..." \ + && python3 /tmp/pin_rocm_dependencies.py /install ${COMMON_WORKDIR}/vllm/requirements/rocm.txt + +# Install dependencies using custom wheels from /install +RUN --mount=type=cache,target=/root/.cache/uv \ + cd vllm \ + && echo "Building vLLM with custom wheels from /install" \ + && uv pip install --system --find-links /install -r requirements/rocm.txt + +# Build wheel using the shared source version to avoid dirty state from modified requirements/rocm.txt +# (setup.py auto-detects ccache/sccache in PATH) +RUN --mount=type=cache,id=vllm-rocm-ccache,target=/root/.cache/ccache \ + cd vllm \ + && export CCACHE_BASEDIR="$PWD" \ + && export SETUPTOOLS_SCM_PRETEND_VERSION="$(cat /tmp/vllm-version.txt)" \ + && echo "Building wheel with version: ${SETUPTOOLS_SCM_PRETEND_VERSION}" \ + && MAX_JOBS="${MAX_JOBS:-$(nproc)}" python3 setup.py bdist_wheel --dist-dir=dist + +FROM scratch AS export_vllm_wheel_release +ARG COMMON_WORKDIR +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/dist/*.whl / +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/requirements /requirements +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/scripts/benchmark_helion_kernels.py /scripts/benchmark_helion_kernels.py +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/ +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_base /docker/Dockerfile.rocm_base +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/ci-rocm.hcl /docker/ci-rocm.hcl +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/docker-bake.hcl /docker/docker-bake.hcl +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/docker-bake-rocm.hcl /docker/docker-bake-rocm.hcl +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/pyproject.toml /pyproject.toml +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/rust /rust +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/rust-toolchain.toml /rust-toolchain.toml +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/vllm/v1 /vllm_v1 + +# ----------------------- +# CI base image (Tier 1) - stable, rarely changing CI dependencies. +# Per-PR test builds pull this as CI_BASE_IMAGE so the test stage only layers +# in the vLLM artifacts for the current commit. +FROM mori_base AS ci_base +ARG COMMON_WORKDIR + +ENV CARGO_HOME=/root/.cargo +ENV RUSTUP_HOME=/root/.rustup +ENV PATH=${CARGO_HOME}/bin:${PATH} + +# Bake the Rust toolchain into ci_base so Rust build/test steps do not +# need to fetch it on each per-commit image build. +COPY --from=rust-toolchain /root/.cargo /root/.cargo +COPY --from=rust-toolchain /root/.rustup /root/.rustup + +# Update rdma-core to support latest rocshmem. +ARG DEEPEP_NIC +RUN if [ "${DEEPEP_NIC}" = "cx7" ] || [ "${DEEPEP_NIC}" = "io" ]; then \ + git clone --branch v62.0 --depth 1 https://github.com/linux-rdma/rdma-core.git /tmp/rdma-core && \ + cd /tmp/rdma-core && \ + mkdir -p build && cd build && \ + cmake -GNinja -DCMAKE_INSTALL_PREFIX=/usr -DNO_MAN_PAGES=1 .. && \ + ninja && ninja install && ldconfig && rm -rf /tmp/rdma-core; \ +fi + +# Install NIXL + DeepEP wheels. +RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \ + --mount=type=bind,from=build_deepep,src=/app/deep_install,target=/deep_install \ + uv pip install --system /nixl_install/*.whl /deep_install/*.whl + +# RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec. +RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \ + librdmacm1 \ + libibverbs1 \ + ibverbs-providers \ + ibverbs-utils \ + util-linux \ + unzip \ + pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \ + libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install torchcodec from source for ROCm/torch ABI compatibility. +COPY tools/install_torchcodec_rocm.sh /tmp/install_torchcodec.sh +RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=cache,target=/root/.cache/pip \ + --mount=type=cache,target=/root/.cache/torchcodec-wheels \ + bash /tmp/install_torchcodec.sh \ + && rm /tmp/install_torchcodec.sh \ + && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Pre-install shared ROCm runtime dependencies. +COPY requirements/common.txt requirements/rocm.txt /tmp/ci-base-requirements/ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -c /etc/rocm-constraints.txt \ + -r /tmp/ci-base-requirements/rocm.txt \ + && rm -rf /tmp/ci-base-requirements + +# Enable fast and less brittle model downloads in tests. +ENV HF_XET_HIGH_PERFORMANCE=1 +ENV HF_HUB_DOWNLOAD_TIMEOUT=60 + +# Keep torch.cuda.is_available() fork-safe (see vllm/env_override.py). +ENV PYTORCH_NVML_BASED_CUDA_CHECK=1 + +# Pre-install vLLM test dependencies. +COPY requirements/test/rocm.txt /tmp/rocm-test-reqs.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -r /tmp/rocm-test-reqs.txt + +# Rebuild fastsafetensors from source so its C++ extension is compiled with +# USE_ROCM and can detect libamdhip64.so at runtime. +RUN --mount=type=cache,target=/root/.cache/pip \ + FASTSAFETENSORS_REQ="$(grep -E '^fastsafetensors(==| @ )' /tmp/rocm-test-reqs.txt | head -1)" \ + && test -n "${FASTSAFETENSORS_REQ}" \ + && python3 -m pip install --force-reinstall --no-deps \ + --no-binary fastsafetensors "${FASTSAFETENSORS_REQ}" \ + && rm /tmp/rocm-test-reqs.txt + +# Set MIOPEN ENVS to resolve performance regressions in MIOpen 3D convolution kernel. +# See: https://github.com/pytorch/pytorch/issues/169857 +ENV MIOPEN_DEBUG_CONV_DIRECT=0 +ENV MIOPEN_DEBUG_CONV_GEMM=0 + +# Use legacy IPC mode for HSA to avoid GPU memory pinning issues with UCX rocm_ipc. +# See: https://github.com/ROCm/rocm-libraries/issues/6266 +ENV HSA_ENABLE_IPC_MODE_LEGACY=1 +ENV UCX_RMA_PPLN_ENABLE=y +ENV UCX_ROCM_COPY_SIGPOOL_MAX_ELEMS=inf + +# ROCm profiler limits workaround. +RUN echo "ROCTRACER_MAX_EVENTS=10000000" > ${COMMON_WORKDIR}/libkineto.conf +ENV KINETO_CONFIG="${COMMON_WORKDIR}/libkineto.conf" + +# Install vllm_test_utils in ci_base for ci_base + wheel parity. +COPY tests/vllm_test_utils /tmp/vllm_test_utils +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system /tmp/vllm_test_utils \ + && rm -rf /tmp/vllm_test_utils + +RUN --mount=type=bind,from=build_lmcache,src=/app/install,target=/lmcache_install \ + --mount=type=bind,source=requirements/test/rocm.txt,target=/tmp/lmcache-constraints.txt \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system --constraint /tmp/lmcache-constraints.txt \ + /lmcache_install/*.whl \ + && python3 -c "import lmcache, lmcache.c_ops; print('lmcache', lmcache.__version__)" + +# ----------------------- +# Test vLLM image (Tier 2) - vLLM-only layer on top of ci_base. +FROM ${CI_BASE_IMAGE} AS test +ARG COMMON_WORKDIR + +# Install the vLLM wheel (--no-deps: all deps already in ci_base). +RUN --mount=type=bind,from=export_vllm,src=/,target=/install \ + --mount=type=cache,target=/root/.cache/uv \ + cd /install \ + && uv pip install --system --no-deps *.whl + +# Store the vLLM wheel in the image for python-only install tests. +COPY --from=export_vllm /*.whl /opt/vllm-wheels/ + +WORKDIR /vllm-workspace +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm /vllm-workspace + +# Copy in the v1 package (for python-only install test group). +COPY --from=export_vllm /vllm_v1 /usr/local/lib/python${PYTHON_VERSION}/dist-packages/vllm/v1 + +# Hide source under src/ so it won't shadow the installed package in tests. +RUN mkdir src && mv vllm src/vllm + +# Catch GPU<->CPU syncs in execute_model/sample_tokens during tests. +ENV VLLM_GPU_SYNC_CHECK=error + +# ----------------------- +# Final vLLM image +FROM mori_base AS final_common + +RUN python3 -m pip install --upgrade pip && rm -rf /var/lib/apt/lists/* + +# Clean up sccache from release image (not needed at runtime) +# This removes the binary and wrappers that may have been installed during build +RUN rm -f /usr/bin/sccache || true \ + && rm -rf /opt/sccache-wrappers || true + +# Unset sccache environment variables for the release image +# This prevents S3 bucket config from leaking into production images +ENV SCCACHE_BUCKET= +ENV SCCACHE_REGION= +ENV SCCACHE_ENDPOINT= +ENV SCCACHE_S3_NO_CREDENTIALS= +ENV SCCACHE_IDLE_TIMEOUT= + +# Error related to odd state for numpy 1.20.3 where there is no METADATA etc, but an extra LICENSES_bundled.txt. +# Manually remove it so that later steps of numpy upgrade can continue +RUN case "$(which python3)" in \ + *"/opt/conda/envs/py_3.9"*) \ + rm -rf /opt/conda/envs/py_3.9/lib/python3.9/site-packages/numpy-1.20.3.dist-info/;; \ + *) ;; esac + +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system --upgrade huggingface-hub[cli] + +# Install vLLM using uv (inherited from base stage) +# Note: No -U flag to avoid upgrading PyTorch ROCm to CUDA version +RUN --mount=type=bind,from=export_vllm,src=/,target=/install \ + --mount=type=cache,target=/root/.cache/uv \ + cd /install \ + && uv pip install --system -c /etc/rocm-constraints.txt -r requirements/rocm.txt \ + && pip uninstall -y vllm \ + && uv pip install --system -c /etc/rocm-constraints.txt *.whl + +# Install NIXL ROCm wheel +RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \ + uv pip install --system /nixl_install/*.whl + +# Recompute the base's triton fingerprint to detect resolver interference. +RUN EXPECTED=$(grep '^TRITON_SHA256: ' /app/versions.txt | cut -d' ' -f2) \ + && ACTUAL=$(find $(python3 -c 'import triton, os; print(os.path.dirname(triton.__file__))') \ + -type f \( -name '*.py' -o -name '*.so' \) | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) \ + && echo "triton sha256 expected=${EXPECTED} actual=${ACTUAL}" \ + && test -n "${EXPECTED}" \ + && test "${EXPECTED}" = "${ACTUAL}" \ + && python3 -c "import torch; assert torch.version.hip, f'non-ROCm torch: {torch.__version__}'; print(f'torch={torch.__version__}')" + +# Recompute the base's pybind11 fingerprint to detect resolver interference. +RUN EXPECTED=$(grep '^PYBIND11_SHA256: ' /app/versions.txt | cut -d' ' -f2) \ + && EXPECTED_ABI=$(grep '^PYBIND11_INTERNALS_VERSION: ' /app/versions.txt | cut -d' ' -f2) \ + && INC=$(python3 -c 'import pybind11; print(pybind11.get_include())') \ + && ACTUAL=$(find "${INC}" -type f -name '*.h' | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) \ + && ACTUAL_ABI=$(grep -rhE '^#[[:space:]]*define PYBIND11_INTERNALS_VERSION' "${INC}" | awk '{print $NF}') \ + && echo "pybind11 $(pip show pybind11 | awk '/^Version:/ {print $2}')" \ + "sha256 expected=${EXPECTED} actual=${ACTUAL}" \ + "internals expected=${EXPECTED_ABI} actual=${ACTUAL_ABI}" \ + && test -n "${EXPECTED}" \ + && test "${EXPECTED}" = "${ACTUAL}" \ + && test "${EXPECTED_ABI}" = "${ACTUAL_ABI}" + +ARG COMMON_WORKDIR +ARG BASE_IMAGE +ARG NIC_BACKEND +ARG AINIC_VERSION + +# Copy over the benchmark scripts as well +COPY --from=export_vllm /benchmarks ${COMMON_WORKDIR}/vllm/benchmarks +COPY --from=export_vllm /examples ${COMMON_WORKDIR}/vllm/examples +COPY --from=export_vllm /docker ${COMMON_WORKDIR}/vllm/docker + +# Use legacy IPC mode for HSA to avoid GPU memory pinning issues with UCX rocm_ipc +# See: https://github.com/ROCm/rocm-libraries/issues/6266 +ENV HSA_ENABLE_IPC_MODE_LEGACY=1 +ENV UCX_RMA_PPLN_ENABLE=y +ENV UCX_ROCM_COPY_SIGPOOL_MAX_ELEMS=inf + +ENV TOKENIZERS_PARALLELISM=false + +# ENV that can improve safe tensor loading, and end-to-end time +ENV SAFETENSORS_FAST_GPU=1 + +# Performance environment variable. +ENV HIP_FORCE_DEV_KERNARG=1 + +# Keep torch.cuda.is_available() fork-safe (see vllm/env_override.py). +ENV PYTORCH_NVML_BASED_CUDA_CHECK=1 + +# Workaround for ROCm profiler limits +RUN echo "ROCTRACER_MAX_EVENTS=10000000" > ${COMMON_WORKDIR}/libkineto.conf +ENV KINETO_CONFIG="${COMMON_WORKDIR}/libkineto.conf" +RUN echo "VLLM_BASE_IMAGE=${BASE_IMAGE}" >> ${COMMON_WORKDIR}/versions.txt \ + && echo "MORI_NIC_BACKEND=${NIC_BACKEND}" >> ${COMMON_WORKDIR}/versions.txt \ + && echo "AINIC_VERSION=${AINIC_VERSION}" >> ${COMMON_WORKDIR}/versions.txt + +CMD ["/bin/bash"] + +FROM final_common AS final_lmcache_false + +FROM final_common AS final_lmcache_true +RUN --mount=type=bind,from=build_lmcache,src=/app/install,target=/lmcache_install \ + --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system /lmcache_install/*.whl \ + && test "$(python3 -c 'import numpy; print(numpy.__version__)')" = "2.3.5" \ + && uv pip check --system \ + && python3 -c "import lmcache, lmcache.c_ops; print('lmcache', lmcache.__version__)" + +FROM final_lmcache_${INSTALL_LMCACHE} AS final + +#Set entrypoint for vllm-openai official images +FROM final AS vllm-openai +ENTRYPOINT ["vllm", "serve"] diff --git a/docker/Dockerfile.rock_base b/docker/Dockerfile.rock_base new file mode 100644 index 0000000000..d5e507b62d --- /dev/null +++ b/docker/Dockerfile.rock_base @@ -0,0 +1,360 @@ +ARG BASE_IMAGE=ubuntu:22.04 +ARG PYTHON_VERSION=3.12 +ARG TORCH_VERSION=2.12.0+rocm7.14.0 +ARG TORCHVISION_VERSION=0.27.0+rocm7.14.0 +ARG TORCHAUDIO_VERSION=2.11.0+rocm7.14.0 +ARG TRITON_BRANCH="f0b55c07da61c71775bef6d1a15ebf846430ac75" # release/internal/3.7.x as of 08/13 +ARG TRITON_REPO="https://github.com/ROCm/triton.git" +ARG FA_BRANCH="0e60e394" +ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" +ARG AITER_BRANCH="v0.1.19" +ARG AITER_REPO="https://github.com/ROCm/aiter.git" +ARG MORI_BRANCH="f95cd5e0ca7b730f92f9ffdfb358484d6965d9f7" # TheRock compatible v1.1.0 +ARG MORI_REPO="https://github.com/ROCm/mori.git" + + +# TheRock configuration +ARG SITE_PACKAGES=/usr/local/lib/python${PYTHON_VERSION}/dist-packages +ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL="https://repo.amd.com/rocm/whl-multi-arch/" # For pre-built GPU images, packages, etc. +ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL="https://rocm.nightlies.amd.com/whl-multi-arch/" # For torch, triton, etc. wheels with latest fixes. +ARG ROCM_SDK_VERSION=7.14.0 + +# Build configuration +ARG PYTORCH_ROCM_ARCH="gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151" +ARG AITER_ROCM_ARCH="gfx942;gfx950" + +# Sccache configuration (only used in release pipeline) +ARG USE_SCCACHE +ARG SCCACHE_DOWNLOAD_URL +ARG SCCACHE_ENDPOINT +ARG SCCACHE_BUCKET_NAME=vllm-build-sccache +ARG SCCACHE_REGION_NAME=us-west-2 +ARG SCCACHE_S3_NO_CREDENTIALS=0 + +FROM ${BASE_IMAGE} AS base + +ARG PYTORCH_ROCM_ARCH +ENV PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH}" +ARG AITER_ROCM_ARCH +ENV AITER_ROCM_ARCH="${AITER_ROCM_ARCH}" + +ARG SITE_PACKAGES +ENV SITE_PACKAGES=${SITE_PACKAGES} +ENV SDK_DEV="$SITE_PACKAGES/_rocm_sdk_devel" \ + SDK_CORE="$SITE_PACKAGES/_rocm_sdk_core" +ENV ROCM_PATH="$SDK_DEV" \ + ROCM_HOME="$SDK_DEV" \ + HIP_PATH="$SDK_DEV" \ + HIP_CLANG_PATH="$SDK_DEV/lib/llvm/bin" \ + HIP_DEVICE_LIB_PATH="$SDK_CORE/lib/llvm/amdgcn/bitcode" +ENV LD_LIBRARY_PATH="$SDK_DEV/lib:$SDK_DEV/lib/rocm_sysdeps/lib:$SDK_CORE/lib:$SDK_CORE/lib/rocm_sysdeps/lib" +ENV PATH="$SDK_DEV/bin:$SDK_DEV/lib/llvm/bin:$SDK_CORE/bin:$PATH" + +# Required for RCCL in ROCm7.1 +ENV HSA_NO_SCRATCH_RECLAIM=1 + +ARG PYTHON_VERSION +ENV PYTHON_VERSION=${PYTHON_VERSION} + +RUN mkdir -p /app +WORKDIR /app +ENV DEBIAN_FRONTEND=noninteractive + +# Install Python and other dependencies +RUN apt-get update -y \ + && apt-get install -y software-properties-common build-essential git curl sudo vim less libgfortran5 libopenmpi-dev libpci-dev liblzma-dev pkg-config libdrm-dev libnuma-dev kmod \ + && for i in 1 2 3; do \ + add-apt-repository -y ppa:deadsnakes/ppa && break || \ + { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \ + done \ + && apt-get update -y \ + && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \ + python${PYTHON_VERSION}-lib2to3 python-is-python3 \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \ + && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \ + && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \ + && python3 --version && python3 -m pip --version + +RUN pip install -U packaging 'cmake<4' ninja wheel 'setuptools<80' pybind11==3.0.4 Cython +RUN apt-get update && apt-get install -y libjpeg-dev libsox-dev libsox-fmt-all sox && rm -rf /var/lib/apt/lists/* + +# Install ROCm SDK via TheRock +ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL +ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL +ARG ROCM_SDK_VERSION +RUN pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ + --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \ + "rocm[libraries,devel]${ROCM_SDK_VERSION:+==${ROCM_SDK_VERSION}}" \ + && rocm-sdk init + +# Install PyTorch wheels (nightly for latest torch, release as fallback) +ARG TORCH_VERSION +ARG TORCHAUDIO_VERSION +ARG TORCHVISION_VERSION +RUN pip install \ + --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ + --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \ + "torch==${TORCH_VERSION}" \ + "torchaudio==${TORCHAUDIO_VERSION}" \ + "torchvision==${TORCHVISION_VERSION}" +RUN python3 -c "import torch; print(f'torch={torch.__version__}, hip={torch.version.hip}')" + +# Install the AMD-compiled device kernels corresponding to PYTORCH_ROCM_ARCH: + +# Possible values that can be in PYTORCH_ROCM_ARCH: +# +# gfx90a (MI210/250), gfx942 (MI300), gfx950 (MI350), gfx1100 (7900XTX), +# gfx1101 (7800XT), gfx1200 (9070XT), gfx1201 (9070), gfx1150/1151 (Strix APU) + +# For each arch in PYTORCH_ROCM_ARCH, install the precompiled GPU code that +# PyTorch needs to run on that specific chip. +SHELL ["/bin/bash", "-c"] +RUN TORCH_VER=$(python3 -c "import torch; print(torch.__version__)") \ + && for arch in $(echo "${PYTORCH_ROCM_ARCH}" | tr ';' ' '); do \ + echo "Installing device packages for ${arch}" \ + && pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ + --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \ + amd-torch-device-${arch}==${TORCH_VER} \ + rocm-sdk-device-${arch}==${ROCM_SDK_VERSION}; \ + done + +# Remove gfx90a xnack kpacks (ASAN artifacts that cause hipErrorInvalidImage on MI250) +RUN rm -f "${SITE_PACKAGES}/torch/.kpack/torch_gfx90a:xnack+.kpack" \ + "${SITE_PACKAGES}/torch/.kpack/torch_gfx90a:xnack-.kpack" + +RUN cd $SITE_PACKAGES/_rocm_sdk_core/share/amd_smi && pip install . +ENV PYTHONPATH=$SITE_PACKAGES/_rocm_sdk_core/share/amd_smi + +# TheRock runtime fixes +RUN if [ -f "${SITE_PACKAGES}/rocm_sdk/__init__.py" ]; then \ + sed -i 's/rtld_global: bool = True/rtld_global: bool = False/g' "${SITE_PACKAGES}/rocm_sdk/__init__.py"; \ + fi \ + && if [ -f "${SITE_PACKAGES}/_rocm_sdk_core/lib/librocm_smi64.so.1" ] \ + && [ ! -e "${ROCM_PATH}/lib/librocm_smi64.so.7" ]; then \ + ln -sfn "${SITE_PACKAGES}/_rocm_sdk_core/lib/librocm_smi64.so.1" "${ROCM_PATH}/lib/librocm_smi64.so.7"; \ + fi \ + && printf '%s\n%s\n' "${ROCM_PATH}/lib" "${SITE_PACKAGES}/_rocm_sdk_core/lib" > /etc/ld.so.conf.d/rocm-pip.conf \ + && for f in ${SITE_PACKAGES}/_rocm_sdk_core/lib/libamdhip64.so*; do \ + bn=$(basename "$f"); \ + if [ -f "${ROCM_PATH}/lib/${bn}" ] && [ -f "$f" ] && [ ! -L "$f" ]; then \ + rm -f "$f"; \ + ln -sfn "${ROCM_PATH}/lib/${bn}" "$f"; \ + fi; \ + done \ + && ldconfig + +# Install sccache if USE_SCCACHE is enabled (for release builds) +ARG USE_SCCACHE +ARG SCCACHE_DOWNLOAD_URL +ARG SCCACHE_ENDPOINT +ARG SCCACHE_BUCKET_NAME +ARG SCCACHE_REGION_NAME +ARG SCCACHE_S3_NO_CREDENTIALS +RUN if [ "$USE_SCCACHE" = "1" ]; then \ + echo "Installing sccache..." \ + && SCCACHE_ARCH="x86_64" \ + && SCCACHE_VERSION="v0.8.1" \ + && SCCACHE_DL_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \ + && curl -L -o /tmp/sccache.tar.gz ${SCCACHE_DL_URL} \ + && tar -xzf /tmp/sccache.tar.gz -C /tmp \ + && mv /tmp/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \ + && chmod +x /usr/bin/sccache \ + && rm -rf /tmp/sccache.tar.gz /tmp/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl \ + && sccache --version; \ + fi + +# Setup sccache for HIP compilation via HIP_CLANG_PATH +# This creates wrapper scripts in a separate directory and points HIP to use them +# This avoids modifying the original ROCm binaries which can break detection +# NOTE: HIP_CLANG_PATH is NOT set as ENV to avoid affecting downstream images (Dockerfile.rocm) +# Instead, each build stage should export HIP_CLANG_PATH=/opt/sccache-wrappers if USE_SCCACHE=1 +RUN if [ "$USE_SCCACHE" = "1" ]; then \ + echo "Setting up sccache wrappers for HIP compilation..." \ + && mkdir -p /opt/sccache-wrappers \ + && printf '#!/bin/bash\nexec sccache %s/lib/llvm/bin/clang++ "$@"\n' "${ROCM_PATH}" > /opt/sccache-wrappers/clang++ \ + && chmod +x /opt/sccache-wrappers/clang++ \ + && printf '#!/bin/bash\nexec sccache %s/lib/llvm/bin/clang "$@"\n' "${ROCM_PATH}" > /opt/sccache-wrappers/clang \ + && chmod +x /opt/sccache-wrappers/clang \ + && echo "sccache wrappers created in /opt/sccache-wrappers"; \ + fi + +ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}} +ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}} +ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}} +ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0} + + +### +### Triton Build +### +FROM base AS build_triton +ARG TRITON_BRANCH +ARG TRITON_REPO +RUN git clone ${TRITON_REPO} +RUN cd triton \ + && git checkout ${TRITON_BRANCH} \ + && if [ ! -f setup.py ]; then cd python; fi \ + && python3 setup.py bdist_wheel --dist-dir=dist \ + && mkdir -p /app/install && cp dist/*.whl /app/install + + +### +### MORI Build +### +FROM base AS build_mori +ARG MORI_BRANCH +ARG MORI_REPO +RUN git clone ${MORI_REPO} +RUN cd mori \ + && git checkout ${MORI_BRANCH} \ + && git submodule update --init --recursive \ + && ln -sfn ${ROCM_PATH} /opt/rocm \ + && mkdir -p /usr/lib64 && ln -sf /usr/lib/x86_64-linux-gnu/libc.so /usr/lib64/libc.so \ + && python3 setup.py bdist_wheel --dist-dir=dist && ls /app/mori/dist/*.whl +RUN mkdir -p /app/install && cp /app/mori/dist/*.whl /app/install + +### +### FlashAttention Build +### +FROM base AS build_fa +ARG FA_BRANCH +ARG FA_REPO +ARG USE_SCCACHE +RUN git clone ${FA_REPO} +RUN cd flash-attention \ + && git checkout ${FA_BRANCH} \ + && git submodule update --init \ + && if [ "$USE_SCCACHE" = "1" ]; then \ + export HIP_CLANG_PATH=/opt/sccache-wrappers \ + && sccache --show-stats; \ + fi \ + && GPU_ARCHS=$(echo ${PYTORCH_ROCM_ARCH} | sed -e 's/;gfx1[0-9]\{3\}//g') python3 setup.py bdist_wheel --dist-dir=dist \ + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi +RUN mkdir -p /app/install && cp /app/flash-attention/dist/*.whl /app/install + + +### +### AITER Build +### +FROM base AS build_aiter +ARG AITER_BRANCH +ARG AITER_REPO +ARG USE_SCCACHE +RUN git clone --recursive --branch ${AITER_BRANCH} ${AITER_REPO} +RUN cd aiter \ + && git submodule update --init --recursive \ + && pip install -r requirements.txt +# AITER_USE_SYSTEM_TRITON keeps setup.py off the index triton, which would +# otherwise displace the one we built. +RUN pip install pyyaml && cd aiter \ + && if [ "$USE_SCCACHE" = "1" ]; then \ + export HIP_CLANG_PATH=/opt/sccache-wrappers \ + && sccache --show-stats; \ + fi \ + && PREBUILD_KERNELS=1 AITER_USE_SYSTEM_TRITON=1 GPU_ARCHS=${AITER_ROCM_ARCH} python3 setup.py bdist_wheel --dist-dir=dist \ + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \ + && ls /app/aiter/dist/*.whl +RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install + + +### +### Final Build +### + +# Wheel release stage - +# only includes dependencies used by wheel release pipeline +FROM base AS debs_wheel_release +RUN mkdir /app/debs +RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs + +FROM base AS debs +RUN mkdir /app/debs +RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs + +FROM base AS final + +# Remove the torch wheel's Requires-Dist: triton entry in METADATA and the SHA +# hash (its optional) from RECORD. These will cause Python to reinstall +# Triton and a constraints file can't fix it. This doesn't exist for source +# builds, so it wasn't needed there. +RUN TORCH_DIST_INFO=$(ls -d ${SITE_PACKAGES}/torch-*.dist-info) \ + && grep -q '^Requires-Dist: triton' "$TORCH_DIST_INFO/METADATA" \ + && sed -i '/^Requires-Dist: triton/d' "$TORCH_DIST_INFO/METADATA" \ + && sed -i "s|^\(${TORCH_DIST_INFO##*/}/METADATA\),.*|\1,,|" "$TORCH_DIST_INFO/RECORD" + +# Other wheels first, so they resolve against the index triton. Ours goes in last +# with --no-deps, where it can't pull the resolver anywhere. +RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \ + pip install $(ls /install/*.whl | grep -v '/triton-') \ + && pip install --no-deps --force-reinstall /install/triton-*.whl \ + # With the pin gone this must be clean, so any complaint is real. + && ! pip check 2>&1 | grep -iE '^(torch|triton) ' \ + # Create a constraints file to protect the versions we need, especially + # triton, requirements will try to reinstall repeatedly. + && pip list --format=freeze \ + | grep -E '^(torch|triton|torchaudio|torchvision|pybind11)==' > /etc/rocm-constraints.txt \ + && echo "ROCm constraints:" && cat /etc/rocm-constraints.txt \ + && test "$(wc -l < /etc/rocm-constraints.txt)" -eq 5 \ + && grep -q '^pybind11==3\.0\.' /etc/rocm-constraints.txt \ + && python3 -c "import torch; assert torch.version.hip, f'non-ROCm torch: {torch.__version__}'; print(f'torch={torch.__version__}')" \ + && python3 -c "import triton; print(f'triton={triton.__version__}')" \ + && pip list 2>/dev/null | grep -iE '^(torch|triton) ' + +ARG BASE_IMAGE +ARG PYTHON_VERSION +ARG ROCM_SDK_VERSION +ARG TORCH_VERSION +ARG TORCHAUDIO_VERSION +ARG TORCHVISION_VERSION +ARG PYTORCH_ROCM_ARCH +ARG AITER_ROCM_ARCH +ARG TRITON_BRANCH +ARG TRITON_REPO +ARG FA_BRANCH +ARG FA_REPO +ARG AITER_BRANCH +ARG AITER_REPO +ARG MORI_BRANCH +ARG MORI_REPO +# Record hashes for Triton and pybind11 so we can verify later. +RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ + && echo "PYTHON_VERSION: ${PYTHON_VERSION}" >> /app/versions.txt \ + && echo "ROCM_SDK_VERSION: ${ROCM_SDK_VERSION}" >> /app/versions.txt \ + && echo "TORCH_VERSION: ${TORCH_VERSION}" >> /app/versions.txt \ + && echo "TORCHAUDIO_VERSION: ${TORCHAUDIO_VERSION}" >> /app/versions.txt \ + && echo "TORCHVISION_VERSION: ${TORCHVISION_VERSION}" >> /app/versions.txt \ + && echo "PYTORCH_ROCM_ARCH: ${PYTORCH_ROCM_ARCH}" >> /app/versions.txt \ + && echo "AITER_ROCM_ARCH: ${AITER_ROCM_ARCH}" >> /app/versions.txt \ + && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \ + && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \ + && echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \ + && echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \ + && echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \ + && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt \ + && echo "MORI_BRANCH: ${MORI_BRANCH}" >> /app/versions.txt \ + && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt \ + # Fingerprint our triton so Dockerfile.rocm can prove it survived the build. + && echo "TRITON_VERSION: $(pip show triton | awk '/^Version:/ {print $2}')" >> /app/versions.txt \ + && echo "TRITON_SHA256: $(find $(python3 -c 'import triton, os; print(os.path.dirname(triton.__file__))') \ + -type f \( -name '*.py' -o -name '*.so' \) | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)" \ + >> /app/versions.txt \ + && echo "PYBIND11_VERSION: $(pip show pybind11 | awk '/^Version:/ {print $2}')" >> /app/versions.txt \ + && echo "PYBIND11_INTERNALS_VERSION: $(grep -rhE '^#[[:space:]]*define PYBIND11_INTERNALS_VERSION' \ + $(python3 -c 'import pybind11; print(pybind11.get_include())') | awk '{print $NF}')" \ + >> /app/versions.txt \ + && echo "PYBIND11_SHA256: $(find $(python3 -c 'import pybind11; print(pybind11.get_include())') \ + -type f -name '*.h' | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)" \ + >> /app/versions.txt diff --git a/pyproject.toml b/pyproject.toml index 0766645fc7..d689d50d8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -138,6 +138,7 @@ ignore-hidden = false extend-ignore-identifiers-re = [".*[Uu][Ee][0-9][Mm][0-9].*"] [tool.typos.default.extend-identifiers] +a63ede7 = "a63ede7" bbc5b7ede = "bbc5b7ede" NOOPs = "NOOPs" nin_shortcut = "nin_shortcut" diff --git a/requirements/build/rock.txt b/requirements/build/rock.txt new file mode 100644 index 0000000000..ae4697e612 --- /dev/null +++ b/requirements/build/rock.txt @@ -0,0 +1,19 @@ +# Common dependencies +-r ../common.txt + + +--extra-index-url https://repo.amd.com/rocm/whl-multi-arch/ +torch==2.12.0+rocm7.14.0 +torchvision==0.27.0+rocm7.14.0 +torchaudio==2.11.0+rocm7.14.0 +triton==3.7.1+git0263a6a6.rocm7.14.0 +cmake>=3.26.1,<4 +packaging>=26.2 +setuptools>=77.0.3,<80.0.0 +setuptools-scm>=8 +setuptools-rust>=1.9.0 +wheel +jinja2>=3.1.6 +amdsmi==7.0.2 +timm>=1.0.17 +tilelang==0.1.10 diff --git a/requirements/build/rocm.txt b/requirements/build/rocm.txt index e09bdc078b..aedfa66ab0 100644 --- a/requirements/build/rocm.txt +++ b/requirements/build/rocm.txt @@ -2,13 +2,13 @@ -r ../common.txt ---extra-index-url https://download.pytorch.org/whl/rocm7.1 -torch==2.11.0 -torchvision==0.26.0 -torchaudio==2.11.0 -triton==3.6.0 +--extra-index-url https://repo.amd.com/rocm/whl-multi-arch/ +torch==2.12.0+rocm7.14.0 +torchvision==0.27.0+rocm7.14.0 +torchaudio==2.11.0+rocm7.14.0 +triton==3.7.1+git0263a6a6 cmake>=3.26.1,<4 -packaging>=24.2 +packaging>=26.2 setuptools>=77.0.3,<80.0.0 setuptools-scm>=8 setuptools-rust>=1.9.0 diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index a7986b6e04..d5ca6ffd58 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -39,6 +39,7 @@ transformers==5.15.0 tokenizers==0.22.2 schemathesis>=4.0.0 # Required for openai schema test # quantization +bitsandbytes==0.50.0 buildkite-test-collector==0.1.9 tilelang==0.1.10 diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index 526a2490fc..411bc50259 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -72,6 +72,8 @@ azure-storage-blob==12.28.0 # via runai-model-streamer-azure backoff==2.2.1 # via -r requirements/test/rocm.in +bitsandbytes==0.50.0 + # via -r requirements/test/rocm.in black==26.3.1 # via datamodel-code-generator blake3==1.0.8 @@ -566,6 +568,7 @@ numpy==2.2.6 # -r requirements/test/../common.txt # accelerate # albumentations + # bitsandbytes # bm25s # contourpy # cupy-cuda12x @@ -691,10 +694,11 @@ outlines-core==0.2.14 # via # -c requirements/common.txt # -r requirements/test/../common.txt -packaging==26.0 +packaging==26.2 # via # -c requirements/rocm.txt # accelerate + # bitsandbytes # black # datasets # evaluate