[Frontend][RFC] Rust front-end integration (#40848)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
@@ -231,6 +231,63 @@ ARG torch_cuda_arch_list='7.5 8.0 8.6 8.9 9.0 10.0 11.0 12.0+PTX'
|
||||
ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
|
||||
#################### BUILD BASE IMAGE ####################
|
||||
|
||||
#################### RUST BUILD IMAGE ####################
|
||||
# Build the Rust frontend (`vllm-rs`) in a dedicated stage so the main wheel
|
||||
# build stage doesn't need the rust toolchain, protoc, or the rust source.
|
||||
# This stage runs in parallel with csrc-build/extensions-build.
|
||||
FROM ${BUILD_BASE_IMAGE} AS rust-build
|
||||
ARG BUILD_OS
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Install protoc (used by tonic-build/prost-build) and a basic C toolchain
|
||||
# (some rust crates compile C in their build.rs scripts).
|
||||
RUN if [ "${BUILD_OS}" = "manylinux" ]; then \
|
||||
dnf install -y --setopt=install_weak_deps=False \
|
||||
ca-certificates curl git gcc gcc-c++ make \
|
||||
protobuf-compiler protobuf-devel \
|
||||
&& dnf clean all && rm -rf /var/cache/dnf; \
|
||||
else \
|
||||
apt-get update -y \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl git build-essential \
|
||||
protobuf-compiler libprotobuf-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*; \
|
||||
fi
|
||||
|
||||
# Install rustup; the toolchain itself is pinned by rust/rust-toolchain.toml.
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
|
||||
sh -s -- -y --profile minimal --default-toolchain none
|
||||
ENV PATH="/root/.cargo/bin:${PATH}"
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
# Copy only the rust workspace — the binary is the sole artifact we need.
|
||||
COPY rust rust
|
||||
|
||||
# Fail loudly if the rust submodule was not initialized on the host before
|
||||
# `docker build`. Without this check, cargo would emit a confusing error.
|
||||
RUN if [ ! -f rust/Cargo.toml ]; then \
|
||||
echo "ERROR: rust/ submodule is not initialized."; \
|
||||
echo "Run 'git submodule update --init --recursive' on the host before building."; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# Cap cargo parallelism to avoid exhausting the CI host's open-file limit
|
||||
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
|
||||
ENV CARGO_BUILD_JOBS=4
|
||||
|
||||
# Build the release binary. Cache cargo registry/git and target/, but copy the
|
||||
# binary out of the target/ cache mount so it persists into the image layer
|
||||
# for later COPY --from=rust-build.
|
||||
RUN --mount=type=cache,target=/root/.cargo/registry \
|
||||
--mount=type=cache,target=/root/.cargo/git \
|
||||
--mount=type=cache,target=/workspace/rust/target \
|
||||
cd rust \
|
||||
&& cargo build --release --bin vllm-rs --features native-tls-vendored \
|
||||
&& cp target/release/vllm-rs /workspace/vllm-rs
|
||||
#################### RUST BUILD IMAGE ####################
|
||||
|
||||
#################### CSRC BUILD IMAGE ####################
|
||||
FROM base AS csrc-build
|
||||
ARG TARGETPLATFORM
|
||||
@@ -435,6 +492,10 @@ WORKDIR /workspace
|
||||
COPY --from=csrc-build /workspace/dist /precompiled-wheels
|
||||
COPY . .
|
||||
|
||||
# Drop the pre-built rust frontend binary into the source tree. setup.py
|
||||
# detects it and ships it as-is, skipping the local cargo build.
|
||||
COPY --from=rust-build /workspace/vllm-rs vllm/vllm-rs
|
||||
|
||||
ARG GIT_REPO_CHECK=0
|
||||
RUN --mount=type=bind,source=.git,target=.git \
|
||||
if [ "$GIT_REPO_CHECK" != "0" ]; then bash tools/check_repo.sh ; fi
|
||||
@@ -455,6 +516,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=bind,source=.git,target=.git \
|
||||
if [ "${vllm_target_device}" = "cuda" ]; then \
|
||||
export VLLM_USE_PRECOMPILED=1; \
|
||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=$(ls /precompiled-wheels/*.whl); \
|
||||
fi && \
|
||||
python3 setup.py bdist_wheel --dist-dir=dist --py-limited-api=cp38
|
||||
|
||||
@@ -36,6 +36,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 10 --slave /usr/bin/g++ g++ /usr/bin/g++-12 \
|
||||
&& curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
|
||||
# Compiler and linker environment
|
||||
ENV CC=/usr/bin/gcc-12 CXX=/usr/bin/g++-12
|
||||
ENV CCACHE_DIR=/root/.cache/ccache
|
||||
ENV CMAKE_CXX_COMPILER_LAUNCHER=ccache
|
||||
@@ -80,6 +81,51 @@ FROM base-${TARGETARCH} AS base
|
||||
|
||||
RUN echo 'ulimit -c 0' >> ~/.bashrc
|
||||
|
||||
######################### RUST BUILD IMAGE #########################
|
||||
# Build the Rust frontend (`vllm-rs`) in a dedicated stage so the wheel build
|
||||
# stage doesn't need the rust toolchain or protoc. This stage runs in parallel
|
||||
# with the main vllm-build stage.
|
||||
FROM ubuntu:22.04 AS rust-build
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update -y \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl git build-essential \
|
||||
protobuf-compiler libprotobuf-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install rustup; the toolchain itself is pinned by rust/rust-toolchain.toml.
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
|
||||
sh -s -- -y --profile minimal --default-toolchain none
|
||||
ENV PATH="/root/.cargo/bin:${PATH}"
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
# Copy only the rust workspace — the binary is the sole artifact we need.
|
||||
COPY rust rust
|
||||
|
||||
# Fail loudly if the rust submodule was not initialized on the host before
|
||||
# `docker build`. Without this check, cargo would emit a confusing error.
|
||||
RUN if [ ! -f rust/Cargo.toml ]; then \
|
||||
echo "ERROR: rust/ submodule is not initialized."; \
|
||||
echo "Run 'git submodule update --init --recursive' on the host before building."; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# Cap cargo parallelism to avoid exhausting the CI host's open-file limit
|
||||
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
|
||||
ENV CARGO_BUILD_JOBS=4
|
||||
|
||||
# Build the release binary. Cache cargo registry/git and target/, but copy the
|
||||
# binary out of the target/ cache mount so it persists into the image layer
|
||||
# for later COPY --from=rust-build.
|
||||
RUN --mount=type=cache,target=/root/.cargo/registry \
|
||||
--mount=type=cache,target=/root/.cargo/git \
|
||||
--mount=type=cache,target=/workspace/rust/target \
|
||||
cd rust \
|
||||
&& cargo build --release --bin vllm-rs --features native-tls-vendored \
|
||||
&& cp target/release/vllm-rs /workspace/vllm-rs
|
||||
|
||||
######################### BUILD IMAGE #########################
|
||||
FROM base AS vllm-build
|
||||
|
||||
@@ -114,6 +160,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
|
||||
COPY . .
|
||||
|
||||
# Drop the pre-built rust frontend binary into the source tree. setup.py
|
||||
# detects it and ships it as-is, skipping the local cargo build.
|
||||
COPY --from=rust-build /workspace/vllm-rs vllm/vllm-rs
|
||||
|
||||
RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
|
||||
@@ -94,6 +94,47 @@ RUN cat torch_build_versions.txt
|
||||
|
||||
#################### BASE BUILD IMAGE ####################
|
||||
|
||||
#################### RUST BUILD IMAGE ####################
|
||||
# Build the Rust frontend (`vllm-rs`) in a dedicated stage so the wheel build
|
||||
# stage doesn't need the rust toolchain or protoc.
|
||||
FROM ubuntu:22.04 AS rust-build
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update -y \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl git build-essential \
|
||||
protobuf-compiler libprotobuf-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install rustup; the toolchain itself is pinned by rust/rust-toolchain.toml.
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
|
||||
sh -s -- -y --profile minimal --default-toolchain none
|
||||
ENV PATH="/root/.cargo/bin:${PATH}"
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
COPY rust rust
|
||||
|
||||
# Fail loudly if the rust submodule was not initialized on the host before
|
||||
# `docker build`.
|
||||
RUN if [ ! -f rust/Cargo.toml ]; then \
|
||||
echo "ERROR: rust/ submodule is not initialized."; \
|
||||
echo "Run 'git submodule update --init --recursive' on the host before building."; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# Cap cargo parallelism to avoid exhausting the CI host's open-file limit
|
||||
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
|
||||
ENV CARGO_BUILD_JOBS=4
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cargo/registry \
|
||||
--mount=type=cache,target=/root/.cargo/git \
|
||||
--mount=type=cache,target=/workspace/rust/target \
|
||||
cd rust \
|
||||
&& cargo build --release --bin vllm-rs --features native-tls-vendored \
|
||||
&& cp target/release/vllm-rs /workspace/vllm-rs
|
||||
#################### RUST BUILD IMAGE ####################
|
||||
|
||||
#################### WHEEL BUILD IMAGE ####################
|
||||
FROM base AS build
|
||||
ARG TARGETPLATFORM
|
||||
@@ -104,6 +145,10 @@ ENV UV_HTTP_TIMEOUT=500
|
||||
|
||||
COPY . .
|
||||
|
||||
# Drop the pre-built rust frontend binary into the source tree. setup.py
|
||||
# detects it and ships it as-is, skipping the local cargo build.
|
||||
COPY --from=rust-build /workspace/vllm-rs vllm/vllm-rs
|
||||
|
||||
RUN python3 use_existing_torch.py
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
|
||||
@@ -106,14 +106,61 @@ ONBUILD RUN git clone ${VLLM_REPO} \
|
||||
&& cd vllm \
|
||||
&& git fetch -v --prune -- origin ${VLLM_BRANCH} \
|
||||
&& git checkout FETCH_HEAD \
|
||||
&& git submodule update --init --recursive \
|
||||
&& if [ ${VLLM_REPO} != "https://github.com/vllm-project/vllm.git" ] ; then \
|
||||
git remote add upstream "https://github.com/vllm-project/vllm.git" \
|
||||
&& git fetch upstream ; fi
|
||||
FROM fetch_vllm_${REMOTE_VLLM} AS fetch_vllm
|
||||
|
||||
# -----------------------
|
||||
# Rust build stage
|
||||
# Builds the `vllm-rs` frontend in a dedicated stage so the wheel build stages
|
||||
# don't need the rust toolchain or protoc. Runs in parallel with the main wheel
|
||||
# build for faster end-to-end builds.
|
||||
FROM fetch_vllm AS rust-build
|
||||
ARG COMMON_WORKDIR
|
||||
|
||||
# Fail loudly if the rust submodule was not initialized on the host before
|
||||
# `docker build`. The rust frontend source is brought in via the fetch_vllm
|
||||
# stage, so an uninitialized submodule would otherwise produce a confusing
|
||||
# cargo failure.
|
||||
RUN if [ ! -f ${COMMON_WORKDIR}/vllm/rust/Cargo.toml ]; then \
|
||||
echo "ERROR: rust/ submodule is not initialized."; \
|
||||
echo "Run 'git submodule update --init --recursive' on the host before building."; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# protoc is used by tonic-build/prost-build.
|
||||
RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \
|
||||
protobuf-compiler libprotobuf-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install rustup; the toolchain itself is pinned by rust/rust-toolchain.toml.
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
|
||||
sh -s -- -y --profile minimal --default-toolchain none
|
||||
ENV PATH="/root/.cargo/bin:${PATH}"
|
||||
|
||||
# Cap cargo parallelism to avoid exhausting the AMD CI host's open-file limit
|
||||
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
|
||||
ENV CARGO_BUILD_JOBS=4
|
||||
|
||||
# Build the release binary. Cache cargo registry/git, and copy the binary out
|
||||
# so it persists into the image layer for later COPY --from=rust-build.
|
||||
RUN --mount=type=cache,target=/root/.cargo/registry \
|
||||
--mount=type=cache,target=/root/.cargo/git \
|
||||
cd ${COMMON_WORKDIR}/vllm/rust \
|
||||
&& cargo build --release --bin vllm-rs --features native-tls-vendored \
|
||||
&& cp target/release/vllm-rs /tmp/vllm-rs
|
||||
|
||||
# -----------------------
|
||||
# vLLM build stages
|
||||
FROM fetch_vllm AS build_vllm
|
||||
ARG COMMON_WORKDIR
|
||||
|
||||
# Drop the pre-built rust frontend binary into the source tree. setup.py
|
||||
# detects it and ships it as-is, skipping the local cargo build.
|
||||
COPY --from=rust-build /tmp/vllm-rs ${COMMON_WORKDIR}/vllm/vllm/vllm-rs
|
||||
|
||||
# Build vLLM (setup.py auto-detects sccache in PATH)
|
||||
RUN cd vllm \
|
||||
&& python3 -m pip install -r requirements/rocm.txt \
|
||||
@@ -293,6 +340,10 @@ FROM fetch_vllm AS build_vllm_wheel_release
|
||||
|
||||
ARG COMMON_WORKDIR
|
||||
|
||||
# Drop the pre-built rust frontend binary into the source tree. setup.py
|
||||
# detects it and ships it as-is, skipping the local cargo build.
|
||||
COPY --from=rust-build /tmp/vllm-rs ${COMMON_WORKDIR}/vllm/vllm/vllm-rs
|
||||
|
||||
# Create /install directory for custom wheels
|
||||
RUN mkdir -p /install
|
||||
|
||||
|
||||
@@ -1,3 +1,43 @@
|
||||
######################### RUST BUILD IMAGE #########################
|
||||
# Build the Rust frontend (`vllm-rs`) in a dedicated stage so the main image
|
||||
# doesn't need the rust toolchain or protoc. Runs in parallel with vllm-base.
|
||||
FROM ubuntu:22.04 AS rust-build
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update -y \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl git build-essential \
|
||||
protobuf-compiler libprotobuf-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install rustup; the toolchain itself is pinned by rust/rust-toolchain.toml.
|
||||
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
|
||||
sh -s -- -y --profile minimal --default-toolchain none
|
||||
ENV PATH="/root/.cargo/bin:${PATH}"
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
COPY rust rust
|
||||
|
||||
# Fail loudly if the rust submodule was not initialized on the host before
|
||||
# `docker build`.
|
||||
RUN if [ ! -f rust/Cargo.toml ]; then \
|
||||
echo "ERROR: rust/ submodule is not initialized."; \
|
||||
echo "Run 'git submodule update --init --recursive' on the host before building."; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
# Cap cargo parallelism to avoid exhausting the CI host's open-file limit
|
||||
# (rustc spawns enough concurrent processes to hit RLIMIT_NOFILE otherwise).
|
||||
ENV CARGO_BUILD_JOBS=4
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cargo/registry \
|
||||
--mount=type=cache,target=/root/.cargo/git \
|
||||
--mount=type=cache,target=/workspace/rust/target \
|
||||
cd rust \
|
||||
&& cargo build --release --bin vllm-rs --features native-tls-vendored \
|
||||
&& cp target/release/vllm-rs /workspace/vllm-rs
|
||||
|
||||
FROM intel/deep-learning-essentials:2025.3.2-0-devel-ubuntu24.04 AS vllm-base
|
||||
|
||||
WORKDIR /workspace/
|
||||
@@ -99,6 +139,11 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
|
||||
|
||||
COPY . .
|
||||
|
||||
# Drop the pre-built rust frontend binary into the source tree. setup.py
|
||||
# detects it and ships it as-is, skipping the local cargo build.
|
||||
COPY --from=rust-build /workspace/vllm-rs vllm/vllm-rs
|
||||
|
||||
ARG GIT_REPO_CHECK=0
|
||||
RUN --mount=type=bind,source=.git,target=.git \
|
||||
if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh; fi
|
||||
|
||||
Reference in New Issue
Block a user