From 40e065e86a91b312f5b4b20921cde86fa0e577e3 Mon Sep 17 00:00:00 2001 From: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:19:36 +0800 Subject: [PATCH] [Docker] Fix CUTLASS DSL cu13 install order in Dockerfile (#45204) Signed-off-by: Mohammad Miadh Angkad <176301910+mmangkad@users.noreply.github.com> --- docker/Dockerfile | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 300028cfb2..aa4ef3c309 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -218,6 +218,10 @@ COPY requirements/common.txt requirements/common.txt COPY requirements/cuda.txt requirements/cuda.txt COPY use_existing_torch.py use_existing_torch.py COPY pyproject.toml pyproject.toml +# nvidia-cutlass-dsl[cu13] installs -libs-base and -libs-cu13 wheels that +# share paths with different content. uv can extract them in either order, +# leaving base files that break CUDA 13 CuTe DSL JIT. +# TODO(mmangkad): Remove this after NVIDIA/cutlass#3259 is fixed. RUN --mount=type=cache,target=/opt/uv/cache \ if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "12" ]; then \ sed -i 's/^nvidia-cutlass-dsl\[cu13\]/nvidia-cutlass-dsl/' requirements/cuda.txt; \ @@ -234,6 +238,13 @@ RUN --mount=type=cache,target=/opt/uv/cache \ else \ uv pip install --python /opt/venv/bin/python3 -r requirements/cuda.txt \ --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.'); \ + fi \ + && if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "13" ]; then \ + CUTLASS_DSL_VERSION=$(uv pip show --python /opt/venv/bin/python3 nvidia-cutlass-dsl 2>/dev/null | awk '/^Version:/{print $2}') && \ + if [ -n "$CUTLASS_DSL_VERSION" ]; then \ + uv pip install --python /opt/venv/bin/python3 --force-reinstall --no-deps \ + "nvidia-cutlass-dsl-libs-cu13==${CUTLASS_DSL_VERSION}"; \ + fi; \ fi # Track PyTorch lib versions used during build and match in downstream instances. @@ -745,6 +756,10 @@ ENV VLLM_ENABLE_CUDA_COMPATIBILITY=0 ARG PYTORCH_CUDA_INDEX_BASE_URL COPY requirements/common.txt /tmp/common.txt COPY requirements/cuda.txt /tmp/requirements-cuda.txt +# nvidia-cutlass-dsl[cu13] installs -libs-base and -libs-cu13 wheels that +# share paths with different content. uv can extract them in either order, +# leaving base files that break CUDA 13 CuTe DSL JIT. +# TODO(mmangkad): Remove this after NVIDIA/cutlass#3259 is fixed. RUN --mount=type=cache,target=/opt/uv/cache \ if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "12" ]; then \ sed -i 's/^nvidia-cutlass-dsl\[cu13\]/nvidia-cutlass-dsl/' /tmp/requirements-cuda.txt; \ @@ -752,6 +767,13 @@ RUN --mount=type=cache,target=/opt/uv/cache \ fi && \ uv pip install --system -r /tmp/requirements-cuda.txt \ --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') && \ + if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "13" ]; then \ + CUTLASS_DSL_VERSION=$(uv pip show --system nvidia-cutlass-dsl 2>/dev/null | awk '/^Version:/{print $2}') && \ + if [ -n "$CUTLASS_DSL_VERSION" ]; then \ + uv pip install --system --force-reinstall --no-deps \ + "nvidia-cutlass-dsl-libs-cu13==${CUTLASS_DSL_VERSION}"; \ + fi; \ + fi && \ rm /tmp/requirements-cuda.txt /tmp/common.txt # Install FlashInfer JIT cache (requires CUDA-version-specific index URL) @@ -842,6 +864,19 @@ RUN --mount=type=bind,from=build,src=/tmp/ep_kernels_workspace/dist,target=/vllm uv pip install --system ep_kernels/dist/*.whl --verbose \ --extra-index-url ${PYTORCH_CUDA_INDEX_BASE_URL}/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d '.') +# nvidia-cutlass-dsl[cu13] installs -libs-base and -libs-cu13 wheels that +# share paths with different content. Force -libs-cu13 last after runtime +# dependency installs so uv cannot leave base files behind. +# TODO(mmangkad): Remove this after NVIDIA/cutlass#3259 is fixed. +RUN --mount=type=cache,target=/opt/uv/cache \ + if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "13" ]; then \ + CUTLASS_DSL_VERSION=$(uv pip show --system nvidia-cutlass-dsl 2>/dev/null | awk '/^Version:/{print $2}') && \ + if [ -n "$CUTLASS_DSL_VERSION" ]; then \ + uv pip install --system --force-reinstall --no-deps \ + "nvidia-cutlass-dsl-libs-cu13==${CUTLASS_DSL_VERSION}"; \ + fi; \ + fi + # Download FlashInfer precompiled cubins AFTER all pip installs are done. # This must run after the vLLM wheel and EP kernels installs above, because # those can reinstall/touch flashinfer packages. Downloading cubins earlier