From fa94450b743766d466021a40e5b5f5f5746c92b9 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Thu, 24 Sep 2026 23:18:45 -0500 Subject: [PATCH] [CI][ROCm] Prevent Model Executor apt stalls (#58607) Signed-off-by: Andreas Karatzas Co-authored-by: Codex --- .buildkite/test-amd.yaml | 1 - .buildkite/test_areas/model_executor.yaml | 5 +++++ docker/Dockerfile.rock | 10 +++++++++- docker/Dockerfile.rocm | 10 +++++++++- docker/Dockerfile.rocm_gfx1250 | 10 +++++++++- 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index e0e4388d6c..8183c16e1e 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -2014,7 +2014,6 @@ steps: - tests/model_executor/test_jit_warmup_cutedsl_launcher.py - tests/model_executor/test_jit_warmup_triton_launcher.py commands: - - apt-get update && apt-get install -y curl libsodium23 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - export PYTHONFAULTHANDLER=1 - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread diff --git a/.buildkite/test_areas/model_executor.yaml b/.buildkite/test_areas/model_executor.yaml index f6905f98ef..a23359b768 100644 --- a/.buildkite/test_areas/model_executor.yaml +++ b/.buildkite/test_areas/model_executor.yaml @@ -48,3 +48,8 @@ steps: - tests/entrypoints/openai/completion/test_tensorizer_entrypoint.py - vllm/_aiter_ops.py - vllm/platforms/rocm.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - export PYTHONFAULTHANDLER=1 + - pytest -v -s model_executor -m '(not slow_test)' --timeout=900 --timeout-method=thread + - pytest -v -s entrypoints/openai/completion/test_tensorizer_entrypoint.py --timeout=900 --timeout-method=thread diff --git a/docker/Dockerfile.rock b/docker/Dockerfile.rock index 08e8ad5c68..babc4db100 100644 --- a/docker/Dockerfile.rock +++ b/docker/Dockerfile.rock @@ -776,13 +776,21 @@ RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \ uv pip install --system /nixl_install/*.whl /deep_install/*.whl # RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec. -RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \ +RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ + && apt-get install -q -y --no-install-recommends \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ librdmacm1 \ libibverbs1 \ ibverbs-providers \ ibverbs-utils \ util-linux \ unzip \ + libsodium23 \ pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \ libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index c0671bbf16..5170a71081 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -797,13 +797,21 @@ RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \ COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem # RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec. -RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \ +RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ + && apt-get install -q -y --no-install-recommends \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ librdmacm1 \ libibverbs1 \ ibverbs-providers \ ibverbs-utils \ util-linux \ unzip \ + libsodium23 \ pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \ libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.rocm_gfx1250 b/docker/Dockerfile.rocm_gfx1250 index 916f417c56..ac6dcdff0a 100644 --- a/docker/Dockerfile.rocm_gfx1250 +++ b/docker/Dockerfile.rocm_gfx1250 @@ -560,11 +560,19 @@ RUN --mount=type=bind,from=build_rixl,src=/app/install,target=/rixl_install \ COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem # RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec. -RUN apt-get update -q -y && apt-get install -q -y --no-install-recommends \ +RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ + && apt-get install -q -y --no-install-recommends \ + -o Acquire::Retries=3 \ + -o Acquire::http::Timeout=30 \ + -o Acquire::https::Timeout=30 \ librdmacm1 \ libibverbs1 \ ibverbs-providers \ ibverbs-utils \ + libsodium23 \ pkg-config ffmpeg libavcodec-dev libavformat-dev libavutil-dev \ libswscale-dev libavdevice-dev libavfilter-dev libswresample-dev \ && rm -rf /var/lib/apt/lists/*