From 76ba32160a501e5e8aadb5e1820c51c765714220 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Wed, 2 Sep 2026 02:16:05 -0700 Subject: [PATCH] [CI] Shard CPU jobs above the 24h P90 threshold (#54751) Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> --- .buildkite/hardware_tests/cpu.yaml | 22 +++--- .../scripts/hardware_ci/run-cpu-test-arm.sh | 77 +++++++++++-------- .../test_cpu_linear_attn_chunked_prefix.py | 2 + 3 files changed, 61 insertions(+), 40 deletions(-) diff --git a/.buildkite/hardware_tests/cpu.yaml b/.buildkite/hardware_tests/cpu.yaml index c3806a1795..f4cb2ad0a7 100644 --- a/.buildkite/hardware_tests/cpu.yaml +++ b/.buildkite/hardware_tests/cpu.yaml @@ -62,7 +62,7 @@ steps: # bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 20m " # bash .buildkite/scripts/hardware_ci/run-cpu-compatibility-test.sh" -- label: CPU-Language Generation and Pooling Model Tests +- label: CPU-Language Generation and Pooling Model Tests Shard %N key: cpu-language-generation-and-pooling-model-tests depends_on: [] device: intel_cpu @@ -75,10 +75,9 @@ steps: - tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py commands: - | - bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 50m " - pytest -x -v -s tests/models/language/generation -m cpu_model - pytest -x -v -s tests/models/language/pooling -m cpu_model - pytest -x -v -s tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py" + bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 25m " + pytest -x -v -s tests/models/language/generation tests/models/language/pooling tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py -m cpu_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB" + parallelism: 3 - label: CPU-Quantization Model Tests key: cpu-quantization-model-tests @@ -162,11 +161,16 @@ steps: bash .buildkite/scripts/hardware_ci/run-cpu-test.sh 40m " VLLM_CI_ENV=0 pytest -x -v -s tests/models/multimodal/generation/test_qwen2_5_vl.py" -- label: "Arm CPU Test" +- label: "Arm CPU Test Shard %N" key: arm-cpu-test - depends_on: [] + depends_on: + - cpu-arm64-image-build soft_fail: false device: arm_cpu no_plugin: true - commands: - - bash .buildkite/scripts/hardware_ci/run-cpu-test-arm.sh + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-cpu-test-arm.sh + "$$BUILDKITE_PARALLEL_JOB" + "$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64-cpu" + parallelism: 3 diff --git a/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh b/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh index 7c8ee86eed..052063f2f6 100755 --- a/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh +++ b/.buildkite/scripts/hardware_ci/run-cpu-test-arm.sh @@ -1,8 +1,17 @@ #!/bin/bash -# This script build the CPU docker image and run the offline inference inside the container. -# It serves a sanity check for compilation and basic model usage. -set -ex +# Run the Arm CPU test suites against the image built by cpu-arm64-image-build. +set -euxo pipefail + +SHARD_ID=${1:?Usage: run-cpu-test-arm.sh SHARD_ID IMAGE} +IMAGE=${2:?Usage: run-cpu-test-arm.sh SHARD_ID IMAGE} +if [[ ! "$SHARD_ID" =~ ^[0-2]$ ]]; then + echo "SHARD_ID must be 0, 1, or 2" >&2 + exit 2 +fi + +JOB_SUFFIX=${BUILDKITE_JOB_ID:-local-$SHARD_ID} +CONTAINER_NAME="cpu-test-${JOB_SUFFIX//[^a-zA-Z0-9_.-]/-}" # allow to bind to different cores CORE_RANGE=${CORE_RANGE:-0-31} @@ -13,26 +22,25 @@ export CMAKE_BUILD_PARALLEL_LEVEL=32 # Setup cleanup remove_docker_container() { set -e; - docker rm -f cpu-test || true; + docker rm -f "$CONTAINER_NAME" || true; } trap remove_docker_container EXIT remove_docker_container -# Try building the docker image -docker build --tag cpu-test --target vllm-test -f docker/Dockerfile.cpu . +docker pull "$IMAGE" # Run the image -docker run -itd --cpuset-cpus="$CORE_RANGE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=16 --env VLLM_CPU_CI_ENV=1 -e E2E_OMP_THREADS="$OMP_CORE_RANGE" --shm-size=4g --name cpu-test cpu-test +docker run -itd --cpuset-cpus="$CORE_RANGE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=16 --env VLLM_CPU_CI_ENV=1 -e E2E_OMP_THREADS="$OMP_CORE_RANGE" --shm-size=4g --name "$CONTAINER_NAME" "$IMAGE" -function cpu_tests() { - set -e - - docker exec cpu-test bash -c " +print_packages() { + docker exec "$CONTAINER_NAME" bash -c " set -e pip list" +} - # Run kernel tests - docker exec cpu-test bash -c " +kernel_tests() { + set -e + docker exec "$CONTAINER_NAME" bash -c " set -e pytest -x -v -s tests/kernels/test_onednn.py pytest -x -v -s tests/kernels/attention/test_cpu_attn.py @@ -43,32 +51,37 @@ function cpu_tests() { pytest -x -v -s tests/kernels/mamba/test_cpu_short_conv.py pytest -x -v -s tests/kernels/mamba/test_causal_conv1d.py pytest -x -v -s tests/kernels/mamba/test_mamba_ssm.py" +} - # skip tests requiring model downloads if HF_TOKEN is not set - # due to rate-limits - if [ -z "$HF_TOKEN" ]; then +model_tests() { + set -e + if [ -z "${HF_TOKEN:-}" ]; then echo "Warning: HF_TOKEN is not set. Skipping tests that require model downloads." return fi - # offline inference - docker exec cpu-test bash -c " + docker exec "$CONTAINER_NAME" bash -c " set -e python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m" - # Test encoder-decoder and encoder-only models - docker exec cpu-test bash -c " + docker exec "$CONTAINER_NAME" bash -c " set -e pytest -x -v -s tests/models/multimodal/generation/test_whisper.py -m cpu_model pytest -x -v -s 'tests/models/language/pooling/test_embedding.py::test_models[sentence-transformers/all-MiniLM-L12-v2]'" - # Run quantized model tests - docker exec cpu-test bash -c " + docker exec "$CONTAINER_NAME" bash -c " set -e pytest -x -v -s tests/quantization/test_compressed_tensors.py::test_compressed_tensors_w8a8_logprobs" +} - # basic online serving - docker exec cpu-test bash -c ' +serving_tests() { + set -e + if [ -z "${HF_TOKEN:-}" ]; then + echo "Warning: HF_TOKEN is not set. Skipping tests that require model downloads." + return + fi + + docker exec "$CONTAINER_NAME" bash -c ' set -e VLLM_CPU_OMP_THREADS_BIND=$E2E_OMP_THREADS vllm serve Qwen/Qwen3-0.6B --max-model-len 2048 & server_pid=$! @@ -81,8 +94,7 @@ function cpu_tests() { --endpoint /v1/completions kill -s SIGTERM $server_pid &' - # smoke test for Gated DeltaNet - docker exec cpu-test bash -c ' + docker exec "$CONTAINER_NAME" bash -c ' set -e VLLM_CPU_OMP_THREADS_BIND=$E2E_OMP_THREADS vllm serve Qwen/Qwen3.5-0.8B --max-model-len 2048 & server_pid=$! @@ -94,10 +106,13 @@ function cpu_tests() { --num-prompts 20 \ --endpoint /v1/completions kill -s SIGTERM $server_pid &' - } -# All of CPU tests are expected to be finished less than 40 mins. -export -f cpu_tests -timeout 2h bash -c cpu_tests - +print_packages +export CONTAINER_NAME +export -f kernel_tests model_tests serving_tests +case "$SHARD_ID" in + 0) timeout 30m bash -c kernel_tests ;; + 1) timeout 30m bash -c model_tests ;; + 2) timeout 30m bash -c serving_tests ;; +esac diff --git a/tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py b/tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py index 71484d3b05..1b6a48b7d2 100644 --- a/tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py +++ b/tests/v1/e2e/test_cpu_linear_attn_chunked_prefix.py @@ -10,6 +10,8 @@ from tests.models.utils import check_logprobs_close from vllm import LLM, SamplingParams from vllm.platforms import current_platform +pytestmark = pytest.mark.cpu_model + if not current_platform.is_cpu(): pytest.skip("skipping CPU-only tests", allow_module_level=True)