From 032d39e9b451e858ac9cb18eb0f587ca005361b5 Mon Sep 17 00:00:00 2001 From: xiangdong <40376367+zxd1997066@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:53:37 +0800 Subject: [PATCH] [XPU][CI] Split Intel jobs to improve B50 utilization (#58817) Signed-off-by: zengxian Signed-off-by: xiangdong <40376367+zxd1997066@users.noreply.github.com> --- .buildkite/intel_jobs/lora_intel.yaml | 37 +++++++-- .buildkite/intel_jobs/misc_intel.yaml | 82 +++++++++++++++++-- .../intel_jobs/model_executor_intel.yaml | 48 ++++++++--- .../intel_jobs/models_multimodal_intel.yaml | 13 +-- .buildkite/intel_jobs/samplers_intel.yaml | 29 ++++++- .buildkite/intel_jobs/test-intel.yaml | 35 ++++++-- 6 files changed, 209 insertions(+), 35 deletions(-) diff --git a/.buildkite/intel_jobs/lora_intel.yaml b/.buildkite/intel_jobs/lora_intel.yaml index 2cecbf18f7..885e4b5466 100644 --- a/.buildkite/intel_jobs/lora_intel.yaml +++ b/.buildkite/intel_jobs/lora_intel.yaml @@ -9,7 +9,7 @@ steps: agent_tags: label: production gpu: 1+ - mem: 24+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -33,7 +33,6 @@ steps: pytest -v -s lora/test_peft_helper.py && pytest -v -s lora/test_resolver.py && pytest -v -s lora/test_utils.py && - pytest -v -s lora/test_add_lora.py && pytest -v -s lora/test_worker.py' - label: LoRA Fused/MoE Kernels @@ -120,7 +119,7 @@ steps: agent_tags: label: production gpu: 2+ - mem: 24+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -138,8 +137,7 @@ steps: pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" && pytest -v -s lora/test_transformers_model.py && pytest -v -s lora/test_chatglm3_tp.py && - pytest -v -s lora/test_llama_tp.py::test_llama_lora && - pytest -s -v lora/test_minicpmv_tp.py' + pytest -v -s lora/test_llama_tp.py::test_llama_lora' - label: LoRA Multimodal key: lora-multimodal @@ -148,6 +146,30 @@ steps: agent_tags: label: production gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/lora + - tests/lora + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && + pytest -v -s lora/test_whisper.py' + +- label: LoRA Add + Multimodal + key: lora-add-multimodal + timeout_in_minutes: 45 + device: intel_gpu + agent_tags: + label: production + gpu: 2+ mem: 24+ no_plugin: true working_dir: "." @@ -163,5 +185,6 @@ steps: bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && export VLLM_WORKER_MULTIPROC_METHOD=spawn && - pytest -v -s lora/test_default_mm_loras.py && - pytest -v -s lora/test_whisper.py' + pytest -v -s lora/test_add_lora.py && + pytest -v -s lora/test_default_mm_loras.py && + pytest -s -v lora/test_minicpmv_tp.py' diff --git a/.buildkite/intel_jobs/misc_intel.yaml b/.buildkite/intel_jobs/misc_intel.yaml index 5fa4d833d7..b7ae9f9078 100644 --- a/.buildkite/intel_jobs/misc_intel.yaml +++ b/.buildkite/intel_jobs/misc_intel.yaml @@ -10,7 +10,7 @@ steps: agent_tags: label: production gpu: 2+ - mem: 24+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -34,6 +34,40 @@ steps: key: v1-core-kv-metrics-intel timeout_in_minutes: 45 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/v1/core + - tests/v1/executor + - tests/v1/kv_offload + - tests/v1/worker + - tests/v1/kv_connector/unit + - tests/v1/ec_connector/unit + - tests/v1/metrics + - tests/entrypoints/openai/correctness/test_lmeval.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install -r requirements/kv_connectors.txt && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && + cd tests && + pytest -v -s v1/executor && + VLLM_BATCH_INVARIANT=1 pytest -v -s -m "not cpu_test" v1/ec_connector/unit + --ignore=v1/ec_connector/unit/test_ec_cpu_connector.py' + +- label: EC CPU Connector (XPU) + key: ec-cpu-connector-xpu + timeout_in_minutes: 45 + device: intel_gpu agent_tags: label: production gpu: 1+ @@ -60,8 +94,7 @@ steps: 'pip install -r requirements/kv_connectors.txt && export VLLM_WORKER_MULTIPROC_METHOD=spawn && cd tests && - pytest -v -s v1/executor && - VLLM_BATCH_INVARIANT=1 pytest -v -s -m "not cpu_test" v1/ec_connector/unit' + pytest -v -s v1/ec_connector/unit/test_ec_cpu_connector.py' - label: V1 Sample + Logits key: v1-sample-logits-intel @@ -70,7 +103,7 @@ steps: agent_tags: label: production gpu: 1+ - mem: 24+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -265,7 +298,7 @@ steps: agent_tags: label: production gpu: 1+ - mem: 24+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -293,10 +326,47 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && - pytest -v -s detokenizer && + pytest -v -s detokenizer --ignore=detokenizer/test_stop_strings.py && pytest -v -s -m "not cpu_test" ./multimodal && pytest -v -s utils_ --ignore=utils_/test_mem_utils.py' +- label: Detokenizer Stop Strings (XPU) + key: detokenizer-stop-strings-xpu + timeout_in_minutes: 45 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 24+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/assets/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/detokenizer + - tests/multimodal + - tests/utils_ + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pytest -v -s detokenizer/test_stop_strings.py' + - label: Fusion Unit Tests key: fusion-unit-tests timeout_in_minutes: 30 diff --git a/.buildkite/intel_jobs/model_executor_intel.yaml b/.buildkite/intel_jobs/model_executor_intel.yaml index 98e4b0ff45..5da075e235 100644 --- a/.buildkite/intel_jobs/model_executor_intel.yaml +++ b/.buildkite/intel_jobs/model_executor_intel.yaml @@ -6,6 +6,43 @@ steps: key: model-executor-intel timeout_in_minutes: 45 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/engine/arg_utils.py + - vllm/config/model.py + - vllm/model_executor + - tests/model_executor + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'apt-get update && apt-get install -y curl libsodium23 && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && + export PYTHONFAULTHANDLER=1 && + cd tests && + pytest -v -s model_executor -m "not slow_test" --ignore="model_executor/layers/test_rocm_unquantized_gemm.py" --ignore="model_executor/model_loader/tensorizer_loader/test_tensorizer.py" --deselect="tests/model_executor/model_loader/test_reload.py::test_kv_scale_reload" + --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormModule::test_matches_reference_above_cudnn_grid_limit" + --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormDtypes::test_dtype_combinations[in_dtype0-out_dtype0]" + --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormDtypes::test_dtype_combinations[in_dtype1-out_dtype1]" + --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormDtypes::test_dtype_combinations[in_dtype2-out_dtype2]" + --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormDtypes::test_dtype_combinations[in_dtype3-out_dtype3]" + --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormDtypes::test_dtype_combinations[in_dtype4-out_dtype4]" + --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormDtypes::test_dtype_combinations[in_dtype5-out_dtype5]" + --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormShapes" + --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormInputHandling"' + +- label: Tensorizer Model Loader Tests (XPU) + key: tensorizer-model-loader-tests-xpu + timeout_in_minutes: 45 + device: intel_gpu agent_tags: label: production gpu: 1+ @@ -28,13 +65,4 @@ steps: export VLLM_WORKER_MULTIPROC_METHOD=spawn && export PYTHONFAULTHANDLER=1 && cd tests && - pytest -v -s model_executor -m "not slow_test" --ignore="model_executor/layers/test_rocm_unquantized_gemm.py" --deselect="tests/model_executor/model_loader/test_reload.py::test_kv_scale_reload" - --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormModule::test_matches_reference_above_cudnn_grid_limit" - --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormDtypes::test_dtype_combinations[in_dtype0-out_dtype0]" - --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormDtypes::test_dtype_combinations[in_dtype1-out_dtype1]" - --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormDtypes::test_dtype_combinations[in_dtype2-out_dtype2]" - --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormDtypes::test_dtype_combinations[in_dtype3-out_dtype3]" - --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormDtypes::test_dtype_combinations[in_dtype4-out_dtype4]" - --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormDtypes::test_dtype_combinations[in_dtype5-out_dtype5]" - --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormShapes" - --deselect="tests/model_executor/layers/test_mm_input_norm.py::TestFusedMMInputNormInputHandling"' + pytest -v -s model_executor/model_loader/tensorizer_loader/test_tensorizer.py -m "not slow_test"' diff --git a/.buildkite/intel_jobs/models_multimodal_intel.yaml b/.buildkite/intel_jobs/models_multimodal_intel.yaml index eb36e0c02e..061be1b136 100644 --- a/.buildkite/intel_jobs/models_multimodal_intel.yaml +++ b/.buildkite/intel_jobs/models_multimodal_intel.yaml @@ -9,7 +9,7 @@ steps: agent_tags: label: production gpu: 1+ - mem: 24+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -33,7 +33,7 @@ steps: agent_tags: label: production gpu: 1+ - mem: 24+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -72,7 +72,8 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma"' + pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" && + pytest -v -s models/multimodal/generation/test_gemma_ragged_audio.py -m core_model' - label: "Multi-Modal Models (Standard) 4: other + whisper" key: multi-modal-models-standard-4-other-whisper @@ -81,7 +82,7 @@ steps: agent_tags: label: production gpu: 1+ - mem: 24+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -95,7 +96,7 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && - pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing' + pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_gemma_ragged_audio.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing' - label: Multi-Modal Processor # 44min key: multi-modal-processor @@ -131,7 +132,7 @@ steps: agent_tags: label: production gpu: 1+ - mem: 24+ + mem: 16+ no_plugin: true working_dir: "." env: diff --git a/.buildkite/intel_jobs/samplers_intel.yaml b/.buildkite/intel_jobs/samplers_intel.yaml index 41ec3c4df7..027237225d 100644 --- a/.buildkite/intel_jobs/samplers_intel.yaml +++ b/.buildkite/intel_jobs/samplers_intel.yaml @@ -6,6 +6,33 @@ steps: key: samplers-test-xpu-kernel-intel timeout_in_minutes: 40 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/model_executor/layers + - vllm/sampling_metadata.py + - tests/samplers + - tests/conftest.py + - vllm/entrypoints/generate/beam_search + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + VLLM_XPU_USE_SAMPLER_KERNEL=1 pytest -v -s samplers + --ignore=samplers/test_beam_search.py' + +- label: Samplers Beam Search Test (XPU Kernel) + key: samplers-beam-search-test-xpu-kernel-intel + timeout_in_minutes: 40 + device: intel_gpu agent_tags: label: production gpu: 1+ @@ -26,4 +53,4 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && - VLLM_XPU_USE_SAMPLER_KERNEL=1 pytest -v -s samplers' + VLLM_XPU_USE_SAMPLER_KERNEL=1 pytest -v -s samplers/test_beam_search.py' diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index b1030e372c..a2a0e14be1 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -23,7 +23,7 @@ steps: agent_tags: label: production gpu: 2+ - mem: 24+ + mem: 16+ no_plugin: true env: REGISTRY: "public.ecr.aws/q9t5s3a7" @@ -46,7 +46,6 @@ steps: python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 && python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel && python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 && - VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 && python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC -tp 2 --max-model-len 8192 ' - label: "XPU W8A8 FP8 Linear Examples" @@ -58,7 +57,7 @@ steps: agent_tags: label: production gpu: 1+ - mem: 24+ + mem: 16+ no_plugin: true env: REGISTRY: "public.ecr.aws/q9t5s3a7" @@ -86,7 +85,7 @@ steps: agent_tags: label: production gpu: 1+ - mem: 24+ + mem: 16+ no_plugin: true env: REGISTRY: "public.ecr.aws/q9t5s3a7" @@ -106,8 +105,34 @@ steps: pytest -v -s v1/test_serial_utils.py && pytest -v -s v1/e2e/general/test_correctness_sliding_window.py && pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py --deselect=tests/v1/spec_decode/test_mtp.py::test_glm_mtp_defers_lm_head && - pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector_hma.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]' + pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py --ignore=v1/kv_connector/unit/test_nixl_connector_hma.py --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --deselect=tests/v1/kv_connector/unit/test_nixl_connector.py::test_abort_timeout_on_prefiller[ray]' parallelism: 2 + - label: "XPU Qwen3-30B + V1 KV Connector" + key: xpu-qwen3-30b-v1-kv-connector + depends_on: + - image-build-xpu + timeout_in_minutes: 90 + device: intel_gpu + agent_tags: + label: production + gpu: 2+ + mem: 24+ + no_plugin: true + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + source_file_dependencies: + - vllm/ + - .buildkite/intel_jobs/test-intel.yaml + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pip install dpctl && + pytest -v -s v1/kv_connector/unit/test_multi_connector.py && + pytest -v -s v1/kv_connector/unit/test_example_connector.py && + cd .. && + VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192' - label: "XPU server test" key: xpu-server-test depends_on: