diff --git a/.buildkite/intel_jobs/basic_correctness_intel.yaml b/.buildkite/intel_jobs/basic_correctness_intel.yaml index 292920c47e..173baf40f6 100644 --- a/.buildkite/intel_jobs/basic_correctness_intel.yaml +++ b/.buildkite/intel_jobs/basic_correctness_intel.yaml @@ -3,6 +3,7 @@ depends_on: - image-build-xpu steps: - label: XPU Sleep Mode + key: xpu-sleep-mode timeout_in_minutes: 45 device: intel_gpu agent_tags: diff --git a/.buildkite/intel_jobs/engine_intel.yaml b/.buildkite/intel_jobs/engine_intel.yaml index d0aabd6d32..2619e1c588 100644 --- a/.buildkite/intel_jobs/engine_intel.yaml +++ b/.buildkite/intel_jobs/engine_intel.yaml @@ -41,6 +41,7 @@ steps: pytest -v -s engine/test_arg_utils.py test_sequence.py test_logger.py test_vllm_port.py jit_monitor/test_hooks.py' - label: Engine (1 GPU) + key: engine-1-gpu timeout_in_minutes: 30 device: intel_gpu agent_tags: @@ -65,6 +66,7 @@ steps: VLLM_XPU_ENABLE_XPU_GRAPH=1 pytest -v -s test_config.py' - label: V1 e2e (2 GPUs) + key: v1-e2e-2-gpus timeout_in_minutes: 30 device: intel_gpu agent_tags: diff --git a/.buildkite/intel_jobs/kernels_intel.yaml b/.buildkite/intel_jobs/kernels_intel.yaml index e914c10d8b..da6086c1dc 100644 --- a/.buildkite/intel_jobs/kernels_intel.yaml +++ b/.buildkite/intel_jobs/kernels_intel.yaml @@ -3,6 +3,7 @@ depends_on: - image-build-xpu steps: - label: vLLM IR Tests + key: vllm-ir-tests timeout_in_minutes: 30 device: intel_gpu agent_tags: diff --git a/.buildkite/intel_jobs/lora_intel.yaml b/.buildkite/intel_jobs/lora_intel.yaml index cab0cb66aa..2cecbf18f7 100644 --- a/.buildkite/intel_jobs/lora_intel.yaml +++ b/.buildkite/intel_jobs/lora_intel.yaml @@ -3,6 +3,7 @@ depends_on: - image-build-xpu steps: - label: LoRA Runtime + Utils + key: lora-runtime-utils timeout_in_minutes: 45 device: intel_gpu agent_tags: @@ -36,6 +37,7 @@ steps: pytest -v -s lora/test_worker.py' - label: LoRA Fused/MoE Kernels + key: lora-fused-moe-kernels timeout_in_minutes: 45 device: intel_gpu agent_tags: @@ -60,6 +62,7 @@ steps: pytest -v -s lora/test_moe_lora_align_sum.py --deselect="tests/lora/test_moe_lora_align_sum.py::test_moe_lora_align_block_size_mixed_base_and_lora[1]"' - label: LoRA Punica Kernels + key: lora-punica-kernels timeout_in_minutes: 45 device: intel_gpu agent_tags: @@ -86,6 +89,7 @@ steps: pytest -v -s lora/test_punica_ops.py::test_add_lora_fused_moe_early_exit' - label: LoRA Punica FP8/XPU Ops + key: lora-punica-fp8-xpu-ops timeout_in_minutes: 60 device: intel_gpu agent_tags: @@ -110,6 +114,7 @@ steps: pytest -v -s lora/test_punica_xpu_ops.py' - label: LoRA Models + key: lora-models timeout_in_minutes: 45 device: intel_gpu agent_tags: @@ -137,6 +142,7 @@ steps: pytest -s -v lora/test_minicpmv_tp.py' - label: LoRA Multimodal + key: lora-multimodal timeout_in_minutes: 45 device: intel_gpu agent_tags: diff --git a/.buildkite/intel_jobs/misc_intel.yaml b/.buildkite/intel_jobs/misc_intel.yaml index 752870abe3..3818339662 100644 --- a/.buildkite/intel_jobs/misc_intel.yaml +++ b/.buildkite/intel_jobs/misc_intel.yaml @@ -3,6 +3,7 @@ depends_on: - image-build-xpu steps: - label: V1 Core + KV + Metrics + key: v1-core-kv-metrics-intel timeout_in_minutes: 45 device: intel_gpu agent_tags: @@ -32,9 +33,10 @@ steps: export VLLM_WORKER_MULTIPROC_METHOD=spawn && cd tests && pytest -v -s v1/executor && - VLLM_BATCH_INVARIANT=1 pytest -v -s -m 'not cpu_test' v1/ec_connector/unit' + VLLM_BATCH_INVARIANT=1 pytest -v -s -m "not cpu_test" v1/ec_connector/unit' - label: V1 Sample + Logits + key: v1-sample-logits-intel timeout_in_minutes: 90 device: intel_gpu agent_tags: @@ -80,6 +82,7 @@ steps: parallelism: 4 - label: Basic Models Tests (Initialization) + key: basic-models-tests-initialization timeout_in_minutes: 60 device: intel_gpu agent_tags: @@ -104,6 +107,7 @@ steps: pytest -v -s models/test_initialization.py::test_can_initialize_large_subset[Eagle3MiniMaxM2ForCausalLM]' - label: XPU CPU Offload + key: xpu-cpu-offload timeout_in_minutes: 60 device: intel_gpu agent_tags: @@ -131,6 +135,7 @@ steps: pytest -v -s v1/kv_connector/unit/test_offloading_connector.py' - label: NixlConnector PD accuracy (4 GPUs) + key: nixlconnector-pd-accuracy-4-gpus timeout_in_minutes: 60 num_devices: 4 device: intel_gpu @@ -269,6 +274,7 @@ steps: pytest -v -s utils_ --ignore=utils_/test_mem_utils.py' - label: Fusion Unit Tests + key: fusion-unit-tests timeout_in_minutes: 30 device: intel_gpu agent_tags: diff --git a/.buildkite/intel_jobs/model_runner_v2_intel.yaml b/.buildkite/intel_jobs/model_runner_v2_intel.yaml index 1f68333e97..7f6767eba2 100644 --- a/.buildkite/intel_jobs/model_runner_v2_intel.yaml +++ b/.buildkite/intel_jobs/model_runner_v2_intel.yaml @@ -3,6 +3,7 @@ depends_on: - image-build-xpu steps: - label: Model Runner V2 Core Tests (Intel) + key: model-runner-v2-core-tests-intel timeout_in_minutes: 45 device: intel_gpu agent_tags: @@ -34,6 +35,7 @@ steps: pytest -v -s v1/e2e/general/test_min_tokens.py' - label: Model Runner V2 Examples (Intel) + key: model-runner-v2-examples-intel timeout_in_minutes: 45 device: intel_gpu agent_tags: @@ -64,6 +66,7 @@ steps: python3 features/automatic_prefix_caching/prefix_caching_offline.py' - label: Model Runner V2 Distributed (2 GPUs) + key: model-runner-v2-distributed-2-gpus timeout_in_minutes: 50 device: intel_gpu agent_tags: @@ -90,6 +93,7 @@ steps: TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -m "distributed\(num_gpus=2\)" -k "not ray and not True"' - label: Model Runner V2 Spec Decode + key: model-runner-v2-spec-decode timeout_in_minutes: 50 device: intel_gpu agent_tags: diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index 0bc25261a2..00f172ca88 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -15,6 +15,7 @@ steps: - exit_status: -10 # Agent was lost limit: 2 - label: "XPU example Test" + key: xpu-example-test depends_on: - image-build-xpu timeout_in_minutes: 50 @@ -50,6 +51,7 @@ steps: python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192 ' - label: "XPU W8A8 FP8 Linear Examples" + key: xpu-w8a8-fp8-linear-examples depends_on: - image-build-xpu timeout_in_minutes: 60 @@ -77,6 +79,7 @@ steps: python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 ' - label: "XPU V1 test" + key: xpu-v1-test depends_on: - image-build-xpu timeout_in_minutes: 70 @@ -107,6 +110,7 @@ steps: pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py' parallelism: 2 - label: "XPU server test" + key: xpu-server-test depends_on: - image-build-xpu timeout_in_minutes: 45 @@ -130,6 +134,7 @@ steps: pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py && pytest -v -s benchmarks/test_serve_cli.py' - label: "XPU quantization test" + key: xpu-quantization-test depends_on: - image-build-xpu timeout_in_minutes: 30 @@ -158,6 +163,7 @@ steps: pytest -v -s quantization/test_auto_round.py && pytest -v -s quantization/test_online.py' - label: "XPU compressed tensors FP8 test" + key: xpu-compressed-tensors-fp8-test depends_on: - image-build-xpu timeout_in_minutes: 60 @@ -182,6 +188,7 @@ steps: pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8 && pytest -v -s quantization/test_compressed_tensors.py::test_compressed_tensors_fp8_block_enabled' - label: "XPU Graph" + key: xpu-graph depends_on: - image-build-xpu timeout_in_minutes: 60 diff --git a/.buildkite/test_areas/docker.yaml b/.buildkite/test_areas/docker.yaml index e88f19ad97..ef50062a7f 100644 --- a/.buildkite/test_areas/docker.yaml +++ b/.buildkite/test_areas/docker.yaml @@ -3,6 +3,7 @@ depends_on: - image-build-cpu steps: - label: ":computer: (CPU) Docker Build Metadata" + key: docker-build-metadata timeout_in_minutes: 20 device: cpu-small source_file_dependencies: diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 0863dfd276..dc90dcb852 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -470,6 +470,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_offloading.py -k "deepseek-v4-flash" - label: ":nvidia: (H200 MIG 35GB) MRCR Eval Small Models" + key: mrcr-eval-small-models device: h200_35gb timeout_in_minutes: 25 source_file_dependencies: diff --git a/.buildkite/test_areas/rust_frontend.yaml b/.buildkite/test_areas/rust_frontend.yaml index 8ccc0c50e9..97078512a1 100644 --- a/.buildkite/test_areas/rust_frontend.yaml +++ b/.buildkite/test_areas/rust_frontend.yaml @@ -3,6 +3,7 @@ depends_on: - image-build steps: - label: ":nvidia: (H200 MIG 18GB) Rust Frontend OpenAI Coverage" + key: rust-frontend-openai-coverage timeout_in_minutes: 30 device: h200_18gb working_dir: "/vllm-workspace/tests" @@ -67,6 +68,7 @@ steps: - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 18GB) Rust Frontend Serve/Admin Coverage" + key: rust-frontend-serve-admin-coverage timeout_in_minutes: 25 device: h200_18gb working_dir: "/vllm-workspace/tests" @@ -116,6 +118,7 @@ steps: - vllm/platforms/rocm.py - label: ":nvidia: (H200 MIG 18GB) Rust Frontend Core Correctness" + key: rust-frontend-core-correctness timeout_in_minutes: 20 device: h200_18gb working_dir: "/vllm-workspace/tests" @@ -130,6 +133,7 @@ steps: - pytest -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine - label: ":nvidia: (H200 MIG 35GB) Rust Frontend Tool Use" + key: rust-frontend-tool-use device: h200_35gb timeout_in_minutes: 25 working_dir: "/vllm-workspace/tests" @@ -145,6 +149,7 @@ steps: - pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice" - label: ":nvidia: (L4) Rust Frontend Distributed" + key: rust-frontend-distributed timeout_in_minutes: 25 device: l4 num_devices: 4 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d1d0e9e3d1..b12c8ce508 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -177,6 +177,13 @@ repos: files: ^\.github/mergify\.yml$ pass_filenames: false additional_dependencies: [PyYAML, regex] + - id: check-buildkite-step-keys + name: Check every Buildkite step declares an explicit key + language: python + entry: python tools/pre_commit/check_buildkite_step_keys.py + files: ^\.buildkite/.*\.ya?ml$ + pass_filenames: false + additional_dependencies: [PyYAML] - id: mypy-3.10 # TODO: Use https://github.com/pre-commit/mirrors-mypy when mypy setup is less awkward name: Run mypy for Python 3.10 entry: python tools/pre_commit/mypy.py "3.10" diff --git a/tools/pre_commit/check_buildkite_step_keys.py b/tools/pre_commit/check_buildkite_step_keys.py new file mode 100644 index 0000000000..41d65b4bc6 --- /dev/null +++ b/tools/pre_commit/check_buildkite_step_keys.py @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Check that every Buildkite step the CI generator reads declares a key. + +A step with no `key:` is published under one the generator derives from its +label, which makes the job's address a function of its title: retitling the +step silently renames it and breaks retries, `depends_on:` and build history. +The generator only reads the directories a pipeline config names in +`job_dirs:`, so this walks the same files and fails on any step without a key. + +Usage: + python tools/pre_commit/check_buildkite_step_keys.py +""" + +import sys +from pathlib import Path + +import yaml + +BUILDKITE = Path(".buildkite") +CONFIG_GLOB = "ci_config*.yaml" +JOB_GLOB = "*.yaml" + + +def job_dirs() -> list[Path]: + """Every directory the pipeline generator reads, per the pipeline configs.""" + configs = sorted(BUILDKITE.glob(CONFIG_GLOB)) + if not configs: + raise SystemExit(f"no pipeline config matched {BUILDKITE / CONFIG_GLOB}") + dirs: set[Path] = set() + for config in configs: + with open(config, encoding="utf-8") as f: + doc = yaml.safe_load(f) + declared = doc.get("job_dirs") if isinstance(doc, dict) else None + if not isinstance(declared, list) or not declared: + raise SystemExit(f"{config}: `job_dirs` is missing or not a list") + for name in declared: + job_dir = Path(name) + if not job_dir.is_dir(): + raise SystemExit(f"{config}: job_dir `{name}` does not exist") + dirs.add(job_dir) + return sorted(dirs) + + +def scan(steps: object, source: Path) -> list[str]: + """Label of every step in a `steps:` list that declares no key.""" + if not isinstance(steps, list): + raise SystemExit(f"{source}: `steps:` is not a list") + keyless: list[str] = [] + for i, step in enumerate(steps): + # A scalar shorthand such as `- wait` has nowhere to put a key. Every + # other step does, so there are no other exemptions. A `mirror:` block + # is not a step here: the generator publishes it as `-`. + if not isinstance(step, dict): + continue + label = step.get("label") or step.get("group") or f"step {i}" + if "steps" in step: + raise SystemExit( + f"{source}: `{label}` nests its own `steps:`. The generator has " + "no group step and drops the children without an error, so give " + "each one its own entry." + ) + key = step.get("key") + if not (isinstance(key, str) and key.strip()): + keyless.append(label) + return keyless + + +def keyless_steps(path: Path) -> list[str]: + with open(path, encoding="utf-8") as f: + doc = yaml.safe_load(f) + if not isinstance(doc, dict) or "steps" not in doc: + return [] + return scan(doc["steps"], path) + + +def main() -> int: + files = sorted({p for d in job_dirs() for p in d.rglob(JOB_GLOB)}) + if not files: + raise SystemExit("no pipeline files under the declared job_dirs") + keyless = [(path, label) for path in files for label in keyless_steps(path)] + if not keyless: + return 0 + print(f"{len(keyless)} Buildkite step(s) have no explicit key:\n", file=sys.stderr) + for path, label in keyless: + print(f" {path}: {label}", file=sys.stderr) + print( + "\nAdd a `key:` naming the test each step runs. Without one the pipeline" + "\ngenerator derives the key from the label, so retitling the step renames" + "\nits job and breaks retries and build history.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main())