From aecb717f10bccbaae441d43910ae660a8f2e4e3c Mon Sep 17 00:00:00 2001 From: Turner Jabbour Date: Tue, 6 Oct 2026 22:16:53 -0600 Subject: [PATCH] [CI] Allowlist-shrink batch 2: wire 9 tests/models/ files into Buildkite, fix 2 stale bugs found (#59050) Signed-off-by: Turner Jabbour Co-authored-by: Claude Sonnet 5 --- .buildkite/test_areas/models_basic.yaml | 22 +++- tests/models/test_dspark_mla.py | 1 + .../transformers/test_layer_registry.py | 115 ++---------------- .../transformers/test_layer_registry_e2e.py | 113 +++++++++++++++++ tests/test_zen_cpu_platform_detection.py | 17 --- tools/pre_commit/test_tethering_allowlist.txt | 11 -- vllm/triton_utils/importing.py | 5 +- 7 files changed, 143 insertions(+), 141 deletions(-) create mode 100644 tests/models/transformers/test_layer_registry_e2e.py diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 865156f594..04efae00fb 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -68,8 +68,10 @@ steps: - tests/models/test_deepseek_v4_vl_rocm.py - tests/models/test_deepseek_v41_decoder_replay_layers.py - tests/models/test_deepseek_v41_replay_batch.py + - tests/models/transformers/test_layer_registry_e2e.py + - tests/models/test_deepseek_v41_replay_start.py commands: - - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py models/test_deepseek_v41_decoder_replay_layers.py models/test_deepseek_v41_replay_batch.py + - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py models/transformers/test_layer_registry_e2e.py models/test_deepseek_v41_replay_start.py models/test_deepseek_v41_decoder_replay_layers.py models/test_deepseek_v41_replay_batch.py mirror: amd: label: ":amd: (MI355) Basic Models (Other)" @@ -81,8 +83,15 @@ steps: source_file_dependencies: - tests/models/test_deepseek_v4_vl_rocm.py - tests/models/test_hyv4_rocm.py + - tests/models/transformers/test_layer_registry_e2e.py + - tests/models/test_deepseek_v41_replay_start.py + - tests/models/test_deepseek_v41_decoder_replay_layers.py + - tests/models/test_deepseek_v41_replay_batch.py + - tests/models/test_deepseek_v4_dspark_rocm.py + - tests/models/test_deepseek_v4_rocm_wo_a.py + - tests/models/test_deepseek_v4_rocm_compressor_gemm_fusion.py commands: - - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py + - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py models/test_deepseek_v4_vl_rocm.py models/transformers/test_layer_registry_e2e.py models/test_deepseek_v41_replay_start.py models/test_deepseek_v41_decoder_replay_layers.py models/test_deepseek_v41_replay_batch.py models/test_deepseek_v4_dspark_rocm.py models/test_deepseek_v4_rocm_wo_a.py models/test_deepseek_v4_rocm_compressor_gemm_fusion.py - VLLM_ROCM_USE_AITER=1 pytest -v -s models/test_hyv4_rocm.py -m 'not distributed' - label: ":nvidia: (B200) Inkling" @@ -229,7 +238,7 @@ steps: key: basic-models-test-other-cpu depends_on: - image-build-cpu - timeout_in_minutes: 20 + timeout_in_minutes: 30 source_file_dependencies: - vllm/ - "!vllm/distributed/kv_transfer/" @@ -239,6 +248,11 @@ steps: - tests/models/test_qwen3_5_mtp_config.py - tests/models/transformers/fusers/ - tests/model_executor/layers/test_activation.py + - tests/models/test_language_model_cache_is_weak.py + - tests/models/deepseek_v32/test_sequence_parallel.py + - tests/models/test_deepseek_v4_fi_moe_ep.py + - tests/models/test_dspark_mla.py + - tests/models/transformers/test_layer_registry.py device: cpu-small commands: - - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ model_executor/layers/test_activation.py + - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ model_executor/layers/test_activation.py models/test_language_model_cache_is_weak.py models/deepseek_v32/test_sequence_parallel.py models/test_deepseek_v4_fi_moe_ep.py models/test_dspark_mla.py models/transformers/test_layer_registry.py diff --git a/tests/models/test_dspark_mla.py b/tests/models/test_dspark_mla.py index f77832d608..76af0eb60c 100644 --- a/tests/models/test_dspark_mla.py +++ b/tests/models/test_dspark_mla.py @@ -85,6 +85,7 @@ def test_dspark_markov_head_is_replicated( assert head.markov_w2.tp_size == 1 assert head.markov_w1.weight.shape == (128, 8) assert head.markov_w2.weight.shape == (128, 8) + head.markov_w2.quant_method.process_weights_after_loading(head.markov_w2) def fail_collective(*args, **kwargs): raise AssertionError("replicated Markov head must not invoke TP collectives") diff --git a/tests/models/transformers/test_layer_registry.py b/tests/models/transformers/test_layer_registry.py index 9e20e2ba54..555d87e379 100644 --- a/tests/models/transformers/test_layer_registry.py +++ b/tests/models/transformers/test_layer_registry.py @@ -7,6 +7,9 @@ is set and the symbol exists, and otherwise falls back to `vllm.model_executor.layers.`. These tests pin that contract and the logging that reports which source was used. + +CPU-only: no engine is built. The end-to-end counterpart that serves a model +through `vllm_runner` lives in `test_layer_registry_e2e.py` (GPU job). """ import logging @@ -14,7 +17,6 @@ import sys import types import pytest -import torch from vllm.model_executor.models.transformers import layers @@ -42,16 +44,16 @@ def test_falls_back_to_vllm_when_disabled(monkeypatch, fake_hw_layernorm): assert layers._resolve("layernorm", "RMSNorm") is VllmRMSNorm -def test_uses_hw_agnostic_when_enabled(monkeypatch, fake_hw_layernorm, caplog): +def test_uses_hw_agnostic_when_enabled(monkeypatch, fake_hw_layernorm, caplog_vllm): """Enabled and available: the hw-agnostic class is used and logged.""" monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1") - with caplog.at_level(logging.INFO): + with caplog_vllm.at_level(logging.INFO): resolved = layers._resolve("layernorm", "RMSNorm") assert resolved is fake_hw_layernorm.RMSNorm - assert "Using hw-agnostic layer: RMSNorm" in caplog.text + assert "Using hw-agnostic layer: RMSNorm" in caplog_vllm.text -def test_falls_back_when_symbol_missing(monkeypatch, caplog): +def test_falls_back_when_symbol_missing(monkeypatch, caplog_vllm): """Enabled but the symbol is not ported: fall back to vLLM and warn.""" monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1") # A hw-agnostic module without the requested attribute triggers fallback. @@ -59,10 +61,10 @@ def test_falls_back_when_symbol_missing(monkeypatch, caplog): monkeypatch.setitem(sys.modules, HW_MODULE, empty) from vllm.model_executor.layers.layernorm import RMSNorm as VllmRMSNorm - with caplog.at_level(logging.WARNING): + with caplog_vllm.at_level(logging.WARNING): resolved = layers._resolve("layernorm", "RMSNorm") assert resolved is VllmRMSNorm - assert "falling back to default" in caplog.text + assert "falling back to default" in caplog_vllm.text def test_act_and_mul_falls_back_for_unknown_activation( @@ -76,102 +78,3 @@ def test_act_and_mul_falls_back_for_unknown_activation( from vllm.model_executor.layers.activation import GeluAndMul assert isinstance(layers.get_act_and_mul_fn("gelu"), GeluAndMul) - - -@pytest.fixture(scope="module") -def tiny_llama_path(tmp_path_factory): - """A randomly-initialized microscopic Llama saved to disk (with an ungated - tokenizer) so vLLM can load it like any local checkpoint.""" - from transformers import AutoTokenizer, LlamaConfig, LlamaForCausalLM - - tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") - config = LlamaConfig( - vocab_size=tokenizer.vocab_size, - hidden_size=128, - intermediate_size=256, - num_hidden_layers=2, - num_attention_heads=4, - rms_norm_eps=1e-6, - hidden_act="silu", - ) - torch.manual_seed(0) - model = LlamaForCausalLM(config) - - path = tmp_path_factory.mktemp("tiny_llama") - model.save_pretrained(path) - tokenizer.save_pretrained(path) - return str(path) - - -# Registered names of the layers the backend can -# currently route to hw-agnostic implementations. -_COVERED_LAYERS = ("rms_norm", "silu_and_mul") - - -def _layer_providers(model) -> dict[str, str]: - """Map each covered layer type present in the model to the provider its - implementation came from (``hw_agnostic`` or ``vllm``). - """ - - def provider_of(module) -> str | None: - for cls in type(module).__mro__: - if "hw_agnostic.layers" in cls.__module__: - return "hw_agnostic" - if ".model_executor.layers." in cls.__module__: - return "vllm" - return None - - providers: dict[str, str] = {} - for module in model.modules(): - name = getattr(module, "name", None) - if ( - name in _COVERED_LAYERS - and name not in providers - and (prov := provider_of(module)) is not None - ): - providers[name] = prov - return providers - - -def _serve(vllm_runner, model_path, prompts): - """Serve the model through the backend; return (layer_providers, logprobs).""" - with vllm_runner( - model_path, - model_impl="transformers", - max_model_len=64, - enforce_eager=True, - gpu_memory_utilization=0.3, - ) as runner: - assert runner.llm.llm_engine.model_config.using_transformers_backend() - providers = runner.apply_model(_layer_providers)[0] - outputs = runner.generate_greedy_logprobs( - prompts, max_tokens=32, num_logprobs=5 - ) - return providers, outputs - - -def test_hw_agnostic_matches_vllm_end_to_end(monkeypatch, vllm_runner, tiny_llama_path): - """Serving the tiny model with hw-agnostic layers matches the vLLM baseline.""" - # spawn: worker re-imports layers with the env set (see docstring). - monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") - # apply_model pickles the introspection function. - monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") - from ..utils import check_logprobs_close - - prompts = ["The capital of France is", "vLLM is"] - - monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0") - vllm_providers, vllm_outputs = _serve(vllm_runner, tiny_llama_path, prompts) - # Both replaceable layers present in a Llama block must be vLLM's here. - assert vllm_providers == {"rms_norm": "vllm", "silu_and_mul": "vllm"} - - monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1") - hw_providers, hw_outputs = _serve(vllm_runner, tiny_llama_path, prompts) - assert hw_providers == {"rms_norm": "hw_agnostic", "silu_and_mul": "hw_agnostic"} - - check_logprobs_close( - outputs_0_lst=vllm_outputs, - outputs_1_lst=hw_outputs, - name_0="vllm", - name_1="hw_agnostic", - ) diff --git a/tests/models/transformers/test_layer_registry_e2e.py b/tests/models/transformers/test_layer_registry_e2e.py new file mode 100644 index 0000000000..13d4b80a16 --- /dev/null +++ b/tests/models/transformers/test_layer_registry_e2e.py @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""End-to-end check of the Transformers backend's hw-agnostic layers. + +Serves a tiny Llama through `vllm_runner` with and without +`VLLM_USE_HW_AGNOSTIC` and checks the layer providers and logprobs. Needs a +real engine, so it runs in the GPU job. The CPU-only unit tests for the +resolution logic (`layers._resolve`) are in `test_layer_registry.py`. +""" + +import pytest +import torch + + +@pytest.fixture(scope="module") +def tiny_llama_path(tmp_path_factory): + """A randomly-initialized microscopic Llama saved to disk (with an ungated + tokenizer) so vLLM can load it like any local checkpoint.""" + from transformers import AutoTokenizer, LlamaConfig, LlamaForCausalLM + + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer") + config = LlamaConfig( + vocab_size=tokenizer.vocab_size, + hidden_size=128, + intermediate_size=256, + num_hidden_layers=2, + num_attention_heads=4, + rms_norm_eps=1e-6, + hidden_act="silu", + ) + torch.manual_seed(0) + model = LlamaForCausalLM(config) + + path = tmp_path_factory.mktemp("tiny_llama") + model.save_pretrained(path) + tokenizer.save_pretrained(path) + return str(path) + + +# Registered names of the layers the backend can +# currently route to hw-agnostic implementations. +_COVERED_LAYERS = ("rms_norm", "silu_and_mul") + + +def _layer_providers(model) -> dict[str, str]: + """Map each covered layer type present in the model to the provider its + implementation came from (``hw_agnostic`` or ``vllm``). + """ + + def provider_of(module) -> str | None: + for cls in type(module).__mro__: + if "hw_agnostic.layers" in cls.__module__: + return "hw_agnostic" + if ".model_executor.layers." in cls.__module__: + return "vllm" + return None + + providers: dict[str, str] = {} + for module in model.modules(): + name = getattr(module, "name", None) + if ( + name in _COVERED_LAYERS + and name not in providers + and (prov := provider_of(module)) is not None + ): + providers[name] = prov + return providers + + +def _serve(vllm_runner, model_path, prompts): + """Serve the model through the backend; return (layer_providers, logprobs).""" + with vllm_runner( + model_path, + model_impl="transformers", + max_model_len=64, + enforce_eager=True, + gpu_memory_utilization=0.3, + ) as runner: + assert runner.llm.llm_engine.model_config.using_transformers_backend() + providers = runner.apply_model(_layer_providers)[0] + outputs = runner.generate_greedy_logprobs( + prompts, max_tokens=32, num_logprobs=5 + ) + return providers, outputs + + +def test_hw_agnostic_matches_vllm_end_to_end(monkeypatch, vllm_runner, tiny_llama_path): + """Serving the tiny model with hw-agnostic layers matches the vLLM baseline.""" + # spawn: `layers` resolves its symbols at import time, so the worker must + # re-import it with VLLM_USE_HW_AGNOSTIC already set (fork would inherit + # the parent's already-resolved module). + monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") + # apply_model pickles the introspection function. + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + from ..utils import check_logprobs_close + + prompts = ["The capital of France is", "vLLM is"] + + monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "0") + vllm_providers, vllm_outputs = _serve(vllm_runner, tiny_llama_path, prompts) + # Both replaceable layers present in a Llama block must be vLLM's here. + assert vllm_providers == {"rms_norm": "vllm", "silu_and_mul": "vllm"} + + monkeypatch.setenv("VLLM_USE_HW_AGNOSTIC", "1") + hw_providers, hw_outputs = _serve(vllm_runner, tiny_llama_path, prompts) + assert hw_providers == {"rms_norm": "hw_agnostic", "silu_and_mul": "hw_agnostic"} + + check_logprobs_close( + outputs_0_lst=vllm_outputs, + outputs_1_lst=hw_outputs, + name_0="vllm", + name_1="hw_agnostic", + ) diff --git a/tests/test_zen_cpu_platform_detection.py b/tests/test_zen_cpu_platform_detection.py index 505d2d3bfd..b8a3949923 100644 --- a/tests/test_zen_cpu_platform_detection.py +++ b/tests/test_zen_cpu_platform_detection.py @@ -13,23 +13,6 @@ from vllm.platforms import ( ) -@pytest.fixture -def caplog_vllm(caplog): - """`caplog`, but also captures vLLM's loggers. - - vllm/logger.py configures the "vllm" logger with propagate=False, so - records from child loggers like "vllm.platforms" reach vLLM's own - handler but never bubble up to the root logger where plain `caplog` - listens. Attach caplog's handler directly to "vllm" to see them too. - """ - logger = logging.getLogger("vllm") - logger.addHandler(caplog.handler) - try: - yield caplog - finally: - logger.removeHandler(caplog.handler) - - def test_is_amd_zen_cpu_detects_amd_with_avx512(): cpuinfo = "vendor_id: AuthenticAMD\nflags: avx avx2 avx512f avx512bw" with ( diff --git a/tools/pre_commit/test_tethering_allowlist.txt b/tools/pre_commit/test_tethering_allowlist.txt index d7f19ec79a..e738a79154 100644 --- a/tools/pre_commit/test_tethering_allowlist.txt +++ b/tools/pre_commit/test_tethering_allowlist.txt @@ -48,17 +48,6 @@ tests/kernels/test_shuffle_rows.py # explicitly --ignore'd in kernels.yaml with tests/lora/test_deepseekv2_tp.py # --ignore'd from sharded LoRA run; no dedicated *_tp job tests/lora/test_qwen3moe_tp.py # --ignore'd from sharded LoRA run; no dedicated *_tp job -# tests/models/ - the models_*.yaml jobs run specific paths, not all of models/. -tests/models/deepseek_v32/test_sequence_parallel.py # no models_*.yaml job collects this path -tests/models/test_deepseek_v4_dspark_rocm.py # added by #52362 (2026-09-21), no models_*.yaml job collects this path -tests/models/test_deepseek_v4_fi_moe_ep.py # no models_*.yaml job collects this path -tests/models/test_deepseek_v4_rocm_compressor_gemm_fusion.py # no models_*.yaml job collects this path -tests/models/test_deepseek_v4_rocm_wo_a.py # added by #54894 (2026-09-20), no models_*.yaml job collects this path -tests/models/test_deepseek_v41_replay_start.py # added by #56227 (2026-09-18), no models_*.yaml job collects this path -tests/models/test_dspark_mla.py # no models_*.yaml job collects this path -tests/models/test_language_model_cache_is_weak.py # no models_*.yaml job collects this path -tests/models/transformers/test_layer_registry.py # no models_*.yaml job collects this path - # tests/v1/ - the v1 subtree is otherwise covered by many `pytest v1/...` jobs. tests/v1/ec_connector/integration/test_epd_correctness.py # run_epd_correctness_test.sh exists but no Buildkite job invokes it tests/v1/ec_connector/integration/test_nixl_failure.py # 2-GPU + NIXL E/PD integration test (#55290); no Buildkite job collects ec_connector/integration/ diff --git a/vllm/triton_utils/importing.py b/vllm/triton_utils/importing.py index 12f19cc52b..7e501c94f4 100644 --- a/vllm/triton_utils/importing.py +++ b/vllm/triton_utils/importing.py @@ -3,11 +3,10 @@ import os import types -from importlib.metadata import version from importlib.util import find_spec from vllm.logger import init_logger -from vllm.platforms import current_platform +from vllm.platforms import current_platform, vllm_version_matches_substr from vllm.utils.math_utils import cdiv logger = init_logger(__name__) @@ -35,7 +34,7 @@ if HAS_TRITON: ] # Check Triton CPU - if "cpu" in version("vllm"): + if vllm_version_matches_substr("cpu"): if "cpu" in backends: HAS_TRITON = True # Suppress following warnings on CPU-only platforms