Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes

Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
lesj0610
2026-09-27 18:25:21 +09:00
1165 changed files with 51759 additions and 14579 deletions
@@ -66,7 +66,7 @@ from vllm.platforms import PlatformEnum
"MarlinMxfp8LinearKernel",
"EmulationMxfp8LinearKernel",
init_mxfp8_linear_kernel,
{},
{"weight_shape": (2048, 2048)},
),
(
B12xTensorFP8ScaledMMLinearKernel,
@@ -365,7 +365,7 @@ def test_b12x_tensor_fp8_apply_quantizes_and_uses_packed_weight(
def test_b12x_mxfp8_can_implement_supported_config() -> None:
can_implement, reason = B12xMxfp8LinearKernel.can_implement(
Mxfp8LinearLayerConfig()
Mxfp8LinearLayerConfig(weight_shape=(256, 512))
)
assert can_implement
+68
View File
@@ -0,0 +1,68 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from torch import nn
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
@pytest.mark.parametrize(
"input_shape,bias",
[
pytest.param((256, 3, 14, 14), False, id="packed_patches"),
pytest.param((2, 3, 28, 28), True, id="multiple_spatial_patches"),
],
)
def test_conv2d_patch_embedding_correctness(default_vllm_config, input_shape, bias):
device = "cuda" if torch.cuda.is_available() else "cpu"
dtype = torch.bfloat16 if device == "cuda" else torch.float32
reference = nn.Conv2d(3, 1024, 14, stride=14, bias=bias).to(
device=device, dtype=dtype
)
layer = Conv2dLayer(3, 1024, 14, stride=14, bias=bias, params_dtype=dtype).to(
device
)
layer.load_state_dict(reference.state_dict(), strict=True)
assert layer.enable_linear
x = torch.randn(input_shape, device=device, dtype=dtype)
expected = reference(x)
atol = rtol = 0.02 if dtype == torch.bfloat16 else 1e-5
torch.testing.assert_close(layer.forward_native(x), expected, atol=atol, rtol=rtol)
@pytest.mark.parametrize(
"input_shape,kernel_size,out_channels,bias",
[
pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1280, False, id="minimax_m3"),
pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1152, False, id="qwen2_vl"),
pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1152, True, id="qwen3_vl"),
pytest.param((4, 3, 1, 14, 14), (1, 14, 14), 1536, True, id="glm_vl"),
pytest.param((1, 3, 4, 28, 28), (2, 14, 14), 32, True, id="multiple_patches"),
],
)
def test_conv3d_patch_embedding_correctness(
default_vllm_config, input_shape, kernel_size, out_channels, bias
):
device = "cuda" if torch.cuda.is_available() else "cpu"
dtype = torch.bfloat16 if device == "cuda" else torch.float32
reference = nn.Conv3d(
input_shape[1], out_channels, kernel_size, stride=kernel_size, bias=bias
).to(device=device, dtype=dtype)
layer = Conv3dLayer(
input_shape[1],
out_channels,
kernel_size,
stride=kernel_size,
bias=bias,
params_dtype=dtype,
).to(device)
layer.load_state_dict(reference.state_dict(), strict=True)
assert layer.enable_linear
x = torch.randn(input_shape, device=device, dtype=dtype)
expected = reference(x)
atol = rtol = 0.02 if dtype == torch.bfloat16 else 1e-5
torch.testing.assert_close(layer(x), expected, atol=atol, rtol=rtol)
@@ -1227,7 +1227,6 @@ def test_non_quark_shared_expert_fse_is_incompatible() -> None:
def _fp8_config(**kwargs: Any) -> Fp8Config:
return Fp8Config(
is_checkpoint_fp8_serialized=True,
activation_scheme="dynamic",
weight_block_size=[128, 128],
**kwargs,
@@ -1248,7 +1247,7 @@ def test_block_fp8_shared_expert_fse_is_compatible() -> None:
def test_per_tensor_fp8_shared_expert_fse_is_incompatible() -> None:
"""Per-tensor scales are 0-D, so the shared-expert chunker cannot slice them."""
compatible, reason = is_shared_expert_quant_fse_compatible(
Fp8Config(is_checkpoint_fp8_serialized=True, activation_scheme="dynamic"),
Fp8Config(activation_scheme="dynamic"),
"model.layers.0.mlp.experts",
"model.layers.0.mlp.shared_experts",
)
@@ -677,6 +677,40 @@ def test_candidate_blocks_to_sparse_indices_matches_reference(
assert not end_buf[1 - end_offset :: 2].any()
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
def test_sparse_topk_remap_bounds_nan_rows():
"""DeepSelect writes an out-of-range sentinel to slot 0 of NaN rows (the
rest is left untouched); the remap turns it into -1."""
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
has_deep_select,
sparse_topk_remap,
)
if not has_deep_select():
pytest.skip("DeepSelect extension (vllm._deepselect_C) required")
rows, num_sparse, sbk, topk = 4, 128, 8, 512
width = num_sparse * sbk
logits = torch.randn(rows, width, dtype=torch.bfloat16, device="cuda")
logits[1:, :16] = float("nan")
sparse_indices = torch.arange(num_sparse, dtype=torch.int32, device="cuda")
end = torch.full((rows,), width, dtype=torch.int32, device="cuda")
out = torch.empty(rows, topk, dtype=torch.int32, device="cuda")
cols = torch.zeros_like(out)
sparse_topk_remap(
logits,
sparse_indices.repeat(rows, 1),
end,
torch.zeros_like(end),
sbk,
topk,
out,
col_indices=cols,
)
assert (cols[1:, 0] >= width).all() and (out[1:, 0] == -1).all()
assert (out[0] >= 0).all()
def _skip_unless_sm100_sparse_kernels():
if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10:
pytest.skip("SM100 required")
@@ -0,0 +1,320 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FusedMMInputNorm and the fused affine transform kernel."""
import pytest
import torch
from vllm.model_executor.layers.fusion.mm_input_norm import (
FusedMMInputNorm,
IdentityInputNorm,
NormParams,
fused_mm_input_norm_triton,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from vllm.utils.torch_utils import set_random_seed
# Module-level tests run on any platform: the CPU platform exercises
# forward_native, accelerators exercise the fused kernel. Only the raw
# kernel tests require an accelerator.
_DEVICE = torch.device(current_platform.device_type)
requires_accelerator = pytest.mark.skipif(
current_platform.is_cpu(),
reason="fused Triton kernel requires a CUDA/XPU accelerator",
)
requires_triton = pytest.mark.skipif(not HAS_TRITON, reason="requires Triton")
requires_vllm_config = pytest.mark.usefixtures("default_vllm_config")
def _reference_input_norm(
pixel_values: torch.Tensor,
image_mean: list[float],
image_std: list[float],
rescale_factor: float,
channel: int,
out_dtype: torch.dtype,
) -> torch.Tensor:
"""Straightforward per-channel affine: (x * rescale - mean) / std."""
patches, size = pixel_values.shape
patch_size = size // channel
mean = torch.tensor(
image_mean, dtype=torch.float32, device=pixel_values.device
).view(1, channel, 1)
std = torch.tensor(image_std, dtype=torch.float32, device=pixel_values.device).view(
1, channel, 1
)
x = pixel_values.to(torch.float32).reshape(patches, channel, patch_size)
x = (x * rescale_factor - mean) / std
return x.view(patches, size).to(out_dtype)
# CLIP-style RGB normalization constants reused across tests.
_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073]
_RGB_STD = [0.26862954, 0.26130258, 0.27577711]
_RGB_RESCALE = 1.0 / 255.0
def _check_against_reference(
pixel_values: torch.Tensor,
out_dtype: torch.dtype,
channel: int = 3,
image_mean: list[float] | None = None,
image_std: list[float] | None = None,
rescale_factor: float = _RGB_RESCALE,
**close_kwargs,
):
"""Run ``FusedMMInputNorm`` on ``pixel_values`` and compare against the
eager per-channel affine reference. Uses the RGB constants by default."""
if image_mean is None:
image_mean = list(_RGB_MEAN)
if image_std is None:
image_std = list(_RGB_STD)
norm = FusedMMInputNorm(
image_mean=image_mean,
image_std=image_std,
rescale_factor=rescale_factor,
channel=channel,
).to(_DEVICE)
expected = _reference_input_norm(
pixel_values, image_mean, image_std, rescale_factor, channel, out_dtype
)
torch.testing.assert_close(
norm(pixel_values, visual_dtype=out_dtype), expected, **close_kwargs
)
# ===========================================================================
# Module-level behavior
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormModule:
def test_matches_reference_above_cudnn_grid_limit(self):
"""num_patches above the old cuDNN batch-norm grid limit (~65535)
used to raise CUDNN_STATUS_INTERNAL_ERROR."""
set_random_seed(0)
pixel_values = torch.randint(
0, 256, (70000, 3 * 196), dtype=torch.float32, device=_DEVICE
)
_check_against_reference(pixel_values, torch.float32)
def test_identity_passthrough(self):
norm = IdentityInputNorm()
pixel_values = torch.randn(8, 3 * 196, dtype=torch.float32, device=_DEVICE)
out = norm(pixel_values, visual_dtype=torch.bfloat16)
torch.testing.assert_close(out, pixel_values.to(torch.bfloat16))
# ===========================================================================
# dtype coverage
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormDtypes:
@pytest.mark.parametrize(
"in_dtype,out_dtype",
[
(torch.float32, torch.float32),
(torch.float32, torch.bfloat16),
(torch.float32, torch.float16),
(torch.bfloat16, torch.bfloat16),
(torch.bfloat16, torch.float32),
(torch.float16, torch.float16),
(torch.uint8, torch.bfloat16),
(torch.uint8, torch.float32),
],
)
def test_dtype_combinations(self, in_dtype: torch.dtype, out_dtype: torch.dtype):
set_random_seed(0)
if in_dtype == torch.uint8:
pixel_values = torch.randint(
0, 256, (32, 3 * 16), dtype=torch.uint8, device=_DEVICE
)
else:
pixel_values = torch.rand(
(32, 3 * 16), dtype=torch.float32, device=_DEVICE
).to(in_dtype)
_check_against_reference(
pixel_values,
out_dtype,
image_mean=[0.5, 0.5, 0.5],
image_std=[0.5, 0.5, 0.5],
atol=1e-2,
rtol=1e-2,
)
# ===========================================================================
# Shape / channel coverage
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormShapes:
@pytest.mark.parametrize("channel", [1, 3, 4])
def test_channel_variants(self, channel: int):
set_random_seed(0)
pixel_values = torch.randint(
0, 256, (16, channel * 64), dtype=torch.float32, device=_DEVICE
)
_check_against_reference(
pixel_values,
torch.float32,
channel=channel,
image_mean=[0.5] * channel,
image_std=[0.25] * channel,
)
@pytest.mark.parametrize("patch_size", [1, 2048])
def test_block_boundaries(self, patch_size: int):
"""Masked-tail (12 elements) and exact-multiple (24 blocks) paths."""
set_random_seed(0)
pixel_values = torch.randint(
0, 256, (4, 3 * patch_size), dtype=torch.float32, device=_DEVICE
)
_check_against_reference(pixel_values, torch.float32)
# ===========================================================================
# Input handling: non-contiguous inputs
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormInputHandling:
def test_non_contiguous_input_matches_reference(self):
set_random_seed(0)
base = torch.randint(
0, 256, (8, 3 * 32, 2), dtype=torch.float32, device=_DEVICE
)
non_contig = base[..., 0]
assert not non_contig.is_contiguous()
_check_against_reference(non_contig, torch.float32)
# ===========================================================================
# Raw kernel entry point
# ===========================================================================
@requires_accelerator
@requires_triton
class TestFusedMMInputNormKernel:
@pytest.mark.parametrize(
"N, C, L",
[
(1, 1, 1),
(1, 3, 1),
(3, 3, 7), # tile straddles channel boundaries
(2, 8, 63),
(4, 3, 1000),
],
)
def test_channel_boundary_crossing(self, N: int, C: int, L: int):
"""The 1D kernel recovers ``c = (offs // L) % C`` per lane, so a tile
can straddle a channel boundary and gather weight/bias per lane.
These shapes force that path by keeping ``L`` well below the block."""
set_random_seed(0)
x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE)
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
out = torch.empty_like(x)
fused_mm_input_norm_triton(x, out, w, b)
expected = x * w.view(1, C, 1) + b.view(1, C, 1)
torch.testing.assert_close(out, expected)
def test_non_contiguous_inputs_materialized(self):
"""Read-only tensors are made contiguous internally."""
N, C, L = 3, 3, 100
set_random_seed(0)
base = torch.randn(N, C, L, 2, dtype=torch.float32, device=_DEVICE)
x = base[..., 0]
assert not x.is_contiguous()
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
out = torch.empty(N, C, L, dtype=torch.float32, device=_DEVICE)
fused_mm_input_norm_triton(x, out, w, b)
expected = x * w.view(1, C, 1) + b.view(1, C, 1)
torch.testing.assert_close(out, expected)
def test_rejects_mismatched_output_shape(self):
"""``outputs`` must be shaped exactly like ``inputs``."""
N, C, L = 3, 3, 100
x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE)
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
for out_shape in ((N, C + 1, L), (N, C, L + 1), (N + 2, C, L)):
with pytest.raises(AssertionError):
fused_mm_input_norm_triton(
x,
torch.empty(*out_shape, dtype=torch.float32, device=_DEVICE),
w,
b,
)
# ===========================================================================
# Construction / configuration
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormConstruction:
"""Weight/bias buffer semantics at init time."""
def test_identity_config_buffers(self):
"""Numerically identity parameters still build plain weight/bias
buffers; the identity shortcut lives in ``from_model_config``."""
norm = FusedMMInputNorm(
image_mean=[0.0, 0.0, 0.0],
image_std=[1.0, 1.0, 1.0],
rescale_factor=1.0,
)
torch.testing.assert_close(norm.weight, torch.ones(3))
torch.testing.assert_close(norm.bias, torch.zeros(3))
def test_non_identity_buffers(self):
channel = 3
norm = FusedMMInputNorm(
image_mean=_RGB_MEAN,
image_std=_RGB_STD,
rescale_factor=_RGB_RESCALE,
channel=channel,
)
assert norm.weight.shape == (channel,)
assert norm.bias.shape == (channel,)
mean = torch.tensor(_RGB_MEAN, dtype=torch.float32)
std = torch.tensor(_RGB_STD, dtype=torch.float32)
torch.testing.assert_close(norm.weight, _RGB_RESCALE / std)
torch.testing.assert_close(norm.bias, -mean / std)
def test_is_identity_params(self):
"""The numeric-identity check mirrors the old allclose detection."""
assert NormParams([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0).is_identity
assert not NormParams(_RGB_MEAN, _RGB_STD, _RGB_RESCALE).is_identity
@pytest.mark.parametrize(
("image_mean", "image_std", "rescale_factor"),
[
([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0),
([0.5, 0.5, 0.5], [0.25, 0.25, 0.25], 1 / 255),
],
)
def test_buffers_initialized_on_default_device(
self,
image_mean: list[float],
image_std: list[float],
rescale_factor: float,
):
"""Buffers are built on CPU first, then moved to the caller's
default device, so init never touches the accelerator."""
# The meta device gives the CPU-only test shard the same non-CPU
# default-device semantics without requiring a CUDA build.
default_device = "cuda" if torch.cuda.is_available() else "meta"
with torch.device(default_device):
norm = FusedMMInputNorm(image_mean, image_std, rescale_factor)
assert norm.weight.device.type == default_device
assert norm.bias.device.type == default_device
@@ -9,7 +9,7 @@ from unittest.mock import patch
import numpy as np
import pytest
import torch
from transformers import PretrainedConfig
from transformers import PreTrainedConfig
from vllm.config import PoolerConfig, set_current_vllm_config
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
@@ -315,7 +315,7 @@ def test_dispatch_seq_cls_honors_token_pooling_type(tok_pooling_type):
model_config=SimpleNamespace(
pooler_config=pooler_config,
head_dtype=None,
hf_config=PretrainedConfig(num_labels=3),
hf_config=PreTrainedConfig(num_labels=3),
),
)
classifier = torch.nn.Linear(4, 3)
@@ -17,9 +17,10 @@ if current_platform.is_cuda():
from vllm.model_executor.layers import utils
def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch):
@pytest.mark.parametrize("m", [1, 128])
def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch, m):
x = torch.randn(1, 64, dtype=torch.float16)
weight = torch.randn(128, 64, dtype=torch.float16)
weight = torch.randn(m, 64, dtype=torch.float16)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
@@ -42,6 +43,20 @@ def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch):
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("n", range(1, 6))
def test_rocm_unquantized_gemm_single_output_real_kernel(dtype, n):
torch.manual_seed(0)
x = torch.randn(n, 2048, device="cuda", dtype=dtype)
weight = torch.randn(1, 2048, device="cuda", dtype=dtype)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
def test_rocm_unquantized_gemm_makes_skinny_activation_contiguous(monkeypatch):
x = torch.randn(64, 4, dtype=torch.float16).t()
weight = torch.randn(128, 64, dtype=torch.float16)
@@ -149,11 +164,12 @@ def test_rocm_unquantized_gemm_noncontiguous_activation_real_kernel(monkeypatch,
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch):
@pytest.mark.parametrize("m", [1, 128])
def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch, m):
# wvSplitK skinny GEMM handles n in [1, 5] (see PR #40687); n > 5 must
# fall back to torch.nn.functional.linear.
x = torch.randn(6, 64, dtype=torch.float16)
weight = torch.randn(128, 64, dtype=torch.float16)
weight = torch.randn(m, 64, dtype=torch.float16)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
@@ -0,0 +1,243 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for --mm-encoder-only safetensors shard filtering."""
import json
import os
import tempfile
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
from vllm.model_executor.model_loader.weight_utils import (
filter_mm_encoder_only_safetensors_files,
resolve_mm_encoder_only_lm_prefixes,
)
from vllm.model_executor.models.utils import WeightsMapper
def _write_index(folder: str, weight_map: dict[str, str]) -> None:
with open(os.path.join(folder, SAFE_WEIGHTS_INDEX_NAME), "w") as f:
json.dump({"weight_map": weight_map}, f)
def _qwen25_mapper() -> WeightsMapper:
"""Qwen2.5-VL dual layout mapper (nested + flat catch-all)."""
return WeightsMapper(
orig_to_new_prefix={
"model.language_model.": "language_model.model.",
"model.visual.": "visual.",
"lm_head.": "language_model.lm_head.",
"model.": "language_model.model.",
}
)
def _qwen3_mapper() -> WeightsMapper:
return WeightsMapper(
orig_to_new_prefix={
"model.visual.": "visual.",
"lm_head.": "language_model.lm_head.",
"model.language_model.": "language_model.model.",
}
)
def _molmo_mapper() -> WeightsMapper:
return WeightsMapper(
orig_to_new_prefix={
"model.vision_backbone.": "vision_backbone.",
"model.transformer.blocks.": "model.layers.",
"model.transformer.ln_f.": "model.norm.",
"model.transformer.mlp.down_proj.": "lm_head.",
}
)
def test_keeps_vision_shards_drops_language_only():
with tempfile.TemporaryDirectory() as folder:
files = [
os.path.join(folder, "model-00001-of-000003.safetensors"),
os.path.join(folder, "model-00002-of-000003.safetensors"),
os.path.join(folder, "model-00003-of-000003.safetensors"),
]
for path in files:
open(path, "wb").close()
_write_index(
folder,
{
"language_model.layers.0.weight": "model-00001-of-000003.safetensors",
"language_model.layers.1.weight": "model-00002-of-000003.safetensors",
"vision_tower.blocks.0.weight": "model-00003-of-000003.safetensors",
"mm_projector.weight": "model-00003-of-000003.safetensors",
},
)
kept = filter_mm_encoder_only_safetensors_files(
files,
folder,
SAFE_WEIGHTS_INDEX_NAME,
("language_model.",),
)
assert kept == [files[2]]
def test_keeps_mixed_shard_with_vision_and_lm():
with tempfile.TemporaryDirectory() as folder:
mixed = os.path.join(folder, "model-00001-of-000001.safetensors")
open(mixed, "wb").close()
_write_index(
folder,
{
"language_model.embed.weight": "model-00001-of-000001.safetensors",
"vision_tower.patch.weight": "model-00001-of-000001.safetensors",
},
)
kept = filter_mm_encoder_only_safetensors_files(
[mixed],
folder,
SAFE_WEIGHTS_INDEX_NAME,
("language_model.",),
)
assert kept == [mixed]
def test_no_index_returns_unchanged():
with tempfile.TemporaryDirectory() as folder:
files = [os.path.join(folder, "model.safetensors")]
open(files[0], "wb").close()
kept = filter_mm_encoder_only_safetensors_files(
files,
folder,
SAFE_WEIGHTS_INDEX_NAME,
("language_model.",),
)
assert kept == files
def test_resolve_returns_module_prefixes_without_hf_hardcode():
"""No global Qwen HF nest list — just vLLM module prefixes."""
prefixes = resolve_mm_encoder_only_lm_prefixes(["language_model"])
assert prefixes == ("language_model.",)
# With Qwen mapper, still module prefixes (keys classified via mapper).
assert resolve_mm_encoder_only_lm_prefixes(
["language_model"], weights_mapper=_qwen25_mapper()
) == ("language_model.",)
def test_resolve_empty_language_model_names_fail_closed():
"""Missing/empty ``_language_model_names`` → None (no language_model. default)."""
assert resolve_mm_encoder_only_lm_prefixes(None) is None
assert resolve_mm_encoder_only_lm_prefixes([]) is None
assert resolve_mm_encoder_only_lm_prefixes(()) is None
def test_resolve_skips_filter_for_shared_hf_root():
"""Shared HF root (Molmo-shaped mapper) → fail-closed, not magic name list."""
assert (
resolve_mm_encoder_only_lm_prefixes(["model"], weights_mapper=_molmo_mapper())
is None
)
# Without a mapper, identity layout: module prefix is returned as-is.
assert resolve_mm_encoder_only_lm_prefixes(["model"]) == ("model.",)
assert resolve_mm_encoder_only_lm_prefixes(["model."]) == ("model.",)
def test_resolve_keeps_llm_prefix_without_model_broadening():
prefixes = resolve_mm_encoder_only_lm_prefixes(["llm"])
assert prefixes == ("llm.",)
def test_qwen3_nested_index_skips_lm_keeps_visual():
with tempfile.TemporaryDirectory() as folder:
files = [
os.path.join(folder, "model-00001-of-000002.safetensors"),
os.path.join(folder, "model-00002-of-000002.safetensors"),
]
for path in files:
open(path, "wb").close()
_write_index(
folder,
{
"model.language_model.layers.0.weight": (
"model-00001-of-000002.safetensors"
),
"lm_head.weight": "model-00001-of-000002.safetensors",
"model.visual.blocks.0.weight": "model-00002-of-000002.safetensors",
},
)
prefixes = resolve_mm_encoder_only_lm_prefixes(
["language_model"], weights_mapper=_qwen3_mapper()
)
assert prefixes is not None
kept = filter_mm_encoder_only_safetensors_files(
files,
folder,
SAFE_WEIGHTS_INDEX_NAME,
prefixes,
weights_mapper=_qwen3_mapper(),
)
assert kept == [files[1]]
def test_qwen25_flat_index_skips_lm_keeps_visual():
with tempfile.TemporaryDirectory() as folder:
files = [
os.path.join(folder, "model-00001-of-000002.safetensors"),
os.path.join(folder, "model-00002-of-000002.safetensors"),
]
for path in files:
open(path, "wb").close()
_write_index(
folder,
{
"model.layers.0.self_attn.q_proj.weight": (
"model-00001-of-000002.safetensors"
),
"model.embed_tokens.weight": "model-00001-of-000002.safetensors",
"model.norm.weight": "model-00001-of-000002.safetensors",
"lm_head.weight": "model-00001-of-000002.safetensors",
"visual.blocks.0.weight": "model-00002-of-000002.safetensors",
},
)
prefixes = resolve_mm_encoder_only_lm_prefixes(
["language_model"], weights_mapper=_qwen25_mapper()
)
assert prefixes is not None
kept = filter_mm_encoder_only_safetensors_files(
files,
folder,
SAFE_WEIGHTS_INDEX_NAME,
prefixes,
weights_mapper=_qwen25_mapper(),
)
assert kept == [files[1]]
def test_molmo_shaped_index_not_filtered_when_deny_disabled():
"""Shared-root LM attr → resolve None; callers leave the file list alone."""
assert (
resolve_mm_encoder_only_lm_prefixes(["model"], weights_mapper=_molmo_mapper())
is None
)
with tempfile.TemporaryDirectory() as folder:
files = [
os.path.join(folder, "vision.safetensors"),
os.path.join(folder, "lm.safetensors"),
]
for path in files:
open(path, "wb").close()
_write_index(
folder,
{
"model.vision_backbone.patch.weight": "vision.safetensors",
"model.transformer.blocks.0.weight": "lm.safetensors",
},
)
# Document why raw HF startswith("model.") is unsafe without fail-closed:
# whole-shard deny would also drop vision.
if_applied_naively = filter_mm_encoder_only_safetensors_files(
files, folder, SAFE_WEIGHTS_INDEX_NAME, ("model.",)
)
assert if_applied_naively == []
@@ -79,7 +79,7 @@ def test_default_loader_hf_still_falls_back_to_pt(tmp_path):
# Control: load_format="hf" still picks up .pt weights via fallback.
(tmp_path / "model.pt").write_bytes(b"\x00\x00\x00\x00")
loader = DefaultModelLoader(LoadConfig(load_format="hf"))
_, files, use_safetensors = loader._prepare_weights(
_, files, use_safetensors, _ = loader._prepare_weights(
str(tmp_path),
None,
None,
@@ -1158,13 +1158,13 @@ def test_kv_scale_reload(vllm_runner):
"Qwen/Qwen3-0.6B",
"inference-optimization/Qwen3-0.6B-debug-multiply",
"inference-optimization/Qwen3-0.6B-debug-add",
"fp8",
"fp8_per_tensor",
),
pytest.param(
"inference-optimization/DeepSeek-V3-debug-empty",
"inference-optimization/DeepSeek-V3-debug-multiply",
"inference-optimization/DeepSeek-V3-debug-add",
"fp8",
"fp8_per_tensor",
marks=[pytest.mark.slow_test],
),
pytest.param(
@@ -1192,7 +1192,7 @@ def test_online_quantize_reload(
if current_platform.device_count() < tp_size:
pytest.skip(reason="Not enough GPU devices")
if quantization == "fp8" and _fp8_reload_unsupported():
if quantization == "fp8_per_tensor" and _fp8_reload_unsupported():
pytest.skip(reason="Requires FP8 support")
with vllm_runner(
@@ -7,14 +7,11 @@ warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
identical outputs.
"""
import glob
import os
import shutil
import subprocess
import sys
import tempfile
import threading
import time
from dataclasses import dataclass, field
from typing import Any
@@ -24,8 +21,6 @@ from vllm import SamplingParams
from vllm.assets.image import ImageAsset
from vllm.platforms import current_platform
DAEMON_TIMEOUT_S = 600
class WeightCacheDaemon:
"""Context manager running the real weight cache daemon as a subprocess."""
@@ -35,18 +30,14 @@ class WeightCacheDaemon:
model: str,
tp_size: int,
extra_args: list[str] | None = None,
num_groups: int = 1,
):
# Short base path: Unix socket paths are limited to ~107 characters.
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
self.tp_size = tp_size
# Each daemon group (target plus cached drafts) binds one socket per
# local rank.
self.num_sockets = tp_size * num_groups
self._cmd = [
sys.executable,
"-m",
"vllm.model_executor.model_loader.weight_cache.daemon",
"vllm.entrypoints.cli.main",
"preload",
"--model",
model,
"--tensor-parallel-size",
@@ -57,18 +48,12 @@ class WeightCacheDaemon:
*(extra_args or []),
]
self._proc: subprocess.Popen | None = None
self._lines: list[str] = []
def __enter__(self) -> "WeightCacheDaemon":
self._proc = subprocess.Popen(
self._cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True
)
threading.Thread(target=self._drain_stderr, daemon=True).start()
try:
self._wait_ready(DAEMON_TIMEOUT_S)
except Exception:
self._stop()
raise
return self
def __exit__(self, *exc_info) -> None:
@@ -76,32 +61,7 @@ class WeightCacheDaemon:
def _drain_stderr(self) -> None:
assert self._proc is not None and self._proc.stderr is not None
for line in self._proc.stderr:
self._lines.append(line)
def _logs(self) -> str:
return "".join(self._lines)
def _wait_ready(self, timeout_s: float) -> None:
assert self._proc is not None
# Each rank binds its socket only once the model is fully cached.
# Poll for the socket files rather than a log line: model loading can
# pull in JIT compilers that swap the process's stderr and swallow
# everything logged afterwards, making log-based readiness flaky.
pattern = os.path.join(self.socket_dir, "vllm_weight_cache_*.sock")
deadline = time.monotonic() + timeout_s
while time.monotonic() < deadline:
if self._proc.poll() is not None:
raise RuntimeError(
f"Weight cache daemon exited with {self._proc.returncode}:\n"
f"{self._logs()}"
)
if len(glob.glob(pattern)) >= self.num_sockets:
return
time.sleep(1.0)
raise TimeoutError(
f"Weight cache daemon not ready after {timeout_s}s:\n{self._logs()}"
)
self._proc.stderr.read()
def _stop(self) -> None:
assert self._proc is not None
@@ -121,9 +81,6 @@ class ModelCase:
images: list | None = None
llm_kwargs: dict[str, Any] = field(default_factory=dict)
daemon_args: list[str] = field(default_factory=list)
# Daemon groups the launcher starts; 2 when a cached draft group joins the
# target group (MTP/EAGLE speculative decoding).
daemon_groups: int = 1
def generate(
@@ -203,7 +160,6 @@ QWEN_MTP_CASE = ModelCase(
"--speculative-config",
'{"method": "mtp", "num_speculative_tokens": 1}',
],
daemon_groups=2,
)
@@ -242,7 +198,6 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
case.model,
tp_size=1,
extra_args=case.daemon_args,
num_groups=case.daemon_groups,
) as d:
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
# Warm restart: a second engine lifetime against the same daemon.
@@ -251,3 +206,116 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
assert cold_outputs == baseline_outputs
assert warm_outputs == baseline_outputs
assert restart_outputs == baseline_outputs
def _parallel(**kw):
from types import SimpleNamespace
base = dict(
tensor_parallel_size=1,
pipeline_parallel_size=1,
data_parallel_size=1,
data_parallel_size_local=1,
data_parallel_rank=0,
nnodes=1,
node_rank=0,
)
base.update(kw)
return SimpleNamespace(**base)
def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
"""Each node serves a contiguous global-rank block (DP-major), so without
DP local GPU i is TP rank r*local+i, and single-node DP launchers offset
by --data-parallel-start-rank."""
from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks
tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1)
assert plan_local_ranks(tp_second_node) == [(i, 0, 4 + i) for i in range(4)]
dp_third_node = _parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8
)
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0) for i in range(4)]
dp_tp = _parallel(
tensor_parallel_size=2,
data_parallel_size=4,
data_parallel_size_local=2,
data_parallel_rank=2,
)
assert plan_local_ranks(dp_tp) == [(0, 2, 0), (1, 2, 1), (2, 3, 0), (3, 3, 1)]
# DP + nnodes: node-local TP replicas
dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2)
assert plan_local_ranks(dp_tp_node0) == [(i, 0, i) for i in range(8)]
dp_tp_node1 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1
)
assert plan_local_ranks(dp_tp_node1) == [(i, 1, i) for i in range(8)]
# DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes)
tp_span_node1 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1
)
assert plan_local_ranks(tp_span_node1) == [(i, 0, 4 + i) for i in range(4)]
tp_span_node2 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2
)
assert plan_local_ranks(tp_span_node2) == [(i, 1, i) for i in range(4)]
# DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes)
dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, i % 4) for i in range(8)]
def test_daemon_rejects_unmappable_parallelism():
from vllm.model_executor.model_loader.weight_cache.daemon import (
_reject_unsupported_parallelism,
)
_reject_unsupported_parallelism(
_parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=12
)
)
# DP combines with --nnodes when the world size divides evenly.
_reject_unsupported_parallelism(
_parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
)
with pytest.raises(ValueError, match="pipeline"):
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
with pytest.raises(ValueError, match="evenly divide"):
_reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2))
with pytest.raises(ValueError, match="evenly divide"):
_reject_unsupported_parallelism(
_parallel(tensor_parallel_size=2, data_parallel_size=3, nnodes=4)
)
with pytest.raises(ValueError, match="exceeds"):
_reject_unsupported_parallelism(
_parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=13
)
)
def test_weight_cache_key_distinguishes_dp_ranks():
from dataclasses import replace
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
key = WeightCacheKey(
checkpoint="ckpt",
model_arch="Arch",
tp_size=1,
tp_rank=0,
dtype="bf16",
quantization=None,
quant_config_hash="h",
revision=None,
vllm_version="v",
dp_size=16,
dp_rank=3,
)
assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"]
assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"]
@@ -7,6 +7,7 @@ import torch
from vllm.model_executor.layers import utils
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_default_torch_dtype
@pytest.fixture(scope="module")
@@ -89,3 +90,25 @@ def test_dispatch_cpu_unquantized_gemm_logs_zentorch_dispatch(monkeypatch):
expected_prepacked,
)
]
@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
@pytest.mark.parametrize("weight_dtype", [torch.bfloat16, torch.float16, torch.float32])
def test_dispatch_cpu_unquantized_gemm_remove_weight_keeps_dtype(
monkeypatch, weight_dtype
):
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
layer = torch.nn.Linear(16, 8, bias=False, dtype=weight_dtype)
loading_dtype = (
torch.float32 if weight_dtype is not torch.float32 else torch.bfloat16
)
with set_default_torch_dtype(loading_dtype):
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
assert layer.weight.numel() == 0
assert layer.weight.dtype is weight_dtype
x = torch.randn(4, 16, dtype=weight_dtype)
output = layer.cpu_linear(x, layer.weight, None)
assert not output.isnan().any()
@@ -1,14 +1,17 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from dataclasses import dataclass
from typing import Any
import pytest
import torch
from vllm import envs
from vllm.model_executor.warmup import jit_warmup_triton_helper
from vllm.model_executor.warmup.jit_warmup import (
JitWarmupRegistry,
WarmupChoices,
WarmupIntRange,
kernel_launcher,
@@ -116,6 +119,105 @@ def test_triton_launcher_supports_compile_and_runtime_adapters() -> None:
assert runtime_calls == [(owner.kernel, (2,), ("runtime", 2), {"CONST": 7})]
@pytest.mark.skipif(
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
reason="Requires CUDA Triton async compilation",
)
def test_failed_parallel_warmup_does_not_leak_into_runtime(monkeypatch) -> None:
from triton.runtime._async_compile import active_mode
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
owner = _TestTritonKernel()
owner.kernel = _FakeTritonKernel()
def fail():
raise RuntimeError("warmup failed")
def warmup(**kwargs):
active_mode.get().submit(kwargs["second"], fail, lambda result: None)
monkeypatch.setattr(owner.kernel, "warmup", warmup)
with pytest.raises(RuntimeError, match="warmup failed"):
owner.compile_many(owner.CompileKey(value=i) for i in range(2))
assert active_mode.get() is None
owner("runtime", 3, None)
assert len(owner.kernel.runtime_calls) == 1
def test_autotuning_warmup_stays_serial(monkeypatch) -> None:
owner = _TestTritonKernel()
owner.kernel = _FakeTritonKernel()
owner._run_autotune = True
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 4)
def unexpected_async(*args):
pytest.fail("Autotuning must not use asynchronous compilation")
monkeypatch.setattr(triton, "AsyncCompileMode", unexpected_async, raising=False)
owner.compile_many(owner.CompileKey(value=i) for i in range(2))
assert len(owner.kernel.runtime_calls) == 2
@triton.jit
def _warmup_store_kernel(out, VALUE: tl.constexpr):
tl.store(out, VALUE)
@pytest.mark.skipif(
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
reason="Requires CUDA Triton async compilation",
)
def test_registered_warmup_parallelizes_compile_only_and_runtime_stays_serial(
monkeypatch,
) -> None:
from triton.runtime._async_compile import active_mode
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
def warmup_inputs():
return dict(out=TritonWarmupTensor(torch.int32), value=WarmupChoices(1, 2))
@triton_kernel_dispatcher_with_warmup(
kernel=_warmup_store_kernel, warmup_inputs=warmup_inputs
)
def store(out, value):
return (1,), dict(VALUE=value)
store.get_warmup_keys() # Initialize Triton's binder before wrapping its compiler.
caller = threading.get_ident()
barrier = threading.Barrier(2, timeout=10)
compiled_on = []
compile_kernel = _warmup_store_kernel.compile
def record_compile(*args, **kwargs):
if threading.get_ident() != caller:
barrier.wait()
result = compile_kernel(*args, **kwargs)
compiled_on.append(threading.get_ident())
return result
monkeypatch.setattr(_warmup_store_kernel, "compile", record_compile)
registry = JitWarmupRegistry(None)
with registry.activate():
store.register_warmup()
registry.warmup()
assert len(compiled_on) == 2
assert all(thread != caller for thread in compiled_on)
assert active_mode.get() is None
out = torch.empty(1, dtype=torch.int32, device=current_platform.device_type)
for value in (1, 2):
store(out, value)
assert out.item() == value
assert len(compiled_on) == 2
# A specialization first seen during inference uses the normal blocking JIT.
store(out, 3)
assert out.item() == 3
assert compiled_on[2:] == [caller]
def test_triton_launcher_supports_cpu_function_wrappers() -> None:
calls: list[tuple[Any, ...]] = []
+7 -2
View File
@@ -8,7 +8,7 @@ from unittest.mock import Mock, patch
import pytest
import torch
import torch.nn as nn
from transformers import PretrainedConfig
from transformers import PreTrainedConfig
from vllm.config import ParallelConfig
from vllm.multimodal.processing import InputProcessingContext
@@ -52,7 +52,7 @@ def print_input_ids(input_ids):
@pytest.fixture
def mock_qwen3_omni_config():
"""Create a mock Qwen3OmniMoeThinker config."""
config = Mock(spec=PretrainedConfig)
config = Mock(spec=PreTrainedConfig)
# Token IDs from https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
config.audio_token_id = 151675 # <|audio_pad|>
config.video_token_id = 151656 # <|video_pad|>
@@ -144,6 +144,11 @@ def test_qwen3_omni_get_updates_use_audio_in_video(
# Create a mock context
mock_ctx = Mock(spec=InputProcessingContext)
mock_ctx.tokenizer = mock_tokenizer
# `model_config` is an instance attribute, so it is not covered by the
# spec; the data parser reads it for `allow_missing_mm_embeddings`.
mock_ctx.model_config = SimpleNamespace(
multimodal_config=SimpleNamespace(allow_missing_mm_embeddings=False)
)
# Create processing info
info = Qwen3OmniMoeThinkerProcessingInfo(mock_ctx)
@@ -0,0 +1,222 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the apply-time Triton kernel override registry.
The registry must let out-of-tree platforms replace kernels defined as
plain ``@triton.jit`` functions, patching every hold-site (defining
module, from-import copies, and JIT warmup owners) so that existing call
sites dispatch to the override without any core-side decoration.
"""
import sys
import types
from typing import Any
import pytest
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
_DecoratedTritonJitKernel,
)
from vllm.triton_utils import tl, triton
from vllm.triton_utils.dispatcher import (
KernelOverride,
register_kernels,
)
class _FakeTritonKernel:
"""Stands in for a Triton JITFunction (works without Triton installed)."""
arg_names = ("output_ptr", "input_ptr", "replace_from", "MAX_NUM_TOKENS")
def __getitem__(self, grid: Any) -> Any:
def launch(*args: Any, **kwargs: Any) -> Any:
return ("default", grid, kwargs)
return launch
@pytest.fixture
def _fake_module(monkeypatch: pytest.MonkeyPatch) -> Any:
"""Expose a fake defining module holding the kernel and its warmup owner."""
import types
kernel = _FakeTritonKernel()
owner = _DecoratedTritonJitKernel(
kernel,
lambda: {},
lambda output, input, replace_from: ( # noqa: ARG005
(1,),
dict(MAX_NUM_TOKENS=8),
),
)
module = types.ModuleType("vllm.tests.fake_kernel_home")
module.expand_kernel = kernel # type: ignore[attr-defined]
module._expand = owner # type: ignore[attr-defined]
monkeypatch.setitem(sys.modules, module.__name__, module)
return module
def test_override_patches_defining_module_and_warmup_owner(
_fake_module: Any,
) -> None:
calls: list[tuple[Any, ...]] = []
def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None):
calls.append((output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS))
return "cpu"
register_kernels(
{
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
}
)
overridden = _fake_module.expand_kernel
assert isinstance(overridden, KernelOverride)
# Launch with kernel-arg names like the warmup launch path does.
assert overridden[(4,)](output_ptr=1, input_ptr=2, replace_from=0) == "cpu"
assert calls == [(1, 2, 0, None)]
# The warmup owner's captured kernel is patched too, and its cached
# argument-name introspection is invalidated.
assert _fake_module._expand.kernel is overridden
def test_override_forwards_positionally_for_mismatched_names(
_fake_module: Any,
) -> None:
calls: list[tuple[Any, ...]] = []
def cpu_impl(output, input_val, replace_from, MAX_NUM_TOKENS=None):
calls.append((output, input_val, replace_from, MAX_NUM_TOKENS))
return "cpu"
register_kernels(
{
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
}
)
overridden = _fake_module.expand_kernel
assert overridden[(4,)](output_ptr=1, input_ptr=2, replace_from=0) == "cpu"
assert calls == [(1, 2, 0, None)]
def test_override_covers_from_import_copies(_fake_module: Any) -> None:
import types
importer = types.ModuleType("vllm.tests.fake_kernel_importer")
importer.expand_kernel = _fake_module.expand_kernel # type: ignore[attr-defined]
sys.modules["vllm.tests.fake_kernel_importer"] = importer
def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None):
return "cpu"
register_kernels(
{
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
}
)
assert isinstance(importer.expand_kernel, KernelOverride)
del sys.modules["vllm.tests.fake_kernel_importer"]
def test_override_owner_launch_goes_through_impl(_fake_module: Any) -> None:
launched: list[Any] = []
def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None):
launched.append((output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS))
return "cpu"
register_kernels(
{
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
}
)
owner = _fake_module._expand
# In-place kernels return None from launch; assert on the recorded call.
owner(1, 2, 0)
assert launched == [(1, 2, 0, 8)]
def test_register_kernel_auto_imports_defining_module(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Registering a kernel whose defining module is not imported yet must
import it on the spot and apply the override."""
kernel_name = "vllm.v1.sample.rejection_sampler.expand_kernel"
module_name = kernel_name.rpartition(".")[0]
calls: list[tuple[Any, ...]] = []
def cpu_impl(
output_ptr,
input_ptr,
cu_num_tokens_ptr,
replace_from,
replace_to,
MAX_NUM_TOKENS=None,
):
calls.append((output_ptr, replace_from, MAX_NUM_TOKENS))
return "cpu"
# Ensure the module is NOT imported before registration. Drop it from
# sys.modules; a fresh import creates new kernel/owner objects, which
# is what a cold-start platform process would see.
saved = sys.modules.pop(module_name, None)
# Re-run the registry patch against the original module afterwards so
# other tests that already imported it keep working.
try:
assert module_name not in sys.modules
register_kernels({kernel_name: cpu_impl})
assert module_name in sys.modules
finally:
if saved is None:
# First import in the whole session: keep the fresh module.
saved = sys.modules[module_name]
else:
# Restore the original module object and re-apply the override
# to its (pre-existing) kernel so both generations agree.
sys.modules[module_name] = saved
register_kernels({kernel_name: cpu_impl})
overridden = saved.expand_kernel
assert isinstance(overridden, KernelOverride)
overridden[(2,)](
output_ptr=1,
input_ptr=2,
cu_num_tokens_ptr=3,
replace_from=0,
replace_to=0,
MAX_NUM_TOKENS=8,
)
assert calls == [(1, 0, 8)]
# The warmup owner captured at the fresh import must be patched too
# (when the module existed before, there are two module generations).
fresh = sys.modules[module_name]
if fresh is not saved:
assert fresh._expand.kernel is overridden
def test_real_triton_kernel_override_end_to_end() -> None:
"""Override a real @triton.jit kernel through the registry."""
if not triton.__class__.__module__.startswith("triton"):
pytest.skip("Triton is not installed")
@triton.jit
def _add_one_kernel(x_ptr, out_ptr, BLOCK: tl.constexpr): # pragma: no cover
offsets = tl.arange(0, BLOCK)
tl.store(out_ptr + offsets, tl.load(x_ptr + offsets) + 1)
module = types.ModuleType("vllm.tests.real_jit_kernel_home")
module._add_one_kernel = _add_one_kernel # type: ignore[attr-defined]
sys.modules[module.__name__] = module
def cpu_impl(x_ptr, out_ptr, BLOCK=None): # noqa: ARG001
return "cpu"
register_kernels({"vllm.tests.real_jit_kernel_home._add_one_kernel": cpu_impl})
assert module._add_one_kernel.arg_names == ( # type: ignore[attr-defined]
"x_ptr",
"out_ptr",
"BLOCK",
)
+15
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tempfile
from unittest.mock import Mock
import huggingface_hub.constants
import pytest
@@ -9,6 +10,7 @@ from huggingface_hub.utils import LocalEntryNotFoundError
from vllm.model_executor.model_loader.weight_utils import (
download_weights_from_hf,
drop_checkpoint_cache,
maybe_remap_kv_scale_name,
)
@@ -292,5 +294,18 @@ class TestKvCacheScaleMapper:
)
@pytest.mark.parametrize("fails", [False, True])
def test_drop_checkpoint_cache_releases_local_weight_files(
tmp_path, monkeypatch, fails
):
"""Evict only weights; cache eviction errors must not prevent loading."""
for name in ("weights.safetensors", "config.json"):
(tmp_path / name).touch()
fadvise = Mock(side_effect=OSError("cache eviction failed") if fails else None)
monkeypatch.setattr("os.posix_fadvise", fadvise)
drop_checkpoint_cache(str(tmp_path))
fadvise.assert_called_once()
if __name__ == "__main__":
test_download_weights_from_hf()