Merge remote-tracking branch 'origin/main' into sync/lesj/qwen4exp-bugfixes
Signed-off-by: lesj0610 <lesj0610@users.noreply.github.com>
This commit is contained in:
@@ -66,7 +66,7 @@ from vllm.platforms import PlatformEnum
|
||||
"MarlinMxfp8LinearKernel",
|
||||
"EmulationMxfp8LinearKernel",
|
||||
init_mxfp8_linear_kernel,
|
||||
{},
|
||||
{"weight_shape": (2048, 2048)},
|
||||
),
|
||||
(
|
||||
B12xTensorFP8ScaledMMLinearKernel,
|
||||
@@ -365,7 +365,7 @@ def test_b12x_tensor_fp8_apply_quantizes_and_uses_packed_weight(
|
||||
|
||||
def test_b12x_mxfp8_can_implement_supported_config() -> None:
|
||||
can_implement, reason = B12xMxfp8LinearKernel.can_implement(
|
||||
Mxfp8LinearLayerConfig()
|
||||
Mxfp8LinearLayerConfig(weight_shape=(256, 512))
|
||||
)
|
||||
|
||||
assert can_implement
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_shape,bias",
|
||||
[
|
||||
pytest.param((256, 3, 14, 14), False, id="packed_patches"),
|
||||
pytest.param((2, 3, 28, 28), True, id="multiple_spatial_patches"),
|
||||
],
|
||||
)
|
||||
def test_conv2d_patch_embedding_correctness(default_vllm_config, input_shape, bias):
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
dtype = torch.bfloat16 if device == "cuda" else torch.float32
|
||||
reference = nn.Conv2d(3, 1024, 14, stride=14, bias=bias).to(
|
||||
device=device, dtype=dtype
|
||||
)
|
||||
layer = Conv2dLayer(3, 1024, 14, stride=14, bias=bias, params_dtype=dtype).to(
|
||||
device
|
||||
)
|
||||
layer.load_state_dict(reference.state_dict(), strict=True)
|
||||
assert layer.enable_linear
|
||||
|
||||
x = torch.randn(input_shape, device=device, dtype=dtype)
|
||||
expected = reference(x)
|
||||
atol = rtol = 0.02 if dtype == torch.bfloat16 else 1e-5
|
||||
torch.testing.assert_close(layer.forward_native(x), expected, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"input_shape,kernel_size,out_channels,bias",
|
||||
[
|
||||
pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1280, False, id="minimax_m3"),
|
||||
pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1152, False, id="qwen2_vl"),
|
||||
pytest.param((4, 3, 2, 14, 14), (2, 14, 14), 1152, True, id="qwen3_vl"),
|
||||
pytest.param((4, 3, 1, 14, 14), (1, 14, 14), 1536, True, id="glm_vl"),
|
||||
pytest.param((1, 3, 4, 28, 28), (2, 14, 14), 32, True, id="multiple_patches"),
|
||||
],
|
||||
)
|
||||
def test_conv3d_patch_embedding_correctness(
|
||||
default_vllm_config, input_shape, kernel_size, out_channels, bias
|
||||
):
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
dtype = torch.bfloat16 if device == "cuda" else torch.float32
|
||||
reference = nn.Conv3d(
|
||||
input_shape[1], out_channels, kernel_size, stride=kernel_size, bias=bias
|
||||
).to(device=device, dtype=dtype)
|
||||
layer = Conv3dLayer(
|
||||
input_shape[1],
|
||||
out_channels,
|
||||
kernel_size,
|
||||
stride=kernel_size,
|
||||
bias=bias,
|
||||
params_dtype=dtype,
|
||||
).to(device)
|
||||
layer.load_state_dict(reference.state_dict(), strict=True)
|
||||
assert layer.enable_linear
|
||||
|
||||
x = torch.randn(input_shape, device=device, dtype=dtype)
|
||||
expected = reference(x)
|
||||
atol = rtol = 0.02 if dtype == torch.bfloat16 else 1e-5
|
||||
torch.testing.assert_close(layer(x), expected, atol=atol, rtol=rtol)
|
||||
@@ -1227,7 +1227,6 @@ def test_non_quark_shared_expert_fse_is_incompatible() -> None:
|
||||
|
||||
def _fp8_config(**kwargs: Any) -> Fp8Config:
|
||||
return Fp8Config(
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
activation_scheme="dynamic",
|
||||
weight_block_size=[128, 128],
|
||||
**kwargs,
|
||||
@@ -1248,7 +1247,7 @@ def test_block_fp8_shared_expert_fse_is_compatible() -> None:
|
||||
def test_per_tensor_fp8_shared_expert_fse_is_incompatible() -> None:
|
||||
"""Per-tensor scales are 0-D, so the shared-expert chunker cannot slice them."""
|
||||
compatible, reason = is_shared_expert_quant_fse_compatible(
|
||||
Fp8Config(is_checkpoint_fp8_serialized=True, activation_scheme="dynamic"),
|
||||
Fp8Config(activation_scheme="dynamic"),
|
||||
"model.layers.0.mlp.experts",
|
||||
"model.layers.0.mlp.shared_experts",
|
||||
)
|
||||
|
||||
@@ -677,6 +677,40 @@ def test_candidate_blocks_to_sparse_indices_matches_reference(
|
||||
assert not end_buf[1 - end_offset :: 2].any()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_sparse_topk_remap_bounds_nan_rows():
|
||||
"""DeepSelect writes an out-of-range sentinel to slot 0 of NaN rows (the
|
||||
rest is left untouched); the remap turns it into -1."""
|
||||
from vllm.model_executor.kernels.attention.dsa.sparse_mqa_logits import (
|
||||
has_deep_select,
|
||||
sparse_topk_remap,
|
||||
)
|
||||
|
||||
if not has_deep_select():
|
||||
pytest.skip("DeepSelect extension (vllm._deepselect_C) required")
|
||||
rows, num_sparse, sbk, topk = 4, 128, 8, 512
|
||||
width = num_sparse * sbk
|
||||
logits = torch.randn(rows, width, dtype=torch.bfloat16, device="cuda")
|
||||
logits[1:, :16] = float("nan")
|
||||
sparse_indices = torch.arange(num_sparse, dtype=torch.int32, device="cuda")
|
||||
end = torch.full((rows,), width, dtype=torch.int32, device="cuda")
|
||||
out = torch.empty(rows, topk, dtype=torch.int32, device="cuda")
|
||||
cols = torch.zeros_like(out)
|
||||
|
||||
sparse_topk_remap(
|
||||
logits,
|
||||
sparse_indices.repeat(rows, 1),
|
||||
end,
|
||||
torch.zeros_like(end),
|
||||
sbk,
|
||||
topk,
|
||||
out,
|
||||
col_indices=cols,
|
||||
)
|
||||
assert (cols[1:, 0] >= width).all() and (out[1:, 0] == -1).all()
|
||||
assert (out[0] >= 0).all()
|
||||
|
||||
|
||||
def _skip_unless_sm100_sparse_kernels():
|
||||
if not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10:
|
||||
pytest.skip("SM100 required")
|
||||
|
||||
@@ -0,0 +1,320 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for FusedMMInputNorm and the fused affine transform kernel."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
||||
FusedMMInputNorm,
|
||||
IdentityInputNorm,
|
||||
NormParams,
|
||||
fused_mm_input_norm_triton,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
# Module-level tests run on any platform: the CPU platform exercises
|
||||
# forward_native, accelerators exercise the fused kernel. Only the raw
|
||||
# kernel tests require an accelerator.
|
||||
_DEVICE = torch.device(current_platform.device_type)
|
||||
|
||||
requires_accelerator = pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason="fused Triton kernel requires a CUDA/XPU accelerator",
|
||||
)
|
||||
requires_triton = pytest.mark.skipif(not HAS_TRITON, reason="requires Triton")
|
||||
requires_vllm_config = pytest.mark.usefixtures("default_vllm_config")
|
||||
|
||||
|
||||
def _reference_input_norm(
|
||||
pixel_values: torch.Tensor,
|
||||
image_mean: list[float],
|
||||
image_std: list[float],
|
||||
rescale_factor: float,
|
||||
channel: int,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Straightforward per-channel affine: (x * rescale - mean) / std."""
|
||||
patches, size = pixel_values.shape
|
||||
patch_size = size // channel
|
||||
mean = torch.tensor(
|
||||
image_mean, dtype=torch.float32, device=pixel_values.device
|
||||
).view(1, channel, 1)
|
||||
std = torch.tensor(image_std, dtype=torch.float32, device=pixel_values.device).view(
|
||||
1, channel, 1
|
||||
)
|
||||
x = pixel_values.to(torch.float32).reshape(patches, channel, patch_size)
|
||||
x = (x * rescale_factor - mean) / std
|
||||
return x.view(patches, size).to(out_dtype)
|
||||
|
||||
|
||||
# CLIP-style RGB normalization constants reused across tests.
|
||||
_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073]
|
||||
_RGB_STD = [0.26862954, 0.26130258, 0.27577711]
|
||||
_RGB_RESCALE = 1.0 / 255.0
|
||||
|
||||
|
||||
def _check_against_reference(
|
||||
pixel_values: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
channel: int = 3,
|
||||
image_mean: list[float] | None = None,
|
||||
image_std: list[float] | None = None,
|
||||
rescale_factor: float = _RGB_RESCALE,
|
||||
**close_kwargs,
|
||||
):
|
||||
"""Run ``FusedMMInputNorm`` on ``pixel_values`` and compare against the
|
||||
eager per-channel affine reference. Uses the RGB constants by default."""
|
||||
if image_mean is None:
|
||||
image_mean = list(_RGB_MEAN)
|
||||
if image_std is None:
|
||||
image_std = list(_RGB_STD)
|
||||
norm = FusedMMInputNorm(
|
||||
image_mean=image_mean,
|
||||
image_std=image_std,
|
||||
rescale_factor=rescale_factor,
|
||||
channel=channel,
|
||||
).to(_DEVICE)
|
||||
expected = _reference_input_norm(
|
||||
pixel_values, image_mean, image_std, rescale_factor, channel, out_dtype
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
norm(pixel_values, visual_dtype=out_dtype), expected, **close_kwargs
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Module-level behavior
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormModule:
|
||||
def test_matches_reference_above_cudnn_grid_limit(self):
|
||||
"""num_patches above the old cuDNN batch-norm grid limit (~65535)
|
||||
used to raise CUDNN_STATUS_INTERNAL_ERROR."""
|
||||
set_random_seed(0)
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (70000, 3 * 196), dtype=torch.float32, device=_DEVICE
|
||||
)
|
||||
_check_against_reference(pixel_values, torch.float32)
|
||||
|
||||
def test_identity_passthrough(self):
|
||||
norm = IdentityInputNorm()
|
||||
|
||||
pixel_values = torch.randn(8, 3 * 196, dtype=torch.float32, device=_DEVICE)
|
||||
out = norm(pixel_values, visual_dtype=torch.bfloat16)
|
||||
torch.testing.assert_close(out, pixel_values.to(torch.bfloat16))
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# dtype coverage
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormDtypes:
|
||||
@pytest.mark.parametrize(
|
||||
"in_dtype,out_dtype",
|
||||
[
|
||||
(torch.float32, torch.float32),
|
||||
(torch.float32, torch.bfloat16),
|
||||
(torch.float32, torch.float16),
|
||||
(torch.bfloat16, torch.bfloat16),
|
||||
(torch.bfloat16, torch.float32),
|
||||
(torch.float16, torch.float16),
|
||||
(torch.uint8, torch.bfloat16),
|
||||
(torch.uint8, torch.float32),
|
||||
],
|
||||
)
|
||||
def test_dtype_combinations(self, in_dtype: torch.dtype, out_dtype: torch.dtype):
|
||||
set_random_seed(0)
|
||||
if in_dtype == torch.uint8:
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (32, 3 * 16), dtype=torch.uint8, device=_DEVICE
|
||||
)
|
||||
else:
|
||||
pixel_values = torch.rand(
|
||||
(32, 3 * 16), dtype=torch.float32, device=_DEVICE
|
||||
).to(in_dtype)
|
||||
|
||||
_check_against_reference(
|
||||
pixel_values,
|
||||
out_dtype,
|
||||
image_mean=[0.5, 0.5, 0.5],
|
||||
image_std=[0.5, 0.5, 0.5],
|
||||
atol=1e-2,
|
||||
rtol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Shape / channel coverage
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormShapes:
|
||||
@pytest.mark.parametrize("channel", [1, 3, 4])
|
||||
def test_channel_variants(self, channel: int):
|
||||
set_random_seed(0)
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (16, channel * 64), dtype=torch.float32, device=_DEVICE
|
||||
)
|
||||
_check_against_reference(
|
||||
pixel_values,
|
||||
torch.float32,
|
||||
channel=channel,
|
||||
image_mean=[0.5] * channel,
|
||||
image_std=[0.25] * channel,
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("patch_size", [1, 2048])
|
||||
def test_block_boundaries(self, patch_size: int):
|
||||
"""Masked-tail (12 elements) and exact-multiple (24 blocks) paths."""
|
||||
set_random_seed(0)
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (4, 3 * patch_size), dtype=torch.float32, device=_DEVICE
|
||||
)
|
||||
_check_against_reference(pixel_values, torch.float32)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Input handling: non-contiguous inputs
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormInputHandling:
|
||||
def test_non_contiguous_input_matches_reference(self):
|
||||
set_random_seed(0)
|
||||
base = torch.randint(
|
||||
0, 256, (8, 3 * 32, 2), dtype=torch.float32, device=_DEVICE
|
||||
)
|
||||
non_contig = base[..., 0]
|
||||
assert not non_contig.is_contiguous()
|
||||
|
||||
_check_against_reference(non_contig, torch.float32)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Raw kernel entry point
|
||||
# ===========================================================================
|
||||
@requires_accelerator
|
||||
@requires_triton
|
||||
class TestFusedMMInputNormKernel:
|
||||
@pytest.mark.parametrize(
|
||||
"N, C, L",
|
||||
[
|
||||
(1, 1, 1),
|
||||
(1, 3, 1),
|
||||
(3, 3, 7), # tile straddles channel boundaries
|
||||
(2, 8, 63),
|
||||
(4, 3, 1000),
|
||||
],
|
||||
)
|
||||
def test_channel_boundary_crossing(self, N: int, C: int, L: int):
|
||||
"""The 1D kernel recovers ``c = (offs // L) % C`` per lane, so a tile
|
||||
can straddle a channel boundary and gather weight/bias per lane.
|
||||
These shapes force that path by keeping ``L`` well below the block."""
|
||||
set_random_seed(0)
|
||||
x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE)
|
||||
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
|
||||
out = torch.empty_like(x)
|
||||
fused_mm_input_norm_triton(x, out, w, b)
|
||||
|
||||
expected = x * w.view(1, C, 1) + b.view(1, C, 1)
|
||||
torch.testing.assert_close(out, expected)
|
||||
|
||||
def test_non_contiguous_inputs_materialized(self):
|
||||
"""Read-only tensors are made contiguous internally."""
|
||||
N, C, L = 3, 3, 100
|
||||
set_random_seed(0)
|
||||
base = torch.randn(N, C, L, 2, dtype=torch.float32, device=_DEVICE)
|
||||
x = base[..., 0]
|
||||
assert not x.is_contiguous()
|
||||
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
|
||||
out = torch.empty(N, C, L, dtype=torch.float32, device=_DEVICE)
|
||||
fused_mm_input_norm_triton(x, out, w, b)
|
||||
|
||||
expected = x * w.view(1, C, 1) + b.view(1, C, 1)
|
||||
torch.testing.assert_close(out, expected)
|
||||
|
||||
def test_rejects_mismatched_output_shape(self):
|
||||
"""``outputs`` must be shaped exactly like ``inputs``."""
|
||||
N, C, L = 3, 3, 100
|
||||
x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE)
|
||||
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
|
||||
for out_shape in ((N, C + 1, L), (N, C, L + 1), (N + 2, C, L)):
|
||||
with pytest.raises(AssertionError):
|
||||
fused_mm_input_norm_triton(
|
||||
x,
|
||||
torch.empty(*out_shape, dtype=torch.float32, device=_DEVICE),
|
||||
w,
|
||||
b,
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Construction / configuration
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormConstruction:
|
||||
"""Weight/bias buffer semantics at init time."""
|
||||
|
||||
def test_identity_config_buffers(self):
|
||||
"""Numerically identity parameters still build plain weight/bias
|
||||
buffers; the identity shortcut lives in ``from_model_config``."""
|
||||
norm = FusedMMInputNorm(
|
||||
image_mean=[0.0, 0.0, 0.0],
|
||||
image_std=[1.0, 1.0, 1.0],
|
||||
rescale_factor=1.0,
|
||||
)
|
||||
torch.testing.assert_close(norm.weight, torch.ones(3))
|
||||
torch.testing.assert_close(norm.bias, torch.zeros(3))
|
||||
|
||||
def test_non_identity_buffers(self):
|
||||
channel = 3
|
||||
norm = FusedMMInputNorm(
|
||||
image_mean=_RGB_MEAN,
|
||||
image_std=_RGB_STD,
|
||||
rescale_factor=_RGB_RESCALE,
|
||||
channel=channel,
|
||||
)
|
||||
assert norm.weight.shape == (channel,)
|
||||
assert norm.bias.shape == (channel,)
|
||||
|
||||
mean = torch.tensor(_RGB_MEAN, dtype=torch.float32)
|
||||
std = torch.tensor(_RGB_STD, dtype=torch.float32)
|
||||
torch.testing.assert_close(norm.weight, _RGB_RESCALE / std)
|
||||
torch.testing.assert_close(norm.bias, -mean / std)
|
||||
|
||||
def test_is_identity_params(self):
|
||||
"""The numeric-identity check mirrors the old allclose detection."""
|
||||
assert NormParams([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0).is_identity
|
||||
assert not NormParams(_RGB_MEAN, _RGB_STD, _RGB_RESCALE).is_identity
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("image_mean", "image_std", "rescale_factor"),
|
||||
[
|
||||
([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0),
|
||||
([0.5, 0.5, 0.5], [0.25, 0.25, 0.25], 1 / 255),
|
||||
],
|
||||
)
|
||||
def test_buffers_initialized_on_default_device(
|
||||
self,
|
||||
image_mean: list[float],
|
||||
image_std: list[float],
|
||||
rescale_factor: float,
|
||||
):
|
||||
"""Buffers are built on CPU first, then moved to the caller's
|
||||
default device, so init never touches the accelerator."""
|
||||
# The meta device gives the CPU-only test shard the same non-CPU
|
||||
# default-device semantics without requiring a CUDA build.
|
||||
default_device = "cuda" if torch.cuda.is_available() else "meta"
|
||||
with torch.device(default_device):
|
||||
norm = FusedMMInputNorm(image_mean, image_std, rescale_factor)
|
||||
|
||||
assert norm.weight.device.type == default_device
|
||||
assert norm.bias.device.type == default_device
|
||||
@@ -9,7 +9,7 @@ from unittest.mock import patch
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import PoolerConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
|
||||
@@ -315,7 +315,7 @@ def test_dispatch_seq_cls_honors_token_pooling_type(tok_pooling_type):
|
||||
model_config=SimpleNamespace(
|
||||
pooler_config=pooler_config,
|
||||
head_dtype=None,
|
||||
hf_config=PretrainedConfig(num_labels=3),
|
||||
hf_config=PreTrainedConfig(num_labels=3),
|
||||
),
|
||||
)
|
||||
classifier = torch.nn.Linear(4, 3)
|
||||
|
||||
@@ -17,9 +17,10 @@ if current_platform.is_cuda():
|
||||
from vllm.model_executor.layers import utils
|
||||
|
||||
|
||||
def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch):
|
||||
@pytest.mark.parametrize("m", [1, 128])
|
||||
def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch, m):
|
||||
x = torch.randn(1, 64, dtype=torch.float16)
|
||||
weight = torch.randn(128, 64, dtype=torch.float16)
|
||||
weight = torch.randn(m, 64, dtype=torch.float16)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
@@ -42,6 +43,20 @@ def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch):
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("n", range(1, 6))
|
||||
def test_rocm_unquantized_gemm_single_output_real_kernel(dtype, n):
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(n, 2048, device="cuda", dtype=dtype)
|
||||
weight = torch.randn(1, 2048, device="cuda", dtype=dtype)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def test_rocm_unquantized_gemm_makes_skinny_activation_contiguous(monkeypatch):
|
||||
x = torch.randn(64, 4, dtype=torch.float16).t()
|
||||
weight = torch.randn(128, 64, dtype=torch.float16)
|
||||
@@ -149,11 +164,12 @@ def test_rocm_unquantized_gemm_noncontiguous_activation_real_kernel(monkeypatch,
|
||||
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch):
|
||||
@pytest.mark.parametrize("m", [1, 128])
|
||||
def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch, m):
|
||||
# wvSplitK skinny GEMM handles n in [1, 5] (see PR #40687); n > 5 must
|
||||
# fall back to torch.nn.functional.linear.
|
||||
x = torch.randn(6, 64, dtype=torch.float16)
|
||||
weight = torch.randn(128, 64, dtype=torch.float16)
|
||||
weight = torch.randn(m, 64, dtype=torch.float16)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for --mm-encoder-only safetensors shard filtering."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
from transformers.utils import SAFE_WEIGHTS_INDEX_NAME
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
filter_mm_encoder_only_safetensors_files,
|
||||
resolve_mm_encoder_only_lm_prefixes,
|
||||
)
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
|
||||
def _write_index(folder: str, weight_map: dict[str, str]) -> None:
|
||||
with open(os.path.join(folder, SAFE_WEIGHTS_INDEX_NAME), "w") as f:
|
||||
json.dump({"weight_map": weight_map}, f)
|
||||
|
||||
|
||||
def _qwen25_mapper() -> WeightsMapper:
|
||||
"""Qwen2.5-VL dual layout mapper (nested + flat catch-all)."""
|
||||
return WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"model.language_model.": "language_model.model.",
|
||||
"model.visual.": "visual.",
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
"model.": "language_model.model.",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _qwen3_mapper() -> WeightsMapper:
|
||||
return WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"model.visual.": "visual.",
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
"model.language_model.": "language_model.model.",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _molmo_mapper() -> WeightsMapper:
|
||||
return WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"model.vision_backbone.": "vision_backbone.",
|
||||
"model.transformer.blocks.": "model.layers.",
|
||||
"model.transformer.ln_f.": "model.norm.",
|
||||
"model.transformer.mlp.down_proj.": "lm_head.",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_keeps_vision_shards_drops_language_only():
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
files = [
|
||||
os.path.join(folder, "model-00001-of-000003.safetensors"),
|
||||
os.path.join(folder, "model-00002-of-000003.safetensors"),
|
||||
os.path.join(folder, "model-00003-of-000003.safetensors"),
|
||||
]
|
||||
for path in files:
|
||||
open(path, "wb").close()
|
||||
|
||||
_write_index(
|
||||
folder,
|
||||
{
|
||||
"language_model.layers.0.weight": "model-00001-of-000003.safetensors",
|
||||
"language_model.layers.1.weight": "model-00002-of-000003.safetensors",
|
||||
"vision_tower.blocks.0.weight": "model-00003-of-000003.safetensors",
|
||||
"mm_projector.weight": "model-00003-of-000003.safetensors",
|
||||
},
|
||||
)
|
||||
|
||||
kept = filter_mm_encoder_only_safetensors_files(
|
||||
files,
|
||||
folder,
|
||||
SAFE_WEIGHTS_INDEX_NAME,
|
||||
("language_model.",),
|
||||
)
|
||||
assert kept == [files[2]]
|
||||
|
||||
|
||||
def test_keeps_mixed_shard_with_vision_and_lm():
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
mixed = os.path.join(folder, "model-00001-of-000001.safetensors")
|
||||
open(mixed, "wb").close()
|
||||
_write_index(
|
||||
folder,
|
||||
{
|
||||
"language_model.embed.weight": "model-00001-of-000001.safetensors",
|
||||
"vision_tower.patch.weight": "model-00001-of-000001.safetensors",
|
||||
},
|
||||
)
|
||||
|
||||
kept = filter_mm_encoder_only_safetensors_files(
|
||||
[mixed],
|
||||
folder,
|
||||
SAFE_WEIGHTS_INDEX_NAME,
|
||||
("language_model.",),
|
||||
)
|
||||
assert kept == [mixed]
|
||||
|
||||
|
||||
def test_no_index_returns_unchanged():
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
files = [os.path.join(folder, "model.safetensors")]
|
||||
open(files[0], "wb").close()
|
||||
kept = filter_mm_encoder_only_safetensors_files(
|
||||
files,
|
||||
folder,
|
||||
SAFE_WEIGHTS_INDEX_NAME,
|
||||
("language_model.",),
|
||||
)
|
||||
assert kept == files
|
||||
|
||||
|
||||
def test_resolve_returns_module_prefixes_without_hf_hardcode():
|
||||
"""No global Qwen HF nest list — just vLLM module prefixes."""
|
||||
prefixes = resolve_mm_encoder_only_lm_prefixes(["language_model"])
|
||||
assert prefixes == ("language_model.",)
|
||||
# With Qwen mapper, still module prefixes (keys classified via mapper).
|
||||
assert resolve_mm_encoder_only_lm_prefixes(
|
||||
["language_model"], weights_mapper=_qwen25_mapper()
|
||||
) == ("language_model.",)
|
||||
|
||||
|
||||
def test_resolve_empty_language_model_names_fail_closed():
|
||||
"""Missing/empty ``_language_model_names`` → None (no language_model. default)."""
|
||||
assert resolve_mm_encoder_only_lm_prefixes(None) is None
|
||||
assert resolve_mm_encoder_only_lm_prefixes([]) is None
|
||||
assert resolve_mm_encoder_only_lm_prefixes(()) is None
|
||||
|
||||
|
||||
def test_resolve_skips_filter_for_shared_hf_root():
|
||||
"""Shared HF root (Molmo-shaped mapper) → fail-closed, not magic name list."""
|
||||
assert (
|
||||
resolve_mm_encoder_only_lm_prefixes(["model"], weights_mapper=_molmo_mapper())
|
||||
is None
|
||||
)
|
||||
# Without a mapper, identity layout: module prefix is returned as-is.
|
||||
assert resolve_mm_encoder_only_lm_prefixes(["model"]) == ("model.",)
|
||||
assert resolve_mm_encoder_only_lm_prefixes(["model."]) == ("model.",)
|
||||
|
||||
|
||||
def test_resolve_keeps_llm_prefix_without_model_broadening():
|
||||
prefixes = resolve_mm_encoder_only_lm_prefixes(["llm"])
|
||||
assert prefixes == ("llm.",)
|
||||
|
||||
|
||||
def test_qwen3_nested_index_skips_lm_keeps_visual():
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
files = [
|
||||
os.path.join(folder, "model-00001-of-000002.safetensors"),
|
||||
os.path.join(folder, "model-00002-of-000002.safetensors"),
|
||||
]
|
||||
for path in files:
|
||||
open(path, "wb").close()
|
||||
_write_index(
|
||||
folder,
|
||||
{
|
||||
"model.language_model.layers.0.weight": (
|
||||
"model-00001-of-000002.safetensors"
|
||||
),
|
||||
"lm_head.weight": "model-00001-of-000002.safetensors",
|
||||
"model.visual.blocks.0.weight": "model-00002-of-000002.safetensors",
|
||||
},
|
||||
)
|
||||
prefixes = resolve_mm_encoder_only_lm_prefixes(
|
||||
["language_model"], weights_mapper=_qwen3_mapper()
|
||||
)
|
||||
assert prefixes is not None
|
||||
kept = filter_mm_encoder_only_safetensors_files(
|
||||
files,
|
||||
folder,
|
||||
SAFE_WEIGHTS_INDEX_NAME,
|
||||
prefixes,
|
||||
weights_mapper=_qwen3_mapper(),
|
||||
)
|
||||
assert kept == [files[1]]
|
||||
|
||||
|
||||
def test_qwen25_flat_index_skips_lm_keeps_visual():
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
files = [
|
||||
os.path.join(folder, "model-00001-of-000002.safetensors"),
|
||||
os.path.join(folder, "model-00002-of-000002.safetensors"),
|
||||
]
|
||||
for path in files:
|
||||
open(path, "wb").close()
|
||||
_write_index(
|
||||
folder,
|
||||
{
|
||||
"model.layers.0.self_attn.q_proj.weight": (
|
||||
"model-00001-of-000002.safetensors"
|
||||
),
|
||||
"model.embed_tokens.weight": "model-00001-of-000002.safetensors",
|
||||
"model.norm.weight": "model-00001-of-000002.safetensors",
|
||||
"lm_head.weight": "model-00001-of-000002.safetensors",
|
||||
"visual.blocks.0.weight": "model-00002-of-000002.safetensors",
|
||||
},
|
||||
)
|
||||
prefixes = resolve_mm_encoder_only_lm_prefixes(
|
||||
["language_model"], weights_mapper=_qwen25_mapper()
|
||||
)
|
||||
assert prefixes is not None
|
||||
kept = filter_mm_encoder_only_safetensors_files(
|
||||
files,
|
||||
folder,
|
||||
SAFE_WEIGHTS_INDEX_NAME,
|
||||
prefixes,
|
||||
weights_mapper=_qwen25_mapper(),
|
||||
)
|
||||
assert kept == [files[1]]
|
||||
|
||||
|
||||
def test_molmo_shaped_index_not_filtered_when_deny_disabled():
|
||||
"""Shared-root LM attr → resolve None; callers leave the file list alone."""
|
||||
assert (
|
||||
resolve_mm_encoder_only_lm_prefixes(["model"], weights_mapper=_molmo_mapper())
|
||||
is None
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
files = [
|
||||
os.path.join(folder, "vision.safetensors"),
|
||||
os.path.join(folder, "lm.safetensors"),
|
||||
]
|
||||
for path in files:
|
||||
open(path, "wb").close()
|
||||
_write_index(
|
||||
folder,
|
||||
{
|
||||
"model.vision_backbone.patch.weight": "vision.safetensors",
|
||||
"model.transformer.blocks.0.weight": "lm.safetensors",
|
||||
},
|
||||
)
|
||||
# Document why raw HF startswith("model.") is unsafe without fail-closed:
|
||||
# whole-shard deny would also drop vision.
|
||||
if_applied_naively = filter_mm_encoder_only_safetensors_files(
|
||||
files, folder, SAFE_WEIGHTS_INDEX_NAME, ("model.",)
|
||||
)
|
||||
assert if_applied_naively == []
|
||||
@@ -79,7 +79,7 @@ def test_default_loader_hf_still_falls_back_to_pt(tmp_path):
|
||||
# Control: load_format="hf" still picks up .pt weights via fallback.
|
||||
(tmp_path / "model.pt").write_bytes(b"\x00\x00\x00\x00")
|
||||
loader = DefaultModelLoader(LoadConfig(load_format="hf"))
|
||||
_, files, use_safetensors = loader._prepare_weights(
|
||||
_, files, use_safetensors, _ = loader._prepare_weights(
|
||||
str(tmp_path),
|
||||
None,
|
||||
None,
|
||||
|
||||
@@ -1158,13 +1158,13 @@ def test_kv_scale_reload(vllm_runner):
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"inference-optimization/Qwen3-0.6B-debug-multiply",
|
||||
"inference-optimization/Qwen3-0.6B-debug-add",
|
||||
"fp8",
|
||||
"fp8_per_tensor",
|
||||
),
|
||||
pytest.param(
|
||||
"inference-optimization/DeepSeek-V3-debug-empty",
|
||||
"inference-optimization/DeepSeek-V3-debug-multiply",
|
||||
"inference-optimization/DeepSeek-V3-debug-add",
|
||||
"fp8",
|
||||
"fp8_per_tensor",
|
||||
marks=[pytest.mark.slow_test],
|
||||
),
|
||||
pytest.param(
|
||||
@@ -1192,7 +1192,7 @@ def test_online_quantize_reload(
|
||||
if current_platform.device_count() < tp_size:
|
||||
pytest.skip(reason="Not enough GPU devices")
|
||||
|
||||
if quantization == "fp8" and _fp8_reload_unsupported():
|
||||
if quantization == "fp8_per_tensor" and _fp8_reload_unsupported():
|
||||
pytest.skip(reason="Requires FP8 support")
|
||||
|
||||
with vllm_runner(
|
||||
|
||||
@@ -7,14 +7,11 @@ warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
|
||||
identical outputs.
|
||||
"""
|
||||
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
@@ -24,8 +21,6 @@ from vllm import SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DAEMON_TIMEOUT_S = 600
|
||||
|
||||
|
||||
class WeightCacheDaemon:
|
||||
"""Context manager running the real weight cache daemon as a subprocess."""
|
||||
@@ -35,18 +30,14 @@ class WeightCacheDaemon:
|
||||
model: str,
|
||||
tp_size: int,
|
||||
extra_args: list[str] | None = None,
|
||||
num_groups: int = 1,
|
||||
):
|
||||
# Short base path: Unix socket paths are limited to ~107 characters.
|
||||
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
|
||||
self.tp_size = tp_size
|
||||
# Each daemon group (target plus cached drafts) binds one socket per
|
||||
# local rank.
|
||||
self.num_sockets = tp_size * num_groups
|
||||
self._cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"vllm.model_executor.model_loader.weight_cache.daemon",
|
||||
"vllm.entrypoints.cli.main",
|
||||
"preload",
|
||||
"--model",
|
||||
model,
|
||||
"--tensor-parallel-size",
|
||||
@@ -57,18 +48,12 @@ class WeightCacheDaemon:
|
||||
*(extra_args or []),
|
||||
]
|
||||
self._proc: subprocess.Popen | None = None
|
||||
self._lines: list[str] = []
|
||||
|
||||
def __enter__(self) -> "WeightCacheDaemon":
|
||||
self._proc = subprocess.Popen(
|
||||
self._cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True
|
||||
)
|
||||
threading.Thread(target=self._drain_stderr, daemon=True).start()
|
||||
try:
|
||||
self._wait_ready(DAEMON_TIMEOUT_S)
|
||||
except Exception:
|
||||
self._stop()
|
||||
raise
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info) -> None:
|
||||
@@ -76,32 +61,7 @@ class WeightCacheDaemon:
|
||||
|
||||
def _drain_stderr(self) -> None:
|
||||
assert self._proc is not None and self._proc.stderr is not None
|
||||
for line in self._proc.stderr:
|
||||
self._lines.append(line)
|
||||
|
||||
def _logs(self) -> str:
|
||||
return "".join(self._lines)
|
||||
|
||||
def _wait_ready(self, timeout_s: float) -> None:
|
||||
assert self._proc is not None
|
||||
# Each rank binds its socket only once the model is fully cached.
|
||||
# Poll for the socket files rather than a log line: model loading can
|
||||
# pull in JIT compilers that swap the process's stderr and swallow
|
||||
# everything logged afterwards, making log-based readiness flaky.
|
||||
pattern = os.path.join(self.socket_dir, "vllm_weight_cache_*.sock")
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while time.monotonic() < deadline:
|
||||
if self._proc.poll() is not None:
|
||||
raise RuntimeError(
|
||||
f"Weight cache daemon exited with {self._proc.returncode}:\n"
|
||||
f"{self._logs()}"
|
||||
)
|
||||
if len(glob.glob(pattern)) >= self.num_sockets:
|
||||
return
|
||||
time.sleep(1.0)
|
||||
raise TimeoutError(
|
||||
f"Weight cache daemon not ready after {timeout_s}s:\n{self._logs()}"
|
||||
)
|
||||
self._proc.stderr.read()
|
||||
|
||||
def _stop(self) -> None:
|
||||
assert self._proc is not None
|
||||
@@ -121,9 +81,6 @@ class ModelCase:
|
||||
images: list | None = None
|
||||
llm_kwargs: dict[str, Any] = field(default_factory=dict)
|
||||
daemon_args: list[str] = field(default_factory=list)
|
||||
# Daemon groups the launcher starts; 2 when a cached draft group joins the
|
||||
# target group (MTP/EAGLE speculative decoding).
|
||||
daemon_groups: int = 1
|
||||
|
||||
|
||||
def generate(
|
||||
@@ -203,7 +160,6 @@ QWEN_MTP_CASE = ModelCase(
|
||||
"--speculative-config",
|
||||
'{"method": "mtp", "num_speculative_tokens": 1}',
|
||||
],
|
||||
daemon_groups=2,
|
||||
)
|
||||
|
||||
|
||||
@@ -242,7 +198,6 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
|
||||
case.model,
|
||||
tp_size=1,
|
||||
extra_args=case.daemon_args,
|
||||
num_groups=case.daemon_groups,
|
||||
) as d:
|
||||
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
|
||||
# Warm restart: a second engine lifetime against the same daemon.
|
||||
@@ -251,3 +206,116 @@ def test_ipc_cache_cold_start_and_warm_restart(vllm_runner, case: ModelCase):
|
||||
assert cold_outputs == baseline_outputs
|
||||
assert warm_outputs == baseline_outputs
|
||||
assert restart_outputs == baseline_outputs
|
||||
|
||||
|
||||
def _parallel(**kw):
|
||||
from types import SimpleNamespace
|
||||
|
||||
base = dict(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
data_parallel_size_local=1,
|
||||
data_parallel_rank=0,
|
||||
nnodes=1,
|
||||
node_rank=0,
|
||||
)
|
||||
base.update(kw)
|
||||
return SimpleNamespace(**base)
|
||||
|
||||
|
||||
def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
|
||||
"""Each node serves a contiguous global-rank block (DP-major), so without
|
||||
DP local GPU i is TP rank r*local+i, and single-node DP launchers offset
|
||||
by --data-parallel-start-rank."""
|
||||
from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks
|
||||
|
||||
tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1)
|
||||
assert plan_local_ranks(tp_second_node) == [(i, 0, 4 + i) for i in range(4)]
|
||||
|
||||
dp_third_node = _parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8
|
||||
)
|
||||
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0) for i in range(4)]
|
||||
|
||||
dp_tp = _parallel(
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_size=4,
|
||||
data_parallel_size_local=2,
|
||||
data_parallel_rank=2,
|
||||
)
|
||||
assert plan_local_ranks(dp_tp) == [(0, 2, 0), (1, 2, 1), (2, 3, 0), (3, 3, 1)]
|
||||
|
||||
# DP + nnodes: node-local TP replicas
|
||||
dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2)
|
||||
assert plan_local_ranks(dp_tp_node0) == [(i, 0, i) for i in range(8)]
|
||||
dp_tp_node1 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1
|
||||
)
|
||||
assert plan_local_ranks(dp_tp_node1) == [(i, 1, i) for i in range(8)]
|
||||
|
||||
# DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes)
|
||||
tp_span_node1 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1
|
||||
)
|
||||
assert plan_local_ranks(tp_span_node1) == [(i, 0, 4 + i) for i in range(4)]
|
||||
tp_span_node2 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2
|
||||
)
|
||||
assert plan_local_ranks(tp_span_node2) == [(i, 1, i) for i in range(4)]
|
||||
|
||||
# DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes)
|
||||
dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
|
||||
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, i % 4) for i in range(8)]
|
||||
|
||||
|
||||
def test_daemon_rejects_unmappable_parallelism():
|
||||
from vllm.model_executor.model_loader.weight_cache.daemon import (
|
||||
_reject_unsupported_parallelism,
|
||||
)
|
||||
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=12
|
||||
)
|
||||
)
|
||||
# DP combines with --nnodes when the world size divides evenly.
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
|
||||
)
|
||||
with pytest.raises(ValueError, match="pipeline"):
|
||||
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
|
||||
with pytest.raises(ValueError, match="evenly divide"):
|
||||
_reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2))
|
||||
with pytest.raises(ValueError, match="evenly divide"):
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(tensor_parallel_size=2, data_parallel_size=3, nnodes=4)
|
||||
)
|
||||
with pytest.raises(ValueError, match="exceeds"):
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=13
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_weight_cache_key_distinguishes_dp_ranks():
|
||||
from dataclasses import replace
|
||||
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
|
||||
|
||||
key = WeightCacheKey(
|
||||
checkpoint="ckpt",
|
||||
model_arch="Arch",
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
dtype="bf16",
|
||||
quantization=None,
|
||||
quant_config_hash="h",
|
||||
revision=None,
|
||||
vllm_version="v",
|
||||
dp_size=16,
|
||||
dp_rank=3,
|
||||
)
|
||||
assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"]
|
||||
assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"]
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
|
||||
from vllm.model_executor.layers import utils
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
@@ -89,3 +90,25 @@ def test_dispatch_cpu_unquantized_gemm_logs_zentorch_dispatch(monkeypatch):
|
||||
expected_prepacked,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
|
||||
@pytest.mark.parametrize("weight_dtype", [torch.bfloat16, torch.float16, torch.float32])
|
||||
def test_dispatch_cpu_unquantized_gemm_remove_weight_keeps_dtype(
|
||||
monkeypatch, weight_dtype
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
|
||||
|
||||
layer = torch.nn.Linear(16, 8, bias=False, dtype=weight_dtype)
|
||||
loading_dtype = (
|
||||
torch.float32 if weight_dtype is not torch.float32 else torch.bfloat16
|
||||
)
|
||||
with set_default_torch_dtype(loading_dtype):
|
||||
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
|
||||
|
||||
assert layer.weight.numel() == 0
|
||||
assert layer.weight.dtype is weight_dtype
|
||||
|
||||
x = torch.randn(4, 16, dtype=weight_dtype)
|
||||
output = layer.cpu_linear(x, layer.weight, None)
|
||||
assert not output.isnan().any()
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.model_executor.warmup import jit_warmup_triton_helper
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
JitWarmupRegistry,
|
||||
WarmupChoices,
|
||||
WarmupIntRange,
|
||||
kernel_launcher,
|
||||
@@ -116,6 +119,105 @@ def test_triton_launcher_supports_compile_and_runtime_adapters() -> None:
|
||||
assert runtime_calls == [(owner.kernel, (2,), ("runtime", 2), {"CONST": 7})]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
|
||||
reason="Requires CUDA Triton async compilation",
|
||||
)
|
||||
def test_failed_parallel_warmup_does_not_leak_into_runtime(monkeypatch) -> None:
|
||||
from triton.runtime._async_compile import active_mode
|
||||
|
||||
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
|
||||
owner = _TestTritonKernel()
|
||||
owner.kernel = _FakeTritonKernel()
|
||||
|
||||
def fail():
|
||||
raise RuntimeError("warmup failed")
|
||||
|
||||
def warmup(**kwargs):
|
||||
active_mode.get().submit(kwargs["second"], fail, lambda result: None)
|
||||
|
||||
monkeypatch.setattr(owner.kernel, "warmup", warmup)
|
||||
with pytest.raises(RuntimeError, match="warmup failed"):
|
||||
owner.compile_many(owner.CompileKey(value=i) for i in range(2))
|
||||
|
||||
assert active_mode.get() is None
|
||||
owner("runtime", 3, None)
|
||||
assert len(owner.kernel.runtime_calls) == 1
|
||||
|
||||
|
||||
def test_autotuning_warmup_stays_serial(monkeypatch) -> None:
|
||||
owner = _TestTritonKernel()
|
||||
owner.kernel = _FakeTritonKernel()
|
||||
owner._run_autotune = True
|
||||
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 4)
|
||||
|
||||
def unexpected_async(*args):
|
||||
pytest.fail("Autotuning must not use asynchronous compilation")
|
||||
|
||||
monkeypatch.setattr(triton, "AsyncCompileMode", unexpected_async, raising=False)
|
||||
owner.compile_many(owner.CompileKey(value=i) for i in range(2))
|
||||
assert len(owner.kernel.runtime_calls) == 2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _warmup_store_kernel(out, VALUE: tl.constexpr):
|
||||
tl.store(out, VALUE)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
|
||||
reason="Requires CUDA Triton async compilation",
|
||||
)
|
||||
def test_registered_warmup_parallelizes_compile_only_and_runtime_stays_serial(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
from triton.runtime._async_compile import active_mode
|
||||
|
||||
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
|
||||
|
||||
def warmup_inputs():
|
||||
return dict(out=TritonWarmupTensor(torch.int32), value=WarmupChoices(1, 2))
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=_warmup_store_kernel, warmup_inputs=warmup_inputs
|
||||
)
|
||||
def store(out, value):
|
||||
return (1,), dict(VALUE=value)
|
||||
|
||||
store.get_warmup_keys() # Initialize Triton's binder before wrapping its compiler.
|
||||
caller = threading.get_ident()
|
||||
barrier = threading.Barrier(2, timeout=10)
|
||||
compiled_on = []
|
||||
compile_kernel = _warmup_store_kernel.compile
|
||||
|
||||
def record_compile(*args, **kwargs):
|
||||
if threading.get_ident() != caller:
|
||||
barrier.wait()
|
||||
result = compile_kernel(*args, **kwargs)
|
||||
compiled_on.append(threading.get_ident())
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(_warmup_store_kernel, "compile", record_compile)
|
||||
registry = JitWarmupRegistry(None)
|
||||
with registry.activate():
|
||||
store.register_warmup()
|
||||
registry.warmup()
|
||||
|
||||
assert len(compiled_on) == 2
|
||||
assert all(thread != caller for thread in compiled_on)
|
||||
assert active_mode.get() is None
|
||||
out = torch.empty(1, dtype=torch.int32, device=current_platform.device_type)
|
||||
for value in (1, 2):
|
||||
store(out, value)
|
||||
assert out.item() == value
|
||||
assert len(compiled_on) == 2
|
||||
|
||||
# A specialization first seen during inference uses the normal blocking JIT.
|
||||
store(out, 3)
|
||||
assert out.item() == 3
|
||||
assert compiled_on[2:] == [caller]
|
||||
|
||||
|
||||
def test_triton_launcher_supports_cpu_function_wrappers() -> None:
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from unittest.mock import Mock, patch
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.multimodal.processing import InputProcessingContext
|
||||
@@ -52,7 +52,7 @@ def print_input_ids(input_ids):
|
||||
@pytest.fixture
|
||||
def mock_qwen3_omni_config():
|
||||
"""Create a mock Qwen3OmniMoeThinker config."""
|
||||
config = Mock(spec=PretrainedConfig)
|
||||
config = Mock(spec=PreTrainedConfig)
|
||||
# Token IDs from https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
|
||||
config.audio_token_id = 151675 # <|audio_pad|>
|
||||
config.video_token_id = 151656 # <|video_pad|>
|
||||
@@ -144,6 +144,11 @@ def test_qwen3_omni_get_updates_use_audio_in_video(
|
||||
# Create a mock context
|
||||
mock_ctx = Mock(spec=InputProcessingContext)
|
||||
mock_ctx.tokenizer = mock_tokenizer
|
||||
# `model_config` is an instance attribute, so it is not covered by the
|
||||
# spec; the data parser reads it for `allow_missing_mm_embeddings`.
|
||||
mock_ctx.model_config = SimpleNamespace(
|
||||
multimodal_config=SimpleNamespace(allow_missing_mm_embeddings=False)
|
||||
)
|
||||
|
||||
# Create processing info
|
||||
info = Qwen3OmniMoeThinkerProcessingInfo(mock_ctx)
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the apply-time Triton kernel override registry.
|
||||
|
||||
The registry must let out-of-tree platforms replace kernels defined as
|
||||
plain ``@triton.jit`` functions, patching every hold-site (defining
|
||||
module, from-import copies, and JIT warmup owners) so that existing call
|
||||
sites dispatch to the override without any core-side decoration.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import types
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
_DecoratedTritonJitKernel,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.triton_utils.dispatcher import (
|
||||
KernelOverride,
|
||||
register_kernels,
|
||||
)
|
||||
|
||||
|
||||
class _FakeTritonKernel:
|
||||
"""Stands in for a Triton JITFunction (works without Triton installed)."""
|
||||
|
||||
arg_names = ("output_ptr", "input_ptr", "replace_from", "MAX_NUM_TOKENS")
|
||||
|
||||
def __getitem__(self, grid: Any) -> Any:
|
||||
def launch(*args: Any, **kwargs: Any) -> Any:
|
||||
return ("default", grid, kwargs)
|
||||
|
||||
return launch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _fake_module(monkeypatch: pytest.MonkeyPatch) -> Any:
|
||||
"""Expose a fake defining module holding the kernel and its warmup owner."""
|
||||
import types
|
||||
|
||||
kernel = _FakeTritonKernel()
|
||||
owner = _DecoratedTritonJitKernel(
|
||||
kernel,
|
||||
lambda: {},
|
||||
lambda output, input, replace_from: ( # noqa: ARG005
|
||||
(1,),
|
||||
dict(MAX_NUM_TOKENS=8),
|
||||
),
|
||||
)
|
||||
module = types.ModuleType("vllm.tests.fake_kernel_home")
|
||||
module.expand_kernel = kernel # type: ignore[attr-defined]
|
||||
module._expand = owner # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(sys.modules, module.__name__, module)
|
||||
return module
|
||||
|
||||
|
||||
def test_override_patches_defining_module_and_warmup_owner(
|
||||
_fake_module: Any,
|
||||
) -> None:
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None):
|
||||
calls.append((output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS))
|
||||
return "cpu"
|
||||
|
||||
register_kernels(
|
||||
{
|
||||
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
|
||||
}
|
||||
)
|
||||
|
||||
overridden = _fake_module.expand_kernel
|
||||
assert isinstance(overridden, KernelOverride)
|
||||
# Launch with kernel-arg names like the warmup launch path does.
|
||||
assert overridden[(4,)](output_ptr=1, input_ptr=2, replace_from=0) == "cpu"
|
||||
assert calls == [(1, 2, 0, None)]
|
||||
|
||||
# The warmup owner's captured kernel is patched too, and its cached
|
||||
# argument-name introspection is invalidated.
|
||||
assert _fake_module._expand.kernel is overridden
|
||||
|
||||
|
||||
def test_override_forwards_positionally_for_mismatched_names(
|
||||
_fake_module: Any,
|
||||
) -> None:
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def cpu_impl(output, input_val, replace_from, MAX_NUM_TOKENS=None):
|
||||
calls.append((output, input_val, replace_from, MAX_NUM_TOKENS))
|
||||
return "cpu"
|
||||
|
||||
register_kernels(
|
||||
{
|
||||
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
|
||||
}
|
||||
)
|
||||
overridden = _fake_module.expand_kernel
|
||||
assert overridden[(4,)](output_ptr=1, input_ptr=2, replace_from=0) == "cpu"
|
||||
assert calls == [(1, 2, 0, None)]
|
||||
|
||||
|
||||
def test_override_covers_from_import_copies(_fake_module: Any) -> None:
|
||||
import types
|
||||
|
||||
importer = types.ModuleType("vllm.tests.fake_kernel_importer")
|
||||
importer.expand_kernel = _fake_module.expand_kernel # type: ignore[attr-defined]
|
||||
sys.modules["vllm.tests.fake_kernel_importer"] = importer
|
||||
|
||||
def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None):
|
||||
return "cpu"
|
||||
|
||||
register_kernels(
|
||||
{
|
||||
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
|
||||
}
|
||||
)
|
||||
assert isinstance(importer.expand_kernel, KernelOverride)
|
||||
del sys.modules["vllm.tests.fake_kernel_importer"]
|
||||
|
||||
|
||||
def test_override_owner_launch_goes_through_impl(_fake_module: Any) -> None:
|
||||
launched: list[Any] = []
|
||||
|
||||
def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None):
|
||||
launched.append((output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS))
|
||||
return "cpu"
|
||||
|
||||
register_kernels(
|
||||
{
|
||||
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
|
||||
}
|
||||
)
|
||||
owner = _fake_module._expand
|
||||
# In-place kernels return None from launch; assert on the recorded call.
|
||||
owner(1, 2, 0)
|
||||
assert launched == [(1, 2, 0, 8)]
|
||||
|
||||
|
||||
def test_register_kernel_auto_imports_defining_module(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Registering a kernel whose defining module is not imported yet must
|
||||
import it on the spot and apply the override."""
|
||||
kernel_name = "vllm.v1.sample.rejection_sampler.expand_kernel"
|
||||
module_name = kernel_name.rpartition(".")[0]
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def cpu_impl(
|
||||
output_ptr,
|
||||
input_ptr,
|
||||
cu_num_tokens_ptr,
|
||||
replace_from,
|
||||
replace_to,
|
||||
MAX_NUM_TOKENS=None,
|
||||
):
|
||||
calls.append((output_ptr, replace_from, MAX_NUM_TOKENS))
|
||||
return "cpu"
|
||||
|
||||
# Ensure the module is NOT imported before registration. Drop it from
|
||||
# sys.modules; a fresh import creates new kernel/owner objects, which
|
||||
# is what a cold-start platform process would see.
|
||||
saved = sys.modules.pop(module_name, None)
|
||||
# Re-run the registry patch against the original module afterwards so
|
||||
# other tests that already imported it keep working.
|
||||
try:
|
||||
assert module_name not in sys.modules
|
||||
register_kernels({kernel_name: cpu_impl})
|
||||
assert module_name in sys.modules
|
||||
finally:
|
||||
if saved is None:
|
||||
# First import in the whole session: keep the fresh module.
|
||||
saved = sys.modules[module_name]
|
||||
else:
|
||||
# Restore the original module object and re-apply the override
|
||||
# to its (pre-existing) kernel so both generations agree.
|
||||
sys.modules[module_name] = saved
|
||||
register_kernels({kernel_name: cpu_impl})
|
||||
|
||||
overridden = saved.expand_kernel
|
||||
assert isinstance(overridden, KernelOverride)
|
||||
overridden[(2,)](
|
||||
output_ptr=1,
|
||||
input_ptr=2,
|
||||
cu_num_tokens_ptr=3,
|
||||
replace_from=0,
|
||||
replace_to=0,
|
||||
MAX_NUM_TOKENS=8,
|
||||
)
|
||||
assert calls == [(1, 0, 8)]
|
||||
# The warmup owner captured at the fresh import must be patched too
|
||||
# (when the module existed before, there are two module generations).
|
||||
fresh = sys.modules[module_name]
|
||||
if fresh is not saved:
|
||||
assert fresh._expand.kernel is overridden
|
||||
|
||||
|
||||
def test_real_triton_kernel_override_end_to_end() -> None:
|
||||
"""Override a real @triton.jit kernel through the registry."""
|
||||
if not triton.__class__.__module__.startswith("triton"):
|
||||
pytest.skip("Triton is not installed")
|
||||
|
||||
@triton.jit
|
||||
def _add_one_kernel(x_ptr, out_ptr, BLOCK: tl.constexpr): # pragma: no cover
|
||||
offsets = tl.arange(0, BLOCK)
|
||||
tl.store(out_ptr + offsets, tl.load(x_ptr + offsets) + 1)
|
||||
|
||||
module = types.ModuleType("vllm.tests.real_jit_kernel_home")
|
||||
module._add_one_kernel = _add_one_kernel # type: ignore[attr-defined]
|
||||
sys.modules[module.__name__] = module
|
||||
|
||||
def cpu_impl(x_ptr, out_ptr, BLOCK=None): # noqa: ARG001
|
||||
return "cpu"
|
||||
|
||||
register_kernels({"vllm.tests.real_jit_kernel_home._add_one_kernel": cpu_impl})
|
||||
assert module._add_one_kernel.arg_names == ( # type: ignore[attr-defined]
|
||||
"x_ptr",
|
||||
"out_ptr",
|
||||
"BLOCK",
|
||||
)
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import tempfile
|
||||
from unittest.mock import Mock
|
||||
|
||||
import huggingface_hub.constants
|
||||
import pytest
|
||||
@@ -9,6 +10,7 @@ from huggingface_hub.utils import LocalEntryNotFoundError
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf,
|
||||
drop_checkpoint_cache,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
|
||||
@@ -292,5 +294,18 @@ class TestKvCacheScaleMapper:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fails", [False, True])
|
||||
def test_drop_checkpoint_cache_releases_local_weight_files(
|
||||
tmp_path, monkeypatch, fails
|
||||
):
|
||||
"""Evict only weights; cache eviction errors must not prevent loading."""
|
||||
for name in ("weights.safetensors", "config.json"):
|
||||
(tmp_path / name).touch()
|
||||
fadvise = Mock(side_effect=OSError("cache eviction failed") if fails else None)
|
||||
monkeypatch.setattr("os.posix_fadvise", fadvise)
|
||||
drop_checkpoint_cache(str(tmp_path))
|
||||
fadvise.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_download_weights_from_hf()
|
||||
|
||||
Reference in New Issue
Block a user