[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
File diff suppressed because it is too large Load Diff
@@ -1,38 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
import pytest
import torch
import torch.nn as nn
from vllm.model_executor.layers.activation import GeluAndMul, SiluAndMul
from vllm.platforms import CpuArchEnum, current_platform
@pytest.mark.skipif(
not current_platform.is_cpu()
or current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC,
reason="Native CPU activations require a non-PowerPC CPU platform",
)
@pytest.mark.parametrize(
"activation_factory,reference",
[
(SiluAndMul, nn.SiLU()),
(GeluAndMul, nn.GELU()),
(partial(GeluAndMul, approximate="tanh"), nn.GELU(approximate="tanh")),
],
ids=["silu", "gelu", "gelu_tanh"],
)
def test_cpu_act_and_mul_without_compiled_ops(
default_vllm_config, monkeypatch, activation_factory, reference
):
"""CPU activations must construct and run without compiled wheel operators."""
default_vllm_config.compilation_config.custom_ops = ["all"]
monkeypatch.setattr(torch.ops, "_C", object())
x = torch.tensor([[-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device="cpu")
layer = activation_factory()
torch.testing.assert_close(layer(x), reference(x[:, :4]) * x[:, 4:])
@@ -1,339 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FusedMMInputNorm and the fused affine transform kernel."""
import pytest
import torch
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
from vllm.config.compilation import CompilationMode
from vllm.model_executor.layers.fusion.mm_input_norm import (
FusedMMInputNorm,
IdentityInputNorm,
NormParams,
fused_mm_input_norm_triton,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from vllm.utils.torch_utils import set_random_seed
# Module-level tests run on any platform: the CPU platform exercises
# forward_native, accelerators exercise the fused kernel. Only the raw
# kernel tests require an accelerator.
_DEVICE = torch.device(current_platform.device_type)
requires_accelerator = pytest.mark.skipif(
current_platform.is_cpu(),
reason="fused Triton kernel requires a CUDA/XPU accelerator",
)
requires_triton = pytest.mark.skipif(not HAS_TRITON, reason="requires Triton")
requires_vllm_config = pytest.mark.usefixtures("default_vllm_config")
def _reference_input_norm(
pixel_values: torch.Tensor,
image_mean: list[float],
image_std: list[float],
rescale_factor: float,
channel: int,
out_dtype: torch.dtype,
) -> torch.Tensor:
"""Straightforward per-channel affine: (x * rescale - mean) / std."""
patches, size = pixel_values.shape
patch_size = size // channel
mean = torch.tensor(
image_mean, dtype=torch.float32, device=pixel_values.device
).view(1, channel, 1)
std = torch.tensor(image_std, dtype=torch.float32, device=pixel_values.device).view(
1, channel, 1
)
x = pixel_values.to(torch.float32).reshape(patches, channel, patch_size)
x = (x * rescale_factor - mean) / std
return x.view(patches, size).to(out_dtype)
# CLIP-style RGB normalization constants reused across tests.
_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073]
_RGB_STD = [0.26862954, 0.26130258, 0.27577711]
_RGB_RESCALE = 1.0 / 255.0
def _check_against_reference(
pixel_values: torch.Tensor,
out_dtype: torch.dtype,
channel: int = 3,
image_mean: list[float] | None = None,
image_std: list[float] | None = None,
rescale_factor: float = _RGB_RESCALE,
**close_kwargs,
):
"""Run ``FusedMMInputNorm`` on ``pixel_values`` and compare against the
eager per-channel affine reference. Uses the RGB constants by default."""
if image_mean is None:
image_mean = list(_RGB_MEAN)
if image_std is None:
image_std = list(_RGB_STD)
norm = FusedMMInputNorm(
image_mean=image_mean,
image_std=image_std,
rescale_factor=rescale_factor,
channel=channel,
).to(_DEVICE)
expected = _reference_input_norm(
pixel_values, image_mean, image_std, rescale_factor, channel, out_dtype
)
torch.testing.assert_close(
norm(pixel_values, visual_dtype=out_dtype), expected, **close_kwargs
)
# ===========================================================================
# Module-level behavior
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormModule:
def test_matches_reference_above_cudnn_grid_limit(self):
"""num_patches above the old cuDNN batch-norm grid limit (~65535)
used to raise CUDNN_STATUS_INTERNAL_ERROR."""
set_random_seed(0)
pixel_values = torch.randint(
0, 256, (70000, 3 * 196), dtype=torch.float32, device=_DEVICE
)
_check_against_reference(pixel_values, torch.float32)
def test_identity_passthrough(self):
norm = IdentityInputNorm()
pixel_values = torch.randn(8, 3 * 196, dtype=torch.float32, device=_DEVICE)
out = norm(pixel_values, visual_dtype=torch.bfloat16)
torch.testing.assert_close(out, pixel_values.to(torch.bfloat16))
# ===========================================================================
# dtype coverage
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormDtypes:
@pytest.mark.parametrize(
"in_dtype,out_dtype",
[
(torch.float32, torch.float32),
(torch.float32, torch.bfloat16),
(torch.float32, torch.float16),
(torch.bfloat16, torch.bfloat16),
(torch.bfloat16, torch.float32),
(torch.float16, torch.float16),
(torch.uint8, torch.bfloat16),
(torch.uint8, torch.float32),
],
)
def test_dtype_combinations(self, in_dtype: torch.dtype, out_dtype: torch.dtype):
set_random_seed(0)
if in_dtype == torch.uint8:
pixel_values = torch.randint(
0, 256, (32, 3 * 16), dtype=torch.uint8, device=_DEVICE
)
else:
pixel_values = torch.rand(
(32, 3 * 16), dtype=torch.float32, device=_DEVICE
).to(in_dtype)
_check_against_reference(
pixel_values,
out_dtype,
image_mean=[0.5, 0.5, 0.5],
image_std=[0.5, 0.5, 0.5],
atol=1e-2,
rtol=1e-2,
)
# ===========================================================================
# Shape / channel coverage
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormShapes:
@pytest.mark.parametrize("channel", [1, 3, 4])
def test_channel_variants(self, channel: int):
set_random_seed(0)
pixel_values = torch.randint(
0, 256, (16, channel * 64), dtype=torch.float32, device=_DEVICE
)
_check_against_reference(
pixel_values,
torch.float32,
channel=channel,
image_mean=[0.5] * channel,
image_std=[0.25] * channel,
)
@pytest.mark.parametrize("patch_size", [1, 2048])
def test_block_boundaries(self, patch_size: int):
"""Masked-tail (12 elements) and exact-multiple (24 blocks) paths."""
set_random_seed(0)
pixel_values = torch.randint(
0, 256, (4, 3 * patch_size), dtype=torch.float32, device=_DEVICE
)
_check_against_reference(pixel_values, torch.float32)
# ===========================================================================
# Input handling: non-contiguous inputs
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormInputHandling:
def test_non_contiguous_input_matches_reference(self):
set_random_seed(0)
base = torch.randint(
0, 256, (8, 3 * 32, 2), dtype=torch.float32, device=_DEVICE
)
non_contig = base[..., 0]
assert not non_contig.is_contiguous()
_check_against_reference(non_contig, torch.float32)
# ===========================================================================
# Raw kernel entry point
# ===========================================================================
@requires_accelerator
@requires_triton
class TestFusedMMInputNormKernel:
@pytest.mark.parametrize(
"N, C, L",
[
(1, 1, 1),
(1, 3, 1),
(3, 3, 7), # tile straddles channel boundaries
(2, 8, 63),
(4, 3, 1000),
],
)
def test_channel_boundary_crossing(self, N: int, C: int, L: int):
"""The 1D kernel recovers ``c = (offs // L) % C`` per lane, so a tile
can straddle a channel boundary and gather weight/bias per lane.
These shapes force that path by keeping ``L`` well below the block."""
set_random_seed(0)
x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE)
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
out = torch.empty_like(x)
fused_mm_input_norm_triton(x, out, w, b)
expected = x * w.view(1, C, 1) + b.view(1, C, 1)
torch.testing.assert_close(out, expected)
def test_non_contiguous_inputs_materialized(self):
"""Read-only tensors are made contiguous internally."""
N, C, L = 3, 3, 100
set_random_seed(0)
base = torch.randn(N, C, L, 2, dtype=torch.float32, device=_DEVICE)
x = base[..., 0]
assert not x.is_contiguous()
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
out = torch.empty(N, C, L, dtype=torch.float32, device=_DEVICE)
fused_mm_input_norm_triton(x, out, w, b)
expected = x * w.view(1, C, 1) + b.view(1, C, 1)
torch.testing.assert_close(out, expected)
def test_rejects_mismatched_output_shape(self):
"""``outputs`` must be shaped exactly like ``inputs``."""
N, C, L = 3, 3, 100
x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE)
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
for out_shape in ((N, C + 1, L), (N, C, L + 1), (N + 2, C, L)):
with pytest.raises(AssertionError):
fused_mm_input_norm_triton(
x,
torch.empty(*out_shape, dtype=torch.float32, device=_DEVICE),
w,
b,
)
# ===========================================================================
# Construction / configuration
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormConstruction:
"""Weight/bias buffer semantics at init time."""
def test_input_norm_is_always_enabled(self):
config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
compile_mm_encoder=True,
custom_ops=["none"],
)
)
with set_current_vllm_config(config):
FusedMMInputNorm(_RGB_MEAN, _RGB_STD, _RGB_RESCALE)
assert (
config.compilation_config.enabled_custom_ops["fused_mm_input_norm"] == 1
)
assert not config.compilation_config.disabled_custom_ops[
"fused_mm_input_norm"
]
def test_identity_config_buffers(self):
"""Numerically identity parameters still build plain weight/bias
buffers; the identity shortcut lives in ``from_model_config``."""
norm = FusedMMInputNorm(
image_mean=[0.0, 0.0, 0.0],
image_std=[1.0, 1.0, 1.0],
rescale_factor=1.0,
)
torch.testing.assert_close(norm.weight, torch.ones(3))
torch.testing.assert_close(norm.bias, torch.zeros(3))
def test_non_identity_buffers(self):
channel = 3
norm = FusedMMInputNorm(
image_mean=_RGB_MEAN,
image_std=_RGB_STD,
rescale_factor=_RGB_RESCALE,
channel=channel,
)
assert norm.weight.shape == (channel,)
assert norm.bias.shape == (channel,)
mean = torch.tensor(_RGB_MEAN, dtype=torch.float32)
std = torch.tensor(_RGB_STD, dtype=torch.float32)
torch.testing.assert_close(norm.weight, _RGB_RESCALE / std)
torch.testing.assert_close(norm.bias, -mean / std)
def test_is_identity_params(self):
"""The numeric-identity check mirrors the old allclose detection."""
assert NormParams([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0).is_identity
assert not NormParams(_RGB_MEAN, _RGB_STD, _RGB_RESCALE).is_identity
@pytest.mark.parametrize(
("image_mean", "image_std", "rescale_factor"),
[
([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0),
([0.5, 0.5, 0.5], [0.25, 0.25, 0.25], 1 / 255),
],
)
def test_buffers_initialized_on_default_device(
self,
image_mean: list[float],
image_std: list[float],
rescale_factor: float,
):
"""Buffers are built on CPU first, then moved to the caller's
default device, so init never touches the accelerator."""
# The meta device gives the CPU-only test shard the same non-CPU
# default-device semantics without requiring a CUDA build.
default_device = "cuda" if torch.cuda.is_available() else "meta"
with torch.device(default_device):
norm = FusedMMInputNorm(image_mean, image_std, rescale_factor)
assert norm.weight.device.type == default_device
assert norm.bias.device.type == default_device
@@ -1,234 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for vllm.model_executor.layers.pooler.activations."""
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from vllm.model_executor.layers.pooler.activations import (
LambdaPoolerActivation,
PoolerClassify,
PoolerIdentity,
PoolerMultiLabelClassify,
PoolerNormalize,
get_act_fn,
resolve_classifier_act_fn,
)
# ---------------------------------------------------------------------------
# PoolerIdentity
# ---------------------------------------------------------------------------
class TestPoolerIdentity:
def test_returns_input_unchanged(self):
pooler = PoolerIdentity()
x = torch.randn(4, 128)
out = pooler(x)
assert torch.equal(out, x)
def test_forward_list(self):
pooler = PoolerIdentity()
tensors = [torch.randn(128), torch.randn(256)]
out = pooler(tensors)
assert len(out) == 2
for orig, result in zip(tensors, out):
assert torch.equal(orig, result)
# ---------------------------------------------------------------------------
# PoolerNormalize
# ---------------------------------------------------------------------------
class TestPoolerNormalize:
def test_output_has_unit_norm(self):
pooler = PoolerNormalize()
x = torch.randn(4, 128)
out = pooler(x)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(4), atol=1e-5)
def test_single_vector(self):
pooler = PoolerNormalize()
x = torch.randn(1, 64)
out = pooler(x)
norm = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norm, torch.ones(1), atol=1e-5)
def test_forward_list(self):
pooler = PoolerNormalize()
tensors = [torch.randn(1, 64), torch.randn(1, 128)]
out = pooler(tensors)
for t in out:
norm = torch.linalg.norm(t, dim=-1)
assert torch.allclose(norm, torch.ones(1), atol=1e-5)
# ---------------------------------------------------------------------------
# PoolerMultiLabelClassify
# ---------------------------------------------------------------------------
class TestPoolerMultiLabelClassify:
def test_output_in_zero_one(self):
pooler = PoolerMultiLabelClassify()
x = torch.randn(4, 10)
out = pooler(x)
assert (out >= 0).all() and (out <= 1).all()
def test_large_positive_maps_near_one(self):
pooler = PoolerMultiLabelClassify()
x = torch.full((1, 3), 100.0)
out = pooler(x)
assert torch.allclose(out, torch.ones(1, 3), atol=1e-4)
def test_large_negative_maps_near_zero(self):
pooler = PoolerMultiLabelClassify()
x = torch.full((1, 3), -100.0)
out = pooler(x)
assert torch.allclose(out, torch.zeros(1, 3), atol=1e-4)
# ---------------------------------------------------------------------------
# PoolerClassify
# ---------------------------------------------------------------------------
class TestPoolerClassify:
def test_infers_from_shape_when_num_labels_none(self):
pooler = PoolerClassify(num_labels=None)
assert pooler.num_labels is None
x = torch.randn(2, 5)
out = pooler(x)
sums = out.sum(dim=-1)
assert torch.allclose(sums, torch.ones(2), atol=1e-5)
def test_sigmoid_when_num_labels_lt_2(self):
pooler = PoolerClassify(num_labels=1)
x = torch.zeros(1, 1)
out = pooler(x)
assert torch.allclose(out, torch.tensor([[0.5]]), atol=1e-5)
def test_num_labels_zero_uses_sigmoid(self):
pooler = PoolerClassify(num_labels=0)
assert pooler.num_labels == 0
x = torch.zeros(1, 3)
out = pooler(x)
assert torch.allclose(out, torch.full((1, 3), 0.5), atol=1e-5)
def test_num_labels_ge_2_uses_softmax(self):
pooler = PoolerClassify(num_labels=4)
assert pooler.num_labels == 4
x = torch.randn(2, 4)
out = pooler(x)
sums = out.sum(dim=-1)
assert torch.allclose(sums, torch.ones(2), atol=1e-5)
def test_default_num_labels_is_none(self):
pooler = PoolerClassify()
assert pooler.num_labels is None
# ---------------------------------------------------------------------------
# LambdaPoolerActivation
# ---------------------------------------------------------------------------
class TestLambdaPoolerActivation:
def test_applies_custom_fn(self):
pooler = LambdaPoolerActivation(nn.ReLU())
x = torch.tensor([[-1.0, 2.0, -3.0]])
out = pooler(x)
expected = torch.tensor([[0.0, 2.0, 0.0]])
assert torch.equal(out, expected)
def test_forward_list(self):
pooler = LambdaPoolerActivation(nn.ReLU())
tensors = [torch.tensor([-1.0, 2.0]), torch.tensor([3.0, -4.0])]
out = pooler(tensors)
assert torch.equal(out[0], torch.tensor([0.0, 2.0]))
assert torch.equal(out[1], torch.tensor([3.0, 0.0]))
# ---------------------------------------------------------------------------
# get_act_fn factory
# ---------------------------------------------------------------------------
class TestGetActFn:
@staticmethod
def _make_config(**kwargs):
return SimpleNamespace(**kwargs)
def test_regression(self):
cfg = self._make_config(problem_type="regression")
result = get_act_fn(cfg)
assert isinstance(result, PoolerIdentity)
def test_single_label_classification(self):
cfg = self._make_config(
problem_type="single_label_classification", num_labels=3
)
result = get_act_fn(cfg)
assert isinstance(result, PoolerClassify)
assert result.num_labels == 3
def test_multi_label_classification(self):
cfg = self._make_config(problem_type="multi_label_classification")
result = get_act_fn(cfg)
assert isinstance(result, PoolerMultiLabelClassify)
def test_sentence_transformers_activation(self):
cfg = self._make_config(
problem_type="",
sentence_transformers={
"activation_fn": "torch.nn.modules.activation.Sigmoid"
},
)
result = get_act_fn(cfg)
assert isinstance(result, PoolerClassify)
def test_sbert_activation(self):
cfg = self._make_config(
problem_type="",
sbert_ce_default_activation_function=(
"torch.nn.modules.activation.Sigmoid"
),
)
result = get_act_fn(cfg)
assert isinstance(result, PoolerClassify)
def test_default_fallback(self):
cfg = self._make_config(problem_type="")
result = get_act_fn(cfg)
assert isinstance(result, PoolerClassify)
def test_sentence_transformers_takes_priority(self):
cfg = self._make_config(
problem_type="",
sentence_transformers={"activation_fn": "torch.nn.modules.linear.Identity"},
sbert_ce_default_activation_function=(
"torch.nn.modules.activation.Sigmoid"
),
)
result = get_act_fn(cfg)
assert isinstance(result, PoolerIdentity)
def test_rejects_non_torch_activation(self):
cfg = self._make_config(
problem_type="",
sentence_transformers={"activation_fn": "os.system"},
)
with pytest.raises(ValueError, match="restricted"):
get_act_fn(cfg)
# ---------------------------------------------------------------------------
# resolve_classifier_act_fn
# ---------------------------------------------------------------------------
class TestResolveClassifierActFn:
def test_delegates_to_get_act_fn_when_none(self):
model_config = SimpleNamespace(
hf_config=SimpleNamespace(num_labels=3, problem_type="")
)
result = resolve_classifier_act_fn(model_config, act_fn=None)
assert isinstance(result, PoolerClassify)
assert result.num_labels == 3
def test_passes_through_provided_act_fn(self):
custom = PoolerIdentity()
result = resolve_classifier_act_fn(None, act_fn=custom)
assert result is custom
@@ -1,490 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for sequence and token pooler head classes."""
import torch
import torch.nn as nn
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
from vllm.model_executor.layers.pooler.seqwise.heads import (
ClassifierPoolerHead,
EmbeddingPoolerHead,
)
from vllm.model_executor.layers.pooler.tokwise.heads import (
TokenClassifierPoolerHead,
TokenEmbeddingPoolerHead,
)
from vllm.pooling_params import PoolingParams
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
_HIDDEN = 16
_BATCH = 3
def _make_params(
n: int,
*,
task: PoolingTask = "embed",
dimensions: int | None = None,
use_activation: bool | None = None,
) -> list[PoolingParams]:
return [
PoolingParams(task=task, dimensions=dimensions, use_activation=use_activation)
for _ in range(n)
]
def _make_metadata(pooling_params: list[PoolingParams]) -> PoolingMetadata:
n = len(pooling_params)
return PoolingMetadata(
prompt_lens=torch.ones(n, dtype=torch.long),
prompt_token_ids=None,
prompt_token_ids_cpu=None,
pooling_params=pooling_params,
pooling_states=[PoolingStates() for _ in range(n)],
)
def _linear(in_f: int, out_f: int) -> nn.Linear:
torch.manual_seed(42)
return nn.Linear(in_f, out_f, bias=False)
# ---------------------------------------------------------------------------
# EmbeddingPoolerHead
# ---------------------------------------------------------------------------
class TestEmbeddingPoolerHead:
def test_supported_tasks(self):
head = EmbeddingPoolerHead()
assert head.get_supported_tasks() == {"embed"}
def test_passthrough(self):
head = EmbeddingPoolerHead()
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH))
out = head(x, meta)
assert torch.equal(out, x)
def test_head_dtype(self):
head = EmbeddingPoolerHead(head_dtype=torch.float16)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH))
out = head(x, meta)
assert out.dtype == torch.float16
def test_projector(self):
proj = _linear(_HIDDEN, 8)
head = EmbeddingPoolerHead(projector=proj)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH))
out = head(x, meta)
assert out.shape == (_BATCH, 8)
assert torch.allclose(out, proj(x))
def test_matryoshka_uniform(self):
head = EmbeddingPoolerHead()
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, dimensions=4)
meta = _make_metadata(params)
out = head(x, meta)
assert out.shape == (_BATCH, 4)
assert torch.equal(out, x[..., :4])
def test_matryoshka_mixed(self):
head = EmbeddingPoolerHead()
x = torch.randn(2, _HIDDEN)
params = [
PoolingParams(task="embed", dimensions=4),
PoolingParams(task="embed", dimensions=8),
]
meta = _make_metadata(params)
out = head(x, meta)
assert isinstance(out, list)
assert len(out) == 2
assert out[0].shape[-1] == 4
assert out[1].shape[-1] == 8
def test_matryoshka_mixed_with_none(self):
head = EmbeddingPoolerHead()
x = torch.randn(2, _HIDDEN)
params = [
PoolingParams(task="embed", dimensions=4),
PoolingParams(task="embed", dimensions=None),
]
meta = _make_metadata(params)
out = head(x, meta)
assert isinstance(out, list)
assert out[0].shape[-1] == 4
assert torch.equal(out[1], x[1])
def test_activation_uniform_true(self):
head = EmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, use_activation=True)
meta = _make_metadata(params)
out = head(x, meta)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
def test_activation_uniform_false(self):
head = EmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, use_activation=False)
meta = _make_metadata(params)
out = head(x, meta)
assert torch.equal(out, x)
def test_activation_mixed_flags(self):
head = EmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(2, _HIDDEN)
params = [
PoolingParams(task="embed", use_activation=True),
PoolingParams(task="embed", use_activation=False),
]
meta = _make_metadata(params)
out = head(x, meta)
assert isinstance(out, list)
norm_0 = torch.linalg.norm(out[0], dim=-1)
assert torch.allclose(norm_0, torch.ones(1), atol=1e-5)
assert torch.equal(out[1], x[1])
def test_list_input_gets_stacked(self):
head = EmbeddingPoolerHead()
tensors = [torch.randn(_HIDDEN) for _ in range(_BATCH)]
meta = _make_metadata(_make_params(_BATCH))
out = head(tensors, meta)
assert out.shape == (_BATCH, _HIDDEN)
expected = torch.stack(tensors)
assert torch.equal(out, expected)
def test_projector_then_matryoshka(self):
proj = _linear(_HIDDEN, 8)
head = EmbeddingPoolerHead(projector=proj)
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, dimensions=4)
meta = _make_metadata(params)
out = head(x, meta)
assert out.shape == (_BATCH, 4)
assert torch.equal(out, proj(x)[..., :4])
def test_matryoshka_then_activation(self):
head = EmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, dimensions=4, use_activation=True)
meta = _make_metadata(params)
out = head(x, meta)
assert out.shape == (_BATCH, 4)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
def test_empty_batch(self):
head = EmbeddingPoolerHead()
x = torch.randn(0, _HIDDEN)
meta = _make_metadata([])
out = head(x, meta)
assert out.shape == (0, _HIDDEN)
# ---------------------------------------------------------------------------
# ClassifierPoolerHead
# ---------------------------------------------------------------------------
class TestClassifierPoolerHead:
def test_supported_tasks(self):
head = ClassifierPoolerHead()
assert head.get_supported_tasks() == {"classify"}
def test_passthrough(self):
head = ClassifierPoolerHead()
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
assert torch.equal(out, x)
def test_head_dtype(self):
head = ClassifierPoolerHead(head_dtype=torch.float16)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
assert out.dtype == torch.float16
def test_classifier(self):
clf = _linear(_HIDDEN, 3)
head = ClassifierPoolerHead(classifier=clf)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
assert out.shape == (_BATCH, 3)
assert torch.allclose(out, clf(x))
def test_logit_mean(self):
head = ClassifierPoolerHead(logit_mean=2.0)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
assert torch.allclose(out, x - 2.0)
def test_logit_sigma(self):
head = ClassifierPoolerHead(logit_sigma=0.5)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
assert torch.allclose(out, x / 0.5)
def test_platt_scaling_combined(self):
head = ClassifierPoolerHead(logit_mean=1.0, logit_sigma=2.0)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
assert torch.allclose(out, (x - 1.0) / 2.0)
def test_activation_uniform_true(self):
head = ClassifierPoolerHead(activation=PoolerNormalize())
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, task="classify", use_activation=True)
meta = _make_metadata(params)
out = head(x, meta)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
def test_activation_uniform_false(self):
head = ClassifierPoolerHead(activation=PoolerNormalize())
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, task="classify", use_activation=False)
meta = _make_metadata(params)
out = head(x, meta)
assert torch.equal(out, x)
def test_activation_mixed_flags(self):
head = ClassifierPoolerHead(activation=PoolerNormalize())
x = torch.randn(2, _HIDDEN)
params = [
PoolingParams(task="classify", use_activation=True),
PoolingParams(task="classify", use_activation=False),
]
meta = _make_metadata(params)
out = head(x, meta)
assert isinstance(out, list)
norm_0 = torch.linalg.norm(out[0], dim=-1)
assert torch.allclose(norm_0, torch.ones(1), atol=1e-5)
assert torch.equal(out[1], x[1])
def test_list_input_gets_stacked(self):
head = ClassifierPoolerHead()
tensors = [torch.randn(_HIDDEN) for _ in range(_BATCH)]
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(tensors, meta)
assert out.shape == (_BATCH, _HIDDEN)
expected = torch.stack(tensors)
assert torch.equal(out, expected)
def test_classifier_then_platt_scaling(self):
clf = _linear(_HIDDEN, 3)
head = ClassifierPoolerHead(classifier=clf, logit_mean=1.0, logit_sigma=2.0)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
expected = (clf(x) - 1.0) / 2.0
assert torch.allclose(out, expected)
def test_empty_batch(self):
head = ClassifierPoolerHead()
x = torch.randn(0, _HIDDEN)
meta = _make_metadata([])
out = head(x, meta)
assert out.shape == (0, _HIDDEN)
# ---------------------------------------------------------------------------
# TokenEmbeddingPoolerHead
# ---------------------------------------------------------------------------
class TestTokenEmbeddingPoolerHead:
def test_supported_tasks(self):
head = TokenEmbeddingPoolerHead()
assert head.get_supported_tasks() == {"token_embed"}
def test_passthrough(self):
head = TokenEmbeddingPoolerHead()
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed")
out = head.forward_chunk(x, param)
assert out is x
assert torch.equal(out, x)
def test_none_chunked_prefill(self):
head = TokenEmbeddingPoolerHead()
param = PoolingParams(task="token_embed")
out = head.forward_chunk(None, param)
assert out is None
def test_head_dtype(self):
head = TokenEmbeddingPoolerHead(head_dtype=torch.float16)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed")
out = head.forward_chunk(x, param)
assert out is not None
assert out.dtype == torch.float16
def test_projector(self):
proj = _linear(_HIDDEN, 8)
head = TokenEmbeddingPoolerHead(projector=proj)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed")
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 8)
assert torch.allclose(out, proj(x))
def test_matryoshka_truncation(self):
head = TokenEmbeddingPoolerHead()
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", dimensions=4)
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 4)
assert torch.equal(out, x[..., :4])
def test_activation_true(self):
head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", use_activation=True)
out = head.forward_chunk(x, param)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
def test_activation_false(self):
head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", use_activation=False)
out = head.forward_chunk(x, param)
assert torch.equal(out, x)
def test_projector_then_matryoshka(self):
proj = _linear(_HIDDEN, 8)
head = TokenEmbeddingPoolerHead(projector=proj)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", dimensions=4)
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 4)
assert torch.equal(out, proj(x)[..., :4])
def test_matryoshka_then_activation(self):
head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", dimensions=4, use_activation=True)
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 4)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
def test_forward_mixed_batch_chunked_prefill(self):
head = TokenEmbeddingPoolerHead()
pooled_data = [torch.randn(5, _HIDDEN), None, torch.randn(3, _HIDDEN)]
params = _make_params(3, task="token_embed")
meta = _make_metadata(params)
out = head(pooled_data, meta)
assert len(out) == 3
assert torch.equal(out[0], pooled_data[0])
assert out[1] is None
assert torch.equal(out[2], pooled_data[2])
def test_forward_empty_batch(self):
head = TokenEmbeddingPoolerHead()
meta = _make_metadata([])
out = head([], meta)
assert out == []
# ---------------------------------------------------------------------------
# TokenClassifierPoolerHead
# ---------------------------------------------------------------------------
class TestTokenClassifierPoolerHead:
def test_supported_tasks(self):
head = TokenClassifierPoolerHead()
assert head.get_supported_tasks() == {"token_classify"}
def test_passthrough(self):
head = TokenClassifierPoolerHead()
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert torch.equal(out, x)
def test_none_chunked_prefill(self):
head = TokenClassifierPoolerHead()
param = PoolingParams(task="token_classify")
out = head.forward_chunk(None, param)
assert out is None
def test_head_dtype(self):
head = TokenClassifierPoolerHead(head_dtype=torch.float16)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert out is not None
assert out.dtype == torch.float16
def test_classifier(self):
clf = _linear(_HIDDEN, 3)
head = TokenClassifierPoolerHead(classifier=clf)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 3)
assert torch.allclose(out, clf(x))
def test_logit_mean(self):
head = TokenClassifierPoolerHead(logit_mean=2.0)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert torch.allclose(out, x - 2.0)
def test_logit_sigma(self):
head = TokenClassifierPoolerHead(logit_sigma=0.5)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert torch.allclose(out, x / 0.5)
def test_platt_scaling_combined(self):
head = TokenClassifierPoolerHead(logit_mean=1.0, logit_sigma=2.0)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert torch.allclose(out, (x - 1.0) / 2.0)
def test_activation_true(self):
head = TokenClassifierPoolerHead(activation=PoolerNormalize())
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify", use_activation=True)
out = head.forward_chunk(x, param)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
def test_activation_false(self):
head = TokenClassifierPoolerHead(activation=PoolerNormalize())
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify", use_activation=False)
out = head.forward_chunk(x, param)
assert torch.equal(out, x)
def test_forward_mixed_batch_chunked_prefill(self):
head = TokenClassifierPoolerHead()
pooled_data = [torch.randn(5, _HIDDEN), None, torch.randn(3, _HIDDEN)]
params = _make_params(3, task="token_classify")
meta = _make_metadata(params)
out = head(pooled_data, meta)
assert len(out) == 3
assert torch.equal(out[0], pooled_data[0])
assert out[1] is None
assert torch.equal(out[2], pooled_data[2])
def test_forward_empty_batch(self):
head = TokenClassifierPoolerHead()
meta = _make_metadata([])
out = head([], meta)
assert out == []
@@ -1,747 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for sequence and token pooling methods and their factories."""
from dataclasses import dataclass
from types import SimpleNamespace
from unittest.mock import patch
import numpy as np
import pytest
import torch
from transformers import PreTrainedConfig
from vllm.config import PoolerConfig, set_current_vllm_config
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
from vllm.model_executor.layers.pooler.seqwise.methods import (
CLSPool,
LastPool,
MeanPool,
get_seq_pooling_method,
)
from vllm.model_executor.layers.pooler.special import BOSEOSFilter, DispatchPooler
from vllm.model_executor.layers.pooler.tokwise.heads import TokenEmbeddingPoolerHead
from vllm.model_executor.layers.pooler.tokwise.methods import (
AllPool,
StepPool,
get_tok_pooling_method,
)
from vllm.model_executor.layers.pooler.tokwise.poolers import TokenPooler
from vllm.pooling_params import LateChunkingParams, PoolingParams
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingCursor, PoolingMetadata, PoolingStates
_CPU = torch.device("cpu")
def _make_pooling_cursor(
prompt_lens: list[int],
*,
num_scheduled_tokens: list[int] | None = None,
seq_lens: list[int] | None = None,
device: torch.device = _CPU,
) -> PoolingCursor:
"""Build a PoolingCursor from a list of per-sequence prompt lengths."""
prompt_lens_cpu = torch.tensor(prompt_lens, dtype=torch.long)
if num_scheduled_tokens is None:
num_scheduled_tokens_cpu = prompt_lens_cpu.clone()
else:
num_scheduled_tokens_cpu = torch.tensor(num_scheduled_tokens, dtype=torch.long)
if seq_lens is None:
seq_lens_cpu = prompt_lens_cpu.clone()
else:
seq_lens_cpu = torch.tensor(seq_lens, dtype=torch.long)
cumsum = torch.zeros(len(prompt_lens) + 1, dtype=torch.long, device=device)
torch.cumsum(num_scheduled_tokens_cpu, dim=0, out=cumsum[1:])
return PoolingCursor(
first_token_indices_gpu=cumsum[: len(prompt_lens)].to(device),
last_token_indices_gpu=(cumsum[1:] - 1).to(device),
prompt_lens_cpu=prompt_lens_cpu,
seq_lens_cpu=seq_lens_cpu,
num_scheduled_tokens_cpu=num_scheduled_tokens_cpu,
partial_prefill=not torch.equal(prompt_lens_cpu, num_scheduled_tokens_cpu),
finished_mask=torch.eq(prompt_lens_cpu, seq_lens_cpu).tolist(),
)
def _make_metadata(
prompt_lens: list[int],
*,
tasks: list[PoolingTask] | None = None,
token_ids: list[list[int]] | None = None,
pooling_params: list[PoolingParams] | None = None,
num_scheduled_tokens: list[int] | None = None,
seq_lens: list[int] | None = None,
device: torch.device = _CPU,
) -> PoolingMetadata:
"""Build a minimal PoolingMetadata for testing pooling methods."""
n_seqs = len(prompt_lens)
if tasks is None:
tasks = ["embed"] * n_seqs
if pooling_params is None:
pooling_params = [PoolingParams(task=t) for t in tasks]
prompt_lens_tensor = torch.tensor(prompt_lens, dtype=torch.long)
prompt_token_ids_cpu = None
prompt_token_ids = None
if token_ids is not None:
max_len = max(len(t) for t in token_ids)
padded = [t + [0] * (max_len - len(t)) for t in token_ids]
prompt_token_ids_cpu = torch.tensor(padded, dtype=torch.long)
prompt_token_ids = prompt_token_ids_cpu.to(device)
cursor = _make_pooling_cursor(
prompt_lens,
num_scheduled_tokens=num_scheduled_tokens,
seq_lens=seq_lens,
device=device,
)
pooling_states = [PoolingStates() for _ in range(n_seqs)]
return PoolingMetadata(
prompt_lens=prompt_lens_tensor,
prompt_token_ids=prompt_token_ids,
prompt_token_ids_cpu=prompt_token_ids_cpu,
pooling_params=pooling_params,
pooling_states=pooling_states,
pooling_cursor=cursor,
)
def test_build_pooling_cursor_caches_cpu_status() -> None:
metadata = PoolingMetadata(
prompt_lens=torch.tensor([3, 4, 2], dtype=torch.long),
prompt_token_ids=None,
prompt_token_ids_cpu=None,
pooling_params=[PoolingParams(task="embed") for _ in range(3)],
pooling_states=[PoolingStates() for _ in range(3)],
)
metadata.build_pooling_cursor(
np.array([3, 2, 2], dtype=np.int32),
seq_lens_cpu=torch.tensor([3, 2, 1], dtype=torch.long),
device=_CPU,
)
cursor = metadata.get_pooling_cursor()
assert cursor.is_partial_prefill() is True
assert cursor.get_finished_mask() == [True, False, False]
assert cursor[:1].is_partial_prefill() is False
assert cursor[1:2].is_partial_prefill() is True
assert cursor[1:].get_finished_mask() == [False, False]
def test_pooling_params_update_combines_token_id_requirements() -> None:
update = PoolingParamsUpdate() | PoolingParamsUpdate(requires_token_ids=True)
params = PoolingParams(task="embed")
update.apply(params)
assert params.requires_token_ids is True
# ---------------------------------------------------------------------------
# CLSPool
# ---------------------------------------------------------------------------
class TestCLSPool:
def test_extracts_first_token(self):
hidden = torch.tensor(
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
)
metadata = _make_metadata([2, 3])
pooler = CLSPool()
out = pooler(hidden, metadata)
expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
assert torch.equal(out, expected)
def test_rejects_partial_prefill(self):
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
metadata = _make_metadata([3], num_scheduled_tokens=[2])
pooler = CLSPool()
with pytest.raises(RuntimeError, match="partial prefill"):
pooler(hidden, metadata)
# ---------------------------------------------------------------------------
# LastPool
# ---------------------------------------------------------------------------
class TestLastPool:
def test_extracts_last_token(self):
hidden = torch.tensor(
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
)
metadata = _make_metadata([2, 3])
pooler = LastPool()
out = pooler(hidden, metadata)
expected = torch.tensor([[3.0, 4.0], [9.0, 10.0]])
assert torch.equal(out, expected)
def test_partial_prefill_extracts_last_scheduled(self):
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
metadata = _make_metadata([4], num_scheduled_tokens=[2])
pooler = LastPool()
out = pooler(hidden, metadata)
expected = torch.tensor([[3.0, 4.0]])
assert torch.equal(out, expected)
# ---------------------------------------------------------------------------
# MeanPool
# ---------------------------------------------------------------------------
class TestMeanPool:
def test_computes_mean(self):
hidden = torch.tensor(
[[1.0, 2.0], [3.0, 4.0], [10.0, 20.0]], dtype=torch.float32
)
metadata = _make_metadata([2, 1])
pooler = MeanPool()
out = pooler(hidden, metadata)
expected = torch.tensor([[2.0, 3.0], [10.0, 20.0]], dtype=torch.float32)
assert torch.allclose(out, expected, atol=1e-5)
def test_single_token_is_identity(self):
hidden = torch.tensor([[5.0, 10.0]], dtype=torch.float32)
metadata = _make_metadata([1])
pooler = MeanPool()
out = pooler(hidden, metadata)
assert torch.allclose(out, hidden, atol=1e-5)
def test_uniform_values_return_same(self):
hidden = torch.full((4, 3), 7.0, dtype=torch.float32)
metadata = _make_metadata([4])
pooler = MeanPool()
out = pooler(hidden, metadata)
expected = torch.full((1, 3), 7.0, dtype=torch.float32)
assert torch.allclose(out, expected, atol=1e-5)
def test_multiple_sequences(self):
hidden = torch.tensor(
[
[0.0, 0.0],
[2.0, 4.0],
[4.0, 8.0],
[10.0, 10.0],
],
dtype=torch.float32,
)
metadata = _make_metadata([3, 1])
pooler = MeanPool()
out = pooler(hidden, metadata)
expected = torch.tensor([[2.0, 4.0], [10.0, 10.0]], dtype=torch.float32)
assert torch.allclose(out, expected, atol=1e-5)
def test_empty_batch(self):
hidden = torch.empty((0, 8), dtype=torch.float32)
metadata = _make_metadata([])
pooler = MeanPool()
out = pooler(hidden, metadata)
assert out.shape == (0, 8)
def test_rejects_partial_prefill(self):
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float32)
metadata = _make_metadata([3], num_scheduled_tokens=[2])
pooler = MeanPool()
with pytest.raises(RuntimeError, match="partial prefill"):
pooler(hidden, metadata)
def test_chunked_accumulation(self):
hidden = torch.arange(20, dtype=torch.float32).reshape(5, 4)
metadata = _make_metadata([3, 2])
pooler = MeanPool()
with patch(
"vllm.model_executor.layers.pooler.seqwise.methods"
"._MEAN_POOL_ACCUMULATION_CHUNK_BYTES",
16,
):
out = pooler(hidden, metadata)
expected_seq0 = hidden[:3].float().mean(dim=0, keepdim=True)
expected_seq1 = hidden[3:].float().mean(dim=0, keepdim=True)
expected = torch.cat([expected_seq0, expected_seq1], dim=0)
assert torch.allclose(out, expected, atol=1e-5)
def test_upcasts_to_float32(self):
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float16)
metadata = _make_metadata([2])
pooler = MeanPool()
out = pooler(hidden, metadata)
assert out.dtype == torch.float32
expected = torch.tensor([[2.0, 3.0]], dtype=torch.float32)
assert torch.allclose(out, expected, atol=1e-2)
# ---------------------------------------------------------------------------
# get_seq_pooling_method factory
# ---------------------------------------------------------------------------
class TestGetSeqPoolingMethod:
def test_cls(self):
assert isinstance(get_seq_pooling_method("CLS"), CLSPool)
def test_last(self):
assert isinstance(get_seq_pooling_method("LAST"), LastPool)
def test_mean(self):
assert isinstance(get_seq_pooling_method("MEAN"), MeanPool)
def test_unknown_raises(self):
with pytest.raises(NotImplementedError, match="UNKNOWN"):
get_seq_pooling_method("UNKNOWN")
# ---------------------------------------------------------------------------
# AllPool
# ---------------------------------------------------------------------------
@dataclass
class _FakeSchedulerConfig:
enable_chunked_prefill: bool = False
async_scheduling: bool = False
@dataclass
class _FakeVllmConfig:
scheduler_config: _FakeSchedulerConfig
@pytest.mark.parametrize("tok_pooling_type", ["ALL", "STEP"])
def test_dispatch_seq_cls_honors_token_pooling_type(tok_pooling_type):
"""Filter STEP rows before classification; preserve all rows for resolved ALL."""
pooler_config = PoolerConfig(
seq_pooling_type="LAST", tok_pooling_type=tok_pooling_type
)
config = SimpleNamespace(
scheduler_config=_FakeSchedulerConfig(),
model_config=SimpleNamespace(
pooler_config=pooler_config,
head_dtype=None,
hf_config=PreTrainedConfig(num_labels=3),
),
)
classifier = torch.nn.Linear(4, 3)
with set_current_vllm_config(config):
pooler = DispatchPooler.for_seq_cls(pooler_config, classifier=classifier)
token_ids = [[2, 99, 3, 99], [4, 99], [5, 6]]
params = [
PoolingParams(
task="token_classify",
use_activation=False,
step_tag_id=99 if tok_pooling_type == "STEP" else None,
)
for _ in token_ids
]
metadata = _make_metadata([4, 2, 2], token_ids=token_ids, pooling_params=params)
hidden = torch.arange(32, dtype=torch.float32).reshape(8, 4)
outputs = pooler(hidden, metadata)
expected_rows = (
[[1, 3], [5], []]
if tok_pooling_type == "STEP"
else [[0, 1, 2, 3], [4, 5], [6, 7]]
)
assert len(outputs) == len(expected_rows)
for output, rows in zip(outputs, expected_rows):
torch.testing.assert_close(output, classifier(hidden[rows]))
assert pooler.get_pooling_updates("token_classify").requires_token_ids == (
tok_pooling_type == "STEP"
)
class TestAllPool:
@staticmethod
def _make_all_pool(
*, chunked: bool = False, async_scheduling: bool = False
) -> AllPool:
fake_config = _FakeVllmConfig(
scheduler_config=_FakeSchedulerConfig(
enable_chunked_prefill=chunked,
async_scheduling=async_scheduling,
),
)
with patch(
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
return_value=fake_config,
):
return AllPool()
def test_splits_by_sequence(self):
pooler = self._make_all_pool()
hidden = torch.tensor(
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
)
metadata = _make_metadata([2, 3])
out = pooler(hidden, metadata)
assert len(out) == 2
assert torch.equal(out[0], hidden[:2])
assert torch.equal(out[1], hidden[2:])
def test_single_sequence(self):
pooler = self._make_all_pool()
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
metadata = _make_metadata([3])
out = pooler(hidden, metadata)
assert len(out) == 1
assert torch.equal(out[0], hidden)
def test_chunked_prefill_returns_none_for_unfinished(self):
pooler = self._make_all_pool(chunked=True)
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
metadata = _make_metadata(
[4],
num_scheduled_tokens=[2],
seq_lens=[2],
)
out = pooler(hidden, metadata)
assert len(out) == 1
assert out[0] is None
def test_chunked_prefill_returns_concat_when_finished(self):
pooler = self._make_all_pool(chunked=True)
chunk1 = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
metadata1 = _make_metadata(
[4],
num_scheduled_tokens=[2],
seq_lens=[2],
)
out1 = pooler(chunk1, metadata1)
assert out1[0] is None
expected_chunk1 = chunk1.clone()
chunk1.zero_()
chunk2 = torch.tensor([[5.0, 6.0], [7.0, 8.0]])
metadata2 = _make_metadata(
[4],
num_scheduled_tokens=[2],
seq_lens=[4],
)
metadata2.pooling_states = metadata1.pooling_states
out2 = pooler(chunk2, metadata2)
assert out2[0] is not None
expected = torch.cat([expected_chunk1, chunk2], dim=0)
assert torch.equal(out2[0], expected)
@pytest.mark.parametrize("async_scheduling", [False, True])
def test_chunked_prefill_single_shot_matches_non_chunked(
self, async_scheduling: bool
):
pooler = self._make_all_pool(chunked=True, async_scheduling=async_scheduling)
hidden = torch.tensor(
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
)
expected = [hidden[:2].clone(), hidden[2:].clone()]
metadata = _make_metadata([2, 3])
out = pooler(hidden, metadata)
if async_scheduling:
# Finished outputs must own their storage under async scheduling.
hidden.zero_()
assert len(out) == 2
assert torch.equal(out[0], expected[0])
assert torch.equal(out[1], expected[1])
def test_non_chunked_owns_output_under_async_scheduling(self):
pooler = self._make_all_pool(async_scheduling=True)
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
expected = [hidden[:2].clone(), hidden[2:].clone()]
metadata = _make_metadata([2, 1])
out = pooler(hidden, metadata)
hidden.zero_()
assert len(out) == 2
assert torch.equal(out[0], expected[0])
assert torch.equal(out[1], expected[1])
def test_chunked_prefill_mixed_finished_unfinished(self):
pooler = self._make_all_pool(chunked=True)
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
metadata = _make_metadata(
[2, 4],
num_scheduled_tokens=[2, 1],
seq_lens=[2, 1],
)
out = pooler(hidden, metadata)
assert len(out) == 2
assert torch.equal(out[0], hidden[:2])
assert out[1] is None
# ---------------------------------------------------------------------------
# StepPool
# ---------------------------------------------------------------------------
class TestStepPool:
@staticmethod
def _make_step_pool(*, chunked: bool = False) -> StepPool:
fake_config = _FakeVllmConfig(
scheduler_config=_FakeSchedulerConfig(
enable_chunked_prefill=chunked,
),
)
with patch(
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
return_value=fake_config,
):
return StepPool()
def test_filters_by_step_tag_id(self):
pooler = self._make_step_pool()
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
token_ids = [[10, 99, 10, 20]]
params = [PoolingParams(task="token_classify", step_tag_id=10)]
metadata = _make_metadata([4], token_ids=token_ids, pooling_params=params)
out = pooler(hidden, metadata)
assert len(out) == 1
expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
assert torch.equal(out[0], expected)
def test_filters_by_returned_token_ids(self):
pooler = self._make_step_pool()
hidden = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
token_ids = [[10, 20]]
params = [PoolingParams(task="token_classify", returned_token_ids=[0, 2])]
metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
out = pooler(hidden, metadata)
assert len(out) == 1
expected = torch.tensor([[1.0, 3.0], [4.0, 6.0]])
assert torch.equal(out[0], expected)
def test_no_filtering_without_params(self):
pooler = self._make_step_pool()
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
token_ids = [[10, 20]]
params = [PoolingParams(task="token_classify")]
metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
out = pooler(hidden, metadata)
assert len(out) == 1
assert torch.equal(out[0], hidden)
def test_combined_step_tag_and_returned_token_ids(self):
pooler = self._make_step_pool()
hidden = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]])
token_ids = [[99, 10, 99]]
params = [
PoolingParams(
task="token_classify",
step_tag_id=10,
returned_token_ids=[0, 2],
)
]
metadata = _make_metadata([3], token_ids=token_ids, pooling_params=params)
out = pooler(hidden, metadata)
assert len(out) == 1
expected = torch.tensor([[4.0, 6.0]])
assert torch.equal(out[0], expected)
def test_step_tag_id_no_match_returns_empty(self):
pooler = self._make_step_pool()
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
token_ids = [[10, 20]]
params = [PoolingParams(task="token_classify", step_tag_id=999)]
metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
out = pooler(hidden, metadata)
assert len(out) == 1
assert out[0].shape == (0, 2)
def test_chunked_prefill_propagates_none_for_unfinished(self):
pooler = self._make_step_pool(chunked=True)
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
token_ids = [[10, 20, 30, 40]]
params = [PoolingParams(task="token_classify", step_tag_id=10)]
metadata = _make_metadata(
[4],
token_ids=token_ids,
pooling_params=params,
num_scheduled_tokens=[2],
seq_lens=[2],
)
out = pooler(hidden, metadata)
assert len(out) == 1
assert out[0] is None
def test_chunked_prefill_filters_when_finished(self):
pooler = self._make_step_pool(chunked=True)
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
token_ids = [[10, 99, 10, 20]]
params = [PoolingParams(task="token_classify", step_tag_id=10)]
metadata = _make_metadata([4], token_ids=token_ids, pooling_params=params)
out = pooler(hidden, metadata)
assert len(out) == 1
expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
assert torch.equal(out[0], expected)
def test_requires_token_ids_update(self):
pooler = self._make_step_pool()
update = pooler.get_pooling_updates("token_classify")
assert update.requires_token_ids is True
# ---------------------------------------------------------------------------
# get_tok_pooling_method factory
# ---------------------------------------------------------------------------
class TestGetTokPoolingMethod:
def test_all(self):
fake_config = _FakeVllmConfig(
scheduler_config=_FakeSchedulerConfig(
enable_chunked_prefill=False,
),
)
with patch(
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
return_value=fake_config,
):
assert isinstance(get_tok_pooling_method("ALL"), AllPool)
def test_step(self):
fake_config = _FakeVllmConfig(
scheduler_config=_FakeSchedulerConfig(
enable_chunked_prefill=False,
),
)
with patch(
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
return_value=fake_config,
):
assert isinstance(get_tok_pooling_method("STEP"), StepPool)
def test_unknown_raises(self):
with pytest.raises(NotImplementedError, match="UNKNOWN"):
get_tok_pooling_method("UNKNOWN")
@pytest.mark.parametrize(
("chunk_size", "dtype", "head_dtype"),
[
(1, torch.float32, None),
(2, torch.float16, None),
(2, torch.bfloat16, torch.float32),
(7, torch.float32, None),
(10, torch.float32, None),
(2**63 - 1, torch.float32, None),
],
)
def test_late_chunk_pool_matches_fp32_means_before_projection_and_normalization(
chunk_size, dtype, head_dtype
):
# Unequal norms expose the incorrect normalize-then-average implementation.
hidden = (torch.arange(28).reshape(7, 4) - 10).to(dtype)
projector = torch.nn.Linear(4, 3, bias=True).to(head_dtype or dtype)
head = TokenEmbeddingPoolerHead(head_dtype, projector, PoolerNormalize())
pooler = TokenPooler(TestAllPool._make_all_pool(), head)
params = PoolingParams(
task="token_embed",
late_chunking_params=LateChunkingParams(chunk_size=chunk_size),
use_activation=True,
)
metadata = _make_metadata([7], pooling_params=[params])
actual = pooler(hidden, metadata)[0]
means = torch.stack(
[hidden[a : a + chunk_size].float().mean(0) for a in range(0, 7, chunk_size)]
).to(head_dtype or dtype)
expected = torch.nn.functional.normalize(projector(means), p=2, dim=-1)
torch.testing.assert_close(actual, expected)
assert actual.dtype == (head_dtype or dtype)
def test_late_chunk_pool_reduces_before_nonlinear_projector():
hidden = torch.tensor([[2.0, 0.0], [0.0, 1.0]])
projector = torch.nn.Tanh()
head = TokenEmbeddingPoolerHead(projector=projector, activation=PoolerNormalize())
params = PoolingParams(
task="token_embed",
late_chunking_params=LateChunkingParams(chunk_size=2),
use_activation=True,
)
actual = head.forward_chunk(hidden, params)
expected = torch.nn.functional.normalize(projector(hidden.mean(0, keepdim=True)))
wrong_order = torch.nn.functional.normalize(projector(hidden).mean(0, keepdim=True))
torch.testing.assert_close(actual, expected)
assert not torch.allclose(actual, wrong_order)
def test_late_chunk_pool_isolates_mixed_requests_and_owns_finished_storage():
hidden = torch.arange(36, dtype=torch.float32).reshape(9, 4)
params = [
PoolingParams(
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=2)
),
PoolingParams(task="token_embed"),
PoolingParams(
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=1)
),
]
pooler = TokenPooler(
TestAllPool._make_all_pool(async_scheduling=True), TokenEmbeddingPoolerHead()
)
outputs = pooler(hidden, _make_metadata([3, 4, 2], pooling_params=params))
expected = [
torch.stack([hidden[:2].mean(0), hidden[2]]),
hidden[3:7].clone(),
hidden[7:].clone(),
]
hidden.fill_(-999)
for actual, reference in zip(outputs, expected):
torch.testing.assert_close(actual, reference)
@pytest.mark.parametrize("chunk_size", [1, 2, 8])
@pytest.mark.parametrize("embedding_size", [1, 4])
def test_bos_eos_filter_preserves_chunk_vectors_in_mixed_batch(
chunk_size, embedding_size
):
hidden = torch.arange(10 * embedding_size, dtype=torch.float32).reshape(
10, embedding_size
)
pooler = BOSEOSFilter(
TokenPooler(TestAllPool._make_all_pool(), TokenEmbeddingPoolerHead()),
bos_token_id=101,
eos_token_id=102,
)
metadata = _make_metadata(
[5, 5],
token_ids=[[101, 1, 2, 3, 102]] * 2,
pooling_params=[
PoolingParams(
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size)
),
PoolingParams(task="token_embed"),
],
)
chunked, ordinary = pooler(hidden, metadata)
expected = torch.stack([part.mean(0) for part in hidden[:5].split(chunk_size)])
torch.testing.assert_close(chunked, expected)
torch.testing.assert_close(ordinary, hidden[6:9].squeeze(-1))
def test_bos_eos_filter_waits_for_complete_chunk_vectors():
pooler = BOSEOSFilter(
TokenPooler(
TestAllPool._make_all_pool(chunked=True), TokenEmbeddingPoolerHead()
),
bos_token_id=101,
)
params = [
PoolingParams(task="token_embed", late_chunking_params=LateChunkingParams(3))
]
first = _make_metadata(
[4],
token_ids=[[101, 1, 2, 102]],
pooling_params=params,
num_scheduled_tokens=[2],
seq_lens=[2],
)
hidden = torch.arange(8, dtype=torch.float32).reshape(4, 2)
assert pooler(hidden[:2], first) == [None]
last = _make_metadata(
[4],
token_ids=[[101, 1, 2, 102]],
pooling_params=params,
num_scheduled_tokens=[2],
seq_lens=[4],
)
last.pooling_states = first.pooling_states
actual = pooler(hidden[2:], last)[0]
torch.testing.assert_close(actual, torch.stack([hidden[:3].mean(0), hidden[3]]))
assert not last.pooling_states[0].hidden_states_cache
@@ -1,344 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
import torch
from vllm.platforms import current_platform
if current_platform.is_cuda():
pytest.skip(
"ROCm skinny GEMM tests are not supported on CUDA.",
allow_module_level=True,
)
from vllm.model_executor.layers import utils
def _on_gfx11() -> bool:
"""gfx11 check that is safe to evaluate at collection time on any platform."""
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx11
return on_gfx11()
@pytest.mark.parametrize("m", [1, 128])
def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch, m):
x = torch.randn(1, 64, dtype=torch.float16)
weight = torch.randn(m, 64, dtype=torch.float16)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_called_once()
llmm1_mock.assert_not_called()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("n", range(1, 6))
def test_rocm_unquantized_gemm_single_output_real_kernel(dtype, n):
torch.manual_seed(0)
x = torch.randn(n, 2048, device="cuda", dtype=dtype)
weight = torch.randn(1, 2048, device="cuda", dtype=dtype)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
def test_rocm_unquantized_gemm_makes_skinny_activation_contiguous(monkeypatch):
x = torch.randn(64, 4, dtype=torch.float16).t()
weight = torch.randn(128, 64, dtype=torch.float16)
assert x.shape == (4, 64)
assert x.stride() == (1, 4)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_called_once()
x_view = wvsplitk_mock.call_args.args[1]
assert x_view.is_contiguous()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
def test_rocm_unquantized_gemm_makes_llmm1_activation_contiguous(monkeypatch):
x = torch.randn(1, 128, dtype=torch.float16)[:, ::2]
weight = torch.randn(4, 64, dtype=torch.float16)
assert x.shape == (1, 64)
assert x.stride() == (128, 2)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
llmm1_mock.assert_called_once()
x_view = llmm1_mock.call_args.args[1]
assert x_view.is_contiguous()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
@pytest.mark.parametrize("noncontiguous_operand", ["weight", "bias"])
def test_rocm_unquantized_gemm_rejects_unsupported_skinny_layouts(
monkeypatch, noncontiguous_operand
):
x = torch.randn(4, 64, dtype=torch.float16)
weight = torch.randn(128, 64, dtype=torch.float16)
bias = torch.randn(128, dtype=torch.float16)
if noncontiguous_operand == "weight":
weight = torch.randn(64, 128, dtype=torch.float16).t()
assert not weight.is_contiguous()
else:
bias = torch.randn(256, dtype=torch.float16)[::2]
assert not bias.is_contiguous()
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.rocm_aiter_ops, "is_tgemm_enabled", lambda: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock()
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
llmm1_mock = MagicMock()
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, bias)
ref = torch.nn.functional.linear(x, weight, bias)
wvsplitk_mock.assert_not_called()
llmm1_mock.assert_not_called()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
def _row_stride_padded(weight: torch.Tensor, pad: int) -> torch.Tensor:
"""A weight whose row stride is offset by ``pad`` elements, as produced by
``maybe_pad_weight_avoid_cache_aliasing``."""
padded = torch.nn.functional.pad(weight, (0, pad))[..., :-pad]
assert padded.shape == weight.shape
assert padded.stride() == (weight.shape[1] + pad, 1)
return padded
def test_rocm_unquantized_gemm_wvsplitk_accepts_row_stride_padded_weight(monkeypatch):
# wvSplitK reads the weight row stride, so a row-stride-padded weight must
# still reach it instead of falling back to torch.nn.functional.linear.
x = torch.randn(4, 1024, dtype=torch.float16)
weight = _row_stride_padded(torch.randn(128, 1024, dtype=torch.float16), 64)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.rocm_aiter_ops, "is_tgemm_enabled", lambda: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_called_once()
assert wvsplitk_mock.call_args.args[0].stride() == weight.stride()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
@pytest.mark.parametrize(
"k,element_size,expected",
[
(4096, 2, 64), # 8192 B row stride, aliases
(12288, 2, 64), # 24576 B row stride, aliases
(1024, 2, 64), # smallest aliasing bf16/fp16 K
(2560, 2, 0), # 5120 B, no aliasing
(11008, 2, 0), # 22016 B, no aliasing
(2048, 1, 128), # fp8: 2048 B, aliases, twice the pad
(512, 4, 32), # fp32: 2048 B, aliases, half the pad
],
)
def test_cache_aliasing_pad_elems(k, element_size, expected):
assert utils.cache_aliasing_pad_elems(k, element_size) == expected
def test_maybe_pad_weight_avoid_cache_aliasing_skips_non_gfx11(monkeypatch):
# Padding is gfx11-only, even for a K whose row stride would alias there.
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: False)
weight = torch.randn(128, 4096, dtype=torch.bfloat16)
assert utils.maybe_pad_weight_avoid_cache_aliasing(weight) is weight
def test_maybe_pad_weight_avoid_cache_aliasing(monkeypatch):
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
weight = torch.randn(128, 4096, dtype=torch.bfloat16)
padded = utils.maybe_pad_weight_avoid_cache_aliasing(weight)
assert padded.shape == weight.shape
assert padded.stride() == (4096 + 64, 1)
assert torch.equal(padded, weight)
# Idempotent: the padded stride no longer aliases, so a second call is a no-op
# and preserves the pointer captured by any graph.
repadded = utils.maybe_pad_weight_avoid_cache_aliasing(padded)
assert repadded.data_ptr() == padded.data_ptr()
@pytest.mark.parametrize(
"weight",
[
torch.randn(128, 2560, dtype=torch.bfloat16), # no aliasing
torch.randn(2560, 128, dtype=torch.bfloat16).t(), # not contiguous
torch.randn(2, 128, 4096, dtype=torch.bfloat16), # not 2D
],
)
def test_maybe_pad_weight_avoid_cache_aliasing_skips(monkeypatch, weight):
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
assert (
utils.maybe_pad_weight_avoid_cache_aliasing(weight).data_ptr()
== weight.data_ptr()
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_rocm_unquantized_gemm_noncontiguous_activation_real_kernel(monkeypatch, dtype):
x = torch.randn(64, 4, device="cuda", dtype=dtype).t()
weight = torch.randn(128, 64, device="cuda", dtype=dtype)
assert x.stride() == (1, 4)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
original_wvsplitk = utils.ops.wvSplitK
wvsplitk_mock = MagicMock(side_effect=original_wvsplitk)
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_called_once()
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not _on_gfx11(), reason="cache-aliasing padding only applies on gfx11"
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_rocm_unquantized_gemm_padded_weight_real_kernel(monkeypatch, dtype):
x = torch.randn(4, 4096, device="cuda", dtype=dtype)
weight = _row_stride_padded(torch.randn(128, 4096, device="cuda", dtype=dtype), 64)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
original_wvsplitk = utils.ops.wvSplitK
wvsplitk_mock = MagicMock(side_effect=original_wvsplitk)
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_called_once()
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("m", [1, 128])
def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch, m):
# wvSplitK skinny GEMM handles n in [1, 5] (see PR #40687); n > 5 must
# fall back to torch.nn.functional.linear.
x = torch.randn(6, 64, dtype=torch.float16)
weight = torch.randn(m, 64, dtype=torch.float16)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_not_called()
llmm1_mock.assert_not_called()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
def test_rocm_unquantized_gemm_gfx950_wvsplitkrc_path(monkeypatch):
x = torch.randn(1024, 16, dtype=torch.float16).t()
weight = torch.randn(256, 1024, dtype=torch.float16)
assert x.stride() == (1, 16)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: True)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitkrc_mock = MagicMock(side_effect=lambda x_view, w, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitKrc", wvsplitkrc_mock)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitkrc_mock.assert_called_once()
wvsplitk_mock.assert_not_called()
x_view = wvsplitkrc_mock.call_args.args[0]
assert x_view.is_contiguous()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
@@ -1,28 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm import SamplingParams
from vllm.platforms import current_platform
test_model = "openai-community/gpt2"
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Create a sampling params object.
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="fastsafetensors requires NVIDIA/AMD GPUs",
)
def test_model_loader_download_files(vllm_runner):
with vllm_runner(test_model, load_format="fastsafetensors") as llm:
deserialized_outputs = llm.generate(prompts, sampling_params)
assert deserialized_outputs
@@ -1,49 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import glob
import tempfile
import huggingface_hub.constants
import pytest
import torch
from vllm.model_executor.model_loader.weight_utils import (
download_weights_from_hf,
fastsafetensors_weights_iterator,
safetensors_weights_iterator,
)
from vllm.platforms import current_platform
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="fastsafetensors requires NVIDIA/AMD GPUs",
)
@pytest.mark.parametrize("queue_size", [0, 1])
def test_fastsafetensors_model_loader(monkeypatch, queue_size):
monkeypatch.setenv("VLLM_FASTSAFETENSORS_QUEUE_SIZE", str(queue_size))
with tempfile.TemporaryDirectory() as tmpdir:
huggingface_hub.constants.HF_HUB_OFFLINE = False
download_weights_from_hf(
"openai-community/gpt2", allow_patterns=["*.safetensors"], cache_dir=tmpdir
)
safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
assert len(safetensors) > 0
fastsafetensors_tensors = {}
hf_safetensors_tensors = {}
for name, tensor in fastsafetensors_weights_iterator(safetensors, True):
fastsafetensors_tensors[name] = tensor
for name, tensor in safetensors_weights_iterator(safetensors, True):
hf_safetensors_tensors[name] = tensor
assert len(fastsafetensors_tensors) == len(hf_safetensors_tensors)
for name, fastsafetensors_tensor in fastsafetensors_tensors.items():
fastsafetensors_tensor = fastsafetensors_tensor.to("cpu")
assert fastsafetensors_tensor.dtype == hf_safetensors_tensors[name].dtype
assert fastsafetensors_tensor.shape == hf_safetensors_tensors[name].shape
assert torch.all(fastsafetensors_tensor.eq(hf_safetensors_tensors[name]))
@@ -1,28 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm import SamplingParams
from vllm.platforms import current_platform
test_model = "openai-community/gpt2"
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Create a sampling params object.
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="InstantTensor requires NVIDIA GPUs",
)
def test_model_loader_download_files(vllm_runner):
with vllm_runner(test_model, load_format="instanttensor") as llm:
deserialized_outputs = llm.generate(prompts, sampling_params)
assert deserialized_outputs
@@ -1,104 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import glob
import sys
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from safetensors.torch import save_file
from vllm.model_executor.model_loader.weight_utils import (
download_weights_from_hf,
instanttensor_weights_iterator,
safetensors_weights_iterator,
)
from vllm.platforms import current_platform
@pytest.mark.parametrize(
("weight_name", "local_ids", "collective"),
[
("mlp.experts.gate_proj.weight", {0}, True),
("mlp.experts.0.gate_proj.weight", {0}, False),
("mlp.experts.0.gate_proj.weight", set(), False),
],
)
def test_instanttensor_ep_collective_loading(
monkeypatch, tmp_path, weight_name, local_ids, collective
):
"""Keep fused checkpoints collective and expert decisions rank-independent."""
from vllm.model_executor.model_loader import weight_utils
path = tmp_path / "model.safetensors"
save_file({weight_name: torch.ones(1)}, str(path))
native_open = MagicMock()
reader = native_open.return_value.__enter__.return_value
reader.total_tensor_size = 0
reader.tensors.return_value = ()
monkeypatch.setitem(
sys.modules, "instanttensor", SimpleNamespace(safe_open=native_open)
)
group = SimpleNamespace(world_size=2, device_group=object())
monkeypatch.setattr(weight_utils, "get_world_group", lambda: group)
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(current_platform, "current_device", lambda: 0)
list(instanttensor_weights_iterator([str(path)], False, local_expert_ids=local_ids))
kwargs = native_open.call_args.kwargs
assert kwargs["process_group"] is (group.device_group if collective else None)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="InstantTensor requires NVIDIA GPUs",
)
def test_instanttensor_ep_filter_cuda(tmp_path):
weights = {
"mlp.experts.0.gate_proj.weight": torch.ones(2),
"mlp.experts.1.gate_proj.weight": torch.ones(2),
"mlp.experts.1.gate_proj.weight_scale": torch.ones(1),
"model.layers[0].attn.weight": torch.ones(2),
}
path = tmp_path / "model.safetensors"
save_file(weights, str(path))
loaded = dict(
instanttensor_weights_iterator([str(path)], False, local_expert_ids={0})
)
del weights["mlp.experts.1.gate_proj.weight"]
assert loaded.keys() == weights.keys()
for name, tensor in loaded.items():
torch.testing.assert_close(tensor.cpu(), weights[name], rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda(),
reason="InstantTensor requires NVIDIA GPUs",
)
def test_instanttensor_model_loader():
model_dir = download_weights_from_hf(
"openai-community/gpt2", cache_dir=None, allow_patterns=["*.safetensors"]
)
safetensors = glob.glob(f"{model_dir}/*.safetensors")
assert len(safetensors) > 0
instanttensor_tensors = {}
hf_safetensors_tensors = {}
for name, tensor in instanttensor_weights_iterator(safetensors, True):
instanttensor_tensors[name] = tensor.to("cpu")
for name, tensor in safetensors_weights_iterator(safetensors, True):
hf_safetensors_tensors[name] = tensor
assert len(instanttensor_tensors) == len(hf_safetensors_tensors)
for name, instanttensor_tensor in instanttensor_tensors.items():
assert instanttensor_tensor.dtype == hf_safetensors_tensors[name].dtype
assert instanttensor_tensor.shape == hf_safetensors_tensors[name].shape
assert torch.all(instanttensor_tensor.eq(hf_safetensors_tensors[name]))
if __name__ == "__main__":
test_instanttensor_model_loader()
@@ -1,721 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import os
import types
from datetime import timedelta
from pathlib import Path
from unittest.mock import patch
import pytest
import torch
from safetensors.torch import load_file, save_file
from tests.utils import create_new_process_for_each_test
from vllm import SamplingParams
from vllm.config.load import LoadConfig
from vllm.model_executor.model_loader import get_model_loader
from vllm.model_executor.model_loader import runai_streamer_loader as rsl
from vllm.utils.mem_constants import GiB_bytes
load_format = "runai_streamer"
test_model = "openai-community/gpt2"
# TODO(amacaskill): Replace with a GKE owned GCS bucket.
test_gcs_model = "gs://vertex-model-garden-public-us/codegemma/codegemma-2b/"
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Create a sampling params object.
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
def get_runai_model_loader():
load_config = LoadConfig(load_format=load_format)
return get_model_loader(load_config)
def test_get_model_loader_with_runai_flag():
model_loader = get_runai_model_loader()
assert model_loader.__class__.__name__ == "RunaiModelStreamerLoader"
def test_checkpoint_selection_hook_is_optional_for_model_detection():
from vllm.model_executor.models.interfaces import SupportsMultiModal
from vllm.model_executor.models.interfaces_base import (
VllmModel,
is_text_generation_model,
)
class ModelWithoutHook(torch.nn.Module):
def __init__(self, vllm_config, prefix=""):
super().__init__()
def embed_input_ids(self, input_ids):
return input_ids
def forward(self, input_ids, positions):
return input_ids
def compute_logits(self, hidden_states):
return hidden_states
assert is_text_generation_model(ModelWithoutHook)
assert is_text_generation_model(ModelWithoutHook(None))
class ModelWithDefaultRule(ModelWithoutHook, VllmModel):
pass
assert not ModelWithDefaultRule(None).is_unused_checkpoint_weight("any.weight")
# Structural models do not inherit the protocol's optional default hook.
for names in ([], ["language_model"]):
model = torch.nn.Module()
model.language_model = ModelWithoutHook(None)
model._language_model_names = names
assert SupportsMultiModal.get_language_model(model) is model.language_model
def test_runai_model_loader_download_files(vllm_runner):
with vllm_runner(test_model, load_format=load_format) as llm:
deserialized_outputs = llm.generate(prompts, sampling_params)
assert deserialized_outputs
# In-process MTP compilation must not retain GPU resources in the pytest worker.
@create_new_process_for_each_test("spawn")
def test_runai_mtp_real_checkpoint(vllm_runner, tmp_path, monkeypatch):
"""Skip main-only shards without changing a real MTP model's generation."""
from vllm.model_executor.models.qwen3_5_mtp import Qwen3_5MTP
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
from vllm.v1.metrics.reader import Counter
if not current_platform.is_cuda_alike():
pytest.skip("MTP serving requires CUDA or ROCm")
checkpoint = tmp_path / "checkpoint"
hf_api().snapshot_download(
repo_id="Qwen/Qwen3.5-0.8B-Base", local_dir=str(checkpoint)
)
original_files = list(checkpoint.glob("*.safetensors"))
tensors = {}
for path in original_files:
tensors.update(load_file(path))
# Reshard real, unmodified weights so selection cannot pass vacuously on
# the published single mixed file. This predicate needs no model state.
selector = Qwen3_5MTP.__new__(Qwen3_5MTP)
main: dict[str, torch.Tensor] = {}
draft: dict[str, torch.Tensor] = {}
for name, tensor in tensors.items():
(main if selector.is_unused_checkpoint_weight(name) else draft)[name] = tensor
assert main and draft
files = [checkpoint / "main.safetensors", checkpoint / "draft.safetensors"]
for path, contents in zip(files, (main, draft)):
save_file(contents, path, metadata={"format": "pt"})
for path in original_files:
path.unlink()
(checkpoint / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
name: path.name
for path, data in zip(files, (main, draft))
for name in data
}
}
)
)
del tensors, main, draft
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
opened = []
real_stream = rsl.runai_safetensors_weights_iterator
def stream(paths, *args):
opened.append(list(paths))
yield from real_stream(paths, *args)
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
results = []
for load_format in ("auto", "runai_streamer"):
with vllm_runner(
str(checkpoint),
load_format=load_format,
block_size=None,
enable_chunked_prefill=True,
disable_log_stats=False,
limit_mm_per_prompt={"image": 0, "video": 0},
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
) as runner:
outputs = runner.generate_greedy(prompts, max_tokens=32)
results.append(outputs)
accepted = next(
m.value
for m in runner.llm.get_metrics()
if isinstance(m, Counter)
and m.name == "vllm:spec_decode_num_accepted_tokens"
)
assert accepted > 0
assert len(opened) == 2
assert set(opened[0]) == {str(path) for path in files}
assert opened[1] == [str(files[1])]
matches = sum(a == b for a, b in zip(*results))
assert matches >= 3, "RunAI MTP generation diverged from the Default loader"
@pytest.mark.skip(
reason="Temporarily disabled due to GCS access issues. "
"TODO: Re-enable this test once the underlying issue is resolved."
)
def test_runai_model_loader_download_files_gcs(
vllm_runner, monkeypatch: pytest.MonkeyPatch
):
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "fake-project")
monkeypatch.setenv("RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS", "true")
monkeypatch.setenv(
"CLOUD_STORAGE_EMULATOR_ENDPOINT", "https://storage.googleapis.com"
)
with vllm_runner(test_gcs_model, load_format=load_format) as llm:
deserialized_outputs = llm.generate(prompts, sampling_params)
assert deserialized_outputs
def test_runai_passes_revision_by_name():
# revision must reach download_safetensors_index_file_from_hf as the
# ``revision`` keyword, not the positional ``subfolder`` slot.
fake_self = types.SimpleNamespace(
load_config=types.SimpleNamespace(download_dir="/cache", ignore_patterns=[])
)
with (
patch.object(rsl, "is_runai_obj_uri", return_value=False),
patch.object(rsl, "download_weights_from_hf", return_value="/folder"),
patch.object(
rsl, "list_safetensors", return_value=["/folder/model.safetensors"]
),
patch.object(rsl, "download_safetensors_index_file_from_hf") as mock_idx,
):
rsl.RunaiModelStreamerLoader._prepare_weights(fake_self, "org/model", "myrev")
mock_idx.assert_called_once()
assert mock_idx.call_args.kwargs.get("revision") == "myrev"
assert "myrev" not in mock_idx.call_args.args
def _runai_loader(extra):
return rsl.RunaiModelStreamerLoader(
LoadConfig(load_format="runai_streamer", model_loader_extra_config=extra)
)
@pytest.mark.parametrize(
"extra, match",
[
({"typo_key": 1}, "Unexpected extra config"),
({"distributed": "yes"}, "distributed must be a bool"),
({"concurrency": "16"}, "concurrency must be a positive integer"),
({"concurrency": -1}, "concurrency must be a positive integer"),
],
)
def test_runai_rejects_invalid_extra_config(extra, match):
# The loader used to silently drop unknown keys / wrong types / negatives.
with pytest.raises(ValueError, match=match):
_runai_loader(extra)
def test_runai_accepts_valid_extra_config():
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("RUNAI_STREAMER_CONCURRENCY", None)
os.environ.pop("RUNAI_STREAMER_MEMORY_LIMIT", None)
loader = _runai_loader(
{"distributed": True, "concurrency": 16, "memory_limit": 1024}
)
assert loader._is_distributed is True
assert os.environ["RUNAI_STREAMER_CONCURRENCY"] == "16"
assert os.environ["RUNAI_STREAMER_MEMORY_LIMIT"] == "1024"
def test_runai_invalid_extra_config_leaves_environ_untouched():
# A later invalid key must not leave an earlier valid key applied to
# os.environ (all values are validated before any global mutation).
with patch.dict(os.environ, {}, clear=False):
os.environ.pop("RUNAI_STREAMER_CONCURRENCY", None)
with pytest.raises(ValueError, match="memory_limit must be an integer >= -1"):
_runai_loader({"concurrency": 16, "memory_limit": -5})
assert "RUNAI_STREAMER_CONCURRENCY" not in os.environ
@pytest.mark.parametrize(
"model_config,expected_source",
[
# Object storage: model_weights holds the URI, model the pulled config dir.
(
types.SimpleNamespace(
model="/tmp/pulled-config-files",
model_weights="s3://bucket/weights",
revision="myrev",
),
"s3://bucket/weights",
),
# HF repo or local path: model_weights is empty, model is the source.
(
types.SimpleNamespace(
model="org/model", model_weights="", revision="myrev"
),
"org/model",
),
],
ids=["object_storage", "hf_repo"],
)
def test_runai_get_all_weights_resolves_source(model_config, expected_source):
fake_self = types.SimpleNamespace(
_get_weights_iterator=lambda path, revision, is_unused_weight=None: iter(
[(f"{path}@{revision}", None)]
)
)
weights = list(
rsl.RunaiModelStreamerLoader.get_all_weights(
fake_self, model_config, model=None
)
)
assert weights == [(f"{expected_source}@myrev", None)]
@pytest.mark.parametrize("selection", ["draft", "unadapted", "reject_all"])
def test_runai_selects_shards_before_streaming(tmp_path, monkeypatch, selection):
"""Unused shards must not reach RunAI; mixed shards retain their tensors."""
contents = [
{"main.weight": torch.tensor([1.0])},
{"main.bias": torch.tensor([2.0]), "draft.weight": torch.tensor([3.0])},
{"draft.bias": torch.tensor([4.0])},
]
files = []
for i, tensors in enumerate(contents):
path = str(tmp_path / f"part-{i}.safetensors")
save_file(tensors, path)
files.append(path)
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
opened = []
real_stream = rsl.runai_safetensors_weights_iterator
def stream(paths, *args):
opened.extend(paths)
yield from real_stream(paths, *args)
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
model = torch.nn.Module()
if selection == "draft":
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
elif selection == "reject_all":
model.is_unused_checkpoint_weight = lambda name: True
config = types.SimpleNamespace(model=str(tmp_path), model_weights="", revision=None)
loaded = dict(_runai_loader({}).get_all_weights(config, model))
expected = files[1:] if selection == "draft" else files
assert opened == expected
assert set(loaded) == {
name
for tensors in (contents[1:] if selection == "draft" else contents)
for name in tensors
}
for name, tensor in {**contents[1], **contents[2]}.items():
torch.testing.assert_close(loaded[name], tensor)
def _check_distributed_selection(rank, rendezvous, scenario):
torch.distributed.init_process_group(
"gloo",
init_method=rendezvous,
rank=rank,
world_size=2,
timeout=timedelta(seconds=30),
)
root = "s3://bucket/weights"
if scenario == "different_roots":
root += str(rank)
files = [root + "/main.safetensors", root + "/draft.safetensors"]
def pull_files(source, destination, **kwargs):
if scenario == "metadata_failure" and rank == 1:
raise OSError("Index temporarily unavailable")
(Path(destination) / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
"main.weight": "main.safetensors",
"draft.weight": "draft.safetensors",
}
}
)
)
try:
with pytest.MonkeyPatch.context() as monkeypatch:
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
monkeypatch.setattr(
rsl,
"_get_remote_checkpoint_keys",
lambda paths: [
["main.weight"]
if path.endswith("main.safetensors")
else ["draft.weight"]
for path in paths
],
)
monkeypatch.setattr(
rsl,
"get_world_group",
lambda: types.SimpleNamespace(
world_size=2, cpu_group=torch.distributed.group.WORLD
),
raising=False,
)
monkeypatch.setattr(
rsl,
"get_pp_group",
lambda: types.SimpleNamespace(
world_size=2 if scenario == "pipeline_draft" else 1
),
raising=False,
)
if scenario == "pipeline_draft" and rank == 0:
# Earlier PP stages never invoke the draft loader.
torch.distributed.barrier()
return
# Exercise real selection and Gloo; observe the distributed stream input.
monkeypatch.setattr(
rsl,
"runai_safetensors_weights_iterator",
lambda paths, *args: iter((path, None) for path in paths),
)
model = torch.nn.Module()
if not (scenario == "missing_hook" and rank == 1):
needed = (
"main." if scenario == "different_roles" and rank == 1 else "draft."
)
model.is_unused_checkpoint_weight = lambda name: (
not name.startswith(needed)
)
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
selected = [
path
for path, _ in _runai_loader({"distributed": True}).get_all_weights(
config, model
)
]
expected = files[1:] if scenario == "healthy" else files
assert selected == expected
if scenario == "pipeline_draft":
torch.distributed.barrier()
finally:
torch.distributed.destroy_process_group()
@pytest.mark.parametrize(
"scenario",
[
"healthy",
"metadata_failure",
"different_roles",
"missing_hook",
"different_roots",
"pipeline_draft",
],
)
def test_distributed_runai_preserves_every_ranks_required_shards(tmp_path, scenario):
"""All ranks must stream the same union even if one cannot filter safely."""
torch.multiprocessing.spawn(
_check_distributed_selection,
args=((tmp_path / "rendezvous").as_uri(), scenario),
nprocs=2,
)
@pytest.mark.parametrize("scheme", ["s3", "gs", "az"])
@pytest.mark.parametrize(
"index_kind",
[
"valid",
"missing",
"malformed",
"unavailable",
"unadapted",
"missing_shard",
"stale",
"header_unavailable",
],
)
def test_runai_remote_selection_uses_actual_weight_source(
tmp_path, monkeypatch, scheme, index_kind
):
root = f"{scheme}://bucket/weights"
files = [
root + "/main/part.safetensors",
root + "/draft/part.safetensors",
root + "/mixed.safetensors",
root + "/extra.safetensors",
]
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
pulled = []
def pull_files(source, destination, allow_pattern, ignore_pattern=None):
pulled.append((source.rstrip("/"), allow_pattern))
if index_kind == "unavailable":
# Storage SDKs do not necessarily derive errors from OSError.
raise Exception("Object metadata unavailable")
index = Path(destination) / "model.safetensors.index.json"
if index_kind in ("valid", "missing_shard", "stale", "header_unavailable"):
index.write_text(
json.dumps(
{
"weight_map": {
"main.weight": "main/part.safetensors",
"draft.weight": (
"absent.safetensors"
if index_kind == "missing_shard"
else "draft/part.safetensors"
),
"main.bias": "mixed.safetensors",
"draft.bias": "mixed.safetensors",
}
}
)
)
elif index_kind == "malformed":
index.write_text("not JSON")
monkeypatch.setattr(rsl, "runai_pull_files", pull_files, raising=False)
headers_checked = []
def checkpoint_keys(paths):
headers_checked.extend(paths)
if index_kind == "header_unavailable":
raise OSError("Header temporarily unavailable")
return [
["main.weight", "draft.unindexed_weight"]
if index_kind == "stale"
else ["main.weight"]
for _ in paths
]
monkeypatch.setattr(
rsl, "_get_remote_checkpoint_keys", checkpoint_keys, raising=False
)
opened = []
def stream(paths, *args):
opened.extend(paths)
return iter(())
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
model = torch.nn.Module()
if index_kind != "unadapted":
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
# The config directory is not the checkpoint: using its index could drop
# tensors from an independent model_weights source.
(tmp_path / "model.safetensors.index.json").write_text("not the weights index")
config = types.SimpleNamespace(
model=str(tmp_path), model_weights=root, revision=None
)
list(_runai_loader({}).get_all_weights(config, model))
assert opened == (files[1:] if index_kind == "valid" else files)
assert headers_checked == (
files[:1] if index_kind in ("valid", "stale", "header_unavailable") else []
)
assert pulled == (
[]
if index_kind == "unadapted"
else [
(
root,
[
"weights/model.safetensors.index.json",
"model.safetensors.index.json",
],
)
]
)
@pytest.mark.parametrize(
"weight_map",
[None, {}, [], {"draft.weight": None}, {"draft.weight": "../part.safetensors"}],
)
def test_runai_invalid_weight_map_keeps_original_files(monkeypatch, weight_map):
files = ["s3://bucket/model/part.safetensors"]
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
def pull_files(source, directory, **kwargs):
(Path(directory) / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": weight_map})
)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
monkeypatch.setattr(
rsl, "runai_safetensors_weights_iterator", lambda paths, *args: iter(paths)
)
model = torch.nn.Module()
model.is_unused_checkpoint_weight = lambda name: True
config = types.SimpleNamespace(
model="s3://bucket/model", model_weights="", revision=None
)
assert list(_runai_loader({}).get_all_weights(config, model)) == files
def test_runai_model_rule_error_is_not_hidden_by_metadata_fallback(monkeypatch):
files = ["s3://bucket/model/part.safetensors"]
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
def pull_files(source, directory, **kwargs):
(Path(directory) / "model.safetensors.index.json").write_text(
json.dumps({"weight_map": {"draft.weight": "part.safetensors"}})
)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
model = torch.nn.Module()
def invalid_rule(name):
raise ValueError("Invalid model rule")
model.is_unused_checkpoint_weight = invalid_rule
config = types.SimpleNamespace(
model="s3://bucket/model", model_weights="", revision=None
)
with pytest.raises(ValueError, match="Invalid model rule"):
list(_runai_loader({}).get_all_weights(config, model))
def test_runai_header_reader_uses_actual_tensor_names(tmp_path):
"""The RunAI header reader sees names even when an index omits them."""
path = tmp_path / "mixed.safetensors"
save_file(
{
"main.weight": torch.tensor([1.0]),
"draft.unindexed_weight": torch.tensor([2.0]),
},
path,
metadata={"format": "pt"},
)
(names,) = rsl._get_remote_checkpoint_keys([str(path)])
assert set(names) == {"main.weight", "draft.unindexed_weight"}
@pytest.mark.parametrize("stale", [False, True])
def test_runai_stale_index_keeps_actual_draft_weights(tmp_path, monkeypatch, stale):
"""Verify excluded headers before dropping a shard with real draft weights."""
root = "s3://bucket/checkpoint"
main_path = tmp_path / "main.safetensors"
draft_path = tmp_path / "draft.safetensors"
main_tensors = {"main.weight": torch.tensor([1.0])}
if stale:
main_tensors["draft.unindexed_weight"] = torch.tensor([2.0])
save_file(main_tensors, main_path)
save_file({"draft.weight": torch.tensor([3.0])}, draft_path)
paths = {
root + "/main.safetensors": str(main_path),
root + "/draft.safetensors": str(draft_path),
}
monkeypatch.setattr(rsl, "list_safetensors", lambda path: list(paths))
def pull_files(source, directory, **kwargs):
(Path(directory) / "model.safetensors.index.json").write_text(
json.dumps(
{
"weight_map": {
"main.weight": "main.safetensors",
"draft.weight": "draft.safetensors",
}
}
)
)
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
read_headers = rsl._get_remote_checkpoint_keys
monkeypatch.setattr(
rsl,
"_get_remote_checkpoint_keys",
lambda files: read_headers([paths[path] for path in files]),
)
stream = rsl.runai_safetensors_weights_iterator
def stream_local(files, *args):
for name, tensor in stream([paths[path] for path in files], *args):
yield name, tensor.clone()
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream_local)
model = torch.nn.Module()
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
loaded = dict(_runai_loader({}).get_all_weights(config, model))
assert set(loaded) == (
{"main.weight", "draft.unindexed_weight", "draft.weight"}
if stale
else {"draft.weight"}
)
torch.testing.assert_close(loaded["draft.weight"], torch.tensor([3.0]))
if stale:
torch.testing.assert_close(
loaded["draft.unindexed_weight"], torch.tensor([2.0])
)
@pytest.mark.parametrize(
"base_model,mul_model,add_model",
[
(
"Qwen/Qwen3-0.6B",
"inference-optimization/Qwen3-0.6B-debug-multiply",
"inference-optimization/Qwen3-0.6B-debug-add",
),
],
)
def test_runai_deep_sleep_reload_weights(base_model, mul_model, add_model, vllm_runner):
free, total = torch.accelerator.get_memory_info()
used_bytes_baseline = total - free
def sleep_and_reload(llm, path):
# Level 2 discards the parameter memory, so after wake_up the weights
# can only come from the reload: a tensor the streamer fails to
# deliver changes the output instead of keeping a stale valid value.
llm.get_llm().sleep(level=2)
free, total = torch.accelerator.get_memory_info()
assert total - free - used_bytes_baseline < 3 * GiB_bytes
llm.get_llm().wake_up(tags=["weights"])
llm.collective_rpc("reload_weights", kwargs={"weights_path": path})
free, total = torch.accelerator.get_memory_info()
assert total - free - used_bytes_baseline < 4 * GiB_bytes
llm.get_llm().wake_up(tags=["kv_cache"])
with vllm_runner(
model_name=base_model,
load_format=load_format,
enable_sleep_mode=True,
enable_prefix_caching=False,
max_model_len=16,
max_num_seqs=1,
) as llm:
base_output = llm.generate_greedy(["3 4 ="], max_tokens=4)
sleep_and_reload(llm, mul_model)
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
assert mul_perp < add_perp
sleep_and_reload(llm, add_model)
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
assert add_perp < mul_perp
# Round trip to the original weights over discarded memory must
# reproduce the pre-sleep output.
sleep_and_reload(llm, base_model)
assert llm.generate_greedy(["3 4 ="], max_tokens=4) == base_output
@@ -1,617 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import gc
import json
import os
import pathlib
import subprocess
import sys
from typing import Any
from unittest.mock import Mock
import pytest
import torch
import vllm.model_executor.model_loader.tensorizer
from tests.utils import VLLM_PATH, RemoteOpenAIServer
from vllm import SamplingParams
from vllm.engine.arg_utils import EngineArgs
from vllm.model_executor.model_loader.tensorizer import (
TensorizerConfig,
TensorSerializer,
is_vllm_tensorized,
open_stream,
tensorize_vllm_model,
)
from vllm.model_executor.model_loader.tensorizer_loader import (
BLACKLISTED_TENSORIZER_ARGS,
)
from vllm.utils.import_utils import PlaceholderModule
from .conftest import DummyExecutor, assert_from_collective_rpc
try:
import tensorizer
from tensorizer import EncryptionParams
except ImportError:
tensorizer = PlaceholderModule("tensorizer")
EncryptionParams = tensorizer.placeholder_attr("EncryptionParams")
class TensorizerCaughtError(Exception):
pass
EXAMPLES_PATH = VLLM_PATH / "examples"
pytest_plugins = ("pytest_asyncio",)
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Create a sampling params object.
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
def patch_init_and_catch_error(self, obj, method_name, expected_error: type[Exception]):
original = getattr(obj, method_name, None)
if original is None:
raise ValueError("Method '{}' not found.".format(method_name))
def wrapper(*args, **kwargs):
try:
return original(*args, **kwargs)
except expected_error as err:
raise TensorizerCaughtError from err
setattr(obj, method_name, wrapper)
self.load_model()
def assert_specific_tensorizer_error_is_raised(
executor,
obj: Any,
method_name: str,
expected_error: type[Exception],
):
with pytest.raises(TensorizerCaughtError):
executor.collective_rpc(
patch_init_and_catch_error,
args=(
obj,
method_name,
expected_error,
),
)
def is_curl_installed():
try:
subprocess.check_call(["curl", "--version"])
return True
except (subprocess.CalledProcessError, FileNotFoundError):
return False
def write_keyfile(keyfile_path: str):
encryption_params = EncryptionParams.random()
pathlib.Path(keyfile_path).parent.mkdir(parents=True, exist_ok=True)
with open(keyfile_path, "wb") as f:
f.write(encryption_params.key)
@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
def test_deserialized_encrypted_vllm_model_has_same_outputs(
model_ref, vllm_runner, tmp_path, model_path
):
args = EngineArgs(model=model_ref)
with vllm_runner(model_ref) as vllm_model:
key_path = tmp_path / model_ref / "model.key"
write_keyfile(key_path)
outputs = vllm_model.generate(prompts, sampling_params)
config_for_serializing = TensorizerConfig(
tensorizer_uri=str(model_path), encryption_keyfile=str(key_path)
)
tensorize_vllm_model(args, config_for_serializing)
config_for_deserializing = TensorizerConfig(
tensorizer_uri=str(model_path), encryption_keyfile=str(key_path)
)
with vllm_runner(
model_ref,
load_format="tensorizer",
model_loader_extra_config=config_for_deserializing,
) as loaded_vllm_model: # noqa: E501
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
# noqa: E501
assert outputs == deserialized_outputs
def test_deserialized_hf_model_has_same_outputs(
hf_runner, vllm_runner, tmp_path, model_ref, model_path
):
with hf_runner(model_ref) as hf_model:
max_tokens = 50
outputs = hf_model.generate_greedy(prompts, max_tokens=max_tokens)
with open_stream(model_path, "wb+") as stream:
serializer = TensorSerializer(stream)
serializer.write_module(hf_model.model)
with vllm_runner(
model_ref,
load_format="tensorizer",
model_loader_extra_config=TensorizerConfig(
tensorizer_uri=str(model_path),
num_readers=1,
),
) as loaded_hf_model:
deserialized_outputs = loaded_hf_model.generate_greedy(
prompts, max_tokens=max_tokens
)
assert outputs == deserialized_outputs
def test_load_without_tensorizer_load_format(vllm_runner, capfd, model_ref):
model = None
try:
model = vllm_runner(
model_ref, model_loader_extra_config=TensorizerConfig(tensorizer_uri="test")
)
pytest.fail("Expected RuntimeError for extra config keys")
except RuntimeError:
out, err = capfd.readouterr()
combined_output = out + err
assert (
"ValueError: Unexpected extra config keys for load format auto"
) in combined_output
finally:
del model
gc.collect()
torch.accelerator.empty_cache()
def test_raise_value_error_on_invalid_load_format(vllm_runner, capfd, model_ref):
model = None
try:
model = vllm_runner(
model_ref,
load_format="safetensors",
model_loader_extra_config=TensorizerConfig(tensorizer_uri="test"),
)
pytest.fail("Expected RuntimeError for extra config keys")
except RuntimeError:
out, err = capfd.readouterr()
combined_output = out + err
assert (
"ValueError: Unexpected extra config keys for load format safetensors"
) in combined_output
finally:
del model
gc.collect()
torch.accelerator.empty_cache()
@pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Requires 2 GPUs")
def test_tensorizer_with_tp_path_without_template(vllm_runner, capfd):
try:
model_ref = "EleutherAI/pythia-1.4b"
tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
vllm_runner(
model_ref,
load_format="tensorizer",
model_loader_extra_config=TensorizerConfig(
tensorizer_uri=tensorized_path,
num_readers=1,
s3_endpoint="object.ord1.coreweave.com",
),
tensor_parallel_size=2,
disable_custom_all_reduce=True,
)
except RuntimeError:
out, err = capfd.readouterr()
combined_output = out + err
assert (
"ValueError: For a sharded model, tensorizer_uri "
"should include a string format template like '%04d' "
"to be formatted with the rank "
"of the shard"
) in combined_output
@pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Requires 2 GPUs")
def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(
vllm_runner, tmp_path
):
model_ref = "EleutherAI/pythia-1.4b"
# record outputs from un-sharded un-tensorized model
with vllm_runner(
model_ref,
disable_custom_all_reduce=True,
enforce_eager=True,
) as base_model:
outputs = base_model.generate(prompts, sampling_params)
# load model with two shards and serialize with encryption
model_path = str(tmp_path / model_ref / "model-%02d.tensors")
key_path = tmp_path / (model_ref + ".key")
tensorizer_config = TensorizerConfig(
tensorizer_uri=model_path,
encryption_keyfile=str(key_path),
)
tensorize_vllm_model(
engine_args=EngineArgs(
model=model_ref,
tensor_parallel_size=2,
disable_custom_all_reduce=True,
enforce_eager=True,
),
tensorizer_config=tensorizer_config,
)
assert os.path.isfile(model_path % 0), "Serialization subprocess failed"
assert os.path.isfile(model_path % 1), "Serialization subprocess failed"
with vllm_runner(
model_ref,
tensor_parallel_size=2,
load_format="tensorizer",
disable_custom_all_reduce=True,
enforce_eager=True,
model_loader_extra_config=tensorizer_config,
) as loaded_vllm_model:
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
assert outputs == deserialized_outputs
@pytest.mark.flaky(reruns=3)
def test_vllm_tensorized_model_has_same_outputs(
model_ref, vllm_runner, tmp_path, model_path
):
gc.collect()
torch.accelerator.empty_cache()
config = TensorizerConfig(tensorizer_uri=str(model_path))
args = EngineArgs(model=model_ref)
with vllm_runner(model_ref) as vllm_model:
outputs = vllm_model.generate(prompts, sampling_params)
tensorize_vllm_model(args, config)
assert is_vllm_tensorized(config)
with vllm_runner(
model_ref, load_format="tensorizer", model_loader_extra_config=config
) as loaded_vllm_model:
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
# noqa: E501
assert outputs == deserialized_outputs
def test_load_with_just_model_tensors(just_serialize_model_tensors, model_ref):
# For backwards compatibility, ensure Tensorizer can be still be loaded
# for inference by passing the model reference name, not a local/S3 dir,
# and the location of the model tensors
model_dir = just_serialize_model_tensors
extra_config = {"tensorizer_uri": f"{model_dir}/model.tensors"}
## Start OpenAI API server
args = [
"--load-format",
"tensorizer",
"--model-loader-extra-config",
json.dumps(extra_config),
]
with RemoteOpenAIServer(model_ref, args):
# This test only concerns itself with being able to load the model
# and successfully initialize the server
pass
def test_assert_serialization_kwargs_passed_to_tensor_serializer(tmp_path, vllm_runner):
serialization_params = {
"limit_cpu_concurrency": 2,
}
model_ref = "facebook/opt-125m"
model_path = tmp_path / (model_ref + ".tensors")
config = TensorizerConfig(
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
)
def serialization_test(self, *args, **kwargs):
# This is performed in the ephemeral worker process, so monkey-patching
# will actually work, and cleanup is guaranteed so don't
# need to reset things
original_dict = serialization_params
to_compare = {}
original = tensorizer.serialization.TensorSerializer.__init__
def tensorizer_serializer_wrapper(self, *args, **kwargs):
nonlocal to_compare
to_compare = kwargs.copy()
return original(self, *args, **kwargs)
tensorizer.serialization.TensorSerializer.__init__ = (
tensorizer_serializer_wrapper
)
tensorizer_config = TensorizerConfig(**kwargs["tensorizer_config"])
self.save_tensorized_model(
tensorizer_config=tensorizer_config,
)
return to_compare | original_dict == to_compare
kwargs = {"tensorizer_config": config.to_serializable()}
with vllm_runner(model_ref) as runner:
assert assert_from_collective_rpc(runner.get_llm(), serialization_test, kwargs)
@pytest.mark.parametrize(
(
"serialization_error",
"renderer_shutdown_error",
"engine_shutdown_error",
"expected_error",
),
[
(None, None, None, None),
(RuntimeError("serialization failed"), None, None, "serialization failed"),
(
RuntimeError("serialization failed"),
ValueError("renderer shutdown failed"),
ValueError("engine shutdown failed"),
"serialization failed",
),
(
None,
ValueError("renderer shutdown failed"),
None,
"renderer shutdown failed",
),
(None, None, ValueError("engine shutdown failed"), "engine shutdown failed"),
],
)
def test_tensorize_vllm_model_shuts_down_engine(
monkeypatch,
serialization_error,
renderer_shutdown_error,
engine_shutdown_error,
expected_error,
):
from vllm.v1.engine.llm_engine import LLMEngine
engine_args = Mock()
config = Mock(encryption_keyfile=None)
engine = Mock()
engine.collective_rpc.side_effect = serialization_error
engine.renderer.shutdown.side_effect = renderer_shutdown_error
engine.engine_core.shutdown.side_effect = engine_shutdown_error
monkeypatch.setattr(LLMEngine, "from_vllm_config", Mock(return_value=engine))
monkeypatch.setenv("VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS", "7")
if expected_error is None:
tensorize_vllm_model(engine_args, config)
else:
with pytest.raises(Exception, match=expected_error) as exc_info:
tensorize_vllm_model(engine_args, config)
notes = getattr(exc_info.value, "__notes__", ())
if serialization_error is not None and renderer_shutdown_error is not None:
assert any("renderer shutdown failed" in note for note in notes)
if serialization_error is not None and engine_shutdown_error is not None:
assert any("engine shutdown failed" in note for note in notes)
engine.engine_core.shutdown.assert_called_once_with(timeout=17.0)
engine.renderer.shutdown.assert_called_once_with()
def test_assert_deserialization_kwargs_passed_to_tensor_deserializer(tmp_path, capfd):
deserialization_kwargs = {
"num_readers": "bar", # illegal value
}
serialization_params = {
"limit_cpu_concurrency": 2,
}
model_ref = "facebook/opt-125m"
model_path = tmp_path / (model_ref + ".tensors")
config = TensorizerConfig(
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
)
args = EngineArgs(model=model_ref)
tensorize_vllm_model(args, config)
loader_tc = TensorizerConfig(
tensorizer_uri=str(model_path),
deserialization_kwargs=deserialization_kwargs,
)
engine_args = EngineArgs(
model="facebook/opt-125m",
load_format="tensorizer",
model_loader_extra_config=loader_tc.to_serializable(),
)
vllm_config = engine_args.create_engine_config()
executor = DummyExecutor(vllm_config)
assert_specific_tensorizer_error_is_raised(
executor,
tensorizer.serialization.TensorDeserializer,
"__init__",
TypeError,
)
def test_assert_stream_kwargs_passed_to_tensor_deserializer(tmp_path, capfd):
deserialization_kwargs = {
"num_readers": 1,
}
serialization_params = {
"limit_cpu_concurrency": 2,
}
model_ref = "facebook/opt-125m"
model_path = tmp_path / (model_ref + ".tensors")
config = TensorizerConfig(
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
)
args = EngineArgs(model=model_ref)
tensorize_vllm_model(args, config)
stream_kwargs = {"mode": "foo"}
loader_tc = TensorizerConfig(
tensorizer_uri=str(model_path),
deserialization_kwargs=deserialization_kwargs,
stream_kwargs=stream_kwargs,
)
engine_args = EngineArgs(
model="facebook/opt-125m",
load_format="tensorizer",
model_loader_extra_config=loader_tc.to_serializable(),
)
vllm_config = engine_args.create_engine_config()
executor = DummyExecutor(vllm_config)
assert_specific_tensorizer_error_is_raised(
executor,
vllm.model_executor.model_loader.tensorizer,
"open_stream",
ValueError,
)
@pytest.mark.asyncio
async def test_serialize_and_serve_entrypoints(tmp_path):
model_ref = "facebook/opt-125m"
suffix = "test"
try:
result = subprocess.run(
[
sys.executable,
f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py",
"--model",
model_ref,
"serialize",
"--serialized-directory",
str(tmp_path),
"--suffix",
suffix,
"--serialization-kwargs",
'{"limit_cpu_concurrency": 4}',
],
check=True,
capture_output=True,
text=True,
)
except subprocess.CalledProcessError as e:
print("Tensorizing failed.")
print("STDOUT:\n", e.stdout)
print("STDERR:\n", e.stderr)
raise
assert "Successfully serialized" in result.stdout
# Next, try to serve with vllm serve
model_uri = tmp_path / "vllm" / model_ref / suffix / "model.tensors"
model_loader_extra_config = {
"tensorizer_uri": str(model_uri),
"stream_kwargs": {
"force_http": False,
},
"deserialization_kwargs": {
"verify_hash": True,
"num_readers": 8,
},
}
cmd = [
"-m",
"vllm.entrypoints.cli.main",
"serve",
"--host",
"localhost",
"--load-format",
"tensorizer",
model_ref,
"--model-loader-extra-config",
json.dumps(model_loader_extra_config, indent=2),
]
proc = await asyncio.create_subprocess_exec(
sys.executable,
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT,
)
assert proc.stdout is not None
fut = proc.stdout.readuntil(b"Application startup complete.")
try:
await asyncio.wait_for(fut, 180)
except TimeoutError:
pytest.fail("Server did not start successfully")
finally:
proc.terminate()
await proc.communicate()
@pytest.mark.parametrize("illegal_value", BLACKLISTED_TENSORIZER_ARGS)
def test_blacklisted_parameter_for_loading(tmp_path, vllm_runner, capfd, illegal_value):
serialization_params = {
"limit_cpu_concurrency": 2,
}
model_ref = "facebook/opt-125m"
model_path = tmp_path / (model_ref + ".tensors")
config = TensorizerConfig(
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
)
args = EngineArgs(model=model_ref)
tensorize_vllm_model(args, config)
loader_tc = {"tensorizer_uri": str(model_path), illegal_value: "foo"}
try:
vllm_runner(
model_ref,
load_format="tensorizer",
model_loader_extra_config=loader_tc,
)
except RuntimeError:
out, err = capfd.readouterr()
combined_output = out + err
assert (
f"ValueError: {illegal_value} is not an allowed Tensorizer argument."
) in combined_output
@@ -1,603 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import fnmatch
import multiprocessing as mp
import os
import shutil
from tempfile import TemporaryDirectory
from types import SimpleNamespace
import pytest
import torch
from vllm import LLM, SamplingParams
from vllm.model_executor.model_loader import ShardedStateLoader
from vllm.transformers_utils.repo_utils import hf_api
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
# Create a sampling params object.
sampling_params = SamplingParams(
temperature=0,
max_tokens=256,
ignore_eos=True,
)
def test_filter_subtensors():
state_dict = {
"a": torch.empty(2),
"b": torch.empty((2, 4)),
"c": torch.empty((2, 4, 8)),
}
state_dict.update(
{
"x": state_dict["b"],
"y": state_dict["c"][1, 2, :],
"z": state_dict["c"][1, :, 4],
}
)
filtered_state_dict = ShardedStateLoader._filter_subtensors(state_dict)
assert tuple(filtered_state_dict.keys()) == ("a", "b", "c")
for key, tensor in filtered_state_dict.items():
# NOTE: don't use `equal` here, as the tensor might contain NaNs
assert tensor is state_dict[key]
def _mock_pp_group(
monkeypatch: pytest.MonkeyPatch,
world_size: int,
*,
pp_rank: int = 0,
tp_rank: int = 0,
tp_world_size: int = 1,
) -> None:
"""Stub the parallel-state accessors used by the sharded state loader."""
monkeypatch.setattr(
"vllm.distributed.get_pp_group",
lambda: SimpleNamespace(world_size=world_size, rank_in_group=pp_rank),
)
monkeypatch.setattr(
"vllm.distributed.get_tp_group",
lambda: SimpleNamespace(world_size=tp_world_size, rank_in_group=tp_rank),
)
monkeypatch.setattr(
"vllm.distributed.get_tensor_model_parallel_rank",
lambda: tp_rank,
)
monkeypatch.setattr(
"vllm.distributed.get_pipeline_model_parallel_rank",
lambda: pp_rank,
)
@pytest.mark.parametrize(
"world_size,expected",
[
(1, ShardedStateLoader.TP_ONLY_PATTERN),
(2, ShardedStateLoader.PP_AND_TP_PATTERN),
],
)
def test_get_default_pattern(monkeypatch, world_size, expected):
"""The default pattern is chosen from the pipeline parallel group size."""
_mock_pp_group(monkeypatch, world_size)
assert ShardedStateLoader._get_default_pattern() == expected
def test_render_default_pattern_default_tp(monkeypatch):
"""Without a custom pattern and PP=1, legacy TP-only names are rendered."""
_mock_pp_group(monkeypatch, 1)
assert (
ShardedStateLoader._render_default_pattern(part=0)
== "model-rank-0-part-0.safetensors"
)
def test_render_default_pattern_default_pp(monkeypatch):
"""Without a custom pattern and PP>1, pp/tp-aware names are rendered."""
_mock_pp_group(monkeypatch, 2)
assert (
ShardedStateLoader._render_default_pattern(part=0)
== "model-pp-0-tp-0-part-0.safetensors"
)
def test_render_default_pattern_custom_tp_only(monkeypatch):
"""A custom TP-only pattern is honored when pipeline parallelism is off."""
_mock_pp_group(monkeypatch, 1)
assert (
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.TP_ONLY_PATTERN, part=0
)
== "model-rank-0-part-0.safetensors"
)
def test_render_default_pattern_tp_only_with_pp_fails(monkeypatch):
"""TP-only filenames would collide across pipeline stages; fail fast."""
_mock_pp_group(monkeypatch, 2)
with pytest.raises(ValueError, match="does not embed the pipeline parallel"):
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.TP_ONLY_PATTERN, part=0
)
def test_render_default_pattern_pp_pattern_without_pp_fails(monkeypatch):
"""A pp/tp-aware pattern cannot be rendered when pipeline parallelism is off."""
_mock_pp_group(monkeypatch, 1)
with pytest.raises(ValueError, match="requires pipeline parallelism"):
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
)
def test_render_default_pattern_custom_pp(monkeypatch):
"""A custom pp/tp-aware pattern is honored when pipeline parallelism is on."""
_mock_pp_group(monkeypatch, 2)
assert (
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
)
== "model-pp-0-tp-0-part-0.safetensors"
)
@pytest.mark.parametrize(
"pattern,match",
[
# tp field present but pp field missing: pipeline stages with the
# same tp rank would write the same file.
(
"model-tp-{tp_rank}-part-{part}.safetensors",
"does not embed the pipeline parallel",
),
# pp field present but tp field missing: TP workers would overwrite
# each other's shards.
(
"model-pp-{pp_rank}-part-{part}.safetensors",
"does not embed the tensor parallel",
),
],
)
def test_render_default_pattern_incomplete_custom_patterns_fail(
monkeypatch, pattern, match
):
"""A custom pattern must distinguish every active rank (pp and tp)."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
with pytest.raises(ValueError, match=match):
ShardedStateLoader._render_default_pattern(pattern, part=0)
def test_render_default_pattern_no_rank_field_with_tp_fails(monkeypatch):
"""Without any rank field, TP workers would collide even when PP is off."""
_mock_pp_group(monkeypatch, 1, tp_world_size=2)
with pytest.raises(ValueError, match="does not embed the tensor parallel"):
ShardedStateLoader._render_default_pattern(
"model-part-{part}.safetensors", part=0
)
def test_render_default_pattern_custom_pp_tp1_allows_pp_only(monkeypatch):
"""With TP=1 a pp-only custom pattern is safe and must be accepted."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_world_size=1)
assert (
ShardedStateLoader._render_default_pattern(
"model-pp-{pp_rank}-part-{part}.safetensors", part=0
)
== "model-pp-1-part-0.safetensors"
)
@pytest.mark.parametrize(
"pattern,match",
[
# {{pp_rank}} renders as literal text: both stages would write the
# same file, so it must not count as a pp field.
(
"model-{{pp_rank}}-tp-{tp_rank}-part-{part}.safetensors",
"does not embed the pipeline parallel",
),
# Same for an escaped tp field: TP workers would overwrite each
# other's shards.
(
"model-pp-{pp_rank}-tp-{{tp_rank}}-part-{part}.safetensors",
"does not embed the tensor parallel",
),
],
)
def test_render_default_pattern_escaped_field_not_rank_field(
monkeypatch, pattern, match
):
"""Escaped braces are literal text and cannot pose as rank fields."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
with pytest.raises(ValueError, match=match):
ShardedStateLoader._render_default_pattern(pattern, part=0)
@pytest.mark.parametrize(
"pattern,world_size,tp_world_size,pp_rank,tp_rank,expected",
[
# Mixed tp alias works when pipeline parallelism is on.
(
"model-pp-{pp_rank}-rank-{rank}-part-{part}.safetensors",
2,
2,
1,
0,
"model-pp-1-rank-0-part-0.safetensors",
),
# tp_rank alias works for TP-only checkpoints too.
(
"model-tp-{tp_rank}-part-{part}.safetensors",
1,
2,
0,
3,
"model-tp-3-part-0.safetensors",
),
],
)
def test_render_default_pattern_tp_aliases_accepted(
monkeypatch, pattern, world_size, tp_world_size, pp_rank, tp_rank, expected
):
"""Both tp aliases are supplied at render time, so either works."""
_mock_pp_group(
monkeypatch,
world_size,
pp_rank=pp_rank,
tp_rank=tp_rank,
tp_world_size=tp_world_size,
)
assert ShardedStateLoader._render_default_pattern(pattern, part=0) == expected
def test_render_default_pattern_unsupported_field_fails(monkeypatch):
"""Unknown placeholders are rejected instead of KeyError-ing later."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_world_size=2)
with pytest.raises(ValueError, match="unsupported placeholder"):
ShardedStateLoader._render_default_pattern(
"model-{pp_rank}-{tp_rank}-{epoch}-part-{part}.safetensors", part=0
)
@pytest.mark.parametrize(
"pp_rank,tp_rank,expected",
[
(0, 0, "model-pp-0-tp-0-part-0.safetensors"),
(1, 2, "model-pp-1-tp-2-part-0.safetensors"),
(3, 5, "model-pp-3-tp-5-part-0.safetensors"),
],
)
def test_render_default_pattern_non_zero_ranks(monkeypatch, pp_rank, tp_rank, expected):
"""The rendered filename must embed the actual pp/tp ranks, not zeros."""
_mock_pp_group(monkeypatch, 2, pp_rank=pp_rank, tp_rank=tp_rank)
assert (
ShardedStateLoader._render_default_pattern(
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
)
== expected
)
def test_render_default_pattern_part_wildcard(monkeypatch):
"""The glob wildcard '*' passes through as the part token for file lookup."""
_mock_pp_group(monkeypatch, 1, tp_rank=4)
assert (
ShardedStateLoader._render_default_pattern(part="*")
== "model-rank-4-part-*.safetensors"
)
def test_save_model_pp_aware_filenames(monkeypatch, tmp_path):
"""save_model writes pp/tp-aware filenames for pipeline-parallel models."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=2)
saved: list[str] = []
def _capture(tensors, path, **kwargs):
saved.append(os.path.basename(path))
monkeypatch.setattr("safetensors.torch.save_file", _capture)
model = SimpleNamespace(
state_dict=lambda: {"w": torch.zeros(4, dtype=torch.float32)}
)
ShardedStateLoader.save_model(model, str(tmp_path))
assert saved == ["model-pp-1-tp-2-part-0.safetensors"]
def test_save_model_part_split(monkeypatch, tmp_path):
"""save_model rolls over to a new part when max_size is exceeded."""
_mock_pp_group(monkeypatch, 1, tp_rank=0)
saved: list[str] = []
def _capture(tensors, path, **kwargs):
saved.append(os.path.basename(path))
monkeypatch.setattr("safetensors.torch.save_file", _capture)
# Two float32[4] tensors (16 B each); max_size=20 fits one but not two.
model = SimpleNamespace(
state_dict=lambda: {
"a": torch.zeros(4, dtype=torch.float32),
"b": torch.zeros(4, dtype=torch.float32),
}
)
ShardedStateLoader.save_model(model, str(tmp_path), max_size=20)
assert saved == [
"model-rank-0-part-0.safetensors",
"model-rank-0-part-1.safetensors",
]
def test_save_model_rejects_bad_pattern_before_overwrite(monkeypatch, tmp_path):
"""An unsafe pattern is rejected before any shard file is written."""
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
# With the escaped pp field this pattern renders both stages to the
# same literal name; the validation must fire before any write.
colliding = tmp_path / "model-{pp_rank}-tp-0-part-0.safetensors"
colliding.write_bytes(b"existing shard")
before = sorted(p.name for p in tmp_path.iterdir())
model = SimpleNamespace(
state_dict=lambda: {"w": torch.zeros(4, dtype=torch.float32)}
)
with pytest.raises(ValueError, match="does not embed the pipeline parallel"):
ShardedStateLoader.save_model(
model,
str(tmp_path),
pattern="model-{{pp_rank}}-tp-{tp_rank}-part-{part}.safetensors",
)
# The pre-existing shard is untouched and nothing new was written.
assert colliding.read_bytes() == b"existing shard"
assert sorted(p.name for p in tmp_path.iterdir()) == before
@pytest.fixture(scope="module")
def llama_3p2_1b_files():
input_dir = hf_api().snapshot_download(
"meta-llama/Llama-3.2-1B-Instruct", ignore_patterns=["*.bin*", "original/*"]
)
yield input_dir
def _run_writer(input_dir, output_dir, weights_patterns, **kwargs):
llm_sharded_writer = LLM(model=input_dir, **kwargs)
# Dump worker states to output directory
llm_sharded_writer.llm_engine.engine_core.save_sharded_state(path=output_dir)
# Copy metadata files to output directory
for file in os.listdir(input_dir):
if os.path.isdir(os.path.join(input_dir, file)):
shutil.copytree(
os.path.join(input_dir, file), os.path.join(output_dir, file)
)
elif not any(fnmatch.fnmatch(file, ext) for ext in weights_patterns):
shutil.copy(os.path.join(input_dir, file), output_dir)
def _run_generate(input_dir, queue: mp.Queue, **kwargs):
llm = LLM(model=input_dir, **kwargs)
gen = llm.generate(prompts, sampling_params)
queue.put([g.outputs[0].__dict__ for g in gen])
queue.close()
queue.join_thread()
@pytest.mark.parametrize("enable_lora", [False, True])
@pytest.mark.parametrize("tp_size", [1, 2])
def test_sharded_state_loader(
enable_lora, tp_size, num_gpus_available, llama_3p2_1b_files
):
if num_gpus_available < tp_size:
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
weights_patterns = ("*.safetensors",)
gpu_memory_utilization = 0.8
input_dir = llama_3p2_1b_files
ctx = mp.get_context("spawn")
# Keep batching deterministic: this test compares exact greedy outputs
# across separate engine processes, whose schedulers may otherwise form
# different batches depending on prompt-processing timing.
platform_args = {"max_num_seqs": 1}
# Run in separate processes for memory & CUDA isolation
with TemporaryDirectory() as output_dir:
p = ctx.Process(
target=_run_writer,
args=(input_dir, output_dir, weights_patterns),
kwargs=dict(
tensor_parallel_size=tp_size,
gpu_memory_utilization=gpu_memory_utilization,
enforce_eager=True,
**platform_args,
),
)
p.start()
p.join()
queue = ctx.Queue()
p = ctx.Process(
target=_run_generate,
args=(input_dir, queue),
kwargs=dict(
enable_lora=enable_lora,
gpu_memory_utilization=gpu_memory_utilization,
tensor_parallel_size=tp_size,
**platform_args,
),
)
p.start()
# Call queue.get() before p.join() to prevent deadlock:
# If p.join() is called before queue.get() and the queue is full,
# the child process may block while writing to the queue and never
# terminate, causing the parent to wait indefinitely on p.join().
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
out_before = queue.get()
p.join()
queue.close()
queue.join_thread()
queue = ctx.Queue()
p = ctx.Process(
target=_run_generate,
args=(output_dir, queue),
kwargs=dict(
enable_lora=enable_lora,
gpu_memory_utilization=gpu_memory_utilization,
tensor_parallel_size=tp_size,
load_format="sharded_state",
**platform_args,
),
)
p.start()
# Call queue.get() before p.join() to prevent deadlock:
# If p.join() is called before queue.get() and the queue is full,
# the child process may block while writing to the queue and never
# terminate, causing the parent to wait indefinitely on p.join().
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
out_after = queue.get()
p.join()
queue.close()
queue.join_thread()
assert out_before == out_after
def _run_sharded_state_round_trip(
input_dir: str,
*,
tp_size: int,
pp_size: int,
num_gpus_available: int,
) -> None:
"""Save a TP/PP sharded checkpoint, reload it, and compare outputs.
Args:
input_dir: directory containing the original (un-sharded) model.
tp_size: tensor parallel size used for both save and load.
pp_size: pipeline parallel size used for both save and load.
num_gpus_available: number of GPUs on the current machine.
"""
if num_gpus_available < tp_size * pp_size:
pytest.skip(f"Not enough GPUs for TP={tp_size}, PP={pp_size}")
weights_patterns = ("*.safetensors",)
gpu_memory_utilization = 0.8
ctx = mp.get_context("spawn")
# Keep batching deterministic: this test compares exact greedy outputs
# across separate engine processes, whose schedulers may otherwise form
# different batches depending on prompt-processing timing.
platform_args = {"max_num_seqs": 1}
parallel_args = dict(
tensor_parallel_size=tp_size,
pipeline_parallel_size=pp_size,
)
with TemporaryDirectory() as output_dir:
p = ctx.Process(
target=_run_writer,
args=(input_dir, output_dir, weights_patterns),
kwargs=dict(
**parallel_args,
gpu_memory_utilization=gpu_memory_utilization,
enforce_eager=True,
**platform_args,
),
)
p.start()
p.join()
assert p.exitcode == 0, f"writer process failed with exit code {p.exitcode}"
if pp_size > 1:
# PP checkpoints must use the pp/tp-aware naming scheme so that
# different pipeline stages write distinct files.
sharded_files = [
f for f in os.listdir(output_dir) if f.endswith(".safetensors")
]
assert sharded_files, "no sharded checkpoint files were written"
assert any(
fnmatch.fnmatch(f, "model-pp-*-tp-*-part-*.safetensors")
for f in sharded_files
), f"expected PP-aware filenames, got {sharded_files}"
queue = ctx.Queue()
p = ctx.Process(
target=_run_generate,
args=(input_dir, queue),
kwargs=dict(
enable_lora=False,
gpu_memory_utilization=gpu_memory_utilization,
**parallel_args,
**platform_args,
),
)
p.start()
# Call queue.get() before p.join() to prevent deadlock:
# If p.join() is called before queue.get() and the queue is full,
# the child process may block while writing to the queue and never
# terminate, causing the parent to wait indefinitely on p.join().
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
out_before = queue.get()
p.join()
assert p.exitcode == 0, f"generate process failed with exit code {p.exitcode}"
queue.close()
queue.join_thread()
queue = ctx.Queue()
p = ctx.Process(
target=_run_generate,
args=(output_dir, queue),
kwargs=dict(
enable_lora=False,
gpu_memory_utilization=gpu_memory_utilization,
load_format="sharded_state",
**parallel_args,
**platform_args,
),
)
p.start()
out_after = queue.get()
p.join()
assert p.exitcode == 0, f"generate process failed with exit code {p.exitcode}"
queue.close()
queue.join_thread()
assert out_before == out_after
def test_sharded_state_loader_pp(num_gpus_available, llama_3p2_1b_files):
"""Round-trip sharded state save/load with pipeline parallelism (PP=2)."""
_run_sharded_state_round_trip(
llama_3p2_1b_files,
tp_size=1,
pp_size=2,
num_gpus_available=num_gpus_available,
)
def test_sharded_state_loader_pp_tp(num_gpus_available, llama_3p2_1b_files):
"""Round-trip sharded state save/load with pipeline + tensor parallelism
(PP=2, TP=2)."""
_run_sharded_state_round_trip(
llama_3p2_1b_files,
tp_size=2,
pp_size=2,
num_gpus_available=num_gpus_available,
)
@@ -1,486 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""End-to-end tests for the IPC weight cache loader.
A cold start (no weight cache daemon, so the loader falls back to disk) and
warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
identical outputs.
"""
import shutil
import subprocess
import sys
import tempfile
import threading
import time
import urllib.error
import urllib.request
from dataclasses import dataclass, field
from typing import Any
import pytest
import regex as re
from vllm import SamplingParams
from vllm.assets.image import ImageAsset
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
class WeightCacheDaemon:
"""Context manager running the real weight cache daemon as a subprocess."""
def __init__(
self,
model: str,
tp_size: int,
extra_args: list[str] | None = None,
):
# Short base path: Unix socket paths are limited to ~107 characters.
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
self.health_port = get_open_port()
self._cmd = [
sys.executable,
"-m",
"vllm.entrypoints.cli.main",
"preload",
"--model",
model,
"--tensor-parallel-size",
str(tp_size),
"--weight-cache-socket-dir",
self.socket_dir,
"--enforce-eager",
"--weight-cache-health-host",
"127.0.0.1",
"--weight-cache-health-port",
str(self.health_port),
*(extra_args or []),
]
self._proc: subprocess.Popen | None = None
def __enter__(self) -> "WeightCacheDaemon":
self._proc = subprocess.Popen(
self._cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True
)
threading.Thread(target=self._drain_stderr, daemon=True).start()
return self
def __exit__(self, *exc_info) -> None:
self._stop()
def _drain_stderr(self) -> None:
assert self._proc is not None and self._proc.stderr is not None
self._proc.stderr.read()
def check_health(self) -> None:
url = f"http://127.0.0.1:{self.health_port}/health"
try:
with urllib.request.urlopen(url, timeout=5) as response:
assert response.status == 200
except urllib.error.HTTPError as error:
raise AssertionError(
f"Weight cache daemon is not ready: HTTP {error.code}"
) from error
def wait_until_ready(self, timeout_s: float = 900) -> None:
"""Poll until READY (which covers the autotune tuners) or the preload
process dies."""
deadline = time.monotonic() + timeout_s
while True:
try:
self.check_health()
return
except (AssertionError, urllib.error.URLError):
if time.monotonic() > deadline:
raise
if self._proc is not None and self._proc.poll() is not None:
raise AssertionError("vllm preload exited before ready") from None
time.sleep(2)
def _stop(self) -> None:
assert self._proc is not None
self._proc.terminate()
try:
self._proc.wait(timeout=60)
except subprocess.TimeoutExpired:
self._proc.kill()
self._proc.wait()
shutil.rmtree(self.socket_dir, ignore_errors=True)
@dataclass
class ModelCase:
model: str
prompts: list[str]
images: list | None = None
llm_kwargs: dict[str, Any] = field(default_factory=dict)
daemon_args: list[str] = field(default_factory=list)
# Additionally assert the daemons tune in place before serving and write
# the on-disk FlashInfer autotune cache, and the restart engine reuses it
# (needs SM90+/FlashInfer).
check_flashinfer_cache: bool = False
def generate(
vllm_runner,
case: ModelCase,
socket_dir: str | None,
fallback: bool,
):
extra_config = (
{} if socket_dir is None else {"socket_dir": socket_dir, "fallback": fallback}
)
with vllm_runner(
case.model,
load_format="auto" if socket_dir is None else "ipc_cache",
model_loader_extra_config=extra_config,
# Greedy outputs are compared across engine restarts; cap the batch
# size so scheduling cannot change the results.
max_num_seqs=1,
**case.llm_kwargs,
) as llm:
sampling_params = SamplingParams(temperature=0, max_tokens=16, ignore_eos=True)
return llm.generate(case.prompts, sampling_params, images=case.images)
# Both hybrid-recurrent models need chunked prefill for their mamba-style
# cache mode.
QWEN_CASE = ModelCase(
model="Qwen/Qwen3.5-0.8B",
prompts=[
"Hello, my name is",
"The capital of France is",
],
llm_kwargs=dict(
gpu_memory_utilization=0.3,
enforce_eager=True,
enable_chunked_prefill=True,
),
)
# Kimi K3 adds the mxfp4-pack MoE path (post-load kernel setup runs in
# pre-processed mode) and a vision tower, exercised with an image request.
K3_CASE = ModelCase(
model="tiny-random/kimi-k3",
prompts=[
"The capital of France is",
"def quicksort(arr):",
"<|kimi_image_placeholder|>What is shown in the image?",
],
images=[None, None, ImageAsset("stop_sign").pil_image],
llm_kwargs=dict(
gpu_memory_utilization=0.3,
enforce_eager=True,
trust_remote_code=True,
enable_chunked_prefill=True,
# The image placeholder expands to >1k tokens for the stop_sign asset.
max_model_len=4096,
),
daemon_args=["--trust-remote-code"],
# Kimi K3 has tunable FlashInfer ops (a dense model like Qwen2.5 tunes
# nothing and upstream never writes the cache file for it).
check_flashinfer_cache=True,
)
# Qwen3.5-0.8B ships one MTP layer in the target checkpoint, so method="mtp"
# loads the draft from the same model. The daemon must cache it in its draft
# group for the warm runs (fallback=False) to succeed.
QWEN_MTP_CASE = ModelCase(
model="Qwen/Qwen3.5-0.8B",
prompts=[
"Hello, my name is",
"The capital of France is",
],
llm_kwargs=dict(
gpu_memory_utilization=0.3,
enforce_eager=True,
enable_chunked_prefill=True,
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
),
daemon_args=[
"--speculative-config",
'{"method": "mtp", "num_speculative_tokens": 1}',
],
)
@pytest.mark.parametrize(
"case",
[QWEN_CASE, K3_CASE, QWEN_MTP_CASE],
ids=["qwen3.5", "kimi-k3", "qwen3.5-mtp"],
)
def test_ipc_cache_cold_start_and_warm_restart(
vllm_runner, case: ModelCase, tmp_path, monkeypatch, capfd
):
"""Cold start falls back to disk; warm restarts load weights via CUDA IPC.
All runs must produce outputs identical to a default-loader baseline. The
warm runs disable the disk fallback, so they only pass if the weights
really came from the daemon — for the MTP case, both the target's and the
draft's daemon groups.
Cases with ``check_flashinfer_cache`` additionally cover in-daemon
autotune (default-on with `--enable-flashinfer-autotune`): each daemon
tunes before binding its socket and must write the on-disk FlashInfer
autotune cache, and the restart engine must load tactics from it instead
of re-profiling. The daemon's cache file is keyed with the
OPENAI_API_SERVER batch defaults while these engines run under LLM_CLASS,
so they never hit the daemon's file by design; reuse is proven against
the file the earlier in-process engines wrote.
"""
if not current_platform.is_cuda_alike():
pytest.skip("Weight cache IPC sharing requires CUDA or ROCm")
if case is K3_CASE and not current_platform.is_device_capability_family(100):
pytest.skip("Kimi K3 IPC weight cache requires an SM100 MXFP4 backend")
if case.check_flashinfer_cache:
from vllm.utils.flashinfer import has_flashinfer
if not (has_flashinfer() and current_platform.has_device_capability(90)):
pytest.skip("FlashInfer autotune requires FlashInfer and SM90+")
# Isolate the on-disk autotune cache. The engines are in-process
# (envs read lazily); the daemon subprocesses spawn later and
# inherit it.
monkeypatch.setenv("VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", str(tmp_path))
# Baseline: plain disk loading with the default loader.
baseline_outputs = generate(
vllm_runner,
case,
None,
fallback=True,
)
assert all(text for _, texts in baseline_outputs for text in texts)
# Cold start: no daemon is serving, so the loader falls back to disk.
with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir:
cold_outputs = generate(vllm_runner, case, empty_socket_dir, fallback=True)
if case.check_flashinfer_cache:
# The baseline engine already wrote its own (LLM_CLASS-keyed) table.
engine_cache_files = set(tmp_path.rglob("autotune_configs*.json"))
with WeightCacheDaemon(
case.model,
tp_size=1,
extra_args=case.daemon_args,
) as d:
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
d.wait_until_ready()
if case.check_flashinfer_cache:
# Each daemon tunes before its ready report; exactly one new file
# (their OPENAI_API_SERVER-keyed table) must appear.
tuner_files = set(tmp_path.rglob("autotune_configs*.json"))
tuner_files -= engine_cache_files
assert len(tuner_files) == 1, (
f"expected the tuner's tuned table, got {tuner_files}"
)
# Warm restart: a second engine lifetime against the same daemon.
if case.check_flashinfer_cache:
capfd.readouterr() # drain: the assertion reads the increment
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
if case.check_flashinfer_cache:
# The restart's EngineCore subprocess inherits pytest's fds; the
# flashinfer autotuner logs "[Autotuner]: Loaded N configs from
# <path>" (INFO, not once-deduplicated) on a disk cache hit.
out = capfd.readouterr()
m = re.search(
r"\[Autotuner\]: Loaded (\d+) configs from", out.out + out.err
)
assert m and int(m.group(1)) > 0, (
"restart engine did not load the on-disk autotune cache"
)
assert cold_outputs == baseline_outputs
assert warm_outputs == baseline_outputs
assert restart_outputs == baseline_outputs
def _parallel(**kw):
from types import SimpleNamespace
base = dict(
tensor_parallel_size=1,
pipeline_parallel_size=1,
data_parallel_size=1,
data_parallel_size_local=1,
data_parallel_rank=0,
nnodes=1,
node_rank=0,
)
base.update(kw)
return SimpleNamespace(**base)
def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
"""Each node serves a contiguous global-rank block (DP-major), so without
DP local GPU i is TP rank r*local+i, and single-node DP launchers offset
by --data-parallel-start-rank."""
from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks
tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1)
assert plan_local_ranks(tp_second_node) == [(i, 0, 0, 4 + i) for i in range(4)]
dp_third_node = _parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8
)
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0, 0) for i in range(4)]
dp_tp = _parallel(
tensor_parallel_size=2,
data_parallel_size=4,
data_parallel_size_local=2,
data_parallel_rank=2,
)
assert plan_local_ranks(dp_tp) == [
(0, 2, 0, 0),
(1, 2, 0, 1),
(2, 3, 0, 0),
(3, 3, 0, 1),
]
# DP + nnodes: node-local TP replicas
dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2)
assert plan_local_ranks(dp_tp_node0) == [(i, 0, 0, i) for i in range(8)]
dp_tp_node1 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1
)
assert plan_local_ranks(dp_tp_node1) == [(i, 1, 0, i) for i in range(8)]
# DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes)
tp_span_node1 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1
)
assert plan_local_ranks(tp_span_node1) == [(i, 0, 0, 4 + i) for i in range(4)]
tp_span_node2 = _parallel(
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2
)
assert plan_local_ranks(tp_span_node2) == [(i, 1, 0, i) for i in range(4)]
# DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes)
dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, 0, i % 4) for i in range(8)]
# PP adds one daemon placement per stage while preserving TP rank order.
pp_tp = _parallel(tensor_parallel_size=2, pipeline_parallel_size=2)
assert plan_local_ranks(pp_tp) == [
(0, 0, 0, 0),
(1, 0, 0, 1),
(2, 0, 1, 0),
(3, 0, 1, 1),
]
def test_daemon_rejects_unmappable_parallelism():
from vllm.model_executor.model_loader.weight_cache.daemon import (
_reject_unsupported_parallelism,
)
_reject_unsupported_parallelism(
_parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=12
)
)
# DP combines with --nnodes when the world size divides evenly.
_reject_unsupported_parallelism(
_parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
)
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
with pytest.raises(ValueError, match="evenly divide"):
_reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2))
with pytest.raises(ValueError, match="evenly divide"):
_reject_unsupported_parallelism(
_parallel(tensor_parallel_size=2, data_parallel_size=3, nnodes=4)
)
with pytest.raises(ValueError, match="exceeds"):
_reject_unsupported_parallelism(
_parallel(
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=13
)
)
def test_weight_cache_key_distinguishes_dp_ranks():
from dataclasses import replace
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
key = WeightCacheKey(
checkpoint="ckpt",
model_arch="Arch",
tp_size=1,
tp_rank=0,
dtype="bf16",
quantization=None,
quant_config_hash="h",
revision=None,
vllm_version="v",
dp_size=16,
dp_rank=3,
pp_size=2,
pp_rank=1,
)
assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"]
assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"]
assert key.mismatched_fields(replace(key, pp_rank=0)) == ["pp_rank"]
@pytest.mark.parametrize("hf_quant_config", [None, {"quant_algo": "NVFP4"}])
def test_weight_cache_key_distinguishes_nvfp4_activation_override(
tmp_path, hf_quant_config
):
"""Cache keys isolate activation modes and reject ambiguous legacy hashes."""
import json
from dataclasses import replace
from types import SimpleNamespace
import torch
from vllm.config.quantization import QuantizationConfigArgs
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
from vllm.utils.hashing import safe_hash
model_config = SimpleNamespace(
model=str(tmp_path),
dtype=torch.bfloat16,
quantization="modelopt_fp4",
quantization_config=None,
revision=None,
hf_config=SimpleNamespace(
architectures=["NemotronHForCausalLM"],
quantization_config=hf_quant_config,
),
)
static_key = WeightCacheKey.from_model_config(model_config, tp_size=1, tp_rank=0)
model_config.quantization_config = QuantizationConfigArgs(
moe={"activation": "nvfp4_per_token"}
)
per_token_key = WeightCacheKey.from_model_config(model_config, tp_size=1, tp_rank=0)
assert static_key.mismatched_fields(per_token_key) == ["quant_config_hash"]
legacy_hash = (
""
if hf_quant_config is None
else safe_hash(
json.dumps(hf_quant_config, sort_keys=True).encode(),
usedforsecurity=False,
).hexdigest()
)
assert static_key.mismatched_fields(
replace(static_key, quant_config_hash=legacy_hash)
) == ["quant_config_hash"]
def test_ipc_loader_copy_mode_reports_no_external_weight_memory():
"""Copy mode clones the weights into the engine, so nothing is external
(vllm_config is never touched)."""
from vllm.config import LoadConfig
from vllm.model_executor.model_loader.weight_cache.ipc_loader import (
IpcModelLoader,
)
copy_mode = LoadConfig(
load_format="ipc_cache", model_loader_extra_config={"mode": "copy"}
)
loader = IpcModelLoader(copy_mode)
assert loader.get_external_weight_memory(None) == 0
@@ -1,93 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from torch.testing._internal.two_tensor import TwoTensor
from vllm.config import ParallelConfig
from vllm.model_executor import parameter
from vllm.model_executor.model_loader.weight_checksum import (
compute_tensor_digests,
zero_weights,
)
from vllm.v1.worker import gpu_worker
from vllm.v1.worker.gpu_worker import Worker
pytestmark = pytest.mark.cpu_test
class _Model(nn.Module):
def __init__(self):
super().__init__()
self.linear = nn.Linear(2, 2)
nn.init.ones_(self.linear.weight)
nn.init.ones_(self.linear.bias)
# A strided view, which cannot be reinterpreted as bytes in place.
self.strided = nn.Parameter(torch.arange(1.0, 7.0)[::2])
self.scalar = nn.Parameter(torch.tensor(2.0))
# Loading never restores buffers, so reset must leave them alone.
self.register_buffer("k_scale", torch.tensor(2.0))
def test_digests_cover_every_parameter():
assert set(compute_tensor_digests(_Model())) == {
"linear.weight",
"linear.bias",
"strided",
"scalar",
}
def test_zero_weights_changes_every_parameter_and_no_buffer():
model = _Model()
before = compute_tensor_digests(model)
zero_weights(model)
after = compute_tensor_digests(model)
assert all(before[name] != after[name] for name in before)
assert model.k_scale.item() == 2.0
def test_shared_weight_partitions_are_hashed_and_reset(monkeypatch):
monkeypatch.setattr(parameter, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(parameter, "get_tensor_model_parallel_world_size", lambda: 1)
model = nn.Module()
model.transform = parameter.SharedWeightParameter(weight_loader=None)
model.transform.add_partition(0, object(), 2, 2)
model.transform.partitions[0].data.fill_(3.0)
before = compute_tensor_digests(model)
zero_weights(model)
assert set(before) == {"transform.0"}
assert compute_tensor_digests(model) != before
def test_tensor_subclass_inner_tensors_are_hashed_and_reset():
model = nn.Module()
model.weight = nn.Parameter(
TwoTensor(torch.ones(2, 2), torch.full((2, 2), 2.0)), requires_grad=False
)
before = compute_tensor_digests(model)
zero_weights(model)
assert set(before) == {"weight.a", "weight.b"}
assert all(
before[name] != digest for name, digest in compute_tensor_digests(model).items()
)
def test_dense_dp_replicas_get_distinct_key_prefixes(monkeypatch):
for name in ("get_tp_group", "get_pp_group", "get_pcp_group"):
monkeypatch.setattr(gpu_worker, name, lambda: SimpleNamespace(rank_in_group=0))
prefixes = []
for dp_rank in (0, 1):
worker = object.__new__(Worker)
worker.parallel_config = ParallelConfig(
data_parallel_size=2, data_parallel_rank=dp_rank
)
# Dense DP resets data_parallel_rank to 0 in each engine.
worker.parallel_config.reconfigure_for_independent_dp_rank()
prefixes.append(worker._weight_checksum_prefix())
assert prefixes == ["dp0:pp0:pcp0:tp0:", "dp1:pp0:pcp0:tp0:"]
-266
View File
@@ -1,266 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib
from types import SimpleNamespace
import pytest
import torch
from vllm.model_executor.kernels.linear import (
B12xFp8BlockScaledMMKernel,
B12xMxFp4LinearKernel,
B12xMxfp8LinearKernel,
B12xNvFp4LinearKernel,
B12xTensorFP8ScaledMMLinearKernel,
)
from vllm.model_executor.warmup.b12x_warmup import b12x_warmup
from vllm.utils.b12x import B12xWarmupUnit, b12x_warmup_token_counts
def test_b12x_warmup_token_counts_cover_serving_regimes() -> None:
assert b12x_warmup_token_counts(
max_tokens=2048,
cudagraph_capture_sizes=[1, 2, 8, 128],
) == (1, 2, 8, 128, 2048)
@pytest.mark.parametrize("is_embedding", [False, True])
def test_b12x_block_quant_warmup_covers_each_layers_plans(monkeypatch, is_embedding):
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
ModelOptBlockQuantLinearMethod,
)
from vllm.model_executor.warmup.b12x_warmup import _collect_warmup_units
method = object.__new__(ModelOptBlockQuantLinearMethod)
method.codec = "q8_0"
method.is_embedding = is_embedding
model = torch.nn.ModuleList([torch.nn.Module(), torch.nn.Module()])
calls = []
monkeypatch.setattr(
method, "apply", lambda layer, x: calls.append((layer, x.shape, x.dtype))
)
monkeypatch.setattr(
method,
"embedding",
lambda layer, ids: calls.append((layer, ids.shape, ids.dtype)),
)
for index, layer in enumerate(model):
layer.b12x_warmup_provider = method
if is_embedding:
layer.weight = torch.nn.Parameter(
torch.empty((8, 8, 34), dtype=torch.uint8), requires_grad=False
)
layer.b12x_embedding_plans = {
dtype: SimpleNamespace(
handle=2 * index + offset, query=SimpleNamespace(max_rows=32)
)
for offset, dtype in enumerate((torch.int32, torch.int64))
}
else:
layer.b12x_block_weight = SimpleNamespace(
in_features=256, values=torch.empty(1)
)
layer.b12x_block_plan = SimpleNamespace(
handle=index, token_counts=(1, 4, 32)
)
units = list(_collect_warmup_units(model, (1, 17, 32), torch.bfloat16))
assert len(units) == 2
for unit in units:
unit.compile()
expected = (
[((32,), torch.int32), ((32,), torch.int64)]
if is_embedding
else [((rows, 256), torch.bfloat16) for rows in (1, 4, 17, 32)]
)
assert calls == [
(layer, shape, dtype) for layer in model for shape, dtype in expected
]
@pytest.mark.parametrize(
("kernel_cls", "module_name", "call_name", "layer", "name"),
[
(
B12xMxFp4LinearKernel,
"vllm.model_executor.kernels.linear.mxfp4.b12x",
"_apply_b12x_mxfp4_linear",
SimpleNamespace(
weight=torch.empty((48, 64), dtype=torch.uint8),
weight_scale=torch.empty((128, 4), dtype=torch.uint8),
),
"MXFP4",
),
(
B12xNvFp4LinearKernel,
"vllm.model_executor.kernels.linear.nvfp4.b12x",
"_apply_b12x_nvfp4_linear",
SimpleNamespace(
weight=torch.empty((48, 64), dtype=torch.uint8),
weight_scale=torch.empty((128, 8), dtype=torch.float8_e4m3fn),
input_global_scale_inv=torch.tensor(2.0),
alpha=torch.tensor(0.25),
),
"NVFP4",
),
(
B12xFp8BlockScaledMMKernel,
"vllm.model_executor.kernels.linear.scaled_mm.b12x",
"_run_b12x_fp8_block_scaled_mm",
SimpleNamespace(
weight=torch.empty((256, 128), dtype=torch.float8_e4m3fn),
weight_scale_inv=torch.empty((2, 1), dtype=torch.float32),
),
"block-FP8",
),
],
)
def test_b12x_warmup_units_cover_token_counts(
monkeypatch,
kernel_cls,
module_name: str,
call_name: str,
layer,
name: str,
) -> None:
calls = []
monkeypatch.setattr(
importlib.import_module(module_name),
call_name,
lambda *args: calls.append(args),
)
kernel = object.__new__(kernel_cls)
unit = kernel.get_b12x_warmup_unit(layer, (1, 8), torch.bfloat16)
unit.compile()
assert unit.name == name
assert [args[0].shape[0] for args in calls] == [1, 8]
assert unit.key[-1] == torch.bfloat16
def test_b12x_mxfp8_warmup_unit(monkeypatch) -> None:
import vllm.model_executor.kernels.linear.mxfp8.b12x as b12x_mod
calls = []
monkeypatch.setattr(
b12x_mod,
"_import_b12x_mxfp8",
lambda: SimpleNamespace(
mm=lambda *args, **kwargs: calls.append((args, kwargs))
),
)
monkeypatch.setattr(
b12x_mod,
"current_stream",
lambda: SimpleNamespace(cuda_stream=object()),
)
packed_weight = SimpleNamespace(
in_features=128,
padded_in_features=128,
out_features=256,
weight=SimpleNamespace(values=torch.empty(1)),
)
layer = SimpleNamespace(b12x_mxfp8_packed_weight=packed_weight)
kernel = object.__new__(B12xMxfp8LinearKernel)
unit = kernel.get_b12x_warmup_unit(layer, (1, 8), torch.float16)
unit.compile()
assert [args[0].shape for args, _ in calls] == [(1, 128), (8, 128)]
assert [kwargs["expected_m"] for _, kwargs in calls] == [1, 8]
def test_b12x_tensor_fp8_warmup_unit(monkeypatch) -> None:
import vllm.model_executor.kernels.linear.scaled_mm.b12x as b12x_mod
calls = []
monkeypatch.setattr(
b12x_mod,
"_import_b12x_tensor_fp8",
lambda: SimpleNamespace(
prewarm=lambda *args, **kwargs: calls.append((args, kwargs))
),
)
monkeypatch.setattr(
b12x_mod,
"current_stream",
lambda: SimpleNamespace(cuda_stream=object()),
)
packed_weight = SimpleNamespace(
in_features=128,
padded_in_features=128,
out_features=256,
values=torch.empty(1),
)
layer = SimpleNamespace(b12x_tensor_fp8_packed_weight=packed_weight)
kernel = object.__new__(B12xTensorFP8ScaledMMLinearKernel)
unit = kernel.get_b12x_warmup_unit(layer, (1, 8), torch.bfloat16)
unit.compile()
assert calls[0][0] == (packed_weight, (1, 8))
assert calls[0][1]["out_dtype"] == torch.bfloat16
def test_b12x_warmup_deduplicates_registered_signatures(monkeypatch) -> None:
import vllm.model_executor.warmup.b12x_warmup as warmup_mod
calls: list[tuple[str, tuple[int, ...], torch.dtype]] = []
class Provider:
def get_b12x_warmup_unit(self, layer, token_counts, output_dtype):
return B12xWarmupUnit(
name="fake",
key=(type(self), layer.shape, output_dtype),
compile=lambda: calls.append((layer.name, token_counts, output_dtype)),
)
provider = Provider()
layers = [
SimpleNamespace(name="first", shape=(128, 256), b12x_warmup_provider=provider),
SimpleNamespace(
name="duplicate", shape=(128, 256), b12x_warmup_provider=provider
),
SimpleNamespace(name="second", shape=(256, 256), b12x_warmup_provider=provider),
SimpleNamespace(),
]
scans = 0
def modules():
nonlocal scans
scans += 1
return iter(layers)
worker = SimpleNamespace(
get_model=lambda: SimpleNamespace(modules=modules),
scheduler_config=SimpleNamespace(
max_num_batched_tokens=8,
max_num_scheduled_tokens=32,
),
model_config=SimpleNamespace(dtype=torch.float32),
vllm_config=SimpleNamespace(
compilation_config=SimpleNamespace(compile_sizes=[4, 16])
),
)
platform = SimpleNamespace(
is_cuda=lambda: True,
is_device_capability_family=lambda family: family == 120,
)
synchronized = []
monkeypatch.setattr(warmup_mod, "current_platform", platform)
monkeypatch.setattr(
warmup_mod.torch.accelerator,
"synchronize",
lambda: synchronized.append(True),
)
b12x_warmup(worker, [1, 2])
assert scans == 1
assert calls == [
("first", (1, 2, 4, 8, 16, 32), torch.bfloat16),
("second", (1, 2, 4, 8, 16, 32), torch.bfloat16),
]
assert synchronized == [True]
@@ -1,165 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for CPU unquantized GEMM dispatch behavior."""
import pytest
import torch
from vllm.model_executor.layers import utils
from vllm.platforms import CpuArchEnum, current_platform
from vllm.utils.torch_utils import set_default_torch_dtype
def test_dispatch_prepacks_arm_bf16_causal_conv(monkeypatch):
monkeypatch.setattr(
current_platform, "get_cpu_architecture", lambda: CpuArchEnum.ARM
)
monkeypatch.setattr(torch.cpu, "get_capabilities", lambda: {"bf16": True})
monkeypatch.setattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False)
packed = torch.randn(32, 4, dtype=torch.bfloat16)
pack_inputs = []
def pack(weight):
pack_inputs.append(weight.clone())
return packed
monkeypatch.setattr(utils.ops, "causal_conv1d_weight_pack", pack)
original = torch.randn(32, 1, 4, dtype=torch.bfloat16)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(original.clone(), requires_grad=False)
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
assert len(pack_inputs) == 1
# Check causal_conv1d_weight_pack received the expected weights
torch.testing.assert_close(pack_inputs[0], original.view(32, 4))
# Check we have correctly stashed the unpacked weights
torch.testing.assert_close(layer._cpu_unpacked_conv_weight, original.view(32, 4))
# Check we have correctly stored the packed weights
assert layer.weight.data_ptr() == packed.data_ptr()
def test_dispatch_does_not_pack_3d_expert_weight(monkeypatch):
monkeypatch.setattr(
current_platform, "get_cpu_architecture", lambda: CpuArchEnum.ARM
)
monkeypatch.setattr(torch.cpu, "get_capabilities", lambda: {"bf16": True})
monkeypatch.setattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False)
pack_calls = []
monkeypatch.setattr(
utils.ops,
"causal_conv1d_weight_pack",
lambda weight: pack_calls.append(weight),
)
layer = torch.nn.Module()
layer.weight = torch.nn.Parameter(
torch.randn(8, 32, 4, dtype=torch.bfloat16), requires_grad=False
)
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
assert pack_calls == []
@pytest.fixture(scope="module")
def _mock_zentorch_linear_unary():
"""Register a mock zentorch_linear_unary op when zentorch is not installed.
Allows the dispatch tests to run in CI without a real zentorch build.
Skips registration when zentorch is already available.
"""
if hasattr(torch.ops.zentorch, "zentorch_linear_unary"):
yield
return
lib_def = torch.library.Library("zentorch", "DEF")
lib_def.define(
"zentorch_linear_unary("
"Tensor input, "
"Tensor weight, "
"Tensor? bias, "
"bool is_weight_prepacked=False"
") -> Tensor"
)
lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
lib_impl.impl(
"zentorch_linear_unary",
lambda input, weight, bias, is_weight_prepacked=False: (
torch.nn.functional.linear(input, weight, bias)
),
)
yield
lib_impl._destroy()
lib_def._destroy()
@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
def test_dispatch_cpu_unquantized_gemm_uses_zentorch_on_zen(monkeypatch):
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
layer = torch.nn.Linear(16, 8, bias=True)
x = torch.randn(4, 16)
expected = torch.nn.functional.linear(x, layer.weight, layer.bias)
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
output = layer.cpu_linear(x, layer.weight, layer.bias)
torch.testing.assert_close(output, expected)
@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
def test_dispatch_cpu_unquantized_gemm_zen_remove_weight(monkeypatch):
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
layer = torch.nn.Linear(16, 8, bias=True)
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
assert layer.weight.numel() == 0
@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
def test_dispatch_cpu_unquantized_gemm_logs_zentorch_dispatch(monkeypatch):
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
expected_prepacked = bool(utils.envs.VLLM_ZENTORCH_WEIGHT_PREPACK) and hasattr(
torch.ops.zentorch, "zentorch_weight_prepack_for_linear"
)
log_calls = []
monkeypatch.setattr(
utils.logger, "debug_once", lambda *args: log_calls.append(args)
)
layer = torch.nn.Linear(16, 8, bias=True)
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
assert log_calls == [
(
"CPU unquantized GEMM dispatch: using zentorch_linear_unary (prepacked=%s)",
expected_prepacked,
)
]
@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
@pytest.mark.parametrize("weight_dtype", [torch.bfloat16, torch.float16, torch.float32])
def test_dispatch_cpu_unquantized_gemm_remove_weight_keeps_dtype(
monkeypatch, weight_dtype
):
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
layer = torch.nn.Linear(16, 8, bias=False, dtype=weight_dtype)
loading_dtype = (
torch.float32 if weight_dtype is not torch.float32 else torch.bfloat16
)
with set_default_torch_dtype(loading_dtype):
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
assert layer.weight.numel() == 0
assert layer.weight.dtype is weight_dtype
x = torch.randn(4, 16, dtype=weight_dtype)
output = layer.cpu_linear(x, layer.weight, None)
assert not output.isnan().any()
@@ -1,267 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import (
CompilationConfig,
VllmConfig,
get_cached_compilation_config,
set_current_vllm_config,
)
from vllm.model_executor.custom_op import CustomOp, op_registry
from vllm.model_executor.layers.activation import (
GeluAndMul,
ReLUSquaredActivation,
SiluAndMul,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import (
dispatch_topk_sigmoid_func,
dispatch_topk_softmax_func,
vllm_topk_sigmoid,
vllm_topk_softmax,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.platforms import current_platform
RMS_NORM_SUPPORTED_DTYPES = [torch.float16, torch.bfloat16]
# Registered subclass for test
@CustomOp.register("relu3")
class Relu3(ReLUSquaredActivation):
pass
@pytest.mark.parametrize(
"env, compilation_mode, backend, ops_enabled, default_on",
[
# Default values based on compile level
# - All by default (no Inductor compilation)
(None, 0, "eager", [True] * 4, True),
(None, 1, "eager", [True] * 4, True),
(None, 2, "eager", [True] * 4, True),
(None, 3, "eager", [True] * 4, True),
# - None by default (with Inductor)
(None, 0, "inductor", [True] * 4, True),
# - None by default (with Inductor)
(None, 1, "inductor", [False] * 4, False),
(None, 2, "inductor", [False] * 4, False),
(None, 3, "inductor", [False] * 4, False),
# Explicitly enabling/disabling
#
# Default: all
#
# All but SiluAndMul
("+rms_norm,-silu_and_mul", 0, "inductor", [1, 0, 1, 1], True),
# Only ReLU3
("none,-rms_norm,+relu3", 1, "eager", [0, 0, 0, 1], False),
# All but SiluAndMul
("all,-silu_and_mul", 2, "inductor", [1, 0, 1, 1], True),
# All but ReLU3 (even if ReLU2 is on)
("-relu3,+relu2", 3, "eager", [1, 1, 1, 0], True),
# RMSNorm and SiluAndMul
("none,-relu3,+rms_norm,+silu_and_mul", 3, "eager", [1, 1, 0, 0], False),
# All but RMSNorm
("-rms_norm", 3, "eager", [0, 1, 1, 1], True),
#
# Default: none
#
# Only ReLU3
("none,+relu3", 3, "inductor", [0, 0, 0, 1], False),
# All but RMSNorm
("all,-rms_norm", 3, "inductor", [0, 1, 1, 1], True),
],
)
def test_enabled_ops(
env: str | None,
compilation_mode: int,
backend: str,
ops_enabled: list[int],
default_on: bool,
):
custom_ops = env.split(",") if env else []
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
backend=backend, mode=compilation_mode, custom_ops=custom_ops
)
)
get_cached_compilation_config.cache_clear()
with set_current_vllm_config(vllm_config):
assert CustomOp.default_on() == default_on
ops_enabled = [bool(x) for x in ops_enabled]
assert RMSNorm(1024).enabled() == ops_enabled[0]
assert op_registry["rms_norm"].enabled() == ops_enabled[0]
assert SiluAndMul().enabled() == ops_enabled[1]
assert op_registry["silu_and_mul"].enabled() == ops_enabled[1]
assert GeluAndMul().enabled() == ops_enabled[2]
assert op_registry["gelu_and_mul"].enabled() == ops_enabled[2]
# If registered, subclasses should follow their own name
assert Relu3().enabled() == ops_enabled[3]
assert op_registry["relu3"].enabled() == ops_enabled[3]
# Unregistered subclass
class SiluAndMul2(SiluAndMul):
pass
# Subclasses should not require registration
assert SiluAndMul2().enabled() == SiluAndMul().enabled()
@pytest.mark.parametrize(
"env", ["all,none", "all,+rms_norm,all", "+rms_norm,-rms_norm"]
)
def test_enabled_ops_invalid(env: str):
with pytest.raises(Exception): # noqa
vllm_config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=env.split(","))
)
with set_current_vllm_config(vllm_config):
RMSNorm(1024).enabled()
@pytest.mark.parametrize(
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
)
def test_topk_softmax_dispatch(use_rocm_aiter: bool):
topk_func = dispatch_topk_softmax_func(use_rocm_aiter)
if current_platform.is_rocm() and use_rocm_aiter:
assert topk_func == rocm_aiter_ops.topk_softmax
else:
assert topk_func == vllm_topk_softmax
def _topk_gating_launch(
num_tokens: int = 4,
num_experts: int = 512,
topk: int = 10,
num_shared_experts: int = 0,
weights_width: int | None = None,
ids_width: int | None = None,
contiguous: bool = True,
) -> dict:
"""Tensors for one softmax top-k launch, as dispatch_topk_softmax_func sees them."""
width = topk + num_shared_experts
weights = torch.empty(num_tokens, weights_width or width)
ids = torch.empty(num_tokens, ids_width or width, dtype=torch.int32)
ids = ids[:, :topk]
if contiguous:
gating = torch.empty(num_tokens, num_experts + num_shared_experts)
else:
gating = torch.empty(num_experts + num_shared_experts + 3, num_tokens).t()
return dict(topk_weights=weights, topk_indices=ids, gating_output=gating)
@pytest.mark.parametrize(
"launch, num_shared_experts, scoring_func, expect_gating",
[
pytest.param({}, 0, "", True, id="plain-softmax"),
pytest.param({"num_tokens": 4096}, 0, "", True, id="max-tokens"),
pytest.param({"num_tokens": 4097}, 0, "", False, id="too-many-tokens"),
pytest.param({"contiguous": False}, 0, "", False, id="non-contiguous"),
pytest.param({}, 0, "sigmoid", False, id="scoring-without-shared"),
pytest.param({"num_shared_experts": 1}, 1, "sigmoid", True, id="one-shared"),
pytest.param({"num_shared_experts": 8}, 8, "sigmoid", True, id="eight-shared"),
pytest.param(
{"num_shared_experts": 3}, 3, "sigmoid", False, id="unsupported-count"
),
pytest.param({"num_shared_experts": 1}, 1, "", False, id="no-shared-scoring"),
pytest.param(
{"num_shared_experts": 1}, 1, "softmax", False, id="wrong-shared-scoring"
),
pytest.param(
{"num_shared_experts": 1, "weights_width": 10},
1,
"sigmoid",
False,
id="weights-too-narrow",
),
pytest.param(
{"num_shared_experts": 1, "ids_width": 12},
1,
"sigmoid",
False,
id="row-stride-mismatch",
),
pytest.param(
{"num_tokens": 4097, "num_shared_experts": 1},
1,
"sigmoid",
False,
id="shared-too-many-tokens",
),
],
)
@pytest.mark.parametrize("gating_enabled", [True, False])
def test_topk_softmax_dispatch_aiter_topk_gating(
monkeypatch: pytest.MonkeyPatch,
launch: dict,
num_shared_experts: int,
scoring_func: str,
expect_gating: bool,
gating_enabled: bool,
):
"""AITER launches use topk_gating only where it is enabled, and only when it
supports them.
Every other AITER launch must reach the legacy topk_softmax, so this is the
one place the choice is made.
"""
monkeypatch.setattr(
rocm_aiter_ops, "is_topk_gating_enabled", lambda: gating_enabled
)
topk_func = dispatch_topk_softmax_func(
True,
**_topk_gating_launch(**launch),
num_shared_experts=num_shared_experts,
shared_expert_scoring_func=scoring_func,
)
if expect_gating and gating_enabled:
assert topk_func == rocm_aiter_ops.topk_gating
else:
assert topk_func == rocm_aiter_ops.topk_softmax
@pytest.mark.parametrize(
"cdna, fused_moe_enabled, expected",
[
pytest.param(3, True, True, id="cdna3"),
pytest.param(4, True, True, id="cdna4"),
pytest.param(2, True, False, id="cdna2"),
pytest.param(3, False, False, id="cdna3-aiter-moe-off"),
pytest.param(4, False, False, id="cdna4-aiter-moe-off"),
],
)
def test_topk_gating_enabled_by_cdna(
monkeypatch: pytest.MonkeyPatch,
cdna: int,
fused_moe_enabled: bool,
expected: bool,
):
"""topk_gating is on for CDNA3 and CDNA4 with AITER MoE enabled, nowhere else."""
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: True)
monkeypatch.setattr(
rocm_aiter_ops, "is_fused_moe_enabled", lambda: fused_moe_enabled
)
monkeypatch.setattr("vllm.platforms.rocm.get_cdna_version", lambda: cdna)
assert bool(rocm_aiter_ops.is_topk_gating_enabled()) is expected
@pytest.mark.parametrize(
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
)
def test_topk_sigmoid_dispatch(use_rocm_aiter: bool):
topk_func = dispatch_topk_sigmoid_func(use_rocm_aiter)
if current_platform.is_rocm() and use_rocm_aiter:
assert topk_func == rocm_aiter_ops.topk_sigmoid
else:
assert topk_func == vllm_topk_sigmoid
@@ -1,586 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
from dataclasses import dataclass
from typing import Any
import pytest
import torch
from vllm import envs
from vllm.model_executor.warmup import jit_warmup_triton_helper
from vllm.model_executor.warmup.jit_warmup import (
JitWarmupRegistry,
WarmupChoices,
WarmupIntRange,
kernel_launcher,
)
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
LaunchSpec,
TritonJitKey,
TritonWarmupTensor,
VllmTritonJitKernel,
triton_kernel_dispatcher_with_warmup,
triton_warmup_inputs,
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@triton.jit
def _binder_test_kernel(x, value, BLOCK: tl.constexpr):
pass
@triton.jit
def _second_binder_test_kernel(x, value, BLOCK: tl.constexpr):
pass
@triton.jit
def _pointer_group_test_kernel(x_ptr, value, BLOCK: tl.constexpr):
pass
class _FakeTritonKernel:
arg_names: tuple[str, ...] = ("first", "second", "CONST")
def __init__(self) -> None:
self.warmup_calls: list[dict[str, Any]] = []
self.runtime_calls: list[tuple[Any, tuple[Any, ...], dict[str, Any]]] = []
def warmup(self, **kwargs: Any) -> None:
self.warmup_calls.append(kwargs)
def __getitem__(self, grid: Any) -> Any:
def launch(*args: Any, **kwargs: Any) -> None:
self.runtime_calls.append((grid, args, kwargs))
return launch
def _patch_key_deriver(
monkeypatch: pytest.MonkeyPatch,
derive: Any,
) -> None:
monkeypatch.setattr(
jit_warmup_triton_helper,
"_triton_key_deriver",
lambda kernel: lambda kwargs: derive(kernel, kwargs),
)
class _TestTritonKernel(VllmTritonJitKernel["_TestTritonKernel.CompileKey"]):
kernel = _FakeTritonKernel()
@dataclass(frozen=True)
class CompileKey:
value: int
def dispatch(self, *, value: int) -> CompileKey:
return self.CompileKey(value=value)
def get_warmup_keys(self) -> list[CompileKey]:
return self._trace_dispatch(self.dispatch)(value=1)
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
return dict(first="warmup", second=compile_key.value, runtime_launcher=None)
@kernel_launcher
def __call__(
self,
first: str,
second: int,
runtime_launcher: Any,
) -> LaunchSpec:
return (2,), dict(
CONST=7,
_runtime_launcher=runtime_launcher,
_runtime_launcher_arg_count=2,
)
def test_triton_launcher_supports_compile_and_runtime_adapters() -> None:
owner = _TestTritonKernel()
owner.kernel.warmup_calls.clear()
owner.compile(owner.CompileKey(value=1))
assert owner.kernel.warmup_calls == [
{"grid": (1,), "first": "warmup", "second": 1, "CONST": 7}
]
runtime_calls: list[tuple[Any, ...]] = []
def runtime_launcher(kernel: Any, grid: Any, *args: Any, **kwargs: Any) -> None:
runtime_calls.append((kernel, grid, args, kwargs))
owner("runtime", 2, runtime_launcher)
assert runtime_calls == [(owner.kernel, (2,), ("runtime", 2), {"CONST": 7})]
@pytest.mark.skipif(
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
reason="Requires CUDA Triton async compilation",
)
def test_failed_parallel_warmup_does_not_leak_into_runtime(monkeypatch) -> None:
from triton.runtime._async_compile import active_mode
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
owner = _TestTritonKernel()
owner.kernel = _FakeTritonKernel()
def fail():
raise RuntimeError("warmup failed")
def warmup(**kwargs):
active_mode.get().submit(kwargs["second"], fail, lambda result: None)
monkeypatch.setattr(owner.kernel, "warmup", warmup)
with pytest.raises(RuntimeError, match="warmup failed"):
owner.compile_many(owner.CompileKey(value=i) for i in range(2))
assert active_mode.get() is None
owner("runtime", 3, None)
assert len(owner.kernel.runtime_calls) == 1
def test_autotuning_warmup_stays_serial(monkeypatch) -> None:
owner = _TestTritonKernel()
owner.kernel = _FakeTritonKernel()
owner._run_autotune = True
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 4)
def unexpected_async(*args):
pytest.fail("Autotuning must not use asynchronous compilation")
monkeypatch.setattr(triton, "AsyncCompileMode", unexpected_async, raising=False)
owner.compile_many(owner.CompileKey(value=i) for i in range(2))
assert len(owner.kernel.runtime_calls) == 2
@triton.jit
def _warmup_store_kernel(out, VALUE: tl.constexpr):
tl.store(out, VALUE)
@pytest.mark.skipif(
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
reason="Requires CUDA Triton async compilation",
)
def test_registered_warmup_overlaps_kernels_and_runtime_stays_serial(
monkeypatch,
) -> None:
from triton.runtime._async_compile import active_mode
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
def make_store(value):
def warmup_inputs():
return dict(out=TritonWarmupTensor(torch.int32), value=value)
@triton_kernel_dispatcher_with_warmup(
kernel=_warmup_store_kernel, warmup_inputs=warmup_inputs
)
def store(out, value):
return (1,), dict(VALUE=value)
# Initialize Triton's binder before wrapping its compiler.
store.get_warmup_keys()
return store
stores = [make_store(value) for value in (1, 2)]
caller = threading.get_ident()
barrier = threading.Barrier(2, timeout=10)
compiled_on = []
compile_kernel = _warmup_store_kernel.compile
def record_compile(*args, **kwargs):
if threading.get_ident() != caller:
barrier.wait()
result = compile_kernel(*args, **kwargs)
compiled_on.append(threading.get_ident())
return result
monkeypatch.setattr(_warmup_store_kernel, "compile", record_compile)
registry = JitWarmupRegistry(None)
with registry.activate():
for store in stores:
store.register_warmup()
registry.warmup()
assert len(compiled_on) == 2
assert all(thread != caller for thread in compiled_on)
assert active_mode.get() is None
out = torch.empty(1, dtype=torch.int32, device=current_platform.device_type)
for store, value in zip(stores, (1, 2)):
store(out, value)
assert out.item() == value
assert len(compiled_on) == 2
# A specialization first seen during inference uses the normal blocking JIT.
store(out, 3)
assert out.item() == 3
assert compiled_on[2:] == [caller]
def test_triton_launcher_supports_cpu_function_wrappers() -> None:
calls: list[tuple[Any, ...]] = []
def kernel(first: str, second: int, CONST: int) -> None:
calls.append((first, second, CONST))
class FuncWrapper:
def __init__(self) -> None:
self.func = kernel
def __getitem__(self, _grid: Any) -> Any:
return self.func
class TestCpuKernel(_TestTritonKernel):
kernel: Any = FuncWrapper()
owner = TestCpuKernel()
owner("runtime", 2, None)
assert calls == [("runtime", 2, 7)]
def test_triton_kernel_decorator_returns_launcher(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeTritonKernel()
kernel.arg_names = (
"FIRST_PTR",
"FIRST_STRIDE0",
"aliased_ptr",
"aliased_stride_0",
"SECOND",
"CONST",
)
def warmup_inputs() -> dict[str, Any]:
return dict(
first=TritonWarmupTensor(torch.float32, shape=(2, 3), strides=(5, 1)),
second=WarmupChoices(1, 2),
)
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
def launch(first: str, second: int = 2, config: int = 7) -> LaunchSpec:
return (2,), dict(aliased_ptr=first, CONST=config)
def fake_keys(kernel: Any, kwargs: Any) -> set[TritonJitKey]:
return {TritonJitKey(id(kernel), "fake", 0, kwargs["SECOND"])}
_patch_key_deriver(monkeypatch, fake_keys)
keys = launch.get_warmup_keys()
keys_with_config = launch.get_warmup_keys(vllm_config=object())
assert keys_with_config == keys
assert [dict(key.inputs)["second"] for key in keys] == [1, 2]
launch.compile(keys[0])
assert kernel.warmup_calls == [
{
"grid": (1,),
"FIRST_PTR": TritonWarmupTensor(
torch.float32, shape=(2, 3), strides=(5, 1)
),
"FIRST_STRIDE0": 5,
"aliased_ptr": TritonWarmupTensor(
torch.float32, shape=(2, 3), strides=(5, 1)
),
"aliased_stride_0": 5,
"SECOND": 1,
"CONST": 7,
}
]
first = torch.empty_strided((2, 3), (5, 1))
launch(first)
assert kernel.runtime_calls == [
(
(2,),
(),
{
"FIRST_PTR": first,
"FIRST_STRIDE0": 5,
"aliased_ptr": first,
"aliased_stride_0": 5,
"SECOND": 2,
"CONST": 7,
},
)
]
assert launch.__name__ == "launch"
with pytest.raises(TypeError, match="unexpected keyword"):
launch(first, 1, 7, stale_constexpr=True)
def test_triton_kernel_decorator_without_triton(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setattr(jit_warmup_triton_helper, "HAS_TRITON", False)
monkeypatch.setattr(jit_warmup_triton_helper, "triton", object())
kernel = _FakeTritonKernel()
def warmup_inputs() -> dict[str, Any]:
return dict(first="warmup", second=1)
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
def launch(first: str, second: int) -> LaunchSpec:
return (2,), dict(CONST=7)
launch("runtime", 2)
assert kernel.runtime_calls == [
((2,), (), {"first": "runtime", "second": 2, "CONST": 7})
]
def test_triton_kernel_decorator_exhausts_large_ranges_before_deduplication(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeTritonKernel()
dispatched: list[int] = []
def warmup_inputs() -> dict[str, Any]:
tokens: Any = WarmupIntRange(1, 8193)
return dict(first="warmup", second=tokens)
def specialization(second: int) -> int:
return 7 if second <= 97 else 8
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
def dispatch(first: str, second: int) -> LaunchSpec:
dispatched.append(second)
return (second,), dict(CONST=specialization(second))
def fake_keys(kernel: Any, kwargs: Any) -> set[TritonJitKey]:
return {TritonJitKey(id(kernel), "fake", 0, kwargs["CONST"])}
_patch_key_deriver(monkeypatch, fake_keys)
keys = dispatch.get_warmup_keys()
assert dispatched == list(range(1, 8193))
assert len(keys) == 2
assert {specialization(dict(key.inputs)["second"]) for key in keys} == {7, 8}
def test_triton_kernel_decorator_propagates_dispatch_assertions(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeTritonKernel()
def warmup_inputs() -> dict[str, Any]:
return dict(first="warmup", second=WarmupChoices(1, 2))
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
def dispatch(first: str, second: int) -> LaunchSpec:
assert second != 2, "broken dispatch"
return (1,), dict(CONST=second)
_patch_key_deriver(
monkeypatch,
lambda kernel, kwargs: {TritonJitKey(id(kernel), "fake", 0, kwargs["CONST"])},
)
with pytest.raises(AssertionError, match="broken dispatch"):
dispatch.get_warmup_keys()
def test_triton_kernel_dispatch_uses_device_fake_tensors(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeTritonKernel()
device_type = current_platform.device_type
def warmup_inputs() -> dict[str, Any]:
return dict(
first=TritonWarmupTensor(torch.float32, shape=(2, 3)),
second=1,
)
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
def dispatch(first: torch.Tensor, second: int) -> LaunchSpec:
assert isinstance(first, torch.Tensor)
assert first.device.type == device_type
assert first[0].is_contiguous()
return (first.shape[0],), dict(CONST=first[0].numel())
_patch_key_deriver(
monkeypatch,
lambda kernel, kwargs: {TritonJitKey(id(kernel), "fake", 0, kwargs["CONST"])},
)
keys = dispatch.get_warmup_keys()
assert len(keys) == 1
assert dict(keys[0].inputs)["first"].device.type == device_type
def test_triton_kernel_decorates_native_launchers(
monkeypatch: pytest.MonkeyPatch,
) -> None:
kernel = _FakeTritonKernel()
def warmup_inputs(kernel: Any = kernel) -> dict[str, Any]:
return triton_warmup_inputs(
kernel,
"warmup",
WarmupChoices(1, 2),
grid=(2,),
CONST=7,
)
launcher = triton_kernel_dispatcher_with_warmup(warmup_inputs=warmup_inputs)(kernel)
launcher[(3,)]("runtime", 4, CONST=8)
assert kernel.runtime_calls == [((3,), ("runtime", 4), {"CONST": 8})]
_patch_key_deriver(
monkeypatch,
lambda kernel, kwargs: {
TritonJitKey(id(kernel), "fake", 0, (kwargs["second"], kwargs["CONST"]))
},
)
keys = launcher.get_warmup_keys()
assert len(keys) == 2
launcher.compile(keys[0])
assert kernel.warmup_calls == [
{"grid": (1,), "first": "warmup", "second": 1, "CONST": 7}
]
def test_triton_warmup_inputs_expands_explicit_pointer_dtypes() -> None:
inputs = triton_warmup_inputs(
_pointer_group_test_kernel,
grid=(1,),
pointer_dtypes={torch.float32: ("x_ptr",)},
value=2,
BLOCK=16,
)
assert inputs["x_ptr"] == TritonWarmupTensor(torch.float32)
assert inputs["value"] == 2
with pytest.raises(ValueError, match="not a pointer: value"):
triton_warmup_inputs(
_pointer_group_test_kernel,
grid=(1,),
pointer_dtypes={torch.float32: ("value",)},
x_ptr=TritonWarmupTensor(torch.float32),
BLOCK=16,
)
with pytest.raises(ValueError, match="Missing Triton pointer inputs: x_ptr"):
triton_warmup_inputs(
_pointer_group_test_kernel,
grid=(1,),
pointer_dtypes={},
value=2,
BLOCK=16,
)
def test_triton_key_derivation_applies_wrappers_and_runtime_options(
monkeypatch: pytest.MonkeyPatch,
) -> None:
knobs = triton.knobs
driver = triton.runtime.driver
device = "test-device"
monkeypatch.setattr(driver.active, "get_current_device", lambda: device)
calls: list[dict[str, Any]] = []
def binder(x: Any, value: int, BLOCK: int, **options: Any) -> Any:
calls.append(dict(BLOCK=BLOCK, **options))
specialization = [("pointer", x.dtype), ("i32", value), ("constexpr", BLOCK)]
return {}, specialization, options
cache: dict[Any, Any] = {}
_binder_test_kernel.device_caches[device] = ({}, cache, None, None, binder)
heuristic_kernel = triton.heuristics(
{"BLOCK": lambda args: 16 if args["value"] <= 16 else 32}
)(_binder_test_kernel)
keys = jit_warmup_triton_helper._triton_key_deriver(heuristic_kernel)(
{"x": TritonWarmupTensor(tl.float32), "value": 2}
)
assert len(keys) == 1
expected_options = {
"debug": knobs.runtime.debug,
"instrumentation_mode": knobs.compilation.instrumentation_mode,
}
if hasattr(knobs.compilation, "fpsan_homomorphic_casts"):
expected_options["fpsan_homomorphic_casts"] = (
knobs.compilation.fpsan_homomorphic_casts
)
assert calls == [{"BLOCK": 16, **expected_options}]
def test_triton_key_derivation_covers_autotune_configs_and_jit_identity(
monkeypatch: pytest.MonkeyPatch,
) -> None:
driver = triton.runtime.driver
device = "test-device"
monkeypatch.setattr(driver.active, "get_current_device", lambda: device)
def binder(x: Any, value: int, BLOCK: int, **options: Any) -> Any:
specialization = [("pointer", x.dtype), ("i32", value), ("constexpr", BLOCK)]
return {}, specialization, options
for kernel in (_binder_test_kernel, _second_binder_test_kernel):
kernel.device_caches[device] = ({}, {}, None, None, binder)
autotuned = triton.autotune(
configs=[
triton.Config({"BLOCK": 16}, num_warps=2),
triton.Config({"BLOCK": 32}, num_warps=4),
],
key=["value"],
)(_binder_test_kernel)
inputs = {"x": TritonWarmupTensor(tl.float32), "value": 2}
autotune_keys = jit_warmup_triton_helper._triton_key_deriver(autotuned)(inputs)
other_kernel_keys = jit_warmup_triton_helper._triton_key_deriver(
_second_binder_test_kernel
)(inputs | {"BLOCK": 16})
assert len(autotune_keys) == 2
assert {key.jit_function_key for key in autotune_keys} == {
_binder_test_kernel.cache_key
}
assert {key.jit_function_id for key in autotune_keys} == {id(_binder_test_kernel)}
assert autotune_keys.isdisjoint(other_kernel_keys)
def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
owner = ComputeSlotMappingKernel()
compile_key = owner.CompileKey(
kv_cache_block_size=16,
total_cp_world_size=2,
total_cp_rank=1,
cp_kv_cache_interleave_size=1,
block_table_stride=128,
)
launches: list[tuple[Any, ...]] = []
def launch(launch_spec: Any, inputs: Any) -> None:
launches.append((launch_spec, inputs))
monkeypatch.setattr(owner, "launch", launch)
owner.compile(compile_key)
(grid, kwargs), inputs = launches[0]
assert grid == (2,)
assert inputs["num_tokens"] == 2
assert inputs["block_table_stride"] == 128
assert kwargs == {
"KV_CACHE_BLOCK_SIZE": 16,
"TOTAL_CP_WORLD_SIZE": 2,
"TOTAL_CP_RANK": 1,
"CP_KV_CACHE_INTERLEAVE_SIZE": 1,
"PAD_ID": PAD_SLOT_ID,
"BLOCK_SIZE": owner.triton_block_size,
}
@@ -1,14 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.model_executor.warmup.mamba_triton_warmup import _warm_batch_memcpy_kernel
from vllm.platforms import current_platform
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA is required")
def test_mamba_batch_memcpy_kernel_compiles_on_gpu() -> None:
_warm_batch_memcpy_kernel(torch.device("cuda"))
torch.accelerator.synchronize("cuda")
@@ -1,110 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import multiprocessing
import types
import pytest
from vllm.platforms import current_platform
def _test_oink_availability_impl(
device_capability: tuple[int, int],
has_rmsnorm: bool,
has_fused_add_rms_norm: bool,
expected_available: bool,
expected_fused: bool,
) -> None:
"""Test OINK support detection with mocked state."""
import torch
from vllm import platforms
# Mock device capability (class method, override on class)
dc = platforms.interface.DeviceCapability(*device_capability)
platforms.current_platform.__class__.get_device_capability = lambda device_id=0: dc
# Mock oink ops
oink_ops = types.SimpleNamespace()
if has_rmsnorm:
oink_ops.rmsnorm = lambda x, w, eps: x
if has_fused_add_rms_norm:
oink_ops.fused_add_rms_norm = lambda x, residual, w, eps: None
torch.ops.oink = oink_ops
# Now import vllm modules with mocks in place (fresh import with mocked platform)
import vllm.kernels.oink_ops # noqa: F401
from vllm.ir.ops import fused_add_rms_norm, rms_norm
# Verify support checks
assert rms_norm.impls["oink"].supported is expected_available
assert fused_add_rms_norm.impls["oink"].supported is expected_fused
@pytest.mark.parametrize(
"device_capability,has_rmsnorm,has_fused_add_rms_norm,expected_available,expected_fused",
[
# Case 1: < SM100, ops not supported
((9, 0), True, False, False, False),
# Case 2: CUDA available and SM100, rmsnorm op registered
((10, 0), True, False, True, False),
# Case 3: SM100 with both rmsnorm and fused_add_rms_norm
((10, 0), True, True, True, True),
],
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test on CUDA")
def test_oink_availability_checks(
device_capability: tuple[int, int],
has_rmsnorm: bool,
has_fused_add_rms_norm: bool,
expected_available: bool,
expected_fused: bool,
):
"""Test OINK support detection with clean import state for each parameter set."""
# Use spawn to run function in fresh process with clean imports
# TODO migrate to spawn utility:
# https://github.com/vllm-project/vllm/issues/41415
ctx = multiprocessing.get_context("spawn")
process = ctx.Process(
target=_test_oink_availability_impl,
args=(
device_capability,
has_rmsnorm,
has_fused_add_rms_norm,
expected_available,
expected_fused,
),
)
process.start()
process.join()
if process.exitcode != 0:
raise AssertionError(
f"Subprocess test failed with exit code {process.exitcode}"
)
def test_can_view_as_2d_stride_guard():
# No global import
import torch
# Import the helper from the kernels module.
from vllm.kernels.oink_ops import _can_view_as_2d
x = torch.zeros((2, 3, 4))
assert _can_view_as_2d(x) is True
# Size-1 dims should be ignored by the viewability check.
# Create a tensor where stride(0) != stride(1) * size(1) due to padding,
# but view(-1, H) is still valid because dim 1 has size 1.
base = torch.zeros((2, 10, 4))
x_singleton = base[:, :1, :]
x_singleton.view(-1, x_singleton.shape[-1])
assert _can_view_as_2d(x_singleton) is True
# Middle-dimension stride break: view(-1, hidden) should be invalid.
x2 = x[:, ::2, :]
with pytest.raises(RuntimeError):
x2.view(-1, x2.shape[-1])
assert _can_view_as_2d(x2) is False
@@ -1,165 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock, patch
import numpy as np
import pytest
import torch
from vllm.config.mamba import MambaBackendEnum
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
from vllm.model_executor.warmup import replayssm_warmup as warmup
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda() or not has_flashinfer(),
reason="FlashInfer ReplaySSM warmup tests require CUDA and FlashInfer",
)
PREFILL_KWARGS = {
"num_tokens": 128,
"skip_eplb": True,
"is_profile": True,
"randomize_inputs": True,
}
def _autotune_runner(
*,
use_v2_model_runner: bool = False,
query_len: int = 6,
max_num_seqs: int = 32,
num_blocks: int = 17,
max_num_tokens: int = 100,
use_replayssm: bool = True,
backend: MambaBackendEnum = MambaBackendEnum.FLASHINFER,
) -> SimpleNamespace:
return SimpleNamespace(
vllm_config=SimpleNamespace(
cache_config=SimpleNamespace(use_replayssm=use_replayssm),
mamba_config=SimpleNamespace(backend=backend),
use_v2_model_runner=use_v2_model_runner,
),
uniform_decode_query_len=query_len,
decode_query_len=query_len,
max_num_tokens=max_num_tokens,
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
kv_cache_config=SimpleNamespace(num_blocks=num_blocks),
)
@pytest.mark.parametrize(
("runner_kwargs", "expected_num_reqs"),
[
# max_num_seqs (32) vs max_num_tokens // query_len (16) vs blocks-1 (16).
(dict(query_len=6, use_v2_model_runner=False), 16),
(dict(query_len=6, use_v2_model_runner=True), 16),
# num_blocks - 1 is the binding constraint.
(dict(query_len=1, max_num_tokens=128, max_num_seqs=64, num_blocks=5), 4),
],
ids=["v1", "v2", "clamped_to_state_capacity"],
)
def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
query_len = runner_kwargs["query_len"]
with patch.object(
warmup, "flashinfer_replayssm_autotune_supported", return_value=True
):
result = warmup._replayssm_autotune_kwargs(
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
)
assert result is not None
max_num_reqs, decode_kwargs = result
assert max_num_reqs == expected_num_reqs
expected_kwargs = {
**PREFILL_KWARGS,
"num_tokens": expected_num_reqs * query_len,
"uniform_decode": True,
}
if runner_kwargs.get("use_v2_model_runner"):
expected_kwargs["valid_dummy_state_slots"] = True
else:
expected_kwargs.update(
allow_microbatching=False,
force_attention=True,
profile_seq_lens=query_len + 1,
)
assert decode_kwargs == expected_kwargs
@pytest.mark.parametrize(
("runner_kwargs", "flashinfer_supported"),
[
(dict(use_replayssm=False), True),
(dict(backend=MambaBackendEnum.TRITON), True),
({}, False),
(dict(num_blocks=1), True),
],
ids=[
"replayssm_disabled",
"non_flashinfer_backend",
"kernel_unavailable",
"zero_non_padding_slots",
],
)
def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported):
with patch.object(
warmup,
"flashinfer_replayssm_autotune_supported",
return_value=flashinfer_supported,
):
result = warmup._replayssm_autotune_kwargs(
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
)
assert result is None
def test_replayssm_autotune_slots_restore_state_and_trackers():
mixer = MambaMixer2.__new__(MambaMixer2)
torch.nn.Module.__init__(mixer)
mixer.use_replayssm = True
mixer.replayssm_buffer_len = 16
mixer.kv_cache = (
torch.full((4, 2), 3.0),
torch.full((4, 2), 3.0),
*(torch.full((4, 2, 17), 3.0) for _ in range(3)),
)
mixer._replayssm_ring_start = torch.full((4,), 3, dtype=torch.int32)
mixer._replayssm_prev_num_accepted = torch.full((4,), 3, dtype=torch.int32)
mixer._replayssm_prev_query_len = torch.full((4,), 3, dtype=torch.int32)
tracked = (
*mixer.kv_cache,
mixer._replayssm_ring_start,
mixer._replayssm_prev_num_accepted,
mixer._replayssm_prev_query_len,
)
block_ids = np.arange(10, 14, dtype=np.int32).reshape(4, 1)
original_block_ids = block_ids.copy()
block_table = SimpleNamespace(block_table=SimpleNamespace(np=block_ids))
multi_group_block_table = SimpleNamespace(
block_tables=[block_table], commit_block_table=Mock()
)
runner = SimpleNamespace(
vllm_config=SimpleNamespace(use_v2_model_runner=False),
input_batch=SimpleNamespace(block_table=multi_group_block_table),
get_model=lambda: SimpleNamespace(modules=lambda: (mixer,)),
)
with warmup._temporary_replayssm_autotune_state(runner, 2):
assert block_ids[:2, 0].tolist() == [1, 2]
for tensor in tracked:
tensor[1:3].fill_(9)
assert np.array_equal(block_ids, original_block_ids)
assert multi_group_block_table.commit_block_table.call_args_list == [
((2,), {}),
((2,), {}),
]
for tensor in tracked:
assert torch.count_nonzero(tensor[1:3]) == 0
assert torch.all(tensor[0] == 3)
assert torch.all(tensor[3] == 3)
@@ -1,263 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the apply-time Triton kernel override registry.
The registry must let out-of-tree platforms replace kernels defined as
plain ``@triton.jit`` functions, patching every hold-site (defining
module, from-import copies, and JIT warmup owners) so that existing call
sites dispatch to the override without any core-side decoration.
"""
import sys
import types
from typing import Any
import pytest
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
_DecoratedTritonJitKernel,
)
from vllm.triton_utils import tl, triton
from vllm.triton_utils.dispatcher import (
KernelOverride,
register_kernels,
)
class _FakeTritonKernel:
"""Stands in for a Triton JITFunction (works without Triton installed)."""
arg_names = ("output_ptr", "input_ptr", "replace_from", "MAX_NUM_TOKENS")
def __getitem__(self, grid: Any) -> Any:
def launch(*args: Any, **kwargs: Any) -> Any:
return ("default", grid, kwargs)
return launch
@pytest.fixture
def _fake_module(monkeypatch: pytest.MonkeyPatch) -> Any:
"""Expose a fake defining module holding the kernel and its warmup owner."""
import types
kernel = _FakeTritonKernel()
owner = _DecoratedTritonJitKernel(
kernel,
lambda: {},
lambda output, input, replace_from: ( # noqa: ARG005
(1,),
dict(MAX_NUM_TOKENS=8),
),
)
module = types.ModuleType("vllm.tests.fake_kernel_home")
module.expand_kernel = kernel # type: ignore[attr-defined]
module._expand = owner # type: ignore[attr-defined]
monkeypatch.setitem(sys.modules, module.__name__, module)
return module
def test_override_patches_defining_module_and_warmup_owner(
_fake_module: Any,
) -> None:
calls: list[tuple[Any, ...]] = []
def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None):
calls.append((output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS))
return "cpu"
register_kernels(
{
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
}
)
overridden = _fake_module.expand_kernel
assert isinstance(overridden, KernelOverride)
# Launch with kernel-arg names like the warmup launch path does.
assert overridden[(4,)](output_ptr=1, input_ptr=2, replace_from=0) == "cpu"
assert calls == [(1, 2, 0, None)]
# The warmup owner's captured kernel is patched too, and its cached
# argument-name introspection is invalidated.
assert _fake_module._expand.kernel is overridden
def test_override_forwards_positionally_for_mismatched_names(
_fake_module: Any,
) -> None:
calls: list[tuple[Any, ...]] = []
def cpu_impl(output, input_val, replace_from, MAX_NUM_TOKENS=None):
calls.append((output, input_val, replace_from, MAX_NUM_TOKENS))
return "cpu"
register_kernels(
{
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
}
)
overridden = _fake_module.expand_kernel
assert overridden[(4,)](output_ptr=1, input_ptr=2, replace_from=0) == "cpu"
assert calls == [(1, 2, 0, None)]
def test_override_covers_from_import_copies(_fake_module: Any) -> None:
import types
importer = types.ModuleType("vllm.tests.fake_kernel_importer")
importer.expand_kernel = _fake_module.expand_kernel # type: ignore[attr-defined]
sys.modules["vllm.tests.fake_kernel_importer"] = importer
def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None):
return "cpu"
register_kernels(
{
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
}
)
assert isinstance(importer.expand_kernel, KernelOverride)
del sys.modules["vllm.tests.fake_kernel_importer"]
def test_override_owner_launch_goes_through_impl(_fake_module: Any) -> None:
launched: list[Any] = []
def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None):
launched.append((output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS))
return "cpu"
register_kernels(
{
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
}
)
owner = _fake_module._expand
# In-place kernels return None from launch; assert on the recorded call.
owner(1, 2, 0)
assert launched == [(1, 2, 0, 8)]
def test_register_kernel_auto_imports_defining_module(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Registering a kernel whose defining module is not imported yet must
import it on the spot and apply the override."""
kernel_name = "vllm.v1.sample.rejection_sampler.expand_kernel"
module_name = kernel_name.rpartition(".")[0]
calls: list[tuple[Any, ...]] = []
def cpu_impl(
output_ptr,
input_ptr,
cu_num_tokens_ptr,
replace_from,
replace_to,
MAX_NUM_TOKENS=None,
):
calls.append((output_ptr, replace_from, MAX_NUM_TOKENS))
return "cpu"
# Ensure the module is NOT imported before registration. Drop it from
# sys.modules; a fresh import creates new kernel/owner objects, which
# is what a cold-start platform process would see.
saved = sys.modules.pop(module_name, None)
# Re-run the registry patch against the original module afterwards so
# other tests that already imported it keep working.
try:
assert module_name not in sys.modules
register_kernels({kernel_name: cpu_impl})
assert module_name in sys.modules
finally:
if saved is None:
# First import in the whole session: keep the fresh module.
saved = sys.modules[module_name]
else:
# Restore the original module object and re-apply the override
# to its (pre-existing) kernel so both generations agree.
sys.modules[module_name] = saved
register_kernels({kernel_name: cpu_impl})
overridden = saved.expand_kernel
assert isinstance(overridden, KernelOverride)
overridden[(2,)](
output_ptr=1,
input_ptr=2,
cu_num_tokens_ptr=3,
replace_from=0,
replace_to=0,
MAX_NUM_TOKENS=8,
)
assert calls == [(1, 0, 8)]
# The warmup owner captured at the fresh import must be patched too
# (when the module existed before, there are two module generations).
fresh = sys.modules[module_name]
if fresh is not saved:
assert fresh._expand.kernel is overridden
def test_real_triton_kernel_override_end_to_end() -> None:
"""Override a real @triton.jit kernel through the registry."""
if not triton.__class__.__module__.startswith("triton"):
pytest.skip("Triton is not installed")
@triton.jit
def _add_one_kernel(x_ptr, out_ptr, BLOCK: tl.constexpr): # pragma: no cover
offsets = tl.arange(0, BLOCK)
tl.store(out_ptr + offsets, tl.load(x_ptr + offsets) + 1)
module = types.ModuleType("vllm.tests.real_jit_kernel_home")
module._add_one_kernel = _add_one_kernel # type: ignore[attr-defined]
sys.modules[module.__name__] = module
def cpu_impl(x_ptr, out_ptr, BLOCK=None): # noqa: ARG001
return "cpu"
register_kernels({"vllm.tests.real_jit_kernel_home._add_one_kernel": cpu_impl})
assert module._add_one_kernel.arg_names == ( # type: ignore[attr-defined]
"x_ptr",
"out_ptr",
"BLOCK",
)
def test_cpu_slot_mapping_override_forwards_kernel_arguments(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from vllm.utils import cpu_triton_utils
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
calls: list[tuple[Any, ...]] = []
fake_torch = types.SimpleNamespace(
ops=types.SimpleNamespace(
_C=types.SimpleNamespace(
compute_slot_mapping_kernel_impl=lambda *args: calls.append(args)
)
)
)
monkeypatch.setattr(cpu_triton_utils, "torch", fake_torch)
override = KernelOverride(
ComputeSlotMappingKernel.kernel,
cpu_triton_utils._compute_slot_mapping_kernel_impl,
)
override[(2,)](
num_tokens=2,
max_num_tokens=4,
query_start_loc_ptr="query_start_loc",
positions_ptr="positions",
block_table_ptr="block_table",
block_table_stride=128,
slot_mapping_ptr="slot_mapping",
KV_CACHE_BLOCK_SIZE=16,
TOTAL_CP_WORLD_SIZE=1,
TOTAL_CP_RANK=0,
CP_KV_CACHE_INTERLEAVE_SIZE=1,
PAD_ID=-1,
BLOCK_SIZE=1024,
)
assert calls == [
("query_start_loc", "positions", "block_table", "slot_mapping", 16)
]
@@ -1,300 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The warmup must cover every specialization the sampler will launch.
``_philox_gumbel_kernel`` is keyed on the Philox key halves, on the logits
dtype (model dtype when the sampler passes logits through, fp32 when logits
processing copies them) and on whether a skip mask is passed, so a missing
combination leaves a request paying the compile this module exists to remove.
"""
import logging
from types import SimpleNamespace
import pytest
import torch
from vllm.config.watermarking import WatermarkConfig, derive_watermark_key
from vllm.model_executor.warmup.watermark_sample_warmup import watermark_sample_warmup
from vllm.platforms import current_platform
from vllm.v1.watermarking.factory import create_watermarker
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
from vllm.v1.watermarking.spec_decode import create_speculative_target_watermarker
MASTER_KEY = 20250910
KEY_A = derive_watermark_key(MASTER_KEY, b"key_a")
KEY_B = derive_watermark_key(MASTER_KEY, b"key_b")
@pytest.fixture(autouse=True)
def cuda_alike_platform(monkeypatch: pytest.MonkeyPatch) -> None:
"""The warmup returns early off CUDA, so pin the platform it reads.
Everything here runs on the fake worker's cpu device; without this the
tests below would pass only on a CUDA-alike host and silently warm nothing
anywhere else.
"""
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: True)
def _sampler(
watermark_config: WatermarkConfig,
speculative: bool,
use_fp64_gumbel: bool,
) -> GPUWatermarkSampler:
"""The sampler ``GPUModelRunner`` builds for this config.
Mirrors ``vllm/v1/worker/gpu/model_runner.py:473-486``. ``__new__`` skips
``Sampler.__init__``, which allocates UVA-backed device buffers; the warmup
only reads the three attributes set here.
"""
watermarker = create_watermarker(watermark_config)
if speculative:
watermarker = create_speculative_target_watermarker(watermarker)
sampler = GPUWatermarkSampler.__new__(GPUWatermarkSampler)
sampler.watermarker = watermarker
sampler.deduplicate_contexts = watermark_config.deduplicate_contexts
sampler.use_fp64_gumbel = use_fp64_gumbel
return sampler
def _worker(
watermark_config: WatermarkConfig | None,
speculative: bool = False,
use_fp64_gumbel: bool = False,
with_sampler: bool = True,
) -> SimpleNamespace:
sampler = (
_sampler(watermark_config, speculative, use_fp64_gumbel)
if watermark_config is not None and with_sampler
else None
)
return SimpleNamespace(
device=torch.device("cpu"),
model_runner=SimpleNamespace(sampler=sampler),
vllm_config=SimpleNamespace(
model_config=SimpleNamespace(
get_vocab_size=lambda: 128,
dtype=torch.bfloat16,
),
watermark_config=watermark_config,
),
)
def _capture_calls(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
calls: list[dict] = []
def fake_philox_gumbel_sample(logits, contexts, key, **kwargs):
calls.append({"logits": logits, "contexts": contexts, "key": key, **kwargs})
return torch.zeros(logits.shape[0], dtype=torch.int64)
monkeypatch.setattr(
"vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample",
fake_philox_gumbel_sample,
)
return calls
def _variants(calls: list[dict]) -> set[tuple[int, torch.dtype, bool]]:
return {
(call["key"], call["logits"].dtype, call.get("skip_mask") is not None)
for call in calls
}
def test_no_warmup_without_a_watermark_config(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(_worker(None))
assert calls == []
def test_no_warmup_without_a_watermarked_sampler(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A rank that does not sample launches nothing to warm.
Only the final pipeline stage of a generative model holds a
``GPUWatermarkSampler``; on any other rank the kernel is never launched.
"""
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(
_worker(WatermarkConfig(key=MASTER_KEY), with_sampler=False)
)
assert calls == []
def test_no_warmup_off_cuda(monkeypatch: pytest.MonkeyPatch) -> None:
"""``GumbelWatermarker`` launches the kernel only on a CUDA-alike device.
Elsewhere -- an XPU worker inherits ``kernel_warmup`` from the CUDA worker
-- the sampler takes the torch path, so warming would compile a kernel
nothing launches.
"""
calls = _capture_calls(monkeypatch)
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: False)
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
assert calls == []
def test_single_key_warms_both_logits_dtypes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
assert _variants(calls) == {
(MASTER_KEY, torch.bfloat16, True),
(MASTER_KEY, torch.float32, True),
}
assert len(calls) == len(_variants(calls))
def test_dual_key_warms_both_derived_keys(monkeypatch: pytest.MonkeyPatch) -> None:
"""``DualKeyGumbelWatermarker.sample`` launches the kernel once per key.
The halves of a derived key exceed 2**31, so each key is typed i64/i32 by
magnitude and a wrong key compiles a specialization the engine never uses.
"""
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(
_worker(WatermarkConfig(key=MASTER_KEY, algorithm="dual_key_gumbel"))
)
assert _variants(calls) == {
(KEY_A, torch.bfloat16, True),
(KEY_A, torch.float32, True),
(KEY_B, torch.bfloat16, True),
(KEY_B, torch.float32, True),
}
@pytest.mark.parametrize("alpha,key", [(0.0, KEY_A), (1.0, KEY_B)])
def test_dual_key_alpha_bounds_warm_one_key(
monkeypatch: pytest.MonkeyPatch, alpha: float, key: int
) -> None:
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(
_worker(
WatermarkConfig(key=MASTER_KEY, algorithm="dual_key_gumbel", alpha=alpha)
)
)
assert _variants(calls) == {
(key, torch.bfloat16, True),
(key, torch.float32, True),
}
def test_speculative_config_warms_the_target_role_only(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Under spec decode the sampler holds ``create_target_watermarker()``.
Rows without draft tokens sample through it, so the sampler path launches
key B alone; the draft key runs in the speculator's own sampler.
"""
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(
_worker(
WatermarkConfig(key=MASTER_KEY, algorithm="dual_key_gumbel"),
speculative=True,
)
)
assert _variants(calls) == {
(KEY_B, torch.bfloat16, True),
(KEY_B, torch.float32, True),
}
def test_dedup_none_also_warms_the_mask_free_specialization(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Without deduplication an all-watermarked batch passes no skip mask.
A later batch mixing in an opted-out request passes one, so both
specializations are reachable and both must be warmed.
"""
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(
_worker(WatermarkConfig(key=MASTER_KEY, deduplicate_contexts="none"))
)
assert _variants(calls) == {
(MASTER_KEY, torch.bfloat16, True),
(MASTER_KEY, torch.float32, True),
(MASTER_KEY, torch.bfloat16, False),
(MASTER_KEY, torch.float32, False),
}
for call in calls:
if call.get("skip_mask") is None:
assert "temperatures" not in call
def test_warmup_argument_dtypes_match_the_runtime_buffers(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Contexts are int32, the sampling state int64 indices and fp32 temperatures.
``GPUWatermarkSampler._get_contexts`` gathers from the int32 request-state
token ids, ``expanded_idx_mapping``/``positions``/``seeds`` are int64 and
``SamplingStates.temperature`` is fp32; warming other dtypes compiles
specializations no engine launches.
"""
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY, context_width=3)))
assert calls
for call in calls:
num_tokens = call["logits"].shape[0]
assert call["contexts"].dtype == torch.int32
assert call["contexts"].shape == (num_tokens, 3)
assert call["skip_mask"].dtype == torch.bool
assert call["expanded_idx_mapping"].dtype == torch.int64
assert call["positions"].dtype == torch.int64
assert call["seeds"].dtype == torch.int64
assert call["temperatures"].dtype == torch.float32
def test_use_fp64_follows_the_path_that_forwards_it(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Only the mixed path forwards the sampler's ``use_fp64_gumbel``.
``GumbelWatermarker._sample_watermarked`` launches the mask-free variant
with the ``use_fp64=False`` default, so warming it with fp64 would compile
a specialization the engine never launches and miss the one it does.
"""
calls = _capture_calls(monkeypatch)
watermark_sample_warmup(
_worker(
WatermarkConfig(key=MASTER_KEY, deduplicate_contexts="none"),
use_fp64_gumbel=True,
)
)
assert calls
for call in calls:
assert call["use_fp64"] is (call.get("skip_mask") is not None)
def test_warmup_failure_does_not_abort_startup(
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
) -> None:
def failing_philox_gumbel_sample(*args, **kwargs):
raise RuntimeError("no triton")
monkeypatch.setattr(
"vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample",
failing_philox_gumbel_sample,
)
with caplog.at_level(logging.WARNING):
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
assert "Skipping watermark sampler warmup." in caplog.text
-470
View File
@@ -1,470 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tempfile
from unittest.mock import Mock
import huggingface_hub.constants
import pytest
import torch
from huggingface_hub.utils import LocalEntryNotFoundError
import vllm.model_executor.model_loader.weight_utils as weight_utils
from vllm.model_executor.model_loader.weight_utils import (
download_weights_from_hf,
drop_checkpoint_cache,
initialize_single_dummy_weight,
load_attention_sink,
maybe_remap_kv_scale_name,
)
from vllm.platforms import current_platform
@pytest.mark.parametrize(
"checkpoint_heads,runtime_heads,tp_size",
[(8, 4, 2), (8, 8, 1), (8, 6, 2), (16, 12, 4)],
)
def test_attention_sink_loader_shapes(
monkeypatch, checkpoint_heads, runtime_heads, tp_size
):
"""Each rank loads its checkpoint heads and disables only padded heads."""
monkeypatch.setattr(
weight_utils, "get_tensor_model_parallel_world_size", lambda: tp_size
)
checkpoint = torch.arange(checkpoint_heads, dtype=torch.float32)
for tp_rank in range(tp_size):
monkeypatch.setattr(
weight_utils, "get_tensor_model_parallel_rank", lambda rank=tp_rank: rank
)
param = torch.nn.Parameter(torch.zeros(runtime_heads), requires_grad=False)
load_attention_sink(param, checkpoint)
local_heads = checkpoint.chunk(tp_size)[tp_rank]
expected = torch.cat(
(
local_heads,
torch.full((runtime_heads - local_heads.numel(),), -torch.inf),
)
)
assert torch.equal(param, expected)
@pytest.mark.parametrize(
"checkpoint_shape,runtime_shape,error",
[
((2, 4), (4,), RuntimeError),
((8,), (2, 2), AssertionError),
((7,), (4,), ValueError),
((8,), (3,), AssertionError),
],
)
def test_attention_sink_loader_rejects_invalid_shapes(
monkeypatch, checkpoint_shape, runtime_shape, error
):
"""Invalid sink shapes fail before changing the runtime parameter."""
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 0)
param = torch.nn.Parameter(torch.ones(runtime_shape), requires_grad=False)
with pytest.raises(error):
load_attention_sink(param, torch.zeros(checkpoint_shape))
assert torch.equal(param, torch.ones(runtime_shape))
@pytest.mark.parametrize("param_dtype", [torch.bfloat16, torch.float32])
def test_attention_sink_loader_keeps_param_dtype(monkeypatch, param_dtype):
"""A checkpoint dtype different from the runtime parameter is cast, not copied."""
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 1)
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 0)
param = torch.nn.Parameter(torch.zeros(4, dtype=param_dtype), requires_grad=False)
load_attention_sink(param, torch.arange(4, dtype=torch.float32))
assert param.dtype == param_dtype
assert torch.equal(param.float(), torch.arange(4, dtype=torch.float32))
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires a GPU")
def test_attention_sink_loader_accepts_cpu_checkpoint(monkeypatch):
"""Sinks are normally loaded from CPU checkpoints into device parameters."""
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 2)
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 1)
param = torch.nn.Parameter(
torch.full((6,), 123.0, device="cuda"), requires_grad=False
)
# The test suite sets the default device, so pin the checkpoint to CPU.
checkpoint = torch.arange(8, dtype=torch.float32, device="cpu")
load_attention_sink(param, checkpoint)
assert param.device.type == "cuda"
expected = torch.arange(4, 8, dtype=torch.float32, device="cpu")
assert torch.equal(param[:4].cpu(), expected)
assert torch.isinf(param[4:]).all() and (param[4:] < 0).all()
@pytest.mark.parametrize(
"num_heads,dtype", [(4, torch.float32), (8, torch.bfloat16), (6, None)]
)
def test_make_attention_sink_registers_loader_and_padding(num_heads, dtype):
"""The factory is the single place that creates and registers a sink."""
sink = weight_utils.make_attention_sink(num_heads, dtype)
assert sink.shape == (num_heads,)
assert sink.dtype == (dtype or torch.get_default_dtype())
assert sink.requires_grad is False
assert torch.isinf(sink).all() and (sink < 0).all()
assert sink.weight_loader is load_attention_sink
def test_download_weights_from_hf():
with tempfile.TemporaryDirectory() as tmpdir:
# assert LocalEntryNotFoundError error is thrown
# if offline is set and model is not cached
huggingface_hub.constants.HF_HUB_OFFLINE = True
with pytest.raises(LocalEntryNotFoundError):
download_weights_from_hf(
"facebook/opt-125m",
allow_patterns=["*.safetensors", "*.bin"],
cache_dir=tmpdir,
)
# download the model
huggingface_hub.constants.HF_HUB_OFFLINE = False
download_weights_from_hf(
"facebook/opt-125m",
allow_patterns=["*.safetensors", "*.bin"],
cache_dir=tmpdir,
)
# now it should work offline
huggingface_hub.constants.HF_HUB_OFFLINE = True
assert (
download_weights_from_hf(
"facebook/opt-125m",
allow_patterns=["*.safetensors", "*.bin"],
cache_dir=tmpdir,
)
is not None
)
class TestMaybeRemapKvScaleName:
"""Tests for maybe_remap_kv_scale_name covering all checkpoint formats."""
PARAMS_DICT = {
"model.layers.0.self_attn.attn.k_scale": None,
"model.layers.0.self_attn.attn.v_scale": None,
"model.layers.0.self_attn.attn.q_scale": None,
"model.layers.0.self_attn.qkv_proj.weight": None,
}
def test_qkv_proj_k_scale(self):
"""Qwen3-MoE / llm-compressor format: qkv_proj.k_scale -> attn.k_scale
Regression test for https://github.com/vllm-project/vllm/issues/25047"""
result = maybe_remap_kv_scale_name(
"model.layers.0.self_attn.qkv_proj.k_scale", self.PARAMS_DICT
)
assert result == "model.layers.0.self_attn.attn.k_scale"
def test_qkv_proj_v_scale(self):
"""Qwen3-MoE / llm-compressor format: qkv_proj.v_scale -> attn.v_scale
Regression test for https://github.com/vllm-project/vllm/issues/25047"""
result = maybe_remap_kv_scale_name(
"model.layers.0.self_attn.qkv_proj.v_scale", self.PARAMS_DICT
)
assert result == "model.layers.0.self_attn.attn.v_scale"
def test_modelopt_k_proj_k_scale(self):
"""ModelOpt format: k_proj.k_scale -> attn.k_scale."""
result = maybe_remap_kv_scale_name(
"model.layers.0.self_attn.k_proj.k_scale", self.PARAMS_DICT
)
assert result == "model.layers.0.self_attn.attn.k_scale"
def test_modelopt_v_proj_v_scale(self):
"""ModelOpt format: v_proj.v_scale -> attn.v_scale."""
result = maybe_remap_kv_scale_name(
"model.layers.0.self_attn.v_proj.v_scale", self.PARAMS_DICT
)
assert result == "model.layers.0.self_attn.attn.v_scale"
def test_deprecated_kv_scale(self):
"""Old format: kv_scale -> attn.k_scale (deprecated)."""
result = maybe_remap_kv_scale_name(
"model.layers.0.self_attn.kv_scale", self.PARAMS_DICT
)
assert result == "model.layers.0.self_attn.attn.k_scale"
def test_default_bare_k_scale(self):
"""Default format: .k_scale -> .attn.k_scale."""
result = maybe_remap_kv_scale_name(
"model.layers.0.self_attn.k_scale", self.PARAMS_DICT
)
assert result == "model.layers.0.self_attn.attn.k_scale"
def test_non_scale_name_unchanged(self):
"""Non-scale names should be returned unchanged."""
name = "model.layers.0.self_attn.qkv_proj.weight"
result = maybe_remap_kv_scale_name(name, self.PARAMS_DICT)
assert result == name
def test_nvfp4_modelopt_k_proj_k_scale(self):
"""ModelOpt NVFP4 format (e.g. nvidia/Qwen3-30B-A3B-NVFP4):
k_proj.k_scale -> attn.k_scale.
Validates that NVFP4 checkpoints are not broken by this change."""
result = maybe_remap_kv_scale_name(
"model.layers.0.self_attn.k_proj.k_scale", self.PARAMS_DICT
)
assert result == "model.layers.0.self_attn.attn.k_scale"
def test_nvfp4_modelopt_v_proj_v_scale(self):
"""ModelOpt NVFP4 format (e.g. nvidia/Qwen3-30B-A3B-NVFP4):
v_proj.v_scale -> attn.v_scale.
Validates that NVFP4 checkpoints are not broken by this change."""
result = maybe_remap_kv_scale_name(
"model.layers.0.self_attn.v_proj.v_scale", self.PARAMS_DICT
)
assert result == "model.layers.0.self_attn.attn.v_scale"
def test_qwen3_vl_moe_qkv_proj_k_scale(self):
"""Qwen3-VL-MoE uses the same fused qkv_proj naming as Qwen3-MoE.
Regression test for qwen3_vl_moe.py fix (same bug as #25047)."""
result = maybe_remap_kv_scale_name(
"model.layers.0.self_attn.qkv_proj.k_scale", self.PARAMS_DICT
)
assert result == "model.layers.0.self_attn.attn.k_scale"
def test_qwen3_vl_moe_qkv_proj_v_scale(self):
"""Qwen3-VL-MoE uses the same fused qkv_proj naming as Qwen3-MoE.
Regression test for qwen3_vl_moe.py fix (same bug as #25047)."""
result = maybe_remap_kv_scale_name(
"model.layers.0.self_attn.qkv_proj.v_scale", self.PARAMS_DICT
)
assert result == "model.layers.0.self_attn.attn.v_scale"
def test_nvfp4_weight_scale_not_remapped(self):
"""NVFP4 weight_scale should not be touched by remap (not a kv scale)."""
name = "model.layers.0.self_attn.k_proj.weight_scale"
result = maybe_remap_kv_scale_name(name, self.PARAMS_DICT)
assert result == name
def test_nvfp4_input_scale_not_remapped(self):
"""NVFP4 input_scale should not be touched by remap (not a kv scale)."""
name = "model.layers.0.self_attn.k_proj.input_scale"
result = maybe_remap_kv_scale_name(name, self.PARAMS_DICT)
assert result == name
def test_missing_target_returns_none(self):
"""If remapped name not in params_dict, return None."""
empty_params: dict[str, None] = {}
result = maybe_remap_kv_scale_name(
"model.layers.0.self_attn.qkv_proj.k_scale", empty_params
)
assert result is None
def test_checkpoint_weight_mapper_discards_serialized_g_idx():
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
)
mapper = QuantizationConfig.get_checkpoint_weight_mapper()
assert mapper._map_name("model.layers.0.g_idx") is None
assert mapper._map_name("model.layers.0.qweight") == "model.layers.0.qweight"
class TestKvCacheScaleMapper:
"""The `WeightsMapper` returned by `get_cache_scale_mapper` replaces the
per-model `maybe_remap_kv_scale_name` calls. It must remap the same set of
checkpoint formats (the non-`params_dict`-dependent ones) and be idempotent
so it composes safely with a model's own qkv/gate_up `hf_to_vllm_mapper`."""
def _mapper(self):
# `get_cache_scale_mapper` does not use `self`; call it on the base
# class to get the default (non-config-specific) mapper.
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
)
return QuantizationConfig.get_cache_scale_mapper()
def _map(self, name: str) -> str | None:
return self._mapper()._map_name(name)
@pytest.mark.parametrize(
"name,expected",
[
# Qwen3-MoE / llm-compressor fused qkv_proj
(
"model.layers.0.self_attn.qkv_proj.k_scale",
"model.layers.0.self_attn.attn.k_scale",
),
(
"model.layers.0.self_attn.qkv_proj.v_scale",
"model.layers.0.self_attn.attn.v_scale",
),
# ModelOpt / NVFP4 k_proj/v_proj
(
"model.layers.0.self_attn.k_proj.k_scale",
"model.layers.0.self_attn.attn.k_scale",
),
(
"model.layers.0.self_attn.v_proj.v_scale",
"model.layers.0.self_attn.attn.v_scale",
),
# deprecated fused kv_scale and bare scales
(
"model.layers.0.self_attn.kv_scale",
"model.layers.0.self_attn.attn.k_scale",
),
(
"model.layers.0.self_attn.k_scale",
"model.layers.0.self_attn.attn.k_scale",
),
# NemotronH mixer
(
"model.layers.0.mixer.k_proj.k_scale",
"model.layers.0.mixer.attn.k_scale",
),
# already in vLLM form -> unchanged (idempotent)
(
"model.layers.0.self_attn.attn.k_scale",
"model.layers.0.self_attn.attn.k_scale",
),
# non-kv scales must not be touched
(
"model.layers.0.self_attn.k_proj.weight_scale",
"model.layers.0.self_attn.k_proj.weight_scale",
),
(
"model.layers.0.self_attn.k_proj.input_scale",
"model.layers.0.self_attn.k_proj.input_scale",
),
# regular weights untouched
(
"model.layers.0.self_attn.q_proj.weight",
"model.layers.0.self_attn.q_proj.weight",
),
],
)
def test_remap(self, name, expected):
assert self._map(name) == expected
@pytest.mark.parametrize(
"name",
[
"model.layers.0.self_attn.k_scale",
"model.layers.0.self_attn.k_proj.k_scale",
"model.layers.0.self_attn.qkv_proj.v_scale",
"model.layers.0.mixer.k_proj.k_scale",
],
)
def test_idempotent(self, name):
once = self._map(name)
assert once is not None
assert self._map(once) == once
def test_composes_with_qkv_mapper(self):
"""Applied together with a model's qkv/gate_up mapper, the regex scale
rules run before the substr rename, so scales are normalized to `.attn.`
and regular projections are still fused correctly."""
from vllm.model_executor.models.utils import WeightsMapper
model_mapper = WeightsMapper(
orig_to_new_substr={
".q_proj": ".qkv_proj.q",
".k_proj": ".qkv_proj.k",
".v_proj": ".qkv_proj.v",
}
)
# AutoWeightsLoader does `mapper |= cache_scale_mapper`
combined = model_mapper | self._mapper()
assert (
combined._map_name("model.layers.0.self_attn.q_proj.weight")
== "model.layers.0.self_attn.qkv_proj.q.weight"
)
assert (
combined._map_name("model.layers.0.self_attn.k_proj.k_scale")
== "model.layers.0.self_attn.attn.k_scale"
)
assert (
combined._map_name("model.layers.0.self_attn.k_scale")
== "model.layers.0.self_attn.attn.k_scale"
)
@pytest.mark.parametrize("fails", [False, True])
def test_drop_checkpoint_cache_releases_local_weight_files(
tmp_path, monkeypatch, fails
):
"""Evict only weights; cache eviction errors must not prevent loading."""
for name in ("weights.safetensors", "config.json"):
(tmp_path / name).touch()
fadvise = Mock(side_effect=OSError("cache eviction failed") if fails else None)
monkeypatch.setattr("os.posix_fadvise", fadvise)
drop_checkpoint_cache(str(tmp_path))
fadvise.assert_called_once()
@pytest.mark.parametrize("dtype", [torch.float8_e4m3fn, torch.float8_e5m2])
def test_dummy_fp8_weight_is_not_degenerate(dtype):
"""+-1e-3 underflows in FP8, so 8-bit floats default to +-1."""
param = torch.empty(4096, dtype=dtype)
initialize_single_dummy_weight(param)
values = param.float()
assert values.isfinite().all()
assert values.abs().max() <= 1.0
assert (values != 0).float().mean() > 0.99
@pytest.mark.parametrize(
("dtype", "name"),
[(torch.float8_e8m0fnu, "weight_scale_inv"), (torch.uint8, "w13_weight_scale")],
)
def test_dummy_e8m0_scale_is_fixed_exponent(dtype, name):
"""E8M0 scales, including uint8-stored ones, get a finite 2**-7."""
param = torch.empty(64, dtype=dtype)
initialize_single_dummy_weight(param, name=name)
assert (param.view(torch.uint8) == 120).all()
@pytest.mark.parametrize("dtype", [torch.uint8, torch.int8, torch.float4_e2m1fn_x2])
def test_dummy_packed_weight_gets_random_bytes(dtype):
"""Packed weights are filled deterministically instead of left uninitialized."""
# Some backends can't fill packed FP4 directly, so view zeroed bytes.
params = [torch.zeros(4096, dtype=torch.uint8).view(dtype) for _ in range(2)]
for param in params:
initialize_single_dummy_weight(param, name="w13_weight")
raw = params[0].view(torch.uint8)
assert raw.unique().numel() > 250
assert torch.equal(raw, params[1].view(torch.uint8))
@pytest.mark.skipif(current_platform.is_rocm(), reason="ROCm zeroes integer params")
def test_dummy_wide_integer_param_is_untouched():
"""Wider integer params may be indices, so random values are unsafe."""
param = torch.arange(16, dtype=torch.int32)
initialize_single_dummy_weight(param, name="g_idx")
assert torch.equal(param, torch.arange(16, dtype=torch.int32))
def test_dummy_norm_weights_are_neutral(default_vllm_config):
"""Dummy norms stay identity-like, keeping activations at a realistic scale."""
from vllm.model_executor.layers.layernorm import (
GemmaRMSNorm,
LayerNorm,
RMSNorm,
RMSNormGated,
)
expected = {
RMSNorm(16): {"weight": 1.0},
GemmaRMSNorm(16): {"weight": 0.0},
LayerNorm(16): {"weight": 1.0, "bias": 0.0},
RMSNormGated(16): {"weight": 1.0},
}
for norm, values in expected.items():
for name, param in norm.named_parameters():
param.data.uniform_(-5, 5)
initialize_single_dummy_weight(param, name=name)
assert torch.all(param == values[name]), (type(norm).__name__, name)
if __name__ == "__main__":
test_download_weights_from_hf()