[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,38 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
import pytest
import torch
import torch.nn as nn
from vllm.model_executor.layers.activation import GeluAndMul, SiluAndMul
from vllm.platforms import CpuArchEnum, current_platform
@pytest.mark.skipif(
not current_platform.is_cpu()
or current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC,
reason="Native CPU activations require a non-PowerPC CPU platform",
)
@pytest.mark.parametrize(
"activation_factory,reference",
[
(SiluAndMul, nn.SiLU()),
(GeluAndMul, nn.GELU()),
(partial(GeluAndMul, approximate="tanh"), nn.GELU(approximate="tanh")),
],
ids=["silu", "gelu", "gelu_tanh"],
)
def test_cpu_act_and_mul_without_compiled_ops(
default_vllm_config, monkeypatch, activation_factory, reference
):
"""CPU activations must construct and run without compiled wheel operators."""
default_vllm_config.compilation_config.custom_ops = ["all"]
monkeypatch.setattr(torch.ops, "_C", object())
x = torch.tensor([[-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device="cpu")
layer = activation_factory()
torch.testing.assert_close(layer(x), reference(x[:, :4]) * x[:, 4:])
@@ -1,339 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for FusedMMInputNorm and the fused affine transform kernel."""
import pytest
import torch
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
from vllm.config.compilation import CompilationMode
from vllm.model_executor.layers.fusion.mm_input_norm import (
FusedMMInputNorm,
IdentityInputNorm,
NormParams,
fused_mm_input_norm_triton,
)
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from vllm.utils.torch_utils import set_random_seed
# Module-level tests run on any platform: the CPU platform exercises
# forward_native, accelerators exercise the fused kernel. Only the raw
# kernel tests require an accelerator.
_DEVICE = torch.device(current_platform.device_type)
requires_accelerator = pytest.mark.skipif(
current_platform.is_cpu(),
reason="fused Triton kernel requires a CUDA/XPU accelerator",
)
requires_triton = pytest.mark.skipif(not HAS_TRITON, reason="requires Triton")
requires_vllm_config = pytest.mark.usefixtures("default_vllm_config")
def _reference_input_norm(
pixel_values: torch.Tensor,
image_mean: list[float],
image_std: list[float],
rescale_factor: float,
channel: int,
out_dtype: torch.dtype,
) -> torch.Tensor:
"""Straightforward per-channel affine: (x * rescale - mean) / std."""
patches, size = pixel_values.shape
patch_size = size // channel
mean = torch.tensor(
image_mean, dtype=torch.float32, device=pixel_values.device
).view(1, channel, 1)
std = torch.tensor(image_std, dtype=torch.float32, device=pixel_values.device).view(
1, channel, 1
)
x = pixel_values.to(torch.float32).reshape(patches, channel, patch_size)
x = (x * rescale_factor - mean) / std
return x.view(patches, size).to(out_dtype)
# CLIP-style RGB normalization constants reused across tests.
_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073]
_RGB_STD = [0.26862954, 0.26130258, 0.27577711]
_RGB_RESCALE = 1.0 / 255.0
def _check_against_reference(
pixel_values: torch.Tensor,
out_dtype: torch.dtype,
channel: int = 3,
image_mean: list[float] | None = None,
image_std: list[float] | None = None,
rescale_factor: float = _RGB_RESCALE,
**close_kwargs,
):
"""Run ``FusedMMInputNorm`` on ``pixel_values`` and compare against the
eager per-channel affine reference. Uses the RGB constants by default."""
if image_mean is None:
image_mean = list(_RGB_MEAN)
if image_std is None:
image_std = list(_RGB_STD)
norm = FusedMMInputNorm(
image_mean=image_mean,
image_std=image_std,
rescale_factor=rescale_factor,
channel=channel,
).to(_DEVICE)
expected = _reference_input_norm(
pixel_values, image_mean, image_std, rescale_factor, channel, out_dtype
)
torch.testing.assert_close(
norm(pixel_values, visual_dtype=out_dtype), expected, **close_kwargs
)
# ===========================================================================
# Module-level behavior
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormModule:
def test_matches_reference_above_cudnn_grid_limit(self):
"""num_patches above the old cuDNN batch-norm grid limit (~65535)
used to raise CUDNN_STATUS_INTERNAL_ERROR."""
set_random_seed(0)
pixel_values = torch.randint(
0, 256, (70000, 3 * 196), dtype=torch.float32, device=_DEVICE
)
_check_against_reference(pixel_values, torch.float32)
def test_identity_passthrough(self):
norm = IdentityInputNorm()
pixel_values = torch.randn(8, 3 * 196, dtype=torch.float32, device=_DEVICE)
out = norm(pixel_values, visual_dtype=torch.bfloat16)
torch.testing.assert_close(out, pixel_values.to(torch.bfloat16))
# ===========================================================================
# dtype coverage
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormDtypes:
@pytest.mark.parametrize(
"in_dtype,out_dtype",
[
(torch.float32, torch.float32),
(torch.float32, torch.bfloat16),
(torch.float32, torch.float16),
(torch.bfloat16, torch.bfloat16),
(torch.bfloat16, torch.float32),
(torch.float16, torch.float16),
(torch.uint8, torch.bfloat16),
(torch.uint8, torch.float32),
],
)
def test_dtype_combinations(self, in_dtype: torch.dtype, out_dtype: torch.dtype):
set_random_seed(0)
if in_dtype == torch.uint8:
pixel_values = torch.randint(
0, 256, (32, 3 * 16), dtype=torch.uint8, device=_DEVICE
)
else:
pixel_values = torch.rand(
(32, 3 * 16), dtype=torch.float32, device=_DEVICE
).to(in_dtype)
_check_against_reference(
pixel_values,
out_dtype,
image_mean=[0.5, 0.5, 0.5],
image_std=[0.5, 0.5, 0.5],
atol=1e-2,
rtol=1e-2,
)
# ===========================================================================
# Shape / channel coverage
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormShapes:
@pytest.mark.parametrize("channel", [1, 3, 4])
def test_channel_variants(self, channel: int):
set_random_seed(0)
pixel_values = torch.randint(
0, 256, (16, channel * 64), dtype=torch.float32, device=_DEVICE
)
_check_against_reference(
pixel_values,
torch.float32,
channel=channel,
image_mean=[0.5] * channel,
image_std=[0.25] * channel,
)
@pytest.mark.parametrize("patch_size", [1, 2048])
def test_block_boundaries(self, patch_size: int):
"""Masked-tail (12 elements) and exact-multiple (24 blocks) paths."""
set_random_seed(0)
pixel_values = torch.randint(
0, 256, (4, 3 * patch_size), dtype=torch.float32, device=_DEVICE
)
_check_against_reference(pixel_values, torch.float32)
# ===========================================================================
# Input handling: non-contiguous inputs
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormInputHandling:
def test_non_contiguous_input_matches_reference(self):
set_random_seed(0)
base = torch.randint(
0, 256, (8, 3 * 32, 2), dtype=torch.float32, device=_DEVICE
)
non_contig = base[..., 0]
assert not non_contig.is_contiguous()
_check_against_reference(non_contig, torch.float32)
# ===========================================================================
# Raw kernel entry point
# ===========================================================================
@requires_accelerator
@requires_triton
class TestFusedMMInputNormKernel:
@pytest.mark.parametrize(
"N, C, L",
[
(1, 1, 1),
(1, 3, 1),
(3, 3, 7), # tile straddles channel boundaries
(2, 8, 63),
(4, 3, 1000),
],
)
def test_channel_boundary_crossing(self, N: int, C: int, L: int):
"""The 1D kernel recovers ``c = (offs // L) % C`` per lane, so a tile
can straddle a channel boundary and gather weight/bias per lane.
These shapes force that path by keeping ``L`` well below the block."""
set_random_seed(0)
x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE)
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
out = torch.empty_like(x)
fused_mm_input_norm_triton(x, out, w, b)
expected = x * w.view(1, C, 1) + b.view(1, C, 1)
torch.testing.assert_close(out, expected)
def test_non_contiguous_inputs_materialized(self):
"""Read-only tensors are made contiguous internally."""
N, C, L = 3, 3, 100
set_random_seed(0)
base = torch.randn(N, C, L, 2, dtype=torch.float32, device=_DEVICE)
x = base[..., 0]
assert not x.is_contiguous()
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
out = torch.empty(N, C, L, dtype=torch.float32, device=_DEVICE)
fused_mm_input_norm_triton(x, out, w, b)
expected = x * w.view(1, C, 1) + b.view(1, C, 1)
torch.testing.assert_close(out, expected)
def test_rejects_mismatched_output_shape(self):
"""``outputs`` must be shaped exactly like ``inputs``."""
N, C, L = 3, 3, 100
x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE)
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
for out_shape in ((N, C + 1, L), (N, C, L + 1), (N + 2, C, L)):
with pytest.raises(AssertionError):
fused_mm_input_norm_triton(
x,
torch.empty(*out_shape, dtype=torch.float32, device=_DEVICE),
w,
b,
)
# ===========================================================================
# Construction / configuration
# ===========================================================================
@requires_vllm_config
class TestFusedMMInputNormConstruction:
"""Weight/bias buffer semantics at init time."""
def test_input_norm_is_always_enabled(self):
config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
compile_mm_encoder=True,
custom_ops=["none"],
)
)
with set_current_vllm_config(config):
FusedMMInputNorm(_RGB_MEAN, _RGB_STD, _RGB_RESCALE)
assert (
config.compilation_config.enabled_custom_ops["fused_mm_input_norm"] == 1
)
assert not config.compilation_config.disabled_custom_ops[
"fused_mm_input_norm"
]
def test_identity_config_buffers(self):
"""Numerically identity parameters still build plain weight/bias
buffers; the identity shortcut lives in ``from_model_config``."""
norm = FusedMMInputNorm(
image_mean=[0.0, 0.0, 0.0],
image_std=[1.0, 1.0, 1.0],
rescale_factor=1.0,
)
torch.testing.assert_close(norm.weight, torch.ones(3))
torch.testing.assert_close(norm.bias, torch.zeros(3))
def test_non_identity_buffers(self):
channel = 3
norm = FusedMMInputNorm(
image_mean=_RGB_MEAN,
image_std=_RGB_STD,
rescale_factor=_RGB_RESCALE,
channel=channel,
)
assert norm.weight.shape == (channel,)
assert norm.bias.shape == (channel,)
mean = torch.tensor(_RGB_MEAN, dtype=torch.float32)
std = torch.tensor(_RGB_STD, dtype=torch.float32)
torch.testing.assert_close(norm.weight, _RGB_RESCALE / std)
torch.testing.assert_close(norm.bias, -mean / std)
def test_is_identity_params(self):
"""The numeric-identity check mirrors the old allclose detection."""
assert NormParams([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0).is_identity
assert not NormParams(_RGB_MEAN, _RGB_STD, _RGB_RESCALE).is_identity
@pytest.mark.parametrize(
("image_mean", "image_std", "rescale_factor"),
[
([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0),
([0.5, 0.5, 0.5], [0.25, 0.25, 0.25], 1 / 255),
],
)
def test_buffers_initialized_on_default_device(
self,
image_mean: list[float],
image_std: list[float],
rescale_factor: float,
):
"""Buffers are built on CPU first, then moved to the caller's
default device, so init never touches the accelerator."""
# The meta device gives the CPU-only test shard the same non-CPU
# default-device semantics without requiring a CUDA build.
default_device = "cuda" if torch.cuda.is_available() else "meta"
with torch.device(default_device):
norm = FusedMMInputNorm(image_mean, image_std, rescale_factor)
assert norm.weight.device.type == default_device
assert norm.bias.device.type == default_device
@@ -1,234 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for vllm.model_executor.layers.pooler.activations."""
from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
from vllm.model_executor.layers.pooler.activations import (
LambdaPoolerActivation,
PoolerClassify,
PoolerIdentity,
PoolerMultiLabelClassify,
PoolerNormalize,
get_act_fn,
resolve_classifier_act_fn,
)
# ---------------------------------------------------------------------------
# PoolerIdentity
# ---------------------------------------------------------------------------
class TestPoolerIdentity:
def test_returns_input_unchanged(self):
pooler = PoolerIdentity()
x = torch.randn(4, 128)
out = pooler(x)
assert torch.equal(out, x)
def test_forward_list(self):
pooler = PoolerIdentity()
tensors = [torch.randn(128), torch.randn(256)]
out = pooler(tensors)
assert len(out) == 2
for orig, result in zip(tensors, out):
assert torch.equal(orig, result)
# ---------------------------------------------------------------------------
# PoolerNormalize
# ---------------------------------------------------------------------------
class TestPoolerNormalize:
def test_output_has_unit_norm(self):
pooler = PoolerNormalize()
x = torch.randn(4, 128)
out = pooler(x)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(4), atol=1e-5)
def test_single_vector(self):
pooler = PoolerNormalize()
x = torch.randn(1, 64)
out = pooler(x)
norm = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norm, torch.ones(1), atol=1e-5)
def test_forward_list(self):
pooler = PoolerNormalize()
tensors = [torch.randn(1, 64), torch.randn(1, 128)]
out = pooler(tensors)
for t in out:
norm = torch.linalg.norm(t, dim=-1)
assert torch.allclose(norm, torch.ones(1), atol=1e-5)
# ---------------------------------------------------------------------------
# PoolerMultiLabelClassify
# ---------------------------------------------------------------------------
class TestPoolerMultiLabelClassify:
def test_output_in_zero_one(self):
pooler = PoolerMultiLabelClassify()
x = torch.randn(4, 10)
out = pooler(x)
assert (out >= 0).all() and (out <= 1).all()
def test_large_positive_maps_near_one(self):
pooler = PoolerMultiLabelClassify()
x = torch.full((1, 3), 100.0)
out = pooler(x)
assert torch.allclose(out, torch.ones(1, 3), atol=1e-4)
def test_large_negative_maps_near_zero(self):
pooler = PoolerMultiLabelClassify()
x = torch.full((1, 3), -100.0)
out = pooler(x)
assert torch.allclose(out, torch.zeros(1, 3), atol=1e-4)
# ---------------------------------------------------------------------------
# PoolerClassify
# ---------------------------------------------------------------------------
class TestPoolerClassify:
def test_infers_from_shape_when_num_labels_none(self):
pooler = PoolerClassify(num_labels=None)
assert pooler.num_labels is None
x = torch.randn(2, 5)
out = pooler(x)
sums = out.sum(dim=-1)
assert torch.allclose(sums, torch.ones(2), atol=1e-5)
def test_sigmoid_when_num_labels_lt_2(self):
pooler = PoolerClassify(num_labels=1)
x = torch.zeros(1, 1)
out = pooler(x)
assert torch.allclose(out, torch.tensor([[0.5]]), atol=1e-5)
def test_num_labels_zero_uses_sigmoid(self):
pooler = PoolerClassify(num_labels=0)
assert pooler.num_labels == 0
x = torch.zeros(1, 3)
out = pooler(x)
assert torch.allclose(out, torch.full((1, 3), 0.5), atol=1e-5)
def test_num_labels_ge_2_uses_softmax(self):
pooler = PoolerClassify(num_labels=4)
assert pooler.num_labels == 4
x = torch.randn(2, 4)
out = pooler(x)
sums = out.sum(dim=-1)
assert torch.allclose(sums, torch.ones(2), atol=1e-5)
def test_default_num_labels_is_none(self):
pooler = PoolerClassify()
assert pooler.num_labels is None
# ---------------------------------------------------------------------------
# LambdaPoolerActivation
# ---------------------------------------------------------------------------
class TestLambdaPoolerActivation:
def test_applies_custom_fn(self):
pooler = LambdaPoolerActivation(nn.ReLU())
x = torch.tensor([[-1.0, 2.0, -3.0]])
out = pooler(x)
expected = torch.tensor([[0.0, 2.0, 0.0]])
assert torch.equal(out, expected)
def test_forward_list(self):
pooler = LambdaPoolerActivation(nn.ReLU())
tensors = [torch.tensor([-1.0, 2.0]), torch.tensor([3.0, -4.0])]
out = pooler(tensors)
assert torch.equal(out[0], torch.tensor([0.0, 2.0]))
assert torch.equal(out[1], torch.tensor([3.0, 0.0]))
# ---------------------------------------------------------------------------
# get_act_fn factory
# ---------------------------------------------------------------------------
class TestGetActFn:
@staticmethod
def _make_config(**kwargs):
return SimpleNamespace(**kwargs)
def test_regression(self):
cfg = self._make_config(problem_type="regression")
result = get_act_fn(cfg)
assert isinstance(result, PoolerIdentity)
def test_single_label_classification(self):
cfg = self._make_config(
problem_type="single_label_classification", num_labels=3
)
result = get_act_fn(cfg)
assert isinstance(result, PoolerClassify)
assert result.num_labels == 3
def test_multi_label_classification(self):
cfg = self._make_config(problem_type="multi_label_classification")
result = get_act_fn(cfg)
assert isinstance(result, PoolerMultiLabelClassify)
def test_sentence_transformers_activation(self):
cfg = self._make_config(
problem_type="",
sentence_transformers={
"activation_fn": "torch.nn.modules.activation.Sigmoid"
},
)
result = get_act_fn(cfg)
assert isinstance(result, PoolerClassify)
def test_sbert_activation(self):
cfg = self._make_config(
problem_type="",
sbert_ce_default_activation_function=(
"torch.nn.modules.activation.Sigmoid"
),
)
result = get_act_fn(cfg)
assert isinstance(result, PoolerClassify)
def test_default_fallback(self):
cfg = self._make_config(problem_type="")
result = get_act_fn(cfg)
assert isinstance(result, PoolerClassify)
def test_sentence_transformers_takes_priority(self):
cfg = self._make_config(
problem_type="",
sentence_transformers={"activation_fn": "torch.nn.modules.linear.Identity"},
sbert_ce_default_activation_function=(
"torch.nn.modules.activation.Sigmoid"
),
)
result = get_act_fn(cfg)
assert isinstance(result, PoolerIdentity)
def test_rejects_non_torch_activation(self):
cfg = self._make_config(
problem_type="",
sentence_transformers={"activation_fn": "os.system"},
)
with pytest.raises(ValueError, match="restricted"):
get_act_fn(cfg)
# ---------------------------------------------------------------------------
# resolve_classifier_act_fn
# ---------------------------------------------------------------------------
class TestResolveClassifierActFn:
def test_delegates_to_get_act_fn_when_none(self):
model_config = SimpleNamespace(
hf_config=SimpleNamespace(num_labels=3, problem_type="")
)
result = resolve_classifier_act_fn(model_config, act_fn=None)
assert isinstance(result, PoolerClassify)
assert result.num_labels == 3
def test_passes_through_provided_act_fn(self):
custom = PoolerIdentity()
result = resolve_classifier_act_fn(None, act_fn=custom)
assert result is custom
@@ -1,490 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for sequence and token pooler head classes."""
import torch
import torch.nn as nn
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
from vllm.model_executor.layers.pooler.seqwise.heads import (
ClassifierPoolerHead,
EmbeddingPoolerHead,
)
from vllm.model_executor.layers.pooler.tokwise.heads import (
TokenClassifierPoolerHead,
TokenEmbeddingPoolerHead,
)
from vllm.pooling_params import PoolingParams
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
_HIDDEN = 16
_BATCH = 3
def _make_params(
n: int,
*,
task: PoolingTask = "embed",
dimensions: int | None = None,
use_activation: bool | None = None,
) -> list[PoolingParams]:
return [
PoolingParams(task=task, dimensions=dimensions, use_activation=use_activation)
for _ in range(n)
]
def _make_metadata(pooling_params: list[PoolingParams]) -> PoolingMetadata:
n = len(pooling_params)
return PoolingMetadata(
prompt_lens=torch.ones(n, dtype=torch.long),
prompt_token_ids=None,
prompt_token_ids_cpu=None,
pooling_params=pooling_params,
pooling_states=[PoolingStates() for _ in range(n)],
)
def _linear(in_f: int, out_f: int) -> nn.Linear:
torch.manual_seed(42)
return nn.Linear(in_f, out_f, bias=False)
# ---------------------------------------------------------------------------
# EmbeddingPoolerHead
# ---------------------------------------------------------------------------
class TestEmbeddingPoolerHead:
def test_supported_tasks(self):
head = EmbeddingPoolerHead()
assert head.get_supported_tasks() == {"embed"}
def test_passthrough(self):
head = EmbeddingPoolerHead()
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH))
out = head(x, meta)
assert torch.equal(out, x)
def test_head_dtype(self):
head = EmbeddingPoolerHead(head_dtype=torch.float16)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH))
out = head(x, meta)
assert out.dtype == torch.float16
def test_projector(self):
proj = _linear(_HIDDEN, 8)
head = EmbeddingPoolerHead(projector=proj)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH))
out = head(x, meta)
assert out.shape == (_BATCH, 8)
assert torch.allclose(out, proj(x))
def test_matryoshka_uniform(self):
head = EmbeddingPoolerHead()
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, dimensions=4)
meta = _make_metadata(params)
out = head(x, meta)
assert out.shape == (_BATCH, 4)
assert torch.equal(out, x[..., :4])
def test_matryoshka_mixed(self):
head = EmbeddingPoolerHead()
x = torch.randn(2, _HIDDEN)
params = [
PoolingParams(task="embed", dimensions=4),
PoolingParams(task="embed", dimensions=8),
]
meta = _make_metadata(params)
out = head(x, meta)
assert isinstance(out, list)
assert len(out) == 2
assert out[0].shape[-1] == 4
assert out[1].shape[-1] == 8
def test_matryoshka_mixed_with_none(self):
head = EmbeddingPoolerHead()
x = torch.randn(2, _HIDDEN)
params = [
PoolingParams(task="embed", dimensions=4),
PoolingParams(task="embed", dimensions=None),
]
meta = _make_metadata(params)
out = head(x, meta)
assert isinstance(out, list)
assert out[0].shape[-1] == 4
assert torch.equal(out[1], x[1])
def test_activation_uniform_true(self):
head = EmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, use_activation=True)
meta = _make_metadata(params)
out = head(x, meta)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
def test_activation_uniform_false(self):
head = EmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, use_activation=False)
meta = _make_metadata(params)
out = head(x, meta)
assert torch.equal(out, x)
def test_activation_mixed_flags(self):
head = EmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(2, _HIDDEN)
params = [
PoolingParams(task="embed", use_activation=True),
PoolingParams(task="embed", use_activation=False),
]
meta = _make_metadata(params)
out = head(x, meta)
assert isinstance(out, list)
norm_0 = torch.linalg.norm(out[0], dim=-1)
assert torch.allclose(norm_0, torch.ones(1), atol=1e-5)
assert torch.equal(out[1], x[1])
def test_list_input_gets_stacked(self):
head = EmbeddingPoolerHead()
tensors = [torch.randn(_HIDDEN) for _ in range(_BATCH)]
meta = _make_metadata(_make_params(_BATCH))
out = head(tensors, meta)
assert out.shape == (_BATCH, _HIDDEN)
expected = torch.stack(tensors)
assert torch.equal(out, expected)
def test_projector_then_matryoshka(self):
proj = _linear(_HIDDEN, 8)
head = EmbeddingPoolerHead(projector=proj)
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, dimensions=4)
meta = _make_metadata(params)
out = head(x, meta)
assert out.shape == (_BATCH, 4)
assert torch.equal(out, proj(x)[..., :4])
def test_matryoshka_then_activation(self):
head = EmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, dimensions=4, use_activation=True)
meta = _make_metadata(params)
out = head(x, meta)
assert out.shape == (_BATCH, 4)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
def test_empty_batch(self):
head = EmbeddingPoolerHead()
x = torch.randn(0, _HIDDEN)
meta = _make_metadata([])
out = head(x, meta)
assert out.shape == (0, _HIDDEN)
# ---------------------------------------------------------------------------
# ClassifierPoolerHead
# ---------------------------------------------------------------------------
class TestClassifierPoolerHead:
def test_supported_tasks(self):
head = ClassifierPoolerHead()
assert head.get_supported_tasks() == {"classify"}
def test_passthrough(self):
head = ClassifierPoolerHead()
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
assert torch.equal(out, x)
def test_head_dtype(self):
head = ClassifierPoolerHead(head_dtype=torch.float16)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
assert out.dtype == torch.float16
def test_classifier(self):
clf = _linear(_HIDDEN, 3)
head = ClassifierPoolerHead(classifier=clf)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
assert out.shape == (_BATCH, 3)
assert torch.allclose(out, clf(x))
def test_logit_mean(self):
head = ClassifierPoolerHead(logit_mean=2.0)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
assert torch.allclose(out, x - 2.0)
def test_logit_sigma(self):
head = ClassifierPoolerHead(logit_sigma=0.5)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
assert torch.allclose(out, x / 0.5)
def test_platt_scaling_combined(self):
head = ClassifierPoolerHead(logit_mean=1.0, logit_sigma=2.0)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
assert torch.allclose(out, (x - 1.0) / 2.0)
def test_activation_uniform_true(self):
head = ClassifierPoolerHead(activation=PoolerNormalize())
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, task="classify", use_activation=True)
meta = _make_metadata(params)
out = head(x, meta)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
def test_activation_uniform_false(self):
head = ClassifierPoolerHead(activation=PoolerNormalize())
x = torch.randn(_BATCH, _HIDDEN)
params = _make_params(_BATCH, task="classify", use_activation=False)
meta = _make_metadata(params)
out = head(x, meta)
assert torch.equal(out, x)
def test_activation_mixed_flags(self):
head = ClassifierPoolerHead(activation=PoolerNormalize())
x = torch.randn(2, _HIDDEN)
params = [
PoolingParams(task="classify", use_activation=True),
PoolingParams(task="classify", use_activation=False),
]
meta = _make_metadata(params)
out = head(x, meta)
assert isinstance(out, list)
norm_0 = torch.linalg.norm(out[0], dim=-1)
assert torch.allclose(norm_0, torch.ones(1), atol=1e-5)
assert torch.equal(out[1], x[1])
def test_list_input_gets_stacked(self):
head = ClassifierPoolerHead()
tensors = [torch.randn(_HIDDEN) for _ in range(_BATCH)]
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(tensors, meta)
assert out.shape == (_BATCH, _HIDDEN)
expected = torch.stack(tensors)
assert torch.equal(out, expected)
def test_classifier_then_platt_scaling(self):
clf = _linear(_HIDDEN, 3)
head = ClassifierPoolerHead(classifier=clf, logit_mean=1.0, logit_sigma=2.0)
x = torch.randn(_BATCH, _HIDDEN)
meta = _make_metadata(_make_params(_BATCH, task="classify"))
out = head(x, meta)
expected = (clf(x) - 1.0) / 2.0
assert torch.allclose(out, expected)
def test_empty_batch(self):
head = ClassifierPoolerHead()
x = torch.randn(0, _HIDDEN)
meta = _make_metadata([])
out = head(x, meta)
assert out.shape == (0, _HIDDEN)
# ---------------------------------------------------------------------------
# TokenEmbeddingPoolerHead
# ---------------------------------------------------------------------------
class TestTokenEmbeddingPoolerHead:
def test_supported_tasks(self):
head = TokenEmbeddingPoolerHead()
assert head.get_supported_tasks() == {"token_embed"}
def test_passthrough(self):
head = TokenEmbeddingPoolerHead()
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed")
out = head.forward_chunk(x, param)
assert out is x
assert torch.equal(out, x)
def test_none_chunked_prefill(self):
head = TokenEmbeddingPoolerHead()
param = PoolingParams(task="token_embed")
out = head.forward_chunk(None, param)
assert out is None
def test_head_dtype(self):
head = TokenEmbeddingPoolerHead(head_dtype=torch.float16)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed")
out = head.forward_chunk(x, param)
assert out is not None
assert out.dtype == torch.float16
def test_projector(self):
proj = _linear(_HIDDEN, 8)
head = TokenEmbeddingPoolerHead(projector=proj)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed")
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 8)
assert torch.allclose(out, proj(x))
def test_matryoshka_truncation(self):
head = TokenEmbeddingPoolerHead()
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", dimensions=4)
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 4)
assert torch.equal(out, x[..., :4])
def test_activation_true(self):
head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", use_activation=True)
out = head.forward_chunk(x, param)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
def test_activation_false(self):
head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", use_activation=False)
out = head.forward_chunk(x, param)
assert torch.equal(out, x)
def test_projector_then_matryoshka(self):
proj = _linear(_HIDDEN, 8)
head = TokenEmbeddingPoolerHead(projector=proj)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", dimensions=4)
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 4)
assert torch.equal(out, proj(x)[..., :4])
def test_matryoshka_then_activation(self):
head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_embed", dimensions=4, use_activation=True)
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 4)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
def test_forward_mixed_batch_chunked_prefill(self):
head = TokenEmbeddingPoolerHead()
pooled_data = [torch.randn(5, _HIDDEN), None, torch.randn(3, _HIDDEN)]
params = _make_params(3, task="token_embed")
meta = _make_metadata(params)
out = head(pooled_data, meta)
assert len(out) == 3
assert torch.equal(out[0], pooled_data[0])
assert out[1] is None
assert torch.equal(out[2], pooled_data[2])
def test_forward_empty_batch(self):
head = TokenEmbeddingPoolerHead()
meta = _make_metadata([])
out = head([], meta)
assert out == []
# ---------------------------------------------------------------------------
# TokenClassifierPoolerHead
# ---------------------------------------------------------------------------
class TestTokenClassifierPoolerHead:
def test_supported_tasks(self):
head = TokenClassifierPoolerHead()
assert head.get_supported_tasks() == {"token_classify"}
def test_passthrough(self):
head = TokenClassifierPoolerHead()
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert torch.equal(out, x)
def test_none_chunked_prefill(self):
head = TokenClassifierPoolerHead()
param = PoolingParams(task="token_classify")
out = head.forward_chunk(None, param)
assert out is None
def test_head_dtype(self):
head = TokenClassifierPoolerHead(head_dtype=torch.float16)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert out is not None
assert out.dtype == torch.float16
def test_classifier(self):
clf = _linear(_HIDDEN, 3)
head = TokenClassifierPoolerHead(classifier=clf)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert out is not None
assert out.shape == (5, 3)
assert torch.allclose(out, clf(x))
def test_logit_mean(self):
head = TokenClassifierPoolerHead(logit_mean=2.0)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert torch.allclose(out, x - 2.0)
def test_logit_sigma(self):
head = TokenClassifierPoolerHead(logit_sigma=0.5)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert torch.allclose(out, x / 0.5)
def test_platt_scaling_combined(self):
head = TokenClassifierPoolerHead(logit_mean=1.0, logit_sigma=2.0)
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify")
out = head.forward_chunk(x, param)
assert torch.allclose(out, (x - 1.0) / 2.0)
def test_activation_true(self):
head = TokenClassifierPoolerHead(activation=PoolerNormalize())
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify", use_activation=True)
out = head.forward_chunk(x, param)
norms = torch.linalg.norm(out, dim=-1)
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
def test_activation_false(self):
head = TokenClassifierPoolerHead(activation=PoolerNormalize())
x = torch.randn(5, _HIDDEN)
param = PoolingParams(task="token_classify", use_activation=False)
out = head.forward_chunk(x, param)
assert torch.equal(out, x)
def test_forward_mixed_batch_chunked_prefill(self):
head = TokenClassifierPoolerHead()
pooled_data = [torch.randn(5, _HIDDEN), None, torch.randn(3, _HIDDEN)]
params = _make_params(3, task="token_classify")
meta = _make_metadata(params)
out = head(pooled_data, meta)
assert len(out) == 3
assert torch.equal(out[0], pooled_data[0])
assert out[1] is None
assert torch.equal(out[2], pooled_data[2])
def test_forward_empty_batch(self):
head = TokenClassifierPoolerHead()
meta = _make_metadata([])
out = head([], meta)
assert out == []
@@ -1,747 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for sequence and token pooling methods and their factories."""
from dataclasses import dataclass
from types import SimpleNamespace
from unittest.mock import patch
import numpy as np
import pytest
import torch
from transformers import PreTrainedConfig
from vllm.config import PoolerConfig, set_current_vllm_config
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
from vllm.model_executor.layers.pooler.seqwise.methods import (
CLSPool,
LastPool,
MeanPool,
get_seq_pooling_method,
)
from vllm.model_executor.layers.pooler.special import BOSEOSFilter, DispatchPooler
from vllm.model_executor.layers.pooler.tokwise.heads import TokenEmbeddingPoolerHead
from vllm.model_executor.layers.pooler.tokwise.methods import (
AllPool,
StepPool,
get_tok_pooling_method,
)
from vllm.model_executor.layers.pooler.tokwise.poolers import TokenPooler
from vllm.pooling_params import LateChunkingParams, PoolingParams
from vllm.tasks import PoolingTask
from vllm.v1.pool.metadata import PoolingCursor, PoolingMetadata, PoolingStates
_CPU = torch.device("cpu")
def _make_pooling_cursor(
prompt_lens: list[int],
*,
num_scheduled_tokens: list[int] | None = None,
seq_lens: list[int] | None = None,
device: torch.device = _CPU,
) -> PoolingCursor:
"""Build a PoolingCursor from a list of per-sequence prompt lengths."""
prompt_lens_cpu = torch.tensor(prompt_lens, dtype=torch.long)
if num_scheduled_tokens is None:
num_scheduled_tokens_cpu = prompt_lens_cpu.clone()
else:
num_scheduled_tokens_cpu = torch.tensor(num_scheduled_tokens, dtype=torch.long)
if seq_lens is None:
seq_lens_cpu = prompt_lens_cpu.clone()
else:
seq_lens_cpu = torch.tensor(seq_lens, dtype=torch.long)
cumsum = torch.zeros(len(prompt_lens) + 1, dtype=torch.long, device=device)
torch.cumsum(num_scheduled_tokens_cpu, dim=0, out=cumsum[1:])
return PoolingCursor(
first_token_indices_gpu=cumsum[: len(prompt_lens)].to(device),
last_token_indices_gpu=(cumsum[1:] - 1).to(device),
prompt_lens_cpu=prompt_lens_cpu,
seq_lens_cpu=seq_lens_cpu,
num_scheduled_tokens_cpu=num_scheduled_tokens_cpu,
partial_prefill=not torch.equal(prompt_lens_cpu, num_scheduled_tokens_cpu),
finished_mask=torch.eq(prompt_lens_cpu, seq_lens_cpu).tolist(),
)
def _make_metadata(
prompt_lens: list[int],
*,
tasks: list[PoolingTask] | None = None,
token_ids: list[list[int]] | None = None,
pooling_params: list[PoolingParams] | None = None,
num_scheduled_tokens: list[int] | None = None,
seq_lens: list[int] | None = None,
device: torch.device = _CPU,
) -> PoolingMetadata:
"""Build a minimal PoolingMetadata for testing pooling methods."""
n_seqs = len(prompt_lens)
if tasks is None:
tasks = ["embed"] * n_seqs
if pooling_params is None:
pooling_params = [PoolingParams(task=t) for t in tasks]
prompt_lens_tensor = torch.tensor(prompt_lens, dtype=torch.long)
prompt_token_ids_cpu = None
prompt_token_ids = None
if token_ids is not None:
max_len = max(len(t) for t in token_ids)
padded = [t + [0] * (max_len - len(t)) for t in token_ids]
prompt_token_ids_cpu = torch.tensor(padded, dtype=torch.long)
prompt_token_ids = prompt_token_ids_cpu.to(device)
cursor = _make_pooling_cursor(
prompt_lens,
num_scheduled_tokens=num_scheduled_tokens,
seq_lens=seq_lens,
device=device,
)
pooling_states = [PoolingStates() for _ in range(n_seqs)]
return PoolingMetadata(
prompt_lens=prompt_lens_tensor,
prompt_token_ids=prompt_token_ids,
prompt_token_ids_cpu=prompt_token_ids_cpu,
pooling_params=pooling_params,
pooling_states=pooling_states,
pooling_cursor=cursor,
)
def test_build_pooling_cursor_caches_cpu_status() -> None:
metadata = PoolingMetadata(
prompt_lens=torch.tensor([3, 4, 2], dtype=torch.long),
prompt_token_ids=None,
prompt_token_ids_cpu=None,
pooling_params=[PoolingParams(task="embed") for _ in range(3)],
pooling_states=[PoolingStates() for _ in range(3)],
)
metadata.build_pooling_cursor(
np.array([3, 2, 2], dtype=np.int32),
seq_lens_cpu=torch.tensor([3, 2, 1], dtype=torch.long),
device=_CPU,
)
cursor = metadata.get_pooling_cursor()
assert cursor.is_partial_prefill() is True
assert cursor.get_finished_mask() == [True, False, False]
assert cursor[:1].is_partial_prefill() is False
assert cursor[1:2].is_partial_prefill() is True
assert cursor[1:].get_finished_mask() == [False, False]
def test_pooling_params_update_combines_token_id_requirements() -> None:
update = PoolingParamsUpdate() | PoolingParamsUpdate(requires_token_ids=True)
params = PoolingParams(task="embed")
update.apply(params)
assert params.requires_token_ids is True
# ---------------------------------------------------------------------------
# CLSPool
# ---------------------------------------------------------------------------
class TestCLSPool:
def test_extracts_first_token(self):
hidden = torch.tensor(
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
)
metadata = _make_metadata([2, 3])
pooler = CLSPool()
out = pooler(hidden, metadata)
expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
assert torch.equal(out, expected)
def test_rejects_partial_prefill(self):
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
metadata = _make_metadata([3], num_scheduled_tokens=[2])
pooler = CLSPool()
with pytest.raises(RuntimeError, match="partial prefill"):
pooler(hidden, metadata)
# ---------------------------------------------------------------------------
# LastPool
# ---------------------------------------------------------------------------
class TestLastPool:
def test_extracts_last_token(self):
hidden = torch.tensor(
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
)
metadata = _make_metadata([2, 3])
pooler = LastPool()
out = pooler(hidden, metadata)
expected = torch.tensor([[3.0, 4.0], [9.0, 10.0]])
assert torch.equal(out, expected)
def test_partial_prefill_extracts_last_scheduled(self):
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
metadata = _make_metadata([4], num_scheduled_tokens=[2])
pooler = LastPool()
out = pooler(hidden, metadata)
expected = torch.tensor([[3.0, 4.0]])
assert torch.equal(out, expected)
# ---------------------------------------------------------------------------
# MeanPool
# ---------------------------------------------------------------------------
class TestMeanPool:
def test_computes_mean(self):
hidden = torch.tensor(
[[1.0, 2.0], [3.0, 4.0], [10.0, 20.0]], dtype=torch.float32
)
metadata = _make_metadata([2, 1])
pooler = MeanPool()
out = pooler(hidden, metadata)
expected = torch.tensor([[2.0, 3.0], [10.0, 20.0]], dtype=torch.float32)
assert torch.allclose(out, expected, atol=1e-5)
def test_single_token_is_identity(self):
hidden = torch.tensor([[5.0, 10.0]], dtype=torch.float32)
metadata = _make_metadata([1])
pooler = MeanPool()
out = pooler(hidden, metadata)
assert torch.allclose(out, hidden, atol=1e-5)
def test_uniform_values_return_same(self):
hidden = torch.full((4, 3), 7.0, dtype=torch.float32)
metadata = _make_metadata([4])
pooler = MeanPool()
out = pooler(hidden, metadata)
expected = torch.full((1, 3), 7.0, dtype=torch.float32)
assert torch.allclose(out, expected, atol=1e-5)
def test_multiple_sequences(self):
hidden = torch.tensor(
[
[0.0, 0.0],
[2.0, 4.0],
[4.0, 8.0],
[10.0, 10.0],
],
dtype=torch.float32,
)
metadata = _make_metadata([3, 1])
pooler = MeanPool()
out = pooler(hidden, metadata)
expected = torch.tensor([[2.0, 4.0], [10.0, 10.0]], dtype=torch.float32)
assert torch.allclose(out, expected, atol=1e-5)
def test_empty_batch(self):
hidden = torch.empty((0, 8), dtype=torch.float32)
metadata = _make_metadata([])
pooler = MeanPool()
out = pooler(hidden, metadata)
assert out.shape == (0, 8)
def test_rejects_partial_prefill(self):
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float32)
metadata = _make_metadata([3], num_scheduled_tokens=[2])
pooler = MeanPool()
with pytest.raises(RuntimeError, match="partial prefill"):
pooler(hidden, metadata)
def test_chunked_accumulation(self):
hidden = torch.arange(20, dtype=torch.float32).reshape(5, 4)
metadata = _make_metadata([3, 2])
pooler = MeanPool()
with patch(
"vllm.model_executor.layers.pooler.seqwise.methods"
"._MEAN_POOL_ACCUMULATION_CHUNK_BYTES",
16,
):
out = pooler(hidden, metadata)
expected_seq0 = hidden[:3].float().mean(dim=0, keepdim=True)
expected_seq1 = hidden[3:].float().mean(dim=0, keepdim=True)
expected = torch.cat([expected_seq0, expected_seq1], dim=0)
assert torch.allclose(out, expected, atol=1e-5)
def test_upcasts_to_float32(self):
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float16)
metadata = _make_metadata([2])
pooler = MeanPool()
out = pooler(hidden, metadata)
assert out.dtype == torch.float32
expected = torch.tensor([[2.0, 3.0]], dtype=torch.float32)
assert torch.allclose(out, expected, atol=1e-2)
# ---------------------------------------------------------------------------
# get_seq_pooling_method factory
# ---------------------------------------------------------------------------
class TestGetSeqPoolingMethod:
def test_cls(self):
assert isinstance(get_seq_pooling_method("CLS"), CLSPool)
def test_last(self):
assert isinstance(get_seq_pooling_method("LAST"), LastPool)
def test_mean(self):
assert isinstance(get_seq_pooling_method("MEAN"), MeanPool)
def test_unknown_raises(self):
with pytest.raises(NotImplementedError, match="UNKNOWN"):
get_seq_pooling_method("UNKNOWN")
# ---------------------------------------------------------------------------
# AllPool
# ---------------------------------------------------------------------------
@dataclass
class _FakeSchedulerConfig:
enable_chunked_prefill: bool = False
async_scheduling: bool = False
@dataclass
class _FakeVllmConfig:
scheduler_config: _FakeSchedulerConfig
@pytest.mark.parametrize("tok_pooling_type", ["ALL", "STEP"])
def test_dispatch_seq_cls_honors_token_pooling_type(tok_pooling_type):
"""Filter STEP rows before classification; preserve all rows for resolved ALL."""
pooler_config = PoolerConfig(
seq_pooling_type="LAST", tok_pooling_type=tok_pooling_type
)
config = SimpleNamespace(
scheduler_config=_FakeSchedulerConfig(),
model_config=SimpleNamespace(
pooler_config=pooler_config,
head_dtype=None,
hf_config=PreTrainedConfig(num_labels=3),
),
)
classifier = torch.nn.Linear(4, 3)
with set_current_vllm_config(config):
pooler = DispatchPooler.for_seq_cls(pooler_config, classifier=classifier)
token_ids = [[2, 99, 3, 99], [4, 99], [5, 6]]
params = [
PoolingParams(
task="token_classify",
use_activation=False,
step_tag_id=99 if tok_pooling_type == "STEP" else None,
)
for _ in token_ids
]
metadata = _make_metadata([4, 2, 2], token_ids=token_ids, pooling_params=params)
hidden = torch.arange(32, dtype=torch.float32).reshape(8, 4)
outputs = pooler(hidden, metadata)
expected_rows = (
[[1, 3], [5], []]
if tok_pooling_type == "STEP"
else [[0, 1, 2, 3], [4, 5], [6, 7]]
)
assert len(outputs) == len(expected_rows)
for output, rows in zip(outputs, expected_rows):
torch.testing.assert_close(output, classifier(hidden[rows]))
assert pooler.get_pooling_updates("token_classify").requires_token_ids == (
tok_pooling_type == "STEP"
)
class TestAllPool:
@staticmethod
def _make_all_pool(
*, chunked: bool = False, async_scheduling: bool = False
) -> AllPool:
fake_config = _FakeVllmConfig(
scheduler_config=_FakeSchedulerConfig(
enable_chunked_prefill=chunked,
async_scheduling=async_scheduling,
),
)
with patch(
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
return_value=fake_config,
):
return AllPool()
def test_splits_by_sequence(self):
pooler = self._make_all_pool()
hidden = torch.tensor(
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
)
metadata = _make_metadata([2, 3])
out = pooler(hidden, metadata)
assert len(out) == 2
assert torch.equal(out[0], hidden[:2])
assert torch.equal(out[1], hidden[2:])
def test_single_sequence(self):
pooler = self._make_all_pool()
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
metadata = _make_metadata([3])
out = pooler(hidden, metadata)
assert len(out) == 1
assert torch.equal(out[0], hidden)
def test_chunked_prefill_returns_none_for_unfinished(self):
pooler = self._make_all_pool(chunked=True)
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
metadata = _make_metadata(
[4],
num_scheduled_tokens=[2],
seq_lens=[2],
)
out = pooler(hidden, metadata)
assert len(out) == 1
assert out[0] is None
def test_chunked_prefill_returns_concat_when_finished(self):
pooler = self._make_all_pool(chunked=True)
chunk1 = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
metadata1 = _make_metadata(
[4],
num_scheduled_tokens=[2],
seq_lens=[2],
)
out1 = pooler(chunk1, metadata1)
assert out1[0] is None
expected_chunk1 = chunk1.clone()
chunk1.zero_()
chunk2 = torch.tensor([[5.0, 6.0], [7.0, 8.0]])
metadata2 = _make_metadata(
[4],
num_scheduled_tokens=[2],
seq_lens=[4],
)
metadata2.pooling_states = metadata1.pooling_states
out2 = pooler(chunk2, metadata2)
assert out2[0] is not None
expected = torch.cat([expected_chunk1, chunk2], dim=0)
assert torch.equal(out2[0], expected)
@pytest.mark.parametrize("async_scheduling", [False, True])
def test_chunked_prefill_single_shot_matches_non_chunked(
self, async_scheduling: bool
):
pooler = self._make_all_pool(chunked=True, async_scheduling=async_scheduling)
hidden = torch.tensor(
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
)
expected = [hidden[:2].clone(), hidden[2:].clone()]
metadata = _make_metadata([2, 3])
out = pooler(hidden, metadata)
if async_scheduling:
# Finished outputs must own their storage under async scheduling.
hidden.zero_()
assert len(out) == 2
assert torch.equal(out[0], expected[0])
assert torch.equal(out[1], expected[1])
def test_non_chunked_owns_output_under_async_scheduling(self):
pooler = self._make_all_pool(async_scheduling=True)
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
expected = [hidden[:2].clone(), hidden[2:].clone()]
metadata = _make_metadata([2, 1])
out = pooler(hidden, metadata)
hidden.zero_()
assert len(out) == 2
assert torch.equal(out[0], expected[0])
assert torch.equal(out[1], expected[1])
def test_chunked_prefill_mixed_finished_unfinished(self):
pooler = self._make_all_pool(chunked=True)
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
metadata = _make_metadata(
[2, 4],
num_scheduled_tokens=[2, 1],
seq_lens=[2, 1],
)
out = pooler(hidden, metadata)
assert len(out) == 2
assert torch.equal(out[0], hidden[:2])
assert out[1] is None
# ---------------------------------------------------------------------------
# StepPool
# ---------------------------------------------------------------------------
class TestStepPool:
@staticmethod
def _make_step_pool(*, chunked: bool = False) -> StepPool:
fake_config = _FakeVllmConfig(
scheduler_config=_FakeSchedulerConfig(
enable_chunked_prefill=chunked,
),
)
with patch(
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
return_value=fake_config,
):
return StepPool()
def test_filters_by_step_tag_id(self):
pooler = self._make_step_pool()
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
token_ids = [[10, 99, 10, 20]]
params = [PoolingParams(task="token_classify", step_tag_id=10)]
metadata = _make_metadata([4], token_ids=token_ids, pooling_params=params)
out = pooler(hidden, metadata)
assert len(out) == 1
expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
assert torch.equal(out[0], expected)
def test_filters_by_returned_token_ids(self):
pooler = self._make_step_pool()
hidden = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
token_ids = [[10, 20]]
params = [PoolingParams(task="token_classify", returned_token_ids=[0, 2])]
metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
out = pooler(hidden, metadata)
assert len(out) == 1
expected = torch.tensor([[1.0, 3.0], [4.0, 6.0]])
assert torch.equal(out[0], expected)
def test_no_filtering_without_params(self):
pooler = self._make_step_pool()
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
token_ids = [[10, 20]]
params = [PoolingParams(task="token_classify")]
metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
out = pooler(hidden, metadata)
assert len(out) == 1
assert torch.equal(out[0], hidden)
def test_combined_step_tag_and_returned_token_ids(self):
pooler = self._make_step_pool()
hidden = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]])
token_ids = [[99, 10, 99]]
params = [
PoolingParams(
task="token_classify",
step_tag_id=10,
returned_token_ids=[0, 2],
)
]
metadata = _make_metadata([3], token_ids=token_ids, pooling_params=params)
out = pooler(hidden, metadata)
assert len(out) == 1
expected = torch.tensor([[4.0, 6.0]])
assert torch.equal(out[0], expected)
def test_step_tag_id_no_match_returns_empty(self):
pooler = self._make_step_pool()
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
token_ids = [[10, 20]]
params = [PoolingParams(task="token_classify", step_tag_id=999)]
metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
out = pooler(hidden, metadata)
assert len(out) == 1
assert out[0].shape == (0, 2)
def test_chunked_prefill_propagates_none_for_unfinished(self):
pooler = self._make_step_pool(chunked=True)
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
token_ids = [[10, 20, 30, 40]]
params = [PoolingParams(task="token_classify", step_tag_id=10)]
metadata = _make_metadata(
[4],
token_ids=token_ids,
pooling_params=params,
num_scheduled_tokens=[2],
seq_lens=[2],
)
out = pooler(hidden, metadata)
assert len(out) == 1
assert out[0] is None
def test_chunked_prefill_filters_when_finished(self):
pooler = self._make_step_pool(chunked=True)
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
token_ids = [[10, 99, 10, 20]]
params = [PoolingParams(task="token_classify", step_tag_id=10)]
metadata = _make_metadata([4], token_ids=token_ids, pooling_params=params)
out = pooler(hidden, metadata)
assert len(out) == 1
expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
assert torch.equal(out[0], expected)
def test_requires_token_ids_update(self):
pooler = self._make_step_pool()
update = pooler.get_pooling_updates("token_classify")
assert update.requires_token_ids is True
# ---------------------------------------------------------------------------
# get_tok_pooling_method factory
# ---------------------------------------------------------------------------
class TestGetTokPoolingMethod:
def test_all(self):
fake_config = _FakeVllmConfig(
scheduler_config=_FakeSchedulerConfig(
enable_chunked_prefill=False,
),
)
with patch(
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
return_value=fake_config,
):
assert isinstance(get_tok_pooling_method("ALL"), AllPool)
def test_step(self):
fake_config = _FakeVllmConfig(
scheduler_config=_FakeSchedulerConfig(
enable_chunked_prefill=False,
),
)
with patch(
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
return_value=fake_config,
):
assert isinstance(get_tok_pooling_method("STEP"), StepPool)
def test_unknown_raises(self):
with pytest.raises(NotImplementedError, match="UNKNOWN"):
get_tok_pooling_method("UNKNOWN")
@pytest.mark.parametrize(
("chunk_size", "dtype", "head_dtype"),
[
(1, torch.float32, None),
(2, torch.float16, None),
(2, torch.bfloat16, torch.float32),
(7, torch.float32, None),
(10, torch.float32, None),
(2**63 - 1, torch.float32, None),
],
)
def test_late_chunk_pool_matches_fp32_means_before_projection_and_normalization(
chunk_size, dtype, head_dtype
):
# Unequal norms expose the incorrect normalize-then-average implementation.
hidden = (torch.arange(28).reshape(7, 4) - 10).to(dtype)
projector = torch.nn.Linear(4, 3, bias=True).to(head_dtype or dtype)
head = TokenEmbeddingPoolerHead(head_dtype, projector, PoolerNormalize())
pooler = TokenPooler(TestAllPool._make_all_pool(), head)
params = PoolingParams(
task="token_embed",
late_chunking_params=LateChunkingParams(chunk_size=chunk_size),
use_activation=True,
)
metadata = _make_metadata([7], pooling_params=[params])
actual = pooler(hidden, metadata)[0]
means = torch.stack(
[hidden[a : a + chunk_size].float().mean(0) for a in range(0, 7, chunk_size)]
).to(head_dtype or dtype)
expected = torch.nn.functional.normalize(projector(means), p=2, dim=-1)
torch.testing.assert_close(actual, expected)
assert actual.dtype == (head_dtype or dtype)
def test_late_chunk_pool_reduces_before_nonlinear_projector():
hidden = torch.tensor([[2.0, 0.0], [0.0, 1.0]])
projector = torch.nn.Tanh()
head = TokenEmbeddingPoolerHead(projector=projector, activation=PoolerNormalize())
params = PoolingParams(
task="token_embed",
late_chunking_params=LateChunkingParams(chunk_size=2),
use_activation=True,
)
actual = head.forward_chunk(hidden, params)
expected = torch.nn.functional.normalize(projector(hidden.mean(0, keepdim=True)))
wrong_order = torch.nn.functional.normalize(projector(hidden).mean(0, keepdim=True))
torch.testing.assert_close(actual, expected)
assert not torch.allclose(actual, wrong_order)
def test_late_chunk_pool_isolates_mixed_requests_and_owns_finished_storage():
hidden = torch.arange(36, dtype=torch.float32).reshape(9, 4)
params = [
PoolingParams(
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=2)
),
PoolingParams(task="token_embed"),
PoolingParams(
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=1)
),
]
pooler = TokenPooler(
TestAllPool._make_all_pool(async_scheduling=True), TokenEmbeddingPoolerHead()
)
outputs = pooler(hidden, _make_metadata([3, 4, 2], pooling_params=params))
expected = [
torch.stack([hidden[:2].mean(0), hidden[2]]),
hidden[3:7].clone(),
hidden[7:].clone(),
]
hidden.fill_(-999)
for actual, reference in zip(outputs, expected):
torch.testing.assert_close(actual, reference)
@pytest.mark.parametrize("chunk_size", [1, 2, 8])
@pytest.mark.parametrize("embedding_size", [1, 4])
def test_bos_eos_filter_preserves_chunk_vectors_in_mixed_batch(
chunk_size, embedding_size
):
hidden = torch.arange(10 * embedding_size, dtype=torch.float32).reshape(
10, embedding_size
)
pooler = BOSEOSFilter(
TokenPooler(TestAllPool._make_all_pool(), TokenEmbeddingPoolerHead()),
bos_token_id=101,
eos_token_id=102,
)
metadata = _make_metadata(
[5, 5],
token_ids=[[101, 1, 2, 3, 102]] * 2,
pooling_params=[
PoolingParams(
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size)
),
PoolingParams(task="token_embed"),
],
)
chunked, ordinary = pooler(hidden, metadata)
expected = torch.stack([part.mean(0) for part in hidden[:5].split(chunk_size)])
torch.testing.assert_close(chunked, expected)
torch.testing.assert_close(ordinary, hidden[6:9].squeeze(-1))
def test_bos_eos_filter_waits_for_complete_chunk_vectors():
pooler = BOSEOSFilter(
TokenPooler(
TestAllPool._make_all_pool(chunked=True), TokenEmbeddingPoolerHead()
),
bos_token_id=101,
)
params = [
PoolingParams(task="token_embed", late_chunking_params=LateChunkingParams(3))
]
first = _make_metadata(
[4],
token_ids=[[101, 1, 2, 102]],
pooling_params=params,
num_scheduled_tokens=[2],
seq_lens=[2],
)
hidden = torch.arange(8, dtype=torch.float32).reshape(4, 2)
assert pooler(hidden[:2], first) == [None]
last = _make_metadata(
[4],
token_ids=[[101, 1, 2, 102]],
pooling_params=params,
num_scheduled_tokens=[2],
seq_lens=[4],
)
last.pooling_states = first.pooling_states
actual = pooler(hidden[2:], last)[0]
torch.testing.assert_close(actual, torch.stack([hidden[:3].mean(0), hidden[3]]))
assert not last.pooling_states[0].hidden_states_cache
@@ -1,344 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
import torch
from vllm.platforms import current_platform
if current_platform.is_cuda():
pytest.skip(
"ROCm skinny GEMM tests are not supported on CUDA.",
allow_module_level=True,
)
from vllm.model_executor.layers import utils
def _on_gfx11() -> bool:
"""gfx11 check that is safe to evaluate at collection time on any platform."""
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx11
return on_gfx11()
@pytest.mark.parametrize("m", [1, 128])
def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch, m):
x = torch.randn(1, 64, dtype=torch.float16)
weight = torch.randn(m, 64, dtype=torch.float16)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_called_once()
llmm1_mock.assert_not_called()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("n", range(1, 6))
def test_rocm_unquantized_gemm_single_output_real_kernel(dtype, n):
torch.manual_seed(0)
x = torch.randn(n, 2048, device="cuda", dtype=dtype)
weight = torch.randn(1, 2048, device="cuda", dtype=dtype)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
def test_rocm_unquantized_gemm_makes_skinny_activation_contiguous(monkeypatch):
x = torch.randn(64, 4, dtype=torch.float16).t()
weight = torch.randn(128, 64, dtype=torch.float16)
assert x.shape == (4, 64)
assert x.stride() == (1, 4)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_called_once()
x_view = wvsplitk_mock.call_args.args[1]
assert x_view.is_contiguous()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
def test_rocm_unquantized_gemm_makes_llmm1_activation_contiguous(monkeypatch):
x = torch.randn(1, 128, dtype=torch.float16)[:, ::2]
weight = torch.randn(4, 64, dtype=torch.float16)
assert x.shape == (1, 64)
assert x.stride() == (128, 2)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
llmm1_mock.assert_called_once()
x_view = llmm1_mock.call_args.args[1]
assert x_view.is_contiguous()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
@pytest.mark.parametrize("noncontiguous_operand", ["weight", "bias"])
def test_rocm_unquantized_gemm_rejects_unsupported_skinny_layouts(
monkeypatch, noncontiguous_operand
):
x = torch.randn(4, 64, dtype=torch.float16)
weight = torch.randn(128, 64, dtype=torch.float16)
bias = torch.randn(128, dtype=torch.float16)
if noncontiguous_operand == "weight":
weight = torch.randn(64, 128, dtype=torch.float16).t()
assert not weight.is_contiguous()
else:
bias = torch.randn(256, dtype=torch.float16)[::2]
assert not bias.is_contiguous()
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.rocm_aiter_ops, "is_tgemm_enabled", lambda: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock()
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
llmm1_mock = MagicMock()
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, bias)
ref = torch.nn.functional.linear(x, weight, bias)
wvsplitk_mock.assert_not_called()
llmm1_mock.assert_not_called()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
def _row_stride_padded(weight: torch.Tensor, pad: int) -> torch.Tensor:
"""A weight whose row stride is offset by ``pad`` elements, as produced by
``maybe_pad_weight_avoid_cache_aliasing``."""
padded = torch.nn.functional.pad(weight, (0, pad))[..., :-pad]
assert padded.shape == weight.shape
assert padded.stride() == (weight.shape[1] + pad, 1)
return padded
def test_rocm_unquantized_gemm_wvsplitk_accepts_row_stride_padded_weight(monkeypatch):
# wvSplitK reads the weight row stride, so a row-stride-padded weight must
# still reach it instead of falling back to torch.nn.functional.linear.
x = torch.randn(4, 1024, dtype=torch.float16)
weight = _row_stride_padded(torch.randn(128, 1024, dtype=torch.float16), 64)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.rocm_aiter_ops, "is_tgemm_enabled", lambda: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_called_once()
assert wvsplitk_mock.call_args.args[0].stride() == weight.stride()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
@pytest.mark.parametrize(
"k,element_size,expected",
[
(4096, 2, 64), # 8192 B row stride, aliases
(12288, 2, 64), # 24576 B row stride, aliases
(1024, 2, 64), # smallest aliasing bf16/fp16 K
(2560, 2, 0), # 5120 B, no aliasing
(11008, 2, 0), # 22016 B, no aliasing
(2048, 1, 128), # fp8: 2048 B, aliases, twice the pad
(512, 4, 32), # fp32: 2048 B, aliases, half the pad
],
)
def test_cache_aliasing_pad_elems(k, element_size, expected):
assert utils.cache_aliasing_pad_elems(k, element_size) == expected
def test_maybe_pad_weight_avoid_cache_aliasing_skips_non_gfx11(monkeypatch):
# Padding is gfx11-only, even for a K whose row stride would alias there.
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: False)
weight = torch.randn(128, 4096, dtype=torch.bfloat16)
assert utils.maybe_pad_weight_avoid_cache_aliasing(weight) is weight
def test_maybe_pad_weight_avoid_cache_aliasing(monkeypatch):
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
weight = torch.randn(128, 4096, dtype=torch.bfloat16)
padded = utils.maybe_pad_weight_avoid_cache_aliasing(weight)
assert padded.shape == weight.shape
assert padded.stride() == (4096 + 64, 1)
assert torch.equal(padded, weight)
# Idempotent: the padded stride no longer aliases, so a second call is a no-op
# and preserves the pointer captured by any graph.
repadded = utils.maybe_pad_weight_avoid_cache_aliasing(padded)
assert repadded.data_ptr() == padded.data_ptr()
@pytest.mark.parametrize(
"weight",
[
torch.randn(128, 2560, dtype=torch.bfloat16), # no aliasing
torch.randn(2560, 128, dtype=torch.bfloat16).t(), # not contiguous
torch.randn(2, 128, 4096, dtype=torch.bfloat16), # not 2D
],
)
def test_maybe_pad_weight_avoid_cache_aliasing_skips(monkeypatch, weight):
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
assert (
utils.maybe_pad_weight_avoid_cache_aliasing(weight).data_ptr()
== weight.data_ptr()
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_rocm_unquantized_gemm_noncontiguous_activation_real_kernel(monkeypatch, dtype):
x = torch.randn(64, 4, device="cuda", dtype=dtype).t()
weight = torch.randn(128, 64, device="cuda", dtype=dtype)
assert x.stride() == (1, 4)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
original_wvsplitk = utils.ops.wvSplitK
wvsplitk_mock = MagicMock(side_effect=original_wvsplitk)
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_called_once()
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not _on_gfx11(), reason="cache-aliasing padding only applies on gfx11"
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_rocm_unquantized_gemm_padded_weight_real_kernel(monkeypatch, dtype):
x = torch.randn(4, 4096, device="cuda", dtype=dtype)
weight = _row_stride_padded(torch.randn(128, 4096, device="cuda", dtype=dtype), 64)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
original_wvsplitk = utils.ops.wvSplitK
wvsplitk_mock = MagicMock(side_effect=original_wvsplitk)
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_called_once()
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
@pytest.mark.parametrize("m", [1, 128])
def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch, m):
# wvSplitK skinny GEMM handles n in [1, 5] (see PR #40687); n > 5 must
# fall back to torch.nn.functional.linear.
x = torch.randn(6, 64, dtype=torch.float16)
weight = torch.randn(m, 64, dtype=torch.float16)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitk_mock.assert_not_called()
llmm1_mock.assert_not_called()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
def test_rocm_unquantized_gemm_gfx950_wvsplitkrc_path(monkeypatch):
x = torch.randn(1024, 16, dtype=torch.float16).t()
weight = torch.randn(256, 1024, dtype=torch.float16)
assert x.stride() == (1, 16)
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: True)
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
wvsplitkrc_mock = MagicMock(side_effect=lambda x_view, w, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitKrc", wvsplitkrc_mock)
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
ref = torch.nn.functional.linear(x, weight, None)
wvsplitkrc_mock.assert_called_once()
wvsplitk_mock.assert_not_called()
x_view = wvsplitkrc_mock.call_args.args[0]
assert x_view.is_contiguous()
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)