[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,38 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.layers.activation import GeluAndMul, SiluAndMul
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cpu()
|
||||
or current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC,
|
||||
reason="Native CPU activations require a non-PowerPC CPU platform",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"activation_factory,reference",
|
||||
[
|
||||
(SiluAndMul, nn.SiLU()),
|
||||
(GeluAndMul, nn.GELU()),
|
||||
(partial(GeluAndMul, approximate="tanh"), nn.GELU(approximate="tanh")),
|
||||
],
|
||||
ids=["silu", "gelu", "gelu_tanh"],
|
||||
)
|
||||
def test_cpu_act_and_mul_without_compiled_ops(
|
||||
default_vllm_config, monkeypatch, activation_factory, reference
|
||||
):
|
||||
"""CPU activations must construct and run without compiled wheel operators."""
|
||||
default_vllm_config.compilation_config.custom_ops = ["all"]
|
||||
monkeypatch.setattr(torch.ops, "_C", object())
|
||||
x = torch.tensor([[-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device="cpu")
|
||||
|
||||
layer = activation_factory()
|
||||
|
||||
torch.testing.assert_close(layer(x), reference(x[:, :4]) * x[:, 4:])
|
||||
@@ -1,339 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for FusedMMInputNorm and the fused affine transform kernel."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
||||
FusedMMInputNorm,
|
||||
IdentityInputNorm,
|
||||
NormParams,
|
||||
fused_mm_input_norm_triton,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
# Module-level tests run on any platform: the CPU platform exercises
|
||||
# forward_native, accelerators exercise the fused kernel. Only the raw
|
||||
# kernel tests require an accelerator.
|
||||
_DEVICE = torch.device(current_platform.device_type)
|
||||
|
||||
requires_accelerator = pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason="fused Triton kernel requires a CUDA/XPU accelerator",
|
||||
)
|
||||
requires_triton = pytest.mark.skipif(not HAS_TRITON, reason="requires Triton")
|
||||
requires_vllm_config = pytest.mark.usefixtures("default_vllm_config")
|
||||
|
||||
|
||||
def _reference_input_norm(
|
||||
pixel_values: torch.Tensor,
|
||||
image_mean: list[float],
|
||||
image_std: list[float],
|
||||
rescale_factor: float,
|
||||
channel: int,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Straightforward per-channel affine: (x * rescale - mean) / std."""
|
||||
patches, size = pixel_values.shape
|
||||
patch_size = size // channel
|
||||
mean = torch.tensor(
|
||||
image_mean, dtype=torch.float32, device=pixel_values.device
|
||||
).view(1, channel, 1)
|
||||
std = torch.tensor(image_std, dtype=torch.float32, device=pixel_values.device).view(
|
||||
1, channel, 1
|
||||
)
|
||||
x = pixel_values.to(torch.float32).reshape(patches, channel, patch_size)
|
||||
x = (x * rescale_factor - mean) / std
|
||||
return x.view(patches, size).to(out_dtype)
|
||||
|
||||
|
||||
# CLIP-style RGB normalization constants reused across tests.
|
||||
_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073]
|
||||
_RGB_STD = [0.26862954, 0.26130258, 0.27577711]
|
||||
_RGB_RESCALE = 1.0 / 255.0
|
||||
|
||||
|
||||
def _check_against_reference(
|
||||
pixel_values: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
channel: int = 3,
|
||||
image_mean: list[float] | None = None,
|
||||
image_std: list[float] | None = None,
|
||||
rescale_factor: float = _RGB_RESCALE,
|
||||
**close_kwargs,
|
||||
):
|
||||
"""Run ``FusedMMInputNorm`` on ``pixel_values`` and compare against the
|
||||
eager per-channel affine reference. Uses the RGB constants by default."""
|
||||
if image_mean is None:
|
||||
image_mean = list(_RGB_MEAN)
|
||||
if image_std is None:
|
||||
image_std = list(_RGB_STD)
|
||||
norm = FusedMMInputNorm(
|
||||
image_mean=image_mean,
|
||||
image_std=image_std,
|
||||
rescale_factor=rescale_factor,
|
||||
channel=channel,
|
||||
).to(_DEVICE)
|
||||
expected = _reference_input_norm(
|
||||
pixel_values, image_mean, image_std, rescale_factor, channel, out_dtype
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
norm(pixel_values, visual_dtype=out_dtype), expected, **close_kwargs
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Module-level behavior
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormModule:
|
||||
def test_matches_reference_above_cudnn_grid_limit(self):
|
||||
"""num_patches above the old cuDNN batch-norm grid limit (~65535)
|
||||
used to raise CUDNN_STATUS_INTERNAL_ERROR."""
|
||||
set_random_seed(0)
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (70000, 3 * 196), dtype=torch.float32, device=_DEVICE
|
||||
)
|
||||
_check_against_reference(pixel_values, torch.float32)
|
||||
|
||||
def test_identity_passthrough(self):
|
||||
norm = IdentityInputNorm()
|
||||
|
||||
pixel_values = torch.randn(8, 3 * 196, dtype=torch.float32, device=_DEVICE)
|
||||
out = norm(pixel_values, visual_dtype=torch.bfloat16)
|
||||
torch.testing.assert_close(out, pixel_values.to(torch.bfloat16))
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# dtype coverage
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormDtypes:
|
||||
@pytest.mark.parametrize(
|
||||
"in_dtype,out_dtype",
|
||||
[
|
||||
(torch.float32, torch.float32),
|
||||
(torch.float32, torch.bfloat16),
|
||||
(torch.float32, torch.float16),
|
||||
(torch.bfloat16, torch.bfloat16),
|
||||
(torch.bfloat16, torch.float32),
|
||||
(torch.float16, torch.float16),
|
||||
(torch.uint8, torch.bfloat16),
|
||||
(torch.uint8, torch.float32),
|
||||
],
|
||||
)
|
||||
def test_dtype_combinations(self, in_dtype: torch.dtype, out_dtype: torch.dtype):
|
||||
set_random_seed(0)
|
||||
if in_dtype == torch.uint8:
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (32, 3 * 16), dtype=torch.uint8, device=_DEVICE
|
||||
)
|
||||
else:
|
||||
pixel_values = torch.rand(
|
||||
(32, 3 * 16), dtype=torch.float32, device=_DEVICE
|
||||
).to(in_dtype)
|
||||
|
||||
_check_against_reference(
|
||||
pixel_values,
|
||||
out_dtype,
|
||||
image_mean=[0.5, 0.5, 0.5],
|
||||
image_std=[0.5, 0.5, 0.5],
|
||||
atol=1e-2,
|
||||
rtol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Shape / channel coverage
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormShapes:
|
||||
@pytest.mark.parametrize("channel", [1, 3, 4])
|
||||
def test_channel_variants(self, channel: int):
|
||||
set_random_seed(0)
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (16, channel * 64), dtype=torch.float32, device=_DEVICE
|
||||
)
|
||||
_check_against_reference(
|
||||
pixel_values,
|
||||
torch.float32,
|
||||
channel=channel,
|
||||
image_mean=[0.5] * channel,
|
||||
image_std=[0.25] * channel,
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("patch_size", [1, 2048])
|
||||
def test_block_boundaries(self, patch_size: int):
|
||||
"""Masked-tail (12 elements) and exact-multiple (24 blocks) paths."""
|
||||
set_random_seed(0)
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (4, 3 * patch_size), dtype=torch.float32, device=_DEVICE
|
||||
)
|
||||
_check_against_reference(pixel_values, torch.float32)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Input handling: non-contiguous inputs
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormInputHandling:
|
||||
def test_non_contiguous_input_matches_reference(self):
|
||||
set_random_seed(0)
|
||||
base = torch.randint(
|
||||
0, 256, (8, 3 * 32, 2), dtype=torch.float32, device=_DEVICE
|
||||
)
|
||||
non_contig = base[..., 0]
|
||||
assert not non_contig.is_contiguous()
|
||||
|
||||
_check_against_reference(non_contig, torch.float32)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Raw kernel entry point
|
||||
# ===========================================================================
|
||||
@requires_accelerator
|
||||
@requires_triton
|
||||
class TestFusedMMInputNormKernel:
|
||||
@pytest.mark.parametrize(
|
||||
"N, C, L",
|
||||
[
|
||||
(1, 1, 1),
|
||||
(1, 3, 1),
|
||||
(3, 3, 7), # tile straddles channel boundaries
|
||||
(2, 8, 63),
|
||||
(4, 3, 1000),
|
||||
],
|
||||
)
|
||||
def test_channel_boundary_crossing(self, N: int, C: int, L: int):
|
||||
"""The 1D kernel recovers ``c = (offs // L) % C`` per lane, so a tile
|
||||
can straddle a channel boundary and gather weight/bias per lane.
|
||||
These shapes force that path by keeping ``L`` well below the block."""
|
||||
set_random_seed(0)
|
||||
x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE)
|
||||
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
|
||||
out = torch.empty_like(x)
|
||||
fused_mm_input_norm_triton(x, out, w, b)
|
||||
|
||||
expected = x * w.view(1, C, 1) + b.view(1, C, 1)
|
||||
torch.testing.assert_close(out, expected)
|
||||
|
||||
def test_non_contiguous_inputs_materialized(self):
|
||||
"""Read-only tensors are made contiguous internally."""
|
||||
N, C, L = 3, 3, 100
|
||||
set_random_seed(0)
|
||||
base = torch.randn(N, C, L, 2, dtype=torch.float32, device=_DEVICE)
|
||||
x = base[..., 0]
|
||||
assert not x.is_contiguous()
|
||||
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
|
||||
out = torch.empty(N, C, L, dtype=torch.float32, device=_DEVICE)
|
||||
fused_mm_input_norm_triton(x, out, w, b)
|
||||
|
||||
expected = x * w.view(1, C, 1) + b.view(1, C, 1)
|
||||
torch.testing.assert_close(out, expected)
|
||||
|
||||
def test_rejects_mismatched_output_shape(self):
|
||||
"""``outputs`` must be shaped exactly like ``inputs``."""
|
||||
N, C, L = 3, 3, 100
|
||||
x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE)
|
||||
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
|
||||
for out_shape in ((N, C + 1, L), (N, C, L + 1), (N + 2, C, L)):
|
||||
with pytest.raises(AssertionError):
|
||||
fused_mm_input_norm_triton(
|
||||
x,
|
||||
torch.empty(*out_shape, dtype=torch.float32, device=_DEVICE),
|
||||
w,
|
||||
b,
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Construction / configuration
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormConstruction:
|
||||
"""Weight/bias buffer semantics at init time."""
|
||||
|
||||
def test_input_norm_is_always_enabled(self):
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
compile_mm_encoder=True,
|
||||
custom_ops=["none"],
|
||||
)
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
FusedMMInputNorm(_RGB_MEAN, _RGB_STD, _RGB_RESCALE)
|
||||
assert (
|
||||
config.compilation_config.enabled_custom_ops["fused_mm_input_norm"] == 1
|
||||
)
|
||||
assert not config.compilation_config.disabled_custom_ops[
|
||||
"fused_mm_input_norm"
|
||||
]
|
||||
|
||||
def test_identity_config_buffers(self):
|
||||
"""Numerically identity parameters still build plain weight/bias
|
||||
buffers; the identity shortcut lives in ``from_model_config``."""
|
||||
norm = FusedMMInputNorm(
|
||||
image_mean=[0.0, 0.0, 0.0],
|
||||
image_std=[1.0, 1.0, 1.0],
|
||||
rescale_factor=1.0,
|
||||
)
|
||||
torch.testing.assert_close(norm.weight, torch.ones(3))
|
||||
torch.testing.assert_close(norm.bias, torch.zeros(3))
|
||||
|
||||
def test_non_identity_buffers(self):
|
||||
channel = 3
|
||||
norm = FusedMMInputNorm(
|
||||
image_mean=_RGB_MEAN,
|
||||
image_std=_RGB_STD,
|
||||
rescale_factor=_RGB_RESCALE,
|
||||
channel=channel,
|
||||
)
|
||||
assert norm.weight.shape == (channel,)
|
||||
assert norm.bias.shape == (channel,)
|
||||
|
||||
mean = torch.tensor(_RGB_MEAN, dtype=torch.float32)
|
||||
std = torch.tensor(_RGB_STD, dtype=torch.float32)
|
||||
torch.testing.assert_close(norm.weight, _RGB_RESCALE / std)
|
||||
torch.testing.assert_close(norm.bias, -mean / std)
|
||||
|
||||
def test_is_identity_params(self):
|
||||
"""The numeric-identity check mirrors the old allclose detection."""
|
||||
assert NormParams([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0).is_identity
|
||||
assert not NormParams(_RGB_MEAN, _RGB_STD, _RGB_RESCALE).is_identity
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("image_mean", "image_std", "rescale_factor"),
|
||||
[
|
||||
([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0),
|
||||
([0.5, 0.5, 0.5], [0.25, 0.25, 0.25], 1 / 255),
|
||||
],
|
||||
)
|
||||
def test_buffers_initialized_on_default_device(
|
||||
self,
|
||||
image_mean: list[float],
|
||||
image_std: list[float],
|
||||
rescale_factor: float,
|
||||
):
|
||||
"""Buffers are built on CPU first, then moved to the caller's
|
||||
default device, so init never touches the accelerator."""
|
||||
# The meta device gives the CPU-only test shard the same non-CPU
|
||||
# default-device semantics without requiring a CUDA build.
|
||||
default_device = "cuda" if torch.cuda.is_available() else "meta"
|
||||
with torch.device(default_device):
|
||||
norm = FusedMMInputNorm(image_mean, image_std, rescale_factor)
|
||||
|
||||
assert norm.weight.device.type == default_device
|
||||
assert norm.bias.device.type == default_device
|
||||
@@ -1,234 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for vllm.model_executor.layers.pooler.activations."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.layers.pooler.activations import (
|
||||
LambdaPoolerActivation,
|
||||
PoolerClassify,
|
||||
PoolerIdentity,
|
||||
PoolerMultiLabelClassify,
|
||||
PoolerNormalize,
|
||||
get_act_fn,
|
||||
resolve_classifier_act_fn,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PoolerIdentity
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestPoolerIdentity:
|
||||
def test_returns_input_unchanged(self):
|
||||
pooler = PoolerIdentity()
|
||||
x = torch.randn(4, 128)
|
||||
out = pooler(x)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_forward_list(self):
|
||||
pooler = PoolerIdentity()
|
||||
tensors = [torch.randn(128), torch.randn(256)]
|
||||
out = pooler(tensors)
|
||||
assert len(out) == 2
|
||||
for orig, result in zip(tensors, out):
|
||||
assert torch.equal(orig, result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PoolerNormalize
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestPoolerNormalize:
|
||||
def test_output_has_unit_norm(self):
|
||||
pooler = PoolerNormalize()
|
||||
x = torch.randn(4, 128)
|
||||
out = pooler(x)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(4), atol=1e-5)
|
||||
|
||||
def test_single_vector(self):
|
||||
pooler = PoolerNormalize()
|
||||
x = torch.randn(1, 64)
|
||||
out = pooler(x)
|
||||
norm = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norm, torch.ones(1), atol=1e-5)
|
||||
|
||||
def test_forward_list(self):
|
||||
pooler = PoolerNormalize()
|
||||
tensors = [torch.randn(1, 64), torch.randn(1, 128)]
|
||||
out = pooler(tensors)
|
||||
for t in out:
|
||||
norm = torch.linalg.norm(t, dim=-1)
|
||||
assert torch.allclose(norm, torch.ones(1), atol=1e-5)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PoolerMultiLabelClassify
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestPoolerMultiLabelClassify:
|
||||
def test_output_in_zero_one(self):
|
||||
pooler = PoolerMultiLabelClassify()
|
||||
x = torch.randn(4, 10)
|
||||
out = pooler(x)
|
||||
assert (out >= 0).all() and (out <= 1).all()
|
||||
|
||||
def test_large_positive_maps_near_one(self):
|
||||
pooler = PoolerMultiLabelClassify()
|
||||
x = torch.full((1, 3), 100.0)
|
||||
out = pooler(x)
|
||||
assert torch.allclose(out, torch.ones(1, 3), atol=1e-4)
|
||||
|
||||
def test_large_negative_maps_near_zero(self):
|
||||
pooler = PoolerMultiLabelClassify()
|
||||
x = torch.full((1, 3), -100.0)
|
||||
out = pooler(x)
|
||||
assert torch.allclose(out, torch.zeros(1, 3), atol=1e-4)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PoolerClassify
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestPoolerClassify:
|
||||
def test_infers_from_shape_when_num_labels_none(self):
|
||||
pooler = PoolerClassify(num_labels=None)
|
||||
assert pooler.num_labels is None
|
||||
x = torch.randn(2, 5)
|
||||
out = pooler(x)
|
||||
sums = out.sum(dim=-1)
|
||||
assert torch.allclose(sums, torch.ones(2), atol=1e-5)
|
||||
|
||||
def test_sigmoid_when_num_labels_lt_2(self):
|
||||
pooler = PoolerClassify(num_labels=1)
|
||||
x = torch.zeros(1, 1)
|
||||
out = pooler(x)
|
||||
assert torch.allclose(out, torch.tensor([[0.5]]), atol=1e-5)
|
||||
|
||||
def test_num_labels_zero_uses_sigmoid(self):
|
||||
pooler = PoolerClassify(num_labels=0)
|
||||
assert pooler.num_labels == 0
|
||||
x = torch.zeros(1, 3)
|
||||
out = pooler(x)
|
||||
assert torch.allclose(out, torch.full((1, 3), 0.5), atol=1e-5)
|
||||
|
||||
def test_num_labels_ge_2_uses_softmax(self):
|
||||
pooler = PoolerClassify(num_labels=4)
|
||||
assert pooler.num_labels == 4
|
||||
x = torch.randn(2, 4)
|
||||
out = pooler(x)
|
||||
sums = out.sum(dim=-1)
|
||||
assert torch.allclose(sums, torch.ones(2), atol=1e-5)
|
||||
|
||||
def test_default_num_labels_is_none(self):
|
||||
pooler = PoolerClassify()
|
||||
assert pooler.num_labels is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LambdaPoolerActivation
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestLambdaPoolerActivation:
|
||||
def test_applies_custom_fn(self):
|
||||
pooler = LambdaPoolerActivation(nn.ReLU())
|
||||
x = torch.tensor([[-1.0, 2.0, -3.0]])
|
||||
out = pooler(x)
|
||||
expected = torch.tensor([[0.0, 2.0, 0.0]])
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
def test_forward_list(self):
|
||||
pooler = LambdaPoolerActivation(nn.ReLU())
|
||||
tensors = [torch.tensor([-1.0, 2.0]), torch.tensor([3.0, -4.0])]
|
||||
out = pooler(tensors)
|
||||
assert torch.equal(out[0], torch.tensor([0.0, 2.0]))
|
||||
assert torch.equal(out[1], torch.tensor([3.0, 0.0]))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_act_fn factory
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestGetActFn:
|
||||
@staticmethod
|
||||
def _make_config(**kwargs):
|
||||
return SimpleNamespace(**kwargs)
|
||||
|
||||
def test_regression(self):
|
||||
cfg = self._make_config(problem_type="regression")
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerIdentity)
|
||||
|
||||
def test_single_label_classification(self):
|
||||
cfg = self._make_config(
|
||||
problem_type="single_label_classification", num_labels=3
|
||||
)
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerClassify)
|
||||
assert result.num_labels == 3
|
||||
|
||||
def test_multi_label_classification(self):
|
||||
cfg = self._make_config(problem_type="multi_label_classification")
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerMultiLabelClassify)
|
||||
|
||||
def test_sentence_transformers_activation(self):
|
||||
cfg = self._make_config(
|
||||
problem_type="",
|
||||
sentence_transformers={
|
||||
"activation_fn": "torch.nn.modules.activation.Sigmoid"
|
||||
},
|
||||
)
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerClassify)
|
||||
|
||||
def test_sbert_activation(self):
|
||||
cfg = self._make_config(
|
||||
problem_type="",
|
||||
sbert_ce_default_activation_function=(
|
||||
"torch.nn.modules.activation.Sigmoid"
|
||||
),
|
||||
)
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerClassify)
|
||||
|
||||
def test_default_fallback(self):
|
||||
cfg = self._make_config(problem_type="")
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerClassify)
|
||||
|
||||
def test_sentence_transformers_takes_priority(self):
|
||||
cfg = self._make_config(
|
||||
problem_type="",
|
||||
sentence_transformers={"activation_fn": "torch.nn.modules.linear.Identity"},
|
||||
sbert_ce_default_activation_function=(
|
||||
"torch.nn.modules.activation.Sigmoid"
|
||||
),
|
||||
)
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerIdentity)
|
||||
|
||||
def test_rejects_non_torch_activation(self):
|
||||
cfg = self._make_config(
|
||||
problem_type="",
|
||||
sentence_transformers={"activation_fn": "os.system"},
|
||||
)
|
||||
with pytest.raises(ValueError, match="restricted"):
|
||||
get_act_fn(cfg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_classifier_act_fn
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestResolveClassifierActFn:
|
||||
def test_delegates_to_get_act_fn_when_none(self):
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(num_labels=3, problem_type="")
|
||||
)
|
||||
result = resolve_classifier_act_fn(model_config, act_fn=None)
|
||||
assert isinstance(result, PoolerClassify)
|
||||
assert result.num_labels == 3
|
||||
|
||||
def test_passes_through_provided_act_fn(self):
|
||||
custom = PoolerIdentity()
|
||||
result = resolve_classifier_act_fn(None, act_fn=custom)
|
||||
assert result is custom
|
||||
@@ -1,490 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for sequence and token pooler head classes."""
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
|
||||
from vllm.model_executor.layers.pooler.seqwise.heads import (
|
||||
ClassifierPoolerHead,
|
||||
EmbeddingPoolerHead,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.tokwise.heads import (
|
||||
TokenClassifierPoolerHead,
|
||||
TokenEmbeddingPoolerHead,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
|
||||
_HIDDEN = 16
|
||||
_BATCH = 3
|
||||
|
||||
|
||||
def _make_params(
|
||||
n: int,
|
||||
*,
|
||||
task: PoolingTask = "embed",
|
||||
dimensions: int | None = None,
|
||||
use_activation: bool | None = None,
|
||||
) -> list[PoolingParams]:
|
||||
return [
|
||||
PoolingParams(task=task, dimensions=dimensions, use_activation=use_activation)
|
||||
for _ in range(n)
|
||||
]
|
||||
|
||||
|
||||
def _make_metadata(pooling_params: list[PoolingParams]) -> PoolingMetadata:
|
||||
n = len(pooling_params)
|
||||
return PoolingMetadata(
|
||||
prompt_lens=torch.ones(n, dtype=torch.long),
|
||||
prompt_token_ids=None,
|
||||
prompt_token_ids_cpu=None,
|
||||
pooling_params=pooling_params,
|
||||
pooling_states=[PoolingStates() for _ in range(n)],
|
||||
)
|
||||
|
||||
|
||||
def _linear(in_f: int, out_f: int) -> nn.Linear:
|
||||
torch.manual_seed(42)
|
||||
return nn.Linear(in_f, out_f, bias=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# EmbeddingPoolerHead
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestEmbeddingPoolerHead:
|
||||
def test_supported_tasks(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
assert head.get_supported_tasks() == {"embed"}
|
||||
|
||||
def test_passthrough(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH))
|
||||
out = head(x, meta)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_head_dtype(self):
|
||||
head = EmbeddingPoolerHead(head_dtype=torch.float16)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH))
|
||||
out = head(x, meta)
|
||||
assert out.dtype == torch.float16
|
||||
|
||||
def test_projector(self):
|
||||
proj = _linear(_HIDDEN, 8)
|
||||
head = EmbeddingPoolerHead(projector=proj)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH))
|
||||
out = head(x, meta)
|
||||
assert out.shape == (_BATCH, 8)
|
||||
assert torch.allclose(out, proj(x))
|
||||
|
||||
def test_matryoshka_uniform(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, dimensions=4)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert out.shape == (_BATCH, 4)
|
||||
assert torch.equal(out, x[..., :4])
|
||||
|
||||
def test_matryoshka_mixed(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
x = torch.randn(2, _HIDDEN)
|
||||
params = [
|
||||
PoolingParams(task="embed", dimensions=4),
|
||||
PoolingParams(task="embed", dimensions=8),
|
||||
]
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert isinstance(out, list)
|
||||
assert len(out) == 2
|
||||
assert out[0].shape[-1] == 4
|
||||
assert out[1].shape[-1] == 8
|
||||
|
||||
def test_matryoshka_mixed_with_none(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
x = torch.randn(2, _HIDDEN)
|
||||
params = [
|
||||
PoolingParams(task="embed", dimensions=4),
|
||||
PoolingParams(task="embed", dimensions=None),
|
||||
]
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert isinstance(out, list)
|
||||
assert out[0].shape[-1] == 4
|
||||
assert torch.equal(out[1], x[1])
|
||||
|
||||
def test_activation_uniform_true(self):
|
||||
head = EmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, use_activation=True)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
|
||||
|
||||
def test_activation_uniform_false(self):
|
||||
head = EmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, use_activation=False)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_activation_mixed_flags(self):
|
||||
head = EmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(2, _HIDDEN)
|
||||
params = [
|
||||
PoolingParams(task="embed", use_activation=True),
|
||||
PoolingParams(task="embed", use_activation=False),
|
||||
]
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert isinstance(out, list)
|
||||
norm_0 = torch.linalg.norm(out[0], dim=-1)
|
||||
assert torch.allclose(norm_0, torch.ones(1), atol=1e-5)
|
||||
assert torch.equal(out[1], x[1])
|
||||
|
||||
def test_list_input_gets_stacked(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
tensors = [torch.randn(_HIDDEN) for _ in range(_BATCH)]
|
||||
meta = _make_metadata(_make_params(_BATCH))
|
||||
out = head(tensors, meta)
|
||||
assert out.shape == (_BATCH, _HIDDEN)
|
||||
expected = torch.stack(tensors)
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
def test_projector_then_matryoshka(self):
|
||||
proj = _linear(_HIDDEN, 8)
|
||||
head = EmbeddingPoolerHead(projector=proj)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, dimensions=4)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert out.shape == (_BATCH, 4)
|
||||
assert torch.equal(out, proj(x)[..., :4])
|
||||
|
||||
def test_matryoshka_then_activation(self):
|
||||
head = EmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, dimensions=4, use_activation=True)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert out.shape == (_BATCH, 4)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
|
||||
|
||||
def test_empty_batch(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
x = torch.randn(0, _HIDDEN)
|
||||
meta = _make_metadata([])
|
||||
out = head(x, meta)
|
||||
assert out.shape == (0, _HIDDEN)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ClassifierPoolerHead
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestClassifierPoolerHead:
|
||||
def test_supported_tasks(self):
|
||||
head = ClassifierPoolerHead()
|
||||
assert head.get_supported_tasks() == {"classify"}
|
||||
|
||||
def test_passthrough(self):
|
||||
head = ClassifierPoolerHead()
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_head_dtype(self):
|
||||
head = ClassifierPoolerHead(head_dtype=torch.float16)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
assert out.dtype == torch.float16
|
||||
|
||||
def test_classifier(self):
|
||||
clf = _linear(_HIDDEN, 3)
|
||||
head = ClassifierPoolerHead(classifier=clf)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
assert out.shape == (_BATCH, 3)
|
||||
assert torch.allclose(out, clf(x))
|
||||
|
||||
def test_logit_mean(self):
|
||||
head = ClassifierPoolerHead(logit_mean=2.0)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
assert torch.allclose(out, x - 2.0)
|
||||
|
||||
def test_logit_sigma(self):
|
||||
head = ClassifierPoolerHead(logit_sigma=0.5)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
assert torch.allclose(out, x / 0.5)
|
||||
|
||||
def test_platt_scaling_combined(self):
|
||||
head = ClassifierPoolerHead(logit_mean=1.0, logit_sigma=2.0)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
assert torch.allclose(out, (x - 1.0) / 2.0)
|
||||
|
||||
def test_activation_uniform_true(self):
|
||||
head = ClassifierPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, task="classify", use_activation=True)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
|
||||
|
||||
def test_activation_uniform_false(self):
|
||||
head = ClassifierPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, task="classify", use_activation=False)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_activation_mixed_flags(self):
|
||||
head = ClassifierPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(2, _HIDDEN)
|
||||
params = [
|
||||
PoolingParams(task="classify", use_activation=True),
|
||||
PoolingParams(task="classify", use_activation=False),
|
||||
]
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert isinstance(out, list)
|
||||
norm_0 = torch.linalg.norm(out[0], dim=-1)
|
||||
assert torch.allclose(norm_0, torch.ones(1), atol=1e-5)
|
||||
assert torch.equal(out[1], x[1])
|
||||
|
||||
def test_list_input_gets_stacked(self):
|
||||
head = ClassifierPoolerHead()
|
||||
tensors = [torch.randn(_HIDDEN) for _ in range(_BATCH)]
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(tensors, meta)
|
||||
assert out.shape == (_BATCH, _HIDDEN)
|
||||
expected = torch.stack(tensors)
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
def test_classifier_then_platt_scaling(self):
|
||||
clf = _linear(_HIDDEN, 3)
|
||||
head = ClassifierPoolerHead(classifier=clf, logit_mean=1.0, logit_sigma=2.0)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
expected = (clf(x) - 1.0) / 2.0
|
||||
assert torch.allclose(out, expected)
|
||||
|
||||
def test_empty_batch(self):
|
||||
head = ClassifierPoolerHead()
|
||||
x = torch.randn(0, _HIDDEN)
|
||||
meta = _make_metadata([])
|
||||
out = head(x, meta)
|
||||
assert out.shape == (0, _HIDDEN)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TokenEmbeddingPoolerHead
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestTokenEmbeddingPoolerHead:
|
||||
def test_supported_tasks(self):
|
||||
head = TokenEmbeddingPoolerHead()
|
||||
assert head.get_supported_tasks() == {"token_embed"}
|
||||
|
||||
def test_passthrough(self):
|
||||
head = TokenEmbeddingPoolerHead()
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is x
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_none_chunked_prefill(self):
|
||||
head = TokenEmbeddingPoolerHead()
|
||||
param = PoolingParams(task="token_embed")
|
||||
out = head.forward_chunk(None, param)
|
||||
assert out is None
|
||||
|
||||
def test_head_dtype(self):
|
||||
head = TokenEmbeddingPoolerHead(head_dtype=torch.float16)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.dtype == torch.float16
|
||||
|
||||
def test_projector(self):
|
||||
proj = _linear(_HIDDEN, 8)
|
||||
head = TokenEmbeddingPoolerHead(projector=proj)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 8)
|
||||
assert torch.allclose(out, proj(x))
|
||||
|
||||
def test_matryoshka_truncation(self):
|
||||
head = TokenEmbeddingPoolerHead()
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", dimensions=4)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 4)
|
||||
assert torch.equal(out, x[..., :4])
|
||||
|
||||
def test_activation_true(self):
|
||||
head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", use_activation=True)
|
||||
out = head.forward_chunk(x, param)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
|
||||
|
||||
def test_activation_false(self):
|
||||
head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", use_activation=False)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_projector_then_matryoshka(self):
|
||||
proj = _linear(_HIDDEN, 8)
|
||||
head = TokenEmbeddingPoolerHead(projector=proj)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", dimensions=4)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 4)
|
||||
assert torch.equal(out, proj(x)[..., :4])
|
||||
|
||||
def test_matryoshka_then_activation(self):
|
||||
head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", dimensions=4, use_activation=True)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 4)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
|
||||
|
||||
def test_forward_mixed_batch_chunked_prefill(self):
|
||||
head = TokenEmbeddingPoolerHead()
|
||||
pooled_data = [torch.randn(5, _HIDDEN), None, torch.randn(3, _HIDDEN)]
|
||||
params = _make_params(3, task="token_embed")
|
||||
meta = _make_metadata(params)
|
||||
out = head(pooled_data, meta)
|
||||
assert len(out) == 3
|
||||
assert torch.equal(out[0], pooled_data[0])
|
||||
assert out[1] is None
|
||||
assert torch.equal(out[2], pooled_data[2])
|
||||
|
||||
def test_forward_empty_batch(self):
|
||||
head = TokenEmbeddingPoolerHead()
|
||||
meta = _make_metadata([])
|
||||
out = head([], meta)
|
||||
assert out == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TokenClassifierPoolerHead
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestTokenClassifierPoolerHead:
|
||||
def test_supported_tasks(self):
|
||||
head = TokenClassifierPoolerHead()
|
||||
assert head.get_supported_tasks() == {"token_classify"}
|
||||
|
||||
def test_passthrough(self):
|
||||
head = TokenClassifierPoolerHead()
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_none_chunked_prefill(self):
|
||||
head = TokenClassifierPoolerHead()
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(None, param)
|
||||
assert out is None
|
||||
|
||||
def test_head_dtype(self):
|
||||
head = TokenClassifierPoolerHead(head_dtype=torch.float16)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.dtype == torch.float16
|
||||
|
||||
def test_classifier(self):
|
||||
clf = _linear(_HIDDEN, 3)
|
||||
head = TokenClassifierPoolerHead(classifier=clf)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 3)
|
||||
assert torch.allclose(out, clf(x))
|
||||
|
||||
def test_logit_mean(self):
|
||||
head = TokenClassifierPoolerHead(logit_mean=2.0)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert torch.allclose(out, x - 2.0)
|
||||
|
||||
def test_logit_sigma(self):
|
||||
head = TokenClassifierPoolerHead(logit_sigma=0.5)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert torch.allclose(out, x / 0.5)
|
||||
|
||||
def test_platt_scaling_combined(self):
|
||||
head = TokenClassifierPoolerHead(logit_mean=1.0, logit_sigma=2.0)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert torch.allclose(out, (x - 1.0) / 2.0)
|
||||
|
||||
def test_activation_true(self):
|
||||
head = TokenClassifierPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify", use_activation=True)
|
||||
out = head.forward_chunk(x, param)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
|
||||
|
||||
def test_activation_false(self):
|
||||
head = TokenClassifierPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify", use_activation=False)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_forward_mixed_batch_chunked_prefill(self):
|
||||
head = TokenClassifierPoolerHead()
|
||||
pooled_data = [torch.randn(5, _HIDDEN), None, torch.randn(3, _HIDDEN)]
|
||||
params = _make_params(3, task="token_classify")
|
||||
meta = _make_metadata(params)
|
||||
out = head(pooled_data, meta)
|
||||
assert len(out) == 3
|
||||
assert torch.equal(out[0], pooled_data[0])
|
||||
assert out[1] is None
|
||||
assert torch.equal(out[2], pooled_data[2])
|
||||
|
||||
def test_forward_empty_batch(self):
|
||||
head = TokenClassifierPoolerHead()
|
||||
meta = _make_metadata([])
|
||||
out = head([], meta)
|
||||
assert out == []
|
||||
@@ -1,747 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for sequence and token pooling methods and their factories."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import PoolerConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
|
||||
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
|
||||
from vllm.model_executor.layers.pooler.seqwise.methods import (
|
||||
CLSPool,
|
||||
LastPool,
|
||||
MeanPool,
|
||||
get_seq_pooling_method,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.special import BOSEOSFilter, DispatchPooler
|
||||
from vllm.model_executor.layers.pooler.tokwise.heads import TokenEmbeddingPoolerHead
|
||||
from vllm.model_executor.layers.pooler.tokwise.methods import (
|
||||
AllPool,
|
||||
StepPool,
|
||||
get_tok_pooling_method,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.tokwise.poolers import TokenPooler
|
||||
from vllm.pooling_params import LateChunkingParams, PoolingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingCursor, PoolingMetadata, PoolingStates
|
||||
|
||||
_CPU = torch.device("cpu")
|
||||
|
||||
|
||||
def _make_pooling_cursor(
|
||||
prompt_lens: list[int],
|
||||
*,
|
||||
num_scheduled_tokens: list[int] | None = None,
|
||||
seq_lens: list[int] | None = None,
|
||||
device: torch.device = _CPU,
|
||||
) -> PoolingCursor:
|
||||
"""Build a PoolingCursor from a list of per-sequence prompt lengths."""
|
||||
prompt_lens_cpu = torch.tensor(prompt_lens, dtype=torch.long)
|
||||
if num_scheduled_tokens is None:
|
||||
num_scheduled_tokens_cpu = prompt_lens_cpu.clone()
|
||||
else:
|
||||
num_scheduled_tokens_cpu = torch.tensor(num_scheduled_tokens, dtype=torch.long)
|
||||
if seq_lens is None:
|
||||
seq_lens_cpu = prompt_lens_cpu.clone()
|
||||
else:
|
||||
seq_lens_cpu = torch.tensor(seq_lens, dtype=torch.long)
|
||||
|
||||
cumsum = torch.zeros(len(prompt_lens) + 1, dtype=torch.long, device=device)
|
||||
torch.cumsum(num_scheduled_tokens_cpu, dim=0, out=cumsum[1:])
|
||||
|
||||
return PoolingCursor(
|
||||
first_token_indices_gpu=cumsum[: len(prompt_lens)].to(device),
|
||||
last_token_indices_gpu=(cumsum[1:] - 1).to(device),
|
||||
prompt_lens_cpu=prompt_lens_cpu,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
num_scheduled_tokens_cpu=num_scheduled_tokens_cpu,
|
||||
partial_prefill=not torch.equal(prompt_lens_cpu, num_scheduled_tokens_cpu),
|
||||
finished_mask=torch.eq(prompt_lens_cpu, seq_lens_cpu).tolist(),
|
||||
)
|
||||
|
||||
|
||||
def _make_metadata(
|
||||
prompt_lens: list[int],
|
||||
*,
|
||||
tasks: list[PoolingTask] | None = None,
|
||||
token_ids: list[list[int]] | None = None,
|
||||
pooling_params: list[PoolingParams] | None = None,
|
||||
num_scheduled_tokens: list[int] | None = None,
|
||||
seq_lens: list[int] | None = None,
|
||||
device: torch.device = _CPU,
|
||||
) -> PoolingMetadata:
|
||||
"""Build a minimal PoolingMetadata for testing pooling methods."""
|
||||
n_seqs = len(prompt_lens)
|
||||
if tasks is None:
|
||||
tasks = ["embed"] * n_seqs
|
||||
if pooling_params is None:
|
||||
pooling_params = [PoolingParams(task=t) for t in tasks]
|
||||
|
||||
prompt_lens_tensor = torch.tensor(prompt_lens, dtype=torch.long)
|
||||
|
||||
prompt_token_ids_cpu = None
|
||||
prompt_token_ids = None
|
||||
if token_ids is not None:
|
||||
max_len = max(len(t) for t in token_ids)
|
||||
padded = [t + [0] * (max_len - len(t)) for t in token_ids]
|
||||
prompt_token_ids_cpu = torch.tensor(padded, dtype=torch.long)
|
||||
prompt_token_ids = prompt_token_ids_cpu.to(device)
|
||||
|
||||
cursor = _make_pooling_cursor(
|
||||
prompt_lens,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
seq_lens=seq_lens,
|
||||
device=device,
|
||||
)
|
||||
|
||||
pooling_states = [PoolingStates() for _ in range(n_seqs)]
|
||||
|
||||
return PoolingMetadata(
|
||||
prompt_lens=prompt_lens_tensor,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
prompt_token_ids_cpu=prompt_token_ids_cpu,
|
||||
pooling_params=pooling_params,
|
||||
pooling_states=pooling_states,
|
||||
pooling_cursor=cursor,
|
||||
)
|
||||
|
||||
|
||||
def test_build_pooling_cursor_caches_cpu_status() -> None:
|
||||
metadata = PoolingMetadata(
|
||||
prompt_lens=torch.tensor([3, 4, 2], dtype=torch.long),
|
||||
prompt_token_ids=None,
|
||||
prompt_token_ids_cpu=None,
|
||||
pooling_params=[PoolingParams(task="embed") for _ in range(3)],
|
||||
pooling_states=[PoolingStates() for _ in range(3)],
|
||||
)
|
||||
metadata.build_pooling_cursor(
|
||||
np.array([3, 2, 2], dtype=np.int32),
|
||||
seq_lens_cpu=torch.tensor([3, 2, 1], dtype=torch.long),
|
||||
device=_CPU,
|
||||
)
|
||||
|
||||
cursor = metadata.get_pooling_cursor()
|
||||
|
||||
assert cursor.is_partial_prefill() is True
|
||||
assert cursor.get_finished_mask() == [True, False, False]
|
||||
assert cursor[:1].is_partial_prefill() is False
|
||||
assert cursor[1:2].is_partial_prefill() is True
|
||||
assert cursor[1:].get_finished_mask() == [False, False]
|
||||
|
||||
|
||||
def test_pooling_params_update_combines_token_id_requirements() -> None:
|
||||
update = PoolingParamsUpdate() | PoolingParamsUpdate(requires_token_ids=True)
|
||||
params = PoolingParams(task="embed")
|
||||
|
||||
update.apply(params)
|
||||
|
||||
assert params.requires_token_ids is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLSPool
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestCLSPool:
|
||||
def test_extracts_first_token(self):
|
||||
hidden = torch.tensor(
|
||||
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
|
||||
)
|
||||
metadata = _make_metadata([2, 3])
|
||||
pooler = CLSPool()
|
||||
out = pooler(hidden, metadata)
|
||||
expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
def test_rejects_partial_prefill(self):
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
|
||||
metadata = _make_metadata([3], num_scheduled_tokens=[2])
|
||||
pooler = CLSPool()
|
||||
with pytest.raises(RuntimeError, match="partial prefill"):
|
||||
pooler(hidden, metadata)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LastPool
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestLastPool:
|
||||
def test_extracts_last_token(self):
|
||||
hidden = torch.tensor(
|
||||
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
|
||||
)
|
||||
metadata = _make_metadata([2, 3])
|
||||
pooler = LastPool()
|
||||
out = pooler(hidden, metadata)
|
||||
expected = torch.tensor([[3.0, 4.0], [9.0, 10.0]])
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
def test_partial_prefill_extracts_last_scheduled(self):
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
metadata = _make_metadata([4], num_scheduled_tokens=[2])
|
||||
pooler = LastPool()
|
||||
out = pooler(hidden, metadata)
|
||||
expected = torch.tensor([[3.0, 4.0]])
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MeanPool
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestMeanPool:
|
||||
def test_computes_mean(self):
|
||||
hidden = torch.tensor(
|
||||
[[1.0, 2.0], [3.0, 4.0], [10.0, 20.0]], dtype=torch.float32
|
||||
)
|
||||
metadata = _make_metadata([2, 1])
|
||||
pooler = MeanPool()
|
||||
out = pooler(hidden, metadata)
|
||||
expected = torch.tensor([[2.0, 3.0], [10.0, 20.0]], dtype=torch.float32)
|
||||
assert torch.allclose(out, expected, atol=1e-5)
|
||||
|
||||
def test_single_token_is_identity(self):
|
||||
hidden = torch.tensor([[5.0, 10.0]], dtype=torch.float32)
|
||||
metadata = _make_metadata([1])
|
||||
pooler = MeanPool()
|
||||
out = pooler(hidden, metadata)
|
||||
assert torch.allclose(out, hidden, atol=1e-5)
|
||||
|
||||
def test_uniform_values_return_same(self):
|
||||
hidden = torch.full((4, 3), 7.0, dtype=torch.float32)
|
||||
metadata = _make_metadata([4])
|
||||
pooler = MeanPool()
|
||||
out = pooler(hidden, metadata)
|
||||
expected = torch.full((1, 3), 7.0, dtype=torch.float32)
|
||||
assert torch.allclose(out, expected, atol=1e-5)
|
||||
|
||||
def test_multiple_sequences(self):
|
||||
hidden = torch.tensor(
|
||||
[
|
||||
[0.0, 0.0],
|
||||
[2.0, 4.0],
|
||||
[4.0, 8.0],
|
||||
[10.0, 10.0],
|
||||
],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
metadata = _make_metadata([3, 1])
|
||||
pooler = MeanPool()
|
||||
out = pooler(hidden, metadata)
|
||||
expected = torch.tensor([[2.0, 4.0], [10.0, 10.0]], dtype=torch.float32)
|
||||
assert torch.allclose(out, expected, atol=1e-5)
|
||||
|
||||
def test_empty_batch(self):
|
||||
hidden = torch.empty((0, 8), dtype=torch.float32)
|
||||
metadata = _make_metadata([])
|
||||
pooler = MeanPool()
|
||||
out = pooler(hidden, metadata)
|
||||
assert out.shape == (0, 8)
|
||||
|
||||
def test_rejects_partial_prefill(self):
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float32)
|
||||
metadata = _make_metadata([3], num_scheduled_tokens=[2])
|
||||
pooler = MeanPool()
|
||||
with pytest.raises(RuntimeError, match="partial prefill"):
|
||||
pooler(hidden, metadata)
|
||||
|
||||
def test_chunked_accumulation(self):
|
||||
hidden = torch.arange(20, dtype=torch.float32).reshape(5, 4)
|
||||
metadata = _make_metadata([3, 2])
|
||||
pooler = MeanPool()
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.seqwise.methods"
|
||||
"._MEAN_POOL_ACCUMULATION_CHUNK_BYTES",
|
||||
16,
|
||||
):
|
||||
out = pooler(hidden, metadata)
|
||||
expected_seq0 = hidden[:3].float().mean(dim=0, keepdim=True)
|
||||
expected_seq1 = hidden[3:].float().mean(dim=0, keepdim=True)
|
||||
expected = torch.cat([expected_seq0, expected_seq1], dim=0)
|
||||
assert torch.allclose(out, expected, atol=1e-5)
|
||||
|
||||
def test_upcasts_to_float32(self):
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float16)
|
||||
metadata = _make_metadata([2])
|
||||
pooler = MeanPool()
|
||||
out = pooler(hidden, metadata)
|
||||
assert out.dtype == torch.float32
|
||||
expected = torch.tensor([[2.0, 3.0]], dtype=torch.float32)
|
||||
assert torch.allclose(out, expected, atol=1e-2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_seq_pooling_method factory
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestGetSeqPoolingMethod:
|
||||
def test_cls(self):
|
||||
assert isinstance(get_seq_pooling_method("CLS"), CLSPool)
|
||||
|
||||
def test_last(self):
|
||||
assert isinstance(get_seq_pooling_method("LAST"), LastPool)
|
||||
|
||||
def test_mean(self):
|
||||
assert isinstance(get_seq_pooling_method("MEAN"), MeanPool)
|
||||
|
||||
def test_unknown_raises(self):
|
||||
with pytest.raises(NotImplementedError, match="UNKNOWN"):
|
||||
get_seq_pooling_method("UNKNOWN")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AllPool
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeSchedulerConfig:
|
||||
enable_chunked_prefill: bool = False
|
||||
async_scheduling: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeVllmConfig:
|
||||
scheduler_config: _FakeSchedulerConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tok_pooling_type", ["ALL", "STEP"])
|
||||
def test_dispatch_seq_cls_honors_token_pooling_type(tok_pooling_type):
|
||||
"""Filter STEP rows before classification; preserve all rows for resolved ALL."""
|
||||
pooler_config = PoolerConfig(
|
||||
seq_pooling_type="LAST", tok_pooling_type=tok_pooling_type
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=_FakeSchedulerConfig(),
|
||||
model_config=SimpleNamespace(
|
||||
pooler_config=pooler_config,
|
||||
head_dtype=None,
|
||||
hf_config=PreTrainedConfig(num_labels=3),
|
||||
),
|
||||
)
|
||||
classifier = torch.nn.Linear(4, 3)
|
||||
with set_current_vllm_config(config):
|
||||
pooler = DispatchPooler.for_seq_cls(pooler_config, classifier=classifier)
|
||||
|
||||
token_ids = [[2, 99, 3, 99], [4, 99], [5, 6]]
|
||||
params = [
|
||||
PoolingParams(
|
||||
task="token_classify",
|
||||
use_activation=False,
|
||||
step_tag_id=99 if tok_pooling_type == "STEP" else None,
|
||||
)
|
||||
for _ in token_ids
|
||||
]
|
||||
metadata = _make_metadata([4, 2, 2], token_ids=token_ids, pooling_params=params)
|
||||
hidden = torch.arange(32, dtype=torch.float32).reshape(8, 4)
|
||||
outputs = pooler(hidden, metadata)
|
||||
|
||||
expected_rows = (
|
||||
[[1, 3], [5], []]
|
||||
if tok_pooling_type == "STEP"
|
||||
else [[0, 1, 2, 3], [4, 5], [6, 7]]
|
||||
)
|
||||
assert len(outputs) == len(expected_rows)
|
||||
for output, rows in zip(outputs, expected_rows):
|
||||
torch.testing.assert_close(output, classifier(hidden[rows]))
|
||||
assert pooler.get_pooling_updates("token_classify").requires_token_ids == (
|
||||
tok_pooling_type == "STEP"
|
||||
)
|
||||
|
||||
|
||||
class TestAllPool:
|
||||
@staticmethod
|
||||
def _make_all_pool(
|
||||
*, chunked: bool = False, async_scheduling: bool = False
|
||||
) -> AllPool:
|
||||
fake_config = _FakeVllmConfig(
|
||||
scheduler_config=_FakeSchedulerConfig(
|
||||
enable_chunked_prefill=chunked,
|
||||
async_scheduling=async_scheduling,
|
||||
),
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
|
||||
return_value=fake_config,
|
||||
):
|
||||
return AllPool()
|
||||
|
||||
def test_splits_by_sequence(self):
|
||||
pooler = self._make_all_pool()
|
||||
hidden = torch.tensor(
|
||||
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
|
||||
)
|
||||
metadata = _make_metadata([2, 3])
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 2
|
||||
assert torch.equal(out[0], hidden[:2])
|
||||
assert torch.equal(out[1], hidden[2:])
|
||||
|
||||
def test_single_sequence(self):
|
||||
pooler = self._make_all_pool()
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
|
||||
metadata = _make_metadata([3])
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
assert torch.equal(out[0], hidden)
|
||||
|
||||
def test_chunked_prefill_returns_none_for_unfinished(self):
|
||||
pooler = self._make_all_pool(chunked=True)
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
metadata = _make_metadata(
|
||||
[4],
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[2],
|
||||
)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
assert out[0] is None
|
||||
|
||||
def test_chunked_prefill_returns_concat_when_finished(self):
|
||||
pooler = self._make_all_pool(chunked=True)
|
||||
|
||||
chunk1 = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
metadata1 = _make_metadata(
|
||||
[4],
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[2],
|
||||
)
|
||||
out1 = pooler(chunk1, metadata1)
|
||||
assert out1[0] is None
|
||||
expected_chunk1 = chunk1.clone()
|
||||
chunk1.zero_()
|
||||
|
||||
chunk2 = torch.tensor([[5.0, 6.0], [7.0, 8.0]])
|
||||
metadata2 = _make_metadata(
|
||||
[4],
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[4],
|
||||
)
|
||||
metadata2.pooling_states = metadata1.pooling_states
|
||||
out2 = pooler(chunk2, metadata2)
|
||||
assert out2[0] is not None
|
||||
expected = torch.cat([expected_chunk1, chunk2], dim=0)
|
||||
assert torch.equal(out2[0], expected)
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [False, True])
|
||||
def test_chunked_prefill_single_shot_matches_non_chunked(
|
||||
self, async_scheduling: bool
|
||||
):
|
||||
pooler = self._make_all_pool(chunked=True, async_scheduling=async_scheduling)
|
||||
hidden = torch.tensor(
|
||||
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
|
||||
)
|
||||
expected = [hidden[:2].clone(), hidden[2:].clone()]
|
||||
metadata = _make_metadata([2, 3])
|
||||
out = pooler(hidden, metadata)
|
||||
if async_scheduling:
|
||||
# Finished outputs must own their storage under async scheduling.
|
||||
hidden.zero_()
|
||||
assert len(out) == 2
|
||||
assert torch.equal(out[0], expected[0])
|
||||
assert torch.equal(out[1], expected[1])
|
||||
|
||||
def test_non_chunked_owns_output_under_async_scheduling(self):
|
||||
pooler = self._make_all_pool(async_scheduling=True)
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
|
||||
expected = [hidden[:2].clone(), hidden[2:].clone()]
|
||||
metadata = _make_metadata([2, 1])
|
||||
out = pooler(hidden, metadata)
|
||||
hidden.zero_()
|
||||
assert len(out) == 2
|
||||
assert torch.equal(out[0], expected[0])
|
||||
assert torch.equal(out[1], expected[1])
|
||||
|
||||
def test_chunked_prefill_mixed_finished_unfinished(self):
|
||||
pooler = self._make_all_pool(chunked=True)
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
|
||||
metadata = _make_metadata(
|
||||
[2, 4],
|
||||
num_scheduled_tokens=[2, 1],
|
||||
seq_lens=[2, 1],
|
||||
)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 2
|
||||
assert torch.equal(out[0], hidden[:2])
|
||||
assert out[1] is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# StepPool
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestStepPool:
|
||||
@staticmethod
|
||||
def _make_step_pool(*, chunked: bool = False) -> StepPool:
|
||||
fake_config = _FakeVllmConfig(
|
||||
scheduler_config=_FakeSchedulerConfig(
|
||||
enable_chunked_prefill=chunked,
|
||||
),
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
|
||||
return_value=fake_config,
|
||||
):
|
||||
return StepPool()
|
||||
|
||||
def test_filters_by_step_tag_id(self):
|
||||
pooler = self._make_step_pool()
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
|
||||
token_ids = [[10, 99, 10, 20]]
|
||||
params = [PoolingParams(task="token_classify", step_tag_id=10)]
|
||||
metadata = _make_metadata([4], token_ids=token_ids, pooling_params=params)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
|
||||
assert torch.equal(out[0], expected)
|
||||
|
||||
def test_filters_by_returned_token_ids(self):
|
||||
pooler = self._make_step_pool()
|
||||
hidden = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
|
||||
token_ids = [[10, 20]]
|
||||
params = [PoolingParams(task="token_classify", returned_token_ids=[0, 2])]
|
||||
metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
expected = torch.tensor([[1.0, 3.0], [4.0, 6.0]])
|
||||
assert torch.equal(out[0], expected)
|
||||
|
||||
def test_no_filtering_without_params(self):
|
||||
pooler = self._make_step_pool()
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
token_ids = [[10, 20]]
|
||||
params = [PoolingParams(task="token_classify")]
|
||||
metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
assert torch.equal(out[0], hidden)
|
||||
|
||||
def test_combined_step_tag_and_returned_token_ids(self):
|
||||
pooler = self._make_step_pool()
|
||||
hidden = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]])
|
||||
token_ids = [[99, 10, 99]]
|
||||
params = [
|
||||
PoolingParams(
|
||||
task="token_classify",
|
||||
step_tag_id=10,
|
||||
returned_token_ids=[0, 2],
|
||||
)
|
||||
]
|
||||
metadata = _make_metadata([3], token_ids=token_ids, pooling_params=params)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
expected = torch.tensor([[4.0, 6.0]])
|
||||
assert torch.equal(out[0], expected)
|
||||
|
||||
def test_step_tag_id_no_match_returns_empty(self):
|
||||
pooler = self._make_step_pool()
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
token_ids = [[10, 20]]
|
||||
params = [PoolingParams(task="token_classify", step_tag_id=999)]
|
||||
metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
assert out[0].shape == (0, 2)
|
||||
|
||||
def test_chunked_prefill_propagates_none_for_unfinished(self):
|
||||
pooler = self._make_step_pool(chunked=True)
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
token_ids = [[10, 20, 30, 40]]
|
||||
params = [PoolingParams(task="token_classify", step_tag_id=10)]
|
||||
metadata = _make_metadata(
|
||||
[4],
|
||||
token_ids=token_ids,
|
||||
pooling_params=params,
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[2],
|
||||
)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
assert out[0] is None
|
||||
|
||||
def test_chunked_prefill_filters_when_finished(self):
|
||||
pooler = self._make_step_pool(chunked=True)
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
|
||||
token_ids = [[10, 99, 10, 20]]
|
||||
params = [PoolingParams(task="token_classify", step_tag_id=10)]
|
||||
metadata = _make_metadata([4], token_ids=token_ids, pooling_params=params)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
|
||||
assert torch.equal(out[0], expected)
|
||||
|
||||
def test_requires_token_ids_update(self):
|
||||
pooler = self._make_step_pool()
|
||||
update = pooler.get_pooling_updates("token_classify")
|
||||
assert update.requires_token_ids is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_tok_pooling_method factory
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestGetTokPoolingMethod:
|
||||
def test_all(self):
|
||||
fake_config = _FakeVllmConfig(
|
||||
scheduler_config=_FakeSchedulerConfig(
|
||||
enable_chunked_prefill=False,
|
||||
),
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
|
||||
return_value=fake_config,
|
||||
):
|
||||
assert isinstance(get_tok_pooling_method("ALL"), AllPool)
|
||||
|
||||
def test_step(self):
|
||||
fake_config = _FakeVllmConfig(
|
||||
scheduler_config=_FakeSchedulerConfig(
|
||||
enable_chunked_prefill=False,
|
||||
),
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
|
||||
return_value=fake_config,
|
||||
):
|
||||
assert isinstance(get_tok_pooling_method("STEP"), StepPool)
|
||||
|
||||
def test_unknown_raises(self):
|
||||
with pytest.raises(NotImplementedError, match="UNKNOWN"):
|
||||
get_tok_pooling_method("UNKNOWN")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("chunk_size", "dtype", "head_dtype"),
|
||||
[
|
||||
(1, torch.float32, None),
|
||||
(2, torch.float16, None),
|
||||
(2, torch.bfloat16, torch.float32),
|
||||
(7, torch.float32, None),
|
||||
(10, torch.float32, None),
|
||||
(2**63 - 1, torch.float32, None),
|
||||
],
|
||||
)
|
||||
def test_late_chunk_pool_matches_fp32_means_before_projection_and_normalization(
|
||||
chunk_size, dtype, head_dtype
|
||||
):
|
||||
# Unequal norms expose the incorrect normalize-then-average implementation.
|
||||
hidden = (torch.arange(28).reshape(7, 4) - 10).to(dtype)
|
||||
projector = torch.nn.Linear(4, 3, bias=True).to(head_dtype or dtype)
|
||||
head = TokenEmbeddingPoolerHead(head_dtype, projector, PoolerNormalize())
|
||||
pooler = TokenPooler(TestAllPool._make_all_pool(), head)
|
||||
params = PoolingParams(
|
||||
task="token_embed",
|
||||
late_chunking_params=LateChunkingParams(chunk_size=chunk_size),
|
||||
use_activation=True,
|
||||
)
|
||||
metadata = _make_metadata([7], pooling_params=[params])
|
||||
actual = pooler(hidden, metadata)[0]
|
||||
means = torch.stack(
|
||||
[hidden[a : a + chunk_size].float().mean(0) for a in range(0, 7, chunk_size)]
|
||||
).to(head_dtype or dtype)
|
||||
expected = torch.nn.functional.normalize(projector(means), p=2, dim=-1)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
assert actual.dtype == (head_dtype or dtype)
|
||||
|
||||
|
||||
def test_late_chunk_pool_reduces_before_nonlinear_projector():
|
||||
hidden = torch.tensor([[2.0, 0.0], [0.0, 1.0]])
|
||||
projector = torch.nn.Tanh()
|
||||
head = TokenEmbeddingPoolerHead(projector=projector, activation=PoolerNormalize())
|
||||
params = PoolingParams(
|
||||
task="token_embed",
|
||||
late_chunking_params=LateChunkingParams(chunk_size=2),
|
||||
use_activation=True,
|
||||
)
|
||||
actual = head.forward_chunk(hidden, params)
|
||||
expected = torch.nn.functional.normalize(projector(hidden.mean(0, keepdim=True)))
|
||||
wrong_order = torch.nn.functional.normalize(projector(hidden).mean(0, keepdim=True))
|
||||
torch.testing.assert_close(actual, expected)
|
||||
assert not torch.allclose(actual, wrong_order)
|
||||
|
||||
|
||||
def test_late_chunk_pool_isolates_mixed_requests_and_owns_finished_storage():
|
||||
hidden = torch.arange(36, dtype=torch.float32).reshape(9, 4)
|
||||
params = [
|
||||
PoolingParams(
|
||||
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=2)
|
||||
),
|
||||
PoolingParams(task="token_embed"),
|
||||
PoolingParams(
|
||||
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=1)
|
||||
),
|
||||
]
|
||||
pooler = TokenPooler(
|
||||
TestAllPool._make_all_pool(async_scheduling=True), TokenEmbeddingPoolerHead()
|
||||
)
|
||||
outputs = pooler(hidden, _make_metadata([3, 4, 2], pooling_params=params))
|
||||
expected = [
|
||||
torch.stack([hidden[:2].mean(0), hidden[2]]),
|
||||
hidden[3:7].clone(),
|
||||
hidden[7:].clone(),
|
||||
]
|
||||
hidden.fill_(-999)
|
||||
for actual, reference in zip(outputs, expected):
|
||||
torch.testing.assert_close(actual, reference)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 2, 8])
|
||||
@pytest.mark.parametrize("embedding_size", [1, 4])
|
||||
def test_bos_eos_filter_preserves_chunk_vectors_in_mixed_batch(
|
||||
chunk_size, embedding_size
|
||||
):
|
||||
hidden = torch.arange(10 * embedding_size, dtype=torch.float32).reshape(
|
||||
10, embedding_size
|
||||
)
|
||||
pooler = BOSEOSFilter(
|
||||
TokenPooler(TestAllPool._make_all_pool(), TokenEmbeddingPoolerHead()),
|
||||
bos_token_id=101,
|
||||
eos_token_id=102,
|
||||
)
|
||||
metadata = _make_metadata(
|
||||
[5, 5],
|
||||
token_ids=[[101, 1, 2, 3, 102]] * 2,
|
||||
pooling_params=[
|
||||
PoolingParams(
|
||||
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size)
|
||||
),
|
||||
PoolingParams(task="token_embed"),
|
||||
],
|
||||
)
|
||||
chunked, ordinary = pooler(hidden, metadata)
|
||||
expected = torch.stack([part.mean(0) for part in hidden[:5].split(chunk_size)])
|
||||
torch.testing.assert_close(chunked, expected)
|
||||
torch.testing.assert_close(ordinary, hidden[6:9].squeeze(-1))
|
||||
|
||||
|
||||
def test_bos_eos_filter_waits_for_complete_chunk_vectors():
|
||||
pooler = BOSEOSFilter(
|
||||
TokenPooler(
|
||||
TestAllPool._make_all_pool(chunked=True), TokenEmbeddingPoolerHead()
|
||||
),
|
||||
bos_token_id=101,
|
||||
)
|
||||
params = [
|
||||
PoolingParams(task="token_embed", late_chunking_params=LateChunkingParams(3))
|
||||
]
|
||||
first = _make_metadata(
|
||||
[4],
|
||||
token_ids=[[101, 1, 2, 102]],
|
||||
pooling_params=params,
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[2],
|
||||
)
|
||||
hidden = torch.arange(8, dtype=torch.float32).reshape(4, 2)
|
||||
assert pooler(hidden[:2], first) == [None]
|
||||
last = _make_metadata(
|
||||
[4],
|
||||
token_ids=[[101, 1, 2, 102]],
|
||||
pooling_params=params,
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[4],
|
||||
)
|
||||
last.pooling_states = first.pooling_states
|
||||
actual = pooler(hidden[2:], last)[0]
|
||||
torch.testing.assert_close(actual, torch.stack([hidden[:3].mean(0), hidden[3]]))
|
||||
assert not last.pooling_states[0].hidden_states_cache
|
||||
@@ -1,344 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cuda():
|
||||
pytest.skip(
|
||||
"ROCm skinny GEMM tests are not supported on CUDA.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers import utils
|
||||
|
||||
|
||||
def _on_gfx11() -> bool:
|
||||
"""gfx11 check that is safe to evaluate at collection time on any platform."""
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx11
|
||||
|
||||
return on_gfx11()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [1, 128])
|
||||
def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch, m):
|
||||
x = torch.randn(1, 64, dtype=torch.float16)
|
||||
weight = torch.randn(m, 64, dtype=torch.float16)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_called_once()
|
||||
llmm1_mock.assert_not_called()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("n", range(1, 6))
|
||||
def test_rocm_unquantized_gemm_single_output_real_kernel(dtype, n):
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(n, 2048, device="cuda", dtype=dtype)
|
||||
weight = torch.randn(1, 2048, device="cuda", dtype=dtype)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def test_rocm_unquantized_gemm_makes_skinny_activation_contiguous(monkeypatch):
|
||||
x = torch.randn(64, 4, dtype=torch.float16).t()
|
||||
weight = torch.randn(128, 64, dtype=torch.float16)
|
||||
assert x.shape == (4, 64)
|
||||
assert x.stride() == (1, 4)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_called_once()
|
||||
x_view = wvsplitk_mock.call_args.args[1]
|
||||
assert x_view.is_contiguous()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
def test_rocm_unquantized_gemm_makes_llmm1_activation_contiguous(monkeypatch):
|
||||
x = torch.randn(1, 128, dtype=torch.float16)[:, ::2]
|
||||
weight = torch.randn(4, 64, dtype=torch.float16)
|
||||
assert x.shape == (1, 64)
|
||||
assert x.stride() == (128, 2)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
llmm1_mock.assert_called_once()
|
||||
x_view = llmm1_mock.call_args.args[1]
|
||||
assert x_view.is_contiguous()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("noncontiguous_operand", ["weight", "bias"])
|
||||
def test_rocm_unquantized_gemm_rejects_unsupported_skinny_layouts(
|
||||
monkeypatch, noncontiguous_operand
|
||||
):
|
||||
x = torch.randn(4, 64, dtype=torch.float16)
|
||||
weight = torch.randn(128, 64, dtype=torch.float16)
|
||||
bias = torch.randn(128, dtype=torch.float16)
|
||||
if noncontiguous_operand == "weight":
|
||||
weight = torch.randn(64, 128, dtype=torch.float16).t()
|
||||
assert not weight.is_contiguous()
|
||||
else:
|
||||
bias = torch.randn(256, dtype=torch.float16)[::2]
|
||||
assert not bias.is_contiguous()
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.rocm_aiter_ops, "is_tgemm_enabled", lambda: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock()
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
llmm1_mock = MagicMock()
|
||||
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, bias)
|
||||
ref = torch.nn.functional.linear(x, weight, bias)
|
||||
|
||||
wvsplitk_mock.assert_not_called()
|
||||
llmm1_mock.assert_not_called()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
def _row_stride_padded(weight: torch.Tensor, pad: int) -> torch.Tensor:
|
||||
"""A weight whose row stride is offset by ``pad`` elements, as produced by
|
||||
``maybe_pad_weight_avoid_cache_aliasing``."""
|
||||
padded = torch.nn.functional.pad(weight, (0, pad))[..., :-pad]
|
||||
assert padded.shape == weight.shape
|
||||
assert padded.stride() == (weight.shape[1] + pad, 1)
|
||||
return padded
|
||||
|
||||
|
||||
def test_rocm_unquantized_gemm_wvsplitk_accepts_row_stride_padded_weight(monkeypatch):
|
||||
# wvSplitK reads the weight row stride, so a row-stride-padded weight must
|
||||
# still reach it instead of falling back to torch.nn.functional.linear.
|
||||
x = torch.randn(4, 1024, dtype=torch.float16)
|
||||
weight = _row_stride_padded(torch.randn(128, 1024, dtype=torch.float16), 64)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.rocm_aiter_ops, "is_tgemm_enabled", lambda: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_called_once()
|
||||
assert wvsplitk_mock.call_args.args[0].stride() == weight.stride()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"k,element_size,expected",
|
||||
[
|
||||
(4096, 2, 64), # 8192 B row stride, aliases
|
||||
(12288, 2, 64), # 24576 B row stride, aliases
|
||||
(1024, 2, 64), # smallest aliasing bf16/fp16 K
|
||||
(2560, 2, 0), # 5120 B, no aliasing
|
||||
(11008, 2, 0), # 22016 B, no aliasing
|
||||
(2048, 1, 128), # fp8: 2048 B, aliases, twice the pad
|
||||
(512, 4, 32), # fp32: 2048 B, aliases, half the pad
|
||||
],
|
||||
)
|
||||
def test_cache_aliasing_pad_elems(k, element_size, expected):
|
||||
assert utils.cache_aliasing_pad_elems(k, element_size) == expected
|
||||
|
||||
|
||||
def test_maybe_pad_weight_avoid_cache_aliasing_skips_non_gfx11(monkeypatch):
|
||||
# Padding is gfx11-only, even for a K whose row stride would alias there.
|
||||
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: False)
|
||||
|
||||
weight = torch.randn(128, 4096, dtype=torch.bfloat16)
|
||||
assert utils.maybe_pad_weight_avoid_cache_aliasing(weight) is weight
|
||||
|
||||
|
||||
def test_maybe_pad_weight_avoid_cache_aliasing(monkeypatch):
|
||||
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
|
||||
|
||||
weight = torch.randn(128, 4096, dtype=torch.bfloat16)
|
||||
padded = utils.maybe_pad_weight_avoid_cache_aliasing(weight)
|
||||
|
||||
assert padded.shape == weight.shape
|
||||
assert padded.stride() == (4096 + 64, 1)
|
||||
assert torch.equal(padded, weight)
|
||||
# Idempotent: the padded stride no longer aliases, so a second call is a no-op
|
||||
# and preserves the pointer captured by any graph.
|
||||
repadded = utils.maybe_pad_weight_avoid_cache_aliasing(padded)
|
||||
assert repadded.data_ptr() == padded.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"weight",
|
||||
[
|
||||
torch.randn(128, 2560, dtype=torch.bfloat16), # no aliasing
|
||||
torch.randn(2560, 128, dtype=torch.bfloat16).t(), # not contiguous
|
||||
torch.randn(2, 128, 4096, dtype=torch.bfloat16), # not 2D
|
||||
],
|
||||
)
|
||||
def test_maybe_pad_weight_avoid_cache_aliasing_skips(monkeypatch, weight):
|
||||
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
|
||||
|
||||
assert (
|
||||
utils.maybe_pad_weight_avoid_cache_aliasing(weight).data_ptr()
|
||||
== weight.data_ptr()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_rocm_unquantized_gemm_noncontiguous_activation_real_kernel(monkeypatch, dtype):
|
||||
x = torch.randn(64, 4, device="cuda", dtype=dtype).t()
|
||||
weight = torch.randn(128, 64, device="cuda", dtype=dtype)
|
||||
assert x.stride() == (1, 4)
|
||||
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
original_wvsplitk = utils.ops.wvSplitK
|
||||
wvsplitk_mock = MagicMock(side_effect=original_wvsplitk)
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_called_once()
|
||||
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _on_gfx11(), reason="cache-aliasing padding only applies on gfx11"
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_rocm_unquantized_gemm_padded_weight_real_kernel(monkeypatch, dtype):
|
||||
x = torch.randn(4, 4096, device="cuda", dtype=dtype)
|
||||
weight = _row_stride_padded(torch.randn(128, 4096, device="cuda", dtype=dtype), 64)
|
||||
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
original_wvsplitk = utils.ops.wvSplitK
|
||||
wvsplitk_mock = MagicMock(side_effect=original_wvsplitk)
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_called_once()
|
||||
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [1, 128])
|
||||
def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch, m):
|
||||
# wvSplitK skinny GEMM handles n in [1, 5] (see PR #40687); n > 5 must
|
||||
# fall back to torch.nn.functional.linear.
|
||||
x = torch.randn(6, 64, dtype=torch.float16)
|
||||
weight = torch.randn(m, 64, dtype=torch.float16)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_not_called()
|
||||
llmm1_mock.assert_not_called()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
def test_rocm_unquantized_gemm_gfx950_wvsplitkrc_path(monkeypatch):
|
||||
x = torch.randn(1024, 16, dtype=torch.float16).t()
|
||||
weight = torch.randn(256, 1024, dtype=torch.float16)
|
||||
assert x.stride() == (1, 16)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: True)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitkrc_mock = MagicMock(side_effect=lambda x_view, w, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitKrc", wvsplitkrc_mock)
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitkrc_mock.assert_called_once()
|
||||
wvsplitk_mock.assert_not_called()
|
||||
x_view = wvsplitkrc_mock.call_args.args[0]
|
||||
assert x_view.is_contiguous()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
Reference in New Issue
Block a user