[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -1,38 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.layers.activation import GeluAndMul, SiluAndMul
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cpu()
|
||||
or current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC,
|
||||
reason="Native CPU activations require a non-PowerPC CPU platform",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"activation_factory,reference",
|
||||
[
|
||||
(SiluAndMul, nn.SiLU()),
|
||||
(GeluAndMul, nn.GELU()),
|
||||
(partial(GeluAndMul, approximate="tanh"), nn.GELU(approximate="tanh")),
|
||||
],
|
||||
ids=["silu", "gelu", "gelu_tanh"],
|
||||
)
|
||||
def test_cpu_act_and_mul_without_compiled_ops(
|
||||
default_vllm_config, monkeypatch, activation_factory, reference
|
||||
):
|
||||
"""CPU activations must construct and run without compiled wheel operators."""
|
||||
default_vllm_config.compilation_config.custom_ops = ["all"]
|
||||
monkeypatch.setattr(torch.ops, "_C", object())
|
||||
x = torch.tensor([[-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device="cpu")
|
||||
|
||||
layer = activation_factory()
|
||||
|
||||
torch.testing.assert_close(layer(x), reference(x[:, :4]) * x[:, 4:])
|
||||
@@ -1,339 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Tests for FusedMMInputNorm and the fused affine transform kernel."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.compilation import CompilationMode
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import (
|
||||
FusedMMInputNorm,
|
||||
IdentityInputNorm,
|
||||
NormParams,
|
||||
fused_mm_input_norm_triton,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
# Module-level tests run on any platform: the CPU platform exercises
|
||||
# forward_native, accelerators exercise the fused kernel. Only the raw
|
||||
# kernel tests require an accelerator.
|
||||
_DEVICE = torch.device(current_platform.device_type)
|
||||
|
||||
requires_accelerator = pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason="fused Triton kernel requires a CUDA/XPU accelerator",
|
||||
)
|
||||
requires_triton = pytest.mark.skipif(not HAS_TRITON, reason="requires Triton")
|
||||
requires_vllm_config = pytest.mark.usefixtures("default_vllm_config")
|
||||
|
||||
|
||||
def _reference_input_norm(
|
||||
pixel_values: torch.Tensor,
|
||||
image_mean: list[float],
|
||||
image_std: list[float],
|
||||
rescale_factor: float,
|
||||
channel: int,
|
||||
out_dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
"""Straightforward per-channel affine: (x * rescale - mean) / std."""
|
||||
patches, size = pixel_values.shape
|
||||
patch_size = size // channel
|
||||
mean = torch.tensor(
|
||||
image_mean, dtype=torch.float32, device=pixel_values.device
|
||||
).view(1, channel, 1)
|
||||
std = torch.tensor(image_std, dtype=torch.float32, device=pixel_values.device).view(
|
||||
1, channel, 1
|
||||
)
|
||||
x = pixel_values.to(torch.float32).reshape(patches, channel, patch_size)
|
||||
x = (x * rescale_factor - mean) / std
|
||||
return x.view(patches, size).to(out_dtype)
|
||||
|
||||
|
||||
# CLIP-style RGB normalization constants reused across tests.
|
||||
_RGB_MEAN = [0.48145466, 0.4578275, 0.40821073]
|
||||
_RGB_STD = [0.26862954, 0.26130258, 0.27577711]
|
||||
_RGB_RESCALE = 1.0 / 255.0
|
||||
|
||||
|
||||
def _check_against_reference(
|
||||
pixel_values: torch.Tensor,
|
||||
out_dtype: torch.dtype,
|
||||
channel: int = 3,
|
||||
image_mean: list[float] | None = None,
|
||||
image_std: list[float] | None = None,
|
||||
rescale_factor: float = _RGB_RESCALE,
|
||||
**close_kwargs,
|
||||
):
|
||||
"""Run ``FusedMMInputNorm`` on ``pixel_values`` and compare against the
|
||||
eager per-channel affine reference. Uses the RGB constants by default."""
|
||||
if image_mean is None:
|
||||
image_mean = list(_RGB_MEAN)
|
||||
if image_std is None:
|
||||
image_std = list(_RGB_STD)
|
||||
norm = FusedMMInputNorm(
|
||||
image_mean=image_mean,
|
||||
image_std=image_std,
|
||||
rescale_factor=rescale_factor,
|
||||
channel=channel,
|
||||
).to(_DEVICE)
|
||||
expected = _reference_input_norm(
|
||||
pixel_values, image_mean, image_std, rescale_factor, channel, out_dtype
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
norm(pixel_values, visual_dtype=out_dtype), expected, **close_kwargs
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Module-level behavior
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormModule:
|
||||
def test_matches_reference_above_cudnn_grid_limit(self):
|
||||
"""num_patches above the old cuDNN batch-norm grid limit (~65535)
|
||||
used to raise CUDNN_STATUS_INTERNAL_ERROR."""
|
||||
set_random_seed(0)
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (70000, 3 * 196), dtype=torch.float32, device=_DEVICE
|
||||
)
|
||||
_check_against_reference(pixel_values, torch.float32)
|
||||
|
||||
def test_identity_passthrough(self):
|
||||
norm = IdentityInputNorm()
|
||||
|
||||
pixel_values = torch.randn(8, 3 * 196, dtype=torch.float32, device=_DEVICE)
|
||||
out = norm(pixel_values, visual_dtype=torch.bfloat16)
|
||||
torch.testing.assert_close(out, pixel_values.to(torch.bfloat16))
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# dtype coverage
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormDtypes:
|
||||
@pytest.mark.parametrize(
|
||||
"in_dtype,out_dtype",
|
||||
[
|
||||
(torch.float32, torch.float32),
|
||||
(torch.float32, torch.bfloat16),
|
||||
(torch.float32, torch.float16),
|
||||
(torch.bfloat16, torch.bfloat16),
|
||||
(torch.bfloat16, torch.float32),
|
||||
(torch.float16, torch.float16),
|
||||
(torch.uint8, torch.bfloat16),
|
||||
(torch.uint8, torch.float32),
|
||||
],
|
||||
)
|
||||
def test_dtype_combinations(self, in_dtype: torch.dtype, out_dtype: torch.dtype):
|
||||
set_random_seed(0)
|
||||
if in_dtype == torch.uint8:
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (32, 3 * 16), dtype=torch.uint8, device=_DEVICE
|
||||
)
|
||||
else:
|
||||
pixel_values = torch.rand(
|
||||
(32, 3 * 16), dtype=torch.float32, device=_DEVICE
|
||||
).to(in_dtype)
|
||||
|
||||
_check_against_reference(
|
||||
pixel_values,
|
||||
out_dtype,
|
||||
image_mean=[0.5, 0.5, 0.5],
|
||||
image_std=[0.5, 0.5, 0.5],
|
||||
atol=1e-2,
|
||||
rtol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Shape / channel coverage
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormShapes:
|
||||
@pytest.mark.parametrize("channel", [1, 3, 4])
|
||||
def test_channel_variants(self, channel: int):
|
||||
set_random_seed(0)
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (16, channel * 64), dtype=torch.float32, device=_DEVICE
|
||||
)
|
||||
_check_against_reference(
|
||||
pixel_values,
|
||||
torch.float32,
|
||||
channel=channel,
|
||||
image_mean=[0.5] * channel,
|
||||
image_std=[0.25] * channel,
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("patch_size", [1, 2048])
|
||||
def test_block_boundaries(self, patch_size: int):
|
||||
"""Masked-tail (12 elements) and exact-multiple (24 blocks) paths."""
|
||||
set_random_seed(0)
|
||||
pixel_values = torch.randint(
|
||||
0, 256, (4, 3 * patch_size), dtype=torch.float32, device=_DEVICE
|
||||
)
|
||||
_check_against_reference(pixel_values, torch.float32)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Input handling: non-contiguous inputs
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormInputHandling:
|
||||
def test_non_contiguous_input_matches_reference(self):
|
||||
set_random_seed(0)
|
||||
base = torch.randint(
|
||||
0, 256, (8, 3 * 32, 2), dtype=torch.float32, device=_DEVICE
|
||||
)
|
||||
non_contig = base[..., 0]
|
||||
assert not non_contig.is_contiguous()
|
||||
|
||||
_check_against_reference(non_contig, torch.float32)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Raw kernel entry point
|
||||
# ===========================================================================
|
||||
@requires_accelerator
|
||||
@requires_triton
|
||||
class TestFusedMMInputNormKernel:
|
||||
@pytest.mark.parametrize(
|
||||
"N, C, L",
|
||||
[
|
||||
(1, 1, 1),
|
||||
(1, 3, 1),
|
||||
(3, 3, 7), # tile straddles channel boundaries
|
||||
(2, 8, 63),
|
||||
(4, 3, 1000),
|
||||
],
|
||||
)
|
||||
def test_channel_boundary_crossing(self, N: int, C: int, L: int):
|
||||
"""The 1D kernel recovers ``c = (offs // L) % C`` per lane, so a tile
|
||||
can straddle a channel boundary and gather weight/bias per lane.
|
||||
These shapes force that path by keeping ``L`` well below the block."""
|
||||
set_random_seed(0)
|
||||
x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE)
|
||||
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
|
||||
out = torch.empty_like(x)
|
||||
fused_mm_input_norm_triton(x, out, w, b)
|
||||
|
||||
expected = x * w.view(1, C, 1) + b.view(1, C, 1)
|
||||
torch.testing.assert_close(out, expected)
|
||||
|
||||
def test_non_contiguous_inputs_materialized(self):
|
||||
"""Read-only tensors are made contiguous internally."""
|
||||
N, C, L = 3, 3, 100
|
||||
set_random_seed(0)
|
||||
base = torch.randn(N, C, L, 2, dtype=torch.float32, device=_DEVICE)
|
||||
x = base[..., 0]
|
||||
assert not x.is_contiguous()
|
||||
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
|
||||
out = torch.empty(N, C, L, dtype=torch.float32, device=_DEVICE)
|
||||
fused_mm_input_norm_triton(x, out, w, b)
|
||||
|
||||
expected = x * w.view(1, C, 1) + b.view(1, C, 1)
|
||||
torch.testing.assert_close(out, expected)
|
||||
|
||||
def test_rejects_mismatched_output_shape(self):
|
||||
"""``outputs`` must be shaped exactly like ``inputs``."""
|
||||
N, C, L = 3, 3, 100
|
||||
x = torch.randn(N, C, L, dtype=torch.float32, device=_DEVICE)
|
||||
w = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
b = torch.randn(C, dtype=torch.float32, device=_DEVICE)
|
||||
|
||||
for out_shape in ((N, C + 1, L), (N, C, L + 1), (N + 2, C, L)):
|
||||
with pytest.raises(AssertionError):
|
||||
fused_mm_input_norm_triton(
|
||||
x,
|
||||
torch.empty(*out_shape, dtype=torch.float32, device=_DEVICE),
|
||||
w,
|
||||
b,
|
||||
)
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Construction / configuration
|
||||
# ===========================================================================
|
||||
@requires_vllm_config
|
||||
class TestFusedMMInputNormConstruction:
|
||||
"""Weight/bias buffer semantics at init time."""
|
||||
|
||||
def test_input_norm_is_always_enabled(self):
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
compile_mm_encoder=True,
|
||||
custom_ops=["none"],
|
||||
)
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
FusedMMInputNorm(_RGB_MEAN, _RGB_STD, _RGB_RESCALE)
|
||||
assert (
|
||||
config.compilation_config.enabled_custom_ops["fused_mm_input_norm"] == 1
|
||||
)
|
||||
assert not config.compilation_config.disabled_custom_ops[
|
||||
"fused_mm_input_norm"
|
||||
]
|
||||
|
||||
def test_identity_config_buffers(self):
|
||||
"""Numerically identity parameters still build plain weight/bias
|
||||
buffers; the identity shortcut lives in ``from_model_config``."""
|
||||
norm = FusedMMInputNorm(
|
||||
image_mean=[0.0, 0.0, 0.0],
|
||||
image_std=[1.0, 1.0, 1.0],
|
||||
rescale_factor=1.0,
|
||||
)
|
||||
torch.testing.assert_close(norm.weight, torch.ones(3))
|
||||
torch.testing.assert_close(norm.bias, torch.zeros(3))
|
||||
|
||||
def test_non_identity_buffers(self):
|
||||
channel = 3
|
||||
norm = FusedMMInputNorm(
|
||||
image_mean=_RGB_MEAN,
|
||||
image_std=_RGB_STD,
|
||||
rescale_factor=_RGB_RESCALE,
|
||||
channel=channel,
|
||||
)
|
||||
assert norm.weight.shape == (channel,)
|
||||
assert norm.bias.shape == (channel,)
|
||||
|
||||
mean = torch.tensor(_RGB_MEAN, dtype=torch.float32)
|
||||
std = torch.tensor(_RGB_STD, dtype=torch.float32)
|
||||
torch.testing.assert_close(norm.weight, _RGB_RESCALE / std)
|
||||
torch.testing.assert_close(norm.bias, -mean / std)
|
||||
|
||||
def test_is_identity_params(self):
|
||||
"""The numeric-identity check mirrors the old allclose detection."""
|
||||
assert NormParams([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0).is_identity
|
||||
assert not NormParams(_RGB_MEAN, _RGB_STD, _RGB_RESCALE).is_identity
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("image_mean", "image_std", "rescale_factor"),
|
||||
[
|
||||
([0.0, 0.0, 0.0], [1.0, 1.0, 1.0], 1.0),
|
||||
([0.5, 0.5, 0.5], [0.25, 0.25, 0.25], 1 / 255),
|
||||
],
|
||||
)
|
||||
def test_buffers_initialized_on_default_device(
|
||||
self,
|
||||
image_mean: list[float],
|
||||
image_std: list[float],
|
||||
rescale_factor: float,
|
||||
):
|
||||
"""Buffers are built on CPU first, then moved to the caller's
|
||||
default device, so init never touches the accelerator."""
|
||||
# The meta device gives the CPU-only test shard the same non-CPU
|
||||
# default-device semantics without requiring a CUDA build.
|
||||
default_device = "cuda" if torch.cuda.is_available() else "meta"
|
||||
with torch.device(default_device):
|
||||
norm = FusedMMInputNorm(image_mean, image_std, rescale_factor)
|
||||
|
||||
assert norm.weight.device.type == default_device
|
||||
assert norm.bias.device.type == default_device
|
||||
@@ -1,234 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for vllm.model_executor.layers.pooler.activations."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.layers.pooler.activations import (
|
||||
LambdaPoolerActivation,
|
||||
PoolerClassify,
|
||||
PoolerIdentity,
|
||||
PoolerMultiLabelClassify,
|
||||
PoolerNormalize,
|
||||
get_act_fn,
|
||||
resolve_classifier_act_fn,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PoolerIdentity
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestPoolerIdentity:
|
||||
def test_returns_input_unchanged(self):
|
||||
pooler = PoolerIdentity()
|
||||
x = torch.randn(4, 128)
|
||||
out = pooler(x)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_forward_list(self):
|
||||
pooler = PoolerIdentity()
|
||||
tensors = [torch.randn(128), torch.randn(256)]
|
||||
out = pooler(tensors)
|
||||
assert len(out) == 2
|
||||
for orig, result in zip(tensors, out):
|
||||
assert torch.equal(orig, result)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PoolerNormalize
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestPoolerNormalize:
|
||||
def test_output_has_unit_norm(self):
|
||||
pooler = PoolerNormalize()
|
||||
x = torch.randn(4, 128)
|
||||
out = pooler(x)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(4), atol=1e-5)
|
||||
|
||||
def test_single_vector(self):
|
||||
pooler = PoolerNormalize()
|
||||
x = torch.randn(1, 64)
|
||||
out = pooler(x)
|
||||
norm = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norm, torch.ones(1), atol=1e-5)
|
||||
|
||||
def test_forward_list(self):
|
||||
pooler = PoolerNormalize()
|
||||
tensors = [torch.randn(1, 64), torch.randn(1, 128)]
|
||||
out = pooler(tensors)
|
||||
for t in out:
|
||||
norm = torch.linalg.norm(t, dim=-1)
|
||||
assert torch.allclose(norm, torch.ones(1), atol=1e-5)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PoolerMultiLabelClassify
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestPoolerMultiLabelClassify:
|
||||
def test_output_in_zero_one(self):
|
||||
pooler = PoolerMultiLabelClassify()
|
||||
x = torch.randn(4, 10)
|
||||
out = pooler(x)
|
||||
assert (out >= 0).all() and (out <= 1).all()
|
||||
|
||||
def test_large_positive_maps_near_one(self):
|
||||
pooler = PoolerMultiLabelClassify()
|
||||
x = torch.full((1, 3), 100.0)
|
||||
out = pooler(x)
|
||||
assert torch.allclose(out, torch.ones(1, 3), atol=1e-4)
|
||||
|
||||
def test_large_negative_maps_near_zero(self):
|
||||
pooler = PoolerMultiLabelClassify()
|
||||
x = torch.full((1, 3), -100.0)
|
||||
out = pooler(x)
|
||||
assert torch.allclose(out, torch.zeros(1, 3), atol=1e-4)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# PoolerClassify
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestPoolerClassify:
|
||||
def test_infers_from_shape_when_num_labels_none(self):
|
||||
pooler = PoolerClassify(num_labels=None)
|
||||
assert pooler.num_labels is None
|
||||
x = torch.randn(2, 5)
|
||||
out = pooler(x)
|
||||
sums = out.sum(dim=-1)
|
||||
assert torch.allclose(sums, torch.ones(2), atol=1e-5)
|
||||
|
||||
def test_sigmoid_when_num_labels_lt_2(self):
|
||||
pooler = PoolerClassify(num_labels=1)
|
||||
x = torch.zeros(1, 1)
|
||||
out = pooler(x)
|
||||
assert torch.allclose(out, torch.tensor([[0.5]]), atol=1e-5)
|
||||
|
||||
def test_num_labels_zero_uses_sigmoid(self):
|
||||
pooler = PoolerClassify(num_labels=0)
|
||||
assert pooler.num_labels == 0
|
||||
x = torch.zeros(1, 3)
|
||||
out = pooler(x)
|
||||
assert torch.allclose(out, torch.full((1, 3), 0.5), atol=1e-5)
|
||||
|
||||
def test_num_labels_ge_2_uses_softmax(self):
|
||||
pooler = PoolerClassify(num_labels=4)
|
||||
assert pooler.num_labels == 4
|
||||
x = torch.randn(2, 4)
|
||||
out = pooler(x)
|
||||
sums = out.sum(dim=-1)
|
||||
assert torch.allclose(sums, torch.ones(2), atol=1e-5)
|
||||
|
||||
def test_default_num_labels_is_none(self):
|
||||
pooler = PoolerClassify()
|
||||
assert pooler.num_labels is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LambdaPoolerActivation
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestLambdaPoolerActivation:
|
||||
def test_applies_custom_fn(self):
|
||||
pooler = LambdaPoolerActivation(nn.ReLU())
|
||||
x = torch.tensor([[-1.0, 2.0, -3.0]])
|
||||
out = pooler(x)
|
||||
expected = torch.tensor([[0.0, 2.0, 0.0]])
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
def test_forward_list(self):
|
||||
pooler = LambdaPoolerActivation(nn.ReLU())
|
||||
tensors = [torch.tensor([-1.0, 2.0]), torch.tensor([3.0, -4.0])]
|
||||
out = pooler(tensors)
|
||||
assert torch.equal(out[0], torch.tensor([0.0, 2.0]))
|
||||
assert torch.equal(out[1], torch.tensor([3.0, 0.0]))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_act_fn factory
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestGetActFn:
|
||||
@staticmethod
|
||||
def _make_config(**kwargs):
|
||||
return SimpleNamespace(**kwargs)
|
||||
|
||||
def test_regression(self):
|
||||
cfg = self._make_config(problem_type="regression")
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerIdentity)
|
||||
|
||||
def test_single_label_classification(self):
|
||||
cfg = self._make_config(
|
||||
problem_type="single_label_classification", num_labels=3
|
||||
)
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerClassify)
|
||||
assert result.num_labels == 3
|
||||
|
||||
def test_multi_label_classification(self):
|
||||
cfg = self._make_config(problem_type="multi_label_classification")
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerMultiLabelClassify)
|
||||
|
||||
def test_sentence_transformers_activation(self):
|
||||
cfg = self._make_config(
|
||||
problem_type="",
|
||||
sentence_transformers={
|
||||
"activation_fn": "torch.nn.modules.activation.Sigmoid"
|
||||
},
|
||||
)
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerClassify)
|
||||
|
||||
def test_sbert_activation(self):
|
||||
cfg = self._make_config(
|
||||
problem_type="",
|
||||
sbert_ce_default_activation_function=(
|
||||
"torch.nn.modules.activation.Sigmoid"
|
||||
),
|
||||
)
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerClassify)
|
||||
|
||||
def test_default_fallback(self):
|
||||
cfg = self._make_config(problem_type="")
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerClassify)
|
||||
|
||||
def test_sentence_transformers_takes_priority(self):
|
||||
cfg = self._make_config(
|
||||
problem_type="",
|
||||
sentence_transformers={"activation_fn": "torch.nn.modules.linear.Identity"},
|
||||
sbert_ce_default_activation_function=(
|
||||
"torch.nn.modules.activation.Sigmoid"
|
||||
),
|
||||
)
|
||||
result = get_act_fn(cfg)
|
||||
assert isinstance(result, PoolerIdentity)
|
||||
|
||||
def test_rejects_non_torch_activation(self):
|
||||
cfg = self._make_config(
|
||||
problem_type="",
|
||||
sentence_transformers={"activation_fn": "os.system"},
|
||||
)
|
||||
with pytest.raises(ValueError, match="restricted"):
|
||||
get_act_fn(cfg)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# resolve_classifier_act_fn
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestResolveClassifierActFn:
|
||||
def test_delegates_to_get_act_fn_when_none(self):
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=SimpleNamespace(num_labels=3, problem_type="")
|
||||
)
|
||||
result = resolve_classifier_act_fn(model_config, act_fn=None)
|
||||
assert isinstance(result, PoolerClassify)
|
||||
assert result.num_labels == 3
|
||||
|
||||
def test_passes_through_provided_act_fn(self):
|
||||
custom = PoolerIdentity()
|
||||
result = resolve_classifier_act_fn(None, act_fn=custom)
|
||||
assert result is custom
|
||||
@@ -1,490 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for sequence and token pooler head classes."""
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
|
||||
from vllm.model_executor.layers.pooler.seqwise.heads import (
|
||||
ClassifierPoolerHead,
|
||||
EmbeddingPoolerHead,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.tokwise.heads import (
|
||||
TokenClassifierPoolerHead,
|
||||
TokenEmbeddingPoolerHead,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
|
||||
_HIDDEN = 16
|
||||
_BATCH = 3
|
||||
|
||||
|
||||
def _make_params(
|
||||
n: int,
|
||||
*,
|
||||
task: PoolingTask = "embed",
|
||||
dimensions: int | None = None,
|
||||
use_activation: bool | None = None,
|
||||
) -> list[PoolingParams]:
|
||||
return [
|
||||
PoolingParams(task=task, dimensions=dimensions, use_activation=use_activation)
|
||||
for _ in range(n)
|
||||
]
|
||||
|
||||
|
||||
def _make_metadata(pooling_params: list[PoolingParams]) -> PoolingMetadata:
|
||||
n = len(pooling_params)
|
||||
return PoolingMetadata(
|
||||
prompt_lens=torch.ones(n, dtype=torch.long),
|
||||
prompt_token_ids=None,
|
||||
prompt_token_ids_cpu=None,
|
||||
pooling_params=pooling_params,
|
||||
pooling_states=[PoolingStates() for _ in range(n)],
|
||||
)
|
||||
|
||||
|
||||
def _linear(in_f: int, out_f: int) -> nn.Linear:
|
||||
torch.manual_seed(42)
|
||||
return nn.Linear(in_f, out_f, bias=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# EmbeddingPoolerHead
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestEmbeddingPoolerHead:
|
||||
def test_supported_tasks(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
assert head.get_supported_tasks() == {"embed"}
|
||||
|
||||
def test_passthrough(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH))
|
||||
out = head(x, meta)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_head_dtype(self):
|
||||
head = EmbeddingPoolerHead(head_dtype=torch.float16)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH))
|
||||
out = head(x, meta)
|
||||
assert out.dtype == torch.float16
|
||||
|
||||
def test_projector(self):
|
||||
proj = _linear(_HIDDEN, 8)
|
||||
head = EmbeddingPoolerHead(projector=proj)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH))
|
||||
out = head(x, meta)
|
||||
assert out.shape == (_BATCH, 8)
|
||||
assert torch.allclose(out, proj(x))
|
||||
|
||||
def test_matryoshka_uniform(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, dimensions=4)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert out.shape == (_BATCH, 4)
|
||||
assert torch.equal(out, x[..., :4])
|
||||
|
||||
def test_matryoshka_mixed(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
x = torch.randn(2, _HIDDEN)
|
||||
params = [
|
||||
PoolingParams(task="embed", dimensions=4),
|
||||
PoolingParams(task="embed", dimensions=8),
|
||||
]
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert isinstance(out, list)
|
||||
assert len(out) == 2
|
||||
assert out[0].shape[-1] == 4
|
||||
assert out[1].shape[-1] == 8
|
||||
|
||||
def test_matryoshka_mixed_with_none(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
x = torch.randn(2, _HIDDEN)
|
||||
params = [
|
||||
PoolingParams(task="embed", dimensions=4),
|
||||
PoolingParams(task="embed", dimensions=None),
|
||||
]
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert isinstance(out, list)
|
||||
assert out[0].shape[-1] == 4
|
||||
assert torch.equal(out[1], x[1])
|
||||
|
||||
def test_activation_uniform_true(self):
|
||||
head = EmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, use_activation=True)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
|
||||
|
||||
def test_activation_uniform_false(self):
|
||||
head = EmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, use_activation=False)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_activation_mixed_flags(self):
|
||||
head = EmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(2, _HIDDEN)
|
||||
params = [
|
||||
PoolingParams(task="embed", use_activation=True),
|
||||
PoolingParams(task="embed", use_activation=False),
|
||||
]
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert isinstance(out, list)
|
||||
norm_0 = torch.linalg.norm(out[0], dim=-1)
|
||||
assert torch.allclose(norm_0, torch.ones(1), atol=1e-5)
|
||||
assert torch.equal(out[1], x[1])
|
||||
|
||||
def test_list_input_gets_stacked(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
tensors = [torch.randn(_HIDDEN) for _ in range(_BATCH)]
|
||||
meta = _make_metadata(_make_params(_BATCH))
|
||||
out = head(tensors, meta)
|
||||
assert out.shape == (_BATCH, _HIDDEN)
|
||||
expected = torch.stack(tensors)
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
def test_projector_then_matryoshka(self):
|
||||
proj = _linear(_HIDDEN, 8)
|
||||
head = EmbeddingPoolerHead(projector=proj)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, dimensions=4)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert out.shape == (_BATCH, 4)
|
||||
assert torch.equal(out, proj(x)[..., :4])
|
||||
|
||||
def test_matryoshka_then_activation(self):
|
||||
head = EmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, dimensions=4, use_activation=True)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert out.shape == (_BATCH, 4)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
|
||||
|
||||
def test_empty_batch(self):
|
||||
head = EmbeddingPoolerHead()
|
||||
x = torch.randn(0, _HIDDEN)
|
||||
meta = _make_metadata([])
|
||||
out = head(x, meta)
|
||||
assert out.shape == (0, _HIDDEN)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ClassifierPoolerHead
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestClassifierPoolerHead:
|
||||
def test_supported_tasks(self):
|
||||
head = ClassifierPoolerHead()
|
||||
assert head.get_supported_tasks() == {"classify"}
|
||||
|
||||
def test_passthrough(self):
|
||||
head = ClassifierPoolerHead()
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_head_dtype(self):
|
||||
head = ClassifierPoolerHead(head_dtype=torch.float16)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
assert out.dtype == torch.float16
|
||||
|
||||
def test_classifier(self):
|
||||
clf = _linear(_HIDDEN, 3)
|
||||
head = ClassifierPoolerHead(classifier=clf)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
assert out.shape == (_BATCH, 3)
|
||||
assert torch.allclose(out, clf(x))
|
||||
|
||||
def test_logit_mean(self):
|
||||
head = ClassifierPoolerHead(logit_mean=2.0)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
assert torch.allclose(out, x - 2.0)
|
||||
|
||||
def test_logit_sigma(self):
|
||||
head = ClassifierPoolerHead(logit_sigma=0.5)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
assert torch.allclose(out, x / 0.5)
|
||||
|
||||
def test_platt_scaling_combined(self):
|
||||
head = ClassifierPoolerHead(logit_mean=1.0, logit_sigma=2.0)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
assert torch.allclose(out, (x - 1.0) / 2.0)
|
||||
|
||||
def test_activation_uniform_true(self):
|
||||
head = ClassifierPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, task="classify", use_activation=True)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
|
||||
|
||||
def test_activation_uniform_false(self):
|
||||
head = ClassifierPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
params = _make_params(_BATCH, task="classify", use_activation=False)
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_activation_mixed_flags(self):
|
||||
head = ClassifierPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(2, _HIDDEN)
|
||||
params = [
|
||||
PoolingParams(task="classify", use_activation=True),
|
||||
PoolingParams(task="classify", use_activation=False),
|
||||
]
|
||||
meta = _make_metadata(params)
|
||||
out = head(x, meta)
|
||||
assert isinstance(out, list)
|
||||
norm_0 = torch.linalg.norm(out[0], dim=-1)
|
||||
assert torch.allclose(norm_0, torch.ones(1), atol=1e-5)
|
||||
assert torch.equal(out[1], x[1])
|
||||
|
||||
def test_list_input_gets_stacked(self):
|
||||
head = ClassifierPoolerHead()
|
||||
tensors = [torch.randn(_HIDDEN) for _ in range(_BATCH)]
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(tensors, meta)
|
||||
assert out.shape == (_BATCH, _HIDDEN)
|
||||
expected = torch.stack(tensors)
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
def test_classifier_then_platt_scaling(self):
|
||||
clf = _linear(_HIDDEN, 3)
|
||||
head = ClassifierPoolerHead(classifier=clf, logit_mean=1.0, logit_sigma=2.0)
|
||||
x = torch.randn(_BATCH, _HIDDEN)
|
||||
meta = _make_metadata(_make_params(_BATCH, task="classify"))
|
||||
out = head(x, meta)
|
||||
expected = (clf(x) - 1.0) / 2.0
|
||||
assert torch.allclose(out, expected)
|
||||
|
||||
def test_empty_batch(self):
|
||||
head = ClassifierPoolerHead()
|
||||
x = torch.randn(0, _HIDDEN)
|
||||
meta = _make_metadata([])
|
||||
out = head(x, meta)
|
||||
assert out.shape == (0, _HIDDEN)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TokenEmbeddingPoolerHead
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestTokenEmbeddingPoolerHead:
|
||||
def test_supported_tasks(self):
|
||||
head = TokenEmbeddingPoolerHead()
|
||||
assert head.get_supported_tasks() == {"token_embed"}
|
||||
|
||||
def test_passthrough(self):
|
||||
head = TokenEmbeddingPoolerHead()
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is x
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_none_chunked_prefill(self):
|
||||
head = TokenEmbeddingPoolerHead()
|
||||
param = PoolingParams(task="token_embed")
|
||||
out = head.forward_chunk(None, param)
|
||||
assert out is None
|
||||
|
||||
def test_head_dtype(self):
|
||||
head = TokenEmbeddingPoolerHead(head_dtype=torch.float16)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.dtype == torch.float16
|
||||
|
||||
def test_projector(self):
|
||||
proj = _linear(_HIDDEN, 8)
|
||||
head = TokenEmbeddingPoolerHead(projector=proj)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 8)
|
||||
assert torch.allclose(out, proj(x))
|
||||
|
||||
def test_matryoshka_truncation(self):
|
||||
head = TokenEmbeddingPoolerHead()
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", dimensions=4)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 4)
|
||||
assert torch.equal(out, x[..., :4])
|
||||
|
||||
def test_activation_true(self):
|
||||
head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", use_activation=True)
|
||||
out = head.forward_chunk(x, param)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
|
||||
|
||||
def test_activation_false(self):
|
||||
head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", use_activation=False)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_projector_then_matryoshka(self):
|
||||
proj = _linear(_HIDDEN, 8)
|
||||
head = TokenEmbeddingPoolerHead(projector=proj)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", dimensions=4)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 4)
|
||||
assert torch.equal(out, proj(x)[..., :4])
|
||||
|
||||
def test_matryoshka_then_activation(self):
|
||||
head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_embed", dimensions=4, use_activation=True)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 4)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
|
||||
|
||||
def test_forward_mixed_batch_chunked_prefill(self):
|
||||
head = TokenEmbeddingPoolerHead()
|
||||
pooled_data = [torch.randn(5, _HIDDEN), None, torch.randn(3, _HIDDEN)]
|
||||
params = _make_params(3, task="token_embed")
|
||||
meta = _make_metadata(params)
|
||||
out = head(pooled_data, meta)
|
||||
assert len(out) == 3
|
||||
assert torch.equal(out[0], pooled_data[0])
|
||||
assert out[1] is None
|
||||
assert torch.equal(out[2], pooled_data[2])
|
||||
|
||||
def test_forward_empty_batch(self):
|
||||
head = TokenEmbeddingPoolerHead()
|
||||
meta = _make_metadata([])
|
||||
out = head([], meta)
|
||||
assert out == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TokenClassifierPoolerHead
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestTokenClassifierPoolerHead:
|
||||
def test_supported_tasks(self):
|
||||
head = TokenClassifierPoolerHead()
|
||||
assert head.get_supported_tasks() == {"token_classify"}
|
||||
|
||||
def test_passthrough(self):
|
||||
head = TokenClassifierPoolerHead()
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_none_chunked_prefill(self):
|
||||
head = TokenClassifierPoolerHead()
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(None, param)
|
||||
assert out is None
|
||||
|
||||
def test_head_dtype(self):
|
||||
head = TokenClassifierPoolerHead(head_dtype=torch.float16)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.dtype == torch.float16
|
||||
|
||||
def test_classifier(self):
|
||||
clf = _linear(_HIDDEN, 3)
|
||||
head = TokenClassifierPoolerHead(classifier=clf)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert out is not None
|
||||
assert out.shape == (5, 3)
|
||||
assert torch.allclose(out, clf(x))
|
||||
|
||||
def test_logit_mean(self):
|
||||
head = TokenClassifierPoolerHead(logit_mean=2.0)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert torch.allclose(out, x - 2.0)
|
||||
|
||||
def test_logit_sigma(self):
|
||||
head = TokenClassifierPoolerHead(logit_sigma=0.5)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert torch.allclose(out, x / 0.5)
|
||||
|
||||
def test_platt_scaling_combined(self):
|
||||
head = TokenClassifierPoolerHead(logit_mean=1.0, logit_sigma=2.0)
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify")
|
||||
out = head.forward_chunk(x, param)
|
||||
assert torch.allclose(out, (x - 1.0) / 2.0)
|
||||
|
||||
def test_activation_true(self):
|
||||
head = TokenClassifierPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify", use_activation=True)
|
||||
out = head.forward_chunk(x, param)
|
||||
norms = torch.linalg.norm(out, dim=-1)
|
||||
assert torch.allclose(norms, torch.ones(5), atol=1e-5)
|
||||
|
||||
def test_activation_false(self):
|
||||
head = TokenClassifierPoolerHead(activation=PoolerNormalize())
|
||||
x = torch.randn(5, _HIDDEN)
|
||||
param = PoolingParams(task="token_classify", use_activation=False)
|
||||
out = head.forward_chunk(x, param)
|
||||
assert torch.equal(out, x)
|
||||
|
||||
def test_forward_mixed_batch_chunked_prefill(self):
|
||||
head = TokenClassifierPoolerHead()
|
||||
pooled_data = [torch.randn(5, _HIDDEN), None, torch.randn(3, _HIDDEN)]
|
||||
params = _make_params(3, task="token_classify")
|
||||
meta = _make_metadata(params)
|
||||
out = head(pooled_data, meta)
|
||||
assert len(out) == 3
|
||||
assert torch.equal(out[0], pooled_data[0])
|
||||
assert out[1] is None
|
||||
assert torch.equal(out[2], pooled_data[2])
|
||||
|
||||
def test_forward_empty_batch(self):
|
||||
head = TokenClassifierPoolerHead()
|
||||
meta = _make_metadata([])
|
||||
out = head([], meta)
|
||||
assert out == []
|
||||
@@ -1,747 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for sequence and token pooling methods and their factories."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import PreTrainedConfig
|
||||
|
||||
from vllm.config import PoolerConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.pooler import PoolingParamsUpdate
|
||||
from vllm.model_executor.layers.pooler.activations import PoolerNormalize
|
||||
from vllm.model_executor.layers.pooler.seqwise.methods import (
|
||||
CLSPool,
|
||||
LastPool,
|
||||
MeanPool,
|
||||
get_seq_pooling_method,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.special import BOSEOSFilter, DispatchPooler
|
||||
from vllm.model_executor.layers.pooler.tokwise.heads import TokenEmbeddingPoolerHead
|
||||
from vllm.model_executor.layers.pooler.tokwise.methods import (
|
||||
AllPool,
|
||||
StepPool,
|
||||
get_tok_pooling_method,
|
||||
)
|
||||
from vllm.model_executor.layers.pooler.tokwise.poolers import TokenPooler
|
||||
from vllm.pooling_params import LateChunkingParams, PoolingParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingCursor, PoolingMetadata, PoolingStates
|
||||
|
||||
_CPU = torch.device("cpu")
|
||||
|
||||
|
||||
def _make_pooling_cursor(
|
||||
prompt_lens: list[int],
|
||||
*,
|
||||
num_scheduled_tokens: list[int] | None = None,
|
||||
seq_lens: list[int] | None = None,
|
||||
device: torch.device = _CPU,
|
||||
) -> PoolingCursor:
|
||||
"""Build a PoolingCursor from a list of per-sequence prompt lengths."""
|
||||
prompt_lens_cpu = torch.tensor(prompt_lens, dtype=torch.long)
|
||||
if num_scheduled_tokens is None:
|
||||
num_scheduled_tokens_cpu = prompt_lens_cpu.clone()
|
||||
else:
|
||||
num_scheduled_tokens_cpu = torch.tensor(num_scheduled_tokens, dtype=torch.long)
|
||||
if seq_lens is None:
|
||||
seq_lens_cpu = prompt_lens_cpu.clone()
|
||||
else:
|
||||
seq_lens_cpu = torch.tensor(seq_lens, dtype=torch.long)
|
||||
|
||||
cumsum = torch.zeros(len(prompt_lens) + 1, dtype=torch.long, device=device)
|
||||
torch.cumsum(num_scheduled_tokens_cpu, dim=0, out=cumsum[1:])
|
||||
|
||||
return PoolingCursor(
|
||||
first_token_indices_gpu=cumsum[: len(prompt_lens)].to(device),
|
||||
last_token_indices_gpu=(cumsum[1:] - 1).to(device),
|
||||
prompt_lens_cpu=prompt_lens_cpu,
|
||||
seq_lens_cpu=seq_lens_cpu,
|
||||
num_scheduled_tokens_cpu=num_scheduled_tokens_cpu,
|
||||
partial_prefill=not torch.equal(prompt_lens_cpu, num_scheduled_tokens_cpu),
|
||||
finished_mask=torch.eq(prompt_lens_cpu, seq_lens_cpu).tolist(),
|
||||
)
|
||||
|
||||
|
||||
def _make_metadata(
|
||||
prompt_lens: list[int],
|
||||
*,
|
||||
tasks: list[PoolingTask] | None = None,
|
||||
token_ids: list[list[int]] | None = None,
|
||||
pooling_params: list[PoolingParams] | None = None,
|
||||
num_scheduled_tokens: list[int] | None = None,
|
||||
seq_lens: list[int] | None = None,
|
||||
device: torch.device = _CPU,
|
||||
) -> PoolingMetadata:
|
||||
"""Build a minimal PoolingMetadata for testing pooling methods."""
|
||||
n_seqs = len(prompt_lens)
|
||||
if tasks is None:
|
||||
tasks = ["embed"] * n_seqs
|
||||
if pooling_params is None:
|
||||
pooling_params = [PoolingParams(task=t) for t in tasks]
|
||||
|
||||
prompt_lens_tensor = torch.tensor(prompt_lens, dtype=torch.long)
|
||||
|
||||
prompt_token_ids_cpu = None
|
||||
prompt_token_ids = None
|
||||
if token_ids is not None:
|
||||
max_len = max(len(t) for t in token_ids)
|
||||
padded = [t + [0] * (max_len - len(t)) for t in token_ids]
|
||||
prompt_token_ids_cpu = torch.tensor(padded, dtype=torch.long)
|
||||
prompt_token_ids = prompt_token_ids_cpu.to(device)
|
||||
|
||||
cursor = _make_pooling_cursor(
|
||||
prompt_lens,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
seq_lens=seq_lens,
|
||||
device=device,
|
||||
)
|
||||
|
||||
pooling_states = [PoolingStates() for _ in range(n_seqs)]
|
||||
|
||||
return PoolingMetadata(
|
||||
prompt_lens=prompt_lens_tensor,
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
prompt_token_ids_cpu=prompt_token_ids_cpu,
|
||||
pooling_params=pooling_params,
|
||||
pooling_states=pooling_states,
|
||||
pooling_cursor=cursor,
|
||||
)
|
||||
|
||||
|
||||
def test_build_pooling_cursor_caches_cpu_status() -> None:
|
||||
metadata = PoolingMetadata(
|
||||
prompt_lens=torch.tensor([3, 4, 2], dtype=torch.long),
|
||||
prompt_token_ids=None,
|
||||
prompt_token_ids_cpu=None,
|
||||
pooling_params=[PoolingParams(task="embed") for _ in range(3)],
|
||||
pooling_states=[PoolingStates() for _ in range(3)],
|
||||
)
|
||||
metadata.build_pooling_cursor(
|
||||
np.array([3, 2, 2], dtype=np.int32),
|
||||
seq_lens_cpu=torch.tensor([3, 2, 1], dtype=torch.long),
|
||||
device=_CPU,
|
||||
)
|
||||
|
||||
cursor = metadata.get_pooling_cursor()
|
||||
|
||||
assert cursor.is_partial_prefill() is True
|
||||
assert cursor.get_finished_mask() == [True, False, False]
|
||||
assert cursor[:1].is_partial_prefill() is False
|
||||
assert cursor[1:2].is_partial_prefill() is True
|
||||
assert cursor[1:].get_finished_mask() == [False, False]
|
||||
|
||||
|
||||
def test_pooling_params_update_combines_token_id_requirements() -> None:
|
||||
update = PoolingParamsUpdate() | PoolingParamsUpdate(requires_token_ids=True)
|
||||
params = PoolingParams(task="embed")
|
||||
|
||||
update.apply(params)
|
||||
|
||||
assert params.requires_token_ids is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# CLSPool
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestCLSPool:
|
||||
def test_extracts_first_token(self):
|
||||
hidden = torch.tensor(
|
||||
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
|
||||
)
|
||||
metadata = _make_metadata([2, 3])
|
||||
pooler = CLSPool()
|
||||
out = pooler(hidden, metadata)
|
||||
expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
def test_rejects_partial_prefill(self):
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
|
||||
metadata = _make_metadata([3], num_scheduled_tokens=[2])
|
||||
pooler = CLSPool()
|
||||
with pytest.raises(RuntimeError, match="partial prefill"):
|
||||
pooler(hidden, metadata)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LastPool
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestLastPool:
|
||||
def test_extracts_last_token(self):
|
||||
hidden = torch.tensor(
|
||||
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
|
||||
)
|
||||
metadata = _make_metadata([2, 3])
|
||||
pooler = LastPool()
|
||||
out = pooler(hidden, metadata)
|
||||
expected = torch.tensor([[3.0, 4.0], [9.0, 10.0]])
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
def test_partial_prefill_extracts_last_scheduled(self):
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
metadata = _make_metadata([4], num_scheduled_tokens=[2])
|
||||
pooler = LastPool()
|
||||
out = pooler(hidden, metadata)
|
||||
expected = torch.tensor([[3.0, 4.0]])
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MeanPool
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestMeanPool:
|
||||
def test_computes_mean(self):
|
||||
hidden = torch.tensor(
|
||||
[[1.0, 2.0], [3.0, 4.0], [10.0, 20.0]], dtype=torch.float32
|
||||
)
|
||||
metadata = _make_metadata([2, 1])
|
||||
pooler = MeanPool()
|
||||
out = pooler(hidden, metadata)
|
||||
expected = torch.tensor([[2.0, 3.0], [10.0, 20.0]], dtype=torch.float32)
|
||||
assert torch.allclose(out, expected, atol=1e-5)
|
||||
|
||||
def test_single_token_is_identity(self):
|
||||
hidden = torch.tensor([[5.0, 10.0]], dtype=torch.float32)
|
||||
metadata = _make_metadata([1])
|
||||
pooler = MeanPool()
|
||||
out = pooler(hidden, metadata)
|
||||
assert torch.allclose(out, hidden, atol=1e-5)
|
||||
|
||||
def test_uniform_values_return_same(self):
|
||||
hidden = torch.full((4, 3), 7.0, dtype=torch.float32)
|
||||
metadata = _make_metadata([4])
|
||||
pooler = MeanPool()
|
||||
out = pooler(hidden, metadata)
|
||||
expected = torch.full((1, 3), 7.0, dtype=torch.float32)
|
||||
assert torch.allclose(out, expected, atol=1e-5)
|
||||
|
||||
def test_multiple_sequences(self):
|
||||
hidden = torch.tensor(
|
||||
[
|
||||
[0.0, 0.0],
|
||||
[2.0, 4.0],
|
||||
[4.0, 8.0],
|
||||
[10.0, 10.0],
|
||||
],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
metadata = _make_metadata([3, 1])
|
||||
pooler = MeanPool()
|
||||
out = pooler(hidden, metadata)
|
||||
expected = torch.tensor([[2.0, 4.0], [10.0, 10.0]], dtype=torch.float32)
|
||||
assert torch.allclose(out, expected, atol=1e-5)
|
||||
|
||||
def test_empty_batch(self):
|
||||
hidden = torch.empty((0, 8), dtype=torch.float32)
|
||||
metadata = _make_metadata([])
|
||||
pooler = MeanPool()
|
||||
out = pooler(hidden, metadata)
|
||||
assert out.shape == (0, 8)
|
||||
|
||||
def test_rejects_partial_prefill(self):
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float32)
|
||||
metadata = _make_metadata([3], num_scheduled_tokens=[2])
|
||||
pooler = MeanPool()
|
||||
with pytest.raises(RuntimeError, match="partial prefill"):
|
||||
pooler(hidden, metadata)
|
||||
|
||||
def test_chunked_accumulation(self):
|
||||
hidden = torch.arange(20, dtype=torch.float32).reshape(5, 4)
|
||||
metadata = _make_metadata([3, 2])
|
||||
pooler = MeanPool()
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.seqwise.methods"
|
||||
"._MEAN_POOL_ACCUMULATION_CHUNK_BYTES",
|
||||
16,
|
||||
):
|
||||
out = pooler(hidden, metadata)
|
||||
expected_seq0 = hidden[:3].float().mean(dim=0, keepdim=True)
|
||||
expected_seq1 = hidden[3:].float().mean(dim=0, keepdim=True)
|
||||
expected = torch.cat([expected_seq0, expected_seq1], dim=0)
|
||||
assert torch.allclose(out, expected, atol=1e-5)
|
||||
|
||||
def test_upcasts_to_float32(self):
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float16)
|
||||
metadata = _make_metadata([2])
|
||||
pooler = MeanPool()
|
||||
out = pooler(hidden, metadata)
|
||||
assert out.dtype == torch.float32
|
||||
expected = torch.tensor([[2.0, 3.0]], dtype=torch.float32)
|
||||
assert torch.allclose(out, expected, atol=1e-2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_seq_pooling_method factory
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestGetSeqPoolingMethod:
|
||||
def test_cls(self):
|
||||
assert isinstance(get_seq_pooling_method("CLS"), CLSPool)
|
||||
|
||||
def test_last(self):
|
||||
assert isinstance(get_seq_pooling_method("LAST"), LastPool)
|
||||
|
||||
def test_mean(self):
|
||||
assert isinstance(get_seq_pooling_method("MEAN"), MeanPool)
|
||||
|
||||
def test_unknown_raises(self):
|
||||
with pytest.raises(NotImplementedError, match="UNKNOWN"):
|
||||
get_seq_pooling_method("UNKNOWN")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# AllPool
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeSchedulerConfig:
|
||||
enable_chunked_prefill: bool = False
|
||||
async_scheduling: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class _FakeVllmConfig:
|
||||
scheduler_config: _FakeSchedulerConfig
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tok_pooling_type", ["ALL", "STEP"])
|
||||
def test_dispatch_seq_cls_honors_token_pooling_type(tok_pooling_type):
|
||||
"""Filter STEP rows before classification; preserve all rows for resolved ALL."""
|
||||
pooler_config = PoolerConfig(
|
||||
seq_pooling_type="LAST", tok_pooling_type=tok_pooling_type
|
||||
)
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=_FakeSchedulerConfig(),
|
||||
model_config=SimpleNamespace(
|
||||
pooler_config=pooler_config,
|
||||
head_dtype=None,
|
||||
hf_config=PreTrainedConfig(num_labels=3),
|
||||
),
|
||||
)
|
||||
classifier = torch.nn.Linear(4, 3)
|
||||
with set_current_vllm_config(config):
|
||||
pooler = DispatchPooler.for_seq_cls(pooler_config, classifier=classifier)
|
||||
|
||||
token_ids = [[2, 99, 3, 99], [4, 99], [5, 6]]
|
||||
params = [
|
||||
PoolingParams(
|
||||
task="token_classify",
|
||||
use_activation=False,
|
||||
step_tag_id=99 if tok_pooling_type == "STEP" else None,
|
||||
)
|
||||
for _ in token_ids
|
||||
]
|
||||
metadata = _make_metadata([4, 2, 2], token_ids=token_ids, pooling_params=params)
|
||||
hidden = torch.arange(32, dtype=torch.float32).reshape(8, 4)
|
||||
outputs = pooler(hidden, metadata)
|
||||
|
||||
expected_rows = (
|
||||
[[1, 3], [5], []]
|
||||
if tok_pooling_type == "STEP"
|
||||
else [[0, 1, 2, 3], [4, 5], [6, 7]]
|
||||
)
|
||||
assert len(outputs) == len(expected_rows)
|
||||
for output, rows in zip(outputs, expected_rows):
|
||||
torch.testing.assert_close(output, classifier(hidden[rows]))
|
||||
assert pooler.get_pooling_updates("token_classify").requires_token_ids == (
|
||||
tok_pooling_type == "STEP"
|
||||
)
|
||||
|
||||
|
||||
class TestAllPool:
|
||||
@staticmethod
|
||||
def _make_all_pool(
|
||||
*, chunked: bool = False, async_scheduling: bool = False
|
||||
) -> AllPool:
|
||||
fake_config = _FakeVllmConfig(
|
||||
scheduler_config=_FakeSchedulerConfig(
|
||||
enable_chunked_prefill=chunked,
|
||||
async_scheduling=async_scheduling,
|
||||
),
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
|
||||
return_value=fake_config,
|
||||
):
|
||||
return AllPool()
|
||||
|
||||
def test_splits_by_sequence(self):
|
||||
pooler = self._make_all_pool()
|
||||
hidden = torch.tensor(
|
||||
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
|
||||
)
|
||||
metadata = _make_metadata([2, 3])
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 2
|
||||
assert torch.equal(out[0], hidden[:2])
|
||||
assert torch.equal(out[1], hidden[2:])
|
||||
|
||||
def test_single_sequence(self):
|
||||
pooler = self._make_all_pool()
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
|
||||
metadata = _make_metadata([3])
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
assert torch.equal(out[0], hidden)
|
||||
|
||||
def test_chunked_prefill_returns_none_for_unfinished(self):
|
||||
pooler = self._make_all_pool(chunked=True)
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
metadata = _make_metadata(
|
||||
[4],
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[2],
|
||||
)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
assert out[0] is None
|
||||
|
||||
def test_chunked_prefill_returns_concat_when_finished(self):
|
||||
pooler = self._make_all_pool(chunked=True)
|
||||
|
||||
chunk1 = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
metadata1 = _make_metadata(
|
||||
[4],
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[2],
|
||||
)
|
||||
out1 = pooler(chunk1, metadata1)
|
||||
assert out1[0] is None
|
||||
expected_chunk1 = chunk1.clone()
|
||||
chunk1.zero_()
|
||||
|
||||
chunk2 = torch.tensor([[5.0, 6.0], [7.0, 8.0]])
|
||||
metadata2 = _make_metadata(
|
||||
[4],
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[4],
|
||||
)
|
||||
metadata2.pooling_states = metadata1.pooling_states
|
||||
out2 = pooler(chunk2, metadata2)
|
||||
assert out2[0] is not None
|
||||
expected = torch.cat([expected_chunk1, chunk2], dim=0)
|
||||
assert torch.equal(out2[0], expected)
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [False, True])
|
||||
def test_chunked_prefill_single_shot_matches_non_chunked(
|
||||
self, async_scheduling: bool
|
||||
):
|
||||
pooler = self._make_all_pool(chunked=True, async_scheduling=async_scheduling)
|
||||
hidden = torch.tensor(
|
||||
[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
|
||||
)
|
||||
expected = [hidden[:2].clone(), hidden[2:].clone()]
|
||||
metadata = _make_metadata([2, 3])
|
||||
out = pooler(hidden, metadata)
|
||||
if async_scheduling:
|
||||
# Finished outputs must own their storage under async scheduling.
|
||||
hidden.zero_()
|
||||
assert len(out) == 2
|
||||
assert torch.equal(out[0], expected[0])
|
||||
assert torch.equal(out[1], expected[1])
|
||||
|
||||
def test_non_chunked_owns_output_under_async_scheduling(self):
|
||||
pooler = self._make_all_pool(async_scheduling=True)
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
|
||||
expected = [hidden[:2].clone(), hidden[2:].clone()]
|
||||
metadata = _make_metadata([2, 1])
|
||||
out = pooler(hidden, metadata)
|
||||
hidden.zero_()
|
||||
assert len(out) == 2
|
||||
assert torch.equal(out[0], expected[0])
|
||||
assert torch.equal(out[1], expected[1])
|
||||
|
||||
def test_chunked_prefill_mixed_finished_unfinished(self):
|
||||
pooler = self._make_all_pool(chunked=True)
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
|
||||
metadata = _make_metadata(
|
||||
[2, 4],
|
||||
num_scheduled_tokens=[2, 1],
|
||||
seq_lens=[2, 1],
|
||||
)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 2
|
||||
assert torch.equal(out[0], hidden[:2])
|
||||
assert out[1] is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# StepPool
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestStepPool:
|
||||
@staticmethod
|
||||
def _make_step_pool(*, chunked: bool = False) -> StepPool:
|
||||
fake_config = _FakeVllmConfig(
|
||||
scheduler_config=_FakeSchedulerConfig(
|
||||
enable_chunked_prefill=chunked,
|
||||
),
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
|
||||
return_value=fake_config,
|
||||
):
|
||||
return StepPool()
|
||||
|
||||
def test_filters_by_step_tag_id(self):
|
||||
pooler = self._make_step_pool()
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
|
||||
token_ids = [[10, 99, 10, 20]]
|
||||
params = [PoolingParams(task="token_classify", step_tag_id=10)]
|
||||
metadata = _make_metadata([4], token_ids=token_ids, pooling_params=params)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
|
||||
assert torch.equal(out[0], expected)
|
||||
|
||||
def test_filters_by_returned_token_ids(self):
|
||||
pooler = self._make_step_pool()
|
||||
hidden = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
|
||||
token_ids = [[10, 20]]
|
||||
params = [PoolingParams(task="token_classify", returned_token_ids=[0, 2])]
|
||||
metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
expected = torch.tensor([[1.0, 3.0], [4.0, 6.0]])
|
||||
assert torch.equal(out[0], expected)
|
||||
|
||||
def test_no_filtering_without_params(self):
|
||||
pooler = self._make_step_pool()
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
token_ids = [[10, 20]]
|
||||
params = [PoolingParams(task="token_classify")]
|
||||
metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
assert torch.equal(out[0], hidden)
|
||||
|
||||
def test_combined_step_tag_and_returned_token_ids(self):
|
||||
pooler = self._make_step_pool()
|
||||
hidden = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]])
|
||||
token_ids = [[99, 10, 99]]
|
||||
params = [
|
||||
PoolingParams(
|
||||
task="token_classify",
|
||||
step_tag_id=10,
|
||||
returned_token_ids=[0, 2],
|
||||
)
|
||||
]
|
||||
metadata = _make_metadata([3], token_ids=token_ids, pooling_params=params)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
expected = torch.tensor([[4.0, 6.0]])
|
||||
assert torch.equal(out[0], expected)
|
||||
|
||||
def test_step_tag_id_no_match_returns_empty(self):
|
||||
pooler = self._make_step_pool()
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
token_ids = [[10, 20]]
|
||||
params = [PoolingParams(task="token_classify", step_tag_id=999)]
|
||||
metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
assert out[0].shape == (0, 2)
|
||||
|
||||
def test_chunked_prefill_propagates_none_for_unfinished(self):
|
||||
pooler = self._make_step_pool(chunked=True)
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
|
||||
token_ids = [[10, 20, 30, 40]]
|
||||
params = [PoolingParams(task="token_classify", step_tag_id=10)]
|
||||
metadata = _make_metadata(
|
||||
[4],
|
||||
token_ids=token_ids,
|
||||
pooling_params=params,
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[2],
|
||||
)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
assert out[0] is None
|
||||
|
||||
def test_chunked_prefill_filters_when_finished(self):
|
||||
pooler = self._make_step_pool(chunked=True)
|
||||
hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
|
||||
token_ids = [[10, 99, 10, 20]]
|
||||
params = [PoolingParams(task="token_classify", step_tag_id=10)]
|
||||
metadata = _make_metadata([4], token_ids=token_ids, pooling_params=params)
|
||||
out = pooler(hidden, metadata)
|
||||
assert len(out) == 1
|
||||
expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
|
||||
assert torch.equal(out[0], expected)
|
||||
|
||||
def test_requires_token_ids_update(self):
|
||||
pooler = self._make_step_pool()
|
||||
update = pooler.get_pooling_updates("token_classify")
|
||||
assert update.requires_token_ids is True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# get_tok_pooling_method factory
|
||||
# ---------------------------------------------------------------------------
|
||||
class TestGetTokPoolingMethod:
|
||||
def test_all(self):
|
||||
fake_config = _FakeVllmConfig(
|
||||
scheduler_config=_FakeSchedulerConfig(
|
||||
enable_chunked_prefill=False,
|
||||
),
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
|
||||
return_value=fake_config,
|
||||
):
|
||||
assert isinstance(get_tok_pooling_method("ALL"), AllPool)
|
||||
|
||||
def test_step(self):
|
||||
fake_config = _FakeVllmConfig(
|
||||
scheduler_config=_FakeSchedulerConfig(
|
||||
enable_chunked_prefill=False,
|
||||
),
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
|
||||
return_value=fake_config,
|
||||
):
|
||||
assert isinstance(get_tok_pooling_method("STEP"), StepPool)
|
||||
|
||||
def test_unknown_raises(self):
|
||||
with pytest.raises(NotImplementedError, match="UNKNOWN"):
|
||||
get_tok_pooling_method("UNKNOWN")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("chunk_size", "dtype", "head_dtype"),
|
||||
[
|
||||
(1, torch.float32, None),
|
||||
(2, torch.float16, None),
|
||||
(2, torch.bfloat16, torch.float32),
|
||||
(7, torch.float32, None),
|
||||
(10, torch.float32, None),
|
||||
(2**63 - 1, torch.float32, None),
|
||||
],
|
||||
)
|
||||
def test_late_chunk_pool_matches_fp32_means_before_projection_and_normalization(
|
||||
chunk_size, dtype, head_dtype
|
||||
):
|
||||
# Unequal norms expose the incorrect normalize-then-average implementation.
|
||||
hidden = (torch.arange(28).reshape(7, 4) - 10).to(dtype)
|
||||
projector = torch.nn.Linear(4, 3, bias=True).to(head_dtype or dtype)
|
||||
head = TokenEmbeddingPoolerHead(head_dtype, projector, PoolerNormalize())
|
||||
pooler = TokenPooler(TestAllPool._make_all_pool(), head)
|
||||
params = PoolingParams(
|
||||
task="token_embed",
|
||||
late_chunking_params=LateChunkingParams(chunk_size=chunk_size),
|
||||
use_activation=True,
|
||||
)
|
||||
metadata = _make_metadata([7], pooling_params=[params])
|
||||
actual = pooler(hidden, metadata)[0]
|
||||
means = torch.stack(
|
||||
[hidden[a : a + chunk_size].float().mean(0) for a in range(0, 7, chunk_size)]
|
||||
).to(head_dtype or dtype)
|
||||
expected = torch.nn.functional.normalize(projector(means), p=2, dim=-1)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
assert actual.dtype == (head_dtype or dtype)
|
||||
|
||||
|
||||
def test_late_chunk_pool_reduces_before_nonlinear_projector():
|
||||
hidden = torch.tensor([[2.0, 0.0], [0.0, 1.0]])
|
||||
projector = torch.nn.Tanh()
|
||||
head = TokenEmbeddingPoolerHead(projector=projector, activation=PoolerNormalize())
|
||||
params = PoolingParams(
|
||||
task="token_embed",
|
||||
late_chunking_params=LateChunkingParams(chunk_size=2),
|
||||
use_activation=True,
|
||||
)
|
||||
actual = head.forward_chunk(hidden, params)
|
||||
expected = torch.nn.functional.normalize(projector(hidden.mean(0, keepdim=True)))
|
||||
wrong_order = torch.nn.functional.normalize(projector(hidden).mean(0, keepdim=True))
|
||||
torch.testing.assert_close(actual, expected)
|
||||
assert not torch.allclose(actual, wrong_order)
|
||||
|
||||
|
||||
def test_late_chunk_pool_isolates_mixed_requests_and_owns_finished_storage():
|
||||
hidden = torch.arange(36, dtype=torch.float32).reshape(9, 4)
|
||||
params = [
|
||||
PoolingParams(
|
||||
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=2)
|
||||
),
|
||||
PoolingParams(task="token_embed"),
|
||||
PoolingParams(
|
||||
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size=1)
|
||||
),
|
||||
]
|
||||
pooler = TokenPooler(
|
||||
TestAllPool._make_all_pool(async_scheduling=True), TokenEmbeddingPoolerHead()
|
||||
)
|
||||
outputs = pooler(hidden, _make_metadata([3, 4, 2], pooling_params=params))
|
||||
expected = [
|
||||
torch.stack([hidden[:2].mean(0), hidden[2]]),
|
||||
hidden[3:7].clone(),
|
||||
hidden[7:].clone(),
|
||||
]
|
||||
hidden.fill_(-999)
|
||||
for actual, reference in zip(outputs, expected):
|
||||
torch.testing.assert_close(actual, reference)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chunk_size", [1, 2, 8])
|
||||
@pytest.mark.parametrize("embedding_size", [1, 4])
|
||||
def test_bos_eos_filter_preserves_chunk_vectors_in_mixed_batch(
|
||||
chunk_size, embedding_size
|
||||
):
|
||||
hidden = torch.arange(10 * embedding_size, dtype=torch.float32).reshape(
|
||||
10, embedding_size
|
||||
)
|
||||
pooler = BOSEOSFilter(
|
||||
TokenPooler(TestAllPool._make_all_pool(), TokenEmbeddingPoolerHead()),
|
||||
bos_token_id=101,
|
||||
eos_token_id=102,
|
||||
)
|
||||
metadata = _make_metadata(
|
||||
[5, 5],
|
||||
token_ids=[[101, 1, 2, 3, 102]] * 2,
|
||||
pooling_params=[
|
||||
PoolingParams(
|
||||
task="token_embed", late_chunking_params=LateChunkingParams(chunk_size)
|
||||
),
|
||||
PoolingParams(task="token_embed"),
|
||||
],
|
||||
)
|
||||
chunked, ordinary = pooler(hidden, metadata)
|
||||
expected = torch.stack([part.mean(0) for part in hidden[:5].split(chunk_size)])
|
||||
torch.testing.assert_close(chunked, expected)
|
||||
torch.testing.assert_close(ordinary, hidden[6:9].squeeze(-1))
|
||||
|
||||
|
||||
def test_bos_eos_filter_waits_for_complete_chunk_vectors():
|
||||
pooler = BOSEOSFilter(
|
||||
TokenPooler(
|
||||
TestAllPool._make_all_pool(chunked=True), TokenEmbeddingPoolerHead()
|
||||
),
|
||||
bos_token_id=101,
|
||||
)
|
||||
params = [
|
||||
PoolingParams(task="token_embed", late_chunking_params=LateChunkingParams(3))
|
||||
]
|
||||
first = _make_metadata(
|
||||
[4],
|
||||
token_ids=[[101, 1, 2, 102]],
|
||||
pooling_params=params,
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[2],
|
||||
)
|
||||
hidden = torch.arange(8, dtype=torch.float32).reshape(4, 2)
|
||||
assert pooler(hidden[:2], first) == [None]
|
||||
last = _make_metadata(
|
||||
[4],
|
||||
token_ids=[[101, 1, 2, 102]],
|
||||
pooling_params=params,
|
||||
num_scheduled_tokens=[2],
|
||||
seq_lens=[4],
|
||||
)
|
||||
last.pooling_states = first.pooling_states
|
||||
actual = pooler(hidden[2:], last)[0]
|
||||
torch.testing.assert_close(actual, torch.stack([hidden[:3].mean(0), hidden[3]]))
|
||||
assert not last.pooling_states[0].hidden_states_cache
|
||||
@@ -1,344 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_cuda():
|
||||
pytest.skip(
|
||||
"ROCm skinny GEMM tests are not supported on CUDA.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers import utils
|
||||
|
||||
|
||||
def _on_gfx11() -> bool:
|
||||
"""gfx11 check that is safe to evaluate at collection time on any platform."""
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx11
|
||||
|
||||
return on_gfx11()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [1, 128])
|
||||
def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch, m):
|
||||
x = torch.randn(1, 64, dtype=torch.float16)
|
||||
weight = torch.randn(m, 64, dtype=torch.float16)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_called_once()
|
||||
llmm1_mock.assert_not_called()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("n", range(1, 6))
|
||||
def test_rocm_unquantized_gemm_single_output_real_kernel(dtype, n):
|
||||
torch.manual_seed(0)
|
||||
x = torch.randn(n, 2048, device="cuda", dtype=dtype)
|
||||
weight = torch.randn(1, 2048, device="cuda", dtype=dtype)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def test_rocm_unquantized_gemm_makes_skinny_activation_contiguous(monkeypatch):
|
||||
x = torch.randn(64, 4, dtype=torch.float16).t()
|
||||
weight = torch.randn(128, 64, dtype=torch.float16)
|
||||
assert x.shape == (4, 64)
|
||||
assert x.stride() == (1, 4)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_called_once()
|
||||
x_view = wvsplitk_mock.call_args.args[1]
|
||||
assert x_view.is_contiguous()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
def test_rocm_unquantized_gemm_makes_llmm1_activation_contiguous(monkeypatch):
|
||||
x = torch.randn(1, 128, dtype=torch.float16)[:, ::2]
|
||||
weight = torch.randn(4, 64, dtype=torch.float16)
|
||||
assert x.shape == (1, 64)
|
||||
assert x.stride() == (128, 2)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
llmm1_mock.assert_called_once()
|
||||
x_view = llmm1_mock.call_args.args[1]
|
||||
assert x_view.is_contiguous()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("noncontiguous_operand", ["weight", "bias"])
|
||||
def test_rocm_unquantized_gemm_rejects_unsupported_skinny_layouts(
|
||||
monkeypatch, noncontiguous_operand
|
||||
):
|
||||
x = torch.randn(4, 64, dtype=torch.float16)
|
||||
weight = torch.randn(128, 64, dtype=torch.float16)
|
||||
bias = torch.randn(128, dtype=torch.float16)
|
||||
if noncontiguous_operand == "weight":
|
||||
weight = torch.randn(64, 128, dtype=torch.float16).t()
|
||||
assert not weight.is_contiguous()
|
||||
else:
|
||||
bias = torch.randn(256, dtype=torch.float16)[::2]
|
||||
assert not bias.is_contiguous()
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.rocm_aiter_ops, "is_tgemm_enabled", lambda: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock()
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
llmm1_mock = MagicMock()
|
||||
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, bias)
|
||||
ref = torch.nn.functional.linear(x, weight, bias)
|
||||
|
||||
wvsplitk_mock.assert_not_called()
|
||||
llmm1_mock.assert_not_called()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
def _row_stride_padded(weight: torch.Tensor, pad: int) -> torch.Tensor:
|
||||
"""A weight whose row stride is offset by ``pad`` elements, as produced by
|
||||
``maybe_pad_weight_avoid_cache_aliasing``."""
|
||||
padded = torch.nn.functional.pad(weight, (0, pad))[..., :-pad]
|
||||
assert padded.shape == weight.shape
|
||||
assert padded.stride() == (weight.shape[1] + pad, 1)
|
||||
return padded
|
||||
|
||||
|
||||
def test_rocm_unquantized_gemm_wvsplitk_accepts_row_stride_padded_weight(monkeypatch):
|
||||
# wvSplitK reads the weight row stride, so a row-stride-padded weight must
|
||||
# still reach it instead of falling back to torch.nn.functional.linear.
|
||||
x = torch.randn(4, 1024, dtype=torch.float16)
|
||||
weight = _row_stride_padded(torch.randn(128, 1024, dtype=torch.float16), 64)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.rocm_aiter_ops, "is_tgemm_enabled", lambda: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_called_once()
|
||||
assert wvsplitk_mock.call_args.args[0].stride() == weight.stride()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"k,element_size,expected",
|
||||
[
|
||||
(4096, 2, 64), # 8192 B row stride, aliases
|
||||
(12288, 2, 64), # 24576 B row stride, aliases
|
||||
(1024, 2, 64), # smallest aliasing bf16/fp16 K
|
||||
(2560, 2, 0), # 5120 B, no aliasing
|
||||
(11008, 2, 0), # 22016 B, no aliasing
|
||||
(2048, 1, 128), # fp8: 2048 B, aliases, twice the pad
|
||||
(512, 4, 32), # fp32: 2048 B, aliases, half the pad
|
||||
],
|
||||
)
|
||||
def test_cache_aliasing_pad_elems(k, element_size, expected):
|
||||
assert utils.cache_aliasing_pad_elems(k, element_size) == expected
|
||||
|
||||
|
||||
def test_maybe_pad_weight_avoid_cache_aliasing_skips_non_gfx11(monkeypatch):
|
||||
# Padding is gfx11-only, even for a K whose row stride would alias there.
|
||||
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: False)
|
||||
|
||||
weight = torch.randn(128, 4096, dtype=torch.bfloat16)
|
||||
assert utils.maybe_pad_weight_avoid_cache_aliasing(weight) is weight
|
||||
|
||||
|
||||
def test_maybe_pad_weight_avoid_cache_aliasing(monkeypatch):
|
||||
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
|
||||
|
||||
weight = torch.randn(128, 4096, dtype=torch.bfloat16)
|
||||
padded = utils.maybe_pad_weight_avoid_cache_aliasing(weight)
|
||||
|
||||
assert padded.shape == weight.shape
|
||||
assert padded.stride() == (4096 + 64, 1)
|
||||
assert torch.equal(padded, weight)
|
||||
# Idempotent: the padded stride no longer aliases, so a second call is a no-op
|
||||
# and preserves the pointer captured by any graph.
|
||||
repadded = utils.maybe_pad_weight_avoid_cache_aliasing(padded)
|
||||
assert repadded.data_ptr() == padded.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"weight",
|
||||
[
|
||||
torch.randn(128, 2560, dtype=torch.bfloat16), # no aliasing
|
||||
torch.randn(2560, 128, dtype=torch.bfloat16).t(), # not contiguous
|
||||
torch.randn(2, 128, 4096, dtype=torch.bfloat16), # not 2D
|
||||
],
|
||||
)
|
||||
def test_maybe_pad_weight_avoid_cache_aliasing_skips(monkeypatch, weight):
|
||||
monkeypatch.setattr(utils.current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx11", lambda: True)
|
||||
|
||||
assert (
|
||||
utils.maybe_pad_weight_avoid_cache_aliasing(weight).data_ptr()
|
||||
== weight.data_ptr()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-only kernel test")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_rocm_unquantized_gemm_noncontiguous_activation_real_kernel(monkeypatch, dtype):
|
||||
x = torch.randn(64, 4, device="cuda", dtype=dtype).t()
|
||||
weight = torch.randn(128, 64, device="cuda", dtype=dtype)
|
||||
assert x.stride() == (1, 4)
|
||||
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
original_wvsplitk = utils.ops.wvSplitK
|
||||
wvsplitk_mock = MagicMock(side_effect=original_wvsplitk)
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_called_once()
|
||||
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _on_gfx11(), reason="cache-aliasing padding only applies on gfx11"
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_rocm_unquantized_gemm_padded_weight_real_kernel(monkeypatch, dtype):
|
||||
x = torch.randn(4, 4096, device="cuda", dtype=dtype)
|
||||
weight = _row_stride_padded(torch.randn(128, 4096, device="cuda", dtype=dtype), 64)
|
||||
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
original_wvsplitk = utils.ops.wvSplitK
|
||||
wvsplitk_mock = MagicMock(side_effect=original_wvsplitk)
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_called_once()
|
||||
torch.testing.assert_close(out, ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m", [1, 128])
|
||||
def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch, m):
|
||||
# wvSplitK skinny GEMM handles n in [1, 5] (see PR #40687); n > 5 must
|
||||
# fall back to torch.nn.functional.linear.
|
||||
x = torch.randn(6, 64, dtype=torch.float16)
|
||||
weight = torch.randn(m, 64, dtype=torch.float16)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: False)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitk_mock.assert_not_called()
|
||||
llmm1_mock.assert_not_called()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
|
||||
|
||||
def test_rocm_unquantized_gemm_gfx950_wvsplitkrc_path(monkeypatch):
|
||||
x = torch.randn(1024, 16, dtype=torch.float16).t()
|
||||
weight = torch.randn(256, 1024, dtype=torch.float16)
|
||||
assert x.stride() == (1, 16)
|
||||
|
||||
monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
|
||||
monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx1250", lambda: True)
|
||||
monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
|
||||
|
||||
wvsplitkrc_mock = MagicMock(side_effect=lambda x_view, w, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitKrc", wvsplitkrc_mock)
|
||||
wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
|
||||
monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
|
||||
|
||||
out = utils.rocm_unquantized_gemm_impl(x, weight, None)
|
||||
ref = torch.nn.functional.linear(x, weight, None)
|
||||
|
||||
wvsplitkrc_mock.assert_called_once()
|
||||
wvsplitk_mock.assert_not_called()
|
||||
x_view = wvsplitkrc_mock.call_args.args[0]
|
||||
assert x_view.is_contiguous()
|
||||
assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
|
||||
-28
@@ -1,28 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
test_model = "openai-community/gpt2"
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
# Create a sampling params object.
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="fastsafetensors requires NVIDIA/AMD GPUs",
|
||||
)
|
||||
def test_model_loader_download_files(vllm_runner):
|
||||
with vllm_runner(test_model, load_format="fastsafetensors") as llm:
|
||||
deserialized_outputs = llm.generate(prompts, sampling_params)
|
||||
assert deserialized_outputs
|
||||
@@ -1,49 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import glob
|
||||
import tempfile
|
||||
|
||||
import huggingface_hub.constants
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf,
|
||||
fastsafetensors_weights_iterator,
|
||||
safetensors_weights_iterator,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="fastsafetensors requires NVIDIA/AMD GPUs",
|
||||
)
|
||||
@pytest.mark.parametrize("queue_size", [0, 1])
|
||||
def test_fastsafetensors_model_loader(monkeypatch, queue_size):
|
||||
monkeypatch.setenv("VLLM_FASTSAFETENSORS_QUEUE_SIZE", str(queue_size))
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
huggingface_hub.constants.HF_HUB_OFFLINE = False
|
||||
download_weights_from_hf(
|
||||
"openai-community/gpt2", allow_patterns=["*.safetensors"], cache_dir=tmpdir
|
||||
)
|
||||
safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
|
||||
assert len(safetensors) > 0
|
||||
|
||||
fastsafetensors_tensors = {}
|
||||
hf_safetensors_tensors = {}
|
||||
|
||||
for name, tensor in fastsafetensors_weights_iterator(safetensors, True):
|
||||
fastsafetensors_tensors[name] = tensor
|
||||
|
||||
for name, tensor in safetensors_weights_iterator(safetensors, True):
|
||||
hf_safetensors_tensors[name] = tensor
|
||||
|
||||
assert len(fastsafetensors_tensors) == len(hf_safetensors_tensors)
|
||||
|
||||
for name, fastsafetensors_tensor in fastsafetensors_tensors.items():
|
||||
fastsafetensors_tensor = fastsafetensors_tensor.to("cpu")
|
||||
assert fastsafetensors_tensor.dtype == hf_safetensors_tensors[name].dtype
|
||||
assert fastsafetensors_tensor.shape == hf_safetensors_tensors[name].shape
|
||||
assert torch.all(fastsafetensors_tensor.eq(hf_safetensors_tensors[name]))
|
||||
@@ -1,28 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
test_model = "openai-community/gpt2"
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
# Create a sampling params object.
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="InstantTensor requires NVIDIA GPUs",
|
||||
)
|
||||
def test_model_loader_download_files(vllm_runner):
|
||||
with vllm_runner(test_model, load_format="instanttensor") as llm:
|
||||
deserialized_outputs = llm.generate(prompts, sampling_params)
|
||||
assert deserialized_outputs
|
||||
@@ -1,104 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import glob
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf,
|
||||
instanttensor_weights_iterator,
|
||||
safetensors_weights_iterator,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("weight_name", "local_ids", "collective"),
|
||||
[
|
||||
("mlp.experts.gate_proj.weight", {0}, True),
|
||||
("mlp.experts.0.gate_proj.weight", {0}, False),
|
||||
("mlp.experts.0.gate_proj.weight", set(), False),
|
||||
],
|
||||
)
|
||||
def test_instanttensor_ep_collective_loading(
|
||||
monkeypatch, tmp_path, weight_name, local_ids, collective
|
||||
):
|
||||
"""Keep fused checkpoints collective and expert decisions rank-independent."""
|
||||
from vllm.model_executor.model_loader import weight_utils
|
||||
|
||||
path = tmp_path / "model.safetensors"
|
||||
save_file({weight_name: torch.ones(1)}, str(path))
|
||||
native_open = MagicMock()
|
||||
reader = native_open.return_value.__enter__.return_value
|
||||
reader.total_tensor_size = 0
|
||||
reader.tensors.return_value = ()
|
||||
monkeypatch.setitem(
|
||||
sys.modules, "instanttensor", SimpleNamespace(safe_open=native_open)
|
||||
)
|
||||
group = SimpleNamespace(world_size=2, device_group=object())
|
||||
monkeypatch.setattr(weight_utils, "get_world_group", lambda: group)
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(current_platform, "current_device", lambda: 0)
|
||||
list(instanttensor_weights_iterator([str(path)], False, local_expert_ids=local_ids))
|
||||
kwargs = native_open.call_args.kwargs
|
||||
assert kwargs["process_group"] is (group.device_group if collective else None)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="InstantTensor requires NVIDIA GPUs",
|
||||
)
|
||||
def test_instanttensor_ep_filter_cuda(tmp_path):
|
||||
weights = {
|
||||
"mlp.experts.0.gate_proj.weight": torch.ones(2),
|
||||
"mlp.experts.1.gate_proj.weight": torch.ones(2),
|
||||
"mlp.experts.1.gate_proj.weight_scale": torch.ones(1),
|
||||
"model.layers[0].attn.weight": torch.ones(2),
|
||||
}
|
||||
path = tmp_path / "model.safetensors"
|
||||
save_file(weights, str(path))
|
||||
loaded = dict(
|
||||
instanttensor_weights_iterator([str(path)], False, local_expert_ids={0})
|
||||
)
|
||||
del weights["mlp.experts.1.gate_proj.weight"]
|
||||
assert loaded.keys() == weights.keys()
|
||||
for name, tensor in loaded.items():
|
||||
torch.testing.assert_close(tensor.cpu(), weights[name], rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="InstantTensor requires NVIDIA GPUs",
|
||||
)
|
||||
def test_instanttensor_model_loader():
|
||||
model_dir = download_weights_from_hf(
|
||||
"openai-community/gpt2", cache_dir=None, allow_patterns=["*.safetensors"]
|
||||
)
|
||||
safetensors = glob.glob(f"{model_dir}/*.safetensors")
|
||||
assert len(safetensors) > 0
|
||||
|
||||
instanttensor_tensors = {}
|
||||
hf_safetensors_tensors = {}
|
||||
|
||||
for name, tensor in instanttensor_weights_iterator(safetensors, True):
|
||||
instanttensor_tensors[name] = tensor.to("cpu")
|
||||
|
||||
for name, tensor in safetensors_weights_iterator(safetensors, True):
|
||||
hf_safetensors_tensors[name] = tensor
|
||||
|
||||
assert len(instanttensor_tensors) == len(hf_safetensors_tensors)
|
||||
|
||||
for name, instanttensor_tensor in instanttensor_tensors.items():
|
||||
assert instanttensor_tensor.dtype == hf_safetensors_tensors[name].dtype
|
||||
assert instanttensor_tensor.shape == hf_safetensors_tensors[name].shape
|
||||
assert torch.all(instanttensor_tensor.eq(hf_safetensors_tensors[name]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_instanttensor_model_loader()
|
||||
-721
@@ -1,721 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import os
|
||||
import types
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import load_file, save_file
|
||||
|
||||
from tests.utils import create_new_process_for_each_test
|
||||
from vllm import SamplingParams
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.model_loader import runai_streamer_loader as rsl
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
load_format = "runai_streamer"
|
||||
test_model = "openai-community/gpt2"
|
||||
# TODO(amacaskill): Replace with a GKE owned GCS bucket.
|
||||
test_gcs_model = "gs://vertex-model-garden-public-us/codegemma/codegemma-2b/"
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
# Create a sampling params object.
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
|
||||
|
||||
|
||||
def get_runai_model_loader():
|
||||
load_config = LoadConfig(load_format=load_format)
|
||||
return get_model_loader(load_config)
|
||||
|
||||
|
||||
def test_get_model_loader_with_runai_flag():
|
||||
model_loader = get_runai_model_loader()
|
||||
assert model_loader.__class__.__name__ == "RunaiModelStreamerLoader"
|
||||
|
||||
|
||||
def test_checkpoint_selection_hook_is_optional_for_model_detection():
|
||||
from vllm.model_executor.models.interfaces import SupportsMultiModal
|
||||
from vllm.model_executor.models.interfaces_base import (
|
||||
VllmModel,
|
||||
is_text_generation_model,
|
||||
)
|
||||
|
||||
class ModelWithoutHook(torch.nn.Module):
|
||||
def __init__(self, vllm_config, prefix=""):
|
||||
super().__init__()
|
||||
|
||||
def embed_input_ids(self, input_ids):
|
||||
return input_ids
|
||||
|
||||
def forward(self, input_ids, positions):
|
||||
return input_ids
|
||||
|
||||
def compute_logits(self, hidden_states):
|
||||
return hidden_states
|
||||
|
||||
assert is_text_generation_model(ModelWithoutHook)
|
||||
assert is_text_generation_model(ModelWithoutHook(None))
|
||||
|
||||
class ModelWithDefaultRule(ModelWithoutHook, VllmModel):
|
||||
pass
|
||||
|
||||
assert not ModelWithDefaultRule(None).is_unused_checkpoint_weight("any.weight")
|
||||
|
||||
# Structural models do not inherit the protocol's optional default hook.
|
||||
for names in ([], ["language_model"]):
|
||||
model = torch.nn.Module()
|
||||
model.language_model = ModelWithoutHook(None)
|
||||
model._language_model_names = names
|
||||
assert SupportsMultiModal.get_language_model(model) is model.language_model
|
||||
|
||||
|
||||
def test_runai_model_loader_download_files(vllm_runner):
|
||||
with vllm_runner(test_model, load_format=load_format) as llm:
|
||||
deserialized_outputs = llm.generate(prompts, sampling_params)
|
||||
assert deserialized_outputs
|
||||
|
||||
|
||||
# In-process MTP compilation must not retain GPU resources in the pytest worker.
|
||||
@create_new_process_for_each_test("spawn")
|
||||
def test_runai_mtp_real_checkpoint(vllm_runner, tmp_path, monkeypatch):
|
||||
"""Skip main-only shards without changing a real MTP model's generation."""
|
||||
from vllm.model_executor.models.qwen3_5_mtp import Qwen3_5MTP
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.v1.metrics.reader import Counter
|
||||
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("MTP serving requires CUDA or ROCm")
|
||||
|
||||
checkpoint = tmp_path / "checkpoint"
|
||||
hf_api().snapshot_download(
|
||||
repo_id="Qwen/Qwen3.5-0.8B-Base", local_dir=str(checkpoint)
|
||||
)
|
||||
original_files = list(checkpoint.glob("*.safetensors"))
|
||||
tensors = {}
|
||||
for path in original_files:
|
||||
tensors.update(load_file(path))
|
||||
# Reshard real, unmodified weights so selection cannot pass vacuously on
|
||||
# the published single mixed file. This predicate needs no model state.
|
||||
selector = Qwen3_5MTP.__new__(Qwen3_5MTP)
|
||||
main: dict[str, torch.Tensor] = {}
|
||||
draft: dict[str, torch.Tensor] = {}
|
||||
for name, tensor in tensors.items():
|
||||
(main if selector.is_unused_checkpoint_weight(name) else draft)[name] = tensor
|
||||
assert main and draft
|
||||
files = [checkpoint / "main.safetensors", checkpoint / "draft.safetensors"]
|
||||
for path, contents in zip(files, (main, draft)):
|
||||
save_file(contents, path, metadata={"format": "pt"})
|
||||
for path in original_files:
|
||||
path.unlink()
|
||||
(checkpoint / "model.safetensors.index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"weight_map": {
|
||||
name: path.name
|
||||
for path, data in zip(files, (main, draft))
|
||||
for name in data
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
del tensors, main, draft
|
||||
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
opened = []
|
||||
real_stream = rsl.runai_safetensors_weights_iterator
|
||||
|
||||
def stream(paths, *args):
|
||||
opened.append(list(paths))
|
||||
yield from real_stream(paths, *args)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
|
||||
results = []
|
||||
for load_format in ("auto", "runai_streamer"):
|
||||
with vllm_runner(
|
||||
str(checkpoint),
|
||||
load_format=load_format,
|
||||
block_size=None,
|
||||
enable_chunked_prefill=True,
|
||||
disable_log_stats=False,
|
||||
limit_mm_per_prompt={"image": 0, "video": 0},
|
||||
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
|
||||
) as runner:
|
||||
outputs = runner.generate_greedy(prompts, max_tokens=32)
|
||||
results.append(outputs)
|
||||
accepted = next(
|
||||
m.value
|
||||
for m in runner.llm.get_metrics()
|
||||
if isinstance(m, Counter)
|
||||
and m.name == "vllm:spec_decode_num_accepted_tokens"
|
||||
)
|
||||
assert accepted > 0
|
||||
assert len(opened) == 2
|
||||
assert set(opened[0]) == {str(path) for path in files}
|
||||
assert opened[1] == [str(files[1])]
|
||||
matches = sum(a == b for a, b in zip(*results))
|
||||
assert matches >= 3, "RunAI MTP generation diverged from the Default loader"
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="Temporarily disabled due to GCS access issues. "
|
||||
"TODO: Re-enable this test once the underlying issue is resolved."
|
||||
)
|
||||
def test_runai_model_loader_download_files_gcs(
|
||||
vllm_runner, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "fake-project")
|
||||
monkeypatch.setenv("RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS", "true")
|
||||
monkeypatch.setenv(
|
||||
"CLOUD_STORAGE_EMULATOR_ENDPOINT", "https://storage.googleapis.com"
|
||||
)
|
||||
with vllm_runner(test_gcs_model, load_format=load_format) as llm:
|
||||
deserialized_outputs = llm.generate(prompts, sampling_params)
|
||||
assert deserialized_outputs
|
||||
|
||||
|
||||
def test_runai_passes_revision_by_name():
|
||||
# revision must reach download_safetensors_index_file_from_hf as the
|
||||
# ``revision`` keyword, not the positional ``subfolder`` slot.
|
||||
fake_self = types.SimpleNamespace(
|
||||
load_config=types.SimpleNamespace(download_dir="/cache", ignore_patterns=[])
|
||||
)
|
||||
with (
|
||||
patch.object(rsl, "is_runai_obj_uri", return_value=False),
|
||||
patch.object(rsl, "download_weights_from_hf", return_value="/folder"),
|
||||
patch.object(
|
||||
rsl, "list_safetensors", return_value=["/folder/model.safetensors"]
|
||||
),
|
||||
patch.object(rsl, "download_safetensors_index_file_from_hf") as mock_idx,
|
||||
):
|
||||
rsl.RunaiModelStreamerLoader._prepare_weights(fake_self, "org/model", "myrev")
|
||||
|
||||
mock_idx.assert_called_once()
|
||||
assert mock_idx.call_args.kwargs.get("revision") == "myrev"
|
||||
assert "myrev" not in mock_idx.call_args.args
|
||||
|
||||
|
||||
def _runai_loader(extra):
|
||||
return rsl.RunaiModelStreamerLoader(
|
||||
LoadConfig(load_format="runai_streamer", model_loader_extra_config=extra)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra, match",
|
||||
[
|
||||
({"typo_key": 1}, "Unexpected extra config"),
|
||||
({"distributed": "yes"}, "distributed must be a bool"),
|
||||
({"concurrency": "16"}, "concurrency must be a positive integer"),
|
||||
({"concurrency": -1}, "concurrency must be a positive integer"),
|
||||
],
|
||||
)
|
||||
def test_runai_rejects_invalid_extra_config(extra, match):
|
||||
# The loader used to silently drop unknown keys / wrong types / negatives.
|
||||
with pytest.raises(ValueError, match=match):
|
||||
_runai_loader(extra)
|
||||
|
||||
|
||||
def test_runai_accepts_valid_extra_config():
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("RUNAI_STREAMER_CONCURRENCY", None)
|
||||
os.environ.pop("RUNAI_STREAMER_MEMORY_LIMIT", None)
|
||||
loader = _runai_loader(
|
||||
{"distributed": True, "concurrency": 16, "memory_limit": 1024}
|
||||
)
|
||||
assert loader._is_distributed is True
|
||||
assert os.environ["RUNAI_STREAMER_CONCURRENCY"] == "16"
|
||||
assert os.environ["RUNAI_STREAMER_MEMORY_LIMIT"] == "1024"
|
||||
|
||||
|
||||
def test_runai_invalid_extra_config_leaves_environ_untouched():
|
||||
# A later invalid key must not leave an earlier valid key applied to
|
||||
# os.environ (all values are validated before any global mutation).
|
||||
with patch.dict(os.environ, {}, clear=False):
|
||||
os.environ.pop("RUNAI_STREAMER_CONCURRENCY", None)
|
||||
with pytest.raises(ValueError, match="memory_limit must be an integer >= -1"):
|
||||
_runai_loader({"concurrency": 16, "memory_limit": -5})
|
||||
assert "RUNAI_STREAMER_CONCURRENCY" not in os.environ
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_config,expected_source",
|
||||
[
|
||||
# Object storage: model_weights holds the URI, model the pulled config dir.
|
||||
(
|
||||
types.SimpleNamespace(
|
||||
model="/tmp/pulled-config-files",
|
||||
model_weights="s3://bucket/weights",
|
||||
revision="myrev",
|
||||
),
|
||||
"s3://bucket/weights",
|
||||
),
|
||||
# HF repo or local path: model_weights is empty, model is the source.
|
||||
(
|
||||
types.SimpleNamespace(
|
||||
model="org/model", model_weights="", revision="myrev"
|
||||
),
|
||||
"org/model",
|
||||
),
|
||||
],
|
||||
ids=["object_storage", "hf_repo"],
|
||||
)
|
||||
def test_runai_get_all_weights_resolves_source(model_config, expected_source):
|
||||
fake_self = types.SimpleNamespace(
|
||||
_get_weights_iterator=lambda path, revision, is_unused_weight=None: iter(
|
||||
[(f"{path}@{revision}", None)]
|
||||
)
|
||||
)
|
||||
|
||||
weights = list(
|
||||
rsl.RunaiModelStreamerLoader.get_all_weights(
|
||||
fake_self, model_config, model=None
|
||||
)
|
||||
)
|
||||
|
||||
assert weights == [(f"{expected_source}@myrev", None)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("selection", ["draft", "unadapted", "reject_all"])
|
||||
def test_runai_selects_shards_before_streaming(tmp_path, monkeypatch, selection):
|
||||
"""Unused shards must not reach RunAI; mixed shards retain their tensors."""
|
||||
contents = [
|
||||
{"main.weight": torch.tensor([1.0])},
|
||||
{"main.bias": torch.tensor([2.0]), "draft.weight": torch.tensor([3.0])},
|
||||
{"draft.bias": torch.tensor([4.0])},
|
||||
]
|
||||
files = []
|
||||
for i, tensors in enumerate(contents):
|
||||
path = str(tmp_path / f"part-{i}.safetensors")
|
||||
save_file(tensors, path)
|
||||
files.append(path)
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
|
||||
opened = []
|
||||
real_stream = rsl.runai_safetensors_weights_iterator
|
||||
|
||||
def stream(paths, *args):
|
||||
opened.extend(paths)
|
||||
yield from real_stream(paths, *args)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
|
||||
model = torch.nn.Module()
|
||||
if selection == "draft":
|
||||
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
|
||||
elif selection == "reject_all":
|
||||
model.is_unused_checkpoint_weight = lambda name: True
|
||||
config = types.SimpleNamespace(model=str(tmp_path), model_weights="", revision=None)
|
||||
loaded = dict(_runai_loader({}).get_all_weights(config, model))
|
||||
|
||||
expected = files[1:] if selection == "draft" else files
|
||||
assert opened == expected
|
||||
assert set(loaded) == {
|
||||
name
|
||||
for tensors in (contents[1:] if selection == "draft" else contents)
|
||||
for name in tensors
|
||||
}
|
||||
for name, tensor in {**contents[1], **contents[2]}.items():
|
||||
torch.testing.assert_close(loaded[name], tensor)
|
||||
|
||||
|
||||
def _check_distributed_selection(rank, rendezvous, scenario):
|
||||
torch.distributed.init_process_group(
|
||||
"gloo",
|
||||
init_method=rendezvous,
|
||||
rank=rank,
|
||||
world_size=2,
|
||||
timeout=timedelta(seconds=30),
|
||||
)
|
||||
root = "s3://bucket/weights"
|
||||
if scenario == "different_roots":
|
||||
root += str(rank)
|
||||
files = [root + "/main.safetensors", root + "/draft.safetensors"]
|
||||
|
||||
def pull_files(source, destination, **kwargs):
|
||||
if scenario == "metadata_failure" and rank == 1:
|
||||
raise OSError("Index temporarily unavailable")
|
||||
(Path(destination) / "model.safetensors.index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"weight_map": {
|
||||
"main.weight": "main.safetensors",
|
||||
"draft.weight": "draft.safetensors",
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
try:
|
||||
with pytest.MonkeyPatch.context() as monkeypatch:
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"_get_remote_checkpoint_keys",
|
||||
lambda paths: [
|
||||
["main.weight"]
|
||||
if path.endswith("main.safetensors")
|
||||
else ["draft.weight"]
|
||||
for path in paths
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"get_world_group",
|
||||
lambda: types.SimpleNamespace(
|
||||
world_size=2, cpu_group=torch.distributed.group.WORLD
|
||||
),
|
||||
raising=False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"get_pp_group",
|
||||
lambda: types.SimpleNamespace(
|
||||
world_size=2 if scenario == "pipeline_draft" else 1
|
||||
),
|
||||
raising=False,
|
||||
)
|
||||
if scenario == "pipeline_draft" and rank == 0:
|
||||
# Earlier PP stages never invoke the draft loader.
|
||||
torch.distributed.barrier()
|
||||
return
|
||||
# Exercise real selection and Gloo; observe the distributed stream input.
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"runai_safetensors_weights_iterator",
|
||||
lambda paths, *args: iter((path, None) for path in paths),
|
||||
)
|
||||
model = torch.nn.Module()
|
||||
if not (scenario == "missing_hook" and rank == 1):
|
||||
needed = (
|
||||
"main." if scenario == "different_roles" and rank == 1 else "draft."
|
||||
)
|
||||
model.is_unused_checkpoint_weight = lambda name: (
|
||||
not name.startswith(needed)
|
||||
)
|
||||
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
|
||||
selected = [
|
||||
path
|
||||
for path, _ in _runai_loader({"distributed": True}).get_all_weights(
|
||||
config, model
|
||||
)
|
||||
]
|
||||
expected = files[1:] if scenario == "healthy" else files
|
||||
assert selected == expected
|
||||
if scenario == "pipeline_draft":
|
||||
torch.distributed.barrier()
|
||||
finally:
|
||||
torch.distributed.destroy_process_group()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"scenario",
|
||||
[
|
||||
"healthy",
|
||||
"metadata_failure",
|
||||
"different_roles",
|
||||
"missing_hook",
|
||||
"different_roots",
|
||||
"pipeline_draft",
|
||||
],
|
||||
)
|
||||
def test_distributed_runai_preserves_every_ranks_required_shards(tmp_path, scenario):
|
||||
"""All ranks must stream the same union even if one cannot filter safely."""
|
||||
torch.multiprocessing.spawn(
|
||||
_check_distributed_selection,
|
||||
args=((tmp_path / "rendezvous").as_uri(), scenario),
|
||||
nprocs=2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scheme", ["s3", "gs", "az"])
|
||||
@pytest.mark.parametrize(
|
||||
"index_kind",
|
||||
[
|
||||
"valid",
|
||||
"missing",
|
||||
"malformed",
|
||||
"unavailable",
|
||||
"unadapted",
|
||||
"missing_shard",
|
||||
"stale",
|
||||
"header_unavailable",
|
||||
],
|
||||
)
|
||||
def test_runai_remote_selection_uses_actual_weight_source(
|
||||
tmp_path, monkeypatch, scheme, index_kind
|
||||
):
|
||||
root = f"{scheme}://bucket/weights"
|
||||
files = [
|
||||
root + "/main/part.safetensors",
|
||||
root + "/draft/part.safetensors",
|
||||
root + "/mixed.safetensors",
|
||||
root + "/extra.safetensors",
|
||||
]
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
pulled = []
|
||||
|
||||
def pull_files(source, destination, allow_pattern, ignore_pattern=None):
|
||||
pulled.append((source.rstrip("/"), allow_pattern))
|
||||
if index_kind == "unavailable":
|
||||
# Storage SDKs do not necessarily derive errors from OSError.
|
||||
raise Exception("Object metadata unavailable")
|
||||
index = Path(destination) / "model.safetensors.index.json"
|
||||
if index_kind in ("valid", "missing_shard", "stale", "header_unavailable"):
|
||||
index.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"weight_map": {
|
||||
"main.weight": "main/part.safetensors",
|
||||
"draft.weight": (
|
||||
"absent.safetensors"
|
||||
if index_kind == "missing_shard"
|
||||
else "draft/part.safetensors"
|
||||
),
|
||||
"main.bias": "mixed.safetensors",
|
||||
"draft.bias": "mixed.safetensors",
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
elif index_kind == "malformed":
|
||||
index.write_text("not JSON")
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files, raising=False)
|
||||
headers_checked = []
|
||||
|
||||
def checkpoint_keys(paths):
|
||||
headers_checked.extend(paths)
|
||||
if index_kind == "header_unavailable":
|
||||
raise OSError("Header temporarily unavailable")
|
||||
return [
|
||||
["main.weight", "draft.unindexed_weight"]
|
||||
if index_kind == "stale"
|
||||
else ["main.weight"]
|
||||
for _ in paths
|
||||
]
|
||||
|
||||
monkeypatch.setattr(
|
||||
rsl, "_get_remote_checkpoint_keys", checkpoint_keys, raising=False
|
||||
)
|
||||
opened = []
|
||||
|
||||
def stream(paths, *args):
|
||||
opened.extend(paths)
|
||||
return iter(())
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream)
|
||||
model = torch.nn.Module()
|
||||
if index_kind != "unadapted":
|
||||
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
|
||||
# The config directory is not the checkpoint: using its index could drop
|
||||
# tensors from an independent model_weights source.
|
||||
(tmp_path / "model.safetensors.index.json").write_text("not the weights index")
|
||||
config = types.SimpleNamespace(
|
||||
model=str(tmp_path), model_weights=root, revision=None
|
||||
)
|
||||
list(_runai_loader({}).get_all_weights(config, model))
|
||||
|
||||
assert opened == (files[1:] if index_kind == "valid" else files)
|
||||
assert headers_checked == (
|
||||
files[:1] if index_kind in ("valid", "stale", "header_unavailable") else []
|
||||
)
|
||||
assert pulled == (
|
||||
[]
|
||||
if index_kind == "unadapted"
|
||||
else [
|
||||
(
|
||||
root,
|
||||
[
|
||||
"weights/model.safetensors.index.json",
|
||||
"model.safetensors.index.json",
|
||||
],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"weight_map",
|
||||
[None, {}, [], {"draft.weight": None}, {"draft.weight": "../part.safetensors"}],
|
||||
)
|
||||
def test_runai_invalid_weight_map_keeps_original_files(monkeypatch, weight_map):
|
||||
files = ["s3://bucket/model/part.safetensors"]
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
|
||||
def pull_files(source, directory, **kwargs):
|
||||
(Path(directory) / "model.safetensors.index.json").write_text(
|
||||
json.dumps({"weight_map": weight_map})
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
|
||||
monkeypatch.setattr(
|
||||
rsl, "runai_safetensors_weights_iterator", lambda paths, *args: iter(paths)
|
||||
)
|
||||
model = torch.nn.Module()
|
||||
model.is_unused_checkpoint_weight = lambda name: True
|
||||
config = types.SimpleNamespace(
|
||||
model="s3://bucket/model", model_weights="", revision=None
|
||||
)
|
||||
assert list(_runai_loader({}).get_all_weights(config, model)) == files
|
||||
|
||||
|
||||
def test_runai_model_rule_error_is_not_hidden_by_metadata_fallback(monkeypatch):
|
||||
files = ["s3://bucket/model/part.safetensors"]
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: files)
|
||||
|
||||
def pull_files(source, directory, **kwargs):
|
||||
(Path(directory) / "model.safetensors.index.json").write_text(
|
||||
json.dumps({"weight_map": {"draft.weight": "part.safetensors"}})
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
|
||||
model = torch.nn.Module()
|
||||
|
||||
def invalid_rule(name):
|
||||
raise ValueError("Invalid model rule")
|
||||
|
||||
model.is_unused_checkpoint_weight = invalid_rule
|
||||
config = types.SimpleNamespace(
|
||||
model="s3://bucket/model", model_weights="", revision=None
|
||||
)
|
||||
with pytest.raises(ValueError, match="Invalid model rule"):
|
||||
list(_runai_loader({}).get_all_weights(config, model))
|
||||
|
||||
|
||||
def test_runai_header_reader_uses_actual_tensor_names(tmp_path):
|
||||
"""The RunAI header reader sees names even when an index omits them."""
|
||||
path = tmp_path / "mixed.safetensors"
|
||||
save_file(
|
||||
{
|
||||
"main.weight": torch.tensor([1.0]),
|
||||
"draft.unindexed_weight": torch.tensor([2.0]),
|
||||
},
|
||||
path,
|
||||
metadata={"format": "pt"},
|
||||
)
|
||||
(names,) = rsl._get_remote_checkpoint_keys([str(path)])
|
||||
assert set(names) == {"main.weight", "draft.unindexed_weight"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stale", [False, True])
|
||||
def test_runai_stale_index_keeps_actual_draft_weights(tmp_path, monkeypatch, stale):
|
||||
"""Verify excluded headers before dropping a shard with real draft weights."""
|
||||
root = "s3://bucket/checkpoint"
|
||||
main_path = tmp_path / "main.safetensors"
|
||||
draft_path = tmp_path / "draft.safetensors"
|
||||
main_tensors = {"main.weight": torch.tensor([1.0])}
|
||||
if stale:
|
||||
main_tensors["draft.unindexed_weight"] = torch.tensor([2.0])
|
||||
save_file(main_tensors, main_path)
|
||||
save_file({"draft.weight": torch.tensor([3.0])}, draft_path)
|
||||
paths = {
|
||||
root + "/main.safetensors": str(main_path),
|
||||
root + "/draft.safetensors": str(draft_path),
|
||||
}
|
||||
monkeypatch.setattr(rsl, "list_safetensors", lambda path: list(paths))
|
||||
|
||||
def pull_files(source, directory, **kwargs):
|
||||
(Path(directory) / "model.safetensors.index.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"weight_map": {
|
||||
"main.weight": "main.safetensors",
|
||||
"draft.weight": "draft.safetensors",
|
||||
}
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_pull_files", pull_files)
|
||||
read_headers = rsl._get_remote_checkpoint_keys
|
||||
monkeypatch.setattr(
|
||||
rsl,
|
||||
"_get_remote_checkpoint_keys",
|
||||
lambda files: read_headers([paths[path] for path in files]),
|
||||
)
|
||||
stream = rsl.runai_safetensors_weights_iterator
|
||||
|
||||
def stream_local(files, *args):
|
||||
for name, tensor in stream([paths[path] for path in files], *args):
|
||||
yield name, tensor.clone()
|
||||
|
||||
monkeypatch.setattr(rsl, "runai_safetensors_weights_iterator", stream_local)
|
||||
model = torch.nn.Module()
|
||||
model.is_unused_checkpoint_weight = lambda name: not name.startswith("draft.")
|
||||
config = types.SimpleNamespace(model=root, model_weights="", revision=None)
|
||||
loaded = dict(_runai_loader({}).get_all_weights(config, model))
|
||||
assert set(loaded) == (
|
||||
{"main.weight", "draft.unindexed_weight", "draft.weight"}
|
||||
if stale
|
||||
else {"draft.weight"}
|
||||
)
|
||||
torch.testing.assert_close(loaded["draft.weight"], torch.tensor([3.0]))
|
||||
if stale:
|
||||
torch.testing.assert_close(
|
||||
loaded["draft.unindexed_weight"], torch.tensor([2.0])
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_model,mul_model,add_model",
|
||||
[
|
||||
(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"inference-optimization/Qwen3-0.6B-debug-multiply",
|
||||
"inference-optimization/Qwen3-0.6B-debug-add",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_runai_deep_sleep_reload_weights(base_model, mul_model, add_model, vllm_runner):
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
used_bytes_baseline = total - free
|
||||
|
||||
def sleep_and_reload(llm, path):
|
||||
# Level 2 discards the parameter memory, so after wake_up the weights
|
||||
# can only come from the reload: a tensor the streamer fails to
|
||||
# deliver changes the output instead of keeping a stale valid value.
|
||||
llm.get_llm().sleep(level=2)
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
assert total - free - used_bytes_baseline < 3 * GiB_bytes
|
||||
llm.get_llm().wake_up(tags=["weights"])
|
||||
llm.collective_rpc("reload_weights", kwargs={"weights_path": path})
|
||||
free, total = torch.accelerator.get_memory_info()
|
||||
assert total - free - used_bytes_baseline < 4 * GiB_bytes
|
||||
llm.get_llm().wake_up(tags=["kv_cache"])
|
||||
|
||||
with vllm_runner(
|
||||
model_name=base_model,
|
||||
load_format=load_format,
|
||||
enable_sleep_mode=True,
|
||||
enable_prefix_caching=False,
|
||||
max_model_len=16,
|
||||
max_num_seqs=1,
|
||||
) as llm:
|
||||
base_output = llm.generate_greedy(["3 4 ="], max_tokens=4)
|
||||
|
||||
sleep_and_reload(llm, mul_model)
|
||||
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
|
||||
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
|
||||
assert mul_perp < add_perp
|
||||
|
||||
sleep_and_reload(llm, add_model)
|
||||
mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
|
||||
add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
|
||||
assert add_perp < mul_perp
|
||||
|
||||
# Round trip to the original weights over discarded memory must
|
||||
# reproduce the pre-sleep output.
|
||||
sleep_and_reload(llm, base_model)
|
||||
assert llm.generate_greedy(["3 4 ="], max_tokens=4) == base_output
|
||||
@@ -1,617 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
from typing import Any
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.model_loader.tensorizer
|
||||
from tests.utils import VLLM_PATH, RemoteOpenAIServer
|
||||
from vllm import SamplingParams
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.model_executor.model_loader.tensorizer import (
|
||||
TensorizerConfig,
|
||||
TensorSerializer,
|
||||
is_vllm_tensorized,
|
||||
open_stream,
|
||||
tensorize_vllm_model,
|
||||
)
|
||||
from vllm.model_executor.model_loader.tensorizer_loader import (
|
||||
BLACKLISTED_TENSORIZER_ARGS,
|
||||
)
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
|
||||
from .conftest import DummyExecutor, assert_from_collective_rpc
|
||||
|
||||
try:
|
||||
import tensorizer
|
||||
from tensorizer import EncryptionParams
|
||||
except ImportError:
|
||||
tensorizer = PlaceholderModule("tensorizer")
|
||||
EncryptionParams = tensorizer.placeholder_attr("EncryptionParams")
|
||||
|
||||
|
||||
class TensorizerCaughtError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
EXAMPLES_PATH = VLLM_PATH / "examples"
|
||||
|
||||
pytest_plugins = ("pytest_asyncio",)
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
# Create a sampling params object.
|
||||
sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
|
||||
|
||||
|
||||
def patch_init_and_catch_error(self, obj, method_name, expected_error: type[Exception]):
|
||||
original = getattr(obj, method_name, None)
|
||||
if original is None:
|
||||
raise ValueError("Method '{}' not found.".format(method_name))
|
||||
|
||||
def wrapper(*args, **kwargs):
|
||||
try:
|
||||
return original(*args, **kwargs)
|
||||
except expected_error as err:
|
||||
raise TensorizerCaughtError from err
|
||||
|
||||
setattr(obj, method_name, wrapper)
|
||||
|
||||
self.load_model()
|
||||
|
||||
|
||||
def assert_specific_tensorizer_error_is_raised(
|
||||
executor,
|
||||
obj: Any,
|
||||
method_name: str,
|
||||
expected_error: type[Exception],
|
||||
):
|
||||
with pytest.raises(TensorizerCaughtError):
|
||||
executor.collective_rpc(
|
||||
patch_init_and_catch_error,
|
||||
args=(
|
||||
obj,
|
||||
method_name,
|
||||
expected_error,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def is_curl_installed():
|
||||
try:
|
||||
subprocess.check_call(["curl", "--version"])
|
||||
return True
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
return False
|
||||
|
||||
|
||||
def write_keyfile(keyfile_path: str):
|
||||
encryption_params = EncryptionParams.random()
|
||||
pathlib.Path(keyfile_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(keyfile_path, "wb") as f:
|
||||
f.write(encryption_params.key)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
|
||||
def test_deserialized_encrypted_vllm_model_has_same_outputs(
|
||||
model_ref, vllm_runner, tmp_path, model_path
|
||||
):
|
||||
args = EngineArgs(model=model_ref)
|
||||
with vllm_runner(model_ref) as vllm_model:
|
||||
key_path = tmp_path / model_ref / "model.key"
|
||||
write_keyfile(key_path)
|
||||
|
||||
outputs = vllm_model.generate(prompts, sampling_params)
|
||||
|
||||
config_for_serializing = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path), encryption_keyfile=str(key_path)
|
||||
)
|
||||
|
||||
tensorize_vllm_model(args, config_for_serializing)
|
||||
|
||||
config_for_deserializing = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path), encryption_keyfile=str(key_path)
|
||||
)
|
||||
|
||||
with vllm_runner(
|
||||
model_ref,
|
||||
load_format="tensorizer",
|
||||
model_loader_extra_config=config_for_deserializing,
|
||||
) as loaded_vllm_model: # noqa: E501
|
||||
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
|
||||
# noqa: E501
|
||||
|
||||
assert outputs == deserialized_outputs
|
||||
|
||||
|
||||
def test_deserialized_hf_model_has_same_outputs(
|
||||
hf_runner, vllm_runner, tmp_path, model_ref, model_path
|
||||
):
|
||||
with hf_runner(model_ref) as hf_model:
|
||||
max_tokens = 50
|
||||
outputs = hf_model.generate_greedy(prompts, max_tokens=max_tokens)
|
||||
with open_stream(model_path, "wb+") as stream:
|
||||
serializer = TensorSerializer(stream)
|
||||
serializer.write_module(hf_model.model)
|
||||
|
||||
with vllm_runner(
|
||||
model_ref,
|
||||
load_format="tensorizer",
|
||||
model_loader_extra_config=TensorizerConfig(
|
||||
tensorizer_uri=str(model_path),
|
||||
num_readers=1,
|
||||
),
|
||||
) as loaded_hf_model:
|
||||
deserialized_outputs = loaded_hf_model.generate_greedy(
|
||||
prompts, max_tokens=max_tokens
|
||||
)
|
||||
|
||||
assert outputs == deserialized_outputs
|
||||
|
||||
|
||||
def test_load_without_tensorizer_load_format(vllm_runner, capfd, model_ref):
|
||||
model = None
|
||||
try:
|
||||
model = vllm_runner(
|
||||
model_ref, model_loader_extra_config=TensorizerConfig(tensorizer_uri="test")
|
||||
)
|
||||
pytest.fail("Expected RuntimeError for extra config keys")
|
||||
except RuntimeError:
|
||||
out, err = capfd.readouterr()
|
||||
combined_output = out + err
|
||||
assert (
|
||||
"ValueError: Unexpected extra config keys for load format auto"
|
||||
) in combined_output
|
||||
finally:
|
||||
del model
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
|
||||
def test_raise_value_error_on_invalid_load_format(vllm_runner, capfd, model_ref):
|
||||
model = None
|
||||
try:
|
||||
model = vllm_runner(
|
||||
model_ref,
|
||||
load_format="safetensors",
|
||||
model_loader_extra_config=TensorizerConfig(tensorizer_uri="test"),
|
||||
)
|
||||
pytest.fail("Expected RuntimeError for extra config keys")
|
||||
except RuntimeError:
|
||||
out, err = capfd.readouterr()
|
||||
|
||||
combined_output = out + err
|
||||
assert (
|
||||
"ValueError: Unexpected extra config keys for load format safetensors"
|
||||
) in combined_output
|
||||
finally:
|
||||
del model
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
|
||||
@pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Requires 2 GPUs")
|
||||
def test_tensorizer_with_tp_path_without_template(vllm_runner, capfd):
|
||||
try:
|
||||
model_ref = "EleutherAI/pythia-1.4b"
|
||||
tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
|
||||
|
||||
vllm_runner(
|
||||
model_ref,
|
||||
load_format="tensorizer",
|
||||
model_loader_extra_config=TensorizerConfig(
|
||||
tensorizer_uri=tensorized_path,
|
||||
num_readers=1,
|
||||
s3_endpoint="object.ord1.coreweave.com",
|
||||
),
|
||||
tensor_parallel_size=2,
|
||||
disable_custom_all_reduce=True,
|
||||
)
|
||||
except RuntimeError:
|
||||
out, err = capfd.readouterr()
|
||||
combined_output = out + err
|
||||
assert (
|
||||
"ValueError: For a sharded model, tensorizer_uri "
|
||||
"should include a string format template like '%04d' "
|
||||
"to be formatted with the rank "
|
||||
"of the shard"
|
||||
) in combined_output
|
||||
|
||||
|
||||
@pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Requires 2 GPUs")
|
||||
def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(
|
||||
vllm_runner, tmp_path
|
||||
):
|
||||
model_ref = "EleutherAI/pythia-1.4b"
|
||||
# record outputs from un-sharded un-tensorized model
|
||||
with vllm_runner(
|
||||
model_ref,
|
||||
disable_custom_all_reduce=True,
|
||||
enforce_eager=True,
|
||||
) as base_model:
|
||||
outputs = base_model.generate(prompts, sampling_params)
|
||||
|
||||
# load model with two shards and serialize with encryption
|
||||
model_path = str(tmp_path / model_ref / "model-%02d.tensors")
|
||||
key_path = tmp_path / (model_ref + ".key")
|
||||
|
||||
tensorizer_config = TensorizerConfig(
|
||||
tensorizer_uri=model_path,
|
||||
encryption_keyfile=str(key_path),
|
||||
)
|
||||
|
||||
tensorize_vllm_model(
|
||||
engine_args=EngineArgs(
|
||||
model=model_ref,
|
||||
tensor_parallel_size=2,
|
||||
disable_custom_all_reduce=True,
|
||||
enforce_eager=True,
|
||||
),
|
||||
tensorizer_config=tensorizer_config,
|
||||
)
|
||||
assert os.path.isfile(model_path % 0), "Serialization subprocess failed"
|
||||
assert os.path.isfile(model_path % 1), "Serialization subprocess failed"
|
||||
|
||||
with vllm_runner(
|
||||
model_ref,
|
||||
tensor_parallel_size=2,
|
||||
load_format="tensorizer",
|
||||
disable_custom_all_reduce=True,
|
||||
enforce_eager=True,
|
||||
model_loader_extra_config=tensorizer_config,
|
||||
) as loaded_vllm_model:
|
||||
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
|
||||
|
||||
assert outputs == deserialized_outputs
|
||||
|
||||
|
||||
@pytest.mark.flaky(reruns=3)
|
||||
def test_vllm_tensorized_model_has_same_outputs(
|
||||
model_ref, vllm_runner, tmp_path, model_path
|
||||
):
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
config = TensorizerConfig(tensorizer_uri=str(model_path))
|
||||
args = EngineArgs(model=model_ref)
|
||||
|
||||
with vllm_runner(model_ref) as vllm_model:
|
||||
outputs = vllm_model.generate(prompts, sampling_params)
|
||||
|
||||
tensorize_vllm_model(args, config)
|
||||
assert is_vllm_tensorized(config)
|
||||
|
||||
with vllm_runner(
|
||||
model_ref, load_format="tensorizer", model_loader_extra_config=config
|
||||
) as loaded_vllm_model:
|
||||
deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
|
||||
# noqa: E501
|
||||
|
||||
assert outputs == deserialized_outputs
|
||||
|
||||
|
||||
def test_load_with_just_model_tensors(just_serialize_model_tensors, model_ref):
|
||||
# For backwards compatibility, ensure Tensorizer can be still be loaded
|
||||
# for inference by passing the model reference name, not a local/S3 dir,
|
||||
# and the location of the model tensors
|
||||
|
||||
model_dir = just_serialize_model_tensors
|
||||
|
||||
extra_config = {"tensorizer_uri": f"{model_dir}/model.tensors"}
|
||||
|
||||
## Start OpenAI API server
|
||||
args = [
|
||||
"--load-format",
|
||||
"tensorizer",
|
||||
"--model-loader-extra-config",
|
||||
json.dumps(extra_config),
|
||||
]
|
||||
|
||||
with RemoteOpenAIServer(model_ref, args):
|
||||
# This test only concerns itself with being able to load the model
|
||||
# and successfully initialize the server
|
||||
pass
|
||||
|
||||
|
||||
def test_assert_serialization_kwargs_passed_to_tensor_serializer(tmp_path, vllm_runner):
|
||||
serialization_params = {
|
||||
"limit_cpu_concurrency": 2,
|
||||
}
|
||||
model_ref = "facebook/opt-125m"
|
||||
model_path = tmp_path / (model_ref + ".tensors")
|
||||
config = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
|
||||
)
|
||||
|
||||
def serialization_test(self, *args, **kwargs):
|
||||
# This is performed in the ephemeral worker process, so monkey-patching
|
||||
# will actually work, and cleanup is guaranteed so don't
|
||||
# need to reset things
|
||||
|
||||
original_dict = serialization_params
|
||||
to_compare = {}
|
||||
|
||||
original = tensorizer.serialization.TensorSerializer.__init__
|
||||
|
||||
def tensorizer_serializer_wrapper(self, *args, **kwargs):
|
||||
nonlocal to_compare
|
||||
to_compare = kwargs.copy()
|
||||
return original(self, *args, **kwargs)
|
||||
|
||||
tensorizer.serialization.TensorSerializer.__init__ = (
|
||||
tensorizer_serializer_wrapper
|
||||
)
|
||||
|
||||
tensorizer_config = TensorizerConfig(**kwargs["tensorizer_config"])
|
||||
self.save_tensorized_model(
|
||||
tensorizer_config=tensorizer_config,
|
||||
)
|
||||
return to_compare | original_dict == to_compare
|
||||
|
||||
kwargs = {"tensorizer_config": config.to_serializable()}
|
||||
|
||||
with vllm_runner(model_ref) as runner:
|
||||
assert assert_from_collective_rpc(runner.get_llm(), serialization_test, kwargs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"serialization_error",
|
||||
"renderer_shutdown_error",
|
||||
"engine_shutdown_error",
|
||||
"expected_error",
|
||||
),
|
||||
[
|
||||
(None, None, None, None),
|
||||
(RuntimeError("serialization failed"), None, None, "serialization failed"),
|
||||
(
|
||||
RuntimeError("serialization failed"),
|
||||
ValueError("renderer shutdown failed"),
|
||||
ValueError("engine shutdown failed"),
|
||||
"serialization failed",
|
||||
),
|
||||
(
|
||||
None,
|
||||
ValueError("renderer shutdown failed"),
|
||||
None,
|
||||
"renderer shutdown failed",
|
||||
),
|
||||
(None, None, ValueError("engine shutdown failed"), "engine shutdown failed"),
|
||||
],
|
||||
)
|
||||
def test_tensorize_vllm_model_shuts_down_engine(
|
||||
monkeypatch,
|
||||
serialization_error,
|
||||
renderer_shutdown_error,
|
||||
engine_shutdown_error,
|
||||
expected_error,
|
||||
):
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
engine_args = Mock()
|
||||
config = Mock(encryption_keyfile=None)
|
||||
engine = Mock()
|
||||
engine.collective_rpc.side_effect = serialization_error
|
||||
engine.renderer.shutdown.side_effect = renderer_shutdown_error
|
||||
engine.engine_core.shutdown.side_effect = engine_shutdown_error
|
||||
monkeypatch.setattr(LLMEngine, "from_vllm_config", Mock(return_value=engine))
|
||||
monkeypatch.setenv("VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS", "7")
|
||||
|
||||
if expected_error is None:
|
||||
tensorize_vllm_model(engine_args, config)
|
||||
else:
|
||||
with pytest.raises(Exception, match=expected_error) as exc_info:
|
||||
tensorize_vllm_model(engine_args, config)
|
||||
notes = getattr(exc_info.value, "__notes__", ())
|
||||
if serialization_error is not None and renderer_shutdown_error is not None:
|
||||
assert any("renderer shutdown failed" in note for note in notes)
|
||||
if serialization_error is not None and engine_shutdown_error is not None:
|
||||
assert any("engine shutdown failed" in note for note in notes)
|
||||
|
||||
engine.engine_core.shutdown.assert_called_once_with(timeout=17.0)
|
||||
engine.renderer.shutdown.assert_called_once_with()
|
||||
|
||||
|
||||
def test_assert_deserialization_kwargs_passed_to_tensor_deserializer(tmp_path, capfd):
|
||||
deserialization_kwargs = {
|
||||
"num_readers": "bar", # illegal value
|
||||
}
|
||||
|
||||
serialization_params = {
|
||||
"limit_cpu_concurrency": 2,
|
||||
}
|
||||
|
||||
model_ref = "facebook/opt-125m"
|
||||
model_path = tmp_path / (model_ref + ".tensors")
|
||||
config = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
|
||||
)
|
||||
|
||||
args = EngineArgs(model=model_ref)
|
||||
tensorize_vllm_model(args, config)
|
||||
|
||||
loader_tc = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path),
|
||||
deserialization_kwargs=deserialization_kwargs,
|
||||
)
|
||||
|
||||
engine_args = EngineArgs(
|
||||
model="facebook/opt-125m",
|
||||
load_format="tensorizer",
|
||||
model_loader_extra_config=loader_tc.to_serializable(),
|
||||
)
|
||||
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
executor = DummyExecutor(vllm_config)
|
||||
|
||||
assert_specific_tensorizer_error_is_raised(
|
||||
executor,
|
||||
tensorizer.serialization.TensorDeserializer,
|
||||
"__init__",
|
||||
TypeError,
|
||||
)
|
||||
|
||||
|
||||
def test_assert_stream_kwargs_passed_to_tensor_deserializer(tmp_path, capfd):
|
||||
deserialization_kwargs = {
|
||||
"num_readers": 1,
|
||||
}
|
||||
|
||||
serialization_params = {
|
||||
"limit_cpu_concurrency": 2,
|
||||
}
|
||||
|
||||
model_ref = "facebook/opt-125m"
|
||||
model_path = tmp_path / (model_ref + ".tensors")
|
||||
config = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
|
||||
)
|
||||
|
||||
args = EngineArgs(model=model_ref)
|
||||
tensorize_vllm_model(args, config)
|
||||
|
||||
stream_kwargs = {"mode": "foo"}
|
||||
|
||||
loader_tc = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path),
|
||||
deserialization_kwargs=deserialization_kwargs,
|
||||
stream_kwargs=stream_kwargs,
|
||||
)
|
||||
|
||||
engine_args = EngineArgs(
|
||||
model="facebook/opt-125m",
|
||||
load_format="tensorizer",
|
||||
model_loader_extra_config=loader_tc.to_serializable(),
|
||||
)
|
||||
|
||||
vllm_config = engine_args.create_engine_config()
|
||||
executor = DummyExecutor(vllm_config)
|
||||
|
||||
assert_specific_tensorizer_error_is_raised(
|
||||
executor,
|
||||
vllm.model_executor.model_loader.tensorizer,
|
||||
"open_stream",
|
||||
ValueError,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serialize_and_serve_entrypoints(tmp_path):
|
||||
model_ref = "facebook/opt-125m"
|
||||
|
||||
suffix = "test"
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py",
|
||||
"--model",
|
||||
model_ref,
|
||||
"serialize",
|
||||
"--serialized-directory",
|
||||
str(tmp_path),
|
||||
"--suffix",
|
||||
suffix,
|
||||
"--serialization-kwargs",
|
||||
'{"limit_cpu_concurrency": 4}',
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print("Tensorizing failed.")
|
||||
print("STDOUT:\n", e.stdout)
|
||||
print("STDERR:\n", e.stderr)
|
||||
raise
|
||||
|
||||
assert "Successfully serialized" in result.stdout
|
||||
|
||||
# Next, try to serve with vllm serve
|
||||
model_uri = tmp_path / "vllm" / model_ref / suffix / "model.tensors"
|
||||
|
||||
model_loader_extra_config = {
|
||||
"tensorizer_uri": str(model_uri),
|
||||
"stream_kwargs": {
|
||||
"force_http": False,
|
||||
},
|
||||
"deserialization_kwargs": {
|
||||
"verify_hash": True,
|
||||
"num_readers": 8,
|
||||
},
|
||||
}
|
||||
|
||||
cmd = [
|
||||
"-m",
|
||||
"vllm.entrypoints.cli.main",
|
||||
"serve",
|
||||
"--host",
|
||||
"localhost",
|
||||
"--load-format",
|
||||
"tensorizer",
|
||||
model_ref,
|
||||
"--model-loader-extra-config",
|
||||
json.dumps(model_loader_extra_config, indent=2),
|
||||
]
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.STDOUT,
|
||||
)
|
||||
|
||||
assert proc.stdout is not None
|
||||
fut = proc.stdout.readuntil(b"Application startup complete.")
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(fut, 180)
|
||||
except TimeoutError:
|
||||
pytest.fail("Server did not start successfully")
|
||||
finally:
|
||||
proc.terminate()
|
||||
await proc.communicate()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("illegal_value", BLACKLISTED_TENSORIZER_ARGS)
|
||||
def test_blacklisted_parameter_for_loading(tmp_path, vllm_runner, capfd, illegal_value):
|
||||
serialization_params = {
|
||||
"limit_cpu_concurrency": 2,
|
||||
}
|
||||
|
||||
model_ref = "facebook/opt-125m"
|
||||
model_path = tmp_path / (model_ref + ".tensors")
|
||||
config = TensorizerConfig(
|
||||
tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
|
||||
)
|
||||
|
||||
args = EngineArgs(model=model_ref)
|
||||
tensorize_vllm_model(args, config)
|
||||
|
||||
loader_tc = {"tensorizer_uri": str(model_path), illegal_value: "foo"}
|
||||
|
||||
try:
|
||||
vllm_runner(
|
||||
model_ref,
|
||||
load_format="tensorizer",
|
||||
model_loader_extra_config=loader_tc,
|
||||
)
|
||||
except RuntimeError:
|
||||
out, err = capfd.readouterr()
|
||||
combined_output = out + err
|
||||
assert (
|
||||
f"ValueError: {illegal_value} is not an allowed Tensorizer argument."
|
||||
) in combined_output
|
||||
@@ -1,603 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import fnmatch
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import shutil
|
||||
from tempfile import TemporaryDirectory
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.model_executor.model_loader import ShardedStateLoader
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
|
||||
# Create a sampling params object.
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0,
|
||||
max_tokens=256,
|
||||
ignore_eos=True,
|
||||
)
|
||||
|
||||
|
||||
def test_filter_subtensors():
|
||||
state_dict = {
|
||||
"a": torch.empty(2),
|
||||
"b": torch.empty((2, 4)),
|
||||
"c": torch.empty((2, 4, 8)),
|
||||
}
|
||||
state_dict.update(
|
||||
{
|
||||
"x": state_dict["b"],
|
||||
"y": state_dict["c"][1, 2, :],
|
||||
"z": state_dict["c"][1, :, 4],
|
||||
}
|
||||
)
|
||||
filtered_state_dict = ShardedStateLoader._filter_subtensors(state_dict)
|
||||
assert tuple(filtered_state_dict.keys()) == ("a", "b", "c")
|
||||
for key, tensor in filtered_state_dict.items():
|
||||
# NOTE: don't use `equal` here, as the tensor might contain NaNs
|
||||
assert tensor is state_dict[key]
|
||||
|
||||
|
||||
def _mock_pp_group(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
world_size: int,
|
||||
*,
|
||||
pp_rank: int = 0,
|
||||
tp_rank: int = 0,
|
||||
tp_world_size: int = 1,
|
||||
) -> None:
|
||||
"""Stub the parallel-state accessors used by the sharded state loader."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.get_pp_group",
|
||||
lambda: SimpleNamespace(world_size=world_size, rank_in_group=pp_rank),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.get_tp_group",
|
||||
lambda: SimpleNamespace(world_size=tp_world_size, rank_in_group=tp_rank),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.get_tensor_model_parallel_rank",
|
||||
lambda: tp_rank,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.distributed.get_pipeline_model_parallel_rank",
|
||||
lambda: pp_rank,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"world_size,expected",
|
||||
[
|
||||
(1, ShardedStateLoader.TP_ONLY_PATTERN),
|
||||
(2, ShardedStateLoader.PP_AND_TP_PATTERN),
|
||||
],
|
||||
)
|
||||
def test_get_default_pattern(monkeypatch, world_size, expected):
|
||||
"""The default pattern is chosen from the pipeline parallel group size."""
|
||||
_mock_pp_group(monkeypatch, world_size)
|
||||
assert ShardedStateLoader._get_default_pattern() == expected
|
||||
|
||||
|
||||
def test_render_default_pattern_default_tp(monkeypatch):
|
||||
"""Without a custom pattern and PP=1, legacy TP-only names are rendered."""
|
||||
_mock_pp_group(monkeypatch, 1)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(part=0)
|
||||
== "model-rank-0-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_default_pp(monkeypatch):
|
||||
"""Without a custom pattern and PP>1, pp/tp-aware names are rendered."""
|
||||
_mock_pp_group(monkeypatch, 2)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(part=0)
|
||||
== "model-pp-0-tp-0-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_custom_tp_only(monkeypatch):
|
||||
"""A custom TP-only pattern is honored when pipeline parallelism is off."""
|
||||
_mock_pp_group(monkeypatch, 1)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.TP_ONLY_PATTERN, part=0
|
||||
)
|
||||
== "model-rank-0-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_tp_only_with_pp_fails(monkeypatch):
|
||||
"""TP-only filenames would collide across pipeline stages; fail fast."""
|
||||
_mock_pp_group(monkeypatch, 2)
|
||||
with pytest.raises(ValueError, match="does not embed the pipeline parallel"):
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.TP_ONLY_PATTERN, part=0
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_pp_pattern_without_pp_fails(monkeypatch):
|
||||
"""A pp/tp-aware pattern cannot be rendered when pipeline parallelism is off."""
|
||||
_mock_pp_group(monkeypatch, 1)
|
||||
with pytest.raises(ValueError, match="requires pipeline parallelism"):
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_custom_pp(monkeypatch):
|
||||
"""A custom pp/tp-aware pattern is honored when pipeline parallelism is on."""
|
||||
_mock_pp_group(monkeypatch, 2)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
|
||||
)
|
||||
== "model-pp-0-tp-0-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern,match",
|
||||
[
|
||||
# tp field present but pp field missing: pipeline stages with the
|
||||
# same tp rank would write the same file.
|
||||
(
|
||||
"model-tp-{tp_rank}-part-{part}.safetensors",
|
||||
"does not embed the pipeline parallel",
|
||||
),
|
||||
# pp field present but tp field missing: TP workers would overwrite
|
||||
# each other's shards.
|
||||
(
|
||||
"model-pp-{pp_rank}-part-{part}.safetensors",
|
||||
"does not embed the tensor parallel",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_render_default_pattern_incomplete_custom_patterns_fail(
|
||||
monkeypatch, pattern, match
|
||||
):
|
||||
"""A custom pattern must distinguish every active rank (pp and tp)."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
|
||||
with pytest.raises(ValueError, match=match):
|
||||
ShardedStateLoader._render_default_pattern(pattern, part=0)
|
||||
|
||||
|
||||
def test_render_default_pattern_no_rank_field_with_tp_fails(monkeypatch):
|
||||
"""Without any rank field, TP workers would collide even when PP is off."""
|
||||
_mock_pp_group(monkeypatch, 1, tp_world_size=2)
|
||||
with pytest.raises(ValueError, match="does not embed the tensor parallel"):
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
"model-part-{part}.safetensors", part=0
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_custom_pp_tp1_allows_pp_only(monkeypatch):
|
||||
"""With TP=1 a pp-only custom pattern is safe and must be accepted."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_world_size=1)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
"model-pp-{pp_rank}-part-{part}.safetensors", part=0
|
||||
)
|
||||
== "model-pp-1-part-0.safetensors"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern,match",
|
||||
[
|
||||
# {{pp_rank}} renders as literal text: both stages would write the
|
||||
# same file, so it must not count as a pp field.
|
||||
(
|
||||
"model-{{pp_rank}}-tp-{tp_rank}-part-{part}.safetensors",
|
||||
"does not embed the pipeline parallel",
|
||||
),
|
||||
# Same for an escaped tp field: TP workers would overwrite each
|
||||
# other's shards.
|
||||
(
|
||||
"model-pp-{pp_rank}-tp-{{tp_rank}}-part-{part}.safetensors",
|
||||
"does not embed the tensor parallel",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_render_default_pattern_escaped_field_not_rank_field(
|
||||
monkeypatch, pattern, match
|
||||
):
|
||||
"""Escaped braces are literal text and cannot pose as rank fields."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
|
||||
with pytest.raises(ValueError, match=match):
|
||||
ShardedStateLoader._render_default_pattern(pattern, part=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pattern,world_size,tp_world_size,pp_rank,tp_rank,expected",
|
||||
[
|
||||
# Mixed tp alias works when pipeline parallelism is on.
|
||||
(
|
||||
"model-pp-{pp_rank}-rank-{rank}-part-{part}.safetensors",
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
0,
|
||||
"model-pp-1-rank-0-part-0.safetensors",
|
||||
),
|
||||
# tp_rank alias works for TP-only checkpoints too.
|
||||
(
|
||||
"model-tp-{tp_rank}-part-{part}.safetensors",
|
||||
1,
|
||||
2,
|
||||
0,
|
||||
3,
|
||||
"model-tp-3-part-0.safetensors",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_render_default_pattern_tp_aliases_accepted(
|
||||
monkeypatch, pattern, world_size, tp_world_size, pp_rank, tp_rank, expected
|
||||
):
|
||||
"""Both tp aliases are supplied at render time, so either works."""
|
||||
_mock_pp_group(
|
||||
monkeypatch,
|
||||
world_size,
|
||||
pp_rank=pp_rank,
|
||||
tp_rank=tp_rank,
|
||||
tp_world_size=tp_world_size,
|
||||
)
|
||||
assert ShardedStateLoader._render_default_pattern(pattern, part=0) == expected
|
||||
|
||||
|
||||
def test_render_default_pattern_unsupported_field_fails(monkeypatch):
|
||||
"""Unknown placeholders are rejected instead of KeyError-ing later."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_world_size=2)
|
||||
with pytest.raises(ValueError, match="unsupported placeholder"):
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
"model-{pp_rank}-{tp_rank}-{epoch}-part-{part}.safetensors", part=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pp_rank,tp_rank,expected",
|
||||
[
|
||||
(0, 0, "model-pp-0-tp-0-part-0.safetensors"),
|
||||
(1, 2, "model-pp-1-tp-2-part-0.safetensors"),
|
||||
(3, 5, "model-pp-3-tp-5-part-0.safetensors"),
|
||||
],
|
||||
)
|
||||
def test_render_default_pattern_non_zero_ranks(monkeypatch, pp_rank, tp_rank, expected):
|
||||
"""The rendered filename must embed the actual pp/tp ranks, not zeros."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=pp_rank, tp_rank=tp_rank)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(
|
||||
ShardedStateLoader.PP_AND_TP_PATTERN, part=0
|
||||
)
|
||||
== expected
|
||||
)
|
||||
|
||||
|
||||
def test_render_default_pattern_part_wildcard(monkeypatch):
|
||||
"""The glob wildcard '*' passes through as the part token for file lookup."""
|
||||
_mock_pp_group(monkeypatch, 1, tp_rank=4)
|
||||
assert (
|
||||
ShardedStateLoader._render_default_pattern(part="*")
|
||||
== "model-rank-4-part-*.safetensors"
|
||||
)
|
||||
|
||||
|
||||
def test_save_model_pp_aware_filenames(monkeypatch, tmp_path):
|
||||
"""save_model writes pp/tp-aware filenames for pipeline-parallel models."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=2)
|
||||
|
||||
saved: list[str] = []
|
||||
|
||||
def _capture(tensors, path, **kwargs):
|
||||
saved.append(os.path.basename(path))
|
||||
|
||||
monkeypatch.setattr("safetensors.torch.save_file", _capture)
|
||||
|
||||
model = SimpleNamespace(
|
||||
state_dict=lambda: {"w": torch.zeros(4, dtype=torch.float32)}
|
||||
)
|
||||
ShardedStateLoader.save_model(model, str(tmp_path))
|
||||
assert saved == ["model-pp-1-tp-2-part-0.safetensors"]
|
||||
|
||||
|
||||
def test_save_model_part_split(monkeypatch, tmp_path):
|
||||
"""save_model rolls over to a new part when max_size is exceeded."""
|
||||
_mock_pp_group(monkeypatch, 1, tp_rank=0)
|
||||
|
||||
saved: list[str] = []
|
||||
|
||||
def _capture(tensors, path, **kwargs):
|
||||
saved.append(os.path.basename(path))
|
||||
|
||||
monkeypatch.setattr("safetensors.torch.save_file", _capture)
|
||||
|
||||
# Two float32[4] tensors (16 B each); max_size=20 fits one but not two.
|
||||
model = SimpleNamespace(
|
||||
state_dict=lambda: {
|
||||
"a": torch.zeros(4, dtype=torch.float32),
|
||||
"b": torch.zeros(4, dtype=torch.float32),
|
||||
}
|
||||
)
|
||||
ShardedStateLoader.save_model(model, str(tmp_path), max_size=20)
|
||||
assert saved == [
|
||||
"model-rank-0-part-0.safetensors",
|
||||
"model-rank-0-part-1.safetensors",
|
||||
]
|
||||
|
||||
|
||||
def test_save_model_rejects_bad_pattern_before_overwrite(monkeypatch, tmp_path):
|
||||
"""An unsafe pattern is rejected before any shard file is written."""
|
||||
_mock_pp_group(monkeypatch, 2, pp_rank=1, tp_rank=0, tp_world_size=2)
|
||||
|
||||
# With the escaped pp field this pattern renders both stages to the
|
||||
# same literal name; the validation must fire before any write.
|
||||
colliding = tmp_path / "model-{pp_rank}-tp-0-part-0.safetensors"
|
||||
colliding.write_bytes(b"existing shard")
|
||||
before = sorted(p.name for p in tmp_path.iterdir())
|
||||
|
||||
model = SimpleNamespace(
|
||||
state_dict=lambda: {"w": torch.zeros(4, dtype=torch.float32)}
|
||||
)
|
||||
with pytest.raises(ValueError, match="does not embed the pipeline parallel"):
|
||||
ShardedStateLoader.save_model(
|
||||
model,
|
||||
str(tmp_path),
|
||||
pattern="model-{{pp_rank}}-tp-{tp_rank}-part-{part}.safetensors",
|
||||
)
|
||||
|
||||
# The pre-existing shard is untouched and nothing new was written.
|
||||
assert colliding.read_bytes() == b"existing shard"
|
||||
assert sorted(p.name for p in tmp_path.iterdir()) == before
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llama_3p2_1b_files():
|
||||
input_dir = hf_api().snapshot_download(
|
||||
"meta-llama/Llama-3.2-1B-Instruct", ignore_patterns=["*.bin*", "original/*"]
|
||||
)
|
||||
|
||||
yield input_dir
|
||||
|
||||
|
||||
def _run_writer(input_dir, output_dir, weights_patterns, **kwargs):
|
||||
llm_sharded_writer = LLM(model=input_dir, **kwargs)
|
||||
|
||||
# Dump worker states to output directory
|
||||
llm_sharded_writer.llm_engine.engine_core.save_sharded_state(path=output_dir)
|
||||
|
||||
# Copy metadata files to output directory
|
||||
for file in os.listdir(input_dir):
|
||||
if os.path.isdir(os.path.join(input_dir, file)):
|
||||
shutil.copytree(
|
||||
os.path.join(input_dir, file), os.path.join(output_dir, file)
|
||||
)
|
||||
elif not any(fnmatch.fnmatch(file, ext) for ext in weights_patterns):
|
||||
shutil.copy(os.path.join(input_dir, file), output_dir)
|
||||
|
||||
|
||||
def _run_generate(input_dir, queue: mp.Queue, **kwargs):
|
||||
llm = LLM(model=input_dir, **kwargs)
|
||||
gen = llm.generate(prompts, sampling_params)
|
||||
queue.put([g.outputs[0].__dict__ for g in gen])
|
||||
queue.close()
|
||||
queue.join_thread()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_lora", [False, True])
|
||||
@pytest.mark.parametrize("tp_size", [1, 2])
|
||||
def test_sharded_state_loader(
|
||||
enable_lora, tp_size, num_gpus_available, llama_3p2_1b_files
|
||||
):
|
||||
if num_gpus_available < tp_size:
|
||||
pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
|
||||
|
||||
weights_patterns = ("*.safetensors",)
|
||||
gpu_memory_utilization = 0.8
|
||||
input_dir = llama_3p2_1b_files
|
||||
ctx = mp.get_context("spawn")
|
||||
|
||||
# Keep batching deterministic: this test compares exact greedy outputs
|
||||
# across separate engine processes, whose schedulers may otherwise form
|
||||
# different batches depending on prompt-processing timing.
|
||||
platform_args = {"max_num_seqs": 1}
|
||||
|
||||
# Run in separate processes for memory & CUDA isolation
|
||||
with TemporaryDirectory() as output_dir:
|
||||
p = ctx.Process(
|
||||
target=_run_writer,
|
||||
args=(input_dir, output_dir, weights_patterns),
|
||||
kwargs=dict(
|
||||
tensor_parallel_size=tp_size,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
enforce_eager=True,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
p.join()
|
||||
|
||||
queue = ctx.Queue()
|
||||
|
||||
p = ctx.Process(
|
||||
target=_run_generate,
|
||||
args=(input_dir, queue),
|
||||
kwargs=dict(
|
||||
enable_lora=enable_lora,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
tensor_parallel_size=tp_size,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
# Call queue.get() before p.join() to prevent deadlock:
|
||||
# If p.join() is called before queue.get() and the queue is full,
|
||||
# the child process may block while writing to the queue and never
|
||||
# terminate, causing the parent to wait indefinitely on p.join().
|
||||
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
|
||||
out_before = queue.get()
|
||||
p.join()
|
||||
queue.close()
|
||||
queue.join_thread()
|
||||
|
||||
queue = ctx.Queue()
|
||||
|
||||
p = ctx.Process(
|
||||
target=_run_generate,
|
||||
args=(output_dir, queue),
|
||||
kwargs=dict(
|
||||
enable_lora=enable_lora,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
tensor_parallel_size=tp_size,
|
||||
load_format="sharded_state",
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
# Call queue.get() before p.join() to prevent deadlock:
|
||||
# If p.join() is called before queue.get() and the queue is full,
|
||||
# the child process may block while writing to the queue and never
|
||||
# terminate, causing the parent to wait indefinitely on p.join().
|
||||
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
|
||||
out_after = queue.get()
|
||||
p.join()
|
||||
queue.close()
|
||||
queue.join_thread()
|
||||
|
||||
assert out_before == out_after
|
||||
|
||||
|
||||
def _run_sharded_state_round_trip(
|
||||
input_dir: str,
|
||||
*,
|
||||
tp_size: int,
|
||||
pp_size: int,
|
||||
num_gpus_available: int,
|
||||
) -> None:
|
||||
"""Save a TP/PP sharded checkpoint, reload it, and compare outputs.
|
||||
|
||||
Args:
|
||||
input_dir: directory containing the original (un-sharded) model.
|
||||
tp_size: tensor parallel size used for both save and load.
|
||||
pp_size: pipeline parallel size used for both save and load.
|
||||
num_gpus_available: number of GPUs on the current machine.
|
||||
|
||||
"""
|
||||
if num_gpus_available < tp_size * pp_size:
|
||||
pytest.skip(f"Not enough GPUs for TP={tp_size}, PP={pp_size}")
|
||||
|
||||
weights_patterns = ("*.safetensors",)
|
||||
gpu_memory_utilization = 0.8
|
||||
ctx = mp.get_context("spawn")
|
||||
|
||||
# Keep batching deterministic: this test compares exact greedy outputs
|
||||
# across separate engine processes, whose schedulers may otherwise form
|
||||
# different batches depending on prompt-processing timing.
|
||||
platform_args = {"max_num_seqs": 1}
|
||||
parallel_args = dict(
|
||||
tensor_parallel_size=tp_size,
|
||||
pipeline_parallel_size=pp_size,
|
||||
)
|
||||
|
||||
with TemporaryDirectory() as output_dir:
|
||||
p = ctx.Process(
|
||||
target=_run_writer,
|
||||
args=(input_dir, output_dir, weights_patterns),
|
||||
kwargs=dict(
|
||||
**parallel_args,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
enforce_eager=True,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
p.join()
|
||||
assert p.exitcode == 0, f"writer process failed with exit code {p.exitcode}"
|
||||
|
||||
if pp_size > 1:
|
||||
# PP checkpoints must use the pp/tp-aware naming scheme so that
|
||||
# different pipeline stages write distinct files.
|
||||
sharded_files = [
|
||||
f for f in os.listdir(output_dir) if f.endswith(".safetensors")
|
||||
]
|
||||
assert sharded_files, "no sharded checkpoint files were written"
|
||||
assert any(
|
||||
fnmatch.fnmatch(f, "model-pp-*-tp-*-part-*.safetensors")
|
||||
for f in sharded_files
|
||||
), f"expected PP-aware filenames, got {sharded_files}"
|
||||
|
||||
queue = ctx.Queue()
|
||||
p = ctx.Process(
|
||||
target=_run_generate,
|
||||
args=(input_dir, queue),
|
||||
kwargs=dict(
|
||||
enable_lora=False,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
**parallel_args,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
# Call queue.get() before p.join() to prevent deadlock:
|
||||
# If p.join() is called before queue.get() and the queue is full,
|
||||
# the child process may block while writing to the queue and never
|
||||
# terminate, causing the parent to wait indefinitely on p.join().
|
||||
# See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
|
||||
out_before = queue.get()
|
||||
p.join()
|
||||
assert p.exitcode == 0, f"generate process failed with exit code {p.exitcode}"
|
||||
queue.close()
|
||||
queue.join_thread()
|
||||
|
||||
queue = ctx.Queue()
|
||||
p = ctx.Process(
|
||||
target=_run_generate,
|
||||
args=(output_dir, queue),
|
||||
kwargs=dict(
|
||||
enable_lora=False,
|
||||
gpu_memory_utilization=gpu_memory_utilization,
|
||||
load_format="sharded_state",
|
||||
**parallel_args,
|
||||
**platform_args,
|
||||
),
|
||||
)
|
||||
p.start()
|
||||
out_after = queue.get()
|
||||
p.join()
|
||||
assert p.exitcode == 0, f"generate process failed with exit code {p.exitcode}"
|
||||
queue.close()
|
||||
queue.join_thread()
|
||||
|
||||
assert out_before == out_after
|
||||
|
||||
|
||||
def test_sharded_state_loader_pp(num_gpus_available, llama_3p2_1b_files):
|
||||
"""Round-trip sharded state save/load with pipeline parallelism (PP=2)."""
|
||||
_run_sharded_state_round_trip(
|
||||
llama_3p2_1b_files,
|
||||
tp_size=1,
|
||||
pp_size=2,
|
||||
num_gpus_available=num_gpus_available,
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_state_loader_pp_tp(num_gpus_available, llama_3p2_1b_files):
|
||||
"""Round-trip sharded state save/load with pipeline + tensor parallelism
|
||||
(PP=2, TP=2)."""
|
||||
_run_sharded_state_round_trip(
|
||||
llama_3p2_1b_files,
|
||||
tp_size=2,
|
||||
pp_size=2,
|
||||
num_gpus_available=num_gpus_available,
|
||||
)
|
||||
@@ -1,486 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end tests for the IPC weight cache loader.
|
||||
|
||||
A cold start (no weight cache daemon, so the loader falls back to disk) and
|
||||
warm restarts (weights mapped from the daemon via CUDA IPC) must both serve
|
||||
identical outputs.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
|
||||
class WeightCacheDaemon:
|
||||
"""Context manager running the real weight cache daemon as a subprocess."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
tp_size: int,
|
||||
extra_args: list[str] | None = None,
|
||||
):
|
||||
# Short base path: Unix socket paths are limited to ~107 characters.
|
||||
self.socket_dir = tempfile.mkdtemp(prefix="vllm_ipc_")
|
||||
self.health_port = get_open_port()
|
||||
self._cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"vllm.entrypoints.cli.main",
|
||||
"preload",
|
||||
"--model",
|
||||
model,
|
||||
"--tensor-parallel-size",
|
||||
str(tp_size),
|
||||
"--weight-cache-socket-dir",
|
||||
self.socket_dir,
|
||||
"--enforce-eager",
|
||||
"--weight-cache-health-host",
|
||||
"127.0.0.1",
|
||||
"--weight-cache-health-port",
|
||||
str(self.health_port),
|
||||
*(extra_args or []),
|
||||
]
|
||||
self._proc: subprocess.Popen | None = None
|
||||
|
||||
def __enter__(self) -> "WeightCacheDaemon":
|
||||
self._proc = subprocess.Popen(
|
||||
self._cmd, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True
|
||||
)
|
||||
threading.Thread(target=self._drain_stderr, daemon=True).start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info) -> None:
|
||||
self._stop()
|
||||
|
||||
def _drain_stderr(self) -> None:
|
||||
assert self._proc is not None and self._proc.stderr is not None
|
||||
self._proc.stderr.read()
|
||||
|
||||
def check_health(self) -> None:
|
||||
url = f"http://127.0.0.1:{self.health_port}/health"
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=5) as response:
|
||||
assert response.status == 200
|
||||
except urllib.error.HTTPError as error:
|
||||
raise AssertionError(
|
||||
f"Weight cache daemon is not ready: HTTP {error.code}"
|
||||
) from error
|
||||
|
||||
def wait_until_ready(self, timeout_s: float = 900) -> None:
|
||||
"""Poll until READY (which covers the autotune tuners) or the preload
|
||||
process dies."""
|
||||
deadline = time.monotonic() + timeout_s
|
||||
while True:
|
||||
try:
|
||||
self.check_health()
|
||||
return
|
||||
except (AssertionError, urllib.error.URLError):
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
if self._proc is not None and self._proc.poll() is not None:
|
||||
raise AssertionError("vllm preload exited before ready") from None
|
||||
time.sleep(2)
|
||||
|
||||
def _stop(self) -> None:
|
||||
assert self._proc is not None
|
||||
self._proc.terminate()
|
||||
try:
|
||||
self._proc.wait(timeout=60)
|
||||
except subprocess.TimeoutExpired:
|
||||
self._proc.kill()
|
||||
self._proc.wait()
|
||||
shutil.rmtree(self.socket_dir, ignore_errors=True)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ModelCase:
|
||||
model: str
|
||||
prompts: list[str]
|
||||
images: list | None = None
|
||||
llm_kwargs: dict[str, Any] = field(default_factory=dict)
|
||||
daemon_args: list[str] = field(default_factory=list)
|
||||
# Additionally assert the daemons tune in place before serving and write
|
||||
# the on-disk FlashInfer autotune cache, and the restart engine reuses it
|
||||
# (needs SM90+/FlashInfer).
|
||||
check_flashinfer_cache: bool = False
|
||||
|
||||
|
||||
def generate(
|
||||
vllm_runner,
|
||||
case: ModelCase,
|
||||
socket_dir: str | None,
|
||||
fallback: bool,
|
||||
):
|
||||
extra_config = (
|
||||
{} if socket_dir is None else {"socket_dir": socket_dir, "fallback": fallback}
|
||||
)
|
||||
with vllm_runner(
|
||||
case.model,
|
||||
load_format="auto" if socket_dir is None else "ipc_cache",
|
||||
model_loader_extra_config=extra_config,
|
||||
# Greedy outputs are compared across engine restarts; cap the batch
|
||||
# size so scheduling cannot change the results.
|
||||
max_num_seqs=1,
|
||||
**case.llm_kwargs,
|
||||
) as llm:
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=16, ignore_eos=True)
|
||||
return llm.generate(case.prompts, sampling_params, images=case.images)
|
||||
|
||||
|
||||
# Both hybrid-recurrent models need chunked prefill for their mamba-style
|
||||
# cache mode.
|
||||
QWEN_CASE = ModelCase(
|
||||
model="Qwen/Qwen3.5-0.8B",
|
||||
prompts=[
|
||||
"Hello, my name is",
|
||||
"The capital of France is",
|
||||
],
|
||||
llm_kwargs=dict(
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=True,
|
||||
),
|
||||
)
|
||||
|
||||
# Kimi K3 adds the mxfp4-pack MoE path (post-load kernel setup runs in
|
||||
# pre-processed mode) and a vision tower, exercised with an image request.
|
||||
K3_CASE = ModelCase(
|
||||
model="tiny-random/kimi-k3",
|
||||
prompts=[
|
||||
"The capital of France is",
|
||||
"def quicksort(arr):",
|
||||
"<|kimi_image_placeholder|>What is shown in the image?",
|
||||
],
|
||||
images=[None, None, ImageAsset("stop_sign").pil_image],
|
||||
llm_kwargs=dict(
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
trust_remote_code=True,
|
||||
enable_chunked_prefill=True,
|
||||
# The image placeholder expands to >1k tokens for the stop_sign asset.
|
||||
max_model_len=4096,
|
||||
),
|
||||
daemon_args=["--trust-remote-code"],
|
||||
# Kimi K3 has tunable FlashInfer ops (a dense model like Qwen2.5 tunes
|
||||
# nothing and upstream never writes the cache file for it).
|
||||
check_flashinfer_cache=True,
|
||||
)
|
||||
|
||||
# Qwen3.5-0.8B ships one MTP layer in the target checkpoint, so method="mtp"
|
||||
# loads the draft from the same model. The daemon must cache it in its draft
|
||||
# group for the warm runs (fallback=False) to succeed.
|
||||
QWEN_MTP_CASE = ModelCase(
|
||||
model="Qwen/Qwen3.5-0.8B",
|
||||
prompts=[
|
||||
"Hello, my name is",
|
||||
"The capital of France is",
|
||||
],
|
||||
llm_kwargs=dict(
|
||||
gpu_memory_utilization=0.3,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=True,
|
||||
speculative_config={"method": "mtp", "num_speculative_tokens": 1},
|
||||
),
|
||||
daemon_args=[
|
||||
"--speculative-config",
|
||||
'{"method": "mtp", "num_speculative_tokens": 1}',
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[QWEN_CASE, K3_CASE, QWEN_MTP_CASE],
|
||||
ids=["qwen3.5", "kimi-k3", "qwen3.5-mtp"],
|
||||
)
|
||||
def test_ipc_cache_cold_start_and_warm_restart(
|
||||
vllm_runner, case: ModelCase, tmp_path, monkeypatch, capfd
|
||||
):
|
||||
"""Cold start falls back to disk; warm restarts load weights via CUDA IPC.
|
||||
|
||||
All runs must produce outputs identical to a default-loader baseline. The
|
||||
warm runs disable the disk fallback, so they only pass if the weights
|
||||
really came from the daemon — for the MTP case, both the target's and the
|
||||
draft's daemon groups.
|
||||
|
||||
Cases with ``check_flashinfer_cache`` additionally cover in-daemon
|
||||
autotune (default-on with `--enable-flashinfer-autotune`): each daemon
|
||||
tunes before binding its socket and must write the on-disk FlashInfer
|
||||
autotune cache, and the restart engine must load tactics from it instead
|
||||
of re-profiling. The daemon's cache file is keyed with the
|
||||
OPENAI_API_SERVER batch defaults while these engines run under LLM_CLASS,
|
||||
so they never hit the daemon's file by design; reuse is proven against
|
||||
the file the earlier in-process engines wrote.
|
||||
"""
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Weight cache IPC sharing requires CUDA or ROCm")
|
||||
if case is K3_CASE and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("Kimi K3 IPC weight cache requires an SM100 MXFP4 backend")
|
||||
if case.check_flashinfer_cache:
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
if not (has_flashinfer() and current_platform.has_device_capability(90)):
|
||||
pytest.skip("FlashInfer autotune requires FlashInfer and SM90+")
|
||||
# Isolate the on-disk autotune cache. The engines are in-process
|
||||
# (envs read lazily); the daemon subprocesses spawn later and
|
||||
# inherit it.
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", str(tmp_path))
|
||||
|
||||
# Baseline: plain disk loading with the default loader.
|
||||
baseline_outputs = generate(
|
||||
vllm_runner,
|
||||
case,
|
||||
None,
|
||||
fallback=True,
|
||||
)
|
||||
assert all(text for _, texts in baseline_outputs for text in texts)
|
||||
|
||||
# Cold start: no daemon is serving, so the loader falls back to disk.
|
||||
with tempfile.TemporaryDirectory(prefix="vllm_ipc_empty_") as empty_socket_dir:
|
||||
cold_outputs = generate(vllm_runner, case, empty_socket_dir, fallback=True)
|
||||
|
||||
if case.check_flashinfer_cache:
|
||||
# The baseline engine already wrote its own (LLM_CLASS-keyed) table.
|
||||
engine_cache_files = set(tmp_path.rglob("autotune_configs*.json"))
|
||||
|
||||
with WeightCacheDaemon(
|
||||
case.model,
|
||||
tp_size=1,
|
||||
extra_args=case.daemon_args,
|
||||
) as d:
|
||||
warm_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
|
||||
d.wait_until_ready()
|
||||
if case.check_flashinfer_cache:
|
||||
# Each daemon tunes before its ready report; exactly one new file
|
||||
# (their OPENAI_API_SERVER-keyed table) must appear.
|
||||
tuner_files = set(tmp_path.rglob("autotune_configs*.json"))
|
||||
tuner_files -= engine_cache_files
|
||||
assert len(tuner_files) == 1, (
|
||||
f"expected the tuner's tuned table, got {tuner_files}"
|
||||
)
|
||||
# Warm restart: a second engine lifetime against the same daemon.
|
||||
if case.check_flashinfer_cache:
|
||||
capfd.readouterr() # drain: the assertion reads the increment
|
||||
restart_outputs = generate(vllm_runner, case, d.socket_dir, fallback=False)
|
||||
if case.check_flashinfer_cache:
|
||||
# The restart's EngineCore subprocess inherits pytest's fds; the
|
||||
# flashinfer autotuner logs "[Autotuner]: Loaded N configs from
|
||||
# <path>" (INFO, not once-deduplicated) on a disk cache hit.
|
||||
out = capfd.readouterr()
|
||||
m = re.search(
|
||||
r"\[Autotuner\]: Loaded (\d+) configs from", out.out + out.err
|
||||
)
|
||||
assert m and int(m.group(1)) > 0, (
|
||||
"restart engine did not load the on-disk autotune cache"
|
||||
)
|
||||
|
||||
assert cold_outputs == baseline_outputs
|
||||
assert warm_outputs == baseline_outputs
|
||||
assert restart_outputs == baseline_outputs
|
||||
|
||||
|
||||
def _parallel(**kw):
|
||||
from types import SimpleNamespace
|
||||
|
||||
base = dict(
|
||||
tensor_parallel_size=1,
|
||||
pipeline_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
data_parallel_size_local=1,
|
||||
data_parallel_rank=0,
|
||||
nnodes=1,
|
||||
node_rank=0,
|
||||
)
|
||||
base.update(kw)
|
||||
return SimpleNamespace(**base)
|
||||
|
||||
|
||||
def test_daemon_places_tp_and_dp_ranks_on_local_gpus():
|
||||
"""Each node serves a contiguous global-rank block (DP-major), so without
|
||||
DP local GPU i is TP rank r*local+i, and single-node DP launchers offset
|
||||
by --data-parallel-start-rank."""
|
||||
from vllm.model_executor.model_loader.weight_cache.daemon import plan_local_ranks
|
||||
|
||||
tp_second_node = _parallel(tensor_parallel_size=8, nnodes=2, node_rank=1)
|
||||
assert plan_local_ranks(tp_second_node) == [(i, 0, 0, 4 + i) for i in range(4)]
|
||||
|
||||
dp_third_node = _parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=8
|
||||
)
|
||||
assert plan_local_ranks(dp_third_node) == [(i, 8 + i, 0, 0) for i in range(4)]
|
||||
|
||||
dp_tp = _parallel(
|
||||
tensor_parallel_size=2,
|
||||
data_parallel_size=4,
|
||||
data_parallel_size_local=2,
|
||||
data_parallel_rank=2,
|
||||
)
|
||||
assert plan_local_ranks(dp_tp) == [
|
||||
(0, 2, 0, 0),
|
||||
(1, 2, 0, 1),
|
||||
(2, 3, 0, 0),
|
||||
(3, 3, 0, 1),
|
||||
]
|
||||
|
||||
# DP + nnodes: node-local TP replicas
|
||||
dp_tp_node0 = _parallel(tensor_parallel_size=8, data_parallel_size=2, nnodes=2)
|
||||
assert plan_local_ranks(dp_tp_node0) == [(i, 0, 0, i) for i in range(8)]
|
||||
dp_tp_node1 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=2, node_rank=1
|
||||
)
|
||||
assert plan_local_ranks(dp_tp_node1) == [(i, 1, 0, i) for i in range(8)]
|
||||
|
||||
# DP + nnodes: TP group spans nodes (TP8 x DP2 on 4 nodes)
|
||||
tp_span_node1 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=1
|
||||
)
|
||||
assert plan_local_ranks(tp_span_node1) == [(i, 0, 0, 4 + i) for i in range(4)]
|
||||
tp_span_node2 = _parallel(
|
||||
tensor_parallel_size=8, data_parallel_size=2, nnodes=4, node_rank=2
|
||||
)
|
||||
assert plan_local_ranks(tp_span_node2) == [(i, 1, 0, i) for i in range(4)]
|
||||
|
||||
# DP + nnodes: several DP replicas per node (TP4 x DP4 on 2 nodes)
|
||||
dp_multi_node0 = _parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
|
||||
assert plan_local_ranks(dp_multi_node0) == [(i, i // 4, 0, i % 4) for i in range(8)]
|
||||
|
||||
# PP adds one daemon placement per stage while preserving TP rank order.
|
||||
pp_tp = _parallel(tensor_parallel_size=2, pipeline_parallel_size=2)
|
||||
assert plan_local_ranks(pp_tp) == [
|
||||
(0, 0, 0, 0),
|
||||
(1, 0, 0, 1),
|
||||
(2, 0, 1, 0),
|
||||
(3, 0, 1, 1),
|
||||
]
|
||||
|
||||
|
||||
def test_daemon_rejects_unmappable_parallelism():
|
||||
from vllm.model_executor.model_loader.weight_cache.daemon import (
|
||||
_reject_unsupported_parallelism,
|
||||
)
|
||||
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=12
|
||||
)
|
||||
)
|
||||
# DP combines with --nnodes when the world size divides evenly.
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(tensor_parallel_size=4, data_parallel_size=4, nnodes=2)
|
||||
)
|
||||
_reject_unsupported_parallelism(_parallel(pipeline_parallel_size=2))
|
||||
with pytest.raises(ValueError, match="evenly divide"):
|
||||
_reject_unsupported_parallelism(_parallel(tensor_parallel_size=3, nnodes=2))
|
||||
with pytest.raises(ValueError, match="evenly divide"):
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(tensor_parallel_size=2, data_parallel_size=3, nnodes=4)
|
||||
)
|
||||
with pytest.raises(ValueError, match="exceeds"):
|
||||
_reject_unsupported_parallelism(
|
||||
_parallel(
|
||||
data_parallel_size=16, data_parallel_size_local=4, data_parallel_rank=13
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_weight_cache_key_distinguishes_dp_ranks():
|
||||
from dataclasses import replace
|
||||
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
|
||||
|
||||
key = WeightCacheKey(
|
||||
checkpoint="ckpt",
|
||||
model_arch="Arch",
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
dtype="bf16",
|
||||
quantization=None,
|
||||
quant_config_hash="h",
|
||||
revision=None,
|
||||
vllm_version="v",
|
||||
dp_size=16,
|
||||
dp_rank=3,
|
||||
pp_size=2,
|
||||
pp_rank=1,
|
||||
)
|
||||
assert key.mismatched_fields(replace(key, dp_rank=4)) == ["dp_rank"]
|
||||
assert key.mismatched_fields(replace(key, dp_size=8, dp_rank=3)) == ["dp_size"]
|
||||
assert key.mismatched_fields(replace(key, pp_rank=0)) == ["pp_rank"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hf_quant_config", [None, {"quant_algo": "NVFP4"}])
|
||||
def test_weight_cache_key_distinguishes_nvfp4_activation_override(
|
||||
tmp_path, hf_quant_config
|
||||
):
|
||||
"""Cache keys isolate activation modes and reject ambiguous legacy hashes."""
|
||||
import json
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.model_executor.model_loader.weight_cache.protocol import WeightCacheKey
|
||||
from vllm.utils.hashing import safe_hash
|
||||
|
||||
model_config = SimpleNamespace(
|
||||
model=str(tmp_path),
|
||||
dtype=torch.bfloat16,
|
||||
quantization="modelopt_fp4",
|
||||
quantization_config=None,
|
||||
revision=None,
|
||||
hf_config=SimpleNamespace(
|
||||
architectures=["NemotronHForCausalLM"],
|
||||
quantization_config=hf_quant_config,
|
||||
),
|
||||
)
|
||||
static_key = WeightCacheKey.from_model_config(model_config, tp_size=1, tp_rank=0)
|
||||
model_config.quantization_config = QuantizationConfigArgs(
|
||||
moe={"activation": "nvfp4_per_token"}
|
||||
)
|
||||
per_token_key = WeightCacheKey.from_model_config(model_config, tp_size=1, tp_rank=0)
|
||||
assert static_key.mismatched_fields(per_token_key) == ["quant_config_hash"]
|
||||
|
||||
legacy_hash = (
|
||||
""
|
||||
if hf_quant_config is None
|
||||
else safe_hash(
|
||||
json.dumps(hf_quant_config, sort_keys=True).encode(),
|
||||
usedforsecurity=False,
|
||||
).hexdigest()
|
||||
)
|
||||
assert static_key.mismatched_fields(
|
||||
replace(static_key, quant_config_hash=legacy_hash)
|
||||
) == ["quant_config_hash"]
|
||||
|
||||
|
||||
def test_ipc_loader_copy_mode_reports_no_external_weight_memory():
|
||||
"""Copy mode clones the weights into the engine, so nothing is external
|
||||
(vllm_config is never touched)."""
|
||||
from vllm.config import LoadConfig
|
||||
from vllm.model_executor.model_loader.weight_cache.ipc_loader import (
|
||||
IpcModelLoader,
|
||||
)
|
||||
|
||||
copy_mode = LoadConfig(
|
||||
load_format="ipc_cache", model_loader_extra_config={"mode": "copy"}
|
||||
)
|
||||
loader = IpcModelLoader(copy_mode)
|
||||
assert loader.get_external_weight_memory(None) == 0
|
||||
@@ -1,93 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.testing._internal.two_tensor import TwoTensor
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.model_executor import parameter
|
||||
from vllm.model_executor.model_loader.weight_checksum import (
|
||||
compute_tensor_digests,
|
||||
zero_weights,
|
||||
)
|
||||
from vllm.v1.worker import gpu_worker
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _Model(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.linear = nn.Linear(2, 2)
|
||||
nn.init.ones_(self.linear.weight)
|
||||
nn.init.ones_(self.linear.bias)
|
||||
# A strided view, which cannot be reinterpreted as bytes in place.
|
||||
self.strided = nn.Parameter(torch.arange(1.0, 7.0)[::2])
|
||||
self.scalar = nn.Parameter(torch.tensor(2.0))
|
||||
# Loading never restores buffers, so reset must leave them alone.
|
||||
self.register_buffer("k_scale", torch.tensor(2.0))
|
||||
|
||||
|
||||
def test_digests_cover_every_parameter():
|
||||
assert set(compute_tensor_digests(_Model())) == {
|
||||
"linear.weight",
|
||||
"linear.bias",
|
||||
"strided",
|
||||
"scalar",
|
||||
}
|
||||
|
||||
|
||||
def test_zero_weights_changes_every_parameter_and_no_buffer():
|
||||
model = _Model()
|
||||
before = compute_tensor_digests(model)
|
||||
zero_weights(model)
|
||||
after = compute_tensor_digests(model)
|
||||
assert all(before[name] != after[name] for name in before)
|
||||
assert model.k_scale.item() == 2.0
|
||||
|
||||
|
||||
def test_shared_weight_partitions_are_hashed_and_reset(monkeypatch):
|
||||
monkeypatch.setattr(parameter, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(parameter, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
model = nn.Module()
|
||||
model.transform = parameter.SharedWeightParameter(weight_loader=None)
|
||||
model.transform.add_partition(0, object(), 2, 2)
|
||||
model.transform.partitions[0].data.fill_(3.0)
|
||||
|
||||
before = compute_tensor_digests(model)
|
||||
zero_weights(model)
|
||||
assert set(before) == {"transform.0"}
|
||||
assert compute_tensor_digests(model) != before
|
||||
|
||||
|
||||
def test_tensor_subclass_inner_tensors_are_hashed_and_reset():
|
||||
model = nn.Module()
|
||||
model.weight = nn.Parameter(
|
||||
TwoTensor(torch.ones(2, 2), torch.full((2, 2), 2.0)), requires_grad=False
|
||||
)
|
||||
|
||||
before = compute_tensor_digests(model)
|
||||
zero_weights(model)
|
||||
assert set(before) == {"weight.a", "weight.b"}
|
||||
assert all(
|
||||
before[name] != digest for name, digest in compute_tensor_digests(model).items()
|
||||
)
|
||||
|
||||
|
||||
def test_dense_dp_replicas_get_distinct_key_prefixes(monkeypatch):
|
||||
for name in ("get_tp_group", "get_pp_group", "get_pcp_group"):
|
||||
monkeypatch.setattr(gpu_worker, name, lambda: SimpleNamespace(rank_in_group=0))
|
||||
prefixes = []
|
||||
for dp_rank in (0, 1):
|
||||
worker = object.__new__(Worker)
|
||||
worker.parallel_config = ParallelConfig(
|
||||
data_parallel_size=2, data_parallel_rank=dp_rank
|
||||
)
|
||||
# Dense DP resets data_parallel_rank to 0 in each engine.
|
||||
worker.parallel_config.reconfigure_for_independent_dp_rank()
|
||||
prefixes.append(worker._weight_checksum_prefix())
|
||||
assert prefixes == ["dp0:pp0:pcp0:tp0:", "dp1:pp0:pcp0:tp0:"]
|
||||
@@ -1,266 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
B12xFp8BlockScaledMMKernel,
|
||||
B12xMxFp4LinearKernel,
|
||||
B12xMxfp8LinearKernel,
|
||||
B12xNvFp4LinearKernel,
|
||||
B12xTensorFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.warmup.b12x_warmup import b12x_warmup
|
||||
from vllm.utils.b12x import B12xWarmupUnit, b12x_warmup_token_counts
|
||||
|
||||
|
||||
def test_b12x_warmup_token_counts_cover_serving_regimes() -> None:
|
||||
assert b12x_warmup_token_counts(
|
||||
max_tokens=2048,
|
||||
cudagraph_capture_sizes=[1, 2, 8, 128],
|
||||
) == (1, 2, 8, 128, 2048)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_embedding", [False, True])
|
||||
def test_b12x_block_quant_warmup_covers_each_layers_plans(monkeypatch, is_embedding):
|
||||
from vllm.model_executor.layers.quantization.modelopt_block_quant import (
|
||||
ModelOptBlockQuantLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.warmup.b12x_warmup import _collect_warmup_units
|
||||
|
||||
method = object.__new__(ModelOptBlockQuantLinearMethod)
|
||||
method.codec = "q8_0"
|
||||
method.is_embedding = is_embedding
|
||||
model = torch.nn.ModuleList([torch.nn.Module(), torch.nn.Module()])
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
method, "apply", lambda layer, x: calls.append((layer, x.shape, x.dtype))
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
method,
|
||||
"embedding",
|
||||
lambda layer, ids: calls.append((layer, ids.shape, ids.dtype)),
|
||||
)
|
||||
for index, layer in enumerate(model):
|
||||
layer.b12x_warmup_provider = method
|
||||
if is_embedding:
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.empty((8, 8, 34), dtype=torch.uint8), requires_grad=False
|
||||
)
|
||||
layer.b12x_embedding_plans = {
|
||||
dtype: SimpleNamespace(
|
||||
handle=2 * index + offset, query=SimpleNamespace(max_rows=32)
|
||||
)
|
||||
for offset, dtype in enumerate((torch.int32, torch.int64))
|
||||
}
|
||||
else:
|
||||
layer.b12x_block_weight = SimpleNamespace(
|
||||
in_features=256, values=torch.empty(1)
|
||||
)
|
||||
layer.b12x_block_plan = SimpleNamespace(
|
||||
handle=index, token_counts=(1, 4, 32)
|
||||
)
|
||||
|
||||
units = list(_collect_warmup_units(model, (1, 17, 32), torch.bfloat16))
|
||||
assert len(units) == 2
|
||||
for unit in units:
|
||||
unit.compile()
|
||||
expected = (
|
||||
[((32,), torch.int32), ((32,), torch.int64)]
|
||||
if is_embedding
|
||||
else [((rows, 256), torch.bfloat16) for rows in (1, 4, 17, 32)]
|
||||
)
|
||||
assert calls == [
|
||||
(layer, shape, dtype) for layer in model for shape, dtype in expected
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kernel_cls", "module_name", "call_name", "layer", "name"),
|
||||
[
|
||||
(
|
||||
B12xMxFp4LinearKernel,
|
||||
"vllm.model_executor.kernels.linear.mxfp4.b12x",
|
||||
"_apply_b12x_mxfp4_linear",
|
||||
SimpleNamespace(
|
||||
weight=torch.empty((48, 64), dtype=torch.uint8),
|
||||
weight_scale=torch.empty((128, 4), dtype=torch.uint8),
|
||||
),
|
||||
"MXFP4",
|
||||
),
|
||||
(
|
||||
B12xNvFp4LinearKernel,
|
||||
"vllm.model_executor.kernels.linear.nvfp4.b12x",
|
||||
"_apply_b12x_nvfp4_linear",
|
||||
SimpleNamespace(
|
||||
weight=torch.empty((48, 64), dtype=torch.uint8),
|
||||
weight_scale=torch.empty((128, 8), dtype=torch.float8_e4m3fn),
|
||||
input_global_scale_inv=torch.tensor(2.0),
|
||||
alpha=torch.tensor(0.25),
|
||||
),
|
||||
"NVFP4",
|
||||
),
|
||||
(
|
||||
B12xFp8BlockScaledMMKernel,
|
||||
"vllm.model_executor.kernels.linear.scaled_mm.b12x",
|
||||
"_run_b12x_fp8_block_scaled_mm",
|
||||
SimpleNamespace(
|
||||
weight=torch.empty((256, 128), dtype=torch.float8_e4m3fn),
|
||||
weight_scale_inv=torch.empty((2, 1), dtype=torch.float32),
|
||||
),
|
||||
"block-FP8",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_b12x_warmup_units_cover_token_counts(
|
||||
monkeypatch,
|
||||
kernel_cls,
|
||||
module_name: str,
|
||||
call_name: str,
|
||||
layer,
|
||||
name: str,
|
||||
) -> None:
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
importlib.import_module(module_name),
|
||||
call_name,
|
||||
lambda *args: calls.append(args),
|
||||
)
|
||||
kernel = object.__new__(kernel_cls)
|
||||
|
||||
unit = kernel.get_b12x_warmup_unit(layer, (1, 8), torch.bfloat16)
|
||||
unit.compile()
|
||||
|
||||
assert unit.name == name
|
||||
assert [args[0].shape[0] for args in calls] == [1, 8]
|
||||
assert unit.key[-1] == torch.bfloat16
|
||||
|
||||
|
||||
def test_b12x_mxfp8_warmup_unit(monkeypatch) -> None:
|
||||
import vllm.model_executor.kernels.linear.mxfp8.b12x as b12x_mod
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
b12x_mod,
|
||||
"_import_b12x_mxfp8",
|
||||
lambda: SimpleNamespace(
|
||||
mm=lambda *args, **kwargs: calls.append((args, kwargs))
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
b12x_mod,
|
||||
"current_stream",
|
||||
lambda: SimpleNamespace(cuda_stream=object()),
|
||||
)
|
||||
packed_weight = SimpleNamespace(
|
||||
in_features=128,
|
||||
padded_in_features=128,
|
||||
out_features=256,
|
||||
weight=SimpleNamespace(values=torch.empty(1)),
|
||||
)
|
||||
layer = SimpleNamespace(b12x_mxfp8_packed_weight=packed_weight)
|
||||
kernel = object.__new__(B12xMxfp8LinearKernel)
|
||||
|
||||
unit = kernel.get_b12x_warmup_unit(layer, (1, 8), torch.float16)
|
||||
unit.compile()
|
||||
|
||||
assert [args[0].shape for args, _ in calls] == [(1, 128), (8, 128)]
|
||||
assert [kwargs["expected_m"] for _, kwargs in calls] == [1, 8]
|
||||
|
||||
|
||||
def test_b12x_tensor_fp8_warmup_unit(monkeypatch) -> None:
|
||||
import vllm.model_executor.kernels.linear.scaled_mm.b12x as b12x_mod
|
||||
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
b12x_mod,
|
||||
"_import_b12x_tensor_fp8",
|
||||
lambda: SimpleNamespace(
|
||||
prewarm=lambda *args, **kwargs: calls.append((args, kwargs))
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
b12x_mod,
|
||||
"current_stream",
|
||||
lambda: SimpleNamespace(cuda_stream=object()),
|
||||
)
|
||||
packed_weight = SimpleNamespace(
|
||||
in_features=128,
|
||||
padded_in_features=128,
|
||||
out_features=256,
|
||||
values=torch.empty(1),
|
||||
)
|
||||
layer = SimpleNamespace(b12x_tensor_fp8_packed_weight=packed_weight)
|
||||
kernel = object.__new__(B12xTensorFP8ScaledMMLinearKernel)
|
||||
|
||||
unit = kernel.get_b12x_warmup_unit(layer, (1, 8), torch.bfloat16)
|
||||
unit.compile()
|
||||
|
||||
assert calls[0][0] == (packed_weight, (1, 8))
|
||||
assert calls[0][1]["out_dtype"] == torch.bfloat16
|
||||
|
||||
|
||||
def test_b12x_warmup_deduplicates_registered_signatures(monkeypatch) -> None:
|
||||
import vllm.model_executor.warmup.b12x_warmup as warmup_mod
|
||||
|
||||
calls: list[tuple[str, tuple[int, ...], torch.dtype]] = []
|
||||
|
||||
class Provider:
|
||||
def get_b12x_warmup_unit(self, layer, token_counts, output_dtype):
|
||||
return B12xWarmupUnit(
|
||||
name="fake",
|
||||
key=(type(self), layer.shape, output_dtype),
|
||||
compile=lambda: calls.append((layer.name, token_counts, output_dtype)),
|
||||
)
|
||||
|
||||
provider = Provider()
|
||||
layers = [
|
||||
SimpleNamespace(name="first", shape=(128, 256), b12x_warmup_provider=provider),
|
||||
SimpleNamespace(
|
||||
name="duplicate", shape=(128, 256), b12x_warmup_provider=provider
|
||||
),
|
||||
SimpleNamespace(name="second", shape=(256, 256), b12x_warmup_provider=provider),
|
||||
SimpleNamespace(),
|
||||
]
|
||||
scans = 0
|
||||
|
||||
def modules():
|
||||
nonlocal scans
|
||||
scans += 1
|
||||
return iter(layers)
|
||||
|
||||
worker = SimpleNamespace(
|
||||
get_model=lambda: SimpleNamespace(modules=modules),
|
||||
scheduler_config=SimpleNamespace(
|
||||
max_num_batched_tokens=8,
|
||||
max_num_scheduled_tokens=32,
|
||||
),
|
||||
model_config=SimpleNamespace(dtype=torch.float32),
|
||||
vllm_config=SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(compile_sizes=[4, 16])
|
||||
),
|
||||
)
|
||||
platform = SimpleNamespace(
|
||||
is_cuda=lambda: True,
|
||||
is_device_capability_family=lambda family: family == 120,
|
||||
)
|
||||
synchronized = []
|
||||
monkeypatch.setattr(warmup_mod, "current_platform", platform)
|
||||
monkeypatch.setattr(
|
||||
warmup_mod.torch.accelerator,
|
||||
"synchronize",
|
||||
lambda: synchronized.append(True),
|
||||
)
|
||||
|
||||
b12x_warmup(worker, [1, 2])
|
||||
|
||||
assert scans == 1
|
||||
assert calls == [
|
||||
("first", (1, 2, 4, 8, 16, 32), torch.bfloat16),
|
||||
("second", (1, 2, 4, 8, 16, 32), torch.bfloat16),
|
||||
]
|
||||
assert synchronized == [True]
|
||||
@@ -1,165 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for CPU unquantized GEMM dispatch behavior."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers import utils
|
||||
from vllm.platforms import CpuArchEnum, current_platform
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
|
||||
def test_dispatch_prepacks_arm_bf16_causal_conv(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
current_platform, "get_cpu_architecture", lambda: CpuArchEnum.ARM
|
||||
)
|
||||
monkeypatch.setattr(torch.cpu, "get_capabilities", lambda: {"bf16": True})
|
||||
monkeypatch.setattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False)
|
||||
|
||||
packed = torch.randn(32, 4, dtype=torch.bfloat16)
|
||||
pack_inputs = []
|
||||
|
||||
def pack(weight):
|
||||
pack_inputs.append(weight.clone())
|
||||
return packed
|
||||
|
||||
monkeypatch.setattr(utils.ops, "causal_conv1d_weight_pack", pack)
|
||||
original = torch.randn(32, 1, 4, dtype=torch.bfloat16)
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(original.clone(), requires_grad=False)
|
||||
|
||||
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
|
||||
|
||||
assert len(pack_inputs) == 1
|
||||
# Check causal_conv1d_weight_pack received the expected weights
|
||||
torch.testing.assert_close(pack_inputs[0], original.view(32, 4))
|
||||
# Check we have correctly stashed the unpacked weights
|
||||
torch.testing.assert_close(layer._cpu_unpacked_conv_weight, original.view(32, 4))
|
||||
# Check we have correctly stored the packed weights
|
||||
assert layer.weight.data_ptr() == packed.data_ptr()
|
||||
|
||||
|
||||
def test_dispatch_does_not_pack_3d_expert_weight(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
current_platform, "get_cpu_architecture", lambda: CpuArchEnum.ARM
|
||||
)
|
||||
monkeypatch.setattr(torch.cpu, "get_capabilities", lambda: {"bf16": True})
|
||||
monkeypatch.setattr(torch.cpu, "_is_avx512_bf16_supported", lambda: False)
|
||||
pack_calls = []
|
||||
monkeypatch.setattr(
|
||||
utils.ops,
|
||||
"causal_conv1d_weight_pack",
|
||||
lambda weight: pack_calls.append(weight),
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.randn(8, 32, 4, dtype=torch.bfloat16), requires_grad=False
|
||||
)
|
||||
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
|
||||
assert pack_calls == []
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def _mock_zentorch_linear_unary():
|
||||
"""Register a mock zentorch_linear_unary op when zentorch is not installed.
|
||||
|
||||
Allows the dispatch tests to run in CI without a real zentorch build.
|
||||
Skips registration when zentorch is already available.
|
||||
"""
|
||||
if hasattr(torch.ops.zentorch, "zentorch_linear_unary"):
|
||||
yield
|
||||
return
|
||||
|
||||
lib_def = torch.library.Library("zentorch", "DEF")
|
||||
lib_def.define(
|
||||
"zentorch_linear_unary("
|
||||
"Tensor input, "
|
||||
"Tensor weight, "
|
||||
"Tensor? bias, "
|
||||
"bool is_weight_prepacked=False"
|
||||
") -> Tensor"
|
||||
)
|
||||
|
||||
lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
|
||||
lib_impl.impl(
|
||||
"zentorch_linear_unary",
|
||||
lambda input, weight, bias, is_weight_prepacked=False: (
|
||||
torch.nn.functional.linear(input, weight, bias)
|
||||
),
|
||||
)
|
||||
|
||||
yield
|
||||
|
||||
lib_impl._destroy()
|
||||
lib_def._destroy()
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
|
||||
def test_dispatch_cpu_unquantized_gemm_uses_zentorch_on_zen(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
|
||||
|
||||
layer = torch.nn.Linear(16, 8, bias=True)
|
||||
x = torch.randn(4, 16)
|
||||
expected = torch.nn.functional.linear(x, layer.weight, layer.bias)
|
||||
|
||||
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
|
||||
output = layer.cpu_linear(x, layer.weight, layer.bias)
|
||||
|
||||
torch.testing.assert_close(output, expected)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
|
||||
def test_dispatch_cpu_unquantized_gemm_zen_remove_weight(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
|
||||
|
||||
layer = torch.nn.Linear(16, 8, bias=True)
|
||||
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
|
||||
|
||||
assert layer.weight.numel() == 0
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
|
||||
def test_dispatch_cpu_unquantized_gemm_logs_zentorch_dispatch(monkeypatch):
|
||||
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
|
||||
expected_prepacked = bool(utils.envs.VLLM_ZENTORCH_WEIGHT_PREPACK) and hasattr(
|
||||
torch.ops.zentorch, "zentorch_weight_prepack_for_linear"
|
||||
)
|
||||
|
||||
log_calls = []
|
||||
monkeypatch.setattr(
|
||||
utils.logger, "debug_once", lambda *args: log_calls.append(args)
|
||||
)
|
||||
|
||||
layer = torch.nn.Linear(16, 8, bias=True)
|
||||
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
|
||||
|
||||
assert log_calls == [
|
||||
(
|
||||
"CPU unquantized GEMM dispatch: using zentorch_linear_unary (prepacked=%s)",
|
||||
expected_prepacked,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
|
||||
@pytest.mark.parametrize("weight_dtype", [torch.bfloat16, torch.float16, torch.float32])
|
||||
def test_dispatch_cpu_unquantized_gemm_remove_weight_keeps_dtype(
|
||||
monkeypatch, weight_dtype
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
|
||||
|
||||
layer = torch.nn.Linear(16, 8, bias=False, dtype=weight_dtype)
|
||||
loading_dtype = (
|
||||
torch.float32 if weight_dtype is not torch.float32 else torch.bfloat16
|
||||
)
|
||||
with set_default_torch_dtype(loading_dtype):
|
||||
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
|
||||
|
||||
assert layer.weight.numel() == 0
|
||||
assert layer.weight.dtype is weight_dtype
|
||||
|
||||
x = torch.randn(4, 16, dtype=weight_dtype)
|
||||
output = layer.cpu_linear(x, layer.weight, None)
|
||||
assert not output.isnan().any()
|
||||
@@ -1,267 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
VllmConfig,
|
||||
get_cached_compilation_config,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.model_executor.custom_op import CustomOp, op_registry
|
||||
from vllm.model_executor.layers.activation import (
|
||||
GeluAndMul,
|
||||
ReLUSquaredActivation,
|
||||
SiluAndMul,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_router import (
|
||||
dispatch_topk_sigmoid_func,
|
||||
dispatch_topk_softmax_func,
|
||||
vllm_topk_sigmoid,
|
||||
vllm_topk_softmax,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
RMS_NORM_SUPPORTED_DTYPES = [torch.float16, torch.bfloat16]
|
||||
|
||||
|
||||
# Registered subclass for test
|
||||
@CustomOp.register("relu3")
|
||||
class Relu3(ReLUSquaredActivation):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"env, compilation_mode, backend, ops_enabled, default_on",
|
||||
[
|
||||
# Default values based on compile level
|
||||
# - All by default (no Inductor compilation)
|
||||
(None, 0, "eager", [True] * 4, True),
|
||||
(None, 1, "eager", [True] * 4, True),
|
||||
(None, 2, "eager", [True] * 4, True),
|
||||
(None, 3, "eager", [True] * 4, True),
|
||||
# - None by default (with Inductor)
|
||||
(None, 0, "inductor", [True] * 4, True),
|
||||
# - None by default (with Inductor)
|
||||
(None, 1, "inductor", [False] * 4, False),
|
||||
(None, 2, "inductor", [False] * 4, False),
|
||||
(None, 3, "inductor", [False] * 4, False),
|
||||
# Explicitly enabling/disabling
|
||||
#
|
||||
# Default: all
|
||||
#
|
||||
# All but SiluAndMul
|
||||
("+rms_norm,-silu_and_mul", 0, "inductor", [1, 0, 1, 1], True),
|
||||
# Only ReLU3
|
||||
("none,-rms_norm,+relu3", 1, "eager", [0, 0, 0, 1], False),
|
||||
# All but SiluAndMul
|
||||
("all,-silu_and_mul", 2, "inductor", [1, 0, 1, 1], True),
|
||||
# All but ReLU3 (even if ReLU2 is on)
|
||||
("-relu3,+relu2", 3, "eager", [1, 1, 1, 0], True),
|
||||
# RMSNorm and SiluAndMul
|
||||
("none,-relu3,+rms_norm,+silu_and_mul", 3, "eager", [1, 1, 0, 0], False),
|
||||
# All but RMSNorm
|
||||
("-rms_norm", 3, "eager", [0, 1, 1, 1], True),
|
||||
#
|
||||
# Default: none
|
||||
#
|
||||
# Only ReLU3
|
||||
("none,+relu3", 3, "inductor", [0, 0, 0, 1], False),
|
||||
# All but RMSNorm
|
||||
("all,-rms_norm", 3, "inductor", [0, 1, 1, 1], True),
|
||||
],
|
||||
)
|
||||
def test_enabled_ops(
|
||||
env: str | None,
|
||||
compilation_mode: int,
|
||||
backend: str,
|
||||
ops_enabled: list[int],
|
||||
default_on: bool,
|
||||
):
|
||||
custom_ops = env.split(",") if env else []
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
backend=backend, mode=compilation_mode, custom_ops=custom_ops
|
||||
)
|
||||
)
|
||||
get_cached_compilation_config.cache_clear()
|
||||
with set_current_vllm_config(vllm_config):
|
||||
assert CustomOp.default_on() == default_on
|
||||
|
||||
ops_enabled = [bool(x) for x in ops_enabled]
|
||||
|
||||
assert RMSNorm(1024).enabled() == ops_enabled[0]
|
||||
assert op_registry["rms_norm"].enabled() == ops_enabled[0]
|
||||
|
||||
assert SiluAndMul().enabled() == ops_enabled[1]
|
||||
assert op_registry["silu_and_mul"].enabled() == ops_enabled[1]
|
||||
|
||||
assert GeluAndMul().enabled() == ops_enabled[2]
|
||||
assert op_registry["gelu_and_mul"].enabled() == ops_enabled[2]
|
||||
|
||||
# If registered, subclasses should follow their own name
|
||||
assert Relu3().enabled() == ops_enabled[3]
|
||||
assert op_registry["relu3"].enabled() == ops_enabled[3]
|
||||
|
||||
# Unregistered subclass
|
||||
class SiluAndMul2(SiluAndMul):
|
||||
pass
|
||||
|
||||
# Subclasses should not require registration
|
||||
assert SiluAndMul2().enabled() == SiluAndMul().enabled()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"env", ["all,none", "all,+rms_norm,all", "+rms_norm,-rms_norm"]
|
||||
)
|
||||
def test_enabled_ops_invalid(env: str):
|
||||
with pytest.raises(Exception): # noqa
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(custom_ops=env.split(","))
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
RMSNorm(1024).enabled()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
)
|
||||
def test_topk_softmax_dispatch(use_rocm_aiter: bool):
|
||||
topk_func = dispatch_topk_softmax_func(use_rocm_aiter)
|
||||
|
||||
if current_platform.is_rocm() and use_rocm_aiter:
|
||||
assert topk_func == rocm_aiter_ops.topk_softmax
|
||||
else:
|
||||
assert topk_func == vllm_topk_softmax
|
||||
|
||||
|
||||
def _topk_gating_launch(
|
||||
num_tokens: int = 4,
|
||||
num_experts: int = 512,
|
||||
topk: int = 10,
|
||||
num_shared_experts: int = 0,
|
||||
weights_width: int | None = None,
|
||||
ids_width: int | None = None,
|
||||
contiguous: bool = True,
|
||||
) -> dict:
|
||||
"""Tensors for one softmax top-k launch, as dispatch_topk_softmax_func sees them."""
|
||||
width = topk + num_shared_experts
|
||||
weights = torch.empty(num_tokens, weights_width or width)
|
||||
ids = torch.empty(num_tokens, ids_width or width, dtype=torch.int32)
|
||||
ids = ids[:, :topk]
|
||||
if contiguous:
|
||||
gating = torch.empty(num_tokens, num_experts + num_shared_experts)
|
||||
else:
|
||||
gating = torch.empty(num_experts + num_shared_experts + 3, num_tokens).t()
|
||||
return dict(topk_weights=weights, topk_indices=ids, gating_output=gating)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"launch, num_shared_experts, scoring_func, expect_gating",
|
||||
[
|
||||
pytest.param({}, 0, "", True, id="plain-softmax"),
|
||||
pytest.param({"num_tokens": 4096}, 0, "", True, id="max-tokens"),
|
||||
pytest.param({"num_tokens": 4097}, 0, "", False, id="too-many-tokens"),
|
||||
pytest.param({"contiguous": False}, 0, "", False, id="non-contiguous"),
|
||||
pytest.param({}, 0, "sigmoid", False, id="scoring-without-shared"),
|
||||
pytest.param({"num_shared_experts": 1}, 1, "sigmoid", True, id="one-shared"),
|
||||
pytest.param({"num_shared_experts": 8}, 8, "sigmoid", True, id="eight-shared"),
|
||||
pytest.param(
|
||||
{"num_shared_experts": 3}, 3, "sigmoid", False, id="unsupported-count"
|
||||
),
|
||||
pytest.param({"num_shared_experts": 1}, 1, "", False, id="no-shared-scoring"),
|
||||
pytest.param(
|
||||
{"num_shared_experts": 1}, 1, "softmax", False, id="wrong-shared-scoring"
|
||||
),
|
||||
pytest.param(
|
||||
{"num_shared_experts": 1, "weights_width": 10},
|
||||
1,
|
||||
"sigmoid",
|
||||
False,
|
||||
id="weights-too-narrow",
|
||||
),
|
||||
pytest.param(
|
||||
{"num_shared_experts": 1, "ids_width": 12},
|
||||
1,
|
||||
"sigmoid",
|
||||
False,
|
||||
id="row-stride-mismatch",
|
||||
),
|
||||
pytest.param(
|
||||
{"num_tokens": 4097, "num_shared_experts": 1},
|
||||
1,
|
||||
"sigmoid",
|
||||
False,
|
||||
id="shared-too-many-tokens",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("gating_enabled", [True, False])
|
||||
def test_topk_softmax_dispatch_aiter_topk_gating(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
launch: dict,
|
||||
num_shared_experts: int,
|
||||
scoring_func: str,
|
||||
expect_gating: bool,
|
||||
gating_enabled: bool,
|
||||
):
|
||||
"""AITER launches use topk_gating only where it is enabled, and only when it
|
||||
supports them.
|
||||
|
||||
Every other AITER launch must reach the legacy topk_softmax, so this is the
|
||||
one place the choice is made.
|
||||
"""
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops, "is_topk_gating_enabled", lambda: gating_enabled
|
||||
)
|
||||
topk_func = dispatch_topk_softmax_func(
|
||||
True,
|
||||
**_topk_gating_launch(**launch),
|
||||
num_shared_experts=num_shared_experts,
|
||||
shared_expert_scoring_func=scoring_func,
|
||||
)
|
||||
if expect_gating and gating_enabled:
|
||||
assert topk_func == rocm_aiter_ops.topk_gating
|
||||
else:
|
||||
assert topk_func == rocm_aiter_ops.topk_softmax
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"cdna, fused_moe_enabled, expected",
|
||||
[
|
||||
pytest.param(3, True, True, id="cdna3"),
|
||||
pytest.param(4, True, True, id="cdna4"),
|
||||
pytest.param(2, True, False, id="cdna2"),
|
||||
pytest.param(3, False, False, id="cdna3-aiter-moe-off"),
|
||||
pytest.param(4, False, False, id="cdna4-aiter-moe-off"),
|
||||
],
|
||||
)
|
||||
def test_topk_gating_enabled_by_cdna(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
cdna: int,
|
||||
fused_moe_enabled: bool,
|
||||
expected: bool,
|
||||
):
|
||||
"""topk_gating is on for CDNA3 and CDNA4 with AITER MoE enabled, nowhere else."""
|
||||
monkeypatch.setattr("vllm._aiter_ops.is_aiter_found_and_supported", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops, "is_fused_moe_enabled", lambda: fused_moe_enabled
|
||||
)
|
||||
monkeypatch.setattr("vllm.platforms.rocm.get_cdna_version", lambda: cdna)
|
||||
assert bool(rocm_aiter_ops.is_topk_gating_enabled()) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
|
||||
)
|
||||
def test_topk_sigmoid_dispatch(use_rocm_aiter: bool):
|
||||
topk_func = dispatch_topk_sigmoid_func(use_rocm_aiter)
|
||||
|
||||
if current_platform.is_rocm() and use_rocm_aiter:
|
||||
assert topk_func == rocm_aiter_ops.topk_sigmoid
|
||||
else:
|
||||
assert topk_func == vllm_topk_sigmoid
|
||||
@@ -1,586 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.model_executor.warmup import jit_warmup_triton_helper
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
JitWarmupRegistry,
|
||||
WarmupChoices,
|
||||
WarmupIntRange,
|
||||
kernel_launcher,
|
||||
)
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
LaunchSpec,
|
||||
TritonJitKey,
|
||||
TritonWarmupTensor,
|
||||
VllmTritonJitKernel,
|
||||
triton_kernel_dispatcher_with_warmup,
|
||||
triton_warmup_inputs,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _binder_test_kernel(x, value, BLOCK: tl.constexpr):
|
||||
pass
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _second_binder_test_kernel(x, value, BLOCK: tl.constexpr):
|
||||
pass
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _pointer_group_test_kernel(x_ptr, value, BLOCK: tl.constexpr):
|
||||
pass
|
||||
|
||||
|
||||
class _FakeTritonKernel:
|
||||
arg_names: tuple[str, ...] = ("first", "second", "CONST")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.warmup_calls: list[dict[str, Any]] = []
|
||||
self.runtime_calls: list[tuple[Any, tuple[Any, ...], dict[str, Any]]] = []
|
||||
|
||||
def warmup(self, **kwargs: Any) -> None:
|
||||
self.warmup_calls.append(kwargs)
|
||||
|
||||
def __getitem__(self, grid: Any) -> Any:
|
||||
def launch(*args: Any, **kwargs: Any) -> None:
|
||||
self.runtime_calls.append((grid, args, kwargs))
|
||||
|
||||
return launch
|
||||
|
||||
|
||||
def _patch_key_deriver(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
derive: Any,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
jit_warmup_triton_helper,
|
||||
"_triton_key_deriver",
|
||||
lambda kernel: lambda kwargs: derive(kernel, kwargs),
|
||||
)
|
||||
|
||||
|
||||
class _TestTritonKernel(VllmTritonJitKernel["_TestTritonKernel.CompileKey"]):
|
||||
kernel = _FakeTritonKernel()
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
value: int
|
||||
|
||||
def dispatch(self, *, value: int) -> CompileKey:
|
||||
return self.CompileKey(value=value)
|
||||
|
||||
def get_warmup_keys(self) -> list[CompileKey]:
|
||||
return self._trace_dispatch(self.dispatch)(value=1)
|
||||
|
||||
def warmup_inputs(self, compile_key: CompileKey) -> dict[str, Any]:
|
||||
return dict(first="warmup", second=compile_key.value, runtime_launcher=None)
|
||||
|
||||
@kernel_launcher
|
||||
def __call__(
|
||||
self,
|
||||
first: str,
|
||||
second: int,
|
||||
runtime_launcher: Any,
|
||||
) -> LaunchSpec:
|
||||
return (2,), dict(
|
||||
CONST=7,
|
||||
_runtime_launcher=runtime_launcher,
|
||||
_runtime_launcher_arg_count=2,
|
||||
)
|
||||
|
||||
|
||||
def test_triton_launcher_supports_compile_and_runtime_adapters() -> None:
|
||||
owner = _TestTritonKernel()
|
||||
owner.kernel.warmup_calls.clear()
|
||||
|
||||
owner.compile(owner.CompileKey(value=1))
|
||||
assert owner.kernel.warmup_calls == [
|
||||
{"grid": (1,), "first": "warmup", "second": 1, "CONST": 7}
|
||||
]
|
||||
|
||||
runtime_calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def runtime_launcher(kernel: Any, grid: Any, *args: Any, **kwargs: Any) -> None:
|
||||
runtime_calls.append((kernel, grid, args, kwargs))
|
||||
|
||||
owner("runtime", 2, runtime_launcher)
|
||||
assert runtime_calls == [(owner.kernel, (2,), ("runtime", 2), {"CONST": 7})]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
|
||||
reason="Requires CUDA Triton async compilation",
|
||||
)
|
||||
def test_failed_parallel_warmup_does_not_leak_into_runtime(monkeypatch) -> None:
|
||||
from triton.runtime._async_compile import active_mode
|
||||
|
||||
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
|
||||
owner = _TestTritonKernel()
|
||||
owner.kernel = _FakeTritonKernel()
|
||||
|
||||
def fail():
|
||||
raise RuntimeError("warmup failed")
|
||||
|
||||
def warmup(**kwargs):
|
||||
active_mode.get().submit(kwargs["second"], fail, lambda result: None)
|
||||
|
||||
monkeypatch.setattr(owner.kernel, "warmup", warmup)
|
||||
with pytest.raises(RuntimeError, match="warmup failed"):
|
||||
owner.compile_many(owner.CompileKey(value=i) for i in range(2))
|
||||
|
||||
assert active_mode.get() is None
|
||||
owner("runtime", 3, None)
|
||||
assert len(owner.kernel.runtime_calls) == 1
|
||||
|
||||
|
||||
def test_autotuning_warmup_stays_serial(monkeypatch) -> None:
|
||||
owner = _TestTritonKernel()
|
||||
owner.kernel = _FakeTritonKernel()
|
||||
owner._run_autotune = True
|
||||
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 4)
|
||||
|
||||
def unexpected_async(*args):
|
||||
pytest.fail("Autotuning must not use asynchronous compilation")
|
||||
|
||||
monkeypatch.setattr(triton, "AsyncCompileMode", unexpected_async, raising=False)
|
||||
owner.compile_many(owner.CompileKey(value=i) for i in range(2))
|
||||
assert len(owner.kernel.runtime_calls) == 2
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _warmup_store_kernel(out, VALUE: tl.constexpr):
|
||||
tl.store(out, VALUE)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not hasattr(triton, "AsyncCompileMode"),
|
||||
reason="Requires CUDA Triton async compilation",
|
||||
)
|
||||
def test_registered_warmup_overlaps_kernels_and_runtime_stays_serial(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
from triton.runtime._async_compile import active_mode
|
||||
|
||||
monkeypatch.setattr(envs, "VLLM_TRITON_JIT_WARMUP_NUM_THREADS", 2)
|
||||
|
||||
def make_store(value):
|
||||
def warmup_inputs():
|
||||
return dict(out=TritonWarmupTensor(torch.int32), value=value)
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(
|
||||
kernel=_warmup_store_kernel, warmup_inputs=warmup_inputs
|
||||
)
|
||||
def store(out, value):
|
||||
return (1,), dict(VALUE=value)
|
||||
|
||||
# Initialize Triton's binder before wrapping its compiler.
|
||||
store.get_warmup_keys()
|
||||
return store
|
||||
|
||||
stores = [make_store(value) for value in (1, 2)]
|
||||
caller = threading.get_ident()
|
||||
barrier = threading.Barrier(2, timeout=10)
|
||||
compiled_on = []
|
||||
compile_kernel = _warmup_store_kernel.compile
|
||||
|
||||
def record_compile(*args, **kwargs):
|
||||
if threading.get_ident() != caller:
|
||||
barrier.wait()
|
||||
result = compile_kernel(*args, **kwargs)
|
||||
compiled_on.append(threading.get_ident())
|
||||
return result
|
||||
|
||||
monkeypatch.setattr(_warmup_store_kernel, "compile", record_compile)
|
||||
registry = JitWarmupRegistry(None)
|
||||
with registry.activate():
|
||||
for store in stores:
|
||||
store.register_warmup()
|
||||
registry.warmup()
|
||||
|
||||
assert len(compiled_on) == 2
|
||||
assert all(thread != caller for thread in compiled_on)
|
||||
assert active_mode.get() is None
|
||||
out = torch.empty(1, dtype=torch.int32, device=current_platform.device_type)
|
||||
for store, value in zip(stores, (1, 2)):
|
||||
store(out, value)
|
||||
assert out.item() == value
|
||||
assert len(compiled_on) == 2
|
||||
|
||||
# A specialization first seen during inference uses the normal blocking JIT.
|
||||
store(out, 3)
|
||||
assert out.item() == 3
|
||||
assert compiled_on[2:] == [caller]
|
||||
|
||||
|
||||
def test_triton_launcher_supports_cpu_function_wrappers() -> None:
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def kernel(first: str, second: int, CONST: int) -> None:
|
||||
calls.append((first, second, CONST))
|
||||
|
||||
class FuncWrapper:
|
||||
def __init__(self) -> None:
|
||||
self.func = kernel
|
||||
|
||||
def __getitem__(self, _grid: Any) -> Any:
|
||||
return self.func
|
||||
|
||||
class TestCpuKernel(_TestTritonKernel):
|
||||
kernel: Any = FuncWrapper()
|
||||
|
||||
owner = TestCpuKernel()
|
||||
|
||||
owner("runtime", 2, None)
|
||||
assert calls == [("runtime", 2, 7)]
|
||||
|
||||
|
||||
def test_triton_kernel_decorator_returns_launcher(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
kernel = _FakeTritonKernel()
|
||||
kernel.arg_names = (
|
||||
"FIRST_PTR",
|
||||
"FIRST_STRIDE0",
|
||||
"aliased_ptr",
|
||||
"aliased_stride_0",
|
||||
"SECOND",
|
||||
"CONST",
|
||||
)
|
||||
|
||||
def warmup_inputs() -> dict[str, Any]:
|
||||
return dict(
|
||||
first=TritonWarmupTensor(torch.float32, shape=(2, 3), strides=(5, 1)),
|
||||
second=WarmupChoices(1, 2),
|
||||
)
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
|
||||
def launch(first: str, second: int = 2, config: int = 7) -> LaunchSpec:
|
||||
return (2,), dict(aliased_ptr=first, CONST=config)
|
||||
|
||||
def fake_keys(kernel: Any, kwargs: Any) -> set[TritonJitKey]:
|
||||
return {TritonJitKey(id(kernel), "fake", 0, kwargs["SECOND"])}
|
||||
|
||||
_patch_key_deriver(monkeypatch, fake_keys)
|
||||
|
||||
keys = launch.get_warmup_keys()
|
||||
keys_with_config = launch.get_warmup_keys(vllm_config=object())
|
||||
assert keys_with_config == keys
|
||||
assert [dict(key.inputs)["second"] for key in keys] == [1, 2]
|
||||
launch.compile(keys[0])
|
||||
assert kernel.warmup_calls == [
|
||||
{
|
||||
"grid": (1,),
|
||||
"FIRST_PTR": TritonWarmupTensor(
|
||||
torch.float32, shape=(2, 3), strides=(5, 1)
|
||||
),
|
||||
"FIRST_STRIDE0": 5,
|
||||
"aliased_ptr": TritonWarmupTensor(
|
||||
torch.float32, shape=(2, 3), strides=(5, 1)
|
||||
),
|
||||
"aliased_stride_0": 5,
|
||||
"SECOND": 1,
|
||||
"CONST": 7,
|
||||
}
|
||||
]
|
||||
first = torch.empty_strided((2, 3), (5, 1))
|
||||
launch(first)
|
||||
assert kernel.runtime_calls == [
|
||||
(
|
||||
(2,),
|
||||
(),
|
||||
{
|
||||
"FIRST_PTR": first,
|
||||
"FIRST_STRIDE0": 5,
|
||||
"aliased_ptr": first,
|
||||
"aliased_stride_0": 5,
|
||||
"SECOND": 2,
|
||||
"CONST": 7,
|
||||
},
|
||||
)
|
||||
]
|
||||
assert launch.__name__ == "launch"
|
||||
with pytest.raises(TypeError, match="unexpected keyword"):
|
||||
launch(first, 1, 7, stale_constexpr=True)
|
||||
|
||||
|
||||
def test_triton_kernel_decorator_without_triton(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(jit_warmup_triton_helper, "HAS_TRITON", False)
|
||||
monkeypatch.setattr(jit_warmup_triton_helper, "triton", object())
|
||||
kernel = _FakeTritonKernel()
|
||||
|
||||
def warmup_inputs() -> dict[str, Any]:
|
||||
return dict(first="warmup", second=1)
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
|
||||
def launch(first: str, second: int) -> LaunchSpec:
|
||||
return (2,), dict(CONST=7)
|
||||
|
||||
launch("runtime", 2)
|
||||
assert kernel.runtime_calls == [
|
||||
((2,), (), {"first": "runtime", "second": 2, "CONST": 7})
|
||||
]
|
||||
|
||||
|
||||
def test_triton_kernel_decorator_exhausts_large_ranges_before_deduplication(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
kernel = _FakeTritonKernel()
|
||||
dispatched: list[int] = []
|
||||
|
||||
def warmup_inputs() -> dict[str, Any]:
|
||||
tokens: Any = WarmupIntRange(1, 8193)
|
||||
return dict(first="warmup", second=tokens)
|
||||
|
||||
def specialization(second: int) -> int:
|
||||
return 7 if second <= 97 else 8
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
|
||||
def dispatch(first: str, second: int) -> LaunchSpec:
|
||||
dispatched.append(second)
|
||||
return (second,), dict(CONST=specialization(second))
|
||||
|
||||
def fake_keys(kernel: Any, kwargs: Any) -> set[TritonJitKey]:
|
||||
return {TritonJitKey(id(kernel), "fake", 0, kwargs["CONST"])}
|
||||
|
||||
_patch_key_deriver(monkeypatch, fake_keys)
|
||||
|
||||
keys = dispatch.get_warmup_keys()
|
||||
assert dispatched == list(range(1, 8193))
|
||||
assert len(keys) == 2
|
||||
assert {specialization(dict(key.inputs)["second"]) for key in keys} == {7, 8}
|
||||
|
||||
|
||||
def test_triton_kernel_decorator_propagates_dispatch_assertions(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
kernel = _FakeTritonKernel()
|
||||
|
||||
def warmup_inputs() -> dict[str, Any]:
|
||||
return dict(first="warmup", second=WarmupChoices(1, 2))
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
|
||||
def dispatch(first: str, second: int) -> LaunchSpec:
|
||||
assert second != 2, "broken dispatch"
|
||||
return (1,), dict(CONST=second)
|
||||
|
||||
_patch_key_deriver(
|
||||
monkeypatch,
|
||||
lambda kernel, kwargs: {TritonJitKey(id(kernel), "fake", 0, kwargs["CONST"])},
|
||||
)
|
||||
|
||||
with pytest.raises(AssertionError, match="broken dispatch"):
|
||||
dispatch.get_warmup_keys()
|
||||
|
||||
|
||||
def test_triton_kernel_dispatch_uses_device_fake_tensors(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
kernel = _FakeTritonKernel()
|
||||
device_type = current_platform.device_type
|
||||
|
||||
def warmup_inputs() -> dict[str, Any]:
|
||||
return dict(
|
||||
first=TritonWarmupTensor(torch.float32, shape=(2, 3)),
|
||||
second=1,
|
||||
)
|
||||
|
||||
@triton_kernel_dispatcher_with_warmup(kernel=kernel, warmup_inputs=warmup_inputs)
|
||||
def dispatch(first: torch.Tensor, second: int) -> LaunchSpec:
|
||||
assert isinstance(first, torch.Tensor)
|
||||
assert first.device.type == device_type
|
||||
assert first[0].is_contiguous()
|
||||
return (first.shape[0],), dict(CONST=first[0].numel())
|
||||
|
||||
_patch_key_deriver(
|
||||
monkeypatch,
|
||||
lambda kernel, kwargs: {TritonJitKey(id(kernel), "fake", 0, kwargs["CONST"])},
|
||||
)
|
||||
|
||||
keys = dispatch.get_warmup_keys()
|
||||
assert len(keys) == 1
|
||||
assert dict(keys[0].inputs)["first"].device.type == device_type
|
||||
|
||||
|
||||
def test_triton_kernel_decorates_native_launchers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
kernel = _FakeTritonKernel()
|
||||
|
||||
def warmup_inputs(kernel: Any = kernel) -> dict[str, Any]:
|
||||
return triton_warmup_inputs(
|
||||
kernel,
|
||||
"warmup",
|
||||
WarmupChoices(1, 2),
|
||||
grid=(2,),
|
||||
CONST=7,
|
||||
)
|
||||
|
||||
launcher = triton_kernel_dispatcher_with_warmup(warmup_inputs=warmup_inputs)(kernel)
|
||||
launcher[(3,)]("runtime", 4, CONST=8)
|
||||
|
||||
assert kernel.runtime_calls == [((3,), ("runtime", 4), {"CONST": 8})]
|
||||
|
||||
_patch_key_deriver(
|
||||
monkeypatch,
|
||||
lambda kernel, kwargs: {
|
||||
TritonJitKey(id(kernel), "fake", 0, (kwargs["second"], kwargs["CONST"]))
|
||||
},
|
||||
)
|
||||
|
||||
keys = launcher.get_warmup_keys()
|
||||
assert len(keys) == 2
|
||||
launcher.compile(keys[0])
|
||||
assert kernel.warmup_calls == [
|
||||
{"grid": (1,), "first": "warmup", "second": 1, "CONST": 7}
|
||||
]
|
||||
|
||||
|
||||
def test_triton_warmup_inputs_expands_explicit_pointer_dtypes() -> None:
|
||||
inputs = triton_warmup_inputs(
|
||||
_pointer_group_test_kernel,
|
||||
grid=(1,),
|
||||
pointer_dtypes={torch.float32: ("x_ptr",)},
|
||||
value=2,
|
||||
BLOCK=16,
|
||||
)
|
||||
|
||||
assert inputs["x_ptr"] == TritonWarmupTensor(torch.float32)
|
||||
assert inputs["value"] == 2
|
||||
|
||||
with pytest.raises(ValueError, match="not a pointer: value"):
|
||||
triton_warmup_inputs(
|
||||
_pointer_group_test_kernel,
|
||||
grid=(1,),
|
||||
pointer_dtypes={torch.float32: ("value",)},
|
||||
x_ptr=TritonWarmupTensor(torch.float32),
|
||||
BLOCK=16,
|
||||
)
|
||||
with pytest.raises(ValueError, match="Missing Triton pointer inputs: x_ptr"):
|
||||
triton_warmup_inputs(
|
||||
_pointer_group_test_kernel,
|
||||
grid=(1,),
|
||||
pointer_dtypes={},
|
||||
value=2,
|
||||
BLOCK=16,
|
||||
)
|
||||
|
||||
|
||||
def test_triton_key_derivation_applies_wrappers_and_runtime_options(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
knobs = triton.knobs
|
||||
driver = triton.runtime.driver
|
||||
|
||||
device = "test-device"
|
||||
monkeypatch.setattr(driver.active, "get_current_device", lambda: device)
|
||||
calls: list[dict[str, Any]] = []
|
||||
|
||||
def binder(x: Any, value: int, BLOCK: int, **options: Any) -> Any:
|
||||
calls.append(dict(BLOCK=BLOCK, **options))
|
||||
specialization = [("pointer", x.dtype), ("i32", value), ("constexpr", BLOCK)]
|
||||
return {}, specialization, options
|
||||
|
||||
cache: dict[Any, Any] = {}
|
||||
_binder_test_kernel.device_caches[device] = ({}, cache, None, None, binder)
|
||||
heuristic_kernel = triton.heuristics(
|
||||
{"BLOCK": lambda args: 16 if args["value"] <= 16 else 32}
|
||||
)(_binder_test_kernel)
|
||||
|
||||
keys = jit_warmup_triton_helper._triton_key_deriver(heuristic_kernel)(
|
||||
{"x": TritonWarmupTensor(tl.float32), "value": 2}
|
||||
)
|
||||
|
||||
assert len(keys) == 1
|
||||
expected_options = {
|
||||
"debug": knobs.runtime.debug,
|
||||
"instrumentation_mode": knobs.compilation.instrumentation_mode,
|
||||
}
|
||||
if hasattr(knobs.compilation, "fpsan_homomorphic_casts"):
|
||||
expected_options["fpsan_homomorphic_casts"] = (
|
||||
knobs.compilation.fpsan_homomorphic_casts
|
||||
)
|
||||
assert calls == [{"BLOCK": 16, **expected_options}]
|
||||
|
||||
|
||||
def test_triton_key_derivation_covers_autotune_configs_and_jit_identity(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
driver = triton.runtime.driver
|
||||
|
||||
device = "test-device"
|
||||
monkeypatch.setattr(driver.active, "get_current_device", lambda: device)
|
||||
|
||||
def binder(x: Any, value: int, BLOCK: int, **options: Any) -> Any:
|
||||
specialization = [("pointer", x.dtype), ("i32", value), ("constexpr", BLOCK)]
|
||||
return {}, specialization, options
|
||||
|
||||
for kernel in (_binder_test_kernel, _second_binder_test_kernel):
|
||||
kernel.device_caches[device] = ({}, {}, None, None, binder)
|
||||
autotuned = triton.autotune(
|
||||
configs=[
|
||||
triton.Config({"BLOCK": 16}, num_warps=2),
|
||||
triton.Config({"BLOCK": 32}, num_warps=4),
|
||||
],
|
||||
key=["value"],
|
||||
)(_binder_test_kernel)
|
||||
inputs = {"x": TritonWarmupTensor(tl.float32), "value": 2}
|
||||
|
||||
autotune_keys = jit_warmup_triton_helper._triton_key_deriver(autotuned)(inputs)
|
||||
other_kernel_keys = jit_warmup_triton_helper._triton_key_deriver(
|
||||
_second_binder_test_kernel
|
||||
)(inputs | {"BLOCK": 16})
|
||||
|
||||
assert len(autotune_keys) == 2
|
||||
assert {key.jit_function_key for key in autotune_keys} == {
|
||||
_binder_test_kernel.cache_key
|
||||
}
|
||||
assert {key.jit_function_id for key in autotune_keys} == {id(_binder_test_kernel)}
|
||||
assert autotune_keys.isdisjoint(other_kernel_keys)
|
||||
|
||||
|
||||
def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
|
||||
|
||||
owner = ComputeSlotMappingKernel()
|
||||
compile_key = owner.CompileKey(
|
||||
kv_cache_block_size=16,
|
||||
total_cp_world_size=2,
|
||||
total_cp_rank=1,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
block_table_stride=128,
|
||||
)
|
||||
launches: list[tuple[Any, ...]] = []
|
||||
|
||||
def launch(launch_spec: Any, inputs: Any) -> None:
|
||||
launches.append((launch_spec, inputs))
|
||||
|
||||
monkeypatch.setattr(owner, "launch", launch)
|
||||
owner.compile(compile_key)
|
||||
|
||||
(grid, kwargs), inputs = launches[0]
|
||||
assert grid == (2,)
|
||||
assert inputs["num_tokens"] == 2
|
||||
assert inputs["block_table_stride"] == 128
|
||||
assert kwargs == {
|
||||
"KV_CACHE_BLOCK_SIZE": 16,
|
||||
"TOTAL_CP_WORLD_SIZE": 2,
|
||||
"TOTAL_CP_RANK": 1,
|
||||
"CP_KV_CACHE_INTERLEAVE_SIZE": 1,
|
||||
"PAD_ID": PAD_SLOT_ID,
|
||||
"BLOCK_SIZE": owner.triton_block_size,
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.mamba_triton_warmup import _warm_batch_memcpy_kernel
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA is required")
|
||||
def test_mamba_batch_memcpy_kernel_compiles_on_gpu() -> None:
|
||||
_warm_batch_memcpy_kernel(torch.device("cuda"))
|
||||
torch.accelerator.synchronize("cuda")
|
||||
@@ -1,110 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import multiprocessing
|
||||
import types
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def _test_oink_availability_impl(
|
||||
device_capability: tuple[int, int],
|
||||
has_rmsnorm: bool,
|
||||
has_fused_add_rms_norm: bool,
|
||||
expected_available: bool,
|
||||
expected_fused: bool,
|
||||
) -> None:
|
||||
"""Test OINK support detection with mocked state."""
|
||||
import torch
|
||||
|
||||
from vllm import platforms
|
||||
|
||||
# Mock device capability (class method, override on class)
|
||||
dc = platforms.interface.DeviceCapability(*device_capability)
|
||||
platforms.current_platform.__class__.get_device_capability = lambda device_id=0: dc
|
||||
|
||||
# Mock oink ops
|
||||
oink_ops = types.SimpleNamespace()
|
||||
if has_rmsnorm:
|
||||
oink_ops.rmsnorm = lambda x, w, eps: x
|
||||
if has_fused_add_rms_norm:
|
||||
oink_ops.fused_add_rms_norm = lambda x, residual, w, eps: None
|
||||
|
||||
torch.ops.oink = oink_ops
|
||||
|
||||
# Now import vllm modules with mocks in place (fresh import with mocked platform)
|
||||
import vllm.kernels.oink_ops # noqa: F401
|
||||
from vllm.ir.ops import fused_add_rms_norm, rms_norm
|
||||
|
||||
# Verify support checks
|
||||
assert rms_norm.impls["oink"].supported is expected_available
|
||||
assert fused_add_rms_norm.impls["oink"].supported is expected_fused
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"device_capability,has_rmsnorm,has_fused_add_rms_norm,expected_available,expected_fused",
|
||||
[
|
||||
# Case 1: < SM100, ops not supported
|
||||
((9, 0), True, False, False, False),
|
||||
# Case 2: CUDA available and SM100, rmsnorm op registered
|
||||
((10, 0), True, False, True, False),
|
||||
# Case 3: SM100 with both rmsnorm and fused_add_rms_norm
|
||||
((10, 0), True, True, True, True),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test on CUDA")
|
||||
def test_oink_availability_checks(
|
||||
device_capability: tuple[int, int],
|
||||
has_rmsnorm: bool,
|
||||
has_fused_add_rms_norm: bool,
|
||||
expected_available: bool,
|
||||
expected_fused: bool,
|
||||
):
|
||||
"""Test OINK support detection with clean import state for each parameter set."""
|
||||
# Use spawn to run function in fresh process with clean imports
|
||||
# TODO migrate to spawn utility:
|
||||
# https://github.com/vllm-project/vllm/issues/41415
|
||||
ctx = multiprocessing.get_context("spawn")
|
||||
process = ctx.Process(
|
||||
target=_test_oink_availability_impl,
|
||||
args=(
|
||||
device_capability,
|
||||
has_rmsnorm,
|
||||
has_fused_add_rms_norm,
|
||||
expected_available,
|
||||
expected_fused,
|
||||
),
|
||||
)
|
||||
process.start()
|
||||
process.join()
|
||||
|
||||
if process.exitcode != 0:
|
||||
raise AssertionError(
|
||||
f"Subprocess test failed with exit code {process.exitcode}"
|
||||
)
|
||||
|
||||
|
||||
def test_can_view_as_2d_stride_guard():
|
||||
# No global import
|
||||
import torch
|
||||
|
||||
# Import the helper from the kernels module.
|
||||
from vllm.kernels.oink_ops import _can_view_as_2d
|
||||
|
||||
x = torch.zeros((2, 3, 4))
|
||||
assert _can_view_as_2d(x) is True
|
||||
|
||||
# Size-1 dims should be ignored by the viewability check.
|
||||
# Create a tensor where stride(0) != stride(1) * size(1) due to padding,
|
||||
# but view(-1, H) is still valid because dim 1 has size 1.
|
||||
base = torch.zeros((2, 10, 4))
|
||||
x_singleton = base[:, :1, :]
|
||||
x_singleton.view(-1, x_singleton.shape[-1])
|
||||
assert _can_view_as_2d(x_singleton) is True
|
||||
|
||||
# Middle-dimension stride break: view(-1, hidden) should be invalid.
|
||||
x2 = x[:, ::2, :]
|
||||
with pytest.raises(RuntimeError):
|
||||
x2.view(-1, x2.shape[-1])
|
||||
assert _can_view_as_2d(x2) is False
|
||||
@@ -1,165 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.model_executor.warmup import replayssm_warmup as warmup
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not has_flashinfer(),
|
||||
reason="FlashInfer ReplaySSM warmup tests require CUDA and FlashInfer",
|
||||
)
|
||||
|
||||
PREFILL_KWARGS = {
|
||||
"num_tokens": 128,
|
||||
"skip_eplb": True,
|
||||
"is_profile": True,
|
||||
"randomize_inputs": True,
|
||||
}
|
||||
|
||||
|
||||
def _autotune_runner(
|
||||
*,
|
||||
use_v2_model_runner: bool = False,
|
||||
query_len: int = 6,
|
||||
max_num_seqs: int = 32,
|
||||
num_blocks: int = 17,
|
||||
max_num_tokens: int = 100,
|
||||
use_replayssm: bool = True,
|
||||
backend: MambaBackendEnum = MambaBackendEnum.FLASHINFER,
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(
|
||||
cache_config=SimpleNamespace(use_replayssm=use_replayssm),
|
||||
mamba_config=SimpleNamespace(backend=backend),
|
||||
use_v2_model_runner=use_v2_model_runner,
|
||||
),
|
||||
uniform_decode_query_len=query_len,
|
||||
decode_query_len=query_len,
|
||||
max_num_tokens=max_num_tokens,
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
|
||||
kv_cache_config=SimpleNamespace(num_blocks=num_blocks),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("runner_kwargs", "expected_num_reqs"),
|
||||
[
|
||||
# max_num_seqs (32) vs max_num_tokens // query_len (16) vs blocks-1 (16).
|
||||
(dict(query_len=6, use_v2_model_runner=False), 16),
|
||||
(dict(query_len=6, use_v2_model_runner=True), 16),
|
||||
# num_blocks - 1 is the binding constraint.
|
||||
(dict(query_len=1, max_num_tokens=128, max_num_seqs=64, num_blocks=5), 4),
|
||||
],
|
||||
ids=["v1", "v2", "clamped_to_state_capacity"],
|
||||
)
|
||||
def test_replayssm_autotune_decode_kwargs(runner_kwargs, expected_num_reqs):
|
||||
query_len = runner_kwargs["query_len"]
|
||||
with patch.object(
|
||||
warmup, "flashinfer_replayssm_autotune_supported", return_value=True
|
||||
):
|
||||
result = warmup._replayssm_autotune_kwargs(
|
||||
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
max_num_reqs, decode_kwargs = result
|
||||
assert max_num_reqs == expected_num_reqs
|
||||
expected_kwargs = {
|
||||
**PREFILL_KWARGS,
|
||||
"num_tokens": expected_num_reqs * query_len,
|
||||
"uniform_decode": True,
|
||||
}
|
||||
if runner_kwargs.get("use_v2_model_runner"):
|
||||
expected_kwargs["valid_dummy_state_slots"] = True
|
||||
else:
|
||||
expected_kwargs.update(
|
||||
allow_microbatching=False,
|
||||
force_attention=True,
|
||||
profile_seq_lens=query_len + 1,
|
||||
)
|
||||
assert decode_kwargs == expected_kwargs
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("runner_kwargs", "flashinfer_supported"),
|
||||
[
|
||||
(dict(use_replayssm=False), True),
|
||||
(dict(backend=MambaBackendEnum.TRITON), True),
|
||||
({}, False),
|
||||
(dict(num_blocks=1), True),
|
||||
],
|
||||
ids=[
|
||||
"replayssm_disabled",
|
||||
"non_flashinfer_backend",
|
||||
"kernel_unavailable",
|
||||
"zero_non_padding_slots",
|
||||
],
|
||||
)
|
||||
def test_replayssm_autotune_kwargs_skipped(runner_kwargs, flashinfer_supported):
|
||||
with patch.object(
|
||||
warmup,
|
||||
"flashinfer_replayssm_autotune_supported",
|
||||
return_value=flashinfer_supported,
|
||||
):
|
||||
result = warmup._replayssm_autotune_kwargs(
|
||||
_autotune_runner(**runner_kwargs), PREFILL_KWARGS
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_replayssm_autotune_slots_restore_state_and_trackers():
|
||||
mixer = MambaMixer2.__new__(MambaMixer2)
|
||||
torch.nn.Module.__init__(mixer)
|
||||
mixer.use_replayssm = True
|
||||
mixer.replayssm_buffer_len = 16
|
||||
mixer.kv_cache = (
|
||||
torch.full((4, 2), 3.0),
|
||||
torch.full((4, 2), 3.0),
|
||||
*(torch.full((4, 2, 17), 3.0) for _ in range(3)),
|
||||
)
|
||||
mixer._replayssm_ring_start = torch.full((4,), 3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_num_accepted = torch.full((4,), 3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_query_len = torch.full((4,), 3, dtype=torch.int32)
|
||||
tracked = (
|
||||
*mixer.kv_cache,
|
||||
mixer._replayssm_ring_start,
|
||||
mixer._replayssm_prev_num_accepted,
|
||||
mixer._replayssm_prev_query_len,
|
||||
)
|
||||
|
||||
block_ids = np.arange(10, 14, dtype=np.int32).reshape(4, 1)
|
||||
original_block_ids = block_ids.copy()
|
||||
block_table = SimpleNamespace(block_table=SimpleNamespace(np=block_ids))
|
||||
multi_group_block_table = SimpleNamespace(
|
||||
block_tables=[block_table], commit_block_table=Mock()
|
||||
)
|
||||
runner = SimpleNamespace(
|
||||
vllm_config=SimpleNamespace(use_v2_model_runner=False),
|
||||
input_batch=SimpleNamespace(block_table=multi_group_block_table),
|
||||
get_model=lambda: SimpleNamespace(modules=lambda: (mixer,)),
|
||||
)
|
||||
|
||||
with warmup._temporary_replayssm_autotune_state(runner, 2):
|
||||
assert block_ids[:2, 0].tolist() == [1, 2]
|
||||
for tensor in tracked:
|
||||
tensor[1:3].fill_(9)
|
||||
|
||||
assert np.array_equal(block_ids, original_block_ids)
|
||||
assert multi_group_block_table.commit_block_table.call_args_list == [
|
||||
((2,), {}),
|
||||
((2,), {}),
|
||||
]
|
||||
for tensor in tracked:
|
||||
assert torch.count_nonzero(tensor[1:3]) == 0
|
||||
assert torch.all(tensor[0] == 3)
|
||||
assert torch.all(tensor[3] == 3)
|
||||
@@ -1,263 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the apply-time Triton kernel override registry.
|
||||
|
||||
The registry must let out-of-tree platforms replace kernels defined as
|
||||
plain ``@triton.jit`` functions, patching every hold-site (defining
|
||||
module, from-import copies, and JIT warmup owners) so that existing call
|
||||
sites dispatch to the override without any core-side decoration.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import types
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
_DecoratedTritonJitKernel,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.triton_utils.dispatcher import (
|
||||
KernelOverride,
|
||||
register_kernels,
|
||||
)
|
||||
|
||||
|
||||
class _FakeTritonKernel:
|
||||
"""Stands in for a Triton JITFunction (works without Triton installed)."""
|
||||
|
||||
arg_names = ("output_ptr", "input_ptr", "replace_from", "MAX_NUM_TOKENS")
|
||||
|
||||
def __getitem__(self, grid: Any) -> Any:
|
||||
def launch(*args: Any, **kwargs: Any) -> Any:
|
||||
return ("default", grid, kwargs)
|
||||
|
||||
return launch
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def _fake_module(monkeypatch: pytest.MonkeyPatch) -> Any:
|
||||
"""Expose a fake defining module holding the kernel and its warmup owner."""
|
||||
import types
|
||||
|
||||
kernel = _FakeTritonKernel()
|
||||
owner = _DecoratedTritonJitKernel(
|
||||
kernel,
|
||||
lambda: {},
|
||||
lambda output, input, replace_from: ( # noqa: ARG005
|
||||
(1,),
|
||||
dict(MAX_NUM_TOKENS=8),
|
||||
),
|
||||
)
|
||||
module = types.ModuleType("vllm.tests.fake_kernel_home")
|
||||
module.expand_kernel = kernel # type: ignore[attr-defined]
|
||||
module._expand = owner # type: ignore[attr-defined]
|
||||
monkeypatch.setitem(sys.modules, module.__name__, module)
|
||||
return module
|
||||
|
||||
|
||||
def test_override_patches_defining_module_and_warmup_owner(
|
||||
_fake_module: Any,
|
||||
) -> None:
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None):
|
||||
calls.append((output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS))
|
||||
return "cpu"
|
||||
|
||||
register_kernels(
|
||||
{
|
||||
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
|
||||
}
|
||||
)
|
||||
|
||||
overridden = _fake_module.expand_kernel
|
||||
assert isinstance(overridden, KernelOverride)
|
||||
# Launch with kernel-arg names like the warmup launch path does.
|
||||
assert overridden[(4,)](output_ptr=1, input_ptr=2, replace_from=0) == "cpu"
|
||||
assert calls == [(1, 2, 0, None)]
|
||||
|
||||
# The warmup owner's captured kernel is patched too, and its cached
|
||||
# argument-name introspection is invalidated.
|
||||
assert _fake_module._expand.kernel is overridden
|
||||
|
||||
|
||||
def test_override_forwards_positionally_for_mismatched_names(
|
||||
_fake_module: Any,
|
||||
) -> None:
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def cpu_impl(output, input_val, replace_from, MAX_NUM_TOKENS=None):
|
||||
calls.append((output, input_val, replace_from, MAX_NUM_TOKENS))
|
||||
return "cpu"
|
||||
|
||||
register_kernels(
|
||||
{
|
||||
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
|
||||
}
|
||||
)
|
||||
overridden = _fake_module.expand_kernel
|
||||
assert overridden[(4,)](output_ptr=1, input_ptr=2, replace_from=0) == "cpu"
|
||||
assert calls == [(1, 2, 0, None)]
|
||||
|
||||
|
||||
def test_override_covers_from_import_copies(_fake_module: Any) -> None:
|
||||
import types
|
||||
|
||||
importer = types.ModuleType("vllm.tests.fake_kernel_importer")
|
||||
importer.expand_kernel = _fake_module.expand_kernel # type: ignore[attr-defined]
|
||||
sys.modules["vllm.tests.fake_kernel_importer"] = importer
|
||||
|
||||
def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None):
|
||||
return "cpu"
|
||||
|
||||
register_kernels(
|
||||
{
|
||||
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
|
||||
}
|
||||
)
|
||||
assert isinstance(importer.expand_kernel, KernelOverride)
|
||||
del sys.modules["vllm.tests.fake_kernel_importer"]
|
||||
|
||||
|
||||
def test_override_owner_launch_goes_through_impl(_fake_module: Any) -> None:
|
||||
launched: list[Any] = []
|
||||
|
||||
def cpu_impl(output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS=None):
|
||||
launched.append((output_ptr, input_ptr, replace_from, MAX_NUM_TOKENS))
|
||||
return "cpu"
|
||||
|
||||
register_kernels(
|
||||
{
|
||||
"vllm.tests.fake_kernel_home.expand_kernel": cpu_impl,
|
||||
}
|
||||
)
|
||||
owner = _fake_module._expand
|
||||
# In-place kernels return None from launch; assert on the recorded call.
|
||||
owner(1, 2, 0)
|
||||
assert launched == [(1, 2, 0, 8)]
|
||||
|
||||
|
||||
def test_register_kernel_auto_imports_defining_module(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Registering a kernel whose defining module is not imported yet must
|
||||
import it on the spot and apply the override."""
|
||||
kernel_name = "vllm.v1.sample.rejection_sampler.expand_kernel"
|
||||
module_name = kernel_name.rpartition(".")[0]
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def cpu_impl(
|
||||
output_ptr,
|
||||
input_ptr,
|
||||
cu_num_tokens_ptr,
|
||||
replace_from,
|
||||
replace_to,
|
||||
MAX_NUM_TOKENS=None,
|
||||
):
|
||||
calls.append((output_ptr, replace_from, MAX_NUM_TOKENS))
|
||||
return "cpu"
|
||||
|
||||
# Ensure the module is NOT imported before registration. Drop it from
|
||||
# sys.modules; a fresh import creates new kernel/owner objects, which
|
||||
# is what a cold-start platform process would see.
|
||||
saved = sys.modules.pop(module_name, None)
|
||||
# Re-run the registry patch against the original module afterwards so
|
||||
# other tests that already imported it keep working.
|
||||
try:
|
||||
assert module_name not in sys.modules
|
||||
register_kernels({kernel_name: cpu_impl})
|
||||
assert module_name in sys.modules
|
||||
finally:
|
||||
if saved is None:
|
||||
# First import in the whole session: keep the fresh module.
|
||||
saved = sys.modules[module_name]
|
||||
else:
|
||||
# Restore the original module object and re-apply the override
|
||||
# to its (pre-existing) kernel so both generations agree.
|
||||
sys.modules[module_name] = saved
|
||||
register_kernels({kernel_name: cpu_impl})
|
||||
|
||||
overridden = saved.expand_kernel
|
||||
assert isinstance(overridden, KernelOverride)
|
||||
overridden[(2,)](
|
||||
output_ptr=1,
|
||||
input_ptr=2,
|
||||
cu_num_tokens_ptr=3,
|
||||
replace_from=0,
|
||||
replace_to=0,
|
||||
MAX_NUM_TOKENS=8,
|
||||
)
|
||||
assert calls == [(1, 0, 8)]
|
||||
# The warmup owner captured at the fresh import must be patched too
|
||||
# (when the module existed before, there are two module generations).
|
||||
fresh = sys.modules[module_name]
|
||||
if fresh is not saved:
|
||||
assert fresh._expand.kernel is overridden
|
||||
|
||||
|
||||
def test_real_triton_kernel_override_end_to_end() -> None:
|
||||
"""Override a real @triton.jit kernel through the registry."""
|
||||
if not triton.__class__.__module__.startswith("triton"):
|
||||
pytest.skip("Triton is not installed")
|
||||
|
||||
@triton.jit
|
||||
def _add_one_kernel(x_ptr, out_ptr, BLOCK: tl.constexpr): # pragma: no cover
|
||||
offsets = tl.arange(0, BLOCK)
|
||||
tl.store(out_ptr + offsets, tl.load(x_ptr + offsets) + 1)
|
||||
|
||||
module = types.ModuleType("vllm.tests.real_jit_kernel_home")
|
||||
module._add_one_kernel = _add_one_kernel # type: ignore[attr-defined]
|
||||
sys.modules[module.__name__] = module
|
||||
|
||||
def cpu_impl(x_ptr, out_ptr, BLOCK=None): # noqa: ARG001
|
||||
return "cpu"
|
||||
|
||||
register_kernels({"vllm.tests.real_jit_kernel_home._add_one_kernel": cpu_impl})
|
||||
assert module._add_one_kernel.arg_names == ( # type: ignore[attr-defined]
|
||||
"x_ptr",
|
||||
"out_ptr",
|
||||
"BLOCK",
|
||||
)
|
||||
|
||||
|
||||
def test_cpu_slot_mapping_override_forwards_kernel_arguments(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from vllm.utils import cpu_triton_utils
|
||||
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
|
||||
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
fake_torch = types.SimpleNamespace(
|
||||
ops=types.SimpleNamespace(
|
||||
_C=types.SimpleNamespace(
|
||||
compute_slot_mapping_kernel_impl=lambda *args: calls.append(args)
|
||||
)
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(cpu_triton_utils, "torch", fake_torch)
|
||||
|
||||
override = KernelOverride(
|
||||
ComputeSlotMappingKernel.kernel,
|
||||
cpu_triton_utils._compute_slot_mapping_kernel_impl,
|
||||
)
|
||||
override[(2,)](
|
||||
num_tokens=2,
|
||||
max_num_tokens=4,
|
||||
query_start_loc_ptr="query_start_loc",
|
||||
positions_ptr="positions",
|
||||
block_table_ptr="block_table",
|
||||
block_table_stride=128,
|
||||
slot_mapping_ptr="slot_mapping",
|
||||
KV_CACHE_BLOCK_SIZE=16,
|
||||
TOTAL_CP_WORLD_SIZE=1,
|
||||
TOTAL_CP_RANK=0,
|
||||
CP_KV_CACHE_INTERLEAVE_SIZE=1,
|
||||
PAD_ID=-1,
|
||||
BLOCK_SIZE=1024,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
("query_start_loc", "positions", "block_table", "slot_mapping", 16)
|
||||
]
|
||||
@@ -1,300 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The warmup must cover every specialization the sampler will launch.
|
||||
|
||||
``_philox_gumbel_kernel`` is keyed on the Philox key halves, on the logits
|
||||
dtype (model dtype when the sampler passes logits through, fp32 when logits
|
||||
processing copies them) and on whether a skip mask is passed, so a missing
|
||||
combination leaves a request paying the compile this module exists to remove.
|
||||
"""
|
||||
|
||||
import logging
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.watermarking import WatermarkConfig, derive_watermark_key
|
||||
from vllm.model_executor.warmup.watermark_sample_warmup import watermark_sample_warmup
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.watermarking.factory import create_watermarker
|
||||
from vllm.v1.watermarking.gpu_sampler import GPUWatermarkSampler
|
||||
from vllm.v1.watermarking.spec_decode import create_speculative_target_watermarker
|
||||
|
||||
MASTER_KEY = 20250910
|
||||
KEY_A = derive_watermark_key(MASTER_KEY, b"key_a")
|
||||
KEY_B = derive_watermark_key(MASTER_KEY, b"key_b")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cuda_alike_platform(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The warmup returns early off CUDA, so pin the platform it reads.
|
||||
|
||||
Everything here runs on the fake worker's cpu device; without this the
|
||||
tests below would pass only on a CUDA-alike host and silently warm nothing
|
||||
anywhere else.
|
||||
"""
|
||||
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: True)
|
||||
|
||||
|
||||
def _sampler(
|
||||
watermark_config: WatermarkConfig,
|
||||
speculative: bool,
|
||||
use_fp64_gumbel: bool,
|
||||
) -> GPUWatermarkSampler:
|
||||
"""The sampler ``GPUModelRunner`` builds for this config.
|
||||
|
||||
Mirrors ``vllm/v1/worker/gpu/model_runner.py:473-486``. ``__new__`` skips
|
||||
``Sampler.__init__``, which allocates UVA-backed device buffers; the warmup
|
||||
only reads the three attributes set here.
|
||||
"""
|
||||
watermarker = create_watermarker(watermark_config)
|
||||
if speculative:
|
||||
watermarker = create_speculative_target_watermarker(watermarker)
|
||||
sampler = GPUWatermarkSampler.__new__(GPUWatermarkSampler)
|
||||
sampler.watermarker = watermarker
|
||||
sampler.deduplicate_contexts = watermark_config.deduplicate_contexts
|
||||
sampler.use_fp64_gumbel = use_fp64_gumbel
|
||||
return sampler
|
||||
|
||||
|
||||
def _worker(
|
||||
watermark_config: WatermarkConfig | None,
|
||||
speculative: bool = False,
|
||||
use_fp64_gumbel: bool = False,
|
||||
with_sampler: bool = True,
|
||||
) -> SimpleNamespace:
|
||||
sampler = (
|
||||
_sampler(watermark_config, speculative, use_fp64_gumbel)
|
||||
if watermark_config is not None and with_sampler
|
||||
else None
|
||||
)
|
||||
return SimpleNamespace(
|
||||
device=torch.device("cpu"),
|
||||
model_runner=SimpleNamespace(sampler=sampler),
|
||||
vllm_config=SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_vocab_size=lambda: 128,
|
||||
dtype=torch.bfloat16,
|
||||
),
|
||||
watermark_config=watermark_config,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _capture_calls(monkeypatch: pytest.MonkeyPatch) -> list[dict]:
|
||||
calls: list[dict] = []
|
||||
|
||||
def fake_philox_gumbel_sample(logits, contexts, key, **kwargs):
|
||||
calls.append({"logits": logits, "contexts": contexts, "key": key, **kwargs})
|
||||
return torch.zeros(logits.shape[0], dtype=torch.int64)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample",
|
||||
fake_philox_gumbel_sample,
|
||||
)
|
||||
return calls
|
||||
|
||||
|
||||
def _variants(calls: list[dict]) -> set[tuple[int, torch.dtype, bool]]:
|
||||
return {
|
||||
(call["key"], call["logits"].dtype, call.get("skip_mask") is not None)
|
||||
for call in calls
|
||||
}
|
||||
|
||||
|
||||
def test_no_warmup_without_a_watermark_config(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(_worker(None))
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_no_warmup_without_a_watermarked_sampler(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""A rank that does not sample launches nothing to warm.
|
||||
|
||||
Only the final pipeline stage of a generative model holds a
|
||||
``GPUWatermarkSampler``; on any other rank the kernel is never launched.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(
|
||||
_worker(WatermarkConfig(key=MASTER_KEY), with_sampler=False)
|
||||
)
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_no_warmup_off_cuda(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""``GumbelWatermarker`` launches the kernel only on a CUDA-alike device.
|
||||
|
||||
Elsewhere -- an XPU worker inherits ``kernel_warmup`` from the CUDA worker
|
||||
-- the sampler takes the torch path, so warming would compile a kernel
|
||||
nothing launches.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
monkeypatch.setattr(current_platform, "is_cuda_alike", lambda: False)
|
||||
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
|
||||
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_single_key_warms_both_logits_dtypes(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
|
||||
|
||||
assert _variants(calls) == {
|
||||
(MASTER_KEY, torch.bfloat16, True),
|
||||
(MASTER_KEY, torch.float32, True),
|
||||
}
|
||||
assert len(calls) == len(_variants(calls))
|
||||
|
||||
|
||||
def test_dual_key_warms_both_derived_keys(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""``DualKeyGumbelWatermarker.sample`` launches the kernel once per key.
|
||||
|
||||
The halves of a derived key exceed 2**31, so each key is typed i64/i32 by
|
||||
magnitude and a wrong key compiles a specialization the engine never uses.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(
|
||||
_worker(WatermarkConfig(key=MASTER_KEY, algorithm="dual_key_gumbel"))
|
||||
)
|
||||
|
||||
assert _variants(calls) == {
|
||||
(KEY_A, torch.bfloat16, True),
|
||||
(KEY_A, torch.float32, True),
|
||||
(KEY_B, torch.bfloat16, True),
|
||||
(KEY_B, torch.float32, True),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("alpha,key", [(0.0, KEY_A), (1.0, KEY_B)])
|
||||
def test_dual_key_alpha_bounds_warm_one_key(
|
||||
monkeypatch: pytest.MonkeyPatch, alpha: float, key: int
|
||||
) -> None:
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(
|
||||
_worker(
|
||||
WatermarkConfig(key=MASTER_KEY, algorithm="dual_key_gumbel", alpha=alpha)
|
||||
)
|
||||
)
|
||||
|
||||
assert _variants(calls) == {
|
||||
(key, torch.bfloat16, True),
|
||||
(key, torch.float32, True),
|
||||
}
|
||||
|
||||
|
||||
def test_speculative_config_warms_the_target_role_only(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Under spec decode the sampler holds ``create_target_watermarker()``.
|
||||
|
||||
Rows without draft tokens sample through it, so the sampler path launches
|
||||
key B alone; the draft key runs in the speculator's own sampler.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(
|
||||
_worker(
|
||||
WatermarkConfig(key=MASTER_KEY, algorithm="dual_key_gumbel"),
|
||||
speculative=True,
|
||||
)
|
||||
)
|
||||
|
||||
assert _variants(calls) == {
|
||||
(KEY_B, torch.bfloat16, True),
|
||||
(KEY_B, torch.float32, True),
|
||||
}
|
||||
|
||||
|
||||
def test_dedup_none_also_warms_the_mask_free_specialization(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Without deduplication an all-watermarked batch passes no skip mask.
|
||||
|
||||
A later batch mixing in an opted-out request passes one, so both
|
||||
specializations are reachable and both must be warmed.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(
|
||||
_worker(WatermarkConfig(key=MASTER_KEY, deduplicate_contexts="none"))
|
||||
)
|
||||
|
||||
assert _variants(calls) == {
|
||||
(MASTER_KEY, torch.bfloat16, True),
|
||||
(MASTER_KEY, torch.float32, True),
|
||||
(MASTER_KEY, torch.bfloat16, False),
|
||||
(MASTER_KEY, torch.float32, False),
|
||||
}
|
||||
for call in calls:
|
||||
if call.get("skip_mask") is None:
|
||||
assert "temperatures" not in call
|
||||
|
||||
|
||||
def test_warmup_argument_dtypes_match_the_runtime_buffers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Contexts are int32, the sampling state int64 indices and fp32 temperatures.
|
||||
|
||||
``GPUWatermarkSampler._get_contexts`` gathers from the int32 request-state
|
||||
token ids, ``expanded_idx_mapping``/``positions``/``seeds`` are int64 and
|
||||
``SamplingStates.temperature`` is fp32; warming other dtypes compiles
|
||||
specializations no engine launches.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY, context_width=3)))
|
||||
|
||||
assert calls
|
||||
for call in calls:
|
||||
num_tokens = call["logits"].shape[0]
|
||||
assert call["contexts"].dtype == torch.int32
|
||||
assert call["contexts"].shape == (num_tokens, 3)
|
||||
assert call["skip_mask"].dtype == torch.bool
|
||||
assert call["expanded_idx_mapping"].dtype == torch.int64
|
||||
assert call["positions"].dtype == torch.int64
|
||||
assert call["seeds"].dtype == torch.int64
|
||||
assert call["temperatures"].dtype == torch.float32
|
||||
|
||||
|
||||
def test_use_fp64_follows_the_path_that_forwards_it(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Only the mixed path forwards the sampler's ``use_fp64_gumbel``.
|
||||
|
||||
``GumbelWatermarker._sample_watermarked`` launches the mask-free variant
|
||||
with the ``use_fp64=False`` default, so warming it with fp64 would compile
|
||||
a specialization the engine never launches and miss the one it does.
|
||||
"""
|
||||
calls = _capture_calls(monkeypatch)
|
||||
watermark_sample_warmup(
|
||||
_worker(
|
||||
WatermarkConfig(key=MASTER_KEY, deduplicate_contexts="none"),
|
||||
use_fp64_gumbel=True,
|
||||
)
|
||||
)
|
||||
|
||||
assert calls
|
||||
for call in calls:
|
||||
assert call["use_fp64"] is (call.get("skip_mask") is not None)
|
||||
|
||||
|
||||
def test_warmup_failure_does_not_abort_startup(
|
||||
monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
|
||||
) -> None:
|
||||
def failing_philox_gumbel_sample(*args, **kwargs):
|
||||
raise RuntimeError("no triton")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.worker.gpu.sample.watermark.philox_gumbel_sample",
|
||||
failing_philox_gumbel_sample,
|
||||
)
|
||||
with caplog.at_level(logging.WARNING):
|
||||
watermark_sample_warmup(_worker(WatermarkConfig(key=MASTER_KEY)))
|
||||
|
||||
assert "Skipping watermark sampler warmup." in caplog.text
|
||||
@@ -1,470 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import tempfile
|
||||
from unittest.mock import Mock
|
||||
|
||||
import huggingface_hub.constants
|
||||
import pytest
|
||||
import torch
|
||||
from huggingface_hub.utils import LocalEntryNotFoundError
|
||||
|
||||
import vllm.model_executor.model_loader.weight_utils as weight_utils
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf,
|
||||
drop_checkpoint_cache,
|
||||
initialize_single_dummy_weight,
|
||||
load_attention_sink,
|
||||
maybe_remap_kv_scale_name,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checkpoint_heads,runtime_heads,tp_size",
|
||||
[(8, 4, 2), (8, 8, 1), (8, 6, 2), (16, 12, 4)],
|
||||
)
|
||||
def test_attention_sink_loader_shapes(
|
||||
monkeypatch, checkpoint_heads, runtime_heads, tp_size
|
||||
):
|
||||
"""Each rank loads its checkpoint heads and disables only padded heads."""
|
||||
monkeypatch.setattr(
|
||||
weight_utils, "get_tensor_model_parallel_world_size", lambda: tp_size
|
||||
)
|
||||
checkpoint = torch.arange(checkpoint_heads, dtype=torch.float32)
|
||||
for tp_rank in range(tp_size):
|
||||
monkeypatch.setattr(
|
||||
weight_utils, "get_tensor_model_parallel_rank", lambda rank=tp_rank: rank
|
||||
)
|
||||
param = torch.nn.Parameter(torch.zeros(runtime_heads), requires_grad=False)
|
||||
load_attention_sink(param, checkpoint)
|
||||
local_heads = checkpoint.chunk(tp_size)[tp_rank]
|
||||
expected = torch.cat(
|
||||
(
|
||||
local_heads,
|
||||
torch.full((runtime_heads - local_heads.numel(),), -torch.inf),
|
||||
)
|
||||
)
|
||||
assert torch.equal(param, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checkpoint_shape,runtime_shape,error",
|
||||
[
|
||||
((2, 4), (4,), RuntimeError),
|
||||
((8,), (2, 2), AssertionError),
|
||||
((7,), (4,), ValueError),
|
||||
((8,), (3,), AssertionError),
|
||||
],
|
||||
)
|
||||
def test_attention_sink_loader_rejects_invalid_shapes(
|
||||
monkeypatch, checkpoint_shape, runtime_shape, error
|
||||
):
|
||||
"""Invalid sink shapes fail before changing the runtime parameter."""
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
param = torch.nn.Parameter(torch.ones(runtime_shape), requires_grad=False)
|
||||
with pytest.raises(error):
|
||||
load_attention_sink(param, torch.zeros(checkpoint_shape))
|
||||
assert torch.equal(param, torch.ones(runtime_shape))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param_dtype", [torch.bfloat16, torch.float32])
|
||||
def test_attention_sink_loader_keeps_param_dtype(monkeypatch, param_dtype):
|
||||
"""A checkpoint dtype different from the runtime parameter is cast, not copied."""
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
param = torch.nn.Parameter(torch.zeros(4, dtype=param_dtype), requires_grad=False)
|
||||
load_attention_sink(param, torch.arange(4, dtype=torch.float32))
|
||||
assert param.dtype == param_dtype
|
||||
assert torch.equal(param.float(), torch.arange(4, dtype=torch.float32))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires a GPU")
|
||||
def test_attention_sink_loader_accepts_cpu_checkpoint(monkeypatch):
|
||||
"""Sinks are normally loaded from CPU checkpoints into device parameters."""
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_world_size", lambda: 2)
|
||||
monkeypatch.setattr(weight_utils, "get_tensor_model_parallel_rank", lambda: 1)
|
||||
param = torch.nn.Parameter(
|
||||
torch.full((6,), 123.0, device="cuda"), requires_grad=False
|
||||
)
|
||||
# The test suite sets the default device, so pin the checkpoint to CPU.
|
||||
checkpoint = torch.arange(8, dtype=torch.float32, device="cpu")
|
||||
load_attention_sink(param, checkpoint)
|
||||
assert param.device.type == "cuda"
|
||||
expected = torch.arange(4, 8, dtype=torch.float32, device="cpu")
|
||||
assert torch.equal(param[:4].cpu(), expected)
|
||||
assert torch.isinf(param[4:]).all() and (param[4:] < 0).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_heads,dtype", [(4, torch.float32), (8, torch.bfloat16), (6, None)]
|
||||
)
|
||||
def test_make_attention_sink_registers_loader_and_padding(num_heads, dtype):
|
||||
"""The factory is the single place that creates and registers a sink."""
|
||||
sink = weight_utils.make_attention_sink(num_heads, dtype)
|
||||
assert sink.shape == (num_heads,)
|
||||
assert sink.dtype == (dtype or torch.get_default_dtype())
|
||||
assert sink.requires_grad is False
|
||||
assert torch.isinf(sink).all() and (sink < 0).all()
|
||||
assert sink.weight_loader is load_attention_sink
|
||||
|
||||
|
||||
def test_download_weights_from_hf():
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
# assert LocalEntryNotFoundError error is thrown
|
||||
# if offline is set and model is not cached
|
||||
huggingface_hub.constants.HF_HUB_OFFLINE = True
|
||||
with pytest.raises(LocalEntryNotFoundError):
|
||||
download_weights_from_hf(
|
||||
"facebook/opt-125m",
|
||||
allow_patterns=["*.safetensors", "*.bin"],
|
||||
cache_dir=tmpdir,
|
||||
)
|
||||
|
||||
# download the model
|
||||
huggingface_hub.constants.HF_HUB_OFFLINE = False
|
||||
download_weights_from_hf(
|
||||
"facebook/opt-125m",
|
||||
allow_patterns=["*.safetensors", "*.bin"],
|
||||
cache_dir=tmpdir,
|
||||
)
|
||||
|
||||
# now it should work offline
|
||||
huggingface_hub.constants.HF_HUB_OFFLINE = True
|
||||
assert (
|
||||
download_weights_from_hf(
|
||||
"facebook/opt-125m",
|
||||
allow_patterns=["*.safetensors", "*.bin"],
|
||||
cache_dir=tmpdir,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
|
||||
class TestMaybeRemapKvScaleName:
|
||||
"""Tests for maybe_remap_kv_scale_name covering all checkpoint formats."""
|
||||
|
||||
PARAMS_DICT = {
|
||||
"model.layers.0.self_attn.attn.k_scale": None,
|
||||
"model.layers.0.self_attn.attn.v_scale": None,
|
||||
"model.layers.0.self_attn.attn.q_scale": None,
|
||||
"model.layers.0.self_attn.qkv_proj.weight": None,
|
||||
}
|
||||
|
||||
def test_qkv_proj_k_scale(self):
|
||||
"""Qwen3-MoE / llm-compressor format: qkv_proj.k_scale -> attn.k_scale
|
||||
Regression test for https://github.com/vllm-project/vllm/issues/25047"""
|
||||
result = maybe_remap_kv_scale_name(
|
||||
"model.layers.0.self_attn.qkv_proj.k_scale", self.PARAMS_DICT
|
||||
)
|
||||
assert result == "model.layers.0.self_attn.attn.k_scale"
|
||||
|
||||
def test_qkv_proj_v_scale(self):
|
||||
"""Qwen3-MoE / llm-compressor format: qkv_proj.v_scale -> attn.v_scale
|
||||
Regression test for https://github.com/vllm-project/vllm/issues/25047"""
|
||||
result = maybe_remap_kv_scale_name(
|
||||
"model.layers.0.self_attn.qkv_proj.v_scale", self.PARAMS_DICT
|
||||
)
|
||||
assert result == "model.layers.0.self_attn.attn.v_scale"
|
||||
|
||||
def test_modelopt_k_proj_k_scale(self):
|
||||
"""ModelOpt format: k_proj.k_scale -> attn.k_scale."""
|
||||
result = maybe_remap_kv_scale_name(
|
||||
"model.layers.0.self_attn.k_proj.k_scale", self.PARAMS_DICT
|
||||
)
|
||||
assert result == "model.layers.0.self_attn.attn.k_scale"
|
||||
|
||||
def test_modelopt_v_proj_v_scale(self):
|
||||
"""ModelOpt format: v_proj.v_scale -> attn.v_scale."""
|
||||
result = maybe_remap_kv_scale_name(
|
||||
"model.layers.0.self_attn.v_proj.v_scale", self.PARAMS_DICT
|
||||
)
|
||||
assert result == "model.layers.0.self_attn.attn.v_scale"
|
||||
|
||||
def test_deprecated_kv_scale(self):
|
||||
"""Old format: kv_scale -> attn.k_scale (deprecated)."""
|
||||
result = maybe_remap_kv_scale_name(
|
||||
"model.layers.0.self_attn.kv_scale", self.PARAMS_DICT
|
||||
)
|
||||
assert result == "model.layers.0.self_attn.attn.k_scale"
|
||||
|
||||
def test_default_bare_k_scale(self):
|
||||
"""Default format: .k_scale -> .attn.k_scale."""
|
||||
result = maybe_remap_kv_scale_name(
|
||||
"model.layers.0.self_attn.k_scale", self.PARAMS_DICT
|
||||
)
|
||||
assert result == "model.layers.0.self_attn.attn.k_scale"
|
||||
|
||||
def test_non_scale_name_unchanged(self):
|
||||
"""Non-scale names should be returned unchanged."""
|
||||
name = "model.layers.0.self_attn.qkv_proj.weight"
|
||||
result = maybe_remap_kv_scale_name(name, self.PARAMS_DICT)
|
||||
assert result == name
|
||||
|
||||
def test_nvfp4_modelopt_k_proj_k_scale(self):
|
||||
"""ModelOpt NVFP4 format (e.g. nvidia/Qwen3-30B-A3B-NVFP4):
|
||||
k_proj.k_scale -> attn.k_scale.
|
||||
Validates that NVFP4 checkpoints are not broken by this change."""
|
||||
result = maybe_remap_kv_scale_name(
|
||||
"model.layers.0.self_attn.k_proj.k_scale", self.PARAMS_DICT
|
||||
)
|
||||
assert result == "model.layers.0.self_attn.attn.k_scale"
|
||||
|
||||
def test_nvfp4_modelopt_v_proj_v_scale(self):
|
||||
"""ModelOpt NVFP4 format (e.g. nvidia/Qwen3-30B-A3B-NVFP4):
|
||||
v_proj.v_scale -> attn.v_scale.
|
||||
Validates that NVFP4 checkpoints are not broken by this change."""
|
||||
result = maybe_remap_kv_scale_name(
|
||||
"model.layers.0.self_attn.v_proj.v_scale", self.PARAMS_DICT
|
||||
)
|
||||
assert result == "model.layers.0.self_attn.attn.v_scale"
|
||||
|
||||
def test_qwen3_vl_moe_qkv_proj_k_scale(self):
|
||||
"""Qwen3-VL-MoE uses the same fused qkv_proj naming as Qwen3-MoE.
|
||||
Regression test for qwen3_vl_moe.py fix (same bug as #25047)."""
|
||||
result = maybe_remap_kv_scale_name(
|
||||
"model.layers.0.self_attn.qkv_proj.k_scale", self.PARAMS_DICT
|
||||
)
|
||||
assert result == "model.layers.0.self_attn.attn.k_scale"
|
||||
|
||||
def test_qwen3_vl_moe_qkv_proj_v_scale(self):
|
||||
"""Qwen3-VL-MoE uses the same fused qkv_proj naming as Qwen3-MoE.
|
||||
Regression test for qwen3_vl_moe.py fix (same bug as #25047)."""
|
||||
result = maybe_remap_kv_scale_name(
|
||||
"model.layers.0.self_attn.qkv_proj.v_scale", self.PARAMS_DICT
|
||||
)
|
||||
assert result == "model.layers.0.self_attn.attn.v_scale"
|
||||
|
||||
def test_nvfp4_weight_scale_not_remapped(self):
|
||||
"""NVFP4 weight_scale should not be touched by remap (not a kv scale)."""
|
||||
name = "model.layers.0.self_attn.k_proj.weight_scale"
|
||||
result = maybe_remap_kv_scale_name(name, self.PARAMS_DICT)
|
||||
assert result == name
|
||||
|
||||
def test_nvfp4_input_scale_not_remapped(self):
|
||||
"""NVFP4 input_scale should not be touched by remap (not a kv scale)."""
|
||||
name = "model.layers.0.self_attn.k_proj.input_scale"
|
||||
result = maybe_remap_kv_scale_name(name, self.PARAMS_DICT)
|
||||
assert result == name
|
||||
|
||||
def test_missing_target_returns_none(self):
|
||||
"""If remapped name not in params_dict, return None."""
|
||||
empty_params: dict[str, None] = {}
|
||||
result = maybe_remap_kv_scale_name(
|
||||
"model.layers.0.self_attn.qkv_proj.k_scale", empty_params
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_checkpoint_weight_mapper_discards_serialized_g_idx():
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
|
||||
mapper = QuantizationConfig.get_checkpoint_weight_mapper()
|
||||
|
||||
assert mapper._map_name("model.layers.0.g_idx") is None
|
||||
assert mapper._map_name("model.layers.0.qweight") == "model.layers.0.qweight"
|
||||
|
||||
|
||||
class TestKvCacheScaleMapper:
|
||||
"""The `WeightsMapper` returned by `get_cache_scale_mapper` replaces the
|
||||
per-model `maybe_remap_kv_scale_name` calls. It must remap the same set of
|
||||
checkpoint formats (the non-`params_dict`-dependent ones) and be idempotent
|
||||
so it composes safely with a model's own qkv/gate_up `hf_to_vllm_mapper`."""
|
||||
|
||||
def _mapper(self):
|
||||
# `get_cache_scale_mapper` does not use `self`; call it on the base
|
||||
# class to get the default (non-config-specific) mapper.
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
)
|
||||
|
||||
return QuantizationConfig.get_cache_scale_mapper()
|
||||
|
||||
def _map(self, name: str) -> str | None:
|
||||
return self._mapper()._map_name(name)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name,expected",
|
||||
[
|
||||
# Qwen3-MoE / llm-compressor fused qkv_proj
|
||||
(
|
||||
"model.layers.0.self_attn.qkv_proj.k_scale",
|
||||
"model.layers.0.self_attn.attn.k_scale",
|
||||
),
|
||||
(
|
||||
"model.layers.0.self_attn.qkv_proj.v_scale",
|
||||
"model.layers.0.self_attn.attn.v_scale",
|
||||
),
|
||||
# ModelOpt / NVFP4 k_proj/v_proj
|
||||
(
|
||||
"model.layers.0.self_attn.k_proj.k_scale",
|
||||
"model.layers.0.self_attn.attn.k_scale",
|
||||
),
|
||||
(
|
||||
"model.layers.0.self_attn.v_proj.v_scale",
|
||||
"model.layers.0.self_attn.attn.v_scale",
|
||||
),
|
||||
# deprecated fused kv_scale and bare scales
|
||||
(
|
||||
"model.layers.0.self_attn.kv_scale",
|
||||
"model.layers.0.self_attn.attn.k_scale",
|
||||
),
|
||||
(
|
||||
"model.layers.0.self_attn.k_scale",
|
||||
"model.layers.0.self_attn.attn.k_scale",
|
||||
),
|
||||
# NemotronH mixer
|
||||
(
|
||||
"model.layers.0.mixer.k_proj.k_scale",
|
||||
"model.layers.0.mixer.attn.k_scale",
|
||||
),
|
||||
# already in vLLM form -> unchanged (idempotent)
|
||||
(
|
||||
"model.layers.0.self_attn.attn.k_scale",
|
||||
"model.layers.0.self_attn.attn.k_scale",
|
||||
),
|
||||
# non-kv scales must not be touched
|
||||
(
|
||||
"model.layers.0.self_attn.k_proj.weight_scale",
|
||||
"model.layers.0.self_attn.k_proj.weight_scale",
|
||||
),
|
||||
(
|
||||
"model.layers.0.self_attn.k_proj.input_scale",
|
||||
"model.layers.0.self_attn.k_proj.input_scale",
|
||||
),
|
||||
# regular weights untouched
|
||||
(
|
||||
"model.layers.0.self_attn.q_proj.weight",
|
||||
"model.layers.0.self_attn.q_proj.weight",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_remap(self, name, expected):
|
||||
assert self._map(name) == expected
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name",
|
||||
[
|
||||
"model.layers.0.self_attn.k_scale",
|
||||
"model.layers.0.self_attn.k_proj.k_scale",
|
||||
"model.layers.0.self_attn.qkv_proj.v_scale",
|
||||
"model.layers.0.mixer.k_proj.k_scale",
|
||||
],
|
||||
)
|
||||
def test_idempotent(self, name):
|
||||
once = self._map(name)
|
||||
assert once is not None
|
||||
assert self._map(once) == once
|
||||
|
||||
def test_composes_with_qkv_mapper(self):
|
||||
"""Applied together with a model's qkv/gate_up mapper, the regex scale
|
||||
rules run before the substr rename, so scales are normalized to `.attn.`
|
||||
and regular projections are still fused correctly."""
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
|
||||
model_mapper = WeightsMapper(
|
||||
orig_to_new_substr={
|
||||
".q_proj": ".qkv_proj.q",
|
||||
".k_proj": ".qkv_proj.k",
|
||||
".v_proj": ".qkv_proj.v",
|
||||
}
|
||||
)
|
||||
# AutoWeightsLoader does `mapper |= cache_scale_mapper`
|
||||
combined = model_mapper | self._mapper()
|
||||
|
||||
assert (
|
||||
combined._map_name("model.layers.0.self_attn.q_proj.weight")
|
||||
== "model.layers.0.self_attn.qkv_proj.q.weight"
|
||||
)
|
||||
assert (
|
||||
combined._map_name("model.layers.0.self_attn.k_proj.k_scale")
|
||||
== "model.layers.0.self_attn.attn.k_scale"
|
||||
)
|
||||
assert (
|
||||
combined._map_name("model.layers.0.self_attn.k_scale")
|
||||
== "model.layers.0.self_attn.attn.k_scale"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fails", [False, True])
|
||||
def test_drop_checkpoint_cache_releases_local_weight_files(
|
||||
tmp_path, monkeypatch, fails
|
||||
):
|
||||
"""Evict only weights; cache eviction errors must not prevent loading."""
|
||||
for name in ("weights.safetensors", "config.json"):
|
||||
(tmp_path / name).touch()
|
||||
fadvise = Mock(side_effect=OSError("cache eviction failed") if fails else None)
|
||||
monkeypatch.setattr("os.posix_fadvise", fadvise)
|
||||
drop_checkpoint_cache(str(tmp_path))
|
||||
fadvise.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float8_e4m3fn, torch.float8_e5m2])
|
||||
def test_dummy_fp8_weight_is_not_degenerate(dtype):
|
||||
"""+-1e-3 underflows in FP8, so 8-bit floats default to +-1."""
|
||||
param = torch.empty(4096, dtype=dtype)
|
||||
initialize_single_dummy_weight(param)
|
||||
values = param.float()
|
||||
assert values.isfinite().all()
|
||||
assert values.abs().max() <= 1.0
|
||||
assert (values != 0).float().mean() > 0.99
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dtype", "name"),
|
||||
[(torch.float8_e8m0fnu, "weight_scale_inv"), (torch.uint8, "w13_weight_scale")],
|
||||
)
|
||||
def test_dummy_e8m0_scale_is_fixed_exponent(dtype, name):
|
||||
"""E8M0 scales, including uint8-stored ones, get a finite 2**-7."""
|
||||
param = torch.empty(64, dtype=dtype)
|
||||
initialize_single_dummy_weight(param, name=name)
|
||||
assert (param.view(torch.uint8) == 120).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.uint8, torch.int8, torch.float4_e2m1fn_x2])
|
||||
def test_dummy_packed_weight_gets_random_bytes(dtype):
|
||||
"""Packed weights are filled deterministically instead of left uninitialized."""
|
||||
# Some backends can't fill packed FP4 directly, so view zeroed bytes.
|
||||
params = [torch.zeros(4096, dtype=torch.uint8).view(dtype) for _ in range(2)]
|
||||
for param in params:
|
||||
initialize_single_dummy_weight(param, name="w13_weight")
|
||||
raw = params[0].view(torch.uint8)
|
||||
assert raw.unique().numel() > 250
|
||||
assert torch.equal(raw, params[1].view(torch.uint8))
|
||||
|
||||
|
||||
@pytest.mark.skipif(current_platform.is_rocm(), reason="ROCm zeroes integer params")
|
||||
def test_dummy_wide_integer_param_is_untouched():
|
||||
"""Wider integer params may be indices, so random values are unsafe."""
|
||||
param = torch.arange(16, dtype=torch.int32)
|
||||
initialize_single_dummy_weight(param, name="g_idx")
|
||||
assert torch.equal(param, torch.arange(16, dtype=torch.int32))
|
||||
|
||||
|
||||
def test_dummy_norm_weights_are_neutral(default_vllm_config):
|
||||
"""Dummy norms stay identity-like, keeping activations at a realistic scale."""
|
||||
from vllm.model_executor.layers.layernorm import (
|
||||
GemmaRMSNorm,
|
||||
LayerNorm,
|
||||
RMSNorm,
|
||||
RMSNormGated,
|
||||
)
|
||||
|
||||
expected = {
|
||||
RMSNorm(16): {"weight": 1.0},
|
||||
GemmaRMSNorm(16): {"weight": 0.0},
|
||||
LayerNorm(16): {"weight": 1.0, "bias": 0.0},
|
||||
RMSNormGated(16): {"weight": 1.0},
|
||||
}
|
||||
for norm, values in expected.items():
|
||||
for name, param in norm.named_parameters():
|
||||
param.data.uniform_(-5, 5)
|
||||
initialize_single_dummy_weight(param, name=name)
|
||||
assert torch.all(param == values[name]), (type(norm).__name__, name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_download_weights_from_hf()
|
||||
Reference in New Issue
Block a user