Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
@@ -0,0 +1,104 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Differential test for the AITER MLA uniform-MTP padding decision.
`_uniform_padded_mtp_qo_len` decides whether a full-CUDA-graph decode batch may
synthesize dummy query rows. It used to reach that decision through eight tensor
dispatches -- sum/item, boolean-mask index, all, eq, nonzero, item, any -- on a
CPU tensor holding one entry per request. It now reads the tensor out once and
decides on Python ints.
That rewrite also **reordered** the checks: the `num_decode_tokens <= sum(...)`
early-out moved below the divisibility and `uniform_qo_len <= 1` guards. Every
one of those paths returns 0, so the result should be unchanged -- but "should
be" is exactly the sort of claim worth checking exhaustively, so both
implementations are carried here and compared over the whole small input space.
Pure host arithmetic; no GPU required.
"""
import itertools
import pytest
import torch
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
def _reference(qo_len: torch.Tensor, max_qo_len: int, num_decode_tokens: int) -> int:
"""The pre-#58381 implementation, verbatim."""
num_reqs = qo_len.numel()
if num_reqs == 0 or num_decode_tokens <= 0:
return 0
if num_decode_tokens <= int(qo_len.sum().item()):
return 0
if num_decode_tokens % num_reqs != 0:
return 0
uniform_qo_len = num_decode_tokens // num_reqs
if uniform_qo_len <= 1:
return 0
positive_qo_len = qo_len[qo_len > 0]
if positive_qo_len.numel() == qo_len.numel():
return 0
if positive_qo_len.numel() > 0:
if max_qo_len != uniform_qo_len:
return 0
if not torch.all(positive_qo_len == uniform_qo_len):
return 0
zero_positions = torch.nonzero(qo_len == 0, as_tuple=False).flatten()
if zero_positions.numel() > 0:
first_zero = int(zero_positions[0].item())
if torch.any(qo_len[first_zero:] > 0):
return 0
return uniform_qo_len
_ACTUAL = AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len
@pytest.mark.parametrize("num_reqs", [1, 2, 3, 4])
def test_matches_reference_exhaustively(num_reqs):
"""Every qo_len vector x decode-token count x max_qo_len in a small space."""
checked = 0
for lens in itertools.product(range(0, 4), repeat=num_reqs):
qo = torch.tensor(lens, dtype=torch.int32)
for num_decode_tokens in range(0, 13):
for max_qo_len in range(0, 5):
got = _ACTUAL(qo, max_qo_len, num_decode_tokens)
want = _reference(qo, max_qo_len, num_decode_tokens)
assert got == want, (
f"lens={lens} num_decode_tokens={num_decode_tokens} "
f"max_qo_len={max_qo_len}: got {got}, reference {want}"
)
checked += 1
assert checked > 0
def test_empty_batch():
assert _ACTUAL(torch.zeros(0, dtype=torch.int32), 4, 8) == 0
assert _reference(torch.zeros(0, dtype=torch.int32), 4, 8) == 0
def test_the_padded_mtp_case_that_should_succeed():
"""A live prefix at the uniform width, then zero-padded dummies."""
qo = torch.tensor([4, 4, 0, 0], dtype=torch.int32)
# 4 requests x 4 tokens = 16 captured, 8 live -> dummies are synthesizable.
assert _ACTUAL(qo, 4, 16) == 4
assert _reference(qo, 4, 16) == 4
def test_positive_after_zero_is_rejected():
"""A live request after a padding hole breaks the contiguous-prefix layout."""
qo = torch.tensor([4, 0, 4, 0], dtype=torch.int32)
assert _ACTUAL(qo, 4, 16) == 0
assert _reference(qo, 4, 16) == 0
def test_ragged_live_lengths_are_rejected():
qo = torch.tensor([4, 2, 0, 0], dtype=torch.int32)
assert _ACTUAL(qo, 4, 16) == 0
assert _reference(qo, 4, 16) == 0
+99 -3
View File
@@ -31,6 +31,7 @@ from vllm.utils.torch_utils import (
STR_DTYPE_TO_TORCH_DTYPE,
is_quantized_kv_cache,
is_torch_equal_or_newer,
nvfp4_kv_cache_full_dim,
set_random_seed,
)
from vllm.v1.attention.backend import (
@@ -41,7 +42,7 @@ from vllm.v1.attention.backend import (
max_decode_query_len,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout, KVQuantMode
BACKENDS_TO_TEST = [
AttentionBackendEnum.FLASH_ATTN,
@@ -974,8 +975,6 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
causal=True,
prefill=None,
decode=decode,
use_cascade=False,
cascade_wrapper=None,
)
seen_shapes = {}
@@ -1698,6 +1697,103 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
or not current_platform.is_device_capability_family(120),
reason="NVFP4 XQA decode requires SM12x.",
)
@pytest.mark.parametrize("use_cuda_graph", [False, True])
@pytest.mark.parametrize("q_len", [1, 4])
def test_flashinfer_xqa_nvfp4_decode_correctness(
default_vllm_config, q_len, use_cuda_graph
):
"""Single-token and spec-decode queries over an NVFP4 cache decode with
XQA, eager and replayed, and match the fa2 prefill that wrote the cache."""
import unittest.mock
from vllm.v1.attention.backends import flashinfer as fi
seq_lens, bs, d = [40, 300, 1027], 16, 256
config = create_vllm_config(
model_name="Qwen/Qwen3-0.6B",
max_model_len=max(seq_lens),
block_size=bs,
)
# GQA 6 at head_dim 256, as Qwen3.8-27B at TP=2.
config.model_config.model_arch_config = replace(
config.model_config.model_arch_config,
total_num_attention_heads=24,
total_num_kv_heads=4,
head_size=d,
)
config.cache_config.cache_dtype = "nvfp4"
if q_len > 1:
config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=q_len - 1
)
spec = FullAttentionSpec(
block_size=bs,
num_kv_heads=4,
head_size=d,
dtype=torch.uint8,
kv_quant_mode=KVQuantMode.NVFP4,
)
cg_support = fi.FlashInferMetadataBuilder.get_cudagraph_support(config, spec)
assert cg_support == AttentionCGSupport.UNIFORM_BATCH
device = torch.device(f"{DEVICE_TYPE}:0")
# Request i owns consecutive pages from starts[i]; page 0 stays null.
pages = [0] + [cdiv(s, bs) for s in seq_lens[:-1]]
starts = 1 + torch.tensor(pages).cumsum(0)
max_pages = cdiv(max(seq_lens), bs)
block_table = (starts[:, None] + torch.arange(max_pages)).int().to(device)
full_dim = nvfp4_kv_cache_full_dim(d)
kv_cache = torch.zeros(
int(starts[-1]) + max_pages, bs, 8, full_dim, dtype=torch.uint8, device=device
).transpose(1, 2)
q, k, v = (
torch.randn(sum(seq_lens), h, d).bfloat16().to(device) for h in (24, 4, 4)
)
def run(query_lens, use_cuda_graph=False):
req = torch.repeat_interleave(torch.arange(3), torch.tensor(query_lens))
pos = torch.cat([torch.arange(s - n, s) for s, n in zip(seq_lens, query_lens)])
rows = pos + torch.tensor([0, *seq_lens]).cumsum(0)[req]
metadata = replace(
create_common_attn_metadata(BatchSpec(seq_lens, query_lens), bs, device),
block_table_tensor=block_table,
slot_mapping=(starts[req] * bs + pos).to(device),
)
with set_current_vllm_config(config):
return rows, run_attention_backend(
AttentionBackendEnum.FLASHINFER,
spec,
["placeholder"],
config,
device,
metadata,
q[rows],
k[rows],
v[rows],
kv_cache,
kv_cache_dtype="nvfp4",
use_cuda_graph=use_cuda_graph,
layer_k_scale=0.02,
layer_v_scale=0.03,
)
# The full prefill writes every token through the NVFP4 writer; its fa2
# output for the last q_len tokens of each request is the reference.
_, reference = run(seq_lens)
xqa_fn = fi.flashinfer_xqa_batch_decode_with_kv_cache
with unittest.mock.patch.object(
fi, "flashinfer_xqa_batch_decode_with_kv_cache", wraps=xqa_fn
) as xqa:
rows, output = run([q_len] * len(seq_lens), use_cuda_graph)
assert xqa.called
torch.testing.assert_close(output, reference[rows], atol=2e-2, rtol=2e-2)
if current_platform.is_rocm():
# FLASH_ATTN is not supported on ROCm
SLIDING_WINDOW_BACKENDS_TO_TEST = [
@@ -10,6 +10,10 @@ from vllm.models.deepseek_v4.amd.rocm import (
DeepseekV4ROCMAiterMLASparseMetadataBuilder,
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
)
from vllm.models.deepseek_v41.amd import rocm as v41_rocm
from vllm.models.deepseek_v41.amd.rocm import (
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
)
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backends.mla import indexer
from vllm.v1.attention.backends.mla.indexer import (
@@ -51,7 +55,7 @@ def _make_indexer_builder(*, adaptive: bool, capacity: int = 12):
@pytest.mark.cpu_test
def test_deepseek_v4_rocm_adaptive_builders_support_varlen_full_graphs():
def test_deepseek_rocm_adaptive_builders_support_varlen_full_graphs():
adaptive_config = SimpleNamespace(
speculative_config=SimpleNamespace(enable_adaptive_verification=True)
)
@@ -62,6 +66,8 @@ def test_deepseek_v4_rocm_adaptive_builders_support_varlen_full_graphs():
for builder_cls in (
DeepseekV4ROCMAiterMLASparseMetadataBuilder,
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
v41_rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
):
assert (
builder_cls.get_cudagraph_support(adaptive_config, SimpleNamespace())
@@ -74,20 +80,25 @@ def test_deepseek_v4_rocm_adaptive_builders_support_varlen_full_graphs():
@pytest.mark.cpu_test
def test_deepseek_v4_rocm_adaptive_indexer_support(monkeypatch: pytest.MonkeyPatch):
@pytest.mark.parametrize(
("architecture", "backend_cls"),
[
("DeepseekV4ForCausalLM", DeepseekV4IndexerBackend),
("DeepseekV41ForCausalLM", DeepseekV41IndexerBackend),
],
)
def test_deepseek_rocm_adaptive_indexer_support(
monkeypatch: pytest.MonkeyPatch,
architecture: str,
backend_cls: type[DeepseekV4IndexerBackend],
):
_mock_rocm_platform(monkeypatch)
adaptive_config = _make_indexer_config(
architecture="DeepseekV4ForCausalLM", adaptive=True
)
fixed_config = _make_indexer_config(
architecture="DeepseekV4ForCausalLM", adaptive=False
)
adaptive_config = _make_indexer_config(architecture=architecture, adaptive=True)
fixed_config = _make_indexer_config(architecture=architecture, adaptive=False)
assert DeepseekV4IndexerBackend.supports_device_cpu_query_lens_mismatch()
assert backend_cls.supports_device_cpu_query_lens_mismatch()
assert indexer._rocm_supports_flattened_device_query_lens(adaptive_config)
assert (
DeepseekV4IndexerBackend.get_builder_cls() is DeepseekV32IndexerMetadataBuilder
)
assert backend_cls.get_builder_cls() is DeepseekV32IndexerMetadataBuilder
assert indexer._use_flattening(adaptive_config)
assert (
DeepseekV32IndexerMetadataBuilder.get_cudagraph_support(
@@ -104,12 +115,39 @@ def test_deepseek_v4_rocm_adaptive_indexer_support(monkeypatch: pytest.MonkeyPat
)
@pytest.mark.cpu_test
def test_deepseek_v41_rocm_fixed_k_path_is_unchanged(
monkeypatch: pytest.MonkeyPatch,
):
_mock_rocm_platform(monkeypatch)
fixed_config = _make_indexer_config(
architecture="DeepseekV41ForCausalLM", adaptive=False
)
for builder_cls in (
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
v41_rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
):
assert (
builder_cls.get_cudagraph_support(fixed_config, SimpleNamespace())
== AttentionCGSupport.UNIFORM_BATCH
)
assert not indexer._use_flattening(fixed_config)
assert (
DeepseekV32IndexerMetadataBuilder.get_cudagraph_support(
fixed_config, SimpleNamespace()
)
== AttentionCGSupport.UNIFORM_BATCH
)
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"architecture",
["DeepseekV32ForCausalLM", "DeepseekV41ForCausalLM", "GlmMoeDsaForCausalLM"],
["DeepseekV32ForCausalLM", "GlmMoeDsaForCausalLM"],
)
def test_rocm_adaptive_indexer_flattening_is_scoped_to_deepseek_v4(
def test_rocm_adaptive_indexer_flattening_rejects_other_architectures(
monkeypatch: pytest.MonkeyPatch,
architecture: str,
):
@@ -125,9 +163,6 @@ def test_rocm_adaptive_indexer_flattening_is_scoped_to_deepseek_v4(
== AttentionCGSupport.UNIFORM_BATCH
)
if architecture == "DeepseekV41ForCausalLM":
assert not DeepseekV41IndexerBackend.supports_device_cpu_query_lens_mismatch()
@pytest.mark.cpu_test
def test_rocm_adaptive_indexer_preserves_single_request_uniform_path(
@@ -15,6 +15,7 @@ import inspect
import pytest
import torch
from transformers import DeepseekV4Config
from typing_extensions import TypedDict
from tests.v1.attention.utils import create_vllm_config
@@ -532,26 +533,26 @@ def make_builder(
builder_cls: type[
DeepseekSparseSWAMetadataBuilder
] = DeepseekSparseSWAMetadataBuilder,
mm_prefix_clamp_sliding_window: bool = True,
deepseek_v4_config: bool = True,
bounded_replay: bool = False,
) -> DeepseekSparseSWAMetadataBuilder:
overrides: dict = {"sliding_window": window}
if vision:
# Emulate the V4 vision config, whose mm_prefix_clamp_sliding_window
# gates the in-kernel SWA widening. V4.1 (causal image tokens) leaves
# it off.
overrides.update(
vision_n_layers=2,
vision_max_n_token=max_image_tokens,
mm_prefix_clamp_sliding_window=mm_prefix_clamp_sliding_window,
)
vision_fields = dict(vision_n_layers=2, vision_max_n_token=max_image_tokens)
vllm_config = create_vllm_config(
model_name=model_name,
max_model_len=4096,
max_num_batched_tokens=max_num_batched_tokens,
max_num_seqs=8,
hf_config_override=overrides,
hf_config_override={"sliding_window": window}
| (vision_fields if vision and not deepseek_v4_config else {}),
)
if vision and deepseek_v4_config:
# Only V4 configs widen the SWA window in-kernel over image spans
vllm_config.model_config.hf_config = DeepseekV4Config(
num_hidden_layers=1,
layer_types=["sliding_attention"],
sliding_window=window,
**vision_fields,
)
spec = SlidingWindowMLASpec(
block_size=BLOCK_SIZE,
num_kv_heads=1,
@@ -686,7 +687,7 @@ def test_v41_image_prefill_uses_causal_swa(compress_ratio, query_len):
max_image_tokens,
query_len,
builder_cls=DeepseekV41SparseSWAMetadataBuilder,
mm_prefix_clamp_sliding_window=False,
deepseek_v4_config=False,
)
assert builder.max_image_tokens == 0
md = build_metadata(builder, [seq_len], [query_len], {0: spans})
@@ -828,3 +829,57 @@ def test_v41_flashinfer_mixed_sparse_indices_respect_replay_start():
# Decode row is copied through; prefill rows follow the bounded window.
assert bounded[0].cpu().tolist() == plain[0].cpu().tolist()
assert bounded[1:].cpu().tolist() == rows[1:]
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_dspark_noncausal_draft_decode_update_matches_build():
"""A DSpark draft graph refreshes its non-causal SWA metadata in place
instead of rebuilding it, so the refresh must match a fresh build."""
from vllm.v1.attention.backends.mla.compressor_utils import (
get_dspark_swa_index_width,
)
num_spec = 4
query_lens = [num_spec] * 3
def make_dspark_builder():
builder = make_builder(vision=False)
builder.is_dspark = True
builder.noncausal_index_width = get_dspark_swa_index_width(WINDOW, num_spec)
builder.decode_threshold = num_spec
return builder
def common_metadata(seq_lens):
query_start_loc, seq_lens_t, _, slot_mapping, block_table = make_batch(
seq_lens, query_lens, torch.device("cuda")
)
return CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=seq_lens_t,
seq_lens_cpu_upper_bound=seq_lens_t.cpu(),
num_reqs=len(seq_lens),
num_actual_tokens=int(query_start_loc[-1]),
max_query_len=num_spec,
max_seq_len=max(seq_lens),
block_table_tensor=block_table,
slot_mapping=slot_mapping,
causal=False,
)
captured_seq_lens = [6, 40, 130]
replay_seq_lens = [130, 9, 90]
builder = make_dspark_builder()
captured = common_metadata(captured_seq_lens)
md = builder.build(0, captured)
replay = common_metadata(replay_seq_lens)
expected = make_dspark_builder().build(0, replay)
# A replay sees new inputs through the same device buffers.
captured.seq_lens.copy_(replay.seq_lens)
captured.block_table_tensor.copy_(replay.block_table_tensor)
builder.update_draft_decode_metadata(md)
assert md.decode_swa_width == builder.noncausal_index_width
torch.testing.assert_close(md.decode_swa_lens, expected.decode_swa_lens)
torch.testing.assert_close(md.decode_swa_indices, expected.decode_swa_indices)
@@ -425,7 +425,7 @@ def _skip_if_backend_unavailable(backend_cls, kv_cache_dtype: str, block_size: i
and kv_cache_dtype != "fp8_ds_mla"
):
pytest.skip("FlashMLA Sparse fp8 only supports fp8_ds_mla kv-cache dtype")
if block_size not in backend_cls.get_supported_kernel_block_sizes():
if not backend_cls.supports_block_size(block_size):
pytest.skip(
f"{backend_cls.get_name()} does not support block_size={block_size}"
)
@@ -29,7 +29,7 @@ HEAD = 512
TOPK = 128 # triton convert requires width % 128 == 0
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **kw):
out = torch.full_like(token_indices, -1)
counts = torch.zeros(token_indices.shape[0], dtype=torch.int32)
for t in range(token_indices.shape[0]):
@@ -41,7 +41,7 @@ def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
blk = int(block_table[int(req_id[t]), pos // BLOCK_SIZE])
if blk < 0:
continue
vals.append(blk * BLOCK_SIZE + pos % BLOCK_SIZE)
vals.append(blk * kw["BLOCK_STRIDE_ROWS"] + pos % BLOCK_SIZE)
out[t, : len(vals)] = torch.tensor(vals, dtype=out.dtype)
counts[t] = len(vals)
return out, counts
@@ -133,11 +133,8 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
meta.state = state
q_nope = torch.randn(rows, impl.num_heads, HEAD)
q_rope = torch.randn(rows, impl.num_heads, qk_rope)
cache = torch.zeros(
8 * BLOCK_SIZE,
impl.head_size,
dtype=torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16,
)
dtype = torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16
cache = torch.zeros(8, 2, BLOCK_SIZE, impl.head_size, dtype=dtype)[:, 0]
out, lse = impl.forward_mqa(
(q_nope, q_rope), cache, meta, SimpleNamespace(_k_scale_float=0.5)
@@ -147,7 +144,10 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
# Reserved buffers carry this step's slots; lengths are NOT refreshed
# here (the builder plans them host-side before capture/replay).
ref_slots, ref_counts = ref_convert(
meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer
meta.req_id_per_token,
meta.block_table,
impl.topk_indices_buffer,
BLOCK_STRIDE_ROWS=2 * BLOCK_SIZE,
)
offset = 0
for t in range(rows):
@@ -161,7 +161,7 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
assert state.wrapper.run_args is not None
q_pe, ckv, kpe, kwargs = state.wrapper.run_args[1:]
assert q_pe.shape == (rows, impl.num_heads, qk_rope)
assert ckv.shape == (8 * BLOCK_SIZE, 1, HEAD)
assert ckv.shape == ((8 - 1) * 2 * BLOCK_SIZE + BLOCK_SIZE, 1, HEAD)
assert kpe.shape[-1] == qk_rope
if impl.use_fp8_kv_cache:
assert kwargs["ckv_scale"] == 0.5 and kwargs["kpe_scale"] == 1.0
@@ -24,6 +24,21 @@ from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends.mla.flashmla_sparse import (
QUANTIZED_DS_MLA_CACHE_FORMATS,
FlashMLASparseBackend,
_fp8_kv_pages,
)
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheLayout,
KVCacheTensor,
KVQuantMode,
MLAAttentionSpec,
)
from vllm.v1.worker.utils import (
AttentionGroup,
allocate_kv_cache,
prepare_kernel_block_sizes,
select_common_block_size,
)
SM90 = DeviceCapability(major=9, minor=0)
@@ -83,6 +98,88 @@ def test_supported_head_sizes_include_512():
assert FlashMLASparseBackend.get_supported_head_sizes() == [576, 512]
def test_flashmla_bf16_nope_accepts_packed_manager_blocks():
spec = MLAAttentionSpec(
block_size=1152,
num_kv_heads=1,
head_size=512,
dtype=torch.bfloat16,
cache_dtype_str="bfloat16",
block_stride_alignment=1024,
)
num_blocks = 2
layers = ["layer.0", "layer.1"]
page_size = spec.page_size_bytes
config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=num_blocks * len(layers) * page_size,
layers=layers,
layer_stride=page_size,
block_stride=len(layers) * page_size,
)
],
kv_cache_groups=[KVCacheGroupSpec(layers, spec)],
)
groups = [[AttentionGroup(FlashMLASparseBackend, layers, spec, 0)]]
kernel_block_sizes = prepare_kernel_block_sizes(config, groups)
caches = allocate_kv_cache(
config,
torch.device("cpu"),
KVCacheLayout.BLHNC,
kernel_block_sizes,
)
assert kernel_block_sizes == [1152]
assert caches["layer.0"].shape == (num_blocks, 1, 1152, 512)
@pytest.mark.parametrize("sm100", [False, True])
def test_flashmla_quantized_cache_kernel_pages(monkeypatch, sm100):
"""Quantized blocks are read as 64-token pages; SM100 only row-aligns them
(TMA) and keeps 64-token blocks."""
monkeypatch.setattr(
current_platform,
"is_device_capability_family",
lambda family, device_id=0: sm100 and family == 100,
)
spec = MLAAttentionSpec(
block_size=1152,
num_kv_heads=1,
head_size=576,
dtype=torch.float8_e4m3fn,
cache_dtype_str="fp8_ds_mla",
kv_quant_mode=KVQuantMode.FP8_PER_TENSOR,
state_content_bytes=656,
)
kernel_block_size = 64 if sm100 else 1152
assert (
select_common_block_size(1152, [FlashMLASparseBackend], [spec])
== kernel_block_size
)
def test_fp8_kv_pages_address_packed_blocks():
"""Token t of block b is row b * block_stride_rows + t of the 64-row pages;
64-token blocks are passed through."""
row, block_size, stride_rows = 656, 128, 192
backing = torch.arange(3 * stride_rows).repeat_interleave(row)
cache = backing.view(3, stride_rows, row)[:, :block_size]
pages, block_stride_rows = _fp8_kv_pages(cache, block_size)
assert block_stride_rows == stride_rows
assert pages.shape[1:] == (64, row)
slot = 2 * block_stride_rows + 100
assert torch.equal(pages[slot // 64, slot % 64], cache[2, 100])
pages, block_stride_rows = _fp8_kv_pages(cache[:, :64], 64)
assert block_stride_rows is None
assert torch.equal(pages, cache[:, :64])
def test_quantized_ds_mla_formats_are_the_envelope_set():
assert frozenset({"fp8_ds_mla", "nvfp4_ds_mla"}) == QUANTIZED_DS_MLA_CACHE_FORMATS
@@ -141,6 +238,7 @@ def _sparse_order(kv_cache_dtype, head_size, num_heads=32, capability=SM100):
)
sparse = {
"FLASH_ATTN_MLA_SPARSE",
"FLASH_ATTN_MLA_SPARSE_FA4",
"FLASHMLA_SPARSE",
"FLASHINFER_MLA_SPARSE",
"FLASHINFER_MLA_SPARSE_SM90",
@@ -179,3 +277,36 @@ def test_sm90_nope_fp8_ds_mla_resolves_to_flashmla():
for name in order[: order.index("FLASHMLA_SPARSE")]:
backend_cls = AttentionBackendEnum[name].get_class()
assert not backend_cls.supports_kv_cache_dtype("fp8_ds_mla"), name
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA backends")
def test_sm100_576_bf16_low_heads_falls_back_from_fa4(monkeypatch):
"""With FA4 unusable, low-head bf16 selection lands on FlashInfer."""
import vllm.v1.attention.backends.mla.flashattn_mla_sparse as fa4_sparse
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.v1.attention.backends.registry import AttentionBackendEnum
monkeypatch.setattr(fa4_sparse, "_fa4_cute_mla_available", lambda: "no FA4")
def reasons(name):
backend_cls = AttentionBackendEnum[name].get_class()
return backend_cls.validate_configuration(
head_size=576,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
block_size=64,
use_mla=True,
has_sink=False,
use_sparse=True,
use_mm_prefix=False,
use_per_head_quant_scales=False,
device_capability=SM100,
attn_type="decoder",
)
order = _sparse_order("auto", 576, num_heads=16)
with set_current_vllm_config(VllmConfig()):
# The selector takes the first candidate with no rejection reasons.
selected = next(name for name in order if not reasons(name))
assert order[0] == "FLASH_ATTN_MLA_SPARSE_FA4", order
assert selected == "FLASHINFER_MLA_SPARSE"
+33 -9
View File
@@ -35,7 +35,7 @@ def _make_config(
)
def _resolve_on_rocm(config, *, kernels_available: bool = True):
def _resolve_on_rocm(config, *, kernels_available: bool = True, mi3xx: bool = True):
with (
patch.object(
qwen_gdn_linear_attn.current_platform, "is_rocm", return_value=True
@@ -45,6 +45,7 @@ def _resolve_on_rocm(config, *, kernels_available: bool = True):
"is_gdn_flydsl_prefill_available",
return_value=kernels_available,
),
patch("vllm.platforms.rocm.on_mi3xx", return_value=mi3xx),
):
return _resolve_gdn_prefill_backend(config)
@@ -72,16 +73,38 @@ def test_resolve_aiter_flydsl_gdn_prefill_backend(
assert active == expected
def test_kda_style_model_cannot_select_flydsl():
@pytest.mark.parametrize(
"mi3xx,kernels_available,head_k_dim,expected",
[
(True, True, 128, "aiter_flydsl"),
(False, True, 128, "triton"),
(True, False, 128, "triton"),
(True, True, 64, "triton"),
],
)
def test_auto_defaults_to_aiter_flydsl(
mi3xx: bool, kernels_available: bool, head_k_dim: int, expected: str
):
"""Under auto, FlyDSL is the default only where it was validated and the
model and build can serve it; everything else stays on Triton."""
config = _make_config(backend="auto", head_k_dim=head_k_dim)
assert _resolve_on_rocm(
config, kernels_available=kernels_available, mi3xx=mi3xx
) == ("auto", expected)
@pytest.mark.parametrize("backend", ["aiter_flydsl", "auto"])
def test_kda_style_model_cannot_select_flydsl(backend: str):
"""Kimi K3 KDA reports head dims through linear_attn_config, not these.
Its builder overrides _build_chunk_metadata, which the AITER path skips,
so this pins the reason the two never meet: the resolver turns the model
down before the builder is ever constructed.
"""
config = _make_config(head_k_dim=None, head_v_dim=None)
config = _make_config(backend=backend, head_k_dim=None, head_v_dim=None)
assert _resolve_on_rocm(config) == ("aiter_flydsl", "triton")
assert _resolve_on_rocm(config) == (backend, "triton")
@pytest.mark.parametrize("on_rocm", [True, False], ids=["kernels_missing", "not_rocm"])
@@ -109,15 +132,16 @@ def test_explicit_aiter_flydsl_fails_closed(on_rocm: bool):
def _flydsl_prefill_unavailable() -> bool:
return (
not current_platform.is_rocm()
or not rocm_aiter_ops.is_gdn_flydsl_prefill_available()
)
if not current_platform.is_rocm():
return True
from vllm.platforms.rocm import on_mi3xx
return not on_mi3xx() or not rocm_aiter_ops.is_gdn_flydsl_prefill_available()
@pytest.mark.skipif(
_flydsl_prefill_unavailable(),
reason="needs ROCm with the AITER FlyDSL GDN prefill kernels installed",
reason="needs gfx942/gfx950 with the AITER FlyDSL GDN prefill kernels installed",
)
@pytest.mark.parametrize(
"state_dtype", [torch.float32, torch.bfloat16], ids=["fp32_state", "bf16_state"]
@@ -21,6 +21,7 @@ from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
plan_flashinfer_checkpoints,
)
from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
from vllm.v1.kv_cache_interface import MambaSpec
@@ -488,3 +489,20 @@ def test_update_block_table_regathers_checkpoint(num_spec):
assert not torch.equal(
actual.checkpoint.state_indices, source.checkpoint.state_indices
)
def test_plan_flashinfer_checkpoints_selects_the_state_at_each_offset():
"""Each checkpoint row maps to the state FlashInfer emits at its offset.
The leading decode row is not in the prefill kernel and never checkpoints;
the 50-token prefill is shorter than the stride and emits no states.
"""
every_n_tokens, cu_starts, rows = plan_flashinfer_checkpoints(
offsets=[0, 128, 0, 192], query_lens=[200, 50, 300]
)
assert every_n_tokens == 64
assert cu_starts == [0, 3, 3, 7]
# Row 1: state after 128 tokens of the first prefill. Row 3: after 192
# tokens of the third prefill, whose states start at index 3.
assert rows == [0, 1, 0, 5]
@@ -0,0 +1,40 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-5.3-Flash indexer kernel block sizes are DeepGEMM pool pages."""
from types import SimpleNamespace
import pytest
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla import indexer
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
KPOOL = 4
@pytest.mark.parametrize(
"rocm,sm120,expected",
[
(False, False, [KPOOL * 32, KPOOL * 64]),
# SM120 DeepGEMM takes only 64-entry pages for the fp8 cache.
(False, True, [KPOOL * 64]),
# ROCm's AITER paged-MQA logits take the same pool pages.
(True, False, [KPOOL * 32, KPOOL * 64]),
],
)
def test_glm5next_indexer_kernel_block_sizes(monkeypatch, rocm, sm120, expected):
monkeypatch.setattr(current_platform, "is_rocm", lambda: rocm)
monkeypatch.setattr(
current_platform,
"is_device_capability_family",
lambda family: sm120 and family == 120,
)
model_config = SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=KPOOL))
config = SimpleNamespace(model_config=model_config)
monkeypatch.setattr(indexer, "get_current_vllm_config", lambda: config)
spec = SimpleNamespace(tokens_per_state=KPOOL)
assert Glm5NextIndexerBackend.get_supported_kernel_block_sizes(spec) == expected
# Platform block-size selection queries without a spec.
assert Glm5NextIndexerBackend.get_supported_kernel_block_sizes() == expected
@@ -14,6 +14,7 @@ from vllm.models.deepseek_v4.sparse_mla import DeepseekV4SparseMLABackend
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend,
)
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla import indexer as indexer_module
from vllm.v1.attention.backends.mla.compressor_utils import (
@@ -113,16 +114,22 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
num_reqs, 4
)
num_sms = torch.cuda.get_device_properties(device).multi_processor_count
# SM100's paged MQA logits only take varlen rows.
indices = (
torch.arange(num_reqs, dtype=torch.int32, device=device)
if mqa_logits_use_mx_scales()
else None
)
schedule_metadata = indexer_module.get_paged_mqa_logits_metadata(
torch.div(seq_lens, 16, rounding_mode="floor").unsqueeze(1),
64,
num_sms,
indices=indices,
).clone()
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
builder.dcp_world_size = 1
builder.compress_ratio = 16
builder.kernel_block_size = None
builder.num_sms = num_sms
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
builder.kv_cache_spec = SimpleNamespace(num_states=64)
@@ -140,6 +147,7 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
decode_lens=torch.zeros(num_reqs, dtype=torch.int32, device=device),
requires_padding=False,
schedule_metadata=schedule_metadata,
indices=indices,
),
block_table=block_table,
)
@@ -158,7 +166,7 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
torch.testing.assert_close(
schedule_metadata,
indexer_module.get_paged_mqa_logits_metadata(
expected_decode_seq_lens.unsqueeze(1), 64, num_sms
expected_decode_seq_lens.unsqueeze(1), 64, num_sms, indices=indices
),
)
assert metadata.decode is not None
@@ -170,30 +178,24 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize(
"block_size,kernel_block_size,compress_ratio", [(64, 64, 1), (256, 64, 4)]
)
def test_indexer_draft_decode_refresh_follows_new_batch(
monkeypatch, block_size, kernel_block_size, compress_ratio
):
@pytest.mark.parametrize("compress_ratio", [1, 4])
def test_indexer_draft_decode_refresh_follows_new_batch(monkeypatch, compress_ratio):
"""Without a prior build(), the refresh must not keep the last batch's state."""
monkeypatch.setattr(indexer_module, "has_deep_gemm", lambda: False)
device = torch.device("cuda")
num_reqs, width = 3, 8
factor = block_size // kernel_block_size
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
builder.dcp_world_size = 1
builder.compress_ratio = compress_ratio
builder.kernel_block_size = kernel_block_size
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
builder.kv_cache_spec = SimpleNamespace(block_size=block_size, num_states=64)
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
def stale(*shape: int) -> torch.Tensor:
return torch.full(shape, -7, dtype=torch.int32, device=device)
decode = DeepSeekV32IndexerDecodeMetadata(
block_table=stale(num_reqs, width // factor),
block_table=stale(num_reqs, width),
seq_lens=stale(num_reqs, 1),
decode_lens=stale(num_reqs),
requires_padding=False,
@@ -219,8 +221,7 @@ def test_indexer_draft_decode_refresh_follows_new_batch(
builder.update_draft_decode_metadata(metadata)
expected_block_table = request_block_table[:, ::factor] // factor
torch.testing.assert_close(decode.block_table, expected_block_table)
torch.testing.assert_close(decode.block_table, request_block_table)
torch.testing.assert_close(decode.indices, builder.arange_buffer[:num_reqs])
assert torch.all(decode.per_req_decode_lens == 1)
assert torch.all(decode.decode_lens == 1)
@@ -471,7 +472,6 @@ def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypat
builder.pcp_rank = 0
builder.dcp_world_size = 1
builder.cp_kv_cache_interleave_size = 1
builder.kernel_block_size = None
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64)
@@ -0,0 +1,213 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""TP query-row sharding of the DSA indexer prefill (see PrefillRowShard):
each rank scores a contiguous slice; the group exchanges int32s per row."""
from types import SimpleNamespace
import pytest
import torch
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
import vllm.v1.attention.backends.mla.indexer as indexer
from vllm.config import CUDAGraphMode
from vllm.model_executor.layers.sparse_attn_indexer import PrefillRowShard
def _chunk(start: int, end: int) -> SimpleNamespace:
"""Chunk over rows [start, end); bounds derive from the global row."""
n = end - start
return SimpleNamespace(
token_start=start,
token_end=end,
cu_seqlen_ks=torch.zeros(n, dtype=torch.int32),
cu_seqlen_ke=((torch.arange(n) * 7 + start) % 65).int(),
)
@pytest.mark.parametrize("world", [2, 3, 8])
def test_shard_narrowing_covers_every_row_once(world: int) -> None:
"""Narrowed windows tile the prefill rows exactly once, carry the causal
bounds of those global rows, and skip chunks foreign to this rank."""
decode, rows = 5, 997
edges = [decode, decode + 330, decode + 337, decode + rows]
chunks = [_chunk(lo, hi) for lo, hi in zip(edges, edges[1:])]
sizes = [rows // world + int(r < rows % world) for r in range(world)]
covered = []
for rank in range(world):
shard = PrefillRowShard(decode, sizes, rank)
for chunk in chunks:
window = shard.narrow(chunk)
if window is None:
continue
start, end, ks, ke = window
lo, hi = start - chunk.token_start, end - chunk.token_start
torch.testing.assert_close(ks, chunk.cu_seqlen_ks[lo:hi])
torch.testing.assert_close(ke, chunk.cu_seqlen_ke[lo:hi])
covered.append((start, end))
covered.sort()
# Windows must be disjoint, contiguous and exactly cover the prefill rows.
assert [s for s, _ in covered] == [decode, *[e for _, e in covered][:-1]]
assert covered[-1][1] == decode + rows
def test_shard_exchange_reassembles_rows_only(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Gathered indices land only on real prefill rows: leading decode rows
and trailing CUDA-graph padding stay untouched, and the full k-pool
width (tail columns included) is exchanged."""
decode, pad, topk, kpool = 5, 3, 8, 4
width = topk + kpool - 1
sizes = [17, 9, 21]
buffer = torch.full((decode + sum(sizes) + pad, width), -7, dtype=torch.int32)
gathered = torch.arange(sum(sizes) * width, dtype=torch.int32).reshape(-1, width)
calls = []
def fake_group():
def all_gatherv(local, dim, sizes):
calls.append((local, sizes))
return gathered
return SimpleNamespace(all_gatherv=all_gatherv)
monkeypatch.setattr(sparse_indexer, "get_tp_group", fake_group)
for rank in range(len(sizes)):
PrefillRowShard(decode, sizes, rank).exchange_topk(buffer, width)
assert calls[0][1] == sizes and calls[0][0].is_contiguous()
torch.testing.assert_close(buffer[decode : decode + sum(sizes)], gathered)
assert torch.all(buffer[:decode] == -7), "decode rows stay outside"
assert torch.all(buffer[decode + sum(sizes) :] == -7), "padding untouched"
@pytest.mark.parametrize("tp_size", [2, 4])
def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
"""The split covers every row once with >= 1 row per rank and declines
below the activation floor (or on a single rank)."""
n = indexer.MIN_TP_SHARD_ROWS_PER_RANK * tp_size
shapes = [
([4 * n], [4 * n]), # fresh prompt: the full causal cost ramp
# uneven requests at different context depths
([2 * n, 7, 4 * n, n // 3 + 5], [2 * n, 9007, 4 * n + 500, n // 3 + 60005]),
]
ratio = 4
for query_lens, seq_lens in shapes:
sizes = indexer.balanced_prefill_row_shard(
torch.tensor(seq_lens, dtype=torch.int32),
torch.tensor(query_lens, dtype=torch.int32),
ratio,
tp_size,
)
assert sizes is not None and len(sizes) == tp_size
assert sum(sizes) == sum(query_lens) and min(sizes) >= 1
lens = torch.tensor([n - 1], dtype=torch.int32)
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size) is None
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1) is None
def test_row_sharding_gate_rejects_unsupported_configurations(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The gate is the whole safety envelope; nothing else guards the exchange."""
monkeypatch.setattr(indexer.current_platform, "is_cuda", lambda: True)
def supported(**kwargs):
mode = kwargs.pop("cudagraph_mode", CUDAGraphMode.PIECEWISE)
cfg = SimpleNamespace(compilation_config=SimpleNamespace(cudagraph_mode=mode))
args = {"dcp_world_size": 1, "use_pcp": False, "tp_size": 4, **kwargs}
return indexer.tp_prefill_row_sharding_supported(cfg, **args)
assert supported()
for bad in ({"tp_size": 1}, {"dcp_world_size": 2}, {"use_pcp": True}):
assert not supported(**bad)
for name in (
"VLLM_DISABLE_PYNCCL",
"VLLM_USE_NCCL_SYMM_MEM",
"VLLM_BATCH_INVARIANT",
):
with monkeypatch.context() as m:
m.setattr(indexer.envs, name, True)
assert not supported()
# FULL captures mixed batches whole, so the exchange must stay out.
assert not supported(cudagraph_mode=CUDAGraphMode.FULL)
assert supported(cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE)
def test_profiling_run_reserves_exchange_transient(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""The dummy run returns before the exchange executes, so its all_gatherv
output plus staging copy must be reserved up front when row sharding is
active -- and only then."""
rows, topk = 128, 8
buffer = torch.empty((rows, topk), dtype=torch.int32)
peak = 2 * rows * topk * buffer.element_size()
specs = (((1,), torch.uint8), ((1,), torch.uint8))
numels: list[int] = []
real_empty = torch.empty
def recording_empty(*args, **kwargs):
out = real_empty(*args, **kwargs)
numels.append(out.numel())
return out
def run_dummy(tp_world: int) -> None:
numels.clear()
with monkeypatch.context() as m:
m.setattr(torch, "empty", recording_empty)
m.setattr(
sparse_indexer,
"get_forward_context",
lambda: SimpleNamespace(attn_metadata=None),
)
m.setattr(sparse_indexer, "_gather_workspace_shapes", lambda *a: specs)
m.setattr(
sparse_indexer,
"current_workspace_manager",
lambda: SimpleNamespace(get_simultaneous=lambda *s: None),
)
m.setattr(
sparse_indexer,
"get_tp_group",
lambda: SimpleNamespace(world_size=tp_world),
)
m.setattr(
sparse_indexer,
"get_current_vllm_config_or_none",
lambda: SimpleNamespace(
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.PIECEWISE
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=1,
prefill_context_parallel_size=1,
),
),
)
m.setattr(sparse_indexer.envs, "VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", 1)
m.setattr(indexer.current_platform, "is_cuda", lambda: True)
sparse_indexer.sparse_attn_indexer(
hidden_states=torch.empty(4, 4),
k_cache_prefix="k",
kv_cache=torch.empty(4, 4),
q_quant=torch.empty(4, 4),
q_scale=None,
k=None,
weights=torch.empty(4),
quant_block_size=128,
scale_fmt=None,
topk_tokens=topk,
head_dim=64,
max_model_len=1024,
total_seq_lens=512,
topk_indices_buffer=buffer,
skip_k_cache_insert=True,
use_pcp=False,
pcp_shard_decode_requests=False,
dense_mha_metadata_layer_name="",
)
run_dummy(tp_world=2)
assert peak in numels
run_dummy(tp_world=1)
assert peak not in numels
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU tests for storage-block kernel-block selection in prepare_kernel_block_sizes."""
"""CPU tests for the kpool indexer's kernel block selection."""
from types import SimpleNamespace
@@ -11,20 +11,14 @@ from vllm.config import VllmConfig, set_current_vllm_config
from vllm.models.glm5next.common.attention import Glm5NextIndexerCache
from vllm.v1.attention.backend import AttentionBackend, MultipleOf
from vllm.v1.attention.backends.mla import indexer as indexer_mod
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerBackend,
KpoolTailBackend,
)
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
)
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.utils import prepare_kernel_block_sizes
from vllm.v1.worker.utils import prepare_kernel_block_sizes, select_common_block_size
pytestmark = pytest.mark.cpu_test
# ``index_kpool`` of zai-org/GLM-5.3-Flash: one indexer state pools 4
# compressed states (hence storage_block_size = page * 4).
# compressed states (hence a kernel block is page * 4 tokens).
INDEX_KPOOL = 4
# The hybrid KDA/mamba manager floors at TP8..TP1; 640 is the geometry of the
@@ -59,14 +53,17 @@ def _prepare(
spec: MLAAttentionSpec, backends: list[type[AttentionBackend]]
) -> list[int]:
kv_cache_config = SimpleNamespace(
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec)]
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec, layer_names=[])],
kv_cache_tensors=[],
)
attn_groups = [[SimpleNamespace(backend=backend) for backend in backends]]
attn_groups = [
[SimpleNamespace(backend=backend, kv_cache_spec=spec) for backend in backends]
]
return prepare_kernel_block_sizes(kv_cache_config, attn_groups)
@pytest.mark.parametrize("manager_block", TP_FLOOR_MANAGER_BLOCKS)
def test_prepare_uses_storage_block_for_the_kpool_group(
def test_kpool_group_maps_pool_pages(
monkeypatch: pytest.MonkeyPatch, manager_block: int
):
"""The bug: on ROCm, kpool groups must get 128/256 - not the 640-4352
@@ -78,54 +75,43 @@ def test_prepare_uses_storage_block_for_the_kpool_group(
"""
_mock_rocm_platform(monkeypatch)
spec = _kpool_storage_spec(manager_block)
expected_storage = 256 if manager_block % 256 == 0 else 128
assert spec.storage_block_size == expected_storage
expected = 256 if manager_block % 256 == 0 else 128
selected = _prepare(
spec,
[ROCMAiterMLASparseBackend, DeepseekV32IndexerBackend, KpoolTailBackend],
)
assert selected == [expected_storage]
# The kpool group's AttentionGroup selects its kernel block this way.
selected = [
select_common_block_size(manager_block, [Glm5NextIndexerBackend], [spec])
]
assert selected == [expected]
assert selected != [manager_block]
# The hybrid block-table split stays integral.
assert manager_block % selected[0] == 0
def test_prepare_falls_back_when_storage_block_is_not_supported():
"""No storage block, or a storage block the backends reject (e.g. the
CUDA indexer's exact ``[64]`` vs a 128-token storage block): selection is
`select_common_block_size`'s answer, unchanged from no-fix behavior.
"""
def test_prepare_uses_the_backend_vote():
"""Unpacked groups take `select_common_block_size`'s answer."""
class Fixed64Backend:
@staticmethod
def get_supported_kernel_block_sizes():
def get_supported_kernel_block_sizes(kv_cache_spec=None):
return [64]
@staticmethod
def get_name() -> str:
return "FIXED64"
# Storage block the group's backend does not accept -> backend vote (64).
spec = _kpool_storage_spec(640)
assert spec.storage_block_size == 128
assert _prepare(spec, [Fixed64Backend]) == [64]
# No storage block at all (any non-kpool MLA model) -> backend vote.
plain_spec = MLAAttentionSpec(
block_size=640,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
)
assert plain_spec.storage_block_size is None
assert _prepare(plain_spec, [Fixed64Backend]) == [64]
# ...and with backends that accept the manager block, the manager wins,
# exactly as on main.
class AcceptAllBackend:
@staticmethod
def get_supported_kernel_block_sizes():
def get_supported_kernel_block_sizes(kv_cache_spec=None):
return [MultipleOf(1)]
@staticmethod
@@ -3,7 +3,7 @@
"""CPU tests for the kpool tail slot mapping (no GPU required).
The kpool tail cache is a 1-block-per-request circular ring addressed by
``pos % kpool`` (``KpoolTailSpec`` / ``KpoolTailManager``: exactly one block
``pos % kpool`` (``CircularBufferSpec`` / ``CircularBufferManager``: exactly one block
allocated per request, never grown, so only column 0 of its block table is
ever written; the rest stays zero-initialized).
@@ -32,27 +32,25 @@ from vllm.v1.attention.backends.mla.indexer import (
KpoolTailMetadataBuilder,
compute_kpool_tail_slot_mapping,
)
from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides
from vllm.v1.kv_cache_interface import CircularBufferSpec, compute_layout_strides
from vllm.v1.kv_cache_layout import KVCacheLayout
from vllm.v1.worker.block_table import get_block_table_width
KPOOL = 4
def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
(layout,) = KpoolTailBackend.supported_kv_cache_layouts()
spec = KpoolTailSpec(
spec = CircularBufferSpec(
block_size=KPOOL,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=KPOOL,
)
strides = compute_layout_strides(spec, num_blocks=8, num_layers=3, layout=layout)
_, _, head_stride, state_stride, content_stride = strides
assert layout is KVCacheLayout.LBHNC
assert layout is KVCacheLayout.BLHNC
assert head_stride == KPOOL * 128 * torch.bfloat16.itemsize
assert state_stride == 128 * torch.bfloat16.itemsize
assert content_stride == 1
@@ -77,37 +75,14 @@ def test_tail_ring_divides_the_attention_block(num_speculative_tokens, ring):
SimpleNamespace(num_speculative_tokens=num_speculative_tokens)
)
assert spec.block_size == spec.sliding_window == ring
assert isinstance(spec, CircularBufferSpec) and spec.block_size == ring
assert ring >= KPOOL + num_speculative_tokens
assert 640 % ring == 0
def test_tail_spec_opts_out_of_generic_slot_mapping():
"""The tail row is one block wide (padded to the block-table alignment), so
the generic kernel's ``pos // kpool`` column index runs off the end of the
allocation for long prompts. The spec must opt out of it entirely."""
spec = KpoolTailSpec(
block_size=KPOOL,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=KPOOL,
)
max_len = 1 << 20
width = get_block_table_width(
spec.max_num_blocks_per_req(None, max_len),
spec.block_size,
token_alignment=spec.block_table_token_alignment,
)
assert width * KPOOL < max_len
assert spec.uses_slot_mapping is False
def make_tail_block_table(own_blocks, width=64):
"""Tail-group block table as BlockTables produces it: column 0 holds the
request's single KpoolTailManager block, the remaining columns are never
request's single CircularBufferManager block, the remaining columns are never
written and stay zero."""
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
@@ -447,6 +422,7 @@ def test_triton_mapping_matches_cpu(per_req, own_blocks, num_actual, padded_len)
tokens between the last request boundary and num_actual_tokens (mapped to
the last request), untouched padding beyond num_actual, and PAD for
requests on the null block."""
ring_size = 2 * KPOOL
positions, qsl, slot_mapping, _, num_reqs = make_batch(
per_req, padded_len=padded_len
)
@@ -455,14 +431,17 @@ def test_triton_mapping_matches_cpu(per_req, own_blocks, num_actual, padded_len)
slot_mapping = torch.arange(padded_len, dtype=torch.int64) + 1000
bt = make_tail_block_table(own_blocks)
ref = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
got = circular_tail_slots(
ref = compute_kpool_tail_slot_mapping(
slot_mapping, bt, qsl, positions, num_actual, num_reqs, ring_size
)
got = compute_kpool_tail_slot_mapping(
slot_mapping.cuda(),
bt.cuda(),
qsl.cuda().to(torch.int32),
positions.cuda(),
num_actual,
num_reqs,
ring_size,
)
torch.testing.assert_close(got.cpu(), ref)
for req, blk in enumerate(own_blocks):
+43 -23
View File
@@ -358,6 +358,8 @@ if current_platform.is_cuda():
)
elif current_platform.is_rocm():
PREFILL_BACKENDS_TO_TEST.append(MLAPrefillBackendEnum.ROCM_AITER_FA)
elif current_platform.is_xpu():
PREFILL_BACKENDS_TO_TEST.append(MLAPrefillBackendEnum.FLASH_ATTN)
MLA_DIMENSIONS_TO_TEST = [
("deepseek", 128, 128),
@@ -370,25 +372,20 @@ def _prefill_backend_dimension_params():
params = []
for prefill_backend in PREFILL_BACKENDS_TO_TEST:
for dimensions_id, qk_nope_head_dim, v_head_dim in MLA_DIMENSIONS_TO_TEST:
if device_capability is None:
invalid_reasons = ["device capability unavailable"]
else:
try:
invalid_reasons = (
prefill_backend.get_class().validate_configuration(
device_capability,
MLAPrefillSelectorConfig(
dtype=torch.bfloat16,
mla_dimensions=MLADimensions(
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=64,
v_head_dim=v_head_dim,
),
),
)
)
except ImportError:
invalid_reasons = ["ImportError"]
try:
invalid_reasons = prefill_backend.get_class().validate_configuration(
device_capability,
MLAPrefillSelectorConfig(
dtype=torch.bfloat16,
mla_dimensions=MLADimensions(
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=64,
v_head_dim=v_head_dim,
),
),
)
except ImportError:
invalid_reasons = ["ImportError"]
marks = []
if invalid_reasons:
@@ -1098,6 +1095,7 @@ def test_flashinfer_mla_dspark_dcp_supports_target_and_draft(monkeypatch):
],
)
@pytest.mark.parametrize("cp_interleave_size", [1, 16, 896])
@pytest.mark.parametrize("min_split_kv", [1, 8])
def test_tokenspeed_mla_decode_contract(
monkeypatch,
causal,
@@ -1105,6 +1103,7 @@ def test_tokenspeed_mla_decode_contract(
dcp_world_size,
dcp_rank,
cp_interleave_size,
min_split_kv,
):
decode_call = None
num_decodes = 2
@@ -1148,7 +1147,12 @@ def test_tokenspeed_mla_decode_contract(
impl.scale = 1.0
impl.softmax_scale = 1.0
impl.output_scale = 1.0
impl._workspace_buffer = torch.empty(1, dtype=torch.int8)
impl._min_split_kv = min_split_kv
impl._max_decode_tokens = num_decode_tokens
workspace = torch.empty(1, dtype=torch.int8)
monkeypatch.setattr(
tokenspeed_mla_module, "_get_workspace", lambda *args: workspace
)
metadata = SimpleNamespace(
num_decodes=num_decodes,
@@ -1183,6 +1187,8 @@ def test_tokenspeed_mla_decode_contract(
assert lse.shape == (num_decode_tokens, num_heads)
assert decode_call is not None
assert decode_call["workspace_buffer"] is workspace
assert decode_call["min_split_kv"] == min_split_kv
assert decode_call["query"].shape == (
num_decodes,
tokens_per_decode,
@@ -1217,16 +1223,23 @@ def test_tokenspeed_mla_decode_contract(
pytest.param(896, 8 * 896 + 17, id="block-interleave-full-cycle"),
],
)
def test_tokenspeed_mla_dcp_matches_unsharded_decode(cp_interleave_size, seq_len):
def test_tokenspeed_mla_dcp_matches_unsharded_decode(
monkeypatch, cp_interleave_size, seq_len
):
from tokenspeed_mla import tokenspeed_mla_decode
from vllm.v1.attention.ops.dcp import (
_lse_weighted_combine,
mask_dcp_empty_shards_,
)
from vllm.v1.worker.workspace import WorkspaceManager
torch.manual_seed(7)
device = torch.device("cuda")
workspace_manager = WorkspaceManager(device)
monkeypatch.setattr(
tokenspeed_mla_module, "current_workspace_manager", lambda: workspace_manager
)
dcp_world_size = 8
kernel_block_size = 64
num_heads = 128
@@ -1257,7 +1270,9 @@ def test_tokenspeed_mla_dcp_matches_unsharded_decode(cp_interleave_size, seq_len
global_tokens = (torch.randn(seq_len, head_size, device=device) * 0.1).to(
torch.float8_e4m3fn
)
workspace = tokenspeed_mla_module._get_workspace(device, num_heads, kv_lora_rank)
workspace = tokenspeed_mla_module._get_workspace(
device, num_heads, kv_lora_rank, 1, query.shape[1]
)
global_cache, global_block_table = make_paged_cache(global_tokens)
global_seq_len = torch.tensor([seq_len], dtype=torch.int32, device=device)
reference, _ = tokenspeed_mla_decode(
@@ -1301,7 +1316,8 @@ def test_tokenspeed_mla_dcp_matches_unsharded_decode(cp_interleave_size, seq_len
impl.scale = head_size**-0.5
impl.softmax_scale = None
impl.output_scale = None
impl._workspace_buffer = workspace
impl._min_split_kv = 1
impl._max_decode_tokens = query.shape[1]
metadata = SimpleNamespace(
num_decodes=1,
num_decode_tokens=query.shape[1],
@@ -1649,6 +1665,10 @@ def _run_backend_correctness(
block_size=default_block_size,
hf_config_override=hf_config_override,
)
if hf_config_override and current_platform.is_xpu():
vllm_config.model_config.model_arch_config.total_num_attention_heads = (
hf_config_override["num_attention_heads"]
)
vllm_config.cache_config.cache_dtype = kv_cache_dtype
# For spec decode tests, add a speculative_config to set the reorder_batch_threshold
@@ -2,14 +2,26 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for MLA prefill backend registry."""
import gc
from unittest.mock import MagicMock
import pytest
import torch
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
from vllm.v1.attention.backends.mla.prefill.flashinfer import FlashInferPrefillBackend
from vllm.v1.attention.backends.mla.prefill.registry import (
MLAPrefillBackendEnum,
register_mla_prefill_backend,
)
from vllm.v1.attention.backends.mla.prefill.trtllm_ragged import (
TrtllmRaggedPrefillBackend,
)
from vllm.v1.worker.workspace import (
current_workspace_manager,
init_workspace_manager,
reset_workspace_manager,
)
class CustomMLAPrefillBackend(MLAPrefillBackend):
@@ -50,6 +62,38 @@ def test_prefill_backend_clone_has_isolated_metadata():
assert clone._prefill_metadata is not backend._prefill_metadata
@pytest.mark.parametrize(
"backend_cls", [TrtllmRaggedPrefillBackend, FlashInferPrefillBackend]
)
def test_prefill_backend_does_not_pin_grown_workspace(monkeypatch, backend_cls):
"""Reserving workspace at init must not keep a view of it: when another user
grows the shared workspace, the old buffer must be freed."""
monkeypatch.setenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", "4096")
init_workspace_manager(torch.device("cpu"))
try:
manager = current_workspace_manager()
backend = backend_cls(
num_heads=4,
scale=0.5,
kv_lora_rank=8,
qk_nope_head_dim=16,
qk_rope_head_dim=8,
v_head_dim=32,
vllm_config=MagicMock(),
)
(view,) = manager.get_simultaneous(((1,), torch.uint8))
old_storage = view.untyped_storage()._weak_ref()
del view
manager.get_simultaneous(((1 << 20,), torch.uint8))
gc.collect()
assert torch.UntypedStorage._expired(old_storage)
del backend
finally:
reset_workspace_manager()
@pytest.fixture(autouse=True)
def cleanup_overrides():
"""Clear any overrides after each test."""
-1
View File
@@ -805,7 +805,6 @@ def test_composite_shared_cache_across_image_and_causal_steps(
)
layer.mm_prefix_clamp_sliding_window = True
builder = backend.get_builder_cls()(spec, ["composite_test"], cfg, DEVICE)
builder.set_kernel_block_size(block_size)
# Match production, where unwritten page tails are zero-initialized.
cache = torch.zeros(
num_blocks,
@@ -142,6 +142,13 @@ def _build(monkeypatch, route, gathered_heads, segmented_supported=True):
def init_common_builder(self, *args, **kwargs):
self.num_heads = gathered_heads // DCP
self.q_data_type = torch.bfloat16
self.mla_dims = SimpleNamespace(
kv_lora_rank=512,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
)
self.dcp_world_size = DCP
self.reorder_batch_threshold = 5
@@ -154,6 +161,11 @@ def _build(monkeypatch, route, gathered_heads, segmented_supported=True):
lambda *args: segmented_supported,
)
monkeypatch.setattr(rocm_aiter_mla, "_fp8_mla_prefill_supported", lambda: False)
# These cases pick between cprr and segmented. On a gfx942 runner the real
# arch would add the Triton verify route, which this stub cannot build.
import vllm.platforms.rocm as rocm
monkeypatch.setattr(rocm, "on_gfx942", lambda: False)
monkeypatch.setattr(
rocm_aiter_mla, "get_dcp_group", lambda: SimpleNamespace(rank_in_group=0)
)
@@ -257,25 +269,46 @@ def test_min_cprr_qlen_is_above_two():
@pytest.mark.parametrize(
"supports,causal,qlen,asm,expected",
"segmented,triton,causal,qlen,asm,expected",
[
(True, True, 1, True, Route.PLAIN),
(True, True, 2, True, Route.SEGMENTED),
(True, True, 3, True, Route.CPRR),
(True, True, 4, True, Route.CPRR),
(True, True, 4, False, Route.SEGMENTED),
(True, False, 2, True, Route.PLAIN),
(True, False, 4, False, Route.PLAIN),
(False, True, 3, True, Route.CPRR),
(True, False, True, 1, True, Route.PLAIN),
(True, False, True, 2, True, Route.SEGMENTED),
(True, False, True, 3, True, Route.CPRR),
(True, False, True, 4, True, Route.CPRR),
(True, False, True, 4, False, Route.SEGMENTED),
(True, False, False, 2, True, Route.PLAIN),
(True, False, False, 4, False, Route.PLAIN),
(False, False, True, 3, True, Route.CPRR),
# gfx942: no cprr and no segmented, so causal verify goes to Triton.
(False, True, True, 1, False, Route.PLAIN),
(False, True, True, 2, False, Route.TRITON),
(False, True, True, 4, False, Route.TRITON),
(False, True, False, 4, False, Route.PLAIN),
# cprr still wins at or above its qlen floor.
(False, True, True, 4, True, Route.CPRR),
(False, True, True, 2, True, Route.TRITON),
],
)
def test_dcp_decode_route(supports, causal, qlen, asm, expected):
assert _select_route(supports, causal, qlen, asm) is expected
def test_dcp_decode_route(segmented, triton, causal, qlen, asm, expected):
route = _select_route(
supports_segmented=segmented,
supports_triton=triton,
causal=causal,
max_qo_len=qlen,
asm_selected=asm,
)
assert route is expected
def test_causal_multi_token_batch_without_a_valid_route_fails():
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
_select_route(False, True, 2, True)
_select_route(
supports_segmented=False,
supports_triton=False,
causal=True,
max_qo_len=2,
asm_selected=True,
)
# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs
@@ -536,5 +536,12 @@ def test_cprr_kernel_is_selected_for_the_k3_shape():
is True
)
assert QLEN >= m._MIN_CPRR_QLEN
assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR
route = m._select_dcp_decode_route(
supports_segmented=True,
supports_triton=False,
causal=True,
max_qo_len=QLEN,
asm_selected=True,
)
assert route is m._DCPDecodeRoute.CPRR
assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS
@@ -0,0 +1,331 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Causal target verify under DCP on gfx942 (MI300/MI325).
Reproduces the attention shape of
vllm serve moonshotai/Kimi-K3 --tensor-parallel-size 8 \
--decode-context-parallel-size 8 \
--speculative-config '{"method": "dspark", "num_speculative_tokens": 5}'
96 gathered heads, bf16 KV, qlen 6 and the 768-token blocks the hybrid KDA
layers impose. Segmented MLA tiles those blocks at 128 tokens and needs 128 KiB
of LDS (gfx942 has 64 KiB), so the builder must route this shape to the generic
Triton split-KV kernel. DeepSeek-V3 with MTP and an fp8 KV cache at DCP 8 hits
the same limit through its 128 gathered heads. Shapes segmented MLA fits stay
on it.
The real metadata builder and ``forward_mqa`` run on one GPU with the DCP ranks
simulated one after another. That is sound because this path has no
collective: the query all-gather happens in the layer, before ``forward_mqa``.
"""
from types import SimpleNamespace
import pytest
import torch
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
if not current_platform.is_rocm():
pytest.skip("ROCm AITER MLA tests", allow_module_level=True)
from vllm._aiter_ops import is_aiter_found # noqa: E402
# DeepSeek-R1 supplies the MLA dims, which Kimi-K3 and DeepSeek-V3 share; only
# the head count is overridden.
MODEL = "deepseek-ai/DeepSeek-R1"
KV_LORA_RANK, ROPE_DIM, NOPE_DIM = 512, 64, 128
HEAD_SIZE = KV_LORA_RANK + ROPE_DIM
SCALE = (NOPE_DIM + ROPE_DIM) ** -0.5
DCP = 8
# Each rank holds ~800 tokens per request, so its shard spans several blocks.
NUM_REQS, CTX = 4, 6400
FP8_KV_SCALE = 0.05
def _on_gfx942() -> bool:
if not (is_aiter_found() and torch.accelerator.is_available()):
return False
from vllm.platforms.rocm import on_gfx942
return on_gfx942()
pytestmark = pytest.mark.skipif(not _on_gfx942(), reason="needs ROCm + AITER on gfx942")
class _FakeGroup:
def __init__(self, world_size: int, rank: int):
self.world_size = world_size
self.rank_in_group = self.rank = self.local_rank = rank
def all_gather(self, tensor, dim=0):
raise RuntimeError("DCP verify must not issue a collective")
@pytest.fixture
def fake_dcp_groups(monkeypatch):
"""Point every importer of get_dcp_group at a settable rank stub."""
import vllm.distributed.parallel_state as ps
import vllm.model_executor.layers.attention.mla_attention as mla_attention
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
group = _FakeGroup(DCP, 0)
for module in (ps, mla_attention, backend):
monkeypatch.setattr(module, "get_dcp_group", lambda: group)
monkeypatch.setattr(ps, "get_tp_group", lambda: _FakeGroup(1, 0))
return group
def _vllm_config(heads_per_rank: int, qlen: int, fp8_kv: bool):
from tests.v1.attention.utils import create_vllm_config
config = create_vllm_config(
model_name=MODEL,
max_model_len=CTX + qlen,
max_num_seqs=NUM_REQS,
max_num_batched_tokens=4096,
)
config.model_config.model_arch_config.total_num_attention_heads = heads_per_rank
# The reorder threshold reads only these two fields; without them a
# multi-token row is classified as prefill and never reaches decode.
config.speculative_config = SimpleNamespace(
num_speculative_tokens=qlen - 1, parallel_drafting=False
)
config.parallel_config.decode_context_parallel_size = DCP
config.parallel_config.cp_kv_cache_interleave_size = 1
if fp8_kv:
config.cache_config.cache_dtype = "fp8"
return config
def _ctx_layer():
"""The registered-layer fields MLACommonMetadataBuilder.__init__ reads."""
from vllm.v1.attention.ops.dcp import MLADCPManager
dcp_manager = object.__new__(MLADCPManager)
dcp_manager.init_kv_gather = lambda *args, **kwargs: None
# Only chunked-context prefill uses the cloned backend.
prefill_backend = SimpleNamespace()
prefill_backend.clone = lambda: prefill_backend
return SimpleNamespace(
non_causal_multi_token_decode=False,
q_lora_rank=None,
kv_lora_rank=KV_LORA_RANK,
qk_nope_head_dim=NOPE_DIM,
qk_rope_head_dim=ROPE_DIM,
v_head_dim=NOPE_DIM,
dcp_manager=dcp_manager,
prefill_backend=prefill_backend,
)
def _impl(dcp_rank: int, heads_per_rank: int, fp8_kv: bool):
"""forward_mqa needs these attributes, not the weight-loading __init__."""
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
impl = object.__new__(AiterMLAImpl)
impl.num_heads = heads_per_rank
impl.kv_lora_rank = KV_LORA_RANK
impl.qk_rope_head_dim = ROPE_DIM
impl.dcp_world_size = DCP
impl.dcp_rank = dcp_rank
impl.pcp_world_size = 1
impl.kv_cache_dtype = "fp8" if fp8_kv else "auto"
impl.scale = SCALE
impl._sm_count = current_platform.num_compute_units()
return impl
def _run_rank(config, spec, layer_name, group, rank, q, kv_cache_src, k_scale):
"""Shard the KV round-robin onto ``rank`` and run its verify step."""
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
device = q.device
qlen, heads = q.shape[1], q.shape[2] // DCP
seq_len = CTX + qlen
positions = torch.arange(rank, seq_len, DCP, device=device)
local_len = positions.numel()
# Production block tables span max_model_len, so every rank's table is at
# least as wide as the largest shard needs.
blocks_per_req = cdiv(cdiv(seq_len, DCP), spec.block_size)
kv_cache = torch.zeros(
NUM_REQS * blocks_per_req,
spec.block_size,
HEAD_SIZE,
dtype=kv_cache_src.dtype,
device=device,
)
kv_cache.view(NUM_REQS, -1, HEAD_SIZE)[:, :local_len] = kv_cache_src[:, positions]
block_table = torch.arange(
NUM_REQS * blocks_per_req, dtype=torch.int32, device=device
).view(NUM_REQS, blocks_per_req)
query_start_loc = torch.arange(
0, NUM_REQS * qlen + 1, qlen, dtype=torch.int32, device=device
)
local_lens = torch.full((NUM_REQS,), local_len, dtype=torch.int32)
common = CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc.cpu(),
seq_lens=torch.full((NUM_REQS,), seq_len, dtype=torch.int32, device=device),
num_reqs=NUM_REQS,
num_actual_tokens=NUM_REQS * qlen,
max_query_len=qlen,
max_seq_len=seq_len,
block_table_tensor=block_table,
slot_mapping=torch.arange(NUM_REQS * qlen, dtype=torch.int64, device=device),
causal=True,
dcp_local_seq_lens=local_lens.to(device),
dcp_local_seq_lens_cpu_upper_bound=local_lens,
)
group.rank_in_group = group.rank = group.local_rank = rank
builder = AiterMLAMetadataBuilder(spec, [layer_name], config, device)
metadata = builder.build(0, common)
layer = SimpleNamespace(
_q_scale=torch.tensor(1.0, device=device),
_k_scale=torch.tensor(k_scale, device=device),
)
impl = _impl(rank, heads, kv_cache_src.dtype != torch.bfloat16)
output, lse = impl.forward_mqa(q.flatten(0, 1), kv_cache, metadata, layer)
return metadata.decode.dcp_route.name, output, lse
@pytest.mark.parametrize(
"heads_per_rank, qlen, block_size, fp8_kv, route",
[
(12, 6, 768, False, "TRITON"),
(12, 6, 64, False, "SEGMENTED"),
(16, 2, 16, True, "TRITON"),
],
ids=["kimi-k3-hybrid-kda-blocks", "segmented-fits", "deepseek-v3-mtp-fp8-kv"],
)
@torch.inference_mode()
def test_dcp_verify_matches_attention(
fake_dcp_groups, heads_per_rank, qlen, block_size, fp8_kv, route
):
from vllm.config import set_current_vllm_config
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.workspace import (
init_workspace_manager,
is_workspace_manager_initialized,
)
device = torch.device("cuda", torch.accelerator.current_device_index())
if not is_workspace_manager_initialized():
init_workspace_manager(device)
torch.manual_seed(0)
seq_len = CTX + qlen
q = torch.randn(
NUM_REQS,
qlen,
heads_per_rank * DCP,
HEAD_SIZE,
dtype=torch.bfloat16,
device=device,
)
kv = torch.randn(NUM_REQS, seq_len, HEAD_SIZE, device=device).clamp_(-4, 4)
if fp8_kv:
k_scale = FP8_KV_SCALE
kv_cache_src = (kv / k_scale).to(current_platform.fp8_dtype())
else:
k_scale = 1.0
kv_cache_src = kv.to(torch.bfloat16)
kv_ref = kv_cache_src.float() * k_scale
config = _vllm_config(heads_per_rank, qlen, fp8_kv)
layer_name = "model.layers.0.self_attn.attn"
with set_current_vllm_config(config):
config.compilation_config.static_forward_context[layer_name] = _ctx_layer()
spec = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=HEAD_SIZE,
dtype=kv_cache_src.dtype,
cache_dtype_str="fp8" if fp8_kv else None,
non_causal_multi_token_decode=False,
)
shards = [
_run_rank(
config,
spec,
layer_name,
fake_dcp_groups,
rank,
q,
kv_cache_src,
k_scale,
)
for rank in range(DCP)
]
assert {shard_route for shard_route, _, _ in shards} == {route}
lses = torch.stack([lse.float() for _, _, lse in shards])
weights = torch.exp(lses - lses.max(dim=0).values)
merged = (
weights.unsqueeze(-1) * torch.stack([out.float() for _, out, _ in shards])
).sum(dim=0) / weights.sum(dim=0).unsqueeze(-1)
# Query token t of a request sees the context plus verify tokens 0..t.
scores = torch.einsum("rthd,rld->rthl", q.float(), kv_ref) * SCALE
visible = torch.arange(seq_len, device=device) <= CTX + torch.arange(
qlen, device=device
).view(-1, 1)
scores.masked_fill_(~visible[None, :, None, :], float("-inf"))
reference = torch.einsum(
"rthl,rld->rthd", scores.softmax(dim=-1), kv_ref[..., :KV_LORA_RANK]
)
torch.testing.assert_close(
merged.view_as(reference), reference, rtol=2e-2, atol=2e-2
)
@pytest.mark.parametrize(
"fp8_kv, num_heads, block_size",
[
(False, 96, 768),
(False, 128, 64),
(True, 64, 128),
(True, 128, 16),
],
)
def test_segmented_lds_gate_matches_the_kernel(fp8_kv, num_heads, block_size):
"""The gate must flag exactly the shapes AITER segmented MLA cannot compile."""
from triton.runtime.errors import OutOfResources
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
_get_segmented_mla_decode,
_segmented_mla_fits_lds,
_segmented_mla_page_size,
)
page = _segmented_mla_page_size(block_size)
kv_dtype = current_platform.fp8_dtype() if fp8_kv else torch.bfloat16
q = torch.zeros(1, num_heads, HEAD_SIZE, dtype=torch.bfloat16, device="cuda")
kv = torch.zeros(1, page, 1, HEAD_SIZE, dtype=kv_dtype, device="cuda")
one_row = torch.tensor([0, 1], dtype=torch.int32, device="cuda")
try:
_get_segmented_mla_decode()(
q,
kv,
None,
one_row,
torch.tensor([page], dtype=torch.int32, device="cuda"),
page,
torch.zeros(1, 1, dtype=torch.int32, device="cuda"),
SCALE,
KV_LORA_RANK,
ROPE_DIM,
causal=True,
q_descale=None,
kv_descale=torch.tensor(1.0, device="cuda"),
skip_reduce=True,
)
compiles = True
except OutOfResources:
compiles = False
assert _segmented_mla_fits_lds(num_heads, block_size, fp8_kv) == compiles
@@ -20,6 +20,10 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( # noqa: E402
AiterMLAImpl,
AiterMLAMetadataBuilder,
)
from vllm.v1.attention.ops.dcp import ( # noqa: E402
CPTritonContext,
correct_attn_out,
)
from vllm.v1.attention.ops.rocm_aiter_mla_merge import ( # noqa: E402
merge_mla_segments_triton,
)
@@ -121,8 +125,11 @@ def _builder(
# segmented route is available, query length decides per batch. CPRR
# preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back.
_supports_segmented_dcp_verify=supports_segmented_dcp_verify,
_use_triton_dcp_verify=False,
# Derived once in the real constructor, so derive it once here too.
_segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size),
_dcp_verify_page_size=rocm_aiter_mla._segmented_mla_page_size(
kernel_block_size
),
_dcp_verify_buffers=None,
_graph_seq_lens=None,
_kv_cache_dtype_str=kv_cache_dtype,
@@ -630,6 +637,99 @@ def test_segmented_verify_reduce_returns_natural_lse_and_masks_empty_rows():
assert lse[1].item() == float("-inf")
def test_triton_verify_empty_local_shard_merges_through_dcp_combine():
"""Empty rows need no masking: -inf LSE gives them zero combine weight.
Row 0 has KV on both ranks, row 1 only on rank 0, row 2 on neither (graph
padding). The raw rows come back NaN; the production AG+RS combine must
still produce full-context attention and zeros for the padded row.
"""
torch.manual_seed(0)
num_ranks, num_rows, num_heads = 2, 3, 16
kv_lora_rank, head_dim, block_size = 512, 576, 16
scale = head_dim**-0.5
local_lens = [[5, 4, 0], [5, 0, 0]]
q = torch.randn(num_rows, num_heads, head_dim, dtype=torch.bfloat16, device="cuda")
kv_caches = [
torch.randn(1, block_size, head_dim, dtype=torch.bfloat16, device="cuda")
for _ in range(num_ranks)
]
impl = object.__new__(AiterMLAImpl)
impl.scale = scale
impl.kv_lora_rank = kv_lora_rank
impl._sm_count = current_platform.num_compute_units()
layer = SimpleNamespace(_k_scale=torch.tensor(1.0, device="cuda"))
outputs, lses = [], []
for rank in range(num_ranks):
verify = SimpleNamespace(
block_table=torch.zeros(num_rows, 1, dtype=torch.int32, device="cuda"),
row_lens=torch.tensor(local_lens[rank], dtype=torch.int32, device="cuda"),
max_kv_seq_len=max(local_lens[rank]),
)
output, lse = impl._forward_triton_dcp_verify(
q, verify, kv_caches[rank], layer, torch.bfloat16
)
outputs.append(output)
lses.append(lse)
assert torch.isneginf(lses[1][1]).all()
assert torch.isneginf(lses[0][2]).all() and torch.isneginf(lses[1][2]).all()
all_lses = torch.stack(lses)
merged = sum(
correct_attn_out(outputs[rank], all_lses, rank, CPTritonContext())[0].float()
for rank in range(num_ranks)
)
reference = torch.zeros(num_rows, num_heads, kv_lora_rank, device="cuda")
for row in range(num_rows):
keys = torch.cat(
[kv_caches[rank][0, : local_lens[rank][row]] for rank in range(num_ranks)]
).float()
if keys.shape[0]:
probs = torch.softmax(q[row].float() @ keys.T * scale, dim=-1)
reference[row] = probs @ keys[:, :kv_lora_rank]
assert torch.isfinite(merged).all()
torch.testing.assert_close(merged, reference, rtol=2e-2, atol=2e-2)
def test_batch_aware_kv_splits_fit_an_int32_safe_reservation(monkeypatch):
# MI325X DCP8 verify at 128K context: 128 gathered heads, 16K local tokens.
sm_count, heads, local_seq_len, kv_lora_rank = 304, 128, 16384, 512
max_rows = 4096
reserved = []
monkeypatch.setattr(
rocm_aiter_mla, "is_workspace_manager_initialized", lambda: True
)
monkeypatch.setattr(
rocm_aiter_mla,
"current_workspace_manager",
lambda: SimpleNamespace(get_simultaneous=lambda *specs: reserved.extend(specs)),
)
rocm_aiter_mla._reserve_triton_dcp_verify_workspace(
max_rows, heads, local_seq_len, kv_lora_rank, sm_count
)
((shape, _),) = reserved
# The kernel's attn_logits offsets are int32; seq-only splits overflow them.
seq_only = rocm_aiter_mla._compute_num_kv_splits(local_seq_len, sm_count)
assert max_rows * heads * seq_only * (kv_lora_rank + 1) >= 2**31
assert math.prod(shape) < 2**31
counts = [
rocm_aiter_mla._triton_dcp_verify_num_kv_splits(
local_seq_len, rows, heads, sm_count
)
for rows in range(1, max_rows + 1)
]
# Small batches keep the seq-only split count.
assert counts[:4] == [seq_only] * 4
assert counts == sorted(counts, reverse=True)
for rows, splits in enumerate(counts, start=1):
assert rows * heads * splits * (kv_lora_rank + 1) <= math.prod(shape)
def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
"""Exercise the target qlen>1 path and merge two rank-local results."""
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
@@ -740,26 +840,34 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
torch.testing.assert_close(output, reference, rtol=3e-2, atol=3e-2)
@pytest.mark.parametrize("num_heads", [8, 16, 24, 32, 64, 128])
@pytest.mark.parametrize(
"spec_method, parallel_drafting",
"num_heads,spec_method,parallel_drafting,group_cache_dtype,needs_row_map",
[
("deepseek_mtp", False),
# A drafter that is not one of the historically recognized MTP methods,
# and a parallel one, so the threshold is 1 + 2 * num_spec rather than
# 1 + num_spec. Sizing the metadata off a method name instead leaves
# these at qlen=1 while the router still admits the full range.
("custom", True),
("eagle", False),
*[
(num_heads, method, parallel, None, True)
for num_heads in [8, 16, 24, 32, 64, 128]
for method, parallel in [
("deepseek_mtp", False),
# Parallel drafting needs qlen=1+2*num_spec, regardless of name.
("custom", True),
("eagle", False),
]
],
pytest.param(16, "deepseek_mtp", False, "auto", False, id="bf16-group"),
pytest.param(16, "deepseek_mtp", False, "fp8_e5m2", False, id="e5m2-group"),
],
)
def test_mtp_builder_init_sizes_native_fp8_metadata(
monkeypatch, num_heads, spec_method, parallel_drafting
def test_builder_init_sizes_decode_metadata_and_filters_dcp_maps(
monkeypatch,
num_heads,
spec_method,
parallel_drafting,
group_cache_dtype,
needs_row_map,
):
"""Aiter init sizes the metadata for every query length decode can be handed.
"""Size decode metadata for padded heads/qlen; skip unsupported DCP maps.
Sweeping num_heads asserts metadata is sized for the padded decode shape,
covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding.
The group's raw cache format must override the global FP8 configuration.
"""
dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16")
info_calls = []
@@ -789,6 +897,13 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
def init_common_builder(self, *args, **kwargs):
self.num_heads = num_heads
self.q_data_type = config.model_config.dtype
self.mla_dims = SimpleNamespace(
kv_lora_rank=512,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
)
self.dcp_world_size = 1
self.cp_kv_cache_interleave_size = 1
# Mirror what _init_reorder_batch_threshold would have left behind: the
@@ -813,6 +928,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
init_common_builder,
)
monkeypatch.setattr(rocm_aiter_mla, "_fp8_mla_prefill_supported", lambda: False)
monkeypatch.setattr(rocm_aiter_mla, "is_aiter_found_and_supported", lambda: True)
config = SimpleNamespace(
speculative_config=SimpleNamespace(
@@ -840,7 +956,9 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
),
)
builder = AiterMLAMetadataBuilder(
kv_cache_spec=SimpleNamespace(block_size=1, dtype=torch.bfloat16),
kv_cache_spec=SimpleNamespace(
block_size=1, dtype=torch.bfloat16, cache_dtype_str=group_cache_dtype
),
layer_names=["layer.0"],
vllm_config=config,
device=torch.device("cpu"),
@@ -860,6 +978,35 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
assert builder._mla_q_dtype == dtypes.fp8
assert builder._mla_kv_dtype == dtypes.fp8
builder.dcp_world_size = 2
chunk = object()
rows = torch.tensor([0], dtype=torch.int64)
build_rows = mock.Mock(return_value=rows)
monkeypatch.setattr(rocm_aiter_mla, "context_row_indices", build_rows)
monkeypatch.setattr(
rocm_aiter_mla,
"_supports_fused_dcp_prefill_config",
mock.Mock(side_effect=AssertionError("Support must be cached at init")),
)
monkeypatch.setattr(
rocm_aiter_mla.MLACommonMetadataBuilder,
"build",
lambda *args: SimpleNamespace(
prefill=SimpleNamespace(
chunked_context=SimpleNamespace(chunks=[chunk]),
block_table=torch.empty(0),
),
decode=None,
dcp_context_row_indices=None,
),
)
for _ in range(2):
metadata = builder.build(0, SimpleNamespace(causal=True))
assert metadata.dcp_context_row_indices == ([rows] if needs_row_map else None)
assert build_rows.call_args_list == (
[mock.call(chunk, torch.device("cpu"))] * 2 if needs_row_map else []
)
def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch):
get_mla_metadata_v1 = mock.MagicMock()
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.platforms import current_platform
if not current_platform.is_rocm() or not is_aiter_found_and_supported():
pytest.skip(
"ROCm AITER sparse indexer test requires ROCm with AITER.",
allow_module_level=True,
)
from vllm._aiter_ops import rocm_aiter_ops
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as sparse_mod
def test_rocm_fp8_mqa_logits_disables_clean_logits(monkeypatch):
captured = {}
expected = torch.empty((1, 1))
def fake_fp8_mqa_logits(*args, clean_logits):
captured["clean_logits"] = clean_logits
return expected
monkeypatch.setattr(sparse_mod, "_ON_GFX942", False)
monkeypatch.setattr(rocm_aiter_ops, "is_enabled", lambda: True)
monkeypatch.setattr(
sparse_mod,
"mqa_logits_module",
lambda: SimpleNamespace(fp8_mqa_logits=fake_fp8_mqa_logits),
)
q = torch.empty((1, 1, 1))
k_fp8 = torch.empty((1, 1))
scale = torch.empty((1,))
weights = torch.empty((1, 1))
cu_seqlen_ks = torch.zeros((1,), dtype=torch.int32)
cu_seqlen_ke = torch.ones((1,), dtype=torch.int32)
result = sparse_mod.rocm_fp8_mqa_logits(
q,
(k_fp8, scale),
weights,
cu_seqlen_ks,
cu_seqlen_ke,
)
assert result is expected
assert captured["clean_logits"] is False
@@ -96,6 +96,8 @@ def test_forward_mqa_prepares_triton_sparse_mla_inputs(
impl.sinks = torch.randn(num_heads, generator=gen).to(device)
impl.kv_cache_dtype = kv_cache_dtype
impl.topk_indices_buffer = topk_indices.to(device)
impl.owns_indexer = True
impl.index_epoch = 0
impl.use_aiter_sparse_mla = True
layer = SimpleNamespace(
_q_scale=torch.tensor([0.02], device=device),
@@ -61,6 +61,31 @@ def test_aiter_unified_attention_uses_dedicated_metadata_builder():
)
def test_rocm_backends_do_not_support_mm_prefix():
"""Regression test: ROCm backends must not claim mm_prefix support.
If they do, prefix-LM multimodal models that attend vision tokens
bidirectionally (e.g. Gemma3 4B) will use bidirectional attention
via the ROCm backend instead of falling back to Triton, producing
incorrect results. Remove this test only if support is properly added.
"""
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
RocmAiterUnifiedAttentionBackend,
)
from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend
assert not RocmAttentionBackend.supports_mm_prefix(), (
"RocmAttentionBackend does not support mm_prefix — "
"defer to Triton unified attention instead for now"
"See https://github.com/vllm-project/vllm/issues/48172"
)
assert not RocmAiterUnifiedAttentionBackend.supports_mm_prefix(), (
"RocmAiterUnifiedAttentionBackend does not support mm_prefix — "
"defer to Triton unified attention instead for now"
"See https://github.com/ROCm/aiter/issues/4168"
)
def test_segmented_attention_uses_dedicated_backend_components():
from vllm.v1.attention.backends.rocm_segmented_attn import (
RocmSegmentedAttentionBackend,
@@ -181,7 +206,9 @@ def test_segmented_triton_cache_survives_standalone_compilation(
RocmSegmentedAttentionImpl(8, 128, 128**-0.5, 2, None, None, "auto")
assert os.environ["TRITON_CACHE_DIR"] == str(expected)
assert expected.is_dir()
InductorStandaloneAdaptor("binary").initialize_cache(str(tmp_path / model))
InductorStandaloneAdaptor(
use_aot_compile=True, save_format="binary"
).initialize_cache(str(tmp_path / model))
assert os.environ["TRITON_CACHE_DIR"] == str(expected)
@@ -250,7 +277,6 @@ def test_segmented_attention_forward_uses_dedicated_dispatch(monkeypatch):
output = torch.empty_like(query)
kv_cache = torch.empty(2, 2, 16, 256, dtype=torch.bfloat16)
metadata = SimpleNamespace(
use_cascade=False,
num_actual_tokens=3,
block_table=torch.zeros(1, 1, dtype=torch.int32),
query_start_loc=torch.tensor([0, 3], dtype=torch.int32),
@@ -11,6 +11,7 @@ from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
_use_rocm_sparse_triton,
fit_kpool_indices_to_aiter,
)
@@ -19,6 +20,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_validate_dsv4_sparse_dims,
_validate_sparse_dims,
)
from vllm.v1.kv_cache_interface import KVCacheLayout
@triton.jit
@@ -104,6 +106,10 @@ def test_fit_kpool_indices_rejects_narrow_input():
)
def test_rocm_sparse_mla_supports_glm_packed_layout():
assert KVCacheLayout.BLHNC in ROCMAiterMLASparseBackend.supported_kv_cache_layouts()
@pytest.mark.parametrize(
(
"kv_cache_dtype",
@@ -149,8 +155,86 @@ def test_rocm_sparse_triton_route(
)
@pytest.mark.parametrize(
("gfx942", "fp8", "num_prefills", "num_decodes", "prefill_max_seq_len", "route"),
[
pytest.param(True, False, 1, 0, 129, "triton", id="above-topk"),
pytest.param(False, False, 1, 0, 129, "aiter", id="gfx950"),
pytest.param(True, True, 1, 0, 129, "aiter", id="fp8"),
pytest.param(True, False, 0, 2, 129, "aiter", id="decode-only"),
pytest.param(True, False, 1, 1, 129, "aiter", id="mixed"),
pytest.param(True, False, 1, 0, 128, "aiter", id="equal-topk"),
pytest.param(True, False, 1, 0, 127, "aiter", id="below-topk"),
pytest.param(True, False, 0, 0, 129, "aiter", id="empty"),
pytest.param(
True, False, 1, 0, None, "missing", id="required-prefill-metadata"
),
],
)
def test_rocm_long_bf16_prefill_dispatch(
monkeypatch, gfx942, fp8, num_prefills, num_decodes, prefill_max_seq_len, route
):
"""Check the new dispatch gate using CPU tensors and intercepted kernels."""
monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: gfx942)
class KernelSelected(Exception):
pass
def triton_kernel(**kwargs):
raise KernelSelected("triton")
def aiter_kernel(*args, **kwargs):
raise KernelSelected("aiter")
monkeypatch.setattr(sparse_mod, "rocm_sparse_attn_prefill", triton_kernel)
monkeypatch.setattr(sparse_mod.rocm_aiter_ops, "mla_decode_fwd", aiter_kernel)
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
impl.num_heads = 64
impl.kv_lora_rank = 512
impl.kv_cache_dtype = "fp8" if fp8 else "auto"
impl.scale = 576**-0.5
impl.sinks = None
dtype = torch.float8_e4m3fn if fp8 else torch.bfloat16
q = torch.empty(2, 64, 576, dtype=dtype)
kv = torch.empty(4, 1, 576, dtype=dtype)
metadata = SimpleNamespace(
attn_out_dtype=torch.bfloat16,
num_prefills=num_prefills,
num_decodes=num_decodes,
num_decode_tokens=num_decodes,
max_query_len=2,
prefill_max_seq_len=prefill_max_seq_len,
topk_tokens=128,
work_meta_data=None,
qo_indptr=torch.arange(3, dtype=torch.int32),
paged_kv_indices=torch.empty(0, dtype=torch.int32),
paged_kv_indptr=torch.zeros(3, dtype=torch.int32),
paged_kv_last_page_len=torch.ones(2, dtype=torch.int32),
)
# Irrelevant routes must not inspect prefill-only metadata.
if (
not gfx942
or fp8
or num_decodes
or not num_prefills
or prefill_max_seq_len is None
):
del metadata.prefill_max_seq_len
del metadata.topk_tokens
layer = SimpleNamespace(_q_scale=1.0, _k_scale=1.0)
error = AttributeError if route == "missing" else KernelSelected
match = "prefill_max_seq_len" if route == "missing" else f"^{route}$"
with pytest.raises(error, match=match):
impl._forward_mla(layer, q, kv, metadata)
@pytest.mark.parametrize("gfx942", [False, True])
@pytest.mark.parametrize("num_heads", [8, 12])
def test_rocm_sparse_triton_route_preserves_padded_sinks(monkeypatch, num_heads):
def test_rocm_sparse_triton_route_preserves_padded_sinks(
monkeypatch, num_heads, gfx942
):
monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: gfx942)
captured = {}
def fake_rocm_sparse_attn_prefill(**kwargs):
@@ -0,0 +1,445 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from tests.utils import multi_gpu_only
from vllm.platforms import current_platform
from vllm.v1.attention.ops.rocm_aiter_mla_prefill import context_row_indices
@pytest.mark.parametrize(
"padded,lengths,starts,total,expected",
[
([3, 2], [[7, 6], [2, 1]], [5, 0], 6, [0, 1, 5, 3, 4, 8]),
([3], [[2, 9]], [4], 3, [3, 4, 5]),
([3], [[1, 1]], [0], 3, None),
],
ids=["padding_and_chunk_starts", "empty_rank", "inconsistent_total"],
)
def test_dcp_context_indices(padded, lengths, starts, total, expected):
chunk = SimpleNamespace(
padded_local_seq_lens=padded,
local_context_lens_allranks=lengths,
local_starts=starts,
num_local_context_tokens=sum(padded),
num_context_tokens=total,
)
if expected is None:
with pytest.raises(AssertionError):
context_row_indices(chunk, torch.device("cpu"))
else:
rows = context_row_indices(chunk, torch.device("cpu"))
assert rows.tolist() == expected
assert rows.dtype == torch.int32
def test_compressed_gather_preserves_bytes_and_workspace_partition(monkeypatch):
from vllm.v1.attention.ops import rocm_aiter_mla_prefill as prefill
workspace = torch.full((12, 4), -1, dtype=torch.bfloat16)
cache = torch.arange(8, dtype=torch.uint8).reshape(2, 4)
chunk = SimpleNamespace(
num_local_context_tokens=2,
local_context_lens_allranks=[[2, 2]],
padded_local_cu_seq_lens=torch.tensor([0, 2]),
num_requests=1,
starts=torch.tensor([0]),
)
def local_gather(src, dst, *args):
assert src.dtype == dst.dtype == torch.uint8
dst.copy_(src)
def allgather(dst, src):
assert dst.dtype == src.dtype == torch.uint8
assert dst.shape == (4, 4)
assert src.shape == (2, 4)
dst[:2].copy_(src)
dst[2:].copy_(src + 16)
monkeypatch.setattr(prefill.ops, "cp_gather_cache", local_gather)
gathered = prefill.gather_compressed_context(
cache, workspace, torch.empty(1, 1), chunk, allgather, torch.float8_e4m3fn
)
torch.testing.assert_close(
gathered.view(torch.uint8), torch.cat((cache, cache + 16))
)
# Retain the original logical capacity even though the byte view is larger.
assert gathered.data_ptr() - workspace.data_ptr() == 4 * 4
torch.testing.assert_close(workspace.view(torch.uint8).reshape(-1, 4)[:2], cache)
@pytest.mark.parametrize(
"unsupported",
[
"packed_cache",
"e5m2",
"bf16_cache",
"quantized_weight",
"bias",
"no_indices",
"query_dtype",
"prefill_dtype",
"weight_dtype",
"weight_layout",
"cache_layout",
"dimensions",
"no_aiter",
],
)
def test_dcp_prefill_unsupported_formats_use_original_path(monkeypatch, unsupported):
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("ROCm MLA backend")
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
AiterMLAImpl,
MLACommonImpl,
)
if unsupported == "no_aiter":
monkeypatch.setattr(
"vllm.v1.attention.backends.mla.rocm_aiter_mla."
"is_aiter_found_and_supported",
lambda: False,
)
impl = object.__new__(AiterMLAImpl)
impl.kv_cache_dtype = {
"packed_cache": "fp8_ds_mla",
"e5m2": "fp8_e5m2",
"bf16_cache": "auto",
}.get(unsupported, "fp8")
impl.kv_lora_rank = 512
if unsupported == "dimensions":
impl.kv_lora_rank = 256
impl.qk_nope_head_dim = impl.v_head_dim = 128
impl.qk_rope_head_dim = 64
impl.num_heads = 1
impl.kv_b_proj = SimpleNamespace(
weight=torch.empty(256, 512, dtype=torch.bfloat16),
weight_scale=torch.ones(1) if unsupported == "quantized_weight" else None,
bias=torch.ones(256) if unsupported == "bias" else None,
)
if unsupported == "weight_dtype":
impl.kv_b_proj.weight = impl.kv_b_proj.weight.half()
elif unsupported == "weight_layout":
impl.kv_b_proj.weight = torch.empty(512, 256, dtype=torch.bfloat16).T
metadata = SimpleNamespace(
prefill=SimpleNamespace(
q_data_type=torch.float16
if unsupported == "prefill_dtype"
else torch.bfloat16
),
dcp_context_row_indices=None if unsupported == "no_indices" else [],
)
expected = (torch.ones(1), torch.zeros(1))
def fused_mla_kv_concat(kv_nope, k_pe, use_fp8_prefill):
raise AssertionError("The fallback should receive the callback")
def original_path(*args):
assert args[-1] is fused_mla_kv_concat
return expected
monkeypatch.setattr(
MLACommonImpl,
"_context_parallel_compute_prefill_context",
original_path,
)
q = torch.empty(1, 1, 192, dtype=torch.bfloat16)
cache = torch.empty(1, 1536, 576, dtype=torch.uint8)
if unsupported == "query_dtype":
q = q.half()
elif unsupported == "cache_layout":
cache = torch.empty(1, 1536, 1152, dtype=torch.uint8)[..., ::2]
assert (
impl._context_parallel_compute_prefill_context(
q,
cache,
metadata,
torch.ones(1),
8,
fused_mla_kv_concat_fn=fused_mla_kv_concat,
)
is expected
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm MLA backend")
@multi_gpu_only(num_gpus=2)
@pytest.mark.parametrize("interleave", [1, 64])
@pytest.mark.parametrize(
"context_lens,workspace_size",
[([145, 37], 1024), ([545, 0, 3, 0], 256)],
ids=["single_chunk", "continuations_and_empty_context"],
)
def test_dcp_prefill_matches_full_attention(
interleave, context_lens, workspace_size, tmp_path
):
"""Real DCP gather, fused expansion and suffix/chunk merges match dense SDPA."""
import torch.multiprocessing as mp
from vllm._aiter_ops import is_aiter_found_and_supported
if not is_aiter_found_and_supported():
pytest.skip("AITER on CDNA 3 or newer is required")
mp.spawn(
_dcp_prefill_worker,
args=(
f"file://{tmp_path / 'dist_init'}",
interleave,
context_lens,
workspace_size,
),
nprocs=2,
)
@torch.inference_mode()
def _dcp_prefill_worker(rank, init_method, interleave, context_lens, workspace_size):
import torch.nn.functional as F
from tests.v1.attention.test_mla_backends import MockMLAAttentionLayer
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.config import set_current_vllm_config
from vllm.distributed import (
cleanup_dist_env_and_memory,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.v1.attention.backends.mla.prefill.aiter_flash_attn import (
AiterFlashAttnPrefillBackend,
)
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
AiterMLAImpl,
AiterMLAMetadataBuilder,
)
from vllm.v1.attention.ops.dcp import MLADCPManager
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.workspace import init_workspace_manager
torch.accelerator.set_device_index(rank)
torch.set_num_threads(1)
device = torch.device(f"cuda:{rank}")
init_workspace_manager(device)
world_size, num_heads, block_size = 2, 12, 64
config = create_vllm_config(
model_name="deepseek-ai/DeepSeek-V2-Lite",
dtype=torch.bfloat16,
tensor_parallel_size=world_size,
max_model_len=2048,
block_size=block_size,
max_num_seqs=8,
max_num_batched_tokens=2048,
)
config.cache_config.cache_dtype = "fp8"
config.parallel_config.decode_context_parallel_size = world_size
config.parallel_config.cp_kv_cache_interleave_size = interleave
config.attention_config.use_prefill_query_quantization = False
config.kernel_config.enable_jit_warmup = False
with set_current_vllm_config(config):
init_distributed_environment(
world_size=world_size,
rank=rank,
local_rank=rank,
distributed_init_method=init_method,
)
initialize_model_parallel(
tensor_model_parallel_size=world_size,
decode_context_model_parallel_size=world_size,
)
try:
projection = ColumnParallelLinear(
512,
num_heads * world_size * 256,
bias=False,
params_dtype=torch.bfloat16,
).to(device)
torch.manual_seed(101 + rank)
projection.weight.copy_(torch.randn_like(projection.weight) * 0.02)
dims = dict(
kv_lora_rank=512,
qk_nope_head_dim=128,
qk_rope_head_dim=64,
v_head_dim=128,
)
impl = AiterMLAImpl(
num_heads=num_heads,
head_size=576,
scale=192**-0.5,
num_kv_heads=1,
alibi_slopes=None,
sliding_window=None,
kv_cache_dtype="fp8",
logits_soft_cap=None,
attn_type="decoder",
kv_sharing_target_layer_name=None,
q_lora_rank=None,
qk_head_dim=192,
kv_b_proj=projection,
**dims,
)
layer = MockMLAAttentionLayer(
impl=impl,
num_heads=num_heads,
device=device,
kv_b_proj=projection,
q_scale=1.0,
k_scale=0.07,
**dims,
)
layer.prefill_backend = AiterFlashAttnPrefillBackend(
num_heads=num_heads,
scale=impl.scale,
vllm_config=config,
**dims,
)
layer.dcp_manager = MLADCPManager(
config,
device,
num_heads,
576,
512,
torch.bfloat16,
torch.bfloat16,
None,
True,
False,
)
config.compilation_config.static_forward_context["mla"] = layer
spec = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=576,
dtype=current_platform.fp8_dtype(),
cache_dtype_str="fp8",
)
builder = AiterMLAMetadataBuilder(spec, ["mla"], config, device)
builder.chunked_prefill_workspace_size = workspace_size
builder.chunked_prefill_workspace = torch.empty(
(workspace_size + workspace_size // world_size, 576),
dtype=torch.bfloat16,
device=device,
)
layer.dcp_manager.init_kv_gather(
builder.chunked_prefill_workspace, workspace_size
)
query_lens = [i + 2 for i in range(len(context_lens))]
batch = BatchSpec(
seq_lens=[c + q for c, q in zip(context_lens, query_lens)],
query_lens=query_lens,
)
common = create_common_attn_metadata(
batch, block_size, device, arange_block_indices=True
)
metadata = builder.build(0, common)
assert metadata.prefill is not None
chunked = metadata.prefill.chunked_context
assert chunked is not None and metadata.dcp_context_row_indices is not None
if 0 in context_lens:
assert chunked.empty_token_slices
assert any(c.is_continuation for c in chunked.chunks)
else:
assert len(chunked.chunks) == 1
blocks_per_req = common.block_table_tensor.shape[1]
cache = torch.zeros(
(len(context_lens) * blocks_per_req, block_size, 576),
dtype=torch.bfloat16,
device=device,
).to(current_platform.fp8_dtype())
queries, suffixes, references = [], [], []
# All ranks reconstruct the same global cache before sharding it.
torch.manual_seed(42)
for request, (context_len, query_len) in enumerate(
zip(context_lens, query_lens)
):
latent = torch.randn(
context_len, 576, device=device, dtype=torch.bfloat16
)
quantized = (latent.float() / layer._k_scale).to(cache.dtype)
positions = torch.arange(context_len, device=device)
shard = quantized.view(torch.uint8)[
(positions // interleave) % world_size == rank
]
cache.view(len(context_lens), -1, 576).view(torch.uint8)[
request, : len(shard)
].copy_(shard)
suffix = torch.randn(
query_len, 576, device=device, dtype=torch.bfloat16
)
q = (
torch.randn(
query_len, num_heads, 192, device=device, dtype=torch.bfloat16
)
* 0.5
)
full_latent = torch.cat(
(
(quantized.float() * layer._k_scale).to(torch.bfloat16),
suffix,
)
)
kv = F.linear(full_latent[:, :512], projection.weight).view(
-1, num_heads, 256
)
k_nope, v = kv.split(128, dim=-1)
k = torch.cat(
(k_nope, full_latent[:, None, 512:].expand(-1, num_heads, -1)),
dim=-1,
)
mask = torch.arange(context_len + query_len, device=device)[
None, :
] <= (context_len + torch.arange(query_len, device=device)[:, None])
references.append(
F.scaled_dot_product_attention(
q.transpose(0, 1).float(),
k.transpose(0, 1).float(),
v.transpose(0, 1).float(),
attn_mask=mask,
scale=impl.scale,
).transpose(0, 1)
)
queries.append(q)
suffixes.append(suffix)
q, suffix = torch.cat(queries), torch.cat(suffixes)
output = torch.empty(
q.shape[0], num_heads * 128, device=device, dtype=torch.bfloat16
)
# Ensure this test cannot silently succeed through the fallback.
with pytest.MonkeyPatch.context() as patch:
def unexpected_fallback(*args, **kwargs):
raise AssertionError("Expected the compressed DCP path")
patch.setattr(
"vllm.model_executor.layers.attention.mla_attention."
"MLACommonImpl._context_parallel_compute_prefill_context",
unexpected_fallback,
)
impl.forward_mha(
q,
suffix[:, :512],
suffix[:, None, 512:],
cache,
metadata,
layer._k_scale,
output,
)
torch.testing.assert_close(
output.view(-1, num_heads, 128).float(),
torch.cat(references),
rtol=0.03,
atol=0.003,
)
finally:
cleanup_dist_env_and_memory()
@@ -36,6 +36,7 @@ def _chunk(block_table, token_start, token_end, row_ends):
cu_seq_lens=torch.zeros(1, dtype=torch.int32),
token_to_seq=torch.zeros(1, dtype=torch.int32),
total_seq_lens=1,
max_seqlen_k=int(ends.max()),
token_start=token_start,
token_end=token_end,
num_reqs=block_table.shape[0],
File diff suppressed because it is too large Load Diff
@@ -12,20 +12,18 @@ from vllm.config import CacheConfig
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseTRTLLMBackend,
)
from vllm.v1.attention.backends.mla.flashmla_sparse import FlashMLASparseBackend
from vllm.v1.attention.backend import MultipleOf
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend
from vllm.v1.attention.backends.mla.sparse_utils import (
_CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL,
flat_kv_row_view,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
from vllm.v1.core.kv_cache_utils import get_kv_cache_config_from_groups
from vllm.v1.core.kv_cache_planning import get_kv_cache_config_from_groups
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, UniformTypeKVCacheSpecs
from vllm.v1.kv_cache_layout import KVCacheLayout
from vllm.v1.worker.utils import allocate_kv_cache
from vllm.v1.worker.utils import allocate_kv_cache, customize_attention_spec
pytestmark = pytest.mark.skip_global_cleanup
@@ -48,17 +46,27 @@ pytestmark = pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
"backend,cache_dtype,sm100,stride_alignment",
[
(FlashInferMLASparseTRTLLMBackend, "auto", True, 1152),
(FlashInferMLASparseTRTLLMBackend, "fp8", True, 576),
(FlashMLASparseBackend, "auto", True, 1152),
(FlashMLASparseBackend, "fp8_ds_mla", False, None),
(FlashMLASparseBackend, "fp8_ds_mla", True, 656),
(FlashMLASparseBackend, "nvfp4_ds_mla", True, 352),
(AttentionBackendEnum.FLASHINFER_MLA_SPARSE, "auto", True, 32 * 1152),
(AttentionBackendEnum.FLASHINFER_MLA_SPARSE, "fp8", True, 32 * 576),
pytest.param(
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE_FA4,
"auto",
True,
64 * 1152,
marks=pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUDA only"
),
),
(AttentionBackendEnum.FLASHMLA_SPARSE, "auto", True, 1152),
(AttentionBackendEnum.FLASHMLA_SPARSE, "fp8_ds_mla", False, MultipleOf(64)),
(AttentionBackendEnum.FLASHMLA_SPARSE, "fp8_ds_mla", True, 656),
(AttentionBackendEnum.FLASHMLA_SPARSE, "nvfp4_ds_mla", True, 352),
],
)
def test_allocation_and_warmup_follow_addressing_mode(
monkeypatch, layout, backend, cache_dtype, sm100, stride_alignment, device
):
backend = backend.get_class()
monkeypatch.setattr(
current_platform, "is_device_capability_family", lambda family: sm100
)
@@ -79,7 +87,9 @@ def test_allocation_and_warmup_follow_addressing_mode(
non_causal_multi_token_decode=False,
)
layer._uses_flat_kv_cache = MethodType(MLAAttention._uses_flat_kv_cache, layer)
spec = MLAAttention.get_kv_cache_spec(layer, config)
spec = customize_attention_spec(
backend, MLAAttention.get_kv_cache_spec(layer, config)
)
indexer = SimpleNamespace(
cache_config=config.cache_config, head_dim=132, dtype=torch.uint8
)
@@ -102,10 +112,8 @@ def test_allocation_and_warmup_follow_addressing_mode(
MLAAttention.bind_kv_cache(layer, views["mla"])
DeepseekV32IndexerCache.bind_kv_cache(indexer, views["indexer"])
assert indexer.kv_cache.stride(0) == views["indexer"].stride(0)
if stride_alignment is not None:
assert (
views["mla"].stride(0) * views["mla"].element_size() % stride_alignment == 0
)
alignment = spec.get_block_stride_alignment()
assert views["mla"].stride(0) * views["mla"].element_size() % alignment == 0
if not layer._uses_flat_kv_cache():
register.assert_not_called()
else:
@@ -113,7 +121,7 @@ def test_allocation_and_warmup_follow_addressing_mode(
register.assert_called_once_with(config, block_stride_rows=stride)
layer.kv_cache[1, 0].fill_(3)
torch.testing.assert_close(rows[stride], layer.kv_cache[1, 0])
if stride_alignment is None:
if alignment == 1:
assert cache.kv_cache_tensors[0].size == cache.num_blocks * sum(
item.page_size_bytes for item in specs.values()
)
@@ -0,0 +1,154 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import torch
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm.config import CUDAGraphMode
from vllm.platforms import current_platform
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder
from vllm.v1.attention.backends.triton_attn_diffkv import (
TritonAttentionDiffKVMetadataBuilder,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec
# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and
# the 64-segment scratch holds every query token (e.g. multi-query verify).
@pytest.mark.parametrize(
"builder_cls",
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
)
@pytest.mark.parametrize(
"capability,num_kv_heads,num_seqs,query_len,segments",
[
((12, 0), 1, 11, 1, 64),
((12, 0), 1, 12, 1, 16),
((12, 0), 8, 1, 1, 64),
((12, 0), 8, 2, 1, 16),
((9, 0), 1, 1, 1, 16),
((12, 0), 1, 2, 4, 64),
((12, 0), 1, 10, 4, 16),
],
)
def test_split_k_segments_follow_sm_occupancy(
monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments
):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(
current_platform, "is_device_capability", lambda cap: cap == capability
)
monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188)
config = SimpleNamespace(
model_config=SimpleNamespace(
get_num_attention_heads=lambda _: 8,
get_num_kv_heads=lambda _: num_kv_heads,
get_head_size=lambda: 128,
rswa_window=None,
),
parallel_config=None,
scheduler_config=SimpleNamespace(max_num_seqs=16),
speculative_config=None,
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
),
)
spec = FullAttentionSpec(
block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16
)
builder = builder_cls(spec, ["layer.0"], config, "cpu")
batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs)
metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu"))
assert metadata.num_par_softmax_segments == segments
assert metadata.softmax_segm_output.shape[2] == segments
assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len
# DiffKV re-allocates the output; it must stay sized like the base buffers.
assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0]
assert (
metadata.softmax_segm_output.data_ptr()
== builder.softmax_segm_output.data_ptr()
)
def _draft_builder(builder_cls, rswa_window: int | None):
config = SimpleNamespace(
model_config=SimpleNamespace(
get_num_attention_heads=lambda _: 8,
get_num_kv_heads=lambda _: 1,
get_head_size=lambda: 128,
rswa_window=rswa_window,
),
parallel_config=None,
scheduler_config=SimpleNamespace(max_num_seqs=16),
speculative_config=None,
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
),
)
spec = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
)
return builder_cls(spec, ["layer.0"], config, "cpu")
def _draft_decode_batch(seq_lens: list[int]):
batch = BatchSpec(seq_lens=seq_lens, query_lens=[1] * len(seq_lens))
return create_common_attn_metadata(batch, 16, "cpu")
@pytest.mark.parametrize(
"builder_cls",
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
)
@pytest.mark.parametrize("rswa_window", [None, 64])
def test_draft_decode_update_flag_matches_build_writes(builder_cls, rswa_window):
"""The update is a no-op, so the builder may claim support only when
build() writes no builder-owned tensor that a captured graph reads."""
builder = _draft_builder(builder_cls, rswa_window)
common_metadata = _draft_decode_batch([5, 9])
common_metadata.rswa_prefix_lens = torch.tensor([3, 7], dtype=torch.int32)
def owned_bytes():
# Bytes, not values: uninitialized scratch may hold NaNs.
return {
name: value.view(torch.uint8).clone()
for name, value in vars(builder).items()
if isinstance(value, torch.Tensor)
}
before = owned_bytes()
builder.build(0, common_metadata)
after = owned_bytes()
written = [name for name in before if not torch.equal(before[name], after[name])]
assert builder.supports_draft_decode_metadata_update is (not written)
assert builder.supports_draft_decode_metadata_update is (rswa_window is None)
@pytest.mark.parametrize(
"builder_cls",
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
)
def test_draft_decode_update_matches_fresh_build(builder_cls):
"""A replayed graph reads metadata built at capture; after an in-place
update it must match a fresh build of the replayed batch."""
torch.manual_seed(0)
builder = _draft_builder(builder_cls, rswa_window=None)
captured = _draft_decode_batch([100, 120])
metadata = builder.build(0, captured)
replayed = _draft_decode_batch([128, 30])
# The replayed batch arrives through the same persistent input buffers.
captured.seq_lens.copy_(replayed.seq_lens)
captured.block_table_tensor.copy_(replayed.block_table_tensor)
captured.slot_mapping.copy_(replayed.slot_mapping)
builder.update_draft_decode_metadata(metadata)
expected = _draft_builder(builder_cls, rswa_window=None).build(0, replayed)
assert torch.equal(metadata.seq_lens, expected.seq_lens)
assert torch.equal(metadata.block_table, expected.block_table)
assert torch.equal(metadata.slot_mapping, expected.slot_mapping)
assert torch.equal(metadata.query_start_loc, expected.query_start_loc)
+113 -28
View File
@@ -13,29 +13,19 @@ from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilde
from vllm.v1.attention.backends.triton_attn_diffkv import (
TritonAttentionDiffKVMetadataBuilder,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVQuantMode
# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and
# the 64-segment scratch holds every query token (e.g. multi-query verify).
@pytest.mark.parametrize(
"builder_cls",
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
)
@pytest.mark.parametrize(
"capability,num_kv_heads,num_seqs,query_len,segments",
[
((12, 0), 1, 11, 1, 64),
((12, 0), 1, 12, 1, 16),
((12, 0), 8, 1, 1, 64),
((12, 0), 8, 2, 1, 16),
((9, 0), 1, 1, 1, 16),
((12, 0), 1, 2, 4, 64),
((12, 0), 1, 10, 4, 16),
],
)
def test_split_k_segments_follow_sm_occupancy(
monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments
def _build(
monkeypatch,
builder_cls,
capability,
num_kv_heads,
query_lens,
max_num_seqs=16,
num_speculative_tokens=None,
parallel_drafting=False,
kv_quant_mode=KVQuantMode.NONE,
):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(
@@ -50,24 +40,119 @@ def test_split_k_segments_follow_sm_occupancy(
rswa_window=None,
),
parallel_config=None,
scheduler_config=SimpleNamespace(max_num_seqs=16),
speculative_config=None,
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
speculative_config=(
None
if num_speculative_tokens is None
else SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
parallel_drafting=parallel_drafting,
)
),
compilation_config=SimpleNamespace(
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
),
)
spec = FullAttentionSpec(
block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16
block_size=16,
num_kv_heads=num_kv_heads,
head_size=128,
dtype=torch.bfloat16,
kv_quant_mode=kv_quant_mode,
)
builder = builder_cls(spec, ["layer.0"], config, "cpu")
batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs)
batch = BatchSpec(seq_lens=[128] * len(query_lens), query_lens=query_lens)
metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu"))
assert metadata.num_par_softmax_segments == segments
assert metadata.softmax_segm_output.shape[2] == segments
assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len
assert metadata.softmax_segm_output.shape[2] == metadata.num_par_softmax_segments
assert metadata.softmax_segm_max.shape[0] >= sum(query_lens)
# DiffKV re-allocates the output; it must stay sized like the base buffers.
assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0]
assert (
metadata.softmax_segm_output.data_ptr()
== builder.softmax_segm_output.data_ptr()
)
return builder, metadata
BUILDERS = [TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder]
# Verify batches: the 64-segment scratch holds min(11, max_num_seqs) * (1 + K)
# rows, and its 16-segment view covers every 3D-eligible sequence. 64 segments
# follow the request count, not Q blocks: on SM12.0 they stay faster for verify
# batches up to 11 requests (22 Q blocks at K=3).
@pytest.mark.parametrize("builder_cls", BUILDERS)
@pytest.mark.parametrize(
"max_num_seqs,num_speculative_tokens,num_seqs,rows,segments",
[
(16, 3, 10, 44, 64),
(16, 3, 11, 44, 64),
(16, 3, 12, 44, 16),
(16, 3, 16, 44, 16),
(1, 5, 1, 6, 64),
],
)
def test_split_k_segment_rows_cover_verify_queries(
monkeypatch,
builder_cls,
max_num_seqs,
num_speculative_tokens,
num_seqs,
rows,
segments,
):
builder, metadata = _build(
monkeypatch,
builder_cls,
(12, 0),
1,
[1 + num_speculative_tokens] * num_seqs,
max_num_seqs,
num_speculative_tokens,
)
assert builder.softmax_segm_max.shape[:2] == (rows, 8)
assert metadata.num_par_softmax_segments == segments
# Without SM12.0 64-segment reuse, the scratch holds exactly
# min(seq_threshold_3D, max_num_seqs) * query_len rows for split-K query_lens.
@pytest.mark.parametrize(
"num_speculative_tokens,parallel_drafting,max_num_seqs,rows",
[
(None, False, 8, 8),
(4, False, 8, 40),
(7, False, 8, 64),
(8, False, 8, 64),
(3, True, 8, 56),
(4, True, 8, 64),
(5, False, 1, 6),
],
)
@pytest.mark.parametrize(
"kv_quant_mode", [KVQuantMode.NONE, KVQuantMode.INT4_PER_TOKEN_HEAD]
)
def test_split_k_scratch_rows_follow_speculative_config(
monkeypatch,
num_speculative_tokens,
parallel_drafting,
max_num_seqs,
rows,
kv_quant_mode,
):
# The INT4 launcher never runs multi-query split-K.
if kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
rows = max_num_seqs
builder, _ = _build(
monkeypatch,
TritonAttentionMetadataBuilder,
(9, 0),
1,
[1],
max_num_seqs,
num_speculative_tokens,
parallel_drafting,
kv_quant_mode,
)
assert builder.softmax_segm_output.shape == (rows, 8, 16, 128)
assert builder.softmax_segm_max.shape == (rows, 8, 16)
assert builder.softmax_segm_expsum.shape == (rows, 8, 16)
@@ -38,9 +38,7 @@ def _manager(
num_prefill_lookahead=0,
):
init_none_hash(sha256)
config = _make_hybrid_kv_cache_config(
block_size, num_blocks, ["full", "mamba_align"]
)
config = _make_hybrid_kv_cache_config(block_size, num_blocks, ["full", "mamba"])
if eagle_group is not None:
groups = list(config.kv_cache_groups)
groups[eagle_group] = replace(groups[eagle_group], is_eagle_group=True)
@@ -26,8 +26,8 @@ from vllm.v1.core.kv_cache_utils import (
)
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
@@ -2308,12 +2308,11 @@ def test_opted_out_scratch_group_keeps_partial_hash_hits():
),
KVCacheGroupSpec(
["tail"],
KpoolTailSpec(
CircularBufferSpec(
block_size=mamba_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=mamba_block_size,
),
),
],
@@ -2366,12 +2365,11 @@ def test_kpool_tail_supports_128_token_partial_hash_hits():
),
KVCacheGroupSpec(
["tail"],
KpoolTailSpec(
CircularBufferSpec(
block_size=kpool_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=kpool_block_size,
),
),
],
-1
View File
@@ -340,7 +340,6 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
total_num_scheduled_tokens=1,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
+44 -13
View File
@@ -18,13 +18,15 @@ import torch
from vllm.config import CacheConfig
from vllm.v1.core.kv_cache_manager import KVCacheManager
from vllm.v1.core.kv_cache_utils import (
from vllm.v1.core.kv_cache_planning import (
_get_kv_cache_bytes_per_block,
_get_packed_kv_cache_groups,
_pool_bytes_per_block,
generate_scheduler_kv_cache_config,
get_kv_cache_config_from_groups,
get_kv_cache_groups,
)
from vllm.v1.core.kv_cache_utils import (
generate_scheduler_kv_cache_config,
resolve_kv_cache_block_sizes,
)
from vllm.v1.hisparse.layout import _build_hisparse_kv_cache_tensors
@@ -88,7 +90,7 @@ def test_packed_alignment_preserves_hot_pages_and_generic_stride():
hot = HiSparseHotSpec(block_size=16, page_size=5120, blocks_per_request=2)
groups = [KVCacheGroupSpec(["attention"], spec), KVCacheGroupSpec(["hot"], hot)]
alignment = lcm(1152, hot.page_size_bytes)
stride = _pool_bytes_per_block(groups)
stride = _pool_bytes_per_block(_mock_vllm_config("BLHNC"), groups)
assert stride == alignment
layout = KVCacheLayout.BLHNC
tensors = _build_hisparse_kv_cache_tensors(groups, 3, 3 * stride, layout, stride)
@@ -254,7 +256,7 @@ class TestCSALinearGrouping:
def test_every_group_fits_one_packed_block(self):
config = _shared_layout_config()
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
bytes_per_block = _get_kv_cache_bytes_per_block(groups)
bytes_per_block = _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC)
pages = _pages(groups)
for group in groups:
@@ -276,7 +278,14 @@ class TestCSALinearGrouping:
neither divides the hit length nor bounds the (empty) block list."""
config = _shared_layout_config()
config.cache_config.enable_prefix_caching = True
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
config.cache_config.mamba_cache_mode = "align"
specs = {
name: replace(spec, mamba_cache_mode="align")
if isinstance(spec, MambaSpec)
else spec
for name, spec in _make_csa_linear_specs().items()
}
groups = get_kv_cache_groups(config, specs)
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=BYTES_PER_BLOCK * 64
)
@@ -418,7 +427,7 @@ class TestCSALinearGrouping:
groups = _get_packed_kv_cache_groups(config, specs)
gdn = [g for g in groups if g.layer_names[0].startswith("gdn.")]
assert _get_kv_cache_bytes_per_block(groups) == sum(
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == sum(
specs[name].page_size_bytes for name in specs if name.startswith("wide.")
)
# That block holds every GDN state at once, so the repeat pattern alone
@@ -465,7 +474,7 @@ class TestSlidingWindowBucketCap:
pages = _pages(groups)
main = next(g for g in groups if "layers.2.attn" in g.layer_names)
main_bytes = sum(pages[n] for n in main.layer_names)
assert _get_kv_cache_bytes_per_block(groups) == main_bytes
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == main_bytes
swa_groups = [g for g in groups if g.layer_names[0].endswith(".swa")]
per_group = main_bytes // pages["layers.0.swa"]
assert len(swa_groups) == -(-43 // per_group)
@@ -479,9 +488,9 @@ class TestSlidingWindowBucketCap:
class TestDensePacking:
def test_bytes_per_block_is_largest_group(self):
groups, g1, g2 = _mixed_page_groups()
assert _get_kv_cache_bytes_per_block(groups) == _expected_bytes_per_block(
groups
)
assert _get_kv_cache_bytes_per_block(
groups, KVCacheLayout.BLHNC
) == _expected_bytes_per_block(groups)
config = get_kv_cache_config_from_groups(
_mock_vllm_config("BLHNC"), groups, MEMORY
@@ -554,6 +563,20 @@ class TestDensePacking:
assert idx_tensor.offset == 2 * _mla(512).page_size_bytes
assert mla_tensor.block_stride == block_stride
def test_layer_outer_pads_only_unaligned_pages(self):
"""Layer-outer block strides are pages, so only an unaligned page pads."""
mla_page = _mla(512).page_size_bytes
idx = replace(_mla(128), block_stride_alignment=12288)
groups = [_uniform_group({"mla.0": _mla(512), "idx.0": idx})]
config = get_kv_cache_config_from_groups(
_mock_vllm_config("LBNHC"), groups, MEMORY
)
mla_tensor, idx_tensor = config.kv_cache_tensors
assert config.num_blocks == MEMORY // (mla_page + 12288)
assert (mla_tensor.block_stride, idx_tensor.block_stride) == (mla_page, 12288)
assert idx_tensor.offset == mla_page * config.num_blocks
_bind(config, "LBNHC")
def test_overlaid_groups_alias_and_stay_isolated(self):
groups, g1, g2 = _mixed_page_groups()
# Overlay models resolve to a block-outer layout at backend selection (the
@@ -562,7 +585,9 @@ class TestDensePacking:
_mock_vllm_config("BLNHC"), groups, MEMORY
)
assert config.num_blocks == MEMORY // _expected_bytes_per_block(groups)
assert _pool_bytes_per_block(groups) == _expected_bytes_per_block(groups)
assert _pool_bytes_per_block(
_mock_vllm_config("BLNHC"), groups
) == _expected_bytes_per_block(groups)
views = _bind(config, "BLNHC")
assert set(views) == set(g1) | set(g2)
@@ -667,7 +692,10 @@ class TestCompressorRingGroup:
assert len(ring_groups) == 1
assert not ring_groups[0].kv_cache_spec.prefix_cacheable
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
config,
groups,
available_memory=64
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
)
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (128, 64)
manager = KVCacheManager(
@@ -716,7 +744,10 @@ class TestSWABoundedReplayGrouping:
for g in groups
) == [(False, 128)] * 4 + [(True, 0)]
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
config,
groups,
available_memory=64
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
)
manager = KVCacheManager(
generate_scheduler_kv_cache_config([kv_cache_config]),
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+97 -87
View File
@@ -279,6 +279,12 @@ def _allocate_scheduled(
return blocks
def _resident_block_ids(manager: KVCacheManager, request_id: str) -> list[int]:
"""The GPU block HiSparse reads each resident page from; 0 if host-only."""
resident = get_hisparse_coordinator(manager).resident_managers[0]
return [block.block_id for block in resident.get_residency_row(request_id)]
def test_hisparse_does_not_write_back_reprefillable_tokens():
"""Multi-module MTP re-prefills the last tokens, so they are not final."""
manager = make_hisparse_kv_cache_manager(32, 16, num_prefill_lookahead=3)
@@ -319,7 +325,7 @@ def test_hisparse_builds_dma_row_mirrors_across_pages():
def test_hisparse_async_speculation_mirrors_uncertain_position_range():
"""Unresolved drafts must not leave gaps in the eager host mirror."""
coordinator = MagicMock(host_group_id=0)
coordinator.take_block_table_updates.return_value = {}
coordinator.take_residency_updates.return_value = {}
coordinator.build_offload_command.return_value = None
coordinator.build_row_mirrors.return_value = ()
scheduler = HiSparseConnectorScheduler(
@@ -332,7 +338,6 @@ def test_hisparse_async_speculation_mirrors_uncertain_position_range():
connector.connector_scheduler = scheduler
connector.update_state_after_alloc(request, None, 0)
scheduler_output = SimpleNamespace(
block_table_updates=None,
kv_cache_block_copies=None,
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
@@ -370,11 +375,66 @@ def test_hisparse_reports_when_context_is_fully_resident():
hot_manager.require_hot(request.request_id)
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
_publish_hisparse_pages(manager)
coordinator.take_residency_updates([request.request_id])
core_row = manager.get_block_ids(request.request_id)[2]
pool = manager.block_pool
pool.get_new_blocks(pool.get_num_free_blocks())
assert not coordinator.all_context_pages_resident(scheduled)
assert coordinator.take_block_table_updates().keys() == {request.request_id}
# The block-table row stays append-only; the worker learns of the lost
# pages from the residency update instead.
assert manager.get_block_ids(request.request_id)[2] == core_row
resident_ids = _resident_block_ids(manager, request.request_id)
lost = [page for page, block_id in enumerate(resident_ids) if block_id == 0]
assert lost
update = coordinator.take_residency_updates([request.request_id])
assert update[request.request_id].pages == lost
assert update[request.request_id].block_ids[0] == [0] * len(lost)
def test_hisparse_residency_update_sends_only_changed_pages():
"""Losing one early page must not resend the pages after it: per-step
residency updates would otherwise grow with the context length."""
manager = make_hisparse_kv_cache_manager(32, 16)
tokens = list(range(6 * HISPARSE_BLOCK_SIZE))
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens))
coordinator = get_hisparse_coordinator(manager)
for hot_manager in coordinator.hot_managers:
hot_manager.require_hot(request.request_id)
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens))
coordinator.take_residency_updates([request.request_id])
pool = manager.block_pool
num_never_used = pool.get_num_free_blocks()
# Pages unpin as their host copies complete, so page 0 is reused first.
_publish_hisparse_pages(manager)
pool.get_new_blocks(num_never_used + 1)
update = coordinator.take_residency_updates([request.request_id])
assert update[request.request_id].pages == [0]
assert update[request.request_id].block_ids == ([0],)
def test_hisparse_readmitted_request_resends_its_full_residency():
"""A preempted request may resume in a worker state row holding another
request's residency, so its first update must cover every page again."""
manager = make_hisparse_kv_cache_manager(32, 16)
coordinator = get_hisparse_coordinator(manager)
tokens = list(range(2 * HISPARSE_BLOCK_SIZE))
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
assert _allocate_scheduled(manager, request, len(tokens)) is not None
assert request.request_id in coordinator.take_residency_updates(
[request.request_id]
)
manager.free(request)
assert _allocate_scheduled(manager, request, len(tokens)) is not None
update = coordinator.take_residency_updates([request.request_id])
assert update[request.request_id].pages == [0, 1]
assert update[request.request_id].block_ids[0] == _resident_block_ids(
manager, request.request_id
)
def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
@@ -416,9 +476,7 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
source, indexer, resident, hot = manager.get_blocks(resumed.request_id).blocks
assert len(source) == len(indexer) == len(resident) == 4
assert len(hot) == 2
assert not any(block.is_null for block in resident[:2])
assert not resident[2].is_null
assert not resident[3].is_null
assert 0 not in _resident_block_ids(manager, resumed.request_id)
coordinator = get_hisparse_coordinator(manager)
assert not coordinator.build_offload_command().page_transfers
coordinator.finish_host_import(resumed.request_id, failed=False)
@@ -666,9 +724,10 @@ def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_cach
pool.free_blocks(held)
assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None
assert all(m.has_hot("first") for m in coordinator.hot_managers)
coordinator.take_residency_updates(["first"])
pool.get_new_blocks(pool.get_num_free_blocks())
assert not coordinator.all_context_pages_resident(scheduled)
assert "first" in coordinator.take_block_table_updates()
assert "first" in coordinator.take_residency_updates(["first"])
def test_hisparse_full_pool_keeps_pages_pinned_until_preemption():
@@ -1204,11 +1263,7 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages():
num_new_computed_tokens=num_computed,
new_computed_blocks=computed,
)
resident_blocks = manager.get_blocks("resumed").blocks[2]
assert [block.block_id for block in resident_blocks[:3]] == (
original_resident_ids[:3]
)
assert not any(block.is_null for block in resident_blocks[:3])
assert _resident_block_ids(manager, "resumed")[:3] == original_resident_ids[:3]
assert get_hisparse_coordinator(manager).all_context_pages_resident(
((resumed.request_id, num_computed, len(tokens) - num_computed),)
)
@@ -1249,17 +1304,13 @@ def test_hisparse_prefix_hit_under_pressure_adopts_surviving_copies(
new_computed_blocks=computed,
)
resumed_blocks = manager.get_blocks("resumed").blocks
assert [block.block_id for block in resumed_blocks[2][:3]] == [
resident_ids = _resident_block_ids(manager, "resumed")
assert resident_ids[:3] == [
copy_ids[page] if page in adopted_pages else pool.null_block.block_id
for page in range(3)
]
gpu_ids = [
block.block_id
for group in resumed_blocks[1:]
for block in group
if not block.is_null
]
_, indexer, _, hot = manager.get_block_ids("resumed")
gpu_ids = [block_id for block_id in indexer + resident_ids + hot if block_id]
assert len(gpu_ids) == len(set(gpu_ids))
@@ -1488,8 +1539,7 @@ def test_hisparse_reused_copy_is_not_adopted():
num_new_computed_tokens=num_computed,
new_computed_blocks=computed,
)
resident_blocks = manager.get_blocks("resumed").blocks[2]
assert all(block.is_null for block in resident_blocks[:3])
assert _resident_block_ids(manager, "resumed")[:3] == [0, 0, 0]
def test_hisparse_external_import_uses_hard_gpu_footprint():
@@ -1567,6 +1617,7 @@ def make_kv_cache_config_hybrid_model(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
return KVCacheConfig(
@@ -1602,6 +1653,7 @@ def make_kv_cache_config_three_types(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
elif third_spec_type == "sliding_window":
third_spec = SlidingWindowSpec(
@@ -1682,6 +1734,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
@@ -1707,10 +1760,13 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
req0 = make_request("0", common_token_ids + [99] * 7, block_size, hash_fn)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
req0, len(req0.prompt_token_ids), 0, computed_blocks
)
assert blocks is not None
# Align-mode Mamba keeps a state only where a chunk ends, so end the first
# chunk on the shared prefix.
for num_new_tokens in (len(common_token_ids), 7):
blocks = manager.allocate_slots(req0, num_new_tokens, 0, computed_blocks)
assert blocks is not None
req0.num_computed_tokens += num_new_tokens
computed_blocks = None
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
@@ -2235,7 +2291,7 @@ def _make_hybrid_kv_cache_config(
- "full": FullAttentionSpec
- "sliding_window": SlidingWindowSpec with window=2*block_size
- "sliding_window_large": SlidingWindowSpec with window=4*block_size
- "mamba": MambaSpec
- "mamba": MambaSpec in prefix-caching ("align") mode
"""
spec_map = {
@@ -2263,11 +2319,6 @@ def _make_hybrid_kv_cache_config(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
),
"mamba_align": lambda: MambaSpec(
block_size=block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
}
@@ -2377,14 +2428,16 @@ def test_prefill_hybrid_model_combinations(spec_types: list[str]):
assert not computed_blocks.blocks[0] # No cache hit initially
assert num_computed_tokens == 0
blocks = manager.allocate_slots(
req0, 55, len(computed_blocks.blocks[0]) * block_size, computed_blocks
)
assert blocks is not None
# Should have blocks for all groups
assert len(blocks.get_block_ids()) == num_groups
manager.new_step_starts()
# Align-mode Mamba keeps a state only where a chunk ends, so end the first
# chunk on the common prefix.
for num_new_tokens in (len(common_token_ids), len(unique_token_ids)):
blocks = manager.allocate_slots(req0, num_new_tokens, 0, computed_blocks)
assert blocks is not None
# Should have blocks for all groups
assert len(blocks.get_block_ids()) == num_groups
req0.num_computed_tokens += num_new_tokens
computed_blocks = None
manager.new_step_starts()
# Second request: should hit cached blocks for common prefix
req1 = make_request("1", common_token_ids + [4] * 5, block_size, hash_fn)
@@ -2499,7 +2552,7 @@ def test_prefill_hybrid_model_mamba_align():
num_blocks = 30
kv_cache_config = _make_hybrid_kv_cache_config(
block_size, num_blocks, ["full", "mamba_align"]
block_size, num_blocks, ["full", "mamba"]
)
manager = make_kv_cache_manager(
kv_cache_config,
@@ -2535,7 +2588,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
"""
block_size = 16
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 30, ["full", "mamba_align"]),
_make_hybrid_kv_cache_config(block_size, 30, ["full", "mamba"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
@@ -2610,7 +2663,7 @@ def test_hybrid_model_mamba_align_with_dynamic_draft_tokens():
num_blocks = 30
kv_cache_config = _make_hybrid_kv_cache_config(
block_size, num_blocks, ["full", "mamba_align"]
block_size, num_blocks, ["full", "mamba"]
)
manager = KVCacheManager(
kv_cache_config,
@@ -5846,49 +5899,6 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
assert retained(0, None) == {14}
def test_mamba_shared_prefix_survives_zero_retention():
"""Manager-level check of the full wiring: a pinned shared-prefix boundary
(``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style
detection) keeps its Mamba state block cached under
``prefix_cache_retention_interval=0``, which otherwise retains only the
end-of-prompt replay boundary. Without this, a shared prefix (junction
before ``num_prompt``) would be recomputed by every sharing request."""
block_size = 16
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
token_ids = [i for i in range(16) for _ in range(block_size)]
def cached_mamba_blocks(shared_prefix_boundary):
# Fresh manager per scenario so cached blocks don't leak between runs.
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 100, ["full", "mamba"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
retention_interval=0,
)
req = make_request("r", token_ids, block_size, sha256)
req.shared_prefix_boundary = shared_prefix_boundary
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
blocks = manager.allocate_slots(
req, len(token_ids), num_computed, computed_blocks
)
assert blocks is not None
pool = manager.block_pool
return {
i
for i in range(16)
if pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
is not None
}
# Without a pinned boundary, retention=0 keeps only the replay boundary (14).
assert cached_mamba_blocks(0) == {14}
# Pinning the shared prefix at token 96 (state block 5) retains it too, so a
# later request sharing that prefix can hit the Mamba state.
assert cached_mamba_blocks(96) == {5, 14}
def test_mamba_shared_prefix_reuse_under_zero_retention():
"""Full cross-request Marconi flow: a partial shared prefix cached by the
detecting request must stay reusable by a later request under
@@ -5899,7 +5909,7 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
def last_req_hit(retention, pin):
manager = make_kv_cache_manager(
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]),
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba"]),
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
+80 -12
View File
@@ -352,6 +352,12 @@ def test_async_scheduling_pp_allows_rescheduling_with_output_placeholders():
_ = scheduler.schedule()
assert req.num_output_placeholders > 0
if scheduler.use_v2_model_runner:
# MRV2 staggers same-request decodes by pp_size steps to match the
# worker-side sampled-token broadcast cadence.
output = scheduler.schedule()
assert req.request_id not in output.num_scheduled_tokens
# before any update_from_output, we still expect the request can be
# scheduled again (multi-step in-flight).
output = scheduler.schedule()
@@ -735,7 +741,8 @@ def test_same_step_duplicate_encoder_input_stays_cached():
assert "image" not in cache.freeable
def test_no_mm_input_chunking():
@pytest.mark.parametrize("warm_encoder_cache", [False, True])
def test_no_mm_input_chunking(warm_encoder_cache: bool):
# Disable multimodal input chunking.
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
@@ -747,6 +754,16 @@ def test_no_mm_input_chunking():
requests = create_requests(
num_requests=1, num_tokens=1200, mm_positions=mm_positions
)
if warm_encoder_cache:
cached_request = create_requests(
num_requests=1,
num_tokens=1200,
mm_positions=mm_positions,
req_ids=["cache-primer"],
)[0]
scheduler.encoder_cache_manager.allocate(cached_request, 0)
scheduler.encoder_cache_manager.free(cached_request)
for request in requests:
scheduler.add_request(request)
@@ -774,6 +791,8 @@ def test_no_mm_input_chunking():
assert output.scheduled_cached_reqs.num_reqs == 1
assert len(output.finished_req_ids) == 0
assert output.num_scheduled_tokens[requests[0].request_id] == 800
if warm_encoder_cache:
assert not output.scheduled_encoder_inputs
# Test that we fail if we disable chunked mm input and use too small
# of a max_num_batched_tokens for the mm input.
@@ -785,6 +804,66 @@ def test_no_mm_input_chunking():
)
def test_no_mm_input_chunking_for_same_step_duplicate():
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
max_num_batched_tokens=1024,
disable_chunked_mm_input=True,
max_model_len=2048,
)
request = create_requests(
num_requests=1,
num_tokens=1352,
mm_hashes_list=[["image", "image"]],
mm_positions=[
[
PlaceholderRange(offset=100, length=576),
PlaceholderRange(offset=776, length=576),
]
],
)[0]
scheduler.add_request(request)
output = scheduler.schedule()
assert output.num_scheduled_tokens[request.request_id] == 776
assert output.scheduled_encoder_inputs == {request.request_id: [0]}
def test_no_mm_input_chunking_at_item_boundary():
"""A chunk starting exactly at a multimodal item must not split it.
Regression test for issue #52306: the rollback guard only fired while
``num_computed_tokens < start_pos``, so a chunk beginning exactly at the
item skipped the check and could cover only part of it - silently breaking
models with bidirectional attention over the item.
"""
mm_positions = [[PlaceholderRange(offset=400, length=800)]]
def ask(num_computed_tokens: int, num_new_tokens: int) -> int:
# A fresh scheduler per call: the encoder cache is stateful.
scheduler = create_scheduler(
model="llava-hf/llava-1.5-7b-hf",
max_num_batched_tokens=1024,
disable_chunked_mm_input=True,
max_model_len=2048,
)
request = create_requests(
num_requests=1, num_tokens=1200, mm_positions=mm_positions
)[0]
return scheduler._try_schedule_encoder_inputs(
request, num_computed_tokens, num_new_tokens, encoder_compute_budget=100_000
)[1]
# Starts before the item and cannot reach past it: stop just before it.
assert ask(0, 500) == 400
# Starts exactly at the item and cannot cover it: schedule nothing rather
# than half an item.
assert ask(400, 500) == 0
# Starts exactly at the item and covers it: schedule it.
assert ask(400, 800) == 800
@pytest.mark.parametrize("enable_prefix_caching", [True, False])
def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
"""Test scheduling behavior with concurrent partial requests.
@@ -929,7 +1008,6 @@ def test_update_from_output_routes_sampling_masks_by_request():
total_num_scheduled_tokens=3,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -973,7 +1051,6 @@ def test_update_from_output_routes_multi_position_sampling_masks():
total_num_scheduled_tokens=2,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -1021,7 +1098,6 @@ def test_stop_via_update_from_output():
requests[0].request_id: [],
requests[1].request_id: [10],
},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -1067,7 +1143,6 @@ def test_stop_via_update_from_output():
requests[0].request_id: [10, 42],
requests[1].request_id: [13],
},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -1111,7 +1186,6 @@ def test_stop_via_update_from_output():
requests[0].request_id: [10, 11],
requests[1].request_id: [],
},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -1149,7 +1223,6 @@ def test_stop_via_update_from_output():
total_num_scheduled_tokens=3,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={requests[0].request_id: [EOS_TOKEN_ID, 10]},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -4200,7 +4273,6 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
total_num_scheduled_tokens=1,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -5675,7 +5747,6 @@ def test_delayed_kv_connector_free_keeps_scheduler_active():
total_num_scheduled_tokens=0,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -5757,7 +5828,6 @@ def test_scheduler_kv_connector_stats():
total_num_scheduled_tokens=1,
scheduled_spec_decode_tokens={},
scheduled_encoder_inputs={},
num_common_prefix_blocks=[0],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -5838,7 +5908,6 @@ def test_scheduler_ec_connector_stats():
total_num_scheduled_tokens=1,
scheduled_spec_decode_tokens={},
scheduled_encoder_inputs={},
num_common_prefix_blocks=[0],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -5870,7 +5939,6 @@ def test_ec_connector_update_connector_output_called():
total_num_scheduled_tokens=0,
scheduled_encoder_inputs={},
scheduled_spec_decode_tokens={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
)
@@ -20,7 +20,6 @@ from vllm.v1.core.single_type_kv_cache_manager import (
ChunkedLocalAttentionManager,
CircularBufferManager,
FullAttentionManager,
KpoolTailManager,
MambaManager,
RSWAManager,
SlidingWindowManager,
@@ -29,7 +28,6 @@ from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
MambaSpec,
RSWASpec,
SlidingWindowSpec,
@@ -399,22 +397,21 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
manager.cache_blocks(request_id, 1024, replay_boundaries=(0,))
manager.remove_skipped_blocks(request_id, 1024)
assert manager.get_num_common_prefix_blocks(request_id) == 0
assert manager.get_num_skipped_tokens(1024) == 0
assert manager.req_to_blocks[request_id] == blocks
@pytest.mark.parametrize("record_for_zeroing", [True, False])
def test_external_kpool_tail_zeroing(record_for_zeroing):
"""A transferred ring at index zero skips zeroing even with a local hit."""
spec = KpoolTailSpec(
"""The kpool tail ring is never zeroed, transferred or not: a pool only
reads ring slots its own request wrote."""
spec = CircularBufferSpec(
block_size=4,
num_kv_heads=2,
head_size=8,
dtype=torch.bfloat16,
sliding_window=4,
)
manager = KpoolTailManager(
manager = CircularBufferManager(
spec,
block_pool=BlockPool(10, enable_caching=True, hash_block_size=4),
enable_caching=True,
@@ -428,9 +425,7 @@ def test_external_kpool_tail_zeroing(record_for_zeroing):
)
blocks = manager.req_to_blocks["request"]
assert len(blocks) == 1
assert manager.take_new_block_ids() == (
[blocks[0].block_id] if record_for_zeroing else []
)
assert manager.take_new_block_ids() == []
assert manager.allocate_new_blocks("request", 20, 20) == []
assert manager.take_new_block_ids() == []
+75 -1
View File
@@ -36,6 +36,10 @@ from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
KVQuantMode,
MambaSpec,
MLAAttentionSpec,
SlidingWindowSpec,
)
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
@@ -238,7 +242,7 @@ def create_scheduler(
structured_output_manager=StructuredOutputManager(vllm_config),
)
if use_v2_model_runner is None:
use_v2_model_runner = bool(envs.VLLM_USE_V2_MODEL_RUNNER)
use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER is not False
scheduler.use_v2_model_runner = use_v2_model_runner
return scheduler
@@ -333,3 +337,73 @@ def create_requests(
)
requests.append(request)
return requests
def new_kv_cache_spec(
block_size=16,
num_kv_heads=2,
head_size=64,
dtype=torch.float32,
page_size_padded=None,
sliding_window=None,
attention_chunk_size=None,
kv_quant_mode=KVQuantMode.NONE,
):
return FullAttentionSpec(
block_size=block_size,
num_kv_heads=num_kv_heads,
head_size=head_size,
dtype=dtype,
page_size_padded=page_size_padded,
sliding_window=sliding_window,
attention_chunk_size=attention_chunk_size,
kv_quant_mode=kv_quant_mode,
)
def new_sliding_window_spec(
block_size=16,
num_kv_heads=2,
head_size=64,
dtype=torch.float32,
page_size_padded=None,
sliding_window=1,
):
return SlidingWindowSpec(
block_size=block_size,
num_kv_heads=num_kv_heads,
head_size=head_size,
dtype=dtype,
page_size_padded=page_size_padded,
sliding_window=sliding_window,
)
def new_mamba_spec(
block_size=16,
shapes=((2, 512), (3, 32, 32)),
dtypes=(torch.float32, torch.float32),
num_speculative_blocks=2,
mamba_cache_mode="none",
page_size_padded=None,
):
return MambaSpec(
block_size=block_size,
shapes=shapes,
dtypes=dtypes,
page_size_padded=page_size_padded,
mamba_cache_mode=mamba_cache_mode,
num_speculative_blocks=num_speculative_blocks,
)
def new_mla_spec(cache_dtype_str=None, block_size: int = 16):
# head_size = kv_lora_rank(512) + qk_rope_head_dim(64) = 576
return MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
max_tp_shards=1,
head_size=576,
dtype=torch.float32,
cache_dtype_str=cache_dtype_str,
)
+29 -17
View File
@@ -11,6 +11,18 @@ from unittest.mock import MagicMock, patch
import pytest
import torch
from vllm.platforms import current_platform
DEVICE_TYPE = current_platform.device_type
HAS_DEVICE_GRAPH = current_platform.is_cuda_alike() or current_platform.is_xpu()
if current_platform.is_xpu():
from vllm.v1.worker.xpu_model_runner import _torch_cuda_wrapper
# Route torch.cuda graph/stream APIs to torch.xpu, as XPUModelRunner does.
with _torch_cuda_wrapper():
pass
@pytest.fixture(autouse=True)
def _enable_breakable_cudagraph(monkeypatch: pytest.MonkeyPatch):
@@ -96,8 +108,8 @@ def cuda_capture_stream():
stream, so all capture-using tests need to run under
``torch.cuda.stream(...)`` for a separate stream.
"""
if not torch.cuda.is_available():
pytest.skip("CUDA required")
if not HAS_DEVICE_GRAPH:
pytest.skip("CUDA or XPU graph required")
from vllm.utils.torch_utils import _current_stream_tls
prev_stream = getattr(_current_stream_tls, "value", None)
@@ -200,7 +212,7 @@ def test_active_state_isolated_across_threads(cuda_capture_stream):
def test_capture_with_no_eager_break_records_one_graph(cuda_capture_stream):
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
x = torch.zeros(4, device="cuda")
x = torch.zeros(4, device=DEVICE_TYPE)
cap = BreakableCUDAGraphCapture()
with cap:
x.add_(1.0)
@@ -212,7 +224,7 @@ def test_capture_with_no_eager_break_records_one_graph(cuda_capture_stream):
def test_add_eager_creates_alternating_graph_eager_graph(cuda_capture_stream):
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
x = torch.zeros(4, device="cuda")
x = torch.zeros(4, device=DEVICE_TYPE)
counter = {"eager_calls": 0}
def eager_step():
@@ -253,7 +265,7 @@ def test_capture_replay_matches_eager_simple(cuda_capture_stream):
"""
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
x = torch.zeros(8, device="cuda")
x = torch.zeros(8, device=DEVICE_TYPE)
log: list[str] = []
def eager_break_op():
@@ -269,21 +281,21 @@ def test_capture_replay_matches_eager_simple(cuda_capture_stream):
# Capture-time: graph kernels were recorded only; eager segment ran
# once on x == 0, leaving x == 0.
torch.accelerator.synchronize()
assert torch.equal(x, torch.zeros(8, device="cuda"))
assert torch.equal(x, torch.zeros(8, device=DEVICE_TYPE))
assert log == ["eager"]
# Replay with a fresh input: 10 -> 11 -> 22 -> 27.
x.fill_(10.0)
cap.replay()
torch.accelerator.synchronize()
assert torch.equal(x, torch.full((8,), 27.0, device="cuda"))
assert torch.equal(x, torch.full((8,), 27.0, device=DEVICE_TYPE))
assert log == ["eager", "eager"]
# Replay again with another input: 100 -> 101 -> 202 -> 207.
x.fill_(100.0)
cap.replay()
torch.accelerator.synchronize()
assert torch.equal(x, torch.full((8,), 207.0, device="cuda"))
assert torch.equal(x, torch.full((8,), 207.0, device=DEVICE_TYPE))
assert log == ["eager", "eager", "eager"]
@@ -301,13 +313,13 @@ def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
# In-place double; stands in for "real" attention work.
t.mul_(2.0)
x = torch.zeros(4, device="cuda")
x = torch.zeros(4, device=DEVICE_TYPE)
# Outside capture: decorator should just call through.
x.fill_(3.0)
attention_like(x)
torch.accelerator.synchronize()
assert torch.equal(x, torch.full((4,), 6.0, device="cuda"))
assert torch.equal(x, torch.full((4,), 6.0, device=DEVICE_TYPE))
# Inside capture: decorator should split the graph. Only the eager
# segment actually mutates state during capture.
@@ -318,7 +330,7 @@ def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
attention_like(x) # eager: x *= 2 (on x == 0, no-op)
x.add_(1.0) # recorded
torch.accelerator.synchronize()
assert torch.equal(x, torch.zeros(4, device="cuda"))
assert torch.equal(x, torch.zeros(4, device=DEVICE_TYPE))
# 2 graph segments + 1 eager segment, ordered G E G; the arithmetic
# equivalence check below verifies the ordering.
assert len(cap.segments) == 3
@@ -329,7 +341,7 @@ def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
x.fill_(2.0)
cap.replay()
torch.accelerator.synchronize()
assert torch.equal(x, torch.full((4,), 15.0, device="cuda"))
assert torch.equal(x, torch.full((4,), 15.0, device=DEVICE_TYPE))
@pytest.mark.parametrize("keyword", [False, True])
@@ -352,7 +364,7 @@ def test_quantized_activation_replay_does_not_retain_tensors(
data_out.copy_(x.data)
scale_out.copy_(x.scale)
source = torch.ones((128, 128), device="cuda", dtype=torch.bfloat16)
source = torch.ones((128, 128), device=DEVICE_TYPE, dtype=torch.bfloat16)
data_out = source.to(torch.float8_e4m3fn)
scale_out = source[:, ::32].to(torch.uint8)
output = torch.empty_like(source)
@@ -391,7 +403,7 @@ def test_eager_attention_inside_multistream_overlap(cuda_capture_stream):
)
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
x = torch.zeros(1024, device="cuda")
x = torch.zeros(1024, device=DEVICE_TYPE)
output = torch.empty_like(x)
aux_stream = torch.cuda.Stream()
main_event = torch.cuda.Event()
@@ -440,7 +452,7 @@ def test_replay_invokes_eager_segments_in_order(cuda_capture_stream):
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
log: list[str] = []
x = torch.zeros(1, device="cuda")
x = torch.zeros(1, device=DEVICE_TYPE)
def make_eager(name):
def step():
@@ -502,7 +514,7 @@ def test_nested_decorated_op_runs_inline(cuda_capture_stream):
eager_break_during_capture,
)
x = torch.zeros(4, device="cuda")
x = torch.zeros(4, device=DEVICE_TYPE)
inner_calls = 0
@eager_break_during_capture
@@ -533,7 +545,7 @@ def test_nested_decorated_op_runs_inline(cuda_capture_stream):
cap.replay()
torch.accelerator.synchronize()
# 0 -> +2 -> +1 (inner) -> +10 (outer) -> +100 = 113
assert torch.equal(x, torch.full((4,), 113.0, device="cuda"))
assert torch.equal(x, torch.full((4,), 113.0, device=DEVICE_TYPE))
assert inner_calls == 2 # replay invokes the outer's lambda again
+12 -4
View File
@@ -24,6 +24,14 @@ from vllm.platforms import current_platform
from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher
DEVICE_TYPE = current_platform.device_type
HAS_DEVICE_GRAPH = current_platform.is_cuda() or current_platform.is_xpu()
if current_platform.is_xpu():
from vllm.v1.worker.xpu_model_runner import _torch_cuda_wrapper
# Route torch.cuda graph/stream APIs to torch.xpu, as XPUModelRunner does.
with _torch_cuda_wrapper():
pass
# Helper MLP for testing
@@ -50,6 +58,7 @@ def _create_vllm_config(
mock_config.parallel_config = ParallelConfig()
mock_config.speculative_config = None # No speculative decoding
mock_config.num_speculative_tokens = 0
mock_config.use_cumem_cudagraph_pool = False
if not lora_config:
mock_config.lora_config = None
else:
@@ -180,7 +189,6 @@ class TestCudagraphDispatcher:
assert key == BatchDescriptor(num_tokens=15)
# 4. invalid_modes={FULL} should have a fall back mode
# (e.g., cascade attention)
desc_full_exact = BatchDescriptor(num_tokens=8, uniform=False)
rt_mode, key = dispatcher.dispatch(
num_tokens=8,
@@ -268,7 +276,7 @@ class TestCudagraphDispatcher:
assert dispatcher.get_capture_descs() == []
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
@pytest.mark.skipif(not HAS_DEVICE_GRAPH, reason="Requires CUDA or XPU graph")
class TestCUDAGraphWrapper:
def setup_method(self):
self.vllm_config = _create_vllm_config(CompilationConfig())
@@ -415,7 +423,7 @@ def _run_and_monitor_call(
@create_new_process_for_each_test("spawn")
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
@pytest.mark.skipif(not HAS_DEVICE_GRAPH, reason="Requires CUDA or XPU graph")
def test_capture_replay_bypass_logic():
comp_config = CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
@@ -484,7 +492,7 @@ def test_capture_replay_bypass_logic():
@create_new_process_for_each_test("spawn")
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
@pytest.mark.skipif(not HAS_DEVICE_GRAPH, reason="Requires CUDA or XPU graph")
def test_nested_wrappers():
"""Tests a scenario with a PIECEWISE wrapper inside a FULL one."""
comp_config = CompilationConfig(
@@ -132,6 +132,7 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings():
pcp_block_tables = SimpleNamespace(
num_kv_cache_groups=1,
input_block_tables=(torch.zeros(num_reqs * 2, 1, dtype=torch.int32),),
cp_interleave=1,
)
pcp_manager = PCPManager(
pcp_world_size=pcp_world_size,
@@ -174,6 +175,52 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings():
block_tables.get_dummy_slot_mappings.assert_not_called()
@pytest.mark.parametrize("full_cudagraph,expected_context_len", [(True, 2), (False, 0)])
def test_dcp_full_capture_has_kv_context(
monkeypatch, full_cudagraph, expected_context_len
):
"""DCP skips context attention on the host when no request has context, so a
FULL graph captured without context would ignore the KV cache on replay."""
monkeypatch.setattr(
gpu_cudagraph_utils, "prepare_dcp_local_seq_lens", lambda *a, **kw: None
)
num_tokens = num_reqs = 8
buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu"))
block_tables = MagicMock()
block_tables.cp_size = 2
block_tables.cp_interleave = 1
block_tables.block_sizes = [16]
block_tables.get_dummy_block_tables.return_value = (
torch.zeros(num_reqs, 4, dtype=torch.int32),
)
block_tables.get_dummy_slot_mappings.return_value = torch.empty(
1, num_tokens, dtype=torch.int64
)
model_state = MagicMock()
model_state.max_model_len = 1024
kv_cache_config = KVCacheConfig(
num_blocks=8, kv_cache_tensors=[], kv_cache_groups=[]
)
gpu_cudagraph_utils.prepare_inputs_to_capture(
num_reqs,
num_tokens,
model_state,
buffers,
block_tables,
[],
kv_cache_config,
full_cudagraph=full_cudagraph,
max_query_len=1,
)
batch = model_state.prepare_attn.call_args.args[0]
context_lens = batch.seq_lens_cpu_upper_bound[:num_reqs] - torch.from_numpy(
batch.num_scheduled_tokens[:num_reqs]
)
assert (context_lens == expected_context_len).all()
@pytest.mark.parametrize(
"full_cudagraph,max_query_len,expected_max_query_len",
[(False, None, None), (True, None, 8), (True, 1, 1), (True, 3, 3)],
@@ -19,6 +19,13 @@ if current_platform.is_rocm():
("TritonAttn", "FULL", True),
("TritonAttn", "FULL_AND_PIECEWISE", True),
]
elif current_platform.is_xpu():
combo_cases_1 = [
("FA2", "FULL", True),
("FA2", "FULL_AND_PIECEWISE", True),
("TritonAttn", "FULL", True),
("TritonAttn", "FULL_AND_PIECEWISE", True),
]
else:
combo_cases_1 = [
("FA3", "FULL", True),
+15 -12
View File
@@ -32,6 +32,15 @@ from vllm.v1.worker.encoder_cudagraph_defs import (
EncoderItemSpec,
)
HAS_DEVICE_GRAPH = current_platform.is_cuda_alike() or current_platform.is_xpu()
if current_platform.is_xpu():
from vllm.v1.worker.xpu_model_runner import _torch_cuda_wrapper
# Route torch.cuda graph/stream APIs to torch.xpu, as XPUModelRunner does.
with _torch_cuda_wrapper():
pass
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
@@ -662,12 +671,10 @@ def _make_video_mm_kwargs(
# ---------------------------------------------------------------------------
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
)
@pytest.mark.skipif(not HAS_DEVICE_GRAPH, reason="Requires CUDA, ROCm or XPU graph")
class TestEncoderCudaGraphCaptureReplay:
def setup_method(self):
self.device = torch.device("cuda:0")
self.device = torch.device(current_platform.device_type, 0)
self.dtype = torch.float16
self.model = SimpleMockViTModel().to(self.device).half()
self.mgr = _make_manager_for_gpu(
@@ -758,9 +765,7 @@ class TestEncoderCudaGraphCaptureReplay:
# ---------------------------------------------------------------------------
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
)
@pytest.mark.skipif(not HAS_DEVICE_GRAPH, reason="Requires CUDA, ROCm or XPU graph")
@pytest.mark.usefixtures("dist_init", "workspace_init")
@pytest.mark.parametrize("profile_only", [False, True])
@torch.inference_mode()
@@ -776,7 +781,7 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
from vllm.v1.worker.workspace import lock_workspace
device = torch.device("cuda:0")
device = torch.device(current_platform.device_type, 0)
dtype = torch.float16
model = SimpleMockViTModel().to(device).half()
manager = _make_manager_for_gpu(model, _BUDGETS, _MAX_BATCH, device, dtype)
@@ -1047,12 +1052,10 @@ _VIDEO_MAX_BATCH = 4
_VIDEO_MAX_FRAMES = 8 # 2 frames per item at max_batch_size=4
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
)
@pytest.mark.skipif(not HAS_DEVICE_GRAPH, reason="Requires CUDA, ROCm or XPU graph")
class TestEncoderCudaGraphVideoReplay:
def setup_method(self):
self.device = torch.device("cuda:0")
self.device = torch.device(current_platform.device_type, 0)
self.dtype = torch.float16
self.model = SimpleMockViTVideoModel().to(self.device).half()
self.mgr = _make_manager_for_gpu(
@@ -180,6 +180,7 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
llm = LLM(
model=TEST_MODEL,
tensor_parallel_size=tp_size,
disable_custom_all_reduce=os.getenv("VLLM_TEST_DISABLE_CUSTOM_AR") == "1",
max_num_seqs=128,
max_model_len=8192,
dtype="auto", # not everything is supported
@@ -9,7 +9,7 @@ import numpy as np
import pytest
import torch
from PIL import Image
from utils import _extract_step_logprobs, skip_if_not_cuda
from utils import _extract_step_logprobs, skip_if_not_cuda_alike
from vllm import LLM, SamplingParams
from vllm.inputs import TextPrompt
@@ -89,7 +89,7 @@ def _assert_batch_invariant(llm: LLM, inputs: list[TextPrompt], sampling) -> Non
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("input_type", ["image", "video"])
@pytest.mark.parametrize("mm_encoder_attn_backend", ["FLASH_ATTN", "TORCH_SDPA"])
def test_vlm_batch_invariance_bs1_vs_bsN(
@@ -101,7 +101,7 @@ def test_vlm_batch_invariance_bs1_vs_bsN(
_run_vlm_batch_invariance(input_type, mm_encoder_attn_backend, enforce_eager=True)
@skip_if_not_cuda
@skip_if_not_cuda_alike
def test_vlm_batch_invariance_default_execution(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
_run_vlm_batch_invariance("image", "FLASH_ATTN", enforce_eager=False)
@@ -19,8 +19,10 @@ from tests.kernels.moe.utils import make_dummy_moe_config, make_test_quant_confi
from tests.kernels.quantization.nvfp4_utils import get_nvfp4_global_scale
from tests.utils import TestFP8Layer, requires_fp8
from vllm import _custom_ops as ops
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
from vllm.model_executor.kernels.linear import fp8_scaled_mm_is_batch_invariant
from vllm.model_executor.kernels.linear.scaled_mm import (
CutlassFP8ScaledMMLinearKernel,
TritonFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
@@ -31,8 +33,11 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
CutlassExpertsFp4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTensorSym,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_random_seed
@@ -87,16 +92,42 @@ _NVFP4_MOE_BATCH_INVARIANT_CASES = (
@pytest.fixture(autouse=True)
def setup_cuda():
if not current_platform.is_cuda():
pytest.skip("CUTLASS FP8 kernels require CUDA.")
if not current_platform.is_cuda_alike():
pytest.skip("Requires CUDA or ROCm.")
torch.set_default_device("cuda")
@requires_fp8
@pytest.mark.parametrize(
"kernel",
[
pytest.param(
CutlassFP8ScaledMMLinearKernel,
marks=pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUTLASS requires CUDA."
),
),
TritonFP8ScaledMMLinearKernel,
],
)
@pytest.mark.parametrize(
"act_key,weight_key",
[
(kFp8DynamicTokenSym, kFp8StaticTensorSym),
(kFp8DynamicTokenSym, kFp8StaticChannelSym),
(kFp8DynamicTokenSym, kFp8StaticTokenSym),
(kFp8StaticTensorSym, kFp8StaticTensorSym),
# An amax over the whole batch: the GEMM is invariant, the layer is not.
(kFp8DynamicTensorSym, kFp8StaticTensorSym),
],
)
@pytest.mark.parametrize("weight_shape", [(1024, 2048), (4608, 4096)])
@pytest.mark.parametrize("batch_size", [1, 16, 17, 32, 64, 65, 256, 257])
@torch.inference_mode()
def test_cutlass_fp8_batch_invariant_fixed_config(
def test_fp8_scaled_mm_batch_invariant_fixed_config(
kernel: type,
act_key,
weight_key,
weight_shape: tuple[int, int],
batch_size: int,
default_vllm_config,
@@ -108,14 +139,16 @@ def test_cutlass_fp8_batch_invariant_fixed_config(
torch.manual_seed(0)
layer = TestFP8Layer(
weight_shape=weight_shape,
activation_quant_key=kFp8DynamicTokenSym,
weight_quant_key=kFp8StaticTensorSym,
activation_quant_key=act_key,
weight_quant_key=weight_key,
input_dtype=torch.bfloat16,
out_dtype=torch.bfloat16,
device=torch.device("cuda"),
force_kernel=CutlassFP8ScaledMMLinearKernel,
force_kernel=kernel,
)
assert isinstance(layer.kernel, CutlassFP8ScaledMMLinearKernel)
assert isinstance(layer.kernel, kernel)
dynamic_per_tensor = act_key == kFp8DynamicTensorSym
assert fp8_scaled_mm_is_batch_invariant(layer.kernel) != dynamic_per_tensor
in_features = weight_shape[1]
needle = torch.randn((1, in_features), device="cuda", dtype=torch.bfloat16)
@@ -124,6 +157,12 @@ def test_cutlass_fp8_batch_invariant_fixed_config(
filler = torch.randn(
(max(batch_size - 1, 0), in_features), device="cuda", dtype=torch.bfloat16
)
if dynamic_per_tensor:
if batch_size == 1:
pytest.skip("no other rows to move the scale")
front_output = layer(torch.cat([needle, 100 * filler], dim=0))[0]
assert not torch.equal(front_output, baseline)
return
front_batch = torch.cat([needle, filler], dim=0)
back_batch = torch.cat([filler, needle], dim=0)
@@ -135,6 +174,69 @@ def test_cutlass_fp8_batch_invariant_fixed_config(
torch.testing.assert_close(back_output, baseline, rtol=0, atol=0)
@requires_fp8
@pytest.mark.parametrize("scheme", ["fp8_per_tensor", "fp8_per_channel"])
@torch.inference_mode()
def test_online_fp8_linear_batch_invariant(
scheme, monkeypatch, default_vllm_config, dist_init
):
from vllm.config.model import ModelConfig
from vllm.config.quantization import resolve_quantization_config
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
from vllm.utils.torch_utils import set_default_torch_dtype
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
default_vllm_config.model_config = ModelConfig(dtype="bfloat16")
quant_config = OnlineQuantizationConfig(resolve_quantization_config(scheme, None))
set_random_seed(0)
with set_default_torch_dtype(torch.bfloat16), torch.device("cuda"):
layer = ReplicatedLinear(
1024, 768, bias=False, quant_config=quant_config, prefix="mlp.down_proj"
)
layer.weight.weight_loader(layer.weight, torch.randn(768, 1024))
needle = torch.randn(1, 1024)
filler = 100 * torch.randn(63, 1024)
baseline = layer(needle)[0]
assert torch.equal(layer(torch.cat([needle, filler]))[0][:1], baseline)
assert torch.equal(layer(torch.cat([filler, needle]))[0][-1:], baseline)
@requires_fp8
@torch.inference_mode()
def test_block_fp8_batch_invariant_across_tuned_block_size_k(monkeypatch):
"""The block-FP8 kernel applies the block scales once per K tile, so a tuned
BLOCK_SIZE_K below the scale group size reorders the sum."""
from vllm.model_executor.layers.quantization.utils import fp8_utils
base = dict(
BLOCK_SIZE_M=64, BLOCK_SIZE_N=128, GROUP_SIZE_M=1, num_warps=4, num_stages=2
)
table = {256: {**base, "BLOCK_SIZE_K": 128}, 512: {**base, "BLOCK_SIZE_K": 64}}
monkeypatch.setattr(fp8_utils, "get_w8a8_block_fp8_configs", lambda *_: table)
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
N, K, block = 12288, 2048, [128, 128]
set_random_seed(0)
fp8 = current_platform.fp8_dtype()
A = torch.randn(512, K).to(fp8)
B = torch.randn(N, K).to(fp8)
As = torch.rand(512, K // 128) / 100
Bs = torch.rand(N // 128, K // 128) / 100
def run(m):
return fp8_utils.w8a8_triton_block_scaled_mm(
A[:m], B, As[:m], Bs, block, torch.bfloat16
)[:256]
assert torch.equal(run(256), run(512))
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", False)
assert not torch.equal(run(256), run(512))
@_NVFP4_REQUIRES_SM100
@pytest.mark.parametrize("dtype", _NVFP4_LINEAR_DTYPES)
@pytest.mark.parametrize("shape", _NVFP4_LINEAR_CONSISTENCY_SHAPES)
@@ -4,7 +4,7 @@
import pytest
import torch
from utils import skip_if_not_cuda
from utils import skip_if_not_cuda_alike
import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op
import vllm.lora.ops.triton_ops.utils as lora_utils
@@ -39,7 +39,7 @@ def _shrink(inputs, weights, mapping, scaling):
return out
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize(
"hidden_size,rank,nslices,dtype",
[
@@ -68,7 +68,7 @@ def test_lora_shrink_matches_reference(hidden_size, rank, nslices, dtype):
torch.testing.assert_close(out, ref, rtol=5e-3, atol=5e-3)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("nslices", [1, 3])
def test_lora_shrink_invariant_across_batch_sizes(monkeypatch, nslices):
reduce_kernel, reduce_grids = shrink_op._lora_shrink_reduce_kernel, []
@@ -10,7 +10,11 @@ import pytest
import torch
from utils import skip_unsupported
from vllm.model_executor.determinism.batch_invariant import matmul_batch_invariant
import vllm.model_executor.determinism.batch_invariant as bi
from vllm.model_executor.determinism.batch_invariant import (
enable_batch_invariant_mode,
matmul_batch_invariant,
)
from vllm.model_executor.determinism.batch_invariant_configs import (
_BATCH_INVARIANT_MATMUL_TUNED_CONFIGS,
_get_tuned_matmul_arch_family,
@@ -129,3 +133,49 @@ def test_matmul_batch_invariance_across_tuned_m_buckets(m, transpose_b):
batch_output = matmul_batch_invariant(a, b)
assert torch.equal(single_output[0], batch_output[0])
@pytest.fixture
def batch_invariant_mode():
# The overrides are process-global: remove them so later tests compare
# against the stock aten kernels again.
enable_batch_invariant_mode()
yield
assert bi._batch_invariant_LIB is not None
bi._batch_invariant_LIB._destroy()
bi._batch_invariant_MODE = False
@skip_unsupported
@pytest.mark.usefixtures("batch_invariant_mode")
@pytest.mark.parametrize("op", ["mm", "addmm", "bmm"])
def test_compiled_matmul_reaches_batch_invariant_out_overloads(op):
"""Inductor lowers mm/addmm/bmm to ``extern_kernels.<op>(..., out=buf)``,
i.e. the ``.out`` overload, which eager never dispatches."""
if (
op != "bmm"
and current_platform.is_cuda()
and not current_platform.is_device_capability_family(80)
):
pytest.skip("mm/addmm are not overridden on SM90+")
torch.manual_seed(0)
dtype, k, n, needle = torch.bfloat16, 1024, 768, 37
w = torch.randn(4, k, n, device=DEVICE_TYPE, dtype=dtype)
bias = torch.randn(n, device=DEVICE_TYPE, dtype=dtype)
fn = {
"mm": lambda x: torch.mm(x[0], w[0]),
# Bare addmm: a following pointwise op would make Inductor unfuse the bias.
"addmm": lambda x: torch.addmm(bias, x[0], w[0]),
"bmm": lambda x: torch.bmm(x, w),
}[op]
compiled = torch.compile(fn, fullgraph=True)
x = torch.randn(4, 256, k, device=DEVICE_TYPE, dtype=dtype)
row = None
for m in (64, 256, 128):
xs = x[:, :m].clone()
# With dynamic=True, a specialization on M would silently recompile.
torch._dynamo.mark_dynamic(xs, 1)
out = compiled(xs)
assert torch.equal(out, fn(xs)), f"compiled {op} bypassed the BI kernel (M={m})"
row = out[..., needle, :] if row is None else row
assert torch.equal(out[..., needle, :], row)
@@ -0,0 +1,165 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The ROCm native MX linear kernels must not depend on the number of rows."""
import pytest
import torch
from utils import requires_mx
from vllm.utils.torch_utils import set_random_seed
SEED = 0
# E8M0 exponent half-width. With 4-bit operands and a narrow spread the fp32
# partial sums are exact, so a reordering of them would be invisible.
SCALE_SPREAD = 15
# (N, K, use_asm_gemm)
MXFP4_CASES = [
(4096, 2048, False),
(2048, 6144, False),
(4608, 7168, False),
(8192, 2048, True),
(8192, 1024, True),
(8192, 14336, True),
]
MXFP8_CASES = [(4096, 2048), (2048, 6144), (1024, 768), (1536, 1024)]
# The untuned AITER MXFP4 configs switch at M = 9/33/65/129/257/513.
MXFP4_TOKEN_COUNTS = [
1,
2,
8,
9,
16,
32,
33,
64,
65,
128,
129,
256,
257,
512,
513,
1024,
2048,
]
MXFP8_TOKEN_COUNTS = [1, 32, 64, 65, 128, 129, 256, 257, 512, 1024, 1025, 2048]
# Row 0 lands in the first tile of every decomposition, so it can stay
# invariant when the rest of the output does not.
CHECK_ROWS = [0, 1, 2, 3, 7, 15, 31]
def _probe(out: torch.Tensor) -> torch.Tensor:
"""The leading rows of an output, kept so the sweep does not hold every GEMM."""
return out[: CHECK_ROWS[-1] + 1].clone()
def _variant_rows(probes: dict[int, torch.Tensor]) -> list[str]:
"""Rows whose contents changed with the number of rows in the launch."""
failures = []
for row in CHECK_ROWS:
counts = [n for n in sorted(probes) if n > row]
if not counts:
continue
reference = probes[counts[0]][row]
variant = [n for n in counts if not torch.equal(probes[n][row], reference)]
if variant:
failures.append(f"row {row} changed at row counts {variant}")
return failures
def _mxfp4_weights(n: int, k: int, use_asm_gemm: bool):
"""Weights laid out as AiterMxfp4LinearKernel.process_weights_after_loading."""
weight = torch.randint(0, 255, (n, k // 2), dtype=torch.uint8, device="cuda")
weight_scale = torch.randint(
127 - SCALE_SPREAD,
128 + SCALE_SPREAD,
(n, k // 32),
dtype=torch.uint8,
device="cuda",
)
if not use_asm_gemm:
return weight, weight_scale.T.contiguous()
from aiter.ops.shuffle import shuffle_weight
sm, sn = weight_scale.shape
weight_scale = (
weight_scale.view(sm // 32, 2, 16, sn // 8, 2, 4, 1)
.permute(0, 3, 5, 2, 4, 1, 6)
.contiguous()
.view(sm, sn)
)
return shuffle_weight(weight, layout=(16, 16)), weight_scale
def _mxfp8_weights(n: int, k: int):
"""E4M3 weights with E8M0 block scales, as the MXFP8 layer stores them."""
weight = (torch.randn(n, k, device="cuda") / 8).to(torch.float8_e4m3fn)
weight_scale = torch.randint(
127 - SCALE_SPREAD,
128 + SCALE_SPREAD,
(n, k // 32),
dtype=torch.uint8,
device="cuda",
)
return weight, weight_scale
@requires_mx
@pytest.mark.parametrize("n,k,use_asm_gemm", MXFP4_CASES)
def test_mxfp4_linear_is_batch_invariant(n: int, k: int, use_asm_gemm: bool):
pytest.importorskip("aiter")
# Importing the module registers torch.ops.vllm.gemm_with_dynamic_quant.
import vllm.model_executor.kernels.linear.mxfp4.aiter # noqa: F401
set_random_seed(SEED)
weight, weight_scale = _mxfp4_weights(n, k, use_asm_gemm)
x = torch.randn(max(MXFP4_TOKEN_COUNTS), k, device="cuda", dtype=torch.bfloat16)
probes = {
num_tokens: _probe(
torch.ops.vllm.gemm_with_dynamic_quant(
x[:num_tokens], weight, weight_scale, use_asm_gemm, torch.bfloat16
)
)
for num_tokens in MXFP4_TOKEN_COUNTS
}
failures = _variant_rows(probes)
assert not failures, (
f"MXFP4 linear depends on the row count "
f"(N={n}, K={k}, asm={use_asm_gemm}):\n " + "\n ".join(failures)
)
@requires_mx
@pytest.mark.parametrize("n,k", MXFP8_CASES)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_mxfp8_linear_is_batch_invariant(n: int, k: int, dtype: torch.dtype):
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
_mxfp8_dot_scaled_linear,
)
set_random_seed(SEED)
weight, weight_scale = _mxfp8_weights(n, k)
x = torch.randn(max(MXFP8_TOKEN_COUNTS), k, device="cuda", dtype=dtype)
probes = {
num_tokens: _probe(
_mxfp8_dot_scaled_linear(x[:num_tokens], weight, weight_scale)
)
for num_tokens in MXFP8_TOKEN_COUNTS
}
failures = _variant_rows(probes)
assert not failures, (
f"MXFP8 linear depends on the row count (N={n}, K={k}, {dtype}):\n "
+ "\n ".join(failures)
)
@@ -9,10 +9,14 @@ import torch
from utils import (
_extract_step_logprobs,
_random_prompt,
skip_if_not_cuda,
skip_if_not_cuda_alike,
)
from vllm import LLM, SamplingParams
from vllm.platforms import current_platform
# FLASH_ATTN is CUDA-only.
E2E_BACKENDS = ["TRITON_ATTN"] if current_platform.is_rocm() else ["FLASH_ATTN"]
pytestmark = pytest.mark.skipif(
not hasattr(torch, "float8_e4m3fn"),
@@ -40,8 +44,8 @@ def _make_llm(max_num_seqs: int, backend: str) -> LLM:
)
@skip_if_not_cuda
@pytest.mark.parametrize("backend", ["FLASH_ATTN"])
@skip_if_not_cuda_alike
@pytest.mark.parametrize("backend", E2E_BACKENDS)
def test_dense_nvfp4_generation_is_deterministic_across_batch_sizes_e2e(backend):
seed = int(os.getenv("VLLM_TEST_SEED", "12345"))
random.seed(seed)
@@ -6,6 +6,7 @@ vLLM server and compare BS=1 vs BS=N results (tokens and per-step logprobs).
Environment variables:
- VLLM_TEST_MODEL: served model name (e.g., Qwen/Qwen3-1.7B / DeepSeek-R1)
- VLLM_TP_SIZE: tensor parallelism size (e.g., 4)
- VLLM_TEST_SERVER_ARGS: extra server args (e.g., "--data-parallel-size 4")
"""
@@ -16,7 +17,7 @@ from typing import Any
import openai
import pytest
from utils import BACKENDS, TEST_MODEL, _random_prompt, skip_if_not_cuda
from utils import BACKENDS, TEST_MODEL, _random_prompt, skip_if_not_cuda_alike
from tests.utils import RemoteOpenAIServer
@@ -132,7 +133,7 @@ def _compare_bs1_vs_bsn_single_process(
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("backend", BACKENDS)
def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
backend: str,
@@ -156,6 +157,7 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
]
if tp_size:
server_args += ["-tp", tp_size]
server_args += os.getenv("VLLM_TEST_SERVER_ARGS", "").split()
with RemoteOpenAIServer(TEST_MODEL, server_args) as server:
client = server.get_client()
@@ -4,7 +4,7 @@
import pytest
import torch
from utils import skip_if_not_cuda, skip_unsupported
from utils import skip_if_not_cuda_alike, skip_unsupported
from vllm.model_executor.determinism.batch_invariant import (
rms_norm_batch_invariant,
@@ -27,7 +27,7 @@ def _rms_norm_reference(
return (output * weight.float()).to(input_tensor.dtype)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("batch_size", [1, 4, 64, 300])
@pytest.mark.parametrize("hidden_size", [512, 2048, 4096, 8192])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@@ -76,7 +76,7 @@ def test_rms_norm_batch_invariant_vs_reference(
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("hidden_size", [512, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("eps", [1e-6])
@@ -189,7 +189,7 @@ def _assert_rows_bit_identical(small, large, msg):
torch.testing.assert_close(small, large, rtol=0.0, atol=0.0, msg=msg)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("hidden_size", [512, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("seed", list(range(4)))
@@ -221,7 +221,7 @@ def test_rms_norm_batch_invariant_nonresidual_kernel(
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("hidden_size", [512, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("seed", list(range(4)))
@@ -267,7 +267,7 @@ def test_rms_norm_static_fp8_quant_batch_invariant(
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("hidden_size", [512, 4096])
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("seed", list(range(4)))
@@ -303,7 +303,7 @@ def test_rms_norm_per_block_quant_batch_invariant(
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("batch_size", [1, 16, 128])
@pytest.mark.parametrize("seq_len", [1, 32, 512])
@pytest.mark.parametrize("hidden_size", [2048, 4096])
@@ -343,7 +343,7 @@ def test_rms_norm_3d_input(
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
def test_rms_norm_numerical_stability(default_vllm_config):
"""Test RMS norm numerical stability with extreme values.
@@ -431,7 +431,7 @@ def test_rms_norm_formula(default_vllm_config):
)
@skip_if_not_cuda
@skip_if_not_cuda_alike
@pytest.mark.parametrize("hidden_size", [128, 1024, 4096, 16384])
def test_rms_norm_different_hidden_sizes(default_vllm_config, hidden_size: int):
"""Test RMS norm with various hidden sizes to ensure block size handling.
+20 -2
View File
@@ -34,12 +34,16 @@ DEVICE_BACKENDS: dict[str, DeviceConfig] = {
available=current_platform.is_xpu() and HAS_TRITON,
backends=["TRITON_ATTN"],
),
"rocm": DeviceConfig(
available=current_platform.is_rocm() and HAS_TRITON,
backends=["TRITON_ATTN"],
),
}
DEFAULT_MODEL = "Qwen/Qwen3-1.7B"
TEST_MODEL = os.getenv("VLLM_TEST_MODEL", DEFAULT_MODEL)
# Override backends for MLA models (MLA only supported on CUDA).
# Override backends for MLA models.
if os.getenv("VLLM_TEST_MODEL"):
config = get_config(TEST_MODEL, trust_remote_code=False)
if ModelArchConfigConvertorBase(config, config.get_text_config()).is_deepseek_mla():
@@ -52,6 +56,10 @@ if os.getenv("VLLM_TEST_MODEL"):
available=DEVICE_BACKENDS["xpu"].available,
backends=[],
)
DEVICE_BACKENDS["rocm"] = DeviceConfig(
available=DEVICE_BACKENDS["rocm"].available,
backends=["TRITON_MLA"],
)
# Only include backends for devices that are actually available.
BACKENDS: list[str] = sorted(
@@ -60,7 +68,7 @@ BACKENDS: list[str] = sorted(
skip_unsupported = pytest.mark.skipif(
not any(cfg.available for cfg in DEVICE_BACKENDS.values()),
reason="Requires CUDA >= Ampere (SM80) or Intel XPU with Triton",
reason="Requires CUDA >= Ampere (SM80), ROCm, or Intel XPU with Triton",
)
skip_if_not_cuda = pytest.mark.skipif(
@@ -68,6 +76,16 @@ skip_if_not_cuda = pytest.mark.skipif(
reason="Requires CUDA >= Ampere (SM80)",
)
skip_if_not_cuda_alike = pytest.mark.skipif(
not (DEVICE_BACKENDS["cuda"].available or DEVICE_BACKENDS["rocm"].available),
reason="Requires CUDA >= Ampere (SM80) or ROCm",
)
requires_mx = pytest.mark.skipif(
not (current_platform.is_rocm() and current_platform.supports_mx()),
reason="requires a ROCm device with native MX support (gfx95x)",
)
def _random_prompt(min_words: int = 1024, max_words: int = 1024 * 2) -> str:
# Generate more realistic prompts that will actually produce varied tokens
@@ -1,43 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm import LLM, SamplingParams
from vllm.platforms import current_platform
from ....utils import create_new_process_for_each_test
if current_platform.is_rocm():
pytest.skip(
"Cascade attention backends FLASH_ATTN and FLASHINFER are notsupported on ROCm",
allow_module_level=True,
)
@create_new_process_for_each_test()
@pytest.mark.parametrize("attn_backend", ["FLASH_ATTN", "FLASHINFER"])
def test_cascade_attention(example_system_message, attn_backend):
prompt = "\n<User>: Implement fibonacci sequence in Python.\n<Claude>:"
if attn_backend == "FLASHINFER":
pytest.skip(
"This test is failing with FlashInfer backend and "
"needs investigation. See issue #25679."
)
llm = LLM(
model="Qwen/Qwen2-1.5B-Instruct", attention_config={"backend": attn_backend}
)
sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
# No cascade attention.
single_prompt = [example_system_message + prompt]
responses = llm.generate(single_prompt, sampling_params)
ref_output = responses[0].outputs[0].text
# (Probably) Use cascade attention.
prompts = [example_system_message + prompt] * 64
responses = llm.generate(prompts, sampling_params)
for response in responses:
assert response.outputs[0].text == ref_output
+11 -1
View File
@@ -83,13 +83,20 @@ def _get_hisparse_worker(runner: VllmRunner) -> HiSparseConnectorWorker:
not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA"
)
@pytest.mark.parametrize(
"with_offloading", [False, True], ids=["standalone", "offload"]
"with_offloading,attention_backend",
[
pytest.param(False, None, id="standalone"),
pytest.param(True, None, id="offload"),
# Offloading does not touch the attention kernels: FA4 runs standalone.
pytest.param(False, "FLASH_ATTN_MLA_SPARSE_FA4", id="standalone-fa4"),
],
)
@fork_new_process_for_each_test
def test_hisparse_spill_and_prefix_restore(
monkeypatch: pytest.MonkeyPatch,
vllm_runner: type[VllmRunner],
with_offloading: bool,
attention_backend: str | None,
):
"""Spilled prefixes restore and FULL-graph decode writes reach host KV.
@@ -101,6 +108,8 @@ def test_hisparse_spill_and_prefix_restore(
capability = current_platform.get_device_capability()
if capability is None or capability.major < 9:
pytest.skip("Sparse MLA requires Hopper or newer")
if attention_backend and not current_platform.is_device_capability_family(100):
pytest.skip("FA4 sparse MLA requires SM 10.x")
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip")
@@ -142,6 +151,7 @@ def test_hisparse_spill_and_prefix_restore(
)
),
kv_transfer_config=kv_transfer_config,
attention_backend=attention_backend,
block_size=64,
max_model_len=320,
max_num_batched_tokens=1024,
@@ -74,6 +74,12 @@ def test_kv_sharing_fast_prefill(
"ROCm: torch.compile produces incorrect output for gemma-3n's GELU "
"with tanh approximation. Use enforce_eager=True instead."
)
if not enforce_eager and not use_v2_model_runner and current_platform.is_cuda():
pytest.skip(
"cublasCreate fails with CUBLAS_STATUS_ALLOC_FAILED during memory "
"profiling with torch 2.14: "
"https://github.com/pytorch/pytorch/issues/199944"
)
sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
compilation_config = CompilationConfig(
+158 -5
View File
@@ -16,7 +16,10 @@ from tests.utils import create_new_process_for_each_test
from vllm import LLM, SamplingParams, TokensPrompt
from vllm.config import CacheConfig
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateCopyFunc,
is_conv_state_dim_first,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -59,6 +62,10 @@ MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8"
# where block boundaries fall. XPU rounds GDN block sizes up to a multiple of 64
# (XPUPlatform.update_block_size_for_backend), so use one.
BLOCK_SIZE = 576
# The step actions also encode the split at the prompt's last block boundary,
# which a GDN prefill backend with internal checkpoints (FlashInfer) skips; see
# get_internal_checkpoint_step_configs.
GDN_PREFILL_CONFIG = {"gdn_prefill_backend": "triton"}
DEVICE_TYPE = current_platform.device_type
NUM_HIDDEN_LAYERS = 1
cur_step_action_idx = 0
@@ -129,6 +136,10 @@ def get_fake_propose_draft_token_ids_fn():
first_token_id_index = (
num_computed_tokens + 1
) # bonus token isn't considered as computed
# num_accepted_tokens_cpu is filled by a non-blocking D2H copy in the
# same step; wait for it or a lagging GPU yields stale counts.
if self.num_accepted_tokens_event is not None:
self.num_accepted_tokens_event.synchronize()
first_token_id_index += self.input_batch.num_accepted_tokens_cpu[0].item()
proposed_draft_token_ids = [
prompt_token_ids[
@@ -878,6 +889,73 @@ def get_mamba_prefix_cache_step_configs(
return tests
def get_internal_checkpoint_step_configs() -> dict[str, TestConfig]:
"""Async-scheduling steps when the GDN prefill exports the checkpoint at
the prompt's last block boundary itself, so the final chunk runs whole.
The checkpoint goes in the second-to-last block the chunk reaches, keyed to
the boundary: column 1 holds the state at B for a 2B + 10 prompt.
"""
B = BLOCK_SIZE
return {
"prompt_2_block_size": TestConfig(
num_prompt_tokens=B * 2,
num_generated_tokens=10,
num_accepted_tokens=4,
step_actions=[
StepAction(0, B * 2, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(B * 2, 4, [1, 1, 1, 1, 1, 1], (1, 2), (-1, -1)),
],
),
"prompt_2_block_size_10": TestConfig(
num_prompt_tokens=B * 2 + 10,
num_generated_tokens=10,
num_accepted_tokens=4,
step_actions=[
StepAction(0, B * 2 + 10, [0, 1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(B * 2 + 10, 4, [0, 1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
],
),
"prompt_10_block_size": TestConfig(
num_prompt_tokens=B * 10,
num_generated_tokens=10,
num_accepted_tokens=4,
step_actions=[
StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
B * 5,
B * 5,
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1],
(4, 9),
(-1, -1),
),
StepAction(
B * 10,
4,
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1],
(9, 10),
(-1, -1),
),
],
),
"prompt_10_block_size_10": TestConfig(
num_prompt_tokens=B * 10 + 10,
num_generated_tokens=10,
num_accepted_tokens=4,
step_actions=[
StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
B * 5,
B * 5 + 10,
[0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1],
(4, 10),
(-1, -1),
),
],
),
}
def _run_mamba_prefix_cache_mrv1(
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
):
@@ -898,7 +976,7 @@ def _run_mamba_prefix_cache_mrv1(
load_format="dummy",
enable_prefix_caching=True,
block_size=BLOCK_SIZE,
mamba_cache_mode="align",
additional_config=GDN_PREFILL_CONFIG,
speculative_config={
"method": "qwen3_next_mtp",
"num_speculative_tokens": num_speculative_tokens,
@@ -1050,8 +1128,11 @@ def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
def _run_mamba_prefix_cache_mrv2(
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
monkeypatch: pytest.MonkeyPatch,
async_scheduling: bool,
additional_config: dict[str, Any] = GDN_PREFILL_CONFIG,
):
internal_checkpoints = additional_config["gdn_prefill_backend"] == "flashinfer"
global async_scheduling_mode
async_scheduling_mode = async_scheduling
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
@@ -1191,8 +1272,24 @@ def _run_mamba_prefix_cache_mrv2(
with gpu_sync_allowed():
start_pos = input_batch.positions[input_batch.query_start_loc[0]].item()
assert cur_step_action.num_computed_tokens_start == start_pos
if scheduler_output.scheduled_new_reqs and "snapshot_col" in captured:
captured["snapshot"] = block_states(self, captured.pop("snapshot_col"))
return ret
def block_states(runner: MRV2GPUModelRunner, col: int) -> list[list[torch.Tensor]]:
# Each Mamba layer's (conv, temporal) states in block column col.
kv_cache_config = captured["kv_cache_config"]
forward_context = runner.vllm_config.compilation_config.static_forward_context
return [
[
temporal_block(state, captured["block_tables"][group_id], col).clone()
for state in forward_context[layer_name].kv_cache
]
for group_ids in get_mamba_groups(kv_cache_config).values()
for group_id in group_ids
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names
]
def fake_sample(
self: MRV2GPUModelRunner,
hidden_states: torch.Tensor,
@@ -1254,7 +1351,7 @@ def _run_mamba_prefix_cache_mrv2(
skip_tokenizer_init=True,
enable_prefix_caching=True,
block_size=BLOCK_SIZE,
mamba_cache_mode="align",
additional_config=additional_config,
speculative_config={
"method": "qwen3_next_mtp",
"num_speculative_tokens": num_speculative_tokens,
@@ -1267,7 +1364,11 @@ def _run_mamba_prefix_cache_mrv2(
)
try:
tests = get_mamba_prefix_cache_step_configs(async_scheduling)
tests = (
get_internal_checkpoint_step_configs()
if internal_checkpoints
else get_mamba_prefix_cache_step_configs(async_scheduling)
)
global step_actions
global cur_step_action_idx
@@ -1313,6 +1414,35 @@ def _run_mamba_prefix_cache_mrv2(
f"got events={second_events!r}"
)
assert engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache()
if internal_checkpoints:
# The checkpoint a 2B + 10 prefill exports at B (column 1) is the
# final state of a prefill that stops at B (column 0). Distinct
# tokens make a shifted conv window visible.
tokens = [(i * 7919) % 30000 + 1 for i in range(BLOCK_SIZE * 2 + 10)]
scheduler = engine.llm_engine.engine_core.engine_core.scheduler
states = []
for num_tokens, col in ((BLOCK_SIZE, 0), (len(tokens), 1)):
captured["snapshot_col"] = col
_ = engine.generate(
[TokensPrompt(prompt_token_ids=tokens[:num_tokens])],
sampling_params=sampling_params,
)
states.append(captured.pop("snapshot"))
assert scheduler.reset_prefix_cache()
hf_config = engine.llm_engine.model_config.hf_text_config
state_len = hf_config.linear_conv_kernel_dim - 1
for (ref_conv, ref_ssm), (conv, ssm) in zip(*states):
if not is_conv_state_dim_first():
ref_conv, conv = ref_conv.mT, conv.mT
# Under MTP the conv state has num_spec scratch columns after
# the window.
for new, ref in (
(conv[..., :state_len], ref_conv[..., :state_len]),
(ssm, ref_ssm),
):
rel_err = (new.float() - ref.float()).norm() / ref.float().norm()
assert rel_err < 1e-2, rel_err
finally:
del engine
torch.accelerator.empty_cache()
@@ -1327,3 +1457,26 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv2_async(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv2(monkeypatch, async_scheduling=True)
@pytest.mark.skipif(
not current_platform.is_cuda()
or not (
current_platform.is_device_capability(90)
or (
(
current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
)
and current_platform.get_cuda_runtime_major() >= 13
)
),
reason="FlashInfer GDN prefill needs SM90, or SM100/SM120 with CUDA 13.",
)
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv2_async_flashinfer(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv2(
monkeypatch,
async_scheduling=True,
additional_config={"gdn_prefill_backend": "flashinfer"},
)
+115 -37
View File
@@ -8,23 +8,26 @@ request is queued before the first step, so the first prefill has the same
shape in both runs. Give the engine its own process and the comparison turns
into a coin toss: prefill starts before the last requests arrive over IPC, so
one boot batches two requests where the other batches one. The batch shape
sets the GEMM reduction order, logprobs shift by up to ~0.25, and near-ties
flip — measured at up to 5 of 16 completions, with sharding disabled on both
sides. A real sharding bug — rows routed to the wrong request, misaligned
gathers — produces many-nats divergence and scrambled top-k sets."""
sets the GEMM reduction order and can make near-ties flip. A real sharding
bug — rows routed to the wrong request, misaligned gathers — produces
many-nats divergence and scrambled top-k sets."""
import pytest
import torch
from tests.utils import multi_gpu_test
from vllm import LLM, SamplingParams
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.distributed import (
cleanup_dist_env_and_memory,
tensor_model_parallel_all_gather,
)
from vllm.logprobs import Logprob, LogprobsOnePosition
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
GPT_OSS_MODEL = "trl-internal-testing/tiny-GptOssForCausalLM"
GPT_OSS_REVISION = "e82685101216af1953707faa42567d232a5e4d45"
# Misrouted rows move logprobs by nats; anything under this is float noise.
# With matched batches the two boots agreed bitwise over 320 positions, so the
# bound is not expected to be reached.
LOGPROB_TOL = 0.5
PROMPTS = [
@@ -58,18 +61,52 @@ def _is_sharded_sampling_active(worker) -> bool:
return worker.model_runner.batch_sharder is not None
def _sampling_params() -> list[SamplingParams]:
return [SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))]
def _local_logits_match_full(worker) -> dict[str, bool]:
"""Reassemble rank-local logits and compare with the full path."""
model = worker.model_runner.model
hidden_size = model.config.hidden_size
dtype = model.lm_head.weight.dtype
hidden_states = torch.linspace(
-1,
1,
steps=2 * hidden_size,
device=worker.model_runner.device,
dtype=dtype,
).view(2, hidden_size)
full_logits = model.compute_logits(hidden_states)
local_logits = model.compute_logits_local(hidden_states)
assert local_logits.shape[-1] == model.lm_head.num_embeddings_per_partition
gathered_logits = tensor_model_parallel_all_gather(local_logits)
gathered_logits = gathered_logits[..., : model.config.vocab_size]
has_full_logits = full_logits is not None
return {
"has_full_logits": has_full_logits,
"matches": not has_full_logits or torch.equal(full_logits, gathered_logits),
}
def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
def _sampling_params(num_prompts: int) -> list[SamplingParams]:
return [SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(num_prompts)]
def _generate(
disable_sharding: bool,
model: str,
prompts: list[str],
sampling_params: list[SamplingParams],
revision: str | None,
):
llm = LLM(
model=MODEL,
model=model,
revision=revision,
tokenizer_revision=revision,
tensor_parallel_size=2,
enable_batch_sharded_sampling=not disable_sharding,
max_model_len=1024,
# More requests than ranks, so round-robin slot ownership splits the
# batch across both ranks.
# Keep capacity at least as large as the TP size. The actual request
# count may be smaller to exercise an empty rank.
max_num_seqs=len(PROMPTS),
enforce_eager=True,
enable_prefix_caching=False,
@@ -83,12 +120,14 @@ def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
# intended sampling mode.
modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active)
assert all(mode == (not disable_sharding) for mode in modes), modes
params = _sampling_params()
# Two waves: the second reuses the request slots freed by the first
# (in whatever order requests finished), covering the cross-rank
# slot-recycling path that request ownership derives from.
wave1 = llm.generate(PROMPTS, params)
wave2 = llm.generate(PROMPTS[::-1], params[::-1])
if not disable_sharding:
logits_checks = llm.llm_engine.collective_rpc(_local_logits_match_full)
assert any(check["has_full_logits"] for check in logits_checks)
assert all(check["matches"] for check in logits_checks)
# The second wave reuses request slots freed by the first,
# covering the cross-rank slot-recycling path that ownership uses.
wave1 = llm.generate(prompts, sampling_params)
wave2 = llm.generate(prompts[::-1], sampling_params[::-1])
return wave1 + wave2
finally:
del llm
@@ -178,25 +217,11 @@ def test_logprob_comparison_rejects_missing_top_k_rank(rank: int | None):
_assert_logprob_dicts_close(lps, lps, "missing rank", num_logprobs=1)
@multi_gpu_test(num_gpus=2)
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
"""Generation, logprobs, and prompt logprobs match between batch-sharded
sampling and the replicated fallback."""
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
# Required for the collective_rpc mode check below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
# Queue every request before the first step, so both boots schedule the
# same batches -- see the module docstring.
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
ref_outputs = _generate(monkeypatch, disable_sharding=True)
shard_outputs = _generate(monkeypatch, disable_sharding=False)
assert len(ref_outputs) == len(shard_outputs) == 2 * len(PROMPTS)
params = _sampling_params()
def _assert_outputs_match(ref_outputs, shard_outputs, params) -> None:
assert len(ref_outputs) == len(shard_outputs) == len(params)
divergent = []
for i, (ref, out, sampling_params) in enumerate(
zip(ref_outputs, shard_outputs, params + params[::-1])
zip(ref_outputs, shard_outputs, params)
):
ref_completion = ref.outputs[0]
out_completion = out.outputs[0]
@@ -244,7 +269,60 @@ def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
)
assert not divergent, (
f"{len(divergent)}/{2 * len(PROMPTS)} completions diverged at "
f"{len(divergent)}/{len(params)} completions diverged at "
f"(prompt, position) {divergent}; the sampling modes did not agree "
"token for token with in-process scheduling"
)
def _configure_test_environment(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
# Required for the collective_rpc mode check below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
# Queue every request before the first step, so both boots schedule the
# same batches -- see the module docstring.
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
("model", "revision", "prompts"),
[
pytest.param(MODEL, None, PROMPTS, id="llama"),
pytest.param(MODEL, None, PROMPTS[:1], id="llama-empty-rank"),
pytest.param(
GPT_OSS_MODEL,
GPT_OSS_REVISION,
PROMPTS,
id="gpt-oss",
),
],
)
def test_sharded_sampling_outputs_match(
monkeypatch: pytest.MonkeyPatch,
model: str,
revision: str | None,
prompts: list[str],
):
"""Generation, logprobs, and prompt logprobs match between batch-sharded
sampling and the replicated fallback."""
_configure_test_environment(monkeypatch)
sampling_params = _sampling_params(len(prompts))
ref_outputs = _generate(
disable_sharding=True,
model=model,
prompts=prompts,
sampling_params=sampling_params,
revision=revision,
)
shard_outputs = _generate(
disable_sharding=False,
model=model,
prompts=prompts,
sampling_params=sampling_params,
revision=revision,
)
expected_params = sampling_params + sampling_params[::-1]
_assert_outputs_match(ref_outputs, shard_outputs, expected_params)
@@ -1,50 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from tests.evals.gsm8k.gsm8k_eval import _build_gsm8k_prompts
from vllm import SamplingParams
from vllm.config import CompilationConfig
from ...utils import compute_acceptance_rate
def test_medusa_acceptance_rate(
sampling_config: SamplingParams,
vllm_runner,
):
"""Verify a trained Medusa checkpoint achieves nonzero acceptance rate.
Uses the canonical FasterDecoding vicuna-7b checkpoint to confirm the
speculation path actually accepts tokens — unlike test_medusa_correctness,
which uses a random head and only validates output correctness.
"""
target_model = "lmsys/vicuna-7b-v1.3"
medusa_model = "FasterDecoding/medusa-vicuna-7b-v1.3"
prompts = _build_gsm8k_prompts(num_questions=10, num_shots=1)[0]
with vllm_runner(
target_model,
block_size=None,
trust_remote_code=False,
speculative_config={
"method": "medusa",
"model": medusa_model,
"num_speculative_tokens": 3,
},
max_model_len=1024,
enforce_eager=True,
enable_chunked_prefill=None,
disable_log_stats=False,
compilation_config=CompilationConfig(),
) as spec_runner:
spec_runner.llm.generate(prompts, sampling_config)
metrics = spec_runner.llm.get_metrics()
acceptance_rate = compute_acceptance_rate(metrics)
min_acceptance_rate = 0.198
print(f"Medusa acceptance rate: {acceptance_rate:.4f} (min {min_acceptance_rate})")
# Regression guard at 90% of the measured baseline.
assert acceptance_rate >= min_acceptance_rate, (
f"Medusa acceptance rate {acceptance_rate:.4f} below min {min_acceptance_rate}"
)
@@ -18,17 +18,18 @@ from vllm.distributed import cleanup_dist_env_and_memory
from .utils import _skip_if_insufficient_gpus_for_tp, get_test_prompts, greedy_sampling
# Even with in-process scheduling, near-tie tokens still flip on up to 3 of 20
# prompts in CI, greedy and seeded. A sharding bug diverges wholesale instead.
MAX_DIVERGENT_PROMPTS = 3
def _is_sharded_sampling_active(worker) -> bool:
return worker.model_runner.batch_sharder is not None
def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
"""Batch-sharded sampling must be a bit-exact drop-in for replicated
sampling under MTP spec decoding: the collectives move the same logits
bytes, Gumbel keys derive from (request slot, position, seed), and slot
assignment is rank-deterministic. Both runs here are spec decode with
identical math, so outputs must match exactly."""
"""Batch-sharded sampling must match replicated sampling under MTP spec
decoding, up to measured near-tie noise."""
tp_size = 2
_skip_if_insufficient_gpus_for_tp(tp_size)
model_name = "Qwen/Qwen3.5-0.8B-Base"
@@ -96,8 +97,7 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
f" replicated: {ref.outputs[0].text!r}\n"
f" sharded: {out.outputs[0].text!r}"
)
assert not divergent, (
assert len(divergent) <= MAX_DIVERGENT_PROMPTS, (
f"{name}: {len(divergent)}/{len(ref_outputs)} prompts diverged "
f"at {divergent}; the sampling modes did not agree token for "
"token with in-process scheduling"
f"at {divergent}, beyond near-tie noise"
)
@@ -88,7 +88,6 @@ def test_mtp_speculative_mixed_batch_short_prefill(
trust_remote_code=True,
enable_chunked_prefill=True,
enable_prefix_caching=enable_prefix_caching,
mamba_cache_mode="align" if enable_prefix_caching else "none",
) as llm:
sampling_params = SamplingParams(
temperature=0.0,
@@ -161,7 +160,6 @@ def test_qwen_mtp_mamba_prefix_cache_hit_is_bounded(vllm_runner, monkeypatch):
enforce_eager=True,
enable_chunked_prefill=True,
enable_prefix_caching=True,
mamba_cache_mode="align",
speculative_config={"method": "mtp", "num_speculative_tokens": 2},
) as runner:
llm = runner.get_llm()
+32 -19
View File
@@ -60,7 +60,6 @@ def _check_replayssm_parity(
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
tensor_parallel_size=tensor_parallel_size,
mamba_backend=mamba_backend,
)
@@ -122,32 +121,48 @@ def test_replayssm_flashinfer_decode_matches_baseline(
@pytest.mark.parametrize("model_name", FLASHINFER_MODELS)
def test_replayssm_flashinfer_spec_decode_matches_baseline(vllm_runner, model_name):
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
def test_replayssm_flashinfer_spec_decode_matches_baseline(
vllm_runner, model_name, monkeypatch, use_v2_model_runner
):
common = dict(
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
speculative_config={
"method": "ngram",
"num_speculative_tokens": 3,
"prompt_lookup_max": 3,
},
)
with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm:
baseline = llm.generate_greedy_logprobs(
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
)
with vllm_runner(
model_name,
use_replayssm=True,
replayssm_buffer_len=16,
mamba_backend="flashinfer",
**common,
) as llm:
replay = llm.generate_greedy_logprobs(
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
)
try:
with monkeypatch.context() as patch:
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
envs.disable_envs_cache()
with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm:
assert (
llm.llm.llm_engine.vllm_config.use_v2_model_runner
is use_v2_model_runner
)
baseline = llm.generate_greedy_logprobs(
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
)
with vllm_runner(
model_name,
use_replayssm=True,
replayssm_buffer_len=16,
mamba_backend="flashinfer",
**common,
) as llm:
assert (
llm.llm.llm_engine.vllm_config.use_v2_model_runner
is use_v2_model_runner
)
replay = llm.generate_greedy_logprobs(
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
)
finally:
envs.disable_envs_cache()
check_logprobs_close(
outputs_0_lst=baseline,
@@ -165,7 +180,6 @@ def test_replayssm_flashinfer_mtp(vllm_runner, monkeypatch, use_v2_model_runner)
max_model_len=1024,
trust_remote_code=True,
enable_prefix_caching=False,
mamba_cache_mode="none",
mamba_backend="flashinfer",
tensor_parallel_size=2,
disable_log_stats=False,
@@ -233,7 +247,6 @@ def _check_replayssm_prefix_caching_parity(
trust_remote_code=True,
enable_prefix_caching=True,
enable_chunked_prefill=True,
mamba_cache_mode="align",
disable_log_stats=False, # required for llm.get_metrics()
tensor_parallel_size=tensor_parallel_size,
)
@@ -22,12 +22,16 @@ class _Pos:
self.offset = offset
self.length = length
def get_num_embeds(self):
return self.length
class _Feature:
def __init__(self, mm_hash, length=1, identifier=None):
self.mm_hash = mm_hash
self.identifier = identifier if identifier is not None else mm_hash
self.mm_position = _Pos(0, length)
self.modality = "image"
class _Request:
@@ -80,7 +84,7 @@ def _make_scheduler(
def _load_ids(meta) -> list[int]:
"""Transfer ids the scheduler dispatched in this step's metadata."""
return [transfer_id for transfer_id, _ in meta.loads.values()]
return [transfer_id for transfer_id, _, _ in meta.loads.values()]
def _load_blocks(meta, mm_hash: str) -> list[int]:
@@ -90,7 +94,7 @@ def _load_blocks(meta, mm_hash: str) -> list[int]:
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
"""Pre-populate a ready cache entry backed by real blocks."""
s._cache.alloc(mm_hash, n_blocks)
s._cache.alloc(mm_hash, n_blocks, (n_blocks, _BS // s._dtype.itemsize))
s._cache.mark_ready(mm_hash)
@@ -97,10 +97,9 @@ def test_second_instance_opens_existing_file_and_shares_memory():
r1.cleanup()
def test_dense_dp_engines_get_distinct_regions(monkeypatch):
def test_dense_dp_engines_get_distinct_regions():
"""Dense DP engines reset data_parallel_rank to 0 but share instance_id,
so the region must be keyed by data_parallel_index."""
monkeypatch.setattr(common, "_get_encoder_cache_hidden_dim", lambda cfg: 8)
instance_id = str(uuid.uuid4())
regions = []
try:
@@ -113,6 +112,7 @@ def test_dense_dp_engines_get_distinct_regions(monkeypatch):
cfg.instance_id = instance_id
cfg.parallel_config = parallel_config
cfg.model_config.dtype = torch.float16
cfg.model_config.get_inputs_embeds_size.return_value = 8
cfg.ec_transfer_config.ec_connector_extra_config = {"ec_cpu_bytes": 64}
regions.append(common.create_ec_shared_region(cfg))
@@ -91,7 +91,9 @@ def _make_region() -> ECSharedRegion:
def _vllm_config(rank: int = 0) -> Mock:
return create_ec_vllm_config(rank=rank, dtype=_DTYPE)
config = create_ec_vllm_config(rank=rank, dtype=_DTYPE)
config.model_config.get_inputs_embeds_size.return_value = _HIDDEN_DIM
return config
def _meta(
@@ -105,7 +107,7 @@ def _meta(
return ECCPUConnectorMetadata(
saves=saves or {},
loads={
mm_hash: (idx, block_ids)
mm_hash: (idx, block_ids, (len(block_ids), _HIDDEN_DIM))
for idx, (mm_hash, block_ids) in enumerate((loads or {}).items())
},
)
@@ -904,21 +906,29 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
# Build scheduler sharing the same region.
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
scheduler = ECCPUScheduler(_vllm_config())
config = _vllm_config()
# DeepStack-wide output: each embedding spans four blocks.
width = 4 * _HIDDEN_DIM
config.ec_transfer_config.mm_encoder_output_widths = {"image": width}
scheduler = ECCPUScheduler(config)
# -- Step 1: scheduler allocates, worker saves --
n_blocks = 3
src = torch.arange(
n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE
).reshape(n_blocks, _HIDDEN_DIM)
n_tokens = 2
src = torch.arange(n_tokens * width, dtype=_DTYPE, device=DEVICE_TYPE).reshape(
n_tokens, width
)
class _Pos:
offset = 0
length = n_blocks
length = n_tokens
def get_num_embeds(self):
return self.length
class _Feature:
identifier = "img_001"
mm_position = _Pos()
modality = "image"
class _Request:
request_id = "req_e2e"
@@ -948,8 +958,9 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
scheduler.update_state_after_alloc(_Request(), 0)
meta_load = scheduler.build_connector_meta(scheduler_output=None)
assert "img_001" in meta_load.loads
_, load_blocks = meta_load.loads["img_001"]
_, load_blocks, shape = meta_load.loads["img_001"]
assert load_blocks == meta_save.saves["img_001"]
assert shape == tuple(src.shape)
load_cache: dict[str, torch.Tensor] = {}
worker.start_load_caches(load_cache, meta_load)
@@ -45,6 +45,34 @@ KEY = (
SPEC = TensorSpec((2, 2), "torch.float32", 16)
def test_push_shapes_follow_each_modality(monkeypatch):
"""Each push is sized at its modality's measured encoder output width."""
from vllm.distributed.ec_transfer.ec_connector.mooncake import scheduler
monkeypatch.setattr(scheduler, "ensure_mooncake_available", lambda: None)
widths = {"image": 8192, "audio": 2048}
config = create_ec_vllm_config(ec_role="ec_producer", encoder_output_widths=widths)
request = SimpleNamespace(
request_id="mixed",
mm_features=[SimpleNamespace(identifier=m, modality=m) for m in widths],
get_num_encoder_embeds=lambda index: 2,
ec_transfer_params={
"consumer_zmq": "tcp://consumer:1234",
"ec_items": [{"mm_hash": m, "transfer_id": m} for m in widths],
},
)
instance = scheduler.ECMooncakeScheduler(config)
try:
for index in range(len(widths)):
instance.update_state_after_alloc(request, index)
metadata = instance.build_connector_meta(
SimpleNamespace(free_encoder_mm_hashes=[], preempted_req_ids=set())
)
assert [spec.shape for spec in metadata.pushes] == [(2, 8192), (2, 2048)]
finally:
instance.close()
def test_embedding_key_combines_namespace_and_identifier():
config = create_ec_vllm_config(ec_role="ec_producer")
config.ec_transfer_config.ec_connector_extra_config.update(
@@ -246,6 +246,18 @@ class TestCacheExistence:
assert result[1] # Second exists
assert not result[2] # Third doesn't exist
def test_has_cache_item_miss_creates_no_folder(
self, mock_vllm_config_consumer, temp_storage
):
"""Test a cache miss leaves no empty folder in the shared storage."""
connector = ECExampleConnector(
vllm_config=mock_vllm_config_consumer,
role=ECConnectorRole.SCHEDULER,
)
assert not connector.has_cache_item("missing_hash")
assert os.listdir(temp_storage) == []
class TestStateManagement:
"""Test connector state management."""
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Startup measurement of the encoder output width EC producers size by."""
from types import SimpleNamespace
import pytest
import torch
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.distributed.ec_transfer.ec_connector import encoder_output_width as eow
pytestmark = pytest.mark.cpu_test
class _DeepStackModel:
"""Emits each modality at its own width, as Qwen3-Omni does."""
widths = {"image": 4 * 16, "audio": 16}
def embed_multimodal(self, *, modality):
return [torch.zeros(3, self.widths[modality])]
class _Budget:
def __init__(self, *args, **kwargs):
self.mm_max_toks_per_item = dict.fromkeys(_DeepStackModel.widths, 8)
def get_dummy_encoder_profile_inputs(self, modality, max_items_per_batch):
assert max_items_per_batch == 1
return [(modality, None)]
def _worker(first_pp_rank=True, supports_mm=True):
runner = SimpleNamespace(
supports_mm_inputs=supports_mm,
get_model=_DeepStackModel,
device=torch.device("cpu"),
)
pp_group = SimpleNamespace(is_first_rank=first_pp_rank)
return SimpleNamespace(model_runner=runner, vllm_config=None), pp_group
@pytest.fixture
def patched(monkeypatch):
import vllm.distributed.parallel_state as ps
import vllm.multimodal.encoder_budget as budget_mod
import vllm.multimodal.utils as mm_utils
monkeypatch.setattr(budget_mod, "MultiModalBudget", _Budget)
monkeypatch.setattr(
mm_utils,
"group_and_batch_mm_kwargs",
lambda items, **_: iter([(items[0][0], 1, {"modality": items[0][0]})]),
)
return lambda group: monkeypatch.setattr(ps, "get_pp_group", lambda: group)
def test_worker_reports_the_encoder_output_width_per_modality(patched):
"""The width comes from the tensor the encoder emits, not from config."""
worker, group = _worker()
patched(group)
assert eow._measure_on_worker(worker) == {"image": 64, "audio": 16}
@pytest.mark.parametrize("first_pp_rank, supports_mm", [(False, True), (True, False)])
def test_ranks_without_an_encoder_report_nothing(patched, first_pp_rank, supports_mm):
worker, group = _worker(first_pp_rank, supports_mm)
patched(group)
assert eow._measure_on_worker(worker) == {}
def test_producer_records_the_widths_its_encoder_ranks_agree_on():
config = create_ec_vllm_config(ec_role="ec_producer", encoder_output_widths={})
reports = [{"image": 64}, {}, {"image": 64}]
eow.measure_encoder_output_widths(config, lambda fn: reports)
assert config.ec_transfer_config.mm_encoder_output_widths == {"image": 64}
assert eow.get_encoder_output_width(config, "image") == 64
@pytest.mark.parametrize(
"reports, match",
[
([{}, {}], "could not measure"),
([{"image": 64}, {"image": 16}], "disagree"),
],
ids=["no-encoder", "disagree"],
)
def test_producer_refuses_to_start_without_a_trustworthy_width(reports, match):
config = create_ec_vllm_config(ec_role="ec_producer", encoder_output_widths={})
with pytest.raises(ValueError, match=match):
eow.measure_encoder_output_widths(config, lambda fn: reports)
@@ -25,6 +25,9 @@ class _Pos:
def __init__(self, offset, length):
self.offset, self.length = offset, length
def get_num_embeds(self):
return self.length
class _Feature:
def __init__(self, mm_hash, length=1, data=None):
@@ -85,6 +88,9 @@ class _FakeTransport:
pass
_MISSING = object()
def _consumer_sched(monkeypatch):
def _region(cfg):
return ECSharedRegion(
@@ -95,16 +101,15 @@ def _consumer_sched(monkeypatch):
# ec_enable_nixl defaults to False, so this builds gate-off; the fields
# below flip on the NIXL consumer state directly.
s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_consumer"))
# No measured widths: a consumer takes each shape from the announcement.
s = ECCPUScheduler(
create_ec_vllm_config(ec_role="ec_consumer", encoder_output_widths={})
)
# Turn on NIXL consumer state without constructing real transports.
s._nixl_enabled = True
s._transport = _FakeTransport()
s._data = None
s._compat_hash = "c"
# _setup_nixl normally computes these from model_config; set them
# directly since this helper builds gate-off then flips fields on.
s._hidden_dim = _HID
s._element_size = _ES
s._metadata_resolver._cache["image"] = {"image_grid_thw"}
return s
@@ -115,24 +120,32 @@ def _params(mm_hash, length):
"peer_host": "h",
"peer_port": 1,
"size_bytes": length * _HID * _ES,
"shape": [length, _HID],
}
}
def test_new_remote_read_defers_then_completes(monkeypatch):
"""The read is sized from the announced shape, not a local width."""
width = 4 * _HID
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
# Route _start_xfer to our fake session instead of real ZMQ/NIXL, but
# still reserve a real not-ready cache entry so mark_ready works.
def _fake_start(mm_hash, info, size):
entry = s._cache.alloc(mm_hash, 1)
def _fake_start(mm_hash, info, size, shape):
assert size == width * _ES
assert shape == (1, width)
entry = s._cache.alloc(mm_hash, size // _BS, shape)
assert entry is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", _fake_start)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
params = _params("h1", 1)
params["h1"]["size_bytes"] = width * _ES
params["h1"]["shape"] = [1, width]
req = _Request([_Feature("h1", 1)], params=params)
# Step 1: unseen remote item -> read started, request deferred.
assert s.ensure_cache_available(req, 0) is False
@@ -155,6 +168,7 @@ def test_new_remote_read_defers_then_completes(monkeypatch):
s.update_state_after_alloc(req, 0)
meta = s.build_connector_meta(scheduler_output=None)
assert "h1" in meta.loads
assert meta.loads["h1"][2] == (1, width)
assert "h1" not in s._step_completed # cleared by promote
# Step 3: still cached -> admitted directly, no new transfer.
@@ -176,8 +190,8 @@ def test_failed_read_falls_back_only_with_local_input(monkeypatch, payload):
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
def _fake_start(mm_hash, info, size):
entry = s._cache.alloc(mm_hash, 1)
def _fake_start(mm_hash, info, size, shape):
entry = s._cache.alloc(mm_hash, 1, shape)
assert entry is not None
fake.started.append(mm_hash)
return True
@@ -230,8 +244,8 @@ def test_remote_wait_budget_survives_retries_and_long_steps(monkeypatch, retry):
fake = _FakeSession()
s._sessions[("h", 1)] = fake
def start(mm_hash, info, size):
assert s._cache.alloc(mm_hash, 1) is not None
def start(mm_hash, info, size, shape):
assert s._cache.alloc(mm_hash, 1, shape) is not None
fake.started.append(mm_hash)
return True
@@ -261,8 +275,8 @@ def test_retryable_read_re_requests_without_admitting(monkeypatch):
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
def _fake_start(mm_hash, info, size):
entry = s._cache.alloc(mm_hash, 1)
def _fake_start(mm_hash, info, size, shape):
entry = s._cache.alloc(mm_hash, 1, shape)
assert entry is not None
fake.started.append(mm_hash)
return True
@@ -287,14 +301,29 @@ def test_retryable_read_re_requests_without_admitting(monkeypatch):
@pytest.mark.parametrize(
"size_fields",
[{"size_bytes": 999}, {"size_bytes": None}, {"size_bytes": "big"}, {}],
ids=["mismatch", "null-size", "text-size", "no-size"],
"fields",
[
{"size_bytes": 999},
{"size_bytes": None},
{"size_bytes": "big"},
{"size_bytes": _MISSING},
{"shape": _MISSING},
{"shape": [2, _HID]},
],
ids=[
"size-mismatch",
"null-size",
"text-size",
"no-size",
"no-shape",
"wrong-rows",
],
)
def test_invalid_remote_size_fails_the_request(monkeypatch, size_fields):
"""Invalid remote sizes fail the request without raising in the scheduler."""
def test_invalid_announcement_fails_the_request(monkeypatch, fields):
"""An unusable announced size or shape fails the request, not the engine."""
s = _consumer_sched(monkeypatch)
announced = {"peer_host": "h", "peer_port": 1, **size_fields}
announced = {**_params("h1", 1)["h1"], **fields}
announced = {k: v for k, v in announced.items() if v is not _MISSING}
req = _Request([_Feature("h1", 1)], params={"h1": announced})
assert s.ensure_cache_available(req, 0) is False
assert s.take_unavailable_requests() == {"r1"}
@@ -306,7 +335,7 @@ def test_invalid_remote_size_fails_the_request(monkeypatch, size_fields):
def test_deferral_budget_is_per_request(monkeypatch):
"""Shared hashes have independent wait budgets and cancellation cleanup."""
s = _consumer_sched(monkeypatch)
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
monkeypatch.setattr(s._cache, "alloc", lambda key, n, shape=None: None)
params = _params("h1", 1)
old = _Request([_Feature("h1", 1)], params=params, req_id="old")
new = _Request([_Feature("h1", 1)], params=params, req_id="new")
@@ -419,7 +448,7 @@ def test_in_flight_hash_defers_without_second_transfer(monkeypatch):
s = _consumer_sched(monkeypatch)
started: list[str] = []
def _spy_start_xfer(mm_hash, info, size):
def _spy_start_xfer(mm_hash, info, size, shape):
started.append(mm_hash)
return True
@@ -16,6 +16,9 @@ class _Pos:
def __init__(self, offset, length):
self.offset, self.length = offset, length
def get_num_embeds(self):
return self.length
class _Feature:
def __init__(self, mm_hash, length=1):
@@ -53,11 +56,8 @@ def test_request_finished_producer_emits_params(monkeypatch):
# Simulate NIXL-enabled producer bookkeeping without building real NIXL.
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
# _setup_nixl normally computes these from model_config; set them
# directly since this test builds gate-off then flips fields on.
s._hidden_dim, s._element_size = 32, 2
# feature length=2, hidden_dim=32, element_size=2 -> 128 bytes -> 2 blocks.
entry = s._cache.alloc("h1", 2)
entry = s._cache.alloc("h1", 2, (2, 32))
assert entry is not None
s._cache.mark_ready("h1")
@@ -71,6 +71,7 @@ def test_request_finished_producer_emits_params(monkeypatch):
"peer_host": "1.2.3.4",
"peer_port": 5601,
"size_bytes": 2 * 32 * 2,
"shape": [2, 32],
}
}
s.shutdown()
@@ -84,8 +85,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
s = _sched_gate_off(monkeypatch)
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
s._hidden_dim, s._element_size = 32, 2
s._cache.alloc("h1", 2) # allocated but not marked ready
s._cache.alloc("h1", 2, (2, 32)) # allocated but not marked ready
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False
@@ -96,6 +96,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
"peer_host": "1.2.3.4",
"peer_port": 5601,
"size_bytes": 2 * 32 * 2,
"shape": [2, 32],
}
}
s.shutdown()
@@ -105,7 +106,6 @@ def test_request_finished_skips_unallocated_entry(monkeypatch):
s = _sched_gate_off(monkeypatch)
s._nixl_enabled = True
s._peer_host, s._peer_port = "1.2.3.4", 5601
s._hidden_dim, s._element_size = 32, 2
# No alloc() for "h1" at all — e.g. the cache was full at save time.
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
+14 -4
View File
@@ -18,12 +18,16 @@ _EXECUTOR_BACKEND = "mp"
_PCP_SIZE = 1
_EMBED_WIDTH = 32
def create_ec_vllm_config(
*,
ec_role: ECRole = "ec_both",
tensor_parallel_size: int = 1,
rank: int = 0,
dtype: torch.dtype = torch.float16,
encoder_output_widths: dict[str, int] | None = None,
) -> Mock:
"""Build a `VllmConfig` stand-in for EC connector unit tests.
@@ -31,10 +35,8 @@ def create_ec_vllm_config(
role derivation and the rank arithmetic the connector depends on behave as
they do in production.
`model_config` is a stub because nothing under test needs a real one: the
connector reads only `dtype` here, and the paths that inspect the model
itself are not exercised. Building a real `ModelConfig` would resolve an HF
model and inspect its architecture.
`model_config` supplies a dtype and embedding width without resolving an
HF model or inspecting its architecture.
Args:
ec_role: EC role this instance plays. `is_ec_producer` and
@@ -42,6 +44,8 @@ def create_ec_vllm_config(
tensor_parallel_size: TP degree.
rank: Global rank of this worker.
dtype: Encoder cache dtype.
encoder_output_widths: Per-modality encoder output width, as a
producer measures it at startup. Defaults to the embedding width.
Returns:
A `VllmConfig`-specced mock carrying the configs above.
@@ -59,9 +63,15 @@ def create_ec_vllm_config(
ec_role=ec_role,
engine_id=str(uuid.uuid4()),
)
ec_transfer_config.mm_encoder_output_widths = (
dict.fromkeys(("image", "video", "audio"), _EMBED_WIDTH)
if encoder_output_widths is None
else encoder_output_widths
)
model_config = Mock(spec=ModelConfig)
model_config.dtype = dtype
model_config.get_inputs_embeds_size.return_value = _EMBED_WIDTH
vllm_config = Mock(spec=VllmConfig)
vllm_config.ec_transfer_config = ec_transfer_config
+77 -2
View File
@@ -6,6 +6,7 @@
These tests cover:
- OutputProcessor.get_num_queued_tokens() token counting
- AsyncLLM.check_admission() admission control logic
- Admission rejection Prometheus samples
- Exception classes (GracefulHTTPError, QueueOverflowError, MaxQueuedTokensError)
- create_error_response() mapping GracefulHTTPError to HTTP 503
- SchedulerConfig field defaults and validation
@@ -14,6 +15,7 @@ These tests cover:
import argparse
import asyncio
import logging
import multiprocessing
from collections.abc import Awaitable, Callable
from http import HTTPStatus
@@ -21,6 +23,7 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
import pytest
from prometheus_client import REGISTRY
from pydantic import ValidationError
from vllm.config.scheduler import SchedulerConfig
@@ -40,6 +43,10 @@ from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine.admission_control import SharedAdmissionStats
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.output_processor import OutputProcessor
from vllm.v1.metrics.loggers import StatLoggerManager
_ADMISSION_METRIC_MODEL = "admission-metric-test"
_ADMISSION_LOGGER = "vllm.v1.engine.async_llm"
pytestmark = pytest.mark.cpu_test
@@ -70,10 +77,43 @@ def _make_async_llm(
llm.output_processor = MagicMock()
llm.output_processor.get_num_unfinished_requests.return_value = num_unfinished
llm.output_processor.get_num_queued_tokens.return_value = num_queued_tokens
llm.logger_manager = None
llm.admission_stats = None
return llm
def _attach_prometheus_logger(
llm: AsyncLLM, engine_indexes: list[int] | None = None
) -> None:
"""Attach a real Prometheus logger manager to a lightweight AsyncLLM."""
vllm_config = MagicMock()
vllm_config.observability_config.show_hidden_metrics = False
vllm_config.observability_config.kv_cache_metrics = False
vllm_config.observability_config.custom_histogram_buckets = None
vllm_config.model_config.served_model_name = _ADMISSION_METRIC_MODEL
vllm_config.model_config.max_model_len = 2048
vllm_config.model_config.is_diffusion = False
vllm_config.speculative_config = None
vllm_config.kv_transfer_config = None
vllm_config.ec_transfer_config = None
vllm_config.lora_config = None
llm.logger_manager = StatLoggerManager(
vllm_config=vllm_config,
engine_idxs=engine_indexes,
enable_default_loggers=False,
)
def _admission_rejection_count(reason: str) -> float:
"""Read the Prometheus sample for one admission rejection reason."""
value = REGISTRY.get_sample_value(
"vllm:admission_rejections_total",
{"model_name": _ADMISSION_METRIC_MODEL, "reason": reason},
)
assert value is not None
return value
def _make_output_processor(**request_states) -> OutputProcessor:
op = OutputProcessor.__new__(OutputProcessor)
op.request_states = request_states
@@ -157,11 +197,13 @@ def test_shared_request_snapshot_enforces_global_limit():
server_0.set_num_requests(2)
llm = _make_async_llm(max_num_queued_reqs=2)
llm.admission_stats = server_1
_attach_prometheus_logger(llm)
with pytest.raises(QueueOverflowError):
llm.check_admission()
assert server_0.get_num_requests() == 2
assert _admission_rejection_count("max_num_queued_reqs") == 1
# ---------------------------------------------------------------------------
@@ -247,19 +289,35 @@ def test_admission_no_limits_allows_everything():
def test_admission_reqs_allows_when_under_limit():
llm = _make_async_llm(max_num_queued_reqs=10, num_unfinished=5)
_attach_prometheus_logger(llm)
llm.check_admission()
llm.check_admission()
assert _admission_rejection_count("max_num_queued_reqs") == 0
assert _admission_rejection_count("max_num_queued_tokens") == 0
def test_admission_reqs_rejects_at_limit():
def test_admission_reqs_rejects_at_limit(caplog):
llm = _make_async_llm(max_num_queued_reqs=10, num_unfinished=10)
with pytest.raises(QueueOverflowError):
_attach_prometheus_logger(llm, engine_indexes=[0, 1, 2])
with (
caplog.at_level(logging.INFO, logger=_ADMISSION_LOGGER),
pytest.raises(QueueOverflowError),
):
llm.check_admission()
assert _admission_rejection_count("max_num_queued_reqs") == 1
assert _admission_rejection_count("max_num_queued_tokens") == 0
assert not any(
"Request queue full" in record.getMessage() for record in caplog.records
)
def test_admission_reqs_rejects_with_n():
llm = _make_async_llm(max_num_queued_reqs=10, num_unfinished=8)
_attach_prometheus_logger(llm)
with pytest.raises(QueueOverflowError):
llm.check_admission(3)
assert _admission_rejection_count("max_num_queued_reqs") == 1
assert _admission_rejection_count("max_num_queued_tokens") == 0
def test_admission_reqs_allows_n_at_boundary():
@@ -283,8 +341,11 @@ def test_admission_tokens_allows_when_under_limit():
def test_admission_tokens_rejects_at_limit():
llm = _make_async_llm(max_num_queued_tokens=1000, num_queued_tokens=1000)
_attach_prometheus_logger(llm)
with pytest.raises(MaxQueuedTokensError):
llm.check_admission()
assert _admission_rejection_count("max_num_queued_reqs") == 0
assert _admission_rejection_count("max_num_queued_tokens") == 1
def test_admission_tokens_rejects_over_limit():
@@ -320,8 +381,11 @@ def test_admission_req_limit_checked_before_token_limit():
num_unfinished=10,
num_queued_tokens=1000,
)
_attach_prometheus_logger(llm)
with pytest.raises(QueueOverflowError):
llm.check_admission()
assert _admission_rejection_count("max_num_queued_reqs") == 1
assert _admission_rejection_count("max_num_queued_tokens") == 0
# -- n derived from params at the add_request call site ---------------------
@@ -381,6 +445,17 @@ async def test_concurrent_single_request_admission_respects_limit():
assert len(request_states) == 1
def test_admission_preflight_pass_then_reject_records_once():
llm = _make_async_llm(max_num_queued_reqs=1, num_unfinished=0)
_attach_prometheus_logger(llm)
llm.check_admission()
llm.output_processor.get_num_unfinished_requests.return_value = 1
with pytest.raises(QueueOverflowError):
llm.check_admission()
assert _admission_rejection_count("max_num_queued_reqs") == 1
assert _admission_rejection_count("max_num_queued_tokens") == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("first_n", [1, 3])
async def test_parallel_admission_is_all_or_nothing(first_n: int):
+2 -2
View File
@@ -4,7 +4,7 @@
import pytest
import torch
from vllm.v1.core.kv_cache_utils import check_enough_kv_cache_memory
from vllm.v1.core.kv_cache_planning import check_enough_kv_cache_memory
from vllm.v1.kv_cache_interface import FullAttentionSpec
@@ -38,7 +38,7 @@ def test_kv_cache_oom_insufficient_memory(monkeypatch):
config.parallel_config.decode_context_parallel_size = 1
monkeypatch.setattr(
"vllm.v1.core.kv_cache_utils.max_memory_usage_bytes",
"vllm.v1.core.kv_cache_planning.max_memory_usage_bytes",
lambda c, s: 100 * 1024**3, # 100 GiB
)
+147
View File
@@ -6,6 +6,7 @@ import time
from unittest.mock import MagicMock, Mock
import numpy as np
import prometheus_client
import pytest
import torch
@@ -17,6 +18,7 @@ from tests.v1.engine.utils import (
MockEngineCore,
)
from vllm import PoolingParams
from vllm.config.watermarking import WatermarkConfig
from vllm.logprobs import FlatLogprobs, Logprob, PromptLogprobs, SampleLogprobs
from vllm.lora.request import LoRARequest
from vllm.outputs import (
@@ -24,6 +26,7 @@ from vllm.outputs import (
PoolingRequestOutput,
RequestOutput,
SamplingMask,
WeightVersionSpan,
)
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.tokenizers import TokenizerLike
@@ -40,6 +43,9 @@ from vllm.v1.engine.output_processor import (
RequestOutputCollector,
RequestState,
)
from vllm.v1.engine.parallel_sampling import ParentRequest
from vllm.v1.metrics.loggers import PrometheusStatLogger
from vllm.v1.metrics.prometheus import unregister_vllm_metrics
from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats
from vllm.v1.outputs import SamplingMaskLists
@@ -87,6 +93,7 @@ def test_completion_output_preserves_each_sampling_mask_position() -> None:
]
state.routed_experts_chunks = []
state.spec_decode_metrics = None
state.weight_version_starts = []
output = state._new_completion_output([1, 2, 3], FinishReason.LENGTH, None)
@@ -1167,6 +1174,95 @@ def test_iteration_stats(dummy_test_vectors):
assert iteration_stats.num_generation_tokens == num_active
def test_watermark_status_partitions_finished_requests(dummy_test_vectors):
"""Each finished request gets exactly one watermark status."""
output_processor = OutputProcessor(dummy_test_vectors.tokenizer, log_stats=True)
prompt_token_ids = dummy_test_vectors.prompt_tokens[0]
def make_request(
request_id: str,
sampling_params: SamplingParams | None = None,
pooling_params: PoolingParams | None = None,
) -> EngineCoreRequest:
return EngineCoreRequest(
request_id=request_id,
external_req_id=f"{request_id}-ext",
prompt_token_ids=prompt_token_ids,
mm_features=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=sampling_params,
pooling_params=pooling_params,
)
# The frontend resolves the flags before the request reaches the
# OutputProcessor; set them as it does.
def resolved(watermarking: bool, skipped: bool, n: int = 1) -> SamplingParams:
params = SamplingParams(n=n, watermarking=watermarking)
params._watermarking_skipped = skipped
return params
finish_reasons = {
"wm": FinishReason.ABORT,
"skip": FinishReason.STOP,
# A re-resolved request can carry both flags; it is watermarked.
"sticky": FinishReason.STOP,
"off": FinishReason.ERROR,
"pool": FinishReason.STOP,
}
output_processor.add_request(make_request("wm", resolved(True, False)), None)
output_processor.add_request(make_request("skip", resolved(False, True)), None)
output_processor.add_request(make_request("sticky", resolved(True, True)), None)
output_processor.add_request(make_request("off", resolved(False, False)), None)
output_processor.add_request(
make_request("pool", pooling_params=PoolingParams(task="embed")), None
)
parent_req = ParentRequest(make_request("parent", resolved(True, False, n=2)))
for idx in range(2):
child_id, child_params = parent_req.get_child_info(idx)
output_processor.add_request(
make_request(child_id, child_params), None, parent_req, idx
)
finish_reasons[child_id] = FinishReason.LENGTH
outputs = [
EngineCoreOutput(
request_id=request_id,
new_token_ids=[] if request_id == "pool" else [prompt_token_ids[0]],
pooling_output=torch.tensor([1.0]) if request_id == "pool" else None,
finish_reason=finish_reason,
)
for request_id, finish_reason in finish_reasons.items()
]
iteration_stats = IterationStats()
output_processor.process_outputs(outputs, time.monotonic(), iteration_stats)
vllm_config = dummy_test_vectors.vllm_config
vllm_config.watermark_config = WatermarkConfig(key=42)
try:
PrometheusStatLogger(vllm_config).record(
scheduler_stats=None, iteration_stats=iteration_stats
)
totals: dict[str, dict[str, float]] = {}
for metric in prometheus_client.REGISTRY.collect():
for sample in metric.samples:
if sample.name in (
"vllm:watermark_requests_total",
"vllm:request_success_total",
):
label = sample.labels.get("status", "all")
by_label = totals.setdefault(sample.name, {})
by_label[label] = by_label.get(label, 0) + sample.value
finally:
unregister_vllm_metrics()
status = totals["vllm:watermark_requests_total"]
assert status == {"watermarked": 4, "skipped": 1, "not_watermarked": 2}
assert sum(status.values()) == totals["vllm:request_success_total"]["all"]
@pytest.mark.parametrize("log_stats", [True, False])
def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
"""Test LoRA request lifecycle tracking through waiting -> running -> finished."""
@@ -1596,6 +1692,7 @@ def test_sampling_masks_follow_output_kind(output_kind):
state.sampling_mask_chunks = []
state.routed_experts_chunks = []
state.spec_decode_metrics = None
state.weight_version_starts = []
supports = [[10, 11], [20], [30, 31]]
masks = []
@@ -1725,3 +1822,53 @@ def test_async_pooling_cache_miss_does_not_fail_output_processor():
assert output.error is not None
assert output.error.retryable
assert not output_processor.has_unfinished_requests()
@pytest.mark.parametrize(
"output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY]
)
@pytest.mark.parametrize(
"steps,expected",
[
# Merge adjacent equal labels; A -> B -> A remains three spans.
(
[("v7", [10, 11]), ("v7", [12]), ("v8", [13, 14]), ("v7", [15])],
[("v7", 0, 3), ("v8", 3, 5), ("v7", 5, 6)],
),
# An output without tokens has no spans.
([], []),
],
)
def test_weight_version_spans(output_kind: RequestOutputKind, steps, expected):
output_processor = OutputProcessor(tokenizer=None, log_stats=False)
queue = RequestOutputCollector(output_kind, request_id="request-0-int")
request = EngineCoreRequest(
request_id="request-0-int",
external_req_id="request-0",
prompt_token_ids=[1, 2, 3],
mm_features=None,
arrival_time=0,
lora_request=None,
cache_salt=None,
data_parallel_rank=None,
sampling_params=SamplingParams(detokenize=False, output_kind=output_kind),
pooling_params=None,
)
output_processor.add_request(request, prompt=None, queue=queue)
for i, (weight_version, token_ids) in enumerate(steps):
# Copy: merging deltas extends token lists in place.
output = EngineCoreOutput(
request_id="request-0-int", new_token_ids=list(token_ids)
)
output_processor.process_outputs([output], weight_version=weight_version)
if i == 0 and output_kind == RequestOutputKind.DELTA:
intermediate = queue.get_nowait()
assert intermediate is not None
assert intermediate.outputs[0].weight_versions is None
# Unread deltas merge into the abort output without losing spans.
output_processor.abort_requests(["request-0"], internal=False)
final = queue.get_nowait()
assert final is not None and final.finished
assert final.outputs[0].weight_versions == [WeightVersionSpan(*s) for s in expected]
-79
View File
@@ -1,79 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import numpy as np
from vllm.distributed.aux_output_connector.connector import (
AuxRequestOutput,
)
from vllm.v1.executor.ray_utils import detach_zero_copy_from_model_runner_output
from vllm.v1.outputs import (
LogprobsLists,
LogprobsTensors,
ModelRunnerOutput,
)
def _make_readonly(arr: np.ndarray) -> np.ndarray:
arr.setflags(write=False)
return arr
def test_detach_zero_copy_from_model_runner_output_copies_only_numpy_views():
cu_num_generated_tokens = [0, 2]
prompt_logprobs = LogprobsTensors.empty_cpu(1, 2)
output = ModelRunnerOutput(
req_ids=["req-0"],
req_id_to_index={"req-0": 0},
logprobs=LogprobsLists(
logprob_token_ids=_make_readonly(
np.array([[1, 2], [3, 4]], dtype=np.int32)
),
logprobs=_make_readonly(
np.array([[0.1, 0.2], [0.3, 0.4]], dtype=np.float32)
),
sampled_token_ranks=_make_readonly(np.array([1, 2], dtype=np.int32)),
cu_num_generated_tokens=cu_num_generated_tokens,
),
prompt_logprobs_dict={"req-0": prompt_logprobs},
)
original_logprobs = output.logprobs
assert original_logprobs is not None
detach_zero_copy_from_model_runner_output(output)
detached_logprobs = output.logprobs
assert detached_logprobs is not None
assert detached_logprobs is not original_logprobs
assert (
detached_logprobs.logprob_token_ids is not original_logprobs.logprob_token_ids
)
assert detached_logprobs.logprobs is not original_logprobs.logprobs
assert (
detached_logprobs.sampled_token_ranks
is not original_logprobs.sampled_token_ranks
)
assert detached_logprobs.logprob_token_ids.flags.writeable
assert detached_logprobs.logprobs.flags.writeable
assert detached_logprobs.sampled_token_ranks.flags.writeable
assert detached_logprobs.cu_num_generated_tokens is cu_num_generated_tokens
assert output.prompt_logprobs_dict["req-0"] is prompt_logprobs
def test_detach_zero_copy_aux_output_without_logprobs():
rows = _make_readonly(np.arange(12, dtype=np.uint8).reshape(2, 3, 2))
output = ModelRunnerOutput(
req_ids=["req-0"],
req_id_to_index={"req-0": 0},
aux_output_connector_output={"req-0": AuxRequestOutput(0, rows)},
)
detach_zero_copy_from_model_runner_output(output)
aux_output = output.aux_output_connector_output
assert aux_output is not None
detached = aux_output["req-0"].rows
assert detached is not rows
assert detached.flags.writeable
np.testing.assert_array_equal(detached, rows)
@@ -1,8 +1,12 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
from vllm.config import ParallelConfig
from vllm.v1.executor import vllm_net_devices
from vllm.v1.executor.vllm_net_devices import normalize_pci
@@ -74,3 +78,24 @@ def test_normalize_pci_device_out_of_range_raises():
def test_normalize_pci_empty_string_raises():
with pytest.raises(ValueError):
normalize_pci("")
@pytest.mark.parametrize("pcp,pp", [(1, 1), (2, 1), (2, 2)])
def test_set_worker_net_device_uses_pcp_aware_gpu_index(monkeypatch, pcp, pp):
"""NIC lookup uses the same DP-offset GPU index as device selection."""
monkeypatch.setenv("VLLM_GPU_NIC_PCIE_MAPPING", "unused")
monkeypatch.setenv("VLLM_NIC_SELECTION_VARS", "unused")
seen: list[int] = []
monkeypatch.setattr(vllm_net_devices, "set_worker_gpu_nic_mapping", seen.append)
parallel_config = ParallelConfig(
tensor_parallel_size=2,
prefill_context_parallel_size=pcp,
pipeline_parallel_size=pp,
data_parallel_size=2,
data_parallel_rank_local=1,
distributed_executor_backend="mp",
)
vllm_net_devices.set_worker_net_device(
1, SimpleNamespace(parallel_config=parallel_config)
)
assert seen == [1 + 2 * pcp * pp]
@@ -13,6 +13,7 @@ python3 -m pytest -s -x "${IMPORT_CANARY}"
tp_configs=(
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=1"
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2"
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=1 SLEEP_WAKE=1"
)
configs=("${tp_configs[@]}")
@@ -59,6 +59,12 @@ GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2}
BLOCK_SIZE=${BLOCK_SIZE:-128}
# Comma-separated extra args for vllm serve (e.g. --max-model-len,2048)
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
# Set to 1 to put every instance to sleep and wake it up before the eval.
SLEEP_WAKE=${SLEEP_WAKE:-0}
if [[ "$SLEEP_WAKE" == "1" ]]; then
export VLLM_SERVER_DEV_MODE=1
VLLM_SERVE_EXTRA_ARGS="${VLLM_SERVE_EXTRA_ARGS:+${VLLM_SERVE_EXTRA_ARGS},}--enable-sleep-mode"
fi
# Resolve the repository root from the script location instead of `.git`.
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
@@ -226,9 +232,17 @@ run_tests_for_model() {
# Wait for the proxy to start
sleep 5
SLEEP_WAKE_URLS=""
if [[ "$SLEEP_WAKE" == "1" ]]; then
for PORT in "${PREFILL_PORTS[@]}" "${DECODE_PORTS[@]}"; do
SLEEP_WAKE_URLS="${SLEEP_WAKE_URLS:+${SLEEP_WAKE_URLS},}http://localhost:${PORT}"
done
fi
# Run lm eval for this model
echo "Running tests for $model_name"
TEST_MODEL=$model_name PROXY_PORT=$PROXY_PORT python3 -m pytest -s -x \
TEST_MODEL=$model_name PROXY_PORT=$PROXY_PORT SLEEP_WAKE_URLS=$SLEEP_WAKE_URLS \
python3 -m pytest -s -x \
"${GIT_ROOT}/tests/v1/kv_connector/mooncake_integration/test_accuracy.py"
# Clean up before running next model
@@ -4,8 +4,13 @@ import os
import lm_eval
import openai
import requests
BASE_URL = f"http://localhost:{os.environ.get('PROXY_PORT', 8192)}/v1"
# Comma-separated server URLs to sleep and wake up before the eval.
SLEEP_WAKE_URLS = [
url for url in os.environ.get("SLEEP_WAKE_URLS", "").split(",") if url
]
NUM_CONCURRENT = 100
TASK = "gsm8k"
FILTER = "exact_match,strict-match"
@@ -35,9 +40,18 @@ def run_simple_prompt():
print("-" * 50)
def sleep_and_wake_up(urls: list[str]):
for url in urls:
requests.post(f"{url}/sleep", params={"level": 1}).raise_for_status()
for url in urls:
requests.post(f"{url}/wake_up").raise_for_status()
def test_accuracy():
"""Run the end to end accuracy test."""
run_simple_prompt()
if SLEEP_WAKE_URLS:
sleep_and_wake_up(SLEEP_WAKE_URLS)
model_args = (
f"model={MODEL_NAME},"
@@ -60,7 +60,6 @@ vllm serve "$MODEL" \
--block-size 128 \
--trust-remote-code \
--enable-prefix-caching \
--mamba-cache-mode align \
--kv-transfer-config "$KV_CONFIG" \
"${EXTRA_ARGS[@]}" &
@@ -78,7 +77,6 @@ vllm serve "$MODEL" \
--block-size 128 \
--trust-remote-code \
--enable-prefix-caching \
--mamba-cache-mode align \
--kv-transfer-config "$KV_CONFIG" \
"${EXTRA_ARGS[@]}" &
+1 -1
View File
@@ -23,5 +23,5 @@ def _mock_cpu_backend_block_sizes(monkeypatch):
monkeypatch.setattr(
CPUAttentionBackend,
"get_supported_kernel_block_sizes",
staticmethod(lambda: [MultipleOf(16)]),
staticmethod(lambda kv_cache_spec=None: [MultipleOf(16)]),
)
@@ -17,10 +17,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
SchedulerOffloadConfig,
)
from vllm.platforms import current_platform
from vllm.v1.core.kv_cache_utils import (
generate_scheduler_kv_cache_config,
kv_cache_groups_tp_replicas,
)
from vllm.v1.core.kv_cache_planning import kv_cache_groups_tp_replicas
from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
@@ -71,7 +69,7 @@ def _make_vllm_config(
decode_context_parallel_size=decode_context_parallel_size,
)
config.kv_events_config = None
config.use_v2_model_runner = False
config.use_v2_model_runner = True
config.kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
@@ -284,7 +282,7 @@ def _parallelism_agnostic(
kv_cache_groups: list[KVCacheGroupSpec],
*,
canonical: bool = False,
v2: bool = False,
v2: bool = True,
) -> bool:
config = _make_vllm_config(
extra_config={"canonical_layout": True} if canonical else None
@@ -308,7 +306,7 @@ def _replicated_layout(
prefill_context_parallel_size: int = 1,
decode_context_parallel_size: int = 1,
use_mla: bool = True,
use_v2_model_runner: bool = False,
use_v2_model_runner: bool = True,
distributed_executor_backend: Any = "mp",
nnodes: int = 1,
world_size: int | None = None,
@@ -1105,7 +1103,10 @@ def test_replicated_layout_parallel_gate(kwargs: dict[str, Any], case: str):
def test_parallelism_agnostic_for_single_full_attention_group():
assert _parallelism_agnostic([KVCacheGroupSpec(["l0"], _full_attention_spec())])
# The direct (non-canonical) parallelism-agnostic layout is MRV1-only.
assert _parallelism_agnostic(
[KVCacheGroupSpec(["l0"], _full_attention_spec())], v2=False
)
_SWA_SPEC = SlidingWindowSpec(
@@ -1147,7 +1148,9 @@ def _uniform_group(*specs: KVCacheSpec) -> KVCacheGroupSpec:
],
)
def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
assert not _parallelism_agnostic(kv_cache_groups)
# The direct layout is never parallelism-agnostic on MRV2, so check the
# MRV1 gate.
assert not _parallelism_agnostic(kv_cache_groups, v2=False)
@pytest.mark.parametrize(
@@ -1178,7 +1181,7 @@ def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
def test_canonical_layout_gate(kv_cache_groups, certified):
"""The canonical layout certifies portability group by group; none of
these shapes are portable in the direct layout."""
assert not _parallelism_agnostic(kv_cache_groups)
assert not _parallelism_agnostic(kv_cache_groups, v2=False)
assert _parallelism_agnostic(kv_cache_groups, canonical=True) is certified
@@ -1233,7 +1236,7 @@ def test_canonical_layout_certifies_v2_model_runner():
registration, so the static gate must not depend on the model-runner
version — the v2 runner is the case the canonical layout exists for."""
groups = _groups(_full_attention_spec())
assert _parallelism_agnostic(groups, canonical=True, v2=True)
assert _parallelism_agnostic(groups, canonical=True)
@pytest.mark.parametrize("tp_size", [1, 2, 4])
@@ -1275,7 +1278,6 @@ def test_canonical_mla_dsa_rows_are_tp_independent(tp_size, scheduler):
def test_parallelism_agnostic_disabled_on_v2_model_runner():
config = _make_vllm_config()
config.use_v2_model_runner = True
kv_cache_config = KVCacheConfig(
num_blocks=0,
kv_cache_tensors=[],
@@ -2,12 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch
from unittest.mock import MagicMock, patch
import pytest
from prometheus_client import Counter, Gauge, Histogram
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorLogging
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
KV_OFFLOAD_CONFIG_INFO,
OffloadingConnectorStats,
OffloadPromMetrics,
_ConnectorMetricName,
@@ -115,15 +117,21 @@ class _FakeVllmConfig:
def _spec_cls_with_metric_definitions(
metric_definitions: dict[str, Any],
info_keys: tuple[str, ...] = (),
) -> type:
"""Build a fake offloading spec class reporting the given metric
definitions, so tests don't need to patch the real CPU spec."""
definitions and info label names, so tests don't need to patch the real
CPU spec."""
class _FakeOffloadingSpec:
@staticmethod
def build_metric_definitions(extra_config):
return metric_definitions
@staticmethod
def config_info_keys(extra_config):
return info_keys
return _FakeOffloadingSpec
@@ -784,3 +792,333 @@ def test_prom_metrics_rejects_undeclared_metric():
_StatsKey.DATA: {"unknown:metric": {(): 1}},
}
)
def _prom_metrics(
metric_definitions: dict[str, Any] | None = None,
info_keys: tuple[str, ...] = (),
):
"""Build OffloadPromMetrics over fake metric classes, with one engine."""
spec_cls = _spec_cls_with_metric_definitions(metric_definitions or {}, info_keys)
with patch.object(OffloadingSpecFactory, "get_spec_cls", return_value=spec_cls):
return OffloadPromMetrics(
vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type]
metric_types={
Gauge: _FakeMetric,
Counter: _FakeMetric,
Histogram: _FakeMetric,
},
labelnames=["model_name", "engine"],
per_engine_labelvalues={0: ["model", "0"], 1: ["model", "1"]},
)
def test_prom_metrics_keeps_a_gauge_declared_multiprocess_mode():
"""A gauge that declares its own multiprocess_mode keeps it.
The value on OffloadingGaugeMetadata must stay a default. A hardcoded
"mostrecent" in _create_metric passes the info-gauge test below and
silently overrides a gauge that needs another merge, for example a real
cross-frontend sum.
"""
prom_metrics = _prom_metrics(
{
PENDING_STORES: OffloadingGaugeMetadata(
documentation="gauge declaring a non-default multiprocess mode.",
multiprocess_mode="sum",
),
}
)
gauge_def = prom_metrics._offloading_metric_defs[PENDING_STORES]
assert gauge_def.kwargs["multiprocess_mode"] == "sum"
def test_prom_metrics_omits_multiprocess_mode_outside_a_gauge():
"""multiprocess_mode reaches a gauge only.
prometheus_client accepts the argument on Gauge alone, so hoisting the
kwarg out of the gauge branch breaks construction of a real Counter or
Histogram.
"""
prom_metrics = _prom_metrics(
{
MY_COUNTER: OffloadingCounterMetadata(documentation="a counter."),
LOOKUP_LATENCY: OffloadingHistogramMetadata(
documentation="a histogram.",
buckets=(0.1, 1),
),
}
)
counter_def = prom_metrics._offloading_metric_defs[MY_COUNTER]
histogram_def = prom_metrics._offloading_metric_defs[LOOKUP_LATENCY]
assert "multiprocess_mode" not in counter_def.kwargs
assert "multiprocess_mode" not in histogram_def.kwargs
def test_prom_metrics_declares_the_info_gauge_from_the_spec_keys():
"""The API-server process holds no manager, but it does hold the spec, so
it declares the gauge at startup, before any payload."""
prom_metrics = _prom_metrics(info_keys=("cpu_num_chunks", "root_dir_name"))
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert gauge_def.kwargs["name"] == KV_OFFLOAD_CONFIG_INFO
assert gauge_def.kwargs["labelnames"] == [
"model_name",
"engine",
"cpu_num_chunks",
"root_dir_name",
]
# The frontend declares the metric and binds no label values, so an engine
# that never reports exposes no series.
assert gauge_def.children == []
prom_metrics.observe(
{_StatsKey.INFO: [{"cpu_num_chunks": 512, "root_dir_name": "a"}]}
)
# Every value renders as a label value, and the metric itself is pinned
# to 1.
(gauge,) = gauge_def.children
assert gauge.labelvalues == ("model", "0", "512", "a")
assert gauge.set_values == [1]
def test_prom_metrics_reads_the_info_payload_by_label_name():
"""A manager orders config_info() as it likes, and msgpack keeps that
order, so the frontend must key on the name rather than on the position."""
prom_metrics = _prom_metrics(info_keys=("first", "second"))
prom_metrics.observe({_StatsKey.INFO: [{"second": 2, "first": 1}]})
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
(gauge,) = gauge_def.children
assert gauge.labelvalues == ("model", "0", "1", "2")
def test_prom_metrics_reports_one_info_series_for_each_mapping():
"""A manager reports one mapping for each series, and a label of its own
tells the series apart. Two mappings of one kind agree on every other value,
so without that label they collapse into one series."""
prom_metrics = _prom_metrics(info_keys=("source", "root_dir_name"))
prom_metrics.observe(
{
_StatsKey.INFO: [
{"source": "1", "root_dir_name": "a"},
{"source": "2", "root_dir_name": "b"},
]
}
)
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert [child.labelvalues for child in gauge_def.children] == [
("model", "0", "1", "a"),
("model", "0", "2", "b"),
]
assert [child.set_values for child in gauge_def.children] == [[1], [1]]
def test_prom_metrics_declares_the_info_gauge_as_most_recent():
"""The info gauge must merge across frontends by freshest write.
Offloading stats reach one frontend per step as complete per-engine
snapshots, so summing would report the number of participating API-server
processes instead of the 1 an info gauge is pinned to -- wrong only under
multiprocess deployment, and silently, since the facts ride the labels.
"""
prom_metrics = _prom_metrics(info_keys=("cpu_num_chunks",))
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert gauge_def.kwargs["multiprocess_mode"] == "mostrecent"
def test_prom_metrics_reports_the_info_gauge_without_manager_labels():
"""A spec that declares no fact still yields the metric, so its presence
alone answers whether offloading runs."""
prom_metrics = _prom_metrics()
prom_metrics.observe({_StatsKey.INFO: [{}]})
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert gauge_def.kwargs["labelnames"] == ["model_name", "engine"]
(gauge,) = gauge_def.children
assert gauge.labelvalues == ("model", "0")
def test_prom_metrics_empties_a_declared_info_label_no_manager_fills():
"""A name the spec declares and the manager skips must publish empty.
PromQL reads an empty label value as an absent label, so the series still
matches both shapes. This code runs on the engine output path of the API
server, where an exception ends the output handler, so it warns and goes
on rather than drop the whole series.
"""
prom_metrics = _prom_metrics(info_keys=("filled", "skipped"))
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics.logger"
) as mock_logger:
prom_metrics.observe({_StatsKey.INFO: [{"filled": 512}]})
assert mock_logger.warning_once.call_count == 1
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
(gauge,) = gauge_def.children
assert gauge.labelvalues == ("model", "0", "512", "")
def test_prom_metrics_keeps_quiet_when_a_name_belongs_to_another_series():
"""A name that one series owns reads empty on the other series by design.
The declaration holds the names of every series, so the frontend checks the
payload as a whole. A per-series check would read this normal shape as a
disagreement, and warn at every start of a server that reports several
series.
"""
prom_metrics = _prom_metrics(info_keys=("source", "root_dir_name", "bucket"))
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics.logger"
) as mock_logger:
prom_metrics.observe(
{
_StatsKey.INFO: [
{"source": "1", "root_dir_name": "kv_a"},
{"source": "2", "bucket": "kv_bucket"},
]
}
)
mock_logger.warning_once.assert_not_called()
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert [child.labelvalues for child in gauge_def.children] == [
("model", "0", "1", "kv_a", ""),
("model", "0", "2", "", "kv_bucket"),
]
def test_prom_metrics_drops_an_info_label_the_spec_did_not_declare():
"""A Prometheus metric binds its label names once, so a name that the spec
did not declare cannot join the gauge. It warns and publishes the rest."""
prom_metrics = _prom_metrics(info_keys=("declared",))
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics.logger"
) as mock_logger:
prom_metrics.observe({_StatsKey.INFO: [{"declared": 512, "undeclared": 1}]})
assert mock_logger.warning_once.call_count == 1
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert gauge_def.kwargs["labelnames"] == ["model_name", "engine", "declared"]
(gauge,) = gauge_def.children
assert gauge.labelvalues == ("model", "0", "512")
def test_prom_metrics_reports_the_info_of_every_engine():
"""Every engine of one instance runs one offloading config, so they share
the gauge and differ by the engine label alone."""
prom_metrics = _prom_metrics(info_keys=("cpu_num_chunks",))
prom_metrics.observe({_StatsKey.INFO: [{"cpu_num_chunks": 512}]}, engine_idx=0)
prom_metrics.observe({_StatsKey.INFO: [{"cpu_num_chunks": 512}]}, engine_idx=1)
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
assert [child.labelvalues for child in gauge_def.children] == [
("model", "0", "512"),
("model", "1", "512"),
]
def test_stats_hold_the_config_info_without_an_observation():
"""An info-only payload must survive the scheduler gate, which drops an
empty payload, and must add no line to the interval log."""
stats = OffloadingConnectorStats()
assert stats.is_empty()
stats.set_info([{"cpu_num_chunks": 512}])
assert not stats.is_empty()
assert stats.reduce() == {}
def test_aggregate_keeps_the_config_info_of_the_other_payload():
"""The scheduler payload holds the info and merges into the worker payload
(v1/core/sched/scheduler.py), so a dropped info would hide the metric on
every step that also transfers, which is every step that matters."""
worker_stats = OffloadingConnectorStats()
worker_stats.increase_counter(LOAD_BYTES, 100)
scheduler_stats = OffloadingConnectorStats()
scheduler_stats.set_info([{"cpu_num_chunks": 512}])
worker_stats.aggregate(scheduler_stats)
assert worker_stats.data[_StatsKey.INFO] == [{"cpu_num_chunks": 512}]
assert worker_stats.data[_StatsKey.DATA][LOAD_BYTES] == {(): 100}
def test_aggregate_keeps_the_config_info_of_a_later_payload():
"""The scheduler sends the info once, so a later payload must not clear
it."""
stats = OffloadingConnectorStats()
stats.set_info([{"cpu_num_chunks": 512}])
later_stats = OffloadingConnectorStats()
later_stats.increase_counter(LOAD_BYTES, 100)
stats.aggregate(later_stats)
assert stats.data[_StatsKey.INFO] == [{"cpu_num_chunks": 512}]
def test_metric_sections_leave_out_the_config_info():
"""A manager accesses its own metrics by a walk over the metric sections, as
KVCRSecondaryTierManager does in v1/kv_offload/tiering/kvcr/manager.py. The
info section holds label names, not metric names, so it stays out of that
walk."""
stats = OffloadingConnectorStats()
stats.increase_counter(MY_COUNTER, 2, ("remote_deliver",))
stats.set_info([{"info_label": "local"}])
info = stats.data[_StatsKey.INFO]
sections = stats.metric_sections()
assert not any(section is info for section in sections)
assert sections == (stats.data[_StatsKey.TYPES], stats.data[_StatsKey.DATA])
def test_logging_writes_no_line_for_an_info_only_payload():
"""An info-only payload reaches the logging path, and reduce() drops the info.
The info belongs to the Prometheus frontend only. Without the guard in
KVConnectorLogging.log(), such a payload writes "KV Transfer metrics: "
with an empty body.
"""
stats = OffloadingConnectorStats()
stats.set_info([{"info_label": "local"}])
assert not stats.is_empty()
assert stats.reduce() == {}
kv_logging = KVConnectorLogging(None)
kv_logging.transfer_stats_accumulator = stats
log_fn = MagicMock()
kv_logging.log(log_fn=log_fn)
log_fn.assert_not_called()
assert kv_logging.transfer_stats_accumulator is None
def test_scheduler_sends_the_config_info_once(request_runner):
"""MockOffloadingSpec publishes no fact, so the payload holds one empty
mapping, which still yields the metric."""
runner = request_runner(
block_size=4,
num_gpu_blocks=10,
async_scheduling=False,
)
stats = runner.connector_scheduler.get_stats()
assert stats is not None
assert stats.data[_StatsKey.INFO] == [{}]
assert runner.connector_scheduler.get_stats() is None
@@ -1017,6 +1017,129 @@ def test_scheduler_reports_lookup_sync_delay(request_runner):
assert reduced[f"{_ConnectorMetricName.LOOKUP_SYNC_DELAY}_sum"] > 0
def test_on_schedule_end_is_the_last_manager_call_of_a_step(request_runner):
"""on_schedule_end must run after every other manager call of the step.
Callers bound a step's exclusive access to the manager by releasing at
on_schedule_end, so anything issued after it — notably prepare_store, which
evicts — would fall outside that window.
"""
block_size = 4
runner = request_runner(
block_size=block_size,
num_gpu_blocks=8,
async_scheduling=False,
)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output([])
)
connector_scheduler = runner.connector_scheduler
build_meta = connector_scheduler.build_connector_meta
windows: list[list[str]] = []
def spy(scheduler_output):
first = len(runner.manager.mock_calls)
meta = build_meta(scheduler_output)
# Entering and exiting manager.lock brackets the step; only the state
# operations it guards are ordered relative to on_schedule_end.
windows.append(
[
name
for name, _, _ in runner.manager.mock_calls[first:]
if not name.startswith("lock.")
]
)
return meta
connector_scheduler.build_connector_meta = spy
runner.new_request(token_ids=[0] * (block_size * 2))
runner.run(decoded_tokens=[EOS_TOKEN_ID])
assert windows, "build_connector_meta was never called"
assert any("prepare_store" in w for w in windows), (
"step must exercise the calls that follow on_schedule_end today"
)
assert any("on_request_finished" in w for w in windows)
for names in windows:
assert names[-1] == "on_schedule_end", (
f"on_schedule_end must be the last manager call of a step, got {names}"
)
def test_step_releases_the_manager_before_the_model_runs(request_runner):
"""The step-scoped region must close when build_connector_meta returns.
Everything after it -- the model future above all -- runs with the manager
free, which is the only window another thread has to use it.
"""
runner = request_runner(
block_size=4,
num_gpu_blocks=10,
async_scheduling=False,
)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output([])
)
runner.new_request(token_ids=[1] * 4)
runner.run(decoded_tokens=[EOS_TOKEN_ID])
scheduler = runner.connector_scheduler
assert scheduler._step_region is None
lock = runner.manager.lock
assert lock.__enter__.call_count > 0
assert lock.__enter__.call_count == lock.__exit__.call_count
def test_has_pending_push_work_does_not_open_the_step_region(request_runner):
"""It must take a short region, never the step-scoped one.
The engine asks this before schedule() and returns early when the answer is
False, so a step-scoped acquire here would never reach the release in
build_connector_meta and would hold the manager for good.
"""
runner = request_runner(
block_size=4,
num_gpu_blocks=10,
async_scheduling=False,
)
scheduler = runner.connector_scheduler
runner.manager.has_pending_work.return_value = False
assert scheduler.has_pending_push_work() is False
assert scheduler._step_region is None
lock = runner.manager.lock
assert lock.__enter__.call_count == lock.__exit__.call_count
def test_take_events_drains_the_manager_inside_the_region(request_runner):
"""take_events must not hold the manager across its own iteration.
manager.take_events() is a generator whose backing list is cleared once
exhausted, so draining it lazily would both pin the manager for as long as
the consumer takes and lose events appended in the meantime.
"""
runner = request_runner(
block_size=4,
num_gpu_blocks=10,
async_scheduling=False,
)
scheduler = runner.connector_scheduler
runner.manager.take_events.return_value = iter(())
runner.manager.reset_mock()
events = list(scheduler.take_events())
assert events == []
assert runner.manager.take_events.call_count == 1
assert scheduler._step_region is None
lock = runner.manager.lock
assert lock.__enter__.call_count == 1
assert lock.__exit__.call_count == 1
def test_scheduler_reports_lookup_async_delay_on_resolve(request_runner):
"""A deferred lookup reports its async delay once it resolves."""
runner = request_runner(
@@ -132,6 +132,7 @@ class MockOffloadingSpec(OffloadingSpec):
self.manager.lookup.return_value = LookupResult.MISS
self.manager.get_load_source.return_value = CacheHitSource.EXTERNAL_UNSPECIFIED
self.manager.get_stats.return_value = None
self.manager.config_info.return_value = [{}]
self.manager.on_new_request.return_value = RequestOffloadingContext()
self.handler = MockOffloadingWorker()
@@ -51,6 +51,7 @@ from .utils import (
create_scheduler,
create_vllm_config,
make_kv_cache_config,
maybe_update_block_size,
)
pytestmark = pytest.mark.cpu_test
@@ -87,10 +88,11 @@ def _make_connector_with_fake_worker(
hand_shake_latency=0, cycles_before_done=0, do_handshake=True
):
"""Create a NixlConnector with FakeNixlConnectorWorker."""
block_size = maybe_update_block_size(32)
vllm_config = create_vllm_config(
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=32
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=block_size
)
kv_cache_config = make_kv_cache_config(block_size=32, num_blocks=2)
kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=2)
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config,
+28 -2
View File
@@ -101,6 +101,7 @@ def test_kv_connector(
def _build_config(
*,
kv_connector: str | None,
kv_connector_extra_config: dict | None = None,
enable_sleep_mode: bool = False,
enable_cumem_allocator: bool = False,
) -> VllmConfig:
@@ -109,7 +110,11 @@ def _build_config(
from types import SimpleNamespace
kv_transfer_config = (
KVTransferConfig(kv_connector=kv_connector, kv_role="kv_both")
KVTransferConfig(
kv_connector=kv_connector,
kv_role="kv_both",
kv_connector_extra_config=kv_connector_extra_config or {},
)
if kv_connector is not None
else None
)
@@ -141,7 +146,7 @@ def test_kv_connector_allows_expandable_segments_with_sleep_mode(monkeypatch):
"""Sleep mode routes KV allocations through CuMemAllocator's pool, which
auto-disables expandable_segments (see #40812)."""
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
_build_config(kv_connector="NixlConnector", enable_sleep_mode=True)
_build_config(kv_connector="MooncakeConnector", enable_sleep_mode=True)
def test_kv_connector_allows_expandable_segments_with_cumem_allocator(
@@ -152,6 +157,27 @@ def test_kv_connector_allows_expandable_segments_with_cumem_allocator(
_build_config(kv_connector="NixlConnector", enable_cumem_allocator=True)
@pytest.mark.parametrize(
("kv_connector", "extra_config", "env"),
[
("NixlConnector", {}, None),
("MooncakeConnector", {"mooncake_protocol": "tcp"}, None),
("MooncakeConnector", {}, "MC_USE_TENT"),
],
)
def test_sleep_mode_refuses_unsupported_kv_connector(
monkeypatch, kv_connector, extra_config, env
):
if env:
monkeypatch.setenv(env, "1")
with pytest.raises(ValueError, match=f"^{kv_connector} does not support sleep"):
_build_config(
kv_connector=kv_connector,
kv_connector_extra_config=extra_config,
enable_sleep_mode=True,
)
def test_kv_connector_allows_other_alloc_conf(monkeypatch):
"""Other PYTORCH_CUDA_ALLOC_CONF values must not be rejected."""
monkeypatch.setenv(
@@ -10,7 +10,7 @@ import torch
from vllm.distributed.kv_transfer.kv_connector.v1.example_hidden_states_connector import ( # noqa: E501
ExampleHiddenStatesConnector,
)
from vllm.v1.core.kv_cache_utils import get_kv_cache_groups
from vllm.v1.core.kv_cache_planning import get_kv_cache_groups
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
HiddenStateCacheSpec,
@@ -3,7 +3,7 @@
import contextlib
import ctypes
from types import SimpleNamespace
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
import pytest
import torch
@@ -25,14 +25,20 @@ from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
from vllm.v1.hisparse import runtime as runtime_module
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.hisparse.runtime import HiSparseCacheHandle
from vllm.v1.hisparse.runtime import (
HiSparseCacheHandle,
update_hisparse_residency,
)
from vllm.v1.hisparse.types import (
SparseKVOffloadCommand,
SparseKVPageTransfer,
SparseKVResidencyUpdate,
SparseKVRowMirror,
)
from vllm.v1.metrics.stats import KVCacheEvictionEvent
from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector
@@ -100,6 +106,46 @@ def test_scheduler_stats_report_host_pool_usage():
assert stats.data["host_cache_usage_perc"] == [expected]
def test_hisparse_host_residency_reported_as_hisparse_metrics():
"""Host blocks sample into HiSparse's own residency metrics."""
from tests.v1.core.test_prefix_caching import (
make_hisparse_kv_cache_config,
make_kv_cache_manager,
)
from tests.v1.core.utils import create_requests
collector = KVCacheMetricsCollector(sample_rate=1.0)
manager = make_kv_cache_manager(
make_hisparse_kv_cache_config(2, 2),
max_model_len=128,
enable_caching=True,
hash_block_size=16,
metrics_collector=collector,
)
coordinator = get_hisparse_coordinator(manager)
assert coordinator.host_manager is not None
scheduler = HiSparseConnectorScheduler(async_speculative=False)
scheduler.bind_coordinator(coordinator)
device = manager.block_pool
host = coordinator.host_manager.block_pool
request = create_requests(1, num_tokens=16)[0]
for pool, birth in ((device, 1), (host, 2)):
with patch("time.monotonic_ns", return_value=birth * 10**9):
block = pool.get_new_blocks(1)[0]
assert block.block_id == 1
pool.cache_full_blocks(request, [block], 0, 1, 16, 0)
with patch("time.monotonic_ns", return_value=4_000_000_000):
host.free_blocks([host.blocks[1]])
host.evict_blocks({1})
with patch("time.monotonic_ns", return_value=6_000_000_000):
device.evict_blocks({1})
assert collector.drain_events() == [KVCacheEvictionEvent(5.0, 5.0, ())]
stats = scheduler.get_kv_connector_stats()
assert stats.data["host_block_lifetime_seconds"] == [2.0]
assert stats.data["host_block_idle_before_evict_seconds"] == [2.0]
def test_no_forward_enqueues_deferred_hisparse_transfers():
"""A zero-token step must still enqueue deferred post-forward transfers."""
connector = object.__new__(ActiveKVConnector)
@@ -141,7 +187,7 @@ def test_full_graph_step_prepares_host_mirror_outside_model():
connector = object.__new__(HiSparseConnector)
connector.connector_worker = worker
connector._get_connector_metadata = MagicMock(
return_value=HiSparseConnectorMetadata(None, (), (), {}, True)
return_value=HiSparseConnectorMetadata(None, (), (), {}, True, {})
)
req_id_per_token = torch.tensor([0, 1], dtype=torch.int32)
attn_metadata = SimpleNamespace(
@@ -279,6 +325,7 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
)
},
True,
{},
)
)
# A verification step: four query tokens of one request fill the page.
@@ -312,7 +359,7 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
# The next step's start mirrors the drafter's rows, then hands the page over.
compute_stream.wait_event.reset_mock()
connector._get_connector_metadata.return_value = HiSparseConnectorMetadata(
None, (), (), {}, True
None, (), (), {}, True, {}
)
connector.start_load_kv(
SimpleNamespace(),
@@ -354,8 +401,56 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
assert worker_meta.completed_transfer_counts == {transfer_id: 1}
@pytest.mark.parametrize("shared", [True, False])
@pytest.mark.parametrize("num_tokens", [0, 4])
def test_zero_token_step_holds_tp_ranks_after_host_write_wait(
monkeypatch, shared, num_tokens
):
"""Steps without tokens hold every TP rank after its shared event wait."""
calls: list[tuple[str, object]] = []
compute_stream = MagicMock()
compute_stream.wait_event.side_effect = lambda event: calls.append(("wait", event))
tp_group = MagicMock()
tp_group.barrier.side_effect = lambda: calls.append(("barrier", None))
monkeypatch.setattr(worker_module, "current_stream", lambda: compute_stream)
monkeypatch.setattr(worker_module, "get_tp_group", lambda: tp_group)
worker = object.__new__(HiSparseConnectorWorker)
worker.shared_host_region = MagicMock() if shared else None
worker.host_write_events = (MagicMock(), MagicMock())
worker.host_write_event = worker.host_write_events[1]
worker._next_host_write_event = 0
worker._slot_mapping_staging = None
worker.cache_handles = []
worker._pending_invalid_block_ids = []
for name in (
"_stage_row_mirror_mapping",
"_finish_previous_step",
"_release_completed_dma_descriptors",
"_set_row_mirrors",
"_clear_forward_mirror_state",
"_copy_host_blocks",
"_restore_pages",
"_submit_transfers",
):
setattr(worker, name, MagicMock())
worker.start_step(
HiSparseConnectorMetadata(None, (), (), {}, True, {}),
None,
[],
num_tokens=num_tokens,
)
expected = [("wait", worker.host_write_events[1])]
if shared and not num_tokens:
expected.append(("barrier", None))
assert calls == expected
def test_scheduled_prefix_hit_publishes_adopted_copies():
"""Copies adopted after scheduling must reach the worker's block table."""
"""Copies adopted after scheduling reach the worker as a residency update,
leaving the block-table row the scheduler output carries untouched."""
from tests.v1.core.test_prefix_caching import (
HISPARSE_BLOCK_SIZE,
_allocate_scheduled,
@@ -401,7 +496,39 @@ def test_scheduled_prefix_hit_publishes_adopted_copies():
num_scheduled_tokens={resumed.request_id: num_new_tokens},
)
scheduler.build_connector_meta(scheduler_output)
core_row = list(scheduler_output.scheduled_new_reqs[0].block_ids[2])
resident_ids = scheduler_output.block_table_updates[resumed.request_id][2]
assert resident_ids[:3] == copy_ids[:3]
metadata = scheduler.build_connector_meta(scheduler_output)
update = metadata.residency_updates[resumed.request_id]
assert update.pages == [0, 1, 2, 3]
assert update.block_ids[0][:3] == copy_ids[:3]
assert manager.get_block_ids(resumed.request_id)[2] == core_row
assert core_row[:3] == [0, 0, 0]
def test_residency_updates_persist_by_state_row():
"""Updates name requests by batch row but are stored by state row.
An update for a lost page and an appended page must leave the other pages
and the other request's row intact across steps that reorder the batch.
"""
table = torch.zeros((4, 2, 4), dtype=torch.int32)
update_hisparse_residency(
table,
{
"a": SparseKVResidencyUpdate([0, 1, 2], ([1, 2, 3], [11, 12, 13])),
"b": SparseKVResidencyUpdate([0, 1], ([4, 5], [14, 15])),
},
["a", "b"],
torch.tensor([2, 0], dtype=torch.int32),
)
update_hisparse_residency(
table,
{"a": SparseKVResidencyUpdate([1, 3], ([0, 6], [0, 16]))},
["b", "a"],
torch.tensor([0, 2], dtype=torch.int32),
)
assert table[[0, 2], 0].tolist() == [[4, 5, 0, 0], [1, 0, 3, 6]]
assert table[[0, 2], 1].tolist() == [[14, 15, 0, 0], [11, 0, 13, 16]]
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
)
@@ -82,7 +84,8 @@ def test_build_kv_connector_stats_round_trip():
assert rebuilt.reduce() == stats.reduce()
def test_prom_metrics_observe_host_usage_gauges():
@pytest.mark.parametrize("kv_cache_metrics", [True, False])
def test_prom_metrics_observe_host_metrics(kv_cache_metrics: bool):
from types import SimpleNamespace
from typing import Any
@@ -91,12 +94,14 @@ def test_prom_metrics_observe_host_usage_gauges():
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
HiSparsePromMetrics,
)
from vllm.v1.metrics.stats import KVCacheEvictionEvent
class _FakeMetric:
def __init__(self, **kwargs: Any):
self.kwargs = kwargs
self.increments: list[int | float] = []
self.set_values: list[int | float] = []
self.observed: list[float] = []
def labels(self, *labelvalues: object) -> "_FakeMetric":
return self
@@ -107,6 +112,9 @@ def test_prom_metrics_observe_host_usage_gauges():
def set(self, value: int | float) -> None:
self.set_values.append(value)
def observe(self, value: float) -> None:
self.observed.append(value)
created: dict[str, _FakeMetric] = {}
class _NamedFake(_FakeMetric):
@@ -115,7 +123,12 @@ def test_prom_metrics_observe_host_usage_gauges():
created[kwargs["name"]] = self
prom = HiSparsePromMetrics(
vllm_config=SimpleNamespace(kv_transfer_config=None),
vllm_config=SimpleNamespace(
kv_transfer_config=None,
observability_config=SimpleNamespace(
kv_cache_metrics=kv_cache_metrics, custom_histogram_buckets=None
),
),
metric_types={Gauge: _NamedFake, Counter: _NamedFake, Histogram: _NamedFake},
labelnames=["model_name"],
per_engine_labelvalues={0: ["model"]},
@@ -125,9 +138,21 @@ def test_prom_metrics_observe_host_usage_gauges():
stats.record_snapshot(hits=3, misses=2, host_to_device_bytes=32)
stats.record_host_usage(usage=0.5, pending_page_transfers=1)
stats.record_host_usage(usage=0.75, pending_page_transfers=3)
prom.observe(stats.to_dict())
stats.record_host_evictions([KVCacheEvictionEvent(5.0, 2.0, (1.0, 2.0))])
later = HiSparseKVConnectorStats()
later.record_host_evictions([KVCacheEvictionEvent(3.0, 3.0, ())])
prom.observe(stats.aggregate(later).to_dict())
assert created["vllm:hisparse_cache_hits"].increments == [3]
assert created["vllm:hisparse_host_to_device_bytes"].increments == [32]
assert created["vllm:hisparse_host_cache_usage_perc"].set_values == [0.75]
assert created["vllm:hisparse_pending_page_transfers"].set_values == [3]
if not kv_cache_metrics:
assert not any("host_block" in name for name in created)
return
assert created["vllm:hisparse_host_block_lifetime_seconds"].observed == [5.0, 3.0]
assert created["vllm:hisparse_host_block_idle_before_evict_seconds"].observed == [
2.0,
3.0,
]
assert created["vllm:hisparse_host_block_reuse_gap_seconds"].observed == [1.0, 2.0]
@@ -27,7 +27,6 @@ def _make_empty_scheduler_output():
total_num_scheduled_tokens=0,
scheduled_spec_decode_tokens={},
scheduled_encoder_inputs={},
num_common_prefix_blocks=[],
finished_req_ids=set(),
free_encoder_mm_hashes=[],
kv_connector_metadata=ExampleConnectorMetadata(),
@@ -11,6 +11,7 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import msgspec
import pytest
import torch
import zmq.asyncio
@@ -57,7 +58,12 @@ from vllm.v1.kv_cache_interface import (
)
from vllm.v1.request import RequestStatus
from .utils import create_request, create_scheduler, create_vllm_config
from .utils import (
create_request,
create_scheduler,
create_vllm_config,
maybe_update_block_size,
)
@pytest.mark.parametrize(
@@ -148,6 +154,7 @@ def test_region_length_validation_checks_replicated_gqa_heads(
def _make_test_kv_cache_config() -> KVCacheConfig:
block_size = maybe_update_block_size(16)
return KVCacheConfig(
num_blocks=0,
kv_cache_tensors=[],
@@ -160,7 +167,7 @@ def _make_test_kv_cache_config() -> KVCacheConfig:
"model.layers.1.eagle_attn",
],
FullAttentionSpec(
block_size=16,
block_size=block_size,
num_kv_heads=4,
head_size=64,
dtype=torch.float16,
@@ -170,6 +177,14 @@ def _make_test_kv_cache_config() -> KVCacheConfig:
)
def _make_worker_vllm_config(kv_role: str):
return create_vllm_config(
kv_connector="MooncakeConnector",
kv_role=kv_role,
block_size=maybe_update_block_size(16),
)
def _make_packed_mla_kv_cache_config(
num_blocks: int, *, num_groups: int = 1
) -> KVCacheConfig:
@@ -218,6 +233,9 @@ class FakeMooncakeWrapper:
def batch_register_memory(self, buffer_addresses, capacities) -> int:
return 0
def batch_unregister_memory(self, buffer_addresses) -> int:
return 0
def test_align_transfer_regions_uses_layer_name_occurrences():
"""Repeated layer names should align by occurrence order."""
@@ -429,9 +447,7 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata(
):
"""Producer sends its PP layer shard to the matching consumer layer address."""
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
vllm_config = _make_worker_vllm_config("kv_producer")
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
prefill_connector = MooncakeConnector(
@@ -1205,7 +1221,7 @@ def _xfer_meta(
@contextlib.contextmanager
def mooncake_register_worker(kv_cache_config, *, kv_role: str = "kv_consumer"):
"""Build a worker whose engine registration and sender thread are patched."""
vllm_config = create_vllm_config(kv_connector="MooncakeConnector", kv_role=kv_role)
vllm_config = _make_worker_vllm_config(kv_role)
connector_mod = (
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector"
)
@@ -1241,9 +1257,7 @@ async def test_receive_kv_selects_remote_pp_workers(
expected_addrs: list[str],
):
"""Decode workers should not hard-code producer pp_rank 0."""
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_consumer"
)
vllm_config = _make_worker_vllm_config("kv_consumer")
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
decode_connector = MooncakeConnector(
@@ -1314,7 +1328,7 @@ def test_resolve_need_send_accounts_for_remote_tp_fanout():
@pytest.mark.asyncio
async def test_heterogeneous_pp_waits_for_consumer_without_shared_layers():
"""P4/D2 must retain KV until both D stages finish, including a no-op pull."""
config = create_vllm_config(kv_connector="MooncakeConnector", kv_role="kv_producer")
config = _make_worker_vllm_config("kv_producer")
with (
set_current_vllm_config(config),
patch_worker_dependencies(),
@@ -1386,9 +1400,7 @@ async def test_kv_producer(monkeypatch):
Verifies memory offset calculation: ptr = base_addr + block_id * block_len.
"""
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
vllm_config = _make_worker_vllm_config("kv_producer")
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
prefill_connector = MooncakeConnector(
@@ -1589,9 +1601,7 @@ async def test_kv_consumuer(monkeypatch):
Verifies that MooncakeXferMetadata is correctly serialized and sent via ZMQ.
"""
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_consumer"
)
vllm_config = _make_worker_vllm_config("kv_consumer")
with set_current_vllm_config(vllm_config), patch_worker_dependencies() as mocks:
decode_connector = MooncakeConnector(
@@ -1661,9 +1671,7 @@ async def test_kv_consumuer(monkeypatch):
@pytest.mark.asyncio
async def test_worker_get_finished_timeout(monkeypatch):
"""Tests the cleanup mechanism for requests."""
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
vllm_config = _make_worker_vllm_config("kv_producer")
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
prefill_connector = MooncakeConnector(
vllm_config,
@@ -1713,7 +1721,7 @@ async def test_worker_get_finished_timeout(monkeypatch):
def test_register_kv_caches(layout: KVCacheLayout, separate_kv_head_groups: bool):
"""Tests the memory registration logic with the underlying Mooncake engine."""
spec = FullAttentionSpec(
block_size=16,
block_size=maybe_update_block_size(16),
num_kv_heads=4,
head_size=64,
dtype=torch.float16,
@@ -1794,7 +1802,10 @@ def test_register_noncontiguous_packed_allows_matching_pp():
known. Rejecting every local PP>1 layout at startup is too strict.
"""
spec = FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
block_size=maybe_update_block_size(16),
num_kv_heads=4,
head_size=64,
dtype=torch.float16,
)
layer_names = ["model.layers.0.self_attn", "model.layers.1.self_attn"]
with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, _reg):
@@ -1814,8 +1825,9 @@ def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, reg):
worker.use_mla = True
worker.transfer_topo.is_mla = True
mla_cache = torch.zeros((2, 16, 512), dtype=torch.float16)
eagle_cache = torch.zeros((2, 16, 8, 64), dtype=torch.float16)
block_size = maybe_update_block_size(16)
mla_cache = torch.zeros((2, block_size, 512), dtype=torch.float16)
eagle_cache = torch.zeros((2, block_size, 8, 64), dtype=torch.float16)
kv_caches = {
"model.layers.0.mla_attn": mla_cache,
"model.layers.1.eagle_attn": eagle_cache,
@@ -2218,6 +2230,215 @@ def test_prepare_transfer_regions_reuses_success_and_error():
assert calls == 6
def _short_xfer_meta(worker, layout_id: int, req_blocks: dict) -> MooncakeXferMetadata:
return MooncakeXferMetadata(
remote_hostname="peer",
remote_port=9,
remote_tp_size=worker.tp_size,
remote_tp_rank=0,
remote_pp_size=worker.pp_size,
layout_id=layout_id,
req_blocks=req_blocks,
kv_caches_base_addr=[],
block_lens=[],
kv_block_lens=[],
)
@pytest.mark.asyncio
async def test_steady_pull_omits_layout_and_unknown_id_asks_for_resend():
"""A later pull is much smaller, and an unseen layout_id is not guessed."""
n = 146
names = [f"model.layers.{i}.self_attn" for i in range(n)]
req_blocks = {"d": ("xfer", [[1, 2, 3]])}
full = MooncakeXferMetadata(
remote_hostname="h",
remote_port=1,
remote_tp_size=4,
remote_tp_rank=0,
layout_id=1,
req_blocks=req_blocks,
kv_caches_base_addr=list(range(n)),
block_lens=[4096] * n,
kv_block_lens=[3000] * n,
registered_layer_names=names,
registered_layer_indices=list(range(n)),
registered_group_indices=[i % 5 for i in range(n)],
registered_shared_group_ids=[[] for _ in range(n)],
registered_row_offsets=[i * 100 for i in range(n)],
)
steady = MooncakeXferMetadata(
remote_hostname="h",
remote_port=1,
remote_tp_size=4,
remote_tp_rank=0,
layout_id=1,
req_blocks=req_blocks,
kv_caches_base_addr=[],
block_lens=[],
kv_block_lens=[],
)
encoder = msgspec.msgpack.Encoder()
full_bytes = len(encoder.encode(full))
steady_bytes = len(encoder.encode(steady))
assert steady_bytes < full_bytes * 0.2
worker, _, regions, _ = _register_sliced_packed_mla([0, 1])
calls = 0
original = worker._get_transfer_regions
def counting(*args, **kwargs):
nonlocal calls
calls += 1
return original(*args, **kwargs)
worker._get_transfer_regions = counting
def handshake(layout_id: int) -> MooncakeXferMetadata:
return _xfer_meta(
regions,
req_blocks,
remote_hostname="peer",
remote_port=9,
remote_tp_size=worker.tp_size,
remote_pp_size=worker.pp_size,
layout_id=layout_id,
)
prepared = worker._prepare_transfer_regions(handshake(1))
assert prepared[2] is None
assert calls == 2
stored = worker._layout_by_peer[worker._peer_layout_key(handshake(1))]
assert stored == (prepared[0], prepared[1])
# Same layout under a new id hits the tuple cache and must still be stored.
worker._prepare_transfer_regions(handshake(3))
assert calls == 2
assert worker._peer_layout_key(handshake(3)) in worker._layout_by_peer
sock = MagicMock()
sock.send_multipart = AsyncMock()
await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 5, req_blocks))
_, payload = sock.send_multipart.call_args[0][0]
response = worker._xfer_resp_decoder.decode(payload)
assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS
assert calls == 2
@pytest.mark.asyncio
async def test_peer_layouts_evict_least_recently_used(monkeypatch):
"""Stale layout ids are bounded, and an evicted peer is asked to resend."""
monkeypatch.setattr(mooncake_connector, "_MAX_PEER_LAYOUTS", 2)
worker, _, regions, _ = _register_sliced_packed_mla([0])
req_blocks = {"d": ("xfer", [[1]])}
def handshake(layout_id: int) -> MooncakeXferMetadata:
return _xfer_meta(
regions,
req_blocks,
remote_hostname="peer",
remote_port=9,
remote_tp_size=worker.tp_size,
remote_pp_size=worker.pp_size,
layout_id=layout_id,
)
worker._prepare_transfer_regions(handshake(1))
worker._prepare_transfer_regions(handshake(3))
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks))
worker._prepare_transfer_regions(handshake(5))
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks))
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 5, req_blocks))
sock = MagicMock()
sock.send_multipart = AsyncMock()
await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 3, req_blocks))
_, payload = sock.send_multipart.call_args[0][0]
response = worker._xfer_resp_decoder.decode(payload)
assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS
class _ScriptedSock:
def __init__(self, responses: list[bytes], sent: list[bytes]):
self._responses = responses
self._sent = sent
def setsockopt(self, *_args, **_kwargs):
return None
async def send(self, data: bytes):
self._sent.append(data)
async def recv(self):
return self._responses.pop(0)
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def _single_pull() -> dict[str, PullReqMeta]:
return {
"d": PullReqMeta(
d_req_id="d",
transfer_id="t",
local_block_ids=[[1]],
remote_engine_id="p",
remote_bootstrap_addr="http://bootstrap",
)
}
async def _pull_once(worker, addr: str, responses: list[MooncakeXferResponse]):
sent: list[bytes] = []
encoded = [worker._encoder.encode(response) for response in responses]
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake."
"mooncake_connector.make_zmq_socket",
return_value=_ScriptedSock(encoded, sent),
):
await worker.receive_kv_from_single_worker(addr, _single_pull())
return sent
@pytest.mark.asyncio
async def test_pull_resends_layout_when_prefill_forgot_it():
worker, _, _, _ = _register_sliced_packed_mla([0])
addr = "tcp://peer:1"
worker._acked_layout_peers.add(addr)
sent = await _pull_once(
worker,
addr,
[
MooncakeXferResponse(status=MooncakeXferResponseStatus.LAYOUT_MISS),
MooncakeXferResponse(
status=MooncakeXferResponseStatus.FINISH,
ok_reqs=["d"],
layout_id=worker._kv_layout_id,
),
],
)
assert len(sent) == 2
assert len(sent[1]) > len(sent[0])
assert addr in worker._acked_layout_peers
@pytest.mark.asyncio
async def test_pull_keeps_layout_for_prefill_that_does_not_echo_it():
"""A prefill without layout_id support never gets a layout-less pull."""
worker, _, _, _ = _register_sliced_packed_mla([0])
addr = "tcp://old-peer:1"
finish = MooncakeXferResponse(
status=MooncakeXferResponseStatus.FINISH, ok_reqs=["d"]
)
first = await _pull_once(worker, addr, [finish])
assert addr not in worker._acked_layout_peers
second = await _pull_once(worker, addr, [finish])
assert len(second[0]) == len(first[0])
def test_coalesce_keeps_a_run_inside_its_starting_row():
"""The row bound is the run start, not the previous slice."""
row = 2500
@@ -2240,7 +2461,7 @@ def _register_sliced_packed_mla(
):
"""Register column slices of one MLA packed row. Returns worker fields."""
spec = MLAAttentionSpec(
block_size=16,
block_size=maybe_update_block_size(16),
num_kv_heads=1,
head_size=64,
dtype=torch.uint8,
@@ -2286,7 +2507,7 @@ def test_sliced_packed_mla_registers_each_layer_view():
KVCacheGroupSpec(
[_layer_name(i) for i in layer_idxs],
MLAAttentionSpec(
block_size=16,
block_size=maybe_update_block_size(16),
num_kv_heads=1,
head_size=64,
dtype=torch.uint8,
@@ -2436,9 +2657,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
remote_block_len = LOCAL_BLOCK_LEN * P_TP_SIZE // d_tp_size
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
vllm_config = _make_worker_vllm_config("kv_producer")
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
prefill_connector = MooncakeConnector(
@@ -2568,3 +2787,29 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
prefill_worker.sender_loop = origin_sender_loop
prefill_worker.shutdown()
def test_sleep_unregisters_and_restores_kv_caches():
spec = FullAttentionSpec(
block_size=maybe_update_block_size(16),
num_kv_heads=4,
head_size=64,
dtype=torch.float16,
)
layer_names = ["model.layers.0.self_attn", "model.layers.1.self_attn"]
with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, reg):
for layer_name in layer_names:
worker._layer_specs[layer_name] = spec
raw = torch.zeros(2 * 2 * spec.page_size_bytes, dtype=torch.int8)
views = dense_kv_cache_views(raw, spec, 2, 2, KVCacheLayout.LBHNC)
worker.register_kv_caches(dict(zip(layer_names, views)))
reg.reset_mock()
with patch.object(
worker.engine, "batch_unregister_memory", return_value=0
) as unreg:
worker.unregister_kv_caches()
worker.unregister_kv_caches()
unreg.assert_called_once_with([raw.data_ptr()])
worker.restore_kv_caches()
worker.restore_kv_caches()
reg.assert_called_once_with([raw.data_ptr()], [raw.nbytes])
@@ -30,7 +30,12 @@ from vllm.v1.kv_cache_interface import (
)
from .test_mooncake_connector import FakeMooncakeWrapper, patch_worker_dependencies
from .utils import create_request, create_vllm_config, make_kv_cache_config
from .utils import (
create_request,
create_vllm_config,
make_kv_cache_config,
maybe_update_block_size,
)
# ---------------------------------------------------------------------------
@@ -224,12 +229,11 @@ def test_metadata_hma_block_ids():
async def test_build_transfer_params_multi_group_trimming(monkeypatch):
"""_build_transfer_params trims per-group blocks when local > remote."""
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
kv_cache_config = make_kv_cache_config(
block_size=vllm_config.cache_config.block_size, swa_enabled=True
kv_connector="MooncakeConnector", kv_role="kv_producer", block_size=block_size
)
kv_cache_config = make_kv_cache_config(block_size=block_size, swa_enabled=True)
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
connector = MooncakeConnector(
@@ -320,12 +324,11 @@ async def test_build_transfer_params_multi_group_trimming(monkeypatch):
async def test_build_transfer_params_group_count_mismatch(monkeypatch):
"""_build_transfer_params reports an error when group counts differ."""
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector", kv_role="kv_producer"
)
kv_cache_config = make_kv_cache_config(
block_size=vllm_config.cache_config.block_size, swa_enabled=True
kv_connector="MooncakeConnector", kv_role="kv_producer", block_size=block_size
)
kv_cache_config = make_kv_cache_config(block_size=block_size, swa_enabled=True)
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
connector = MooncakeConnector(
@@ -449,7 +452,7 @@ def test_request_finished_with_hma_groups():
def _make_kv_consumer_worker(
swa_enabled: bool = False, kv_cache_config: KVCacheConfig | None = None
):
block_size = 16
block_size = maybe_update_block_size(16)
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_consumer",
@@ -495,6 +498,7 @@ def test_worker_is_hma_required(swa_enabled, expected_is_hma):
def test_worker_is_hma_required_multiple_full_attention_groups():
"""Two full-attention groups with different block sizes also select
request-level failure reporting, matching the scheduler."""
block_size = maybe_update_block_size(16)
kv_cache_config = KVCacheConfig(
num_blocks=100,
kv_cache_tensors=[],
@@ -502,13 +506,19 @@ def test_worker_is_hma_required_multiple_full_attention_groups():
KVCacheGroupSpec(
["layer0"],
FullAttentionSpec(
block_size=16, num_kv_heads=4, head_size=16, dtype=torch.float16
block_size=block_size,
num_kv_heads=4,
head_size=16,
dtype=torch.float16,
),
),
KVCacheGroupSpec(
["layer1"],
FullAttentionSpec(
block_size=32, num_kv_heads=4, head_size=16, dtype=torch.float16
block_size=2 * block_size,
num_kv_heads=4,
head_size=16,
dtype=torch.float16,
),
),
],
@@ -8,6 +8,7 @@ validated by this test module.
"""
import asyncio
from collections import OrderedDict
from types import SimpleNamespace
from unittest.mock import patch
@@ -36,7 +37,7 @@ from vllm.v1.kv_cache_interface import (
)
from .test_mooncake_connector import patch_worker_dependencies
from .utils import create_request, create_vllm_config
from .utils import create_request, create_vllm_config, maybe_update_block_size
def noop_shutdown():
@@ -147,6 +148,7 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch):
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_consumer",
block_size=maybe_update_block_size(16),
)
kv_cache_config = make_hybrid_gdn_kv_cache_config(
vllm_config.cache_config.block_size
@@ -164,13 +166,12 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch):
fa_spec = kv_cache_config.kv_cache_groups[0].kv_cache_spec
gdn_spec = kv_cache_config.kv_cache_groups[1].kv_cache_spec
fa_raw = torch.empty(num_blocks * fa_spec.page_size_bytes, dtype=torch.int8)
gdn_raw = torch.empty(num_blocks * gdn_spec.page_size_bytes, dtype=torch.int8)
gdn_block_stride = gdn_spec.page_size_bytes + 64
gdn_raw = torch.empty(num_blocks, gdn_block_stride, dtype=torch.int8)
(fa_cache,) = dense_kv_cache_views(
fa_raw, fa_spec, num_blocks, 1, KVCacheLayout.LBHNC
)
(gdn_cache,) = dense_kv_cache_views(
gdn_raw, gdn_spec, num_blocks, 1, KVCacheLayout.LBHNC
)
gdn_cache = gdn_raw[:, : gdn_spec.page_size_bytes]
worker.register_kv_caches(
{
@@ -184,7 +185,7 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch):
"model.layers.0.self_attn",
"model.layers.1.linear_attn",
]
assert worker.block_len_per_layer == [
assert worker.kv_block_len_per_layer == [
fa_spec.page_size_bytes,
gdn_spec.page_size_bytes,
]
@@ -204,6 +205,7 @@ def test_register_kv_caches_scales_attention_len_to_kernel_block(monkeypatch):
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_consumer",
block_size=maybe_update_block_size(16),
)
kv_cache_config = make_hybrid_gdn_kv_cache_config(
vllm_config.cache_config.block_size
@@ -242,8 +244,10 @@ async def test_build_transfer_params_uses_non_overlapping_physical_pages():
kv_cache_config = make_hybrid_gdn_kv_cache_config(block_size=16)
worker = object.__new__(MooncakeConnectorWorker)
worker.shutdown = noop_shutdown
# __new__ skips __init__, where this cache is created. register_kv_caches clears it.
# __new__ skips __init__, where these caches are created.
worker._prepared_transfer_regions = {}
worker._layout_by_peer = OrderedDict()
worker._acked_layout_peers = set()
worker.use_mla = False
worker.engine = SimpleNamespace(batch_register_memory=lambda *_: 0)
worker.is_kv_consumer = True
@@ -393,6 +397,7 @@ def test_register_kv_caches_deduplicates_shared_backing_memory(monkeypatch):
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_consumer",
block_size=maybe_update_block_size(16),
)
kv_cache_config = make_hybrid_gdn_kv_cache_config(
vllm_config.cache_config.block_size
@@ -406,8 +411,13 @@ def test_register_kv_caches_deduplicates_shared_backing_memory(monkeypatch):
)
worker = connector.connector_worker
backing = torch.empty((4, 64), dtype=torch.float16)
fa_cache = backing[:2, :16]
fa_spec = kv_cache_config.kv_cache_groups[0].kv_cache_spec
fa_cache_width = (
fa_spec.page_size_bytes
// torch.tensor([], dtype=torch.float16).element_size()
)
backing = torch.empty((4, fa_cache_width), dtype=torch.float16)
fa_cache = backing[:2]
gdn_cache = backing[:3]
with patch.object(
@@ -439,6 +449,7 @@ def test_hybrid_gdn_transfer_params_preserve_group_identity(monkeypatch):
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_producer",
block_size=maybe_update_block_size(16),
)
kv_cache_config = make_hybrid_gdn_kv_cache_config(
vllm_config.cache_config.block_size
@@ -568,6 +579,7 @@ def test_hybrid_gdn_keeps_packed_fa_and_gdn_regions_whole(
vllm_config = create_vllm_config(
kv_connector="MooncakeConnector",
kv_role="kv_producer",
block_size=maybe_update_block_size(16),
)
kv_cache_config = make_hybrid_gdn_kv_cache_config(
vllm_config.cache_config.block_size
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
MooncakeStoreConnectorStats,
)
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
@@ -141,18 +142,74 @@ def test_validation_rejects_mamba_mode_directly():
)
def test_scheduler_requires_align_mode_for_mamba():
vllm_config = _make_vllm_config()
mamba_spec = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="none",
def _make_mamba_none_kv_cache_config(
attention_block_size: int | None, mamba_block_size: int
) -> KVCacheConfig:
groups = [
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="none",
),
)
]
if attention_block_size is not None:
# Kimi K3 layout without prefix caching: the LCM hash unit exceeds both
# group block sizes, so no lookup coordinator can be built.
groups.insert(
0,
KVCacheGroupSpec(
["attention"],
FullAttentionSpec(
block_size=attention_block_size,
num_kv_heads=8,
head_size=64,
dtype=torch.float32,
),
),
)
return KVCacheConfig(num_blocks=4, kv_cache_tensors=[], kv_cache_groups=groups)
def _make_role_vllm_config(kv_role: str, enable_lookup: bool, save_decode_cache: bool):
vllm_config = create_vllm_config(
kv_connector="MooncakeStoreConnector",
kv_role=kv_role,
kv_connector_extra_config={
"enable_lookup": enable_lookup,
"save_decode_cache": save_decode_cache,
},
)
kv_cache_config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["mamba"], mamba_spec)],
vllm_config.cache_config.enable_prefix_caching = False
return vllm_config
_MAMBA_NONE_LAYOUTS = pytest.mark.parametrize(
"attention_block_size,mamba_block_size",
[(None, 16), (1_048_576, 6_400)],
ids=["mamba-only", "hybrid-no-prefix"],
)
@_MAMBA_NONE_LAYOUTS
@pytest.mark.parametrize(
"kv_role,enable_lookup,save_decode_cache",
[
("kv_consumer", True, False),
("kv_consumer", False, True),
("kv_producer", False, False),
("kv_both", False, False),
],
)
def test_scheduler_requires_align_mode_for_mamba_kv_transfer(
kv_role, enable_lookup, save_decode_cache, attention_block_size, mamba_block_size
):
vllm_config = _make_role_vllm_config(kv_role, enable_lookup, save_decode_cache)
kv_cache_config = _make_mamba_none_kv_cache_config(
attention_block_size, mamba_block_size
)
with (
@@ -165,6 +222,30 @@ def test_scheduler_requires_align_mode_for_mamba():
scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
@_MAMBA_NONE_LAYOUTS
def test_capacity_only_scheduler_allows_mamba_none_mode(
attention_block_size, mamba_block_size
):
"""Capacity-only consumers transfer no KV, so any Mamba mode is allowed."""
vllm_config = _make_role_vllm_config(
"kv_consumer", enable_lookup=False, save_decode_cache=False
)
kv_cache_config = _make_mamba_none_kv_cache_config(
attention_block_size, mamba_block_size
)
with patch(
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
"scheduler.LookupKeyClient"
) as mock_client:
store_scheduler = scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
assert store_scheduler.get_num_new_matched_tokens(MagicMock(), 0) == (0, False)
meta = store_scheduler.build_connector_meta(SchedulerOutput.make_empty())
mock_client.return_value.lookup.assert_not_called()
assert meta.requests == []
def _make_block_stored(
block_hash: bytes = b"hash", group_idx: int | None = None
) -> BlockStored:
@@ -3,6 +3,7 @@
from math import lcm
import pytest
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
@@ -30,7 +31,13 @@ def _mamba_align(block_size=32):
)
def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=None):
def _make_coord(
groups,
hash_block_size,
use_eagle=False,
retention_interval=None,
enable_partial_hash_hits=False,
):
"""Construct a coordinator using the natural LCM of group block sizes as
the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for
the test fixtures."""
@@ -42,6 +49,7 @@ def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=Non
hash_block_size=hash_block_size,
use_eagle=use_eagle,
retention_interval=retention_interval,
enable_partial_hash_hits=enable_partial_hash_hits,
)
@@ -219,8 +227,7 @@ def test_coordinator_fine_grained_partial_tail_hit():
KVCacheGroupSpec(["L0"], _full(32)),
KVCacheGroupSpec(["L1"], _mamba_align(32)),
]
coord = _make_coord(groups, hash_block_size=16)
assert coord.enable_partial_hash_hits
coord = _make_coord(groups, hash_block_size=16, enable_partial_hash_hits=True)
hs = _hashes(4) # 4 hash units of 16 = 64 tokens; block 0 = [0,32), etc.
# Both groups: full block 0 (key = last sub-hash hs[1]) + partial boundary
# at token 48 (key = hs[2]). No hs[3] -> block 1 is not full.
@@ -287,38 +294,6 @@ def test_store_mask_swa_only_window_around_each_lcm_boundary():
assert masks[1] == [False, False, False, True, False, False, False, True]
def test_store_mask_swa_does_not_double_scale_under_dcp():
"""The coordinator receives DCP-resolved specs (worker.py applies
resolve_dcp_kv_cache_spec) and indexes chunks in that already-scaled block
size. So dcp_world_size must NOT further scale reachable_block_mask; the
mask must be identical for any dcp_world_size given the same specs.
Regression: previously the coordinator forwarded its dcp_world_size, which
double-scaled the SWA block size and produced a wrong mask when dcp > 1.
"""
full = _full(32)
swa = _swa(block_size=8, sliding_window=8)
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
scheduler_block_size = lcm(32, 8)
def make(dcp_world_size):
return MooncakeStoreCoordinator(
groups,
scheduler_block_size=scheduler_block_size,
hash_block_size=8,
dcp_world_size=dcp_world_size,
)
mask_dcp1 = make(1).store_mask(64)
mask_dcp2 = make(2).store_mask(64)
# dcp must not change the mask; both match the correct single-block-size
# result (chunks ending at 32 and 64 -> blocks 3 and 7).
expected = [False, False, False, True, False, False, False, True]
assert mask_dcp1[1] == expected
assert mask_dcp2[1] == expected
def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
"""Same hybrid layout but sliding_window=16 (= 2 SWA blocks). Each lcm
boundary should now span two SWA tail blocks."""
@@ -448,6 +423,49 @@ def test_store_mask_retention_interval_zero_keeps_only_replay_boundary():
assert masks[1] == [i == 11 for i in range(16)]
def test_store_mask_retention_interval_zero_eagle_keeps_lookup_tail():
"""With EAGLE the replay boundary sits one scheduler block lower, as in
core's ``replay_boundaries``: the lookup matches that block and drops
back from it. The retained SWA tail must span the ``need`` contiguous
blocks the lookup requires (cdiv(8 - 1, 8) + 1 = 2 here); the non-EAGLE
boundary leaves only one of them inside the saved prefix."""
coord = _make_coord(
_retention_groups(), hash_block_size=8, use_eagle=True, retention_interval=0
)
# num_prompt=100 -> boundary 64 -> chunks 7, 8 (8 is the EAGLE peek).
masks = coord.store_mask(96, num_prompt_tokens=100)
assert masks[0] is None
assert masks[1] == [i in (7, 8) for i in range(12)]
# Aligned num_prompt=96 -> boundaries 32 (resend) and 64 (longer sibling).
masks = coord.store_mask(96, num_prompt_tokens=96)
assert masks[1] == [i in (3, 4, 7, 8) for i in range(12)]
@pytest.mark.parametrize("num_prompt_tokens", [100, 96])
def test_store_mask_retention_interval_zero_eagle_round_trip(num_prompt_tokens):
"""Everything store_mask keeps is in the store; the EAGLE lookup of the
same prompt must then reach the full block below the one it drops (64 of
96). The unaligned prompt hit 0 before the fix; the aligned one already
hit."""
groups = _retention_groups()
coord = _make_coord(groups, hash_block_size=8, use_eagle=True, retention_interval=0)
hashes = _hashes(96 // coord.hash_block_size)
exists: set[tuple[int, bytes]] = set()
store_masks = coord.store_mask(96, num_prompt_tokens=num_prompt_tokens)
for gid, (group, mask) in enumerate(zip(groups, store_masks, strict=True)):
group_hashes = coord.block_hashes_for_spec(hashes, group.kv_cache_spec)
for chunk_id, block_hash in enumerate(group_hashes):
if mask is None or mask[chunk_id]:
exists.add((gid, bytes(block_hash)))
_masks, hit = coord.find_longest_cache_hit(
hashes,
max_length=96,
cached_block_pool=ExternalCachedBlockPool(coord.hash_block_size, exists),
)
assert hit == 64
def test_store_mask_retention_interval_keeps_segment_and_replay_tails():
"""Sparse segment tails (interval=64 -> chunks 7,15) plus the replay
boundary tail (num_prompt=100 -> chunk 11) coexist."""
@@ -18,11 +18,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator imp
MooncakeStoreCoordinator,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
BoundaryPut,
ChunkedTokenDatabase,
KeyMetadata,
LoadSpec,
ReqMeta,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import ( # noqa: E501
_partial_tail_non_mamba_puts,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501
KVCacheStoreRecvingThread,
KVCacheStoreSendingThread,
@@ -31,7 +35,6 @@ from vllm.v1.core.kv_cache_utils import BlockHash
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
@@ -379,6 +382,15 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size():
assert out[1][2].hex() == fine_hashes[6].hex()
def _resolve_partial_tail(thread, req: ReqMeta) -> ReqMeta:
"""Add the tail's non-Mamba puts as the scheduler does before the worker."""
req.boundary_puts = [BoundaryPut(*put) for put in req.boundary_puts or []]
req.boundary_puts[:0] = _partial_tail_non_mamba_puts(
thread.coord, req, [db.block_size for db in thread.token_databases]
)
return req
def test_sub_block_partial_tail_offload_reads_cow_block():
"""Sub-block prompt (the 900/128/1536 shape, scaled to 12/4/16): the
partial tail is offloaded for both groups under the boundary sub-hash. The
@@ -395,7 +407,12 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
]
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
coord = MooncakeStoreCoordinator(
groups,
scheduler_block_size=16,
hash_block_size=4,
enable_partial_hash_hits=True,
)
assert coord.enable_partial_hash_hits
class _RecordingStore(_DictStore):
@@ -442,11 +459,13 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
block_ids=([1], [2]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=20,
boundary_state_offloads=[(1, mamba_cow_block, 12)],
num_prompt_tokens=13,
prefill_end_tokens=20,
completed_token_len=20,
boundary_puts=[(1, mamba_cow_block, 12)],
)
send._maybe_offload_boundary_states(req)
send._maybe_offload_boundary_states(_resolve_partial_tail(send, req))
# boundary = 12 // 4 * 4 = 12 -> keyed by hs[12 // 4 - 1] = hs[2].
partial_hash = hs[2]
@@ -474,7 +493,12 @@ def test_offload_syncs_event_before_put():
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
]
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
coord = MooncakeStoreCoordinator(
groups,
scheduler_block_size=16,
hash_block_size=4,
enable_partial_hash_hits=True,
)
event = MagicMock()
class _FencedStore(_DictStore):
@@ -514,12 +538,13 @@ def test_offload_syncs_event_before_put():
block_hashes=hs,
can_save=True,
num_prompt_tokens=12,
completed_token_len=12,
store_job_id=1,
boundary_state_offloads=[(1, 7, 12)],
boundary_puts=[(1, 7, 8)],
)
req.current_event = event
send.add_request(req)
send.add_request(_resolve_partial_tail(send, req))
send._handle_request(send.request_queue.get())
assert send.request_queue.qsize() == 0
assert store._data
@@ -533,7 +558,8 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
offload must persist every FA block up to the boundary — the normal save
floors to the lcm, so those blocks are otherwise never written and the
consumer's per-group lookup would miss. The mamba boundary block still
reads the core-provided CoW target."""
reads the core-provided CoW target. Without EAGLE, the 12-token prompt's
resend-safe checkpoint is 8, leaving two attention blocks to save."""
full = FullAttentionSpec(block_size=4, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
@@ -545,7 +571,12 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
]
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
coord = MooncakeStoreCoordinator(
groups,
scheduler_block_size=16,
hash_block_size=4,
enable_partial_hash_hits=True,
)
assert coord.enable_partial_hash_hits
class _RecordingStore(_DictStore):
@@ -591,19 +622,19 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
block_hashes=hs,
can_save=True,
num_prompt_tokens=12,
boundary_state_offloads=[(1, mamba_cow_block, 12)],
completed_token_len=12,
boundary_puts=[(1, mamba_cow_block, 8)],
)
send._maybe_offload_boundary_states(req)
send._maybe_offload_boundary_states(_resolve_partial_tail(send, req))
# FA (block 4): full blocks ending at 4, 8 and 12, keyed by their normal
# FA (block 4): full blocks ending at 4 and 8, keyed by their normal
# block-end hashes; mamba (block 16): the partial boundary block under
# the boundary sub-hash, read from the CoW target.
expected = {
token_dbs[0].key_for(hs[0]): [1 * 512],
token_dbs[0].key_for(hs[1]): [2 * 512],
token_dbs[0].key_for(hs[2]): [3 * 512],
token_dbs[1].key_for(hs[2]): [10_000 + mamba_cow_block * 512],
token_dbs[1].key_for(hs[1]): [10_000 + mamba_cow_block * 512],
}
assert store.puts == expected
@@ -612,11 +643,10 @@ def test_worker_lookup_hits_sub_block_partial_tail():
"""worker.lookup must query sub-block keys when partial hash hits are on.
Regression test: ``lookup`` hard-coded ``fine_grained = False``, so the
sub-block keys persisted by ``_sub_block_tail_puts`` were never probed and
partial prefix hits silently returned 0 while the store side kept writing
them. Here the mamba block (16) exceeds the hash unit (4), so
``enable_partial_hash_hits`` is on and the lookup must find the stored
boundary at 12.
partial tail's sub-block keys were never probed and partial prefix hits
silently returned 0 while the store side kept writing them. Here the
mamba block (16) exceeds the hash unit (4), so ``enable_partial_hash_hits``
is on and the lookup must find the stored boundary at 8.
"""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
@@ -645,6 +675,8 @@ def test_worker_lookup_hits_sub_block_partial_tail():
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
],
# The hit alignment the engine core resolves for this config.
cache_hit_alignment_tokens=4,
)
vllm_config = _minimal_vllm_config(cache_block_size=16)
# Hash unit 4 < mamba block 16 -> partial hash hits are enabled.
@@ -673,7 +705,7 @@ def test_worker_lookup_hits_sub_block_partial_tail():
replicate_config=MagicMock(),
)
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//4-1]).
# A 12-token prompt leaves a resend-safe sub-block checkpoint at 8.
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(5)]
req = ReqMeta(
req_id="r0",
@@ -681,15 +713,16 @@ def test_worker_lookup_hits_sub_block_partial_tail():
block_ids=([1], [2]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=20,
boundary_state_offloads=[(1, 7, 12)],
num_prompt_tokens=12,
completed_token_len=12,
boundary_puts=[(1, 7, 8)],
)
send_thread._maybe_offload_boundary_states(req)
send_thread._maybe_offload_boundary_states(_resolve_partial_tail(send_thread, req))
worker.store = store
# A 13-token prompt sharing the prefix must hit the stored boundary at 12.
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 12
# A 13-token prompt sharing the prefix must hit the stored boundary at 8.
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 8
def test_worker_setup_tolerates_finer_scratch_group():
@@ -711,13 +744,12 @@ def test_worker_setup_tolerates_finer_scratch_group():
mamba_cache_mode="align",
)
# Scratch block 4 is not divisible by the hash unit 8 below.
scratch = KpoolTailSpec(
scratch = CircularBufferSpec(
block_size=4,
num_kv_heads=2,
head_size=64,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=4,
)
cfg = KVCacheConfig(
num_blocks=4,
@@ -746,6 +778,8 @@ def test_worker_setup_tolerates_finer_scratch_group():
KVCacheGroupSpec(["L1"], mamba),
KVCacheGroupSpec(["L2"], scratch),
],
# The hit alignment the engine core resolves for this config.
cache_hit_alignment_tokens=8,
)
vllm_config = _minimal_vllm_config(cache_block_size=16)
# Hash unit 8 divides the participating groups (16) but not the scratch
@@ -779,7 +813,7 @@ def test_worker_setup_tolerates_finer_scratch_group():
group_participates=[True, True],
)
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//8-1]).
# Persist the sub-block partial tail at boundary 8 (keyed by hs[8//8-1]).
hs = [BlockHash(bytes([i + 1]) * 8) for i in range(3)]
req = ReqMeta(
req_id="r0",
@@ -787,10 +821,11 @@ def test_worker_setup_tolerates_finer_scratch_group():
block_ids=([1], [2]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=20,
boundary_state_offloads=[(1, 7, 12)],
num_prompt_tokens=12,
completed_token_len=12,
boundary_puts=[(1, 7, 8)],
)
send_thread._maybe_offload_boundary_states(req)
send_thread._maybe_offload_boundary_states(_resolve_partial_tail(send_thread, req))
worker.store = store

Some files were not shown because too many files have changed in this diff Show More