Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -0,0 +1,104 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Differential test for the AITER MLA uniform-MTP padding decision.
|
||||
|
||||
`_uniform_padded_mtp_qo_len` decides whether a full-CUDA-graph decode batch may
|
||||
synthesize dummy query rows. It used to reach that decision through eight tensor
|
||||
dispatches -- sum/item, boolean-mask index, all, eq, nonzero, item, any -- on a
|
||||
CPU tensor holding one entry per request. It now reads the tensor out once and
|
||||
decides on Python ints.
|
||||
|
||||
That rewrite also **reordered** the checks: the `num_decode_tokens <= sum(...)`
|
||||
early-out moved below the divisibility and `uniform_qo_len <= 1` guards. Every
|
||||
one of those paths returns 0, so the result should be unchanged -- but "should
|
||||
be" is exactly the sort of claim worth checking exhaustively, so both
|
||||
implementations are carried here and compared over the whole small input space.
|
||||
|
||||
Pure host arithmetic; no GPU required.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
|
||||
|
||||
|
||||
def _reference(qo_len: torch.Tensor, max_qo_len: int, num_decode_tokens: int) -> int:
|
||||
"""The pre-#58381 implementation, verbatim."""
|
||||
num_reqs = qo_len.numel()
|
||||
if num_reqs == 0 or num_decode_tokens <= 0:
|
||||
return 0
|
||||
if num_decode_tokens <= int(qo_len.sum().item()):
|
||||
return 0
|
||||
if num_decode_tokens % num_reqs != 0:
|
||||
return 0
|
||||
|
||||
uniform_qo_len = num_decode_tokens // num_reqs
|
||||
if uniform_qo_len <= 1:
|
||||
return 0
|
||||
|
||||
positive_qo_len = qo_len[qo_len > 0]
|
||||
if positive_qo_len.numel() == qo_len.numel():
|
||||
return 0
|
||||
if positive_qo_len.numel() > 0:
|
||||
if max_qo_len != uniform_qo_len:
|
||||
return 0
|
||||
if not torch.all(positive_qo_len == uniform_qo_len):
|
||||
return 0
|
||||
|
||||
zero_positions = torch.nonzero(qo_len == 0, as_tuple=False).flatten()
|
||||
if zero_positions.numel() > 0:
|
||||
first_zero = int(zero_positions[0].item())
|
||||
if torch.any(qo_len[first_zero:] > 0):
|
||||
return 0
|
||||
|
||||
return uniform_qo_len
|
||||
|
||||
|
||||
_ACTUAL = AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_reqs", [1, 2, 3, 4])
|
||||
def test_matches_reference_exhaustively(num_reqs):
|
||||
"""Every qo_len vector x decode-token count x max_qo_len in a small space."""
|
||||
checked = 0
|
||||
for lens in itertools.product(range(0, 4), repeat=num_reqs):
|
||||
qo = torch.tensor(lens, dtype=torch.int32)
|
||||
for num_decode_tokens in range(0, 13):
|
||||
for max_qo_len in range(0, 5):
|
||||
got = _ACTUAL(qo, max_qo_len, num_decode_tokens)
|
||||
want = _reference(qo, max_qo_len, num_decode_tokens)
|
||||
assert got == want, (
|
||||
f"lens={lens} num_decode_tokens={num_decode_tokens} "
|
||||
f"max_qo_len={max_qo_len}: got {got}, reference {want}"
|
||||
)
|
||||
checked += 1
|
||||
assert checked > 0
|
||||
|
||||
|
||||
def test_empty_batch():
|
||||
assert _ACTUAL(torch.zeros(0, dtype=torch.int32), 4, 8) == 0
|
||||
assert _reference(torch.zeros(0, dtype=torch.int32), 4, 8) == 0
|
||||
|
||||
|
||||
def test_the_padded_mtp_case_that_should_succeed():
|
||||
"""A live prefix at the uniform width, then zero-padded dummies."""
|
||||
qo = torch.tensor([4, 4, 0, 0], dtype=torch.int32)
|
||||
# 4 requests x 4 tokens = 16 captured, 8 live -> dummies are synthesizable.
|
||||
assert _ACTUAL(qo, 4, 16) == 4
|
||||
assert _reference(qo, 4, 16) == 4
|
||||
|
||||
|
||||
def test_positive_after_zero_is_rejected():
|
||||
"""A live request after a padding hole breaks the contiguous-prefix layout."""
|
||||
qo = torch.tensor([4, 0, 4, 0], dtype=torch.int32)
|
||||
assert _ACTUAL(qo, 4, 16) == 0
|
||||
assert _reference(qo, 4, 16) == 0
|
||||
|
||||
|
||||
def test_ragged_live_lengths_are_rejected():
|
||||
qo = torch.tensor([4, 2, 0, 0], dtype=torch.int32)
|
||||
assert _ACTUAL(qo, 4, 16) == 0
|
||||
assert _reference(qo, 4, 16) == 0
|
||||
@@ -31,6 +31,7 @@ from vllm.utils.torch_utils import (
|
||||
STR_DTYPE_TO_TORCH_DTYPE,
|
||||
is_quantized_kv_cache,
|
||||
is_torch_equal_or_newer,
|
||||
nvfp4_kv_cache_full_dim,
|
||||
set_random_seed,
|
||||
)
|
||||
from vllm.v1.attention.backend import (
|
||||
@@ -41,7 +42,7 @@ from vllm.v1.attention.backend import (
|
||||
max_decode_query_len,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout, KVQuantMode
|
||||
|
||||
BACKENDS_TO_TEST = [
|
||||
AttentionBackendEnum.FLASH_ATTN,
|
||||
@@ -974,8 +975,6 @@ def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypa
|
||||
causal=True,
|
||||
prefill=None,
|
||||
decode=decode,
|
||||
use_cascade=False,
|
||||
cascade_wrapper=None,
|
||||
)
|
||||
|
||||
seen_shapes = {}
|
||||
@@ -1698,6 +1697,103 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
|
||||
or not current_platform.is_device_capability_family(120),
|
||||
reason="NVFP4 XQA decode requires SM12x.",
|
||||
)
|
||||
@pytest.mark.parametrize("use_cuda_graph", [False, True])
|
||||
@pytest.mark.parametrize("q_len", [1, 4])
|
||||
def test_flashinfer_xqa_nvfp4_decode_correctness(
|
||||
default_vllm_config, q_len, use_cuda_graph
|
||||
):
|
||||
"""Single-token and spec-decode queries over an NVFP4 cache decode with
|
||||
XQA, eager and replayed, and match the fa2 prefill that wrote the cache."""
|
||||
import unittest.mock
|
||||
|
||||
from vllm.v1.attention.backends import flashinfer as fi
|
||||
|
||||
seq_lens, bs, d = [40, 300, 1027], 16, 256
|
||||
config = create_vllm_config(
|
||||
model_name="Qwen/Qwen3-0.6B",
|
||||
max_model_len=max(seq_lens),
|
||||
block_size=bs,
|
||||
)
|
||||
# GQA 6 at head_dim 256, as Qwen3.8-27B at TP=2.
|
||||
config.model_config.model_arch_config = replace(
|
||||
config.model_config.model_arch_config,
|
||||
total_num_attention_heads=24,
|
||||
total_num_kv_heads=4,
|
||||
head_size=d,
|
||||
)
|
||||
config.cache_config.cache_dtype = "nvfp4"
|
||||
if q_len > 1:
|
||||
config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=q_len - 1
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=bs,
|
||||
num_kv_heads=4,
|
||||
head_size=d,
|
||||
dtype=torch.uint8,
|
||||
kv_quant_mode=KVQuantMode.NVFP4,
|
||||
)
|
||||
cg_support = fi.FlashInferMetadataBuilder.get_cudagraph_support(config, spec)
|
||||
assert cg_support == AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
# Request i owns consecutive pages from starts[i]; page 0 stays null.
|
||||
pages = [0] + [cdiv(s, bs) for s in seq_lens[:-1]]
|
||||
starts = 1 + torch.tensor(pages).cumsum(0)
|
||||
max_pages = cdiv(max(seq_lens), bs)
|
||||
block_table = (starts[:, None] + torch.arange(max_pages)).int().to(device)
|
||||
full_dim = nvfp4_kv_cache_full_dim(d)
|
||||
kv_cache = torch.zeros(
|
||||
int(starts[-1]) + max_pages, bs, 8, full_dim, dtype=torch.uint8, device=device
|
||||
).transpose(1, 2)
|
||||
q, k, v = (
|
||||
torch.randn(sum(seq_lens), h, d).bfloat16().to(device) for h in (24, 4, 4)
|
||||
)
|
||||
|
||||
def run(query_lens, use_cuda_graph=False):
|
||||
req = torch.repeat_interleave(torch.arange(3), torch.tensor(query_lens))
|
||||
pos = torch.cat([torch.arange(s - n, s) for s, n in zip(seq_lens, query_lens)])
|
||||
rows = pos + torch.tensor([0, *seq_lens]).cumsum(0)[req]
|
||||
metadata = replace(
|
||||
create_common_attn_metadata(BatchSpec(seq_lens, query_lens), bs, device),
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=(starts[req] * bs + pos).to(device),
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
return rows, run_attention_backend(
|
||||
AttentionBackendEnum.FLASHINFER,
|
||||
spec,
|
||||
["placeholder"],
|
||||
config,
|
||||
device,
|
||||
metadata,
|
||||
q[rows],
|
||||
k[rows],
|
||||
v[rows],
|
||||
kv_cache,
|
||||
kv_cache_dtype="nvfp4",
|
||||
use_cuda_graph=use_cuda_graph,
|
||||
layer_k_scale=0.02,
|
||||
layer_v_scale=0.03,
|
||||
)
|
||||
|
||||
# The full prefill writes every token through the NVFP4 writer; its fa2
|
||||
# output for the last q_len tokens of each request is the reference.
|
||||
_, reference = run(seq_lens)
|
||||
xqa_fn = fi.flashinfer_xqa_batch_decode_with_kv_cache
|
||||
with unittest.mock.patch.object(
|
||||
fi, "flashinfer_xqa_batch_decode_with_kv_cache", wraps=xqa_fn
|
||||
) as xqa:
|
||||
rows, output = run([q_len] * len(seq_lens), use_cuda_graph)
|
||||
assert xqa.called
|
||||
torch.testing.assert_close(output, reference[rows], atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
if current_platform.is_rocm():
|
||||
# FLASH_ATTN is not supported on ROCm
|
||||
SLIDING_WINDOW_BACKENDS_TO_TEST = [
|
||||
|
||||
@@ -10,6 +10,10 @@ from vllm.models.deepseek_v4.amd.rocm import (
|
||||
DeepseekV4ROCMAiterMLASparseMetadataBuilder,
|
||||
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
)
|
||||
from vllm.models.deepseek_v41.amd import rocm as v41_rocm
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backends.mla import indexer
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
@@ -51,7 +55,7 @@ def _make_indexer_builder(*, adaptive: bool, capacity: int = 12):
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_deepseek_v4_rocm_adaptive_builders_support_varlen_full_graphs():
|
||||
def test_deepseek_rocm_adaptive_builders_support_varlen_full_graphs():
|
||||
adaptive_config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(enable_adaptive_verification=True)
|
||||
)
|
||||
@@ -62,6 +66,8 @@ def test_deepseek_v4_rocm_adaptive_builders_support_varlen_full_graphs():
|
||||
for builder_cls in (
|
||||
DeepseekV4ROCMAiterMLASparseMetadataBuilder,
|
||||
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
|
||||
v41_rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
):
|
||||
assert (
|
||||
builder_cls.get_cudagraph_support(adaptive_config, SimpleNamespace())
|
||||
@@ -74,20 +80,25 @@ def test_deepseek_v4_rocm_adaptive_builders_support_varlen_full_graphs():
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_deepseek_v4_rocm_adaptive_indexer_support(monkeypatch: pytest.MonkeyPatch):
|
||||
@pytest.mark.parametrize(
|
||||
("architecture", "backend_cls"),
|
||||
[
|
||||
("DeepseekV4ForCausalLM", DeepseekV4IndexerBackend),
|
||||
("DeepseekV41ForCausalLM", DeepseekV41IndexerBackend),
|
||||
],
|
||||
)
|
||||
def test_deepseek_rocm_adaptive_indexer_support(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
architecture: str,
|
||||
backend_cls: type[DeepseekV4IndexerBackend],
|
||||
):
|
||||
_mock_rocm_platform(monkeypatch)
|
||||
adaptive_config = _make_indexer_config(
|
||||
architecture="DeepseekV4ForCausalLM", adaptive=True
|
||||
)
|
||||
fixed_config = _make_indexer_config(
|
||||
architecture="DeepseekV4ForCausalLM", adaptive=False
|
||||
)
|
||||
adaptive_config = _make_indexer_config(architecture=architecture, adaptive=True)
|
||||
fixed_config = _make_indexer_config(architecture=architecture, adaptive=False)
|
||||
|
||||
assert DeepseekV4IndexerBackend.supports_device_cpu_query_lens_mismatch()
|
||||
assert backend_cls.supports_device_cpu_query_lens_mismatch()
|
||||
assert indexer._rocm_supports_flattened_device_query_lens(adaptive_config)
|
||||
assert (
|
||||
DeepseekV4IndexerBackend.get_builder_cls() is DeepseekV32IndexerMetadataBuilder
|
||||
)
|
||||
assert backend_cls.get_builder_cls() is DeepseekV32IndexerMetadataBuilder
|
||||
assert indexer._use_flattening(adaptive_config)
|
||||
assert (
|
||||
DeepseekV32IndexerMetadataBuilder.get_cudagraph_support(
|
||||
@@ -104,12 +115,39 @@ def test_deepseek_v4_rocm_adaptive_indexer_support(monkeypatch: pytest.MonkeyPat
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_deepseek_v41_rocm_fixed_k_path_is_unchanged(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_mock_rocm_platform(monkeypatch)
|
||||
fixed_config = _make_indexer_config(
|
||||
architecture="DeepseekV41ForCausalLM", adaptive=False
|
||||
)
|
||||
|
||||
for builder_cls in (
|
||||
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
|
||||
v41_rocm.DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
):
|
||||
assert (
|
||||
builder_cls.get_cudagraph_support(fixed_config, SimpleNamespace())
|
||||
== AttentionCGSupport.UNIFORM_BATCH
|
||||
)
|
||||
|
||||
assert not indexer._use_flattening(fixed_config)
|
||||
assert (
|
||||
DeepseekV32IndexerMetadataBuilder.get_cudagraph_support(
|
||||
fixed_config, SimpleNamespace()
|
||||
)
|
||||
== AttentionCGSupport.UNIFORM_BATCH
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"architecture",
|
||||
["DeepseekV32ForCausalLM", "DeepseekV41ForCausalLM", "GlmMoeDsaForCausalLM"],
|
||||
["DeepseekV32ForCausalLM", "GlmMoeDsaForCausalLM"],
|
||||
)
|
||||
def test_rocm_adaptive_indexer_flattening_is_scoped_to_deepseek_v4(
|
||||
def test_rocm_adaptive_indexer_flattening_rejects_other_architectures(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
architecture: str,
|
||||
):
|
||||
@@ -125,9 +163,6 @@ def test_rocm_adaptive_indexer_flattening_is_scoped_to_deepseek_v4(
|
||||
== AttentionCGSupport.UNIFORM_BATCH
|
||||
)
|
||||
|
||||
if architecture == "DeepseekV41ForCausalLM":
|
||||
assert not DeepseekV41IndexerBackend.supports_device_cpu_query_lens_mismatch()
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_rocm_adaptive_indexer_preserves_single_request_uniform_path(
|
||||
|
||||
@@ -15,6 +15,7 @@ import inspect
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import DeepseekV4Config
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
@@ -532,26 +533,26 @@ def make_builder(
|
||||
builder_cls: type[
|
||||
DeepseekSparseSWAMetadataBuilder
|
||||
] = DeepseekSparseSWAMetadataBuilder,
|
||||
mm_prefix_clamp_sliding_window: bool = True,
|
||||
deepseek_v4_config: bool = True,
|
||||
bounded_replay: bool = False,
|
||||
) -> DeepseekSparseSWAMetadataBuilder:
|
||||
overrides: dict = {"sliding_window": window}
|
||||
if vision:
|
||||
# Emulate the V4 vision config, whose mm_prefix_clamp_sliding_window
|
||||
# gates the in-kernel SWA widening. V4.1 (causal image tokens) leaves
|
||||
# it off.
|
||||
overrides.update(
|
||||
vision_n_layers=2,
|
||||
vision_max_n_token=max_image_tokens,
|
||||
mm_prefix_clamp_sliding_window=mm_prefix_clamp_sliding_window,
|
||||
)
|
||||
vision_fields = dict(vision_n_layers=2, vision_max_n_token=max_image_tokens)
|
||||
vllm_config = create_vllm_config(
|
||||
model_name=model_name,
|
||||
max_model_len=4096,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
max_num_seqs=8,
|
||||
hf_config_override=overrides,
|
||||
hf_config_override={"sliding_window": window}
|
||||
| (vision_fields if vision and not deepseek_v4_config else {}),
|
||||
)
|
||||
if vision and deepseek_v4_config:
|
||||
# Only V4 configs widen the SWA window in-kernel over image spans
|
||||
vllm_config.model_config.hf_config = DeepseekV4Config(
|
||||
num_hidden_layers=1,
|
||||
layer_types=["sliding_attention"],
|
||||
sliding_window=window,
|
||||
**vision_fields,
|
||||
)
|
||||
spec = SlidingWindowMLASpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=1,
|
||||
@@ -686,7 +687,7 @@ def test_v41_image_prefill_uses_causal_swa(compress_ratio, query_len):
|
||||
max_image_tokens,
|
||||
query_len,
|
||||
builder_cls=DeepseekV41SparseSWAMetadataBuilder,
|
||||
mm_prefix_clamp_sliding_window=False,
|
||||
deepseek_v4_config=False,
|
||||
)
|
||||
assert builder.max_image_tokens == 0
|
||||
md = build_metadata(builder, [seq_len], [query_len], {0: spans})
|
||||
@@ -828,3 +829,57 @@ def test_v41_flashinfer_mixed_sparse_indices_respect_replay_start():
|
||||
# Decode row is copied through; prefill rows follow the bounded window.
|
||||
assert bounded[0].cpu().tolist() == plain[0].cpu().tolist()
|
||||
assert bounded[1:].cpu().tolist() == rows[1:]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_dspark_noncausal_draft_decode_update_matches_build():
|
||||
"""A DSpark draft graph refreshes its non-causal SWA metadata in place
|
||||
instead of rebuilding it, so the refresh must match a fresh build."""
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
get_dspark_swa_index_width,
|
||||
)
|
||||
|
||||
num_spec = 4
|
||||
query_lens = [num_spec] * 3
|
||||
|
||||
def make_dspark_builder():
|
||||
builder = make_builder(vision=False)
|
||||
builder.is_dspark = True
|
||||
builder.noncausal_index_width = get_dspark_swa_index_width(WINDOW, num_spec)
|
||||
builder.decode_threshold = num_spec
|
||||
return builder
|
||||
|
||||
def common_metadata(seq_lens):
|
||||
query_start_loc, seq_lens_t, _, slot_mapping, block_table = make_batch(
|
||||
seq_lens, query_lens, torch.device("cuda")
|
||||
)
|
||||
return CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=seq_lens_t,
|
||||
seq_lens_cpu_upper_bound=seq_lens_t.cpu(),
|
||||
num_reqs=len(seq_lens),
|
||||
num_actual_tokens=int(query_start_loc[-1]),
|
||||
max_query_len=num_spec,
|
||||
max_seq_len=max(seq_lens),
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=slot_mapping,
|
||||
causal=False,
|
||||
)
|
||||
|
||||
captured_seq_lens = [6, 40, 130]
|
||||
replay_seq_lens = [130, 9, 90]
|
||||
builder = make_dspark_builder()
|
||||
captured = common_metadata(captured_seq_lens)
|
||||
md = builder.build(0, captured)
|
||||
replay = common_metadata(replay_seq_lens)
|
||||
expected = make_dspark_builder().build(0, replay)
|
||||
|
||||
# A replay sees new inputs through the same device buffers.
|
||||
captured.seq_lens.copy_(replay.seq_lens)
|
||||
captured.block_table_tensor.copy_(replay.block_table_tensor)
|
||||
builder.update_draft_decode_metadata(md)
|
||||
|
||||
assert md.decode_swa_width == builder.noncausal_index_width
|
||||
torch.testing.assert_close(md.decode_swa_lens, expected.decode_swa_lens)
|
||||
torch.testing.assert_close(md.decode_swa_indices, expected.decode_swa_indices)
|
||||
|
||||
@@ -425,7 +425,7 @@ def _skip_if_backend_unavailable(backend_cls, kv_cache_dtype: str, block_size: i
|
||||
and kv_cache_dtype != "fp8_ds_mla"
|
||||
):
|
||||
pytest.skip("FlashMLA Sparse fp8 only supports fp8_ds_mla kv-cache dtype")
|
||||
if block_size not in backend_cls.get_supported_kernel_block_sizes():
|
||||
if not backend_cls.supports_block_size(block_size):
|
||||
pytest.skip(
|
||||
f"{backend_cls.get_name()} does not support block_size={block_size}"
|
||||
)
|
||||
|
||||
@@ -29,7 +29,7 @@ HEAD = 512
|
||||
TOPK = 128 # triton convert requires width % 128 == 0
|
||||
|
||||
|
||||
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
|
||||
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **kw):
|
||||
out = torch.full_like(token_indices, -1)
|
||||
counts = torch.zeros(token_indices.shape[0], dtype=torch.int32)
|
||||
for t in range(token_indices.shape[0]):
|
||||
@@ -41,7 +41,7 @@ def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
|
||||
blk = int(block_table[int(req_id[t]), pos // BLOCK_SIZE])
|
||||
if blk < 0:
|
||||
continue
|
||||
vals.append(blk * BLOCK_SIZE + pos % BLOCK_SIZE)
|
||||
vals.append(blk * kw["BLOCK_STRIDE_ROWS"] + pos % BLOCK_SIZE)
|
||||
out[t, : len(vals)] = torch.tensor(vals, dtype=out.dtype)
|
||||
counts[t] = len(vals)
|
||||
return out, counts
|
||||
@@ -133,11 +133,8 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
meta.state = state
|
||||
q_nope = torch.randn(rows, impl.num_heads, HEAD)
|
||||
q_rope = torch.randn(rows, impl.num_heads, qk_rope)
|
||||
cache = torch.zeros(
|
||||
8 * BLOCK_SIZE,
|
||||
impl.head_size,
|
||||
dtype=torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16,
|
||||
)
|
||||
dtype = torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16
|
||||
cache = torch.zeros(8, 2, BLOCK_SIZE, impl.head_size, dtype=dtype)[:, 0]
|
||||
|
||||
out, lse = impl.forward_mqa(
|
||||
(q_nope, q_rope), cache, meta, SimpleNamespace(_k_scale_float=0.5)
|
||||
@@ -147,7 +144,10 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
# Reserved buffers carry this step's slots; lengths are NOT refreshed
|
||||
# here (the builder plans them host-side before capture/replay).
|
||||
ref_slots, ref_counts = ref_convert(
|
||||
meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer
|
||||
meta.req_id_per_token,
|
||||
meta.block_table,
|
||||
impl.topk_indices_buffer,
|
||||
BLOCK_STRIDE_ROWS=2 * BLOCK_SIZE,
|
||||
)
|
||||
offset = 0
|
||||
for t in range(rows):
|
||||
@@ -161,7 +161,7 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
assert state.wrapper.run_args is not None
|
||||
q_pe, ckv, kpe, kwargs = state.wrapper.run_args[1:]
|
||||
assert q_pe.shape == (rows, impl.num_heads, qk_rope)
|
||||
assert ckv.shape == (8 * BLOCK_SIZE, 1, HEAD)
|
||||
assert ckv.shape == ((8 - 1) * 2 * BLOCK_SIZE + BLOCK_SIZE, 1, HEAD)
|
||||
assert kpe.shape[-1] == qk_rope
|
||||
if impl.use_fp8_kv_cache:
|
||||
assert kwargs["ckv_scale"] == 0.5 and kwargs["kpe_scale"] == 1.0
|
||||
|
||||
@@ -24,6 +24,21 @@ from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
QUANTIZED_DS_MLA_CACHE_FORMATS,
|
||||
FlashMLASparseBackend,
|
||||
_fp8_kv_pages,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheTensor,
|
||||
KVQuantMode,
|
||||
MLAAttentionSpec,
|
||||
)
|
||||
from vllm.v1.worker.utils import (
|
||||
AttentionGroup,
|
||||
allocate_kv_cache,
|
||||
prepare_kernel_block_sizes,
|
||||
select_common_block_size,
|
||||
)
|
||||
|
||||
SM90 = DeviceCapability(major=9, minor=0)
|
||||
@@ -83,6 +98,88 @@ def test_supported_head_sizes_include_512():
|
||||
assert FlashMLASparseBackend.get_supported_head_sizes() == [576, 512]
|
||||
|
||||
|
||||
def test_flashmla_bf16_nope_accepts_packed_manager_blocks():
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=1152,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.bfloat16,
|
||||
cache_dtype_str="bfloat16",
|
||||
block_stride_alignment=1024,
|
||||
)
|
||||
num_blocks = 2
|
||||
layers = ["layer.0", "layer.1"]
|
||||
page_size = spec.page_size_bytes
|
||||
config = KVCacheConfig(
|
||||
num_blocks=num_blocks,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=num_blocks * len(layers) * page_size,
|
||||
layers=layers,
|
||||
layer_stride=page_size,
|
||||
block_stride=len(layers) * page_size,
|
||||
)
|
||||
],
|
||||
kv_cache_groups=[KVCacheGroupSpec(layers, spec)],
|
||||
)
|
||||
groups = [[AttentionGroup(FlashMLASparseBackend, layers, spec, 0)]]
|
||||
|
||||
kernel_block_sizes = prepare_kernel_block_sizes(config, groups)
|
||||
caches = allocate_kv_cache(
|
||||
config,
|
||||
torch.device("cpu"),
|
||||
KVCacheLayout.BLHNC,
|
||||
kernel_block_sizes,
|
||||
)
|
||||
|
||||
assert kernel_block_sizes == [1152]
|
||||
assert caches["layer.0"].shape == (num_blocks, 1, 1152, 512)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sm100", [False, True])
|
||||
def test_flashmla_quantized_cache_kernel_pages(monkeypatch, sm100):
|
||||
"""Quantized blocks are read as 64-token pages; SM100 only row-aligns them
|
||||
(TMA) and keeps 64-token blocks."""
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
lambda family, device_id=0: sm100 and family == 100,
|
||||
)
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=1152,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
kv_quant_mode=KVQuantMode.FP8_PER_TENSOR,
|
||||
state_content_bytes=656,
|
||||
)
|
||||
|
||||
kernel_block_size = 64 if sm100 else 1152
|
||||
assert (
|
||||
select_common_block_size(1152, [FlashMLASparseBackend], [spec])
|
||||
== kernel_block_size
|
||||
)
|
||||
|
||||
|
||||
def test_fp8_kv_pages_address_packed_blocks():
|
||||
"""Token t of block b is row b * block_stride_rows + t of the 64-row pages;
|
||||
64-token blocks are passed through."""
|
||||
row, block_size, stride_rows = 656, 128, 192
|
||||
backing = torch.arange(3 * stride_rows).repeat_interleave(row)
|
||||
cache = backing.view(3, stride_rows, row)[:, :block_size]
|
||||
|
||||
pages, block_stride_rows = _fp8_kv_pages(cache, block_size)
|
||||
assert block_stride_rows == stride_rows
|
||||
assert pages.shape[1:] == (64, row)
|
||||
slot = 2 * block_stride_rows + 100
|
||||
assert torch.equal(pages[slot // 64, slot % 64], cache[2, 100])
|
||||
|
||||
pages, block_stride_rows = _fp8_kv_pages(cache[:, :64], 64)
|
||||
assert block_stride_rows is None
|
||||
assert torch.equal(pages, cache[:, :64])
|
||||
|
||||
|
||||
def test_quantized_ds_mla_formats_are_the_envelope_set():
|
||||
assert frozenset({"fp8_ds_mla", "nvfp4_ds_mla"}) == QUANTIZED_DS_MLA_CACHE_FORMATS
|
||||
|
||||
@@ -141,6 +238,7 @@ def _sparse_order(kv_cache_dtype, head_size, num_heads=32, capability=SM100):
|
||||
)
|
||||
sparse = {
|
||||
"FLASH_ATTN_MLA_SPARSE",
|
||||
"FLASH_ATTN_MLA_SPARSE_FA4",
|
||||
"FLASHMLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE",
|
||||
"FLASHINFER_MLA_SPARSE_SM90",
|
||||
@@ -179,3 +277,36 @@ def test_sm90_nope_fp8_ds_mla_resolves_to_flashmla():
|
||||
for name in order[: order.index("FLASHMLA_SPARSE")]:
|
||||
backend_cls = AttentionBackendEnum[name].get_class()
|
||||
assert not backend_cls.supports_kv_cache_dtype("fp8_ds_mla"), name
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA backends")
|
||||
def test_sm100_576_bf16_low_heads_falls_back_from_fa4(monkeypatch):
|
||||
"""With FA4 unusable, low-head bf16 selection lands on FlashInfer."""
|
||||
import vllm.v1.attention.backends.mla.flashattn_mla_sparse as fa4_sparse
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
monkeypatch.setattr(fa4_sparse, "_fa4_cute_mla_available", lambda: "no FA4")
|
||||
|
||||
def reasons(name):
|
||||
backend_cls = AttentionBackendEnum[name].get_class()
|
||||
return backend_cls.validate_configuration(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
block_size=64,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=True,
|
||||
use_mm_prefix=False,
|
||||
use_per_head_quant_scales=False,
|
||||
device_capability=SM100,
|
||||
attn_type="decoder",
|
||||
)
|
||||
|
||||
order = _sparse_order("auto", 576, num_heads=16)
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
# The selector takes the first candidate with no rejection reasons.
|
||||
selected = next(name for name in order if not reasons(name))
|
||||
assert order[0] == "FLASH_ATTN_MLA_SPARSE_FA4", order
|
||||
assert selected == "FLASHINFER_MLA_SPARSE"
|
||||
|
||||
@@ -35,7 +35,7 @@ def _make_config(
|
||||
)
|
||||
|
||||
|
||||
def _resolve_on_rocm(config, *, kernels_available: bool = True):
|
||||
def _resolve_on_rocm(config, *, kernels_available: bool = True, mi3xx: bool = True):
|
||||
with (
|
||||
patch.object(
|
||||
qwen_gdn_linear_attn.current_platform, "is_rocm", return_value=True
|
||||
@@ -45,6 +45,7 @@ def _resolve_on_rocm(config, *, kernels_available: bool = True):
|
||||
"is_gdn_flydsl_prefill_available",
|
||||
return_value=kernels_available,
|
||||
),
|
||||
patch("vllm.platforms.rocm.on_mi3xx", return_value=mi3xx),
|
||||
):
|
||||
return _resolve_gdn_prefill_backend(config)
|
||||
|
||||
@@ -72,16 +73,38 @@ def test_resolve_aiter_flydsl_gdn_prefill_backend(
|
||||
assert active == expected
|
||||
|
||||
|
||||
def test_kda_style_model_cannot_select_flydsl():
|
||||
@pytest.mark.parametrize(
|
||||
"mi3xx,kernels_available,head_k_dim,expected",
|
||||
[
|
||||
(True, True, 128, "aiter_flydsl"),
|
||||
(False, True, 128, "triton"),
|
||||
(True, False, 128, "triton"),
|
||||
(True, True, 64, "triton"),
|
||||
],
|
||||
)
|
||||
def test_auto_defaults_to_aiter_flydsl(
|
||||
mi3xx: bool, kernels_available: bool, head_k_dim: int, expected: str
|
||||
):
|
||||
"""Under auto, FlyDSL is the default only where it was validated and the
|
||||
model and build can serve it; everything else stays on Triton."""
|
||||
config = _make_config(backend="auto", head_k_dim=head_k_dim)
|
||||
|
||||
assert _resolve_on_rocm(
|
||||
config, kernels_available=kernels_available, mi3xx=mi3xx
|
||||
) == ("auto", expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["aiter_flydsl", "auto"])
|
||||
def test_kda_style_model_cannot_select_flydsl(backend: str):
|
||||
"""Kimi K3 KDA reports head dims through linear_attn_config, not these.
|
||||
|
||||
Its builder overrides _build_chunk_metadata, which the AITER path skips,
|
||||
so this pins the reason the two never meet: the resolver turns the model
|
||||
down before the builder is ever constructed.
|
||||
"""
|
||||
config = _make_config(head_k_dim=None, head_v_dim=None)
|
||||
config = _make_config(backend=backend, head_k_dim=None, head_v_dim=None)
|
||||
|
||||
assert _resolve_on_rocm(config) == ("aiter_flydsl", "triton")
|
||||
assert _resolve_on_rocm(config) == (backend, "triton")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("on_rocm", [True, False], ids=["kernels_missing", "not_rocm"])
|
||||
@@ -109,15 +132,16 @@ def test_explicit_aiter_flydsl_fails_closed(on_rocm: bool):
|
||||
|
||||
|
||||
def _flydsl_prefill_unavailable() -> bool:
|
||||
return (
|
||||
not current_platform.is_rocm()
|
||||
or not rocm_aiter_ops.is_gdn_flydsl_prefill_available()
|
||||
)
|
||||
if not current_platform.is_rocm():
|
||||
return True
|
||||
from vllm.platforms.rocm import on_mi3xx
|
||||
|
||||
return not on_mi3xx() or not rocm_aiter_ops.is_gdn_flydsl_prefill_available()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
_flydsl_prefill_unavailable(),
|
||||
reason="needs ROCm with the AITER FlyDSL GDN prefill kernels installed",
|
||||
reason="needs gfx942/gfx950 with the AITER FlyDSL GDN prefill kernels installed",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"state_dtype", [torch.float32, torch.bfloat16], ids=["fp32_state", "bf16_state"]
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.gdn_attn import (
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
plan_flashinfer_checkpoints,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
@@ -488,3 +489,20 @@ def test_update_block_table_regathers_checkpoint(num_spec):
|
||||
assert not torch.equal(
|
||||
actual.checkpoint.state_indices, source.checkpoint.state_indices
|
||||
)
|
||||
|
||||
|
||||
def test_plan_flashinfer_checkpoints_selects_the_state_at_each_offset():
|
||||
"""Each checkpoint row maps to the state FlashInfer emits at its offset.
|
||||
|
||||
The leading decode row is not in the prefill kernel and never checkpoints;
|
||||
the 50-token prefill is shorter than the stride and emits no states.
|
||||
"""
|
||||
every_n_tokens, cu_starts, rows = plan_flashinfer_checkpoints(
|
||||
offsets=[0, 128, 0, 192], query_lens=[200, 50, 300]
|
||||
)
|
||||
|
||||
assert every_n_tokens == 64
|
||||
assert cu_starts == [0, 3, 3, 7]
|
||||
# Row 1: state after 128 tokens of the first prefill. Row 3: after 192
|
||||
# tokens of the third prefill, whose states start at index 3.
|
||||
assert rows == [0, 1, 0, 5]
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLM-5.3-Flash indexer kernel block sizes are DeepGEMM pool pages."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla import indexer
|
||||
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
|
||||
|
||||
KPOOL = 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rocm,sm120,expected",
|
||||
[
|
||||
(False, False, [KPOOL * 32, KPOOL * 64]),
|
||||
# SM120 DeepGEMM takes only 64-entry pages for the fp8 cache.
|
||||
(False, True, [KPOOL * 64]),
|
||||
# ROCm's AITER paged-MQA logits take the same pool pages.
|
||||
(True, False, [KPOOL * 32, KPOOL * 64]),
|
||||
],
|
||||
)
|
||||
def test_glm5next_indexer_kernel_block_sizes(monkeypatch, rocm, sm120, expected):
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: rocm)
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
lambda family: sm120 and family == 120,
|
||||
)
|
||||
model_config = SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=KPOOL))
|
||||
config = SimpleNamespace(model_config=model_config)
|
||||
monkeypatch.setattr(indexer, "get_current_vllm_config", lambda: config)
|
||||
|
||||
spec = SimpleNamespace(tokens_per_state=KPOOL)
|
||||
assert Glm5NextIndexerBackend.get_supported_kernel_block_sizes(spec) == expected
|
||||
# Platform block-size selection queries without a spec.
|
||||
assert Glm5NextIndexerBackend.get_supported_kernel_block_sizes() == expected
|
||||
@@ -14,6 +14,7 @@ from vllm.models.deepseek_v4.sparse_mla import DeepseekV4SparseMLABackend
|
||||
from vllm.models.deepseek_v41.sparse_mla import (
|
||||
DeepseekV4SparseMLABackend as DeepseekV41SparseMLABackend,
|
||||
)
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla import indexer as indexer_module
|
||||
from vllm.v1.attention.backends.mla.compressor_utils import (
|
||||
@@ -113,16 +114,22 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
num_reqs, 4
|
||||
)
|
||||
num_sms = torch.cuda.get_device_properties(device).multi_processor_count
|
||||
# SM100's paged MQA logits only take varlen rows.
|
||||
indices = (
|
||||
torch.arange(num_reqs, dtype=torch.int32, device=device)
|
||||
if mqa_logits_use_mx_scales()
|
||||
else None
|
||||
)
|
||||
schedule_metadata = indexer_module.get_paged_mqa_logits_metadata(
|
||||
torch.div(seq_lens, 16, rounding_mode="floor").unsqueeze(1),
|
||||
64,
|
||||
num_sms,
|
||||
indices=indices,
|
||||
).clone()
|
||||
|
||||
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
|
||||
builder.dcp_world_size = 1
|
||||
builder.compress_ratio = 16
|
||||
builder.kernel_block_size = None
|
||||
builder.num_sms = num_sms
|
||||
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
builder.kv_cache_spec = SimpleNamespace(num_states=64)
|
||||
@@ -140,6 +147,7 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
decode_lens=torch.zeros(num_reqs, dtype=torch.int32, device=device),
|
||||
requires_padding=False,
|
||||
schedule_metadata=schedule_metadata,
|
||||
indices=indices,
|
||||
),
|
||||
block_table=block_table,
|
||||
)
|
||||
@@ -158,7 +166,7 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
torch.testing.assert_close(
|
||||
schedule_metadata,
|
||||
indexer_module.get_paged_mqa_logits_metadata(
|
||||
expected_decode_seq_lens.unsqueeze(1), 64, num_sms
|
||||
expected_decode_seq_lens.unsqueeze(1), 64, num_sms, indices=indices
|
||||
),
|
||||
)
|
||||
assert metadata.decode is not None
|
||||
@@ -170,30 +178,24 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"block_size,kernel_block_size,compress_ratio", [(64, 64, 1), (256, 64, 4)]
|
||||
)
|
||||
def test_indexer_draft_decode_refresh_follows_new_batch(
|
||||
monkeypatch, block_size, kernel_block_size, compress_ratio
|
||||
):
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 4])
|
||||
def test_indexer_draft_decode_refresh_follows_new_batch(monkeypatch, compress_ratio):
|
||||
"""Without a prior build(), the refresh must not keep the last batch's state."""
|
||||
monkeypatch.setattr(indexer_module, "has_deep_gemm", lambda: False)
|
||||
device = torch.device("cuda")
|
||||
num_reqs, width = 3, 8
|
||||
factor = block_size // kernel_block_size
|
||||
|
||||
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
|
||||
builder.dcp_world_size = 1
|
||||
builder.compress_ratio = compress_ratio
|
||||
builder.kernel_block_size = kernel_block_size
|
||||
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=block_size, num_states=64)
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
|
||||
|
||||
def stale(*shape: int) -> torch.Tensor:
|
||||
return torch.full(shape, -7, dtype=torch.int32, device=device)
|
||||
|
||||
decode = DeepSeekV32IndexerDecodeMetadata(
|
||||
block_table=stale(num_reqs, width // factor),
|
||||
block_table=stale(num_reqs, width),
|
||||
seq_lens=stale(num_reqs, 1),
|
||||
decode_lens=stale(num_reqs),
|
||||
requires_padding=False,
|
||||
@@ -219,8 +221,7 @@ def test_indexer_draft_decode_refresh_follows_new_batch(
|
||||
|
||||
builder.update_draft_decode_metadata(metadata)
|
||||
|
||||
expected_block_table = request_block_table[:, ::factor] // factor
|
||||
torch.testing.assert_close(decode.block_table, expected_block_table)
|
||||
torch.testing.assert_close(decode.block_table, request_block_table)
|
||||
torch.testing.assert_close(decode.indices, builder.arange_buffer[:num_reqs])
|
||||
assert torch.all(decode.per_req_decode_lens == 1)
|
||||
assert torch.all(decode.decode_lens == 1)
|
||||
@@ -471,7 +472,6 @@ def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypat
|
||||
builder.pcp_rank = 0
|
||||
builder.dcp_world_size = 1
|
||||
builder.cp_kv_cache_interleave_size = 1
|
||||
builder.kernel_block_size = None
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
|
||||
builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64)
|
||||
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""TP query-row sharding of the DSA indexer prefill (see PrefillRowShard):
|
||||
each rank scores a contiguous slice; the group exchanges int32s per row."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
|
||||
import vllm.v1.attention.backends.mla.indexer as indexer
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import PrefillRowShard
|
||||
|
||||
|
||||
def _chunk(start: int, end: int) -> SimpleNamespace:
|
||||
"""Chunk over rows [start, end); bounds derive from the global row."""
|
||||
n = end - start
|
||||
return SimpleNamespace(
|
||||
token_start=start,
|
||||
token_end=end,
|
||||
cu_seqlen_ks=torch.zeros(n, dtype=torch.int32),
|
||||
cu_seqlen_ke=((torch.arange(n) * 7 + start) % 65).int(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("world", [2, 3, 8])
|
||||
def test_shard_narrowing_covers_every_row_once(world: int) -> None:
|
||||
"""Narrowed windows tile the prefill rows exactly once, carry the causal
|
||||
bounds of those global rows, and skip chunks foreign to this rank."""
|
||||
decode, rows = 5, 997
|
||||
edges = [decode, decode + 330, decode + 337, decode + rows]
|
||||
chunks = [_chunk(lo, hi) for lo, hi in zip(edges, edges[1:])]
|
||||
sizes = [rows // world + int(r < rows % world) for r in range(world)]
|
||||
covered = []
|
||||
for rank in range(world):
|
||||
shard = PrefillRowShard(decode, sizes, rank)
|
||||
for chunk in chunks:
|
||||
window = shard.narrow(chunk)
|
||||
if window is None:
|
||||
continue
|
||||
start, end, ks, ke = window
|
||||
lo, hi = start - chunk.token_start, end - chunk.token_start
|
||||
torch.testing.assert_close(ks, chunk.cu_seqlen_ks[lo:hi])
|
||||
torch.testing.assert_close(ke, chunk.cu_seqlen_ke[lo:hi])
|
||||
covered.append((start, end))
|
||||
covered.sort()
|
||||
# Windows must be disjoint, contiguous and exactly cover the prefill rows.
|
||||
assert [s for s, _ in covered] == [decode, *[e for _, e in covered][:-1]]
|
||||
assert covered[-1][1] == decode + rows
|
||||
|
||||
|
||||
def test_shard_exchange_reassembles_rows_only(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""Gathered indices land only on real prefill rows: leading decode rows
|
||||
and trailing CUDA-graph padding stay untouched, and the full k-pool
|
||||
width (tail columns included) is exchanged."""
|
||||
decode, pad, topk, kpool = 5, 3, 8, 4
|
||||
width = topk + kpool - 1
|
||||
sizes = [17, 9, 21]
|
||||
buffer = torch.full((decode + sum(sizes) + pad, width), -7, dtype=torch.int32)
|
||||
gathered = torch.arange(sum(sizes) * width, dtype=torch.int32).reshape(-1, width)
|
||||
calls = []
|
||||
|
||||
def fake_group():
|
||||
def all_gatherv(local, dim, sizes):
|
||||
calls.append((local, sizes))
|
||||
return gathered
|
||||
|
||||
return SimpleNamespace(all_gatherv=all_gatherv)
|
||||
|
||||
monkeypatch.setattr(sparse_indexer, "get_tp_group", fake_group)
|
||||
for rank in range(len(sizes)):
|
||||
PrefillRowShard(decode, sizes, rank).exchange_topk(buffer, width)
|
||||
assert calls[0][1] == sizes and calls[0][0].is_contiguous()
|
||||
torch.testing.assert_close(buffer[decode : decode + sum(sizes)], gathered)
|
||||
assert torch.all(buffer[:decode] == -7), "decode rows stay outside"
|
||||
assert torch.all(buffer[decode + sum(sizes) :] == -7), "padding untouched"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
def test_balanced_row_shard_partitions_and_declines(tp_size: int) -> None:
|
||||
"""The split covers every row once with >= 1 row per rank and declines
|
||||
below the activation floor (or on a single rank)."""
|
||||
n = indexer.MIN_TP_SHARD_ROWS_PER_RANK * tp_size
|
||||
shapes = [
|
||||
([4 * n], [4 * n]), # fresh prompt: the full causal cost ramp
|
||||
# uneven requests at different context depths
|
||||
([2 * n, 7, 4 * n, n // 3 + 5], [2 * n, 9007, 4 * n + 500, n // 3 + 60005]),
|
||||
]
|
||||
ratio = 4
|
||||
for query_lens, seq_lens in shapes:
|
||||
sizes = indexer.balanced_prefill_row_shard(
|
||||
torch.tensor(seq_lens, dtype=torch.int32),
|
||||
torch.tensor(query_lens, dtype=torch.int32),
|
||||
ratio,
|
||||
tp_size,
|
||||
)
|
||||
assert sizes is not None and len(sizes) == tp_size
|
||||
assert sum(sizes) == sum(query_lens) and min(sizes) >= 1
|
||||
lens = torch.tensor([n - 1], dtype=torch.int32)
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, tp_size) is None
|
||||
assert indexer.balanced_prefill_row_shard(lens, lens, ratio, 1) is None
|
||||
|
||||
|
||||
def test_row_sharding_gate_rejects_unsupported_configurations(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The gate is the whole safety envelope; nothing else guards the exchange."""
|
||||
monkeypatch.setattr(indexer.current_platform, "is_cuda", lambda: True)
|
||||
|
||||
def supported(**kwargs):
|
||||
mode = kwargs.pop("cudagraph_mode", CUDAGraphMode.PIECEWISE)
|
||||
cfg = SimpleNamespace(compilation_config=SimpleNamespace(cudagraph_mode=mode))
|
||||
args = {"dcp_world_size": 1, "use_pcp": False, "tp_size": 4, **kwargs}
|
||||
return indexer.tp_prefill_row_sharding_supported(cfg, **args)
|
||||
|
||||
assert supported()
|
||||
for bad in ({"tp_size": 1}, {"dcp_world_size": 2}, {"use_pcp": True}):
|
||||
assert not supported(**bad)
|
||||
for name in (
|
||||
"VLLM_DISABLE_PYNCCL",
|
||||
"VLLM_USE_NCCL_SYMM_MEM",
|
||||
"VLLM_BATCH_INVARIANT",
|
||||
):
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(indexer.envs, name, True)
|
||||
assert not supported()
|
||||
# FULL captures mixed batches whole, so the exchange must stay out.
|
||||
assert not supported(cudagraph_mode=CUDAGraphMode.FULL)
|
||||
assert supported(cudagraph_mode=CUDAGraphMode.FULL_AND_PIECEWISE)
|
||||
|
||||
|
||||
def test_profiling_run_reserves_exchange_transient(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""The dummy run returns before the exchange executes, so its all_gatherv
|
||||
output plus staging copy must be reserved up front when row sharding is
|
||||
active -- and only then."""
|
||||
rows, topk = 128, 8
|
||||
buffer = torch.empty((rows, topk), dtype=torch.int32)
|
||||
peak = 2 * rows * topk * buffer.element_size()
|
||||
specs = (((1,), torch.uint8), ((1,), torch.uint8))
|
||||
numels: list[int] = []
|
||||
real_empty = torch.empty
|
||||
|
||||
def recording_empty(*args, **kwargs):
|
||||
out = real_empty(*args, **kwargs)
|
||||
numels.append(out.numel())
|
||||
return out
|
||||
|
||||
def run_dummy(tp_world: int) -> None:
|
||||
numels.clear()
|
||||
with monkeypatch.context() as m:
|
||||
m.setattr(torch, "empty", recording_empty)
|
||||
m.setattr(
|
||||
sparse_indexer,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata=None),
|
||||
)
|
||||
m.setattr(sparse_indexer, "_gather_workspace_shapes", lambda *a: specs)
|
||||
m.setattr(
|
||||
sparse_indexer,
|
||||
"current_workspace_manager",
|
||||
lambda: SimpleNamespace(get_simultaneous=lambda *s: None),
|
||||
)
|
||||
m.setattr(
|
||||
sparse_indexer,
|
||||
"get_tp_group",
|
||||
lambda: SimpleNamespace(world_size=tp_world),
|
||||
)
|
||||
m.setattr(
|
||||
sparse_indexer,
|
||||
"get_current_vllm_config_or_none",
|
||||
lambda: SimpleNamespace(
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.PIECEWISE
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=1,
|
||||
prefill_context_parallel_size=1,
|
||||
),
|
||||
),
|
||||
)
|
||||
m.setattr(sparse_indexer.envs, "VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", 1)
|
||||
m.setattr(indexer.current_platform, "is_cuda", lambda: True)
|
||||
sparse_indexer.sparse_attn_indexer(
|
||||
hidden_states=torch.empty(4, 4),
|
||||
k_cache_prefix="k",
|
||||
kv_cache=torch.empty(4, 4),
|
||||
q_quant=torch.empty(4, 4),
|
||||
q_scale=None,
|
||||
k=None,
|
||||
weights=torch.empty(4),
|
||||
quant_block_size=128,
|
||||
scale_fmt=None,
|
||||
topk_tokens=topk,
|
||||
head_dim=64,
|
||||
max_model_len=1024,
|
||||
total_seq_lens=512,
|
||||
topk_indices_buffer=buffer,
|
||||
skip_k_cache_insert=True,
|
||||
use_pcp=False,
|
||||
pcp_shard_decode_requests=False,
|
||||
dense_mha_metadata_layer_name="",
|
||||
)
|
||||
|
||||
run_dummy(tp_world=2)
|
||||
assert peak in numels
|
||||
run_dummy(tp_world=1)
|
||||
assert peak not in numels
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for storage-block kernel-block selection in prepare_kernel_block_sizes."""
|
||||
"""CPU tests for the kpool indexer's kernel block selection."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -11,20 +11,14 @@ from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.models.glm5next.common.attention import Glm5NextIndexerCache
|
||||
from vllm.v1.attention.backend import AttentionBackend, MultipleOf
|
||||
from vllm.v1.attention.backends.mla import indexer as indexer_mod
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerBackend,
|
||||
KpoolTailBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.utils import prepare_kernel_block_sizes
|
||||
from vllm.v1.worker.utils import prepare_kernel_block_sizes, select_common_block_size
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
# ``index_kpool`` of zai-org/GLM-5.3-Flash: one indexer state pools 4
|
||||
# compressed states (hence storage_block_size = page * 4).
|
||||
# compressed states (hence a kernel block is page * 4 tokens).
|
||||
INDEX_KPOOL = 4
|
||||
|
||||
# The hybrid KDA/mamba manager floors at TP8..TP1; 640 is the geometry of the
|
||||
@@ -59,14 +53,17 @@ def _prepare(
|
||||
spec: MLAAttentionSpec, backends: list[type[AttentionBackend]]
|
||||
) -> list[int]:
|
||||
kv_cache_config = SimpleNamespace(
|
||||
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec)]
|
||||
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec, layer_names=[])],
|
||||
kv_cache_tensors=[],
|
||||
)
|
||||
attn_groups = [[SimpleNamespace(backend=backend) for backend in backends]]
|
||||
attn_groups = [
|
||||
[SimpleNamespace(backend=backend, kv_cache_spec=spec) for backend in backends]
|
||||
]
|
||||
return prepare_kernel_block_sizes(kv_cache_config, attn_groups)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("manager_block", TP_FLOOR_MANAGER_BLOCKS)
|
||||
def test_prepare_uses_storage_block_for_the_kpool_group(
|
||||
def test_kpool_group_maps_pool_pages(
|
||||
monkeypatch: pytest.MonkeyPatch, manager_block: int
|
||||
):
|
||||
"""The bug: on ROCm, kpool groups must get 128/256 - not the 640-4352
|
||||
@@ -78,54 +75,43 @@ def test_prepare_uses_storage_block_for_the_kpool_group(
|
||||
"""
|
||||
_mock_rocm_platform(monkeypatch)
|
||||
spec = _kpool_storage_spec(manager_block)
|
||||
expected_storage = 256 if manager_block % 256 == 0 else 128
|
||||
assert spec.storage_block_size == expected_storage
|
||||
expected = 256 if manager_block % 256 == 0 else 128
|
||||
|
||||
selected = _prepare(
|
||||
spec,
|
||||
[ROCMAiterMLASparseBackend, DeepseekV32IndexerBackend, KpoolTailBackend],
|
||||
)
|
||||
assert selected == [expected_storage]
|
||||
# The kpool group's AttentionGroup selects its kernel block this way.
|
||||
selected = [
|
||||
select_common_block_size(manager_block, [Glm5NextIndexerBackend], [spec])
|
||||
]
|
||||
assert selected == [expected]
|
||||
assert selected != [manager_block]
|
||||
# The hybrid block-table split stays integral.
|
||||
assert manager_block % selected[0] == 0
|
||||
|
||||
|
||||
def test_prepare_falls_back_when_storage_block_is_not_supported():
|
||||
"""No storage block, or a storage block the backends reject (e.g. the
|
||||
CUDA indexer's exact ``[64]`` vs a 128-token storage block): selection is
|
||||
`select_common_block_size`'s answer, unchanged from no-fix behavior.
|
||||
"""
|
||||
def test_prepare_uses_the_backend_vote():
|
||||
"""Unpacked groups take `select_common_block_size`'s answer."""
|
||||
|
||||
class Fixed64Backend:
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None):
|
||||
return [64]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FIXED64"
|
||||
|
||||
# Storage block the group's backend does not accept -> backend vote (64).
|
||||
spec = _kpool_storage_spec(640)
|
||||
assert spec.storage_block_size == 128
|
||||
assert _prepare(spec, [Fixed64Backend]) == [64]
|
||||
|
||||
# No storage block at all (any non-kpool MLA model) -> backend vote.
|
||||
plain_spec = MLAAttentionSpec(
|
||||
block_size=640,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
assert plain_spec.storage_block_size is None
|
||||
assert _prepare(plain_spec, [Fixed64Backend]) == [64]
|
||||
|
||||
# ...and with backends that accept the manager block, the manager wins,
|
||||
# exactly as on main.
|
||||
class AcceptAllBackend:
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None):
|
||||
return [MultipleOf(1)]
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""CPU tests for the kpool tail slot mapping (no GPU required).
|
||||
|
||||
The kpool tail cache is a 1-block-per-request circular ring addressed by
|
||||
``pos % kpool`` (``KpoolTailSpec`` / ``KpoolTailManager``: exactly one block
|
||||
``pos % kpool`` (``CircularBufferSpec`` / ``CircularBufferManager``: exactly one block
|
||||
allocated per request, never grown, so only column 0 of its block table is
|
||||
ever written; the rest stays zero-initialized).
|
||||
|
||||
@@ -32,27 +32,25 @@ from vllm.v1.attention.backends.mla.indexer import (
|
||||
KpoolTailMetadataBuilder,
|
||||
compute_kpool_tail_slot_mapping,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides
|
||||
from vllm.v1.kv_cache_interface import CircularBufferSpec, compute_layout_strides
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
|
||||
KPOOL = 4
|
||||
|
||||
|
||||
def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
|
||||
(layout,) = KpoolTailBackend.supported_kv_cache_layouts()
|
||||
spec = KpoolTailSpec(
|
||||
spec = CircularBufferSpec(
|
||||
block_size=KPOOL,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=KPOOL,
|
||||
)
|
||||
strides = compute_layout_strides(spec, num_blocks=8, num_layers=3, layout=layout)
|
||||
_, _, head_stride, state_stride, content_stride = strides
|
||||
|
||||
assert layout is KVCacheLayout.LBHNC
|
||||
assert layout is KVCacheLayout.BLHNC
|
||||
assert head_stride == KPOOL * 128 * torch.bfloat16.itemsize
|
||||
assert state_stride == 128 * torch.bfloat16.itemsize
|
||||
assert content_stride == 1
|
||||
@@ -77,37 +75,14 @@ def test_tail_ring_divides_the_attention_block(num_speculative_tokens, ring):
|
||||
SimpleNamespace(num_speculative_tokens=num_speculative_tokens)
|
||||
)
|
||||
|
||||
assert spec.block_size == spec.sliding_window == ring
|
||||
assert isinstance(spec, CircularBufferSpec) and spec.block_size == ring
|
||||
assert ring >= KPOOL + num_speculative_tokens
|
||||
assert 640 % ring == 0
|
||||
|
||||
|
||||
def test_tail_spec_opts_out_of_generic_slot_mapping():
|
||||
"""The tail row is one block wide (padded to the block-table alignment), so
|
||||
the generic kernel's ``pos // kpool`` column index runs off the end of the
|
||||
allocation for long prompts. The spec must opt out of it entirely."""
|
||||
spec = KpoolTailSpec(
|
||||
block_size=KPOOL,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=KPOOL,
|
||||
)
|
||||
max_len = 1 << 20
|
||||
width = get_block_table_width(
|
||||
spec.max_num_blocks_per_req(None, max_len),
|
||||
spec.block_size,
|
||||
token_alignment=spec.block_table_token_alignment,
|
||||
)
|
||||
|
||||
assert width * KPOOL < max_len
|
||||
assert spec.uses_slot_mapping is False
|
||||
|
||||
|
||||
def make_tail_block_table(own_blocks, width=64):
|
||||
"""Tail-group block table as BlockTables produces it: column 0 holds the
|
||||
request's single KpoolTailManager block, the remaining columns are never
|
||||
request's single CircularBufferManager block, the remaining columns are never
|
||||
written and stay zero."""
|
||||
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
|
||||
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
|
||||
@@ -447,6 +422,7 @@ def test_triton_mapping_matches_cpu(per_req, own_blocks, num_actual, padded_len)
|
||||
tokens between the last request boundary and num_actual_tokens (mapped to
|
||||
the last request), untouched padding beyond num_actual, and PAD for
|
||||
requests on the null block."""
|
||||
ring_size = 2 * KPOOL
|
||||
positions, qsl, slot_mapping, _, num_reqs = make_batch(
|
||||
per_req, padded_len=padded_len
|
||||
)
|
||||
@@ -455,14 +431,17 @@ def test_triton_mapping_matches_cpu(per_req, own_blocks, num_actual, padded_len)
|
||||
slot_mapping = torch.arange(padded_len, dtype=torch.int64) + 1000
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
ref = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
|
||||
got = circular_tail_slots(
|
||||
ref = compute_kpool_tail_slot_mapping(
|
||||
slot_mapping, bt, qsl, positions, num_actual, num_reqs, ring_size
|
||||
)
|
||||
got = compute_kpool_tail_slot_mapping(
|
||||
slot_mapping.cuda(),
|
||||
bt.cuda(),
|
||||
qsl.cuda().to(torch.int32),
|
||||
positions.cuda(),
|
||||
num_actual,
|
||||
num_reqs,
|
||||
ring_size,
|
||||
)
|
||||
torch.testing.assert_close(got.cpu(), ref)
|
||||
for req, blk in enumerate(own_blocks):
|
||||
|
||||
@@ -358,6 +358,8 @@ if current_platform.is_cuda():
|
||||
)
|
||||
elif current_platform.is_rocm():
|
||||
PREFILL_BACKENDS_TO_TEST.append(MLAPrefillBackendEnum.ROCM_AITER_FA)
|
||||
elif current_platform.is_xpu():
|
||||
PREFILL_BACKENDS_TO_TEST.append(MLAPrefillBackendEnum.FLASH_ATTN)
|
||||
|
||||
MLA_DIMENSIONS_TO_TEST = [
|
||||
("deepseek", 128, 128),
|
||||
@@ -370,25 +372,20 @@ def _prefill_backend_dimension_params():
|
||||
params = []
|
||||
for prefill_backend in PREFILL_BACKENDS_TO_TEST:
|
||||
for dimensions_id, qk_nope_head_dim, v_head_dim in MLA_DIMENSIONS_TO_TEST:
|
||||
if device_capability is None:
|
||||
invalid_reasons = ["device capability unavailable"]
|
||||
else:
|
||||
try:
|
||||
invalid_reasons = (
|
||||
prefill_backend.get_class().validate_configuration(
|
||||
device_capability,
|
||||
MLAPrefillSelectorConfig(
|
||||
dtype=torch.bfloat16,
|
||||
mla_dimensions=MLADimensions(
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=v_head_dim,
|
||||
),
|
||||
),
|
||||
)
|
||||
)
|
||||
except ImportError:
|
||||
invalid_reasons = ["ImportError"]
|
||||
try:
|
||||
invalid_reasons = prefill_backend.get_class().validate_configuration(
|
||||
device_capability,
|
||||
MLAPrefillSelectorConfig(
|
||||
dtype=torch.bfloat16,
|
||||
mla_dimensions=MLADimensions(
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=v_head_dim,
|
||||
),
|
||||
),
|
||||
)
|
||||
except ImportError:
|
||||
invalid_reasons = ["ImportError"]
|
||||
|
||||
marks = []
|
||||
if invalid_reasons:
|
||||
@@ -1098,6 +1095,7 @@ def test_flashinfer_mla_dspark_dcp_supports_target_and_draft(monkeypatch):
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("cp_interleave_size", [1, 16, 896])
|
||||
@pytest.mark.parametrize("min_split_kv", [1, 8])
|
||||
def test_tokenspeed_mla_decode_contract(
|
||||
monkeypatch,
|
||||
causal,
|
||||
@@ -1105,6 +1103,7 @@ def test_tokenspeed_mla_decode_contract(
|
||||
dcp_world_size,
|
||||
dcp_rank,
|
||||
cp_interleave_size,
|
||||
min_split_kv,
|
||||
):
|
||||
decode_call = None
|
||||
num_decodes = 2
|
||||
@@ -1148,7 +1147,12 @@ def test_tokenspeed_mla_decode_contract(
|
||||
impl.scale = 1.0
|
||||
impl.softmax_scale = 1.0
|
||||
impl.output_scale = 1.0
|
||||
impl._workspace_buffer = torch.empty(1, dtype=torch.int8)
|
||||
impl._min_split_kv = min_split_kv
|
||||
impl._max_decode_tokens = num_decode_tokens
|
||||
workspace = torch.empty(1, dtype=torch.int8)
|
||||
monkeypatch.setattr(
|
||||
tokenspeed_mla_module, "_get_workspace", lambda *args: workspace
|
||||
)
|
||||
|
||||
metadata = SimpleNamespace(
|
||||
num_decodes=num_decodes,
|
||||
@@ -1183,6 +1187,8 @@ def test_tokenspeed_mla_decode_contract(
|
||||
assert lse.shape == (num_decode_tokens, num_heads)
|
||||
|
||||
assert decode_call is not None
|
||||
assert decode_call["workspace_buffer"] is workspace
|
||||
assert decode_call["min_split_kv"] == min_split_kv
|
||||
assert decode_call["query"].shape == (
|
||||
num_decodes,
|
||||
tokens_per_decode,
|
||||
@@ -1217,16 +1223,23 @@ def test_tokenspeed_mla_decode_contract(
|
||||
pytest.param(896, 8 * 896 + 17, id="block-interleave-full-cycle"),
|
||||
],
|
||||
)
|
||||
def test_tokenspeed_mla_dcp_matches_unsharded_decode(cp_interleave_size, seq_len):
|
||||
def test_tokenspeed_mla_dcp_matches_unsharded_decode(
|
||||
monkeypatch, cp_interleave_size, seq_len
|
||||
):
|
||||
from tokenspeed_mla import tokenspeed_mla_decode
|
||||
|
||||
from vllm.v1.attention.ops.dcp import (
|
||||
_lse_weighted_combine,
|
||||
mask_dcp_empty_shards_,
|
||||
)
|
||||
from vllm.v1.worker.workspace import WorkspaceManager
|
||||
|
||||
torch.manual_seed(7)
|
||||
device = torch.device("cuda")
|
||||
workspace_manager = WorkspaceManager(device)
|
||||
monkeypatch.setattr(
|
||||
tokenspeed_mla_module, "current_workspace_manager", lambda: workspace_manager
|
||||
)
|
||||
dcp_world_size = 8
|
||||
kernel_block_size = 64
|
||||
num_heads = 128
|
||||
@@ -1257,7 +1270,9 @@ def test_tokenspeed_mla_dcp_matches_unsharded_decode(cp_interleave_size, seq_len
|
||||
global_tokens = (torch.randn(seq_len, head_size, device=device) * 0.1).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
workspace = tokenspeed_mla_module._get_workspace(device, num_heads, kv_lora_rank)
|
||||
workspace = tokenspeed_mla_module._get_workspace(
|
||||
device, num_heads, kv_lora_rank, 1, query.shape[1]
|
||||
)
|
||||
global_cache, global_block_table = make_paged_cache(global_tokens)
|
||||
global_seq_len = torch.tensor([seq_len], dtype=torch.int32, device=device)
|
||||
reference, _ = tokenspeed_mla_decode(
|
||||
@@ -1301,7 +1316,8 @@ def test_tokenspeed_mla_dcp_matches_unsharded_decode(cp_interleave_size, seq_len
|
||||
impl.scale = head_size**-0.5
|
||||
impl.softmax_scale = None
|
||||
impl.output_scale = None
|
||||
impl._workspace_buffer = workspace
|
||||
impl._min_split_kv = 1
|
||||
impl._max_decode_tokens = query.shape[1]
|
||||
metadata = SimpleNamespace(
|
||||
num_decodes=1,
|
||||
num_decode_tokens=query.shape[1],
|
||||
@@ -1649,6 +1665,10 @@ def _run_backend_correctness(
|
||||
block_size=default_block_size,
|
||||
hf_config_override=hf_config_override,
|
||||
)
|
||||
if hf_config_override and current_platform.is_xpu():
|
||||
vllm_config.model_config.model_arch_config.total_num_attention_heads = (
|
||||
hf_config_override["num_attention_heads"]
|
||||
)
|
||||
vllm_config.cache_config.cache_dtype = kv_cache_dtype
|
||||
|
||||
# For spec decode tests, add a speculative_config to set the reorder_batch_threshold
|
||||
|
||||
@@ -2,14 +2,26 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for MLA prefill backend registry."""
|
||||
|
||||
import gc
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backends.mla.prefill.base import MLAPrefillBackend
|
||||
from vllm.v1.attention.backends.mla.prefill.flashinfer import FlashInferPrefillBackend
|
||||
from vllm.v1.attention.backends.mla.prefill.registry import (
|
||||
MLAPrefillBackendEnum,
|
||||
register_mla_prefill_backend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.prefill.trtllm_ragged import (
|
||||
TrtllmRaggedPrefillBackend,
|
||||
)
|
||||
from vllm.v1.worker.workspace import (
|
||||
current_workspace_manager,
|
||||
init_workspace_manager,
|
||||
reset_workspace_manager,
|
||||
)
|
||||
|
||||
|
||||
class CustomMLAPrefillBackend(MLAPrefillBackend):
|
||||
@@ -50,6 +62,38 @@ def test_prefill_backend_clone_has_isolated_metadata():
|
||||
assert clone._prefill_metadata is not backend._prefill_metadata
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_cls", [TrtllmRaggedPrefillBackend, FlashInferPrefillBackend]
|
||||
)
|
||||
def test_prefill_backend_does_not_pin_grown_workspace(monkeypatch, backend_cls):
|
||||
"""Reserving workspace at init must not keep a view of it: when another user
|
||||
grows the shared workspace, the old buffer must be freed."""
|
||||
monkeypatch.setenv("VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE", "4096")
|
||||
init_workspace_manager(torch.device("cpu"))
|
||||
try:
|
||||
manager = current_workspace_manager()
|
||||
backend = backend_cls(
|
||||
num_heads=4,
|
||||
scale=0.5,
|
||||
kv_lora_rank=8,
|
||||
qk_nope_head_dim=16,
|
||||
qk_rope_head_dim=8,
|
||||
v_head_dim=32,
|
||||
vllm_config=MagicMock(),
|
||||
)
|
||||
(view,) = manager.get_simultaneous(((1,), torch.uint8))
|
||||
old_storage = view.untyped_storage()._weak_ref()
|
||||
del view
|
||||
|
||||
manager.get_simultaneous(((1 << 20,), torch.uint8))
|
||||
gc.collect()
|
||||
|
||||
assert torch.UntypedStorage._expired(old_storage)
|
||||
del backend
|
||||
finally:
|
||||
reset_workspace_manager()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def cleanup_overrides():
|
||||
"""Clear any overrides after each test."""
|
||||
|
||||
@@ -805,7 +805,6 @@ def test_composite_shared_cache_across_image_and_causal_steps(
|
||||
)
|
||||
layer.mm_prefix_clamp_sliding_window = True
|
||||
builder = backend.get_builder_cls()(spec, ["composite_test"], cfg, DEVICE)
|
||||
builder.set_kernel_block_size(block_size)
|
||||
# Match production, where unwritten page tails are zero-initialized.
|
||||
cache = torch.zeros(
|
||||
num_blocks,
|
||||
|
||||
@@ -142,6 +142,13 @@ def _build(monkeypatch, route, gathered_heads, segmented_supported=True):
|
||||
|
||||
def init_common_builder(self, *args, **kwargs):
|
||||
self.num_heads = gathered_heads // DCP
|
||||
self.q_data_type = torch.bfloat16
|
||||
self.mla_dims = SimpleNamespace(
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
)
|
||||
self.dcp_world_size = DCP
|
||||
self.reorder_batch_threshold = 5
|
||||
|
||||
@@ -154,6 +161,11 @@ def _build(monkeypatch, route, gathered_heads, segmented_supported=True):
|
||||
lambda *args: segmented_supported,
|
||||
)
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_fp8_mla_prefill_supported", lambda: False)
|
||||
# These cases pick between cprr and segmented. On a gfx942 runner the real
|
||||
# arch would add the Triton verify route, which this stub cannot build.
|
||||
import vllm.platforms.rocm as rocm
|
||||
|
||||
monkeypatch.setattr(rocm, "on_gfx942", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "get_dcp_group", lambda: SimpleNamespace(rank_in_group=0)
|
||||
)
|
||||
@@ -257,25 +269,46 @@ def test_min_cprr_qlen_is_above_two():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"supports,causal,qlen,asm,expected",
|
||||
"segmented,triton,causal,qlen,asm,expected",
|
||||
[
|
||||
(True, True, 1, True, Route.PLAIN),
|
||||
(True, True, 2, True, Route.SEGMENTED),
|
||||
(True, True, 3, True, Route.CPRR),
|
||||
(True, True, 4, True, Route.CPRR),
|
||||
(True, True, 4, False, Route.SEGMENTED),
|
||||
(True, False, 2, True, Route.PLAIN),
|
||||
(True, False, 4, False, Route.PLAIN),
|
||||
(False, True, 3, True, Route.CPRR),
|
||||
(True, False, True, 1, True, Route.PLAIN),
|
||||
(True, False, True, 2, True, Route.SEGMENTED),
|
||||
(True, False, True, 3, True, Route.CPRR),
|
||||
(True, False, True, 4, True, Route.CPRR),
|
||||
(True, False, True, 4, False, Route.SEGMENTED),
|
||||
(True, False, False, 2, True, Route.PLAIN),
|
||||
(True, False, False, 4, False, Route.PLAIN),
|
||||
(False, False, True, 3, True, Route.CPRR),
|
||||
# gfx942: no cprr and no segmented, so causal verify goes to Triton.
|
||||
(False, True, True, 1, False, Route.PLAIN),
|
||||
(False, True, True, 2, False, Route.TRITON),
|
||||
(False, True, True, 4, False, Route.TRITON),
|
||||
(False, True, False, 4, False, Route.PLAIN),
|
||||
# cprr still wins at or above its qlen floor.
|
||||
(False, True, True, 4, True, Route.CPRR),
|
||||
(False, True, True, 2, True, Route.TRITON),
|
||||
],
|
||||
)
|
||||
def test_dcp_decode_route(supports, causal, qlen, asm, expected):
|
||||
assert _select_route(supports, causal, qlen, asm) is expected
|
||||
def test_dcp_decode_route(segmented, triton, causal, qlen, asm, expected):
|
||||
route = _select_route(
|
||||
supports_segmented=segmented,
|
||||
supports_triton=triton,
|
||||
causal=causal,
|
||||
max_qo_len=qlen,
|
||||
asm_selected=asm,
|
||||
)
|
||||
assert route is expected
|
||||
|
||||
|
||||
def test_causal_multi_token_batch_without_a_valid_route_fails():
|
||||
with pytest.raises(RuntimeError, match="requires either segmented MLA"):
|
||||
_select_route(False, True, 2, True)
|
||||
_select_route(
|
||||
supports_segmented=False,
|
||||
supports_triton=False,
|
||||
causal=True,
|
||||
max_qo_len=2,
|
||||
asm_selected=True,
|
||||
)
|
||||
|
||||
|
||||
# Kernel numerics live in test_rocm_aiter_mla_dcp_cprr_numerics.py, which runs
|
||||
|
||||
@@ -536,5 +536,12 @@ def test_cprr_kernel_is_selected_for_the_k3_shape():
|
||||
is True
|
||||
)
|
||||
assert QLEN >= m._MIN_CPRR_QLEN
|
||||
assert m._select_dcp_decode_route(True, True, QLEN, True) is m._DCPDecodeRoute.CPRR
|
||||
route = m._select_dcp_decode_route(
|
||||
supports_segmented=True,
|
||||
supports_triton=False,
|
||||
causal=True,
|
||||
max_qo_len=QLEN,
|
||||
asm_selected=True,
|
||||
)
|
||||
assert route is m._DCPDecodeRoute.CPRR
|
||||
assert m._asm_dcp_verify_heads(96) in m._NATIVE_CPRR_HEADS
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Causal target verify under DCP on gfx942 (MI300/MI325).
|
||||
|
||||
Reproduces the attention shape of
|
||||
|
||||
vllm serve moonshotai/Kimi-K3 --tensor-parallel-size 8 \
|
||||
--decode-context-parallel-size 8 \
|
||||
--speculative-config '{"method": "dspark", "num_speculative_tokens": 5}'
|
||||
|
||||
96 gathered heads, bf16 KV, qlen 6 and the 768-token blocks the hybrid KDA
|
||||
layers impose. Segmented MLA tiles those blocks at 128 tokens and needs 128 KiB
|
||||
of LDS (gfx942 has 64 KiB), so the builder must route this shape to the generic
|
||||
Triton split-KV kernel. DeepSeek-V3 with MTP and an fp8 KV cache at DCP 8 hits
|
||||
the same limit through its 128 gathered heads. Shapes segmented MLA fits stay
|
||||
on it.
|
||||
|
||||
The real metadata builder and ``forward_mqa`` run on one GPU with the DCP ranks
|
||||
simulated one after another. That is sound because this path has no
|
||||
collective: the query all-gather happens in the layer, before ``forward_mqa``.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm AITER MLA tests", allow_module_level=True)
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found # noqa: E402
|
||||
|
||||
# DeepSeek-R1 supplies the MLA dims, which Kimi-K3 and DeepSeek-V3 share; only
|
||||
# the head count is overridden.
|
||||
MODEL = "deepseek-ai/DeepSeek-R1"
|
||||
KV_LORA_RANK, ROPE_DIM, NOPE_DIM = 512, 64, 128
|
||||
HEAD_SIZE = KV_LORA_RANK + ROPE_DIM
|
||||
SCALE = (NOPE_DIM + ROPE_DIM) ** -0.5
|
||||
DCP = 8
|
||||
# Each rank holds ~800 tokens per request, so its shard spans several blocks.
|
||||
NUM_REQS, CTX = 4, 6400
|
||||
FP8_KV_SCALE = 0.05
|
||||
|
||||
|
||||
def _on_gfx942() -> bool:
|
||||
if not (is_aiter_found() and torch.accelerator.is_available()):
|
||||
return False
|
||||
from vllm.platforms.rocm import on_gfx942
|
||||
|
||||
return on_gfx942()
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(not _on_gfx942(), reason="needs ROCm + AITER on gfx942")
|
||||
|
||||
|
||||
class _FakeGroup:
|
||||
def __init__(self, world_size: int, rank: int):
|
||||
self.world_size = world_size
|
||||
self.rank_in_group = self.rank = self.local_rank = rank
|
||||
|
||||
def all_gather(self, tensor, dim=0):
|
||||
raise RuntimeError("DCP verify must not issue a collective")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_dcp_groups(monkeypatch):
|
||||
"""Point every importer of get_dcp_group at a settable rank stub."""
|
||||
import vllm.distributed.parallel_state as ps
|
||||
import vllm.model_executor.layers.attention.mla_attention as mla_attention
|
||||
import vllm.v1.attention.backends.mla.rocm_aiter_mla as backend
|
||||
|
||||
group = _FakeGroup(DCP, 0)
|
||||
for module in (ps, mla_attention, backend):
|
||||
monkeypatch.setattr(module, "get_dcp_group", lambda: group)
|
||||
monkeypatch.setattr(ps, "get_tp_group", lambda: _FakeGroup(1, 0))
|
||||
return group
|
||||
|
||||
|
||||
def _vllm_config(heads_per_rank: int, qlen: int, fp8_kv: bool):
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
|
||||
config = create_vllm_config(
|
||||
model_name=MODEL,
|
||||
max_model_len=CTX + qlen,
|
||||
max_num_seqs=NUM_REQS,
|
||||
max_num_batched_tokens=4096,
|
||||
)
|
||||
config.model_config.model_arch_config.total_num_attention_heads = heads_per_rank
|
||||
# The reorder threshold reads only these two fields; without them a
|
||||
# multi-token row is classified as prefill and never reaches decode.
|
||||
config.speculative_config = SimpleNamespace(
|
||||
num_speculative_tokens=qlen - 1, parallel_drafting=False
|
||||
)
|
||||
config.parallel_config.decode_context_parallel_size = DCP
|
||||
config.parallel_config.cp_kv_cache_interleave_size = 1
|
||||
if fp8_kv:
|
||||
config.cache_config.cache_dtype = "fp8"
|
||||
return config
|
||||
|
||||
|
||||
def _ctx_layer():
|
||||
"""The registered-layer fields MLACommonMetadataBuilder.__init__ reads."""
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
|
||||
dcp_manager = object.__new__(MLADCPManager)
|
||||
dcp_manager.init_kv_gather = lambda *args, **kwargs: None
|
||||
# Only chunked-context prefill uses the cloned backend.
|
||||
prefill_backend = SimpleNamespace()
|
||||
prefill_backend.clone = lambda: prefill_backend
|
||||
return SimpleNamespace(
|
||||
non_causal_multi_token_decode=False,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=NOPE_DIM,
|
||||
qk_rope_head_dim=ROPE_DIM,
|
||||
v_head_dim=NOPE_DIM,
|
||||
dcp_manager=dcp_manager,
|
||||
prefill_backend=prefill_backend,
|
||||
)
|
||||
|
||||
|
||||
def _impl(dcp_rank: int, heads_per_rank: int, fp8_kv: bool):
|
||||
"""forward_mqa needs these attributes, not the weight-loading __init__."""
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.num_heads = heads_per_rank
|
||||
impl.kv_lora_rank = KV_LORA_RANK
|
||||
impl.qk_rope_head_dim = ROPE_DIM
|
||||
impl.dcp_world_size = DCP
|
||||
impl.dcp_rank = dcp_rank
|
||||
impl.pcp_world_size = 1
|
||||
impl.kv_cache_dtype = "fp8" if fp8_kv else "auto"
|
||||
impl.scale = SCALE
|
||||
impl._sm_count = current_platform.num_compute_units()
|
||||
return impl
|
||||
|
||||
|
||||
def _run_rank(config, spec, layer_name, group, rank, q, kv_cache_src, k_scale):
|
||||
"""Shard the KV round-robin onto ``rank`` and run its verify step."""
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAMetadataBuilder
|
||||
|
||||
device = q.device
|
||||
qlen, heads = q.shape[1], q.shape[2] // DCP
|
||||
seq_len = CTX + qlen
|
||||
positions = torch.arange(rank, seq_len, DCP, device=device)
|
||||
local_len = positions.numel()
|
||||
# Production block tables span max_model_len, so every rank's table is at
|
||||
# least as wide as the largest shard needs.
|
||||
blocks_per_req = cdiv(cdiv(seq_len, DCP), spec.block_size)
|
||||
kv_cache = torch.zeros(
|
||||
NUM_REQS * blocks_per_req,
|
||||
spec.block_size,
|
||||
HEAD_SIZE,
|
||||
dtype=kv_cache_src.dtype,
|
||||
device=device,
|
||||
)
|
||||
kv_cache.view(NUM_REQS, -1, HEAD_SIZE)[:, :local_len] = kv_cache_src[:, positions]
|
||||
block_table = torch.arange(
|
||||
NUM_REQS * blocks_per_req, dtype=torch.int32, device=device
|
||||
).view(NUM_REQS, blocks_per_req)
|
||||
|
||||
query_start_loc = torch.arange(
|
||||
0, NUM_REQS * qlen + 1, qlen, dtype=torch.int32, device=device
|
||||
)
|
||||
local_lens = torch.full((NUM_REQS,), local_len, dtype=torch.int32)
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc.cpu(),
|
||||
seq_lens=torch.full((NUM_REQS,), seq_len, dtype=torch.int32, device=device),
|
||||
num_reqs=NUM_REQS,
|
||||
num_actual_tokens=NUM_REQS * qlen,
|
||||
max_query_len=qlen,
|
||||
max_seq_len=seq_len,
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=torch.arange(NUM_REQS * qlen, dtype=torch.int64, device=device),
|
||||
causal=True,
|
||||
dcp_local_seq_lens=local_lens.to(device),
|
||||
dcp_local_seq_lens_cpu_upper_bound=local_lens,
|
||||
)
|
||||
|
||||
group.rank_in_group = group.rank = group.local_rank = rank
|
||||
builder = AiterMLAMetadataBuilder(spec, [layer_name], config, device)
|
||||
metadata = builder.build(0, common)
|
||||
layer = SimpleNamespace(
|
||||
_q_scale=torch.tensor(1.0, device=device),
|
||||
_k_scale=torch.tensor(k_scale, device=device),
|
||||
)
|
||||
impl = _impl(rank, heads, kv_cache_src.dtype != torch.bfloat16)
|
||||
output, lse = impl.forward_mqa(q.flatten(0, 1), kv_cache, metadata, layer)
|
||||
return metadata.decode.dcp_route.name, output, lse
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"heads_per_rank, qlen, block_size, fp8_kv, route",
|
||||
[
|
||||
(12, 6, 768, False, "TRITON"),
|
||||
(12, 6, 64, False, "SEGMENTED"),
|
||||
(16, 2, 16, True, "TRITON"),
|
||||
],
|
||||
ids=["kimi-k3-hybrid-kda-blocks", "segmented-fits", "deepseek-v3-mtp-fp8-kv"],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_dcp_verify_matches_attention(
|
||||
fake_dcp_groups, heads_per_rank, qlen, block_size, fp8_kv, route
|
||||
):
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.workspace import (
|
||||
init_workspace_manager,
|
||||
is_workspace_manager_initialized,
|
||||
)
|
||||
|
||||
device = torch.device("cuda", torch.accelerator.current_device_index())
|
||||
if not is_workspace_manager_initialized():
|
||||
init_workspace_manager(device)
|
||||
torch.manual_seed(0)
|
||||
seq_len = CTX + qlen
|
||||
q = torch.randn(
|
||||
NUM_REQS,
|
||||
qlen,
|
||||
heads_per_rank * DCP,
|
||||
HEAD_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
kv = torch.randn(NUM_REQS, seq_len, HEAD_SIZE, device=device).clamp_(-4, 4)
|
||||
if fp8_kv:
|
||||
k_scale = FP8_KV_SCALE
|
||||
kv_cache_src = (kv / k_scale).to(current_platform.fp8_dtype())
|
||||
else:
|
||||
k_scale = 1.0
|
||||
kv_cache_src = kv.to(torch.bfloat16)
|
||||
kv_ref = kv_cache_src.float() * k_scale
|
||||
|
||||
config = _vllm_config(heads_per_rank, qlen, fp8_kv)
|
||||
layer_name = "model.layers.0.self_attn.attn"
|
||||
with set_current_vllm_config(config):
|
||||
config.compilation_config.static_forward_context[layer_name] = _ctx_layer()
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=HEAD_SIZE,
|
||||
dtype=kv_cache_src.dtype,
|
||||
cache_dtype_str="fp8" if fp8_kv else None,
|
||||
non_causal_multi_token_decode=False,
|
||||
)
|
||||
shards = [
|
||||
_run_rank(
|
||||
config,
|
||||
spec,
|
||||
layer_name,
|
||||
fake_dcp_groups,
|
||||
rank,
|
||||
q,
|
||||
kv_cache_src,
|
||||
k_scale,
|
||||
)
|
||||
for rank in range(DCP)
|
||||
]
|
||||
|
||||
assert {shard_route for shard_route, _, _ in shards} == {route}
|
||||
lses = torch.stack([lse.float() for _, _, lse in shards])
|
||||
weights = torch.exp(lses - lses.max(dim=0).values)
|
||||
merged = (
|
||||
weights.unsqueeze(-1) * torch.stack([out.float() for _, out, _ in shards])
|
||||
).sum(dim=0) / weights.sum(dim=0).unsqueeze(-1)
|
||||
|
||||
# Query token t of a request sees the context plus verify tokens 0..t.
|
||||
scores = torch.einsum("rthd,rld->rthl", q.float(), kv_ref) * SCALE
|
||||
visible = torch.arange(seq_len, device=device) <= CTX + torch.arange(
|
||||
qlen, device=device
|
||||
).view(-1, 1)
|
||||
scores.masked_fill_(~visible[None, :, None, :], float("-inf"))
|
||||
reference = torch.einsum(
|
||||
"rthl,rld->rthd", scores.softmax(dim=-1), kv_ref[..., :KV_LORA_RANK]
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
merged.view_as(reference), reference, rtol=2e-2, atol=2e-2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"fp8_kv, num_heads, block_size",
|
||||
[
|
||||
(False, 96, 768),
|
||||
(False, 128, 64),
|
||||
(True, 64, 128),
|
||||
(True, 128, 16),
|
||||
],
|
||||
)
|
||||
def test_segmented_lds_gate_matches_the_kernel(fp8_kv, num_heads, block_size):
|
||||
"""The gate must flag exactly the shapes AITER segmented MLA cannot compile."""
|
||||
from triton.runtime.errors import OutOfResources
|
||||
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
_get_segmented_mla_decode,
|
||||
_segmented_mla_fits_lds,
|
||||
_segmented_mla_page_size,
|
||||
)
|
||||
|
||||
page = _segmented_mla_page_size(block_size)
|
||||
kv_dtype = current_platform.fp8_dtype() if fp8_kv else torch.bfloat16
|
||||
q = torch.zeros(1, num_heads, HEAD_SIZE, dtype=torch.bfloat16, device="cuda")
|
||||
kv = torch.zeros(1, page, 1, HEAD_SIZE, dtype=kv_dtype, device="cuda")
|
||||
one_row = torch.tensor([0, 1], dtype=torch.int32, device="cuda")
|
||||
try:
|
||||
_get_segmented_mla_decode()(
|
||||
q,
|
||||
kv,
|
||||
None,
|
||||
one_row,
|
||||
torch.tensor([page], dtype=torch.int32, device="cuda"),
|
||||
page,
|
||||
torch.zeros(1, 1, dtype=torch.int32, device="cuda"),
|
||||
SCALE,
|
||||
KV_LORA_RANK,
|
||||
ROPE_DIM,
|
||||
causal=True,
|
||||
q_descale=None,
|
||||
kv_descale=torch.tensor(1.0, device="cuda"),
|
||||
skip_reduce=True,
|
||||
)
|
||||
compiles = True
|
||||
except OutOfResources:
|
||||
compiles = False
|
||||
assert _segmented_mla_fits_lds(num_heads, block_size, fp8_kv) == compiles
|
||||
@@ -20,6 +20,10 @@ from vllm.v1.attention.backends.mla.rocm_aiter_mla import ( # noqa: E402
|
||||
AiterMLAImpl,
|
||||
AiterMLAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.ops.dcp import ( # noqa: E402
|
||||
CPTritonContext,
|
||||
correct_attn_out,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_merge import ( # noqa: E402
|
||||
merge_mla_segments_triton,
|
||||
)
|
||||
@@ -121,8 +125,11 @@ def _builder(
|
||||
# segmented route is available, query length decides per batch. CPRR
|
||||
# preferred still keeps this True so qlen < _MIN_CPRR_QLEN can fall back.
|
||||
_supports_segmented_dcp_verify=supports_segmented_dcp_verify,
|
||||
_use_triton_dcp_verify=False,
|
||||
# Derived once in the real constructor, so derive it once here too.
|
||||
_segmented_page_size=rocm_aiter_mla._segmented_mla_page_size(kernel_block_size),
|
||||
_dcp_verify_page_size=rocm_aiter_mla._segmented_mla_page_size(
|
||||
kernel_block_size
|
||||
),
|
||||
_dcp_verify_buffers=None,
|
||||
_graph_seq_lens=None,
|
||||
_kv_cache_dtype_str=kv_cache_dtype,
|
||||
@@ -630,6 +637,99 @@ def test_segmented_verify_reduce_returns_natural_lse_and_masks_empty_rows():
|
||||
assert lse[1].item() == float("-inf")
|
||||
|
||||
|
||||
def test_triton_verify_empty_local_shard_merges_through_dcp_combine():
|
||||
"""Empty rows need no masking: -inf LSE gives them zero combine weight.
|
||||
|
||||
Row 0 has KV on both ranks, row 1 only on rank 0, row 2 on neither (graph
|
||||
padding). The raw rows come back NaN; the production AG+RS combine must
|
||||
still produce full-context attention and zeros for the padded row.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_ranks, num_rows, num_heads = 2, 3, 16
|
||||
kv_lora_rank, head_dim, block_size = 512, 576, 16
|
||||
scale = head_dim**-0.5
|
||||
local_lens = [[5, 4, 0], [5, 0, 0]]
|
||||
|
||||
q = torch.randn(num_rows, num_heads, head_dim, dtype=torch.bfloat16, device="cuda")
|
||||
kv_caches = [
|
||||
torch.randn(1, block_size, head_dim, dtype=torch.bfloat16, device="cuda")
|
||||
for _ in range(num_ranks)
|
||||
]
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.scale = scale
|
||||
impl.kv_lora_rank = kv_lora_rank
|
||||
impl._sm_count = current_platform.num_compute_units()
|
||||
layer = SimpleNamespace(_k_scale=torch.tensor(1.0, device="cuda"))
|
||||
outputs, lses = [], []
|
||||
for rank in range(num_ranks):
|
||||
verify = SimpleNamespace(
|
||||
block_table=torch.zeros(num_rows, 1, dtype=torch.int32, device="cuda"),
|
||||
row_lens=torch.tensor(local_lens[rank], dtype=torch.int32, device="cuda"),
|
||||
max_kv_seq_len=max(local_lens[rank]),
|
||||
)
|
||||
output, lse = impl._forward_triton_dcp_verify(
|
||||
q, verify, kv_caches[rank], layer, torch.bfloat16
|
||||
)
|
||||
outputs.append(output)
|
||||
lses.append(lse)
|
||||
|
||||
assert torch.isneginf(lses[1][1]).all()
|
||||
assert torch.isneginf(lses[0][2]).all() and torch.isneginf(lses[1][2]).all()
|
||||
|
||||
all_lses = torch.stack(lses)
|
||||
merged = sum(
|
||||
correct_attn_out(outputs[rank], all_lses, rank, CPTritonContext())[0].float()
|
||||
for rank in range(num_ranks)
|
||||
)
|
||||
|
||||
reference = torch.zeros(num_rows, num_heads, kv_lora_rank, device="cuda")
|
||||
for row in range(num_rows):
|
||||
keys = torch.cat(
|
||||
[kv_caches[rank][0, : local_lens[rank][row]] for rank in range(num_ranks)]
|
||||
).float()
|
||||
if keys.shape[0]:
|
||||
probs = torch.softmax(q[row].float() @ keys.T * scale, dim=-1)
|
||||
reference[row] = probs @ keys[:, :kv_lora_rank]
|
||||
|
||||
assert torch.isfinite(merged).all()
|
||||
torch.testing.assert_close(merged, reference, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
def test_batch_aware_kv_splits_fit_an_int32_safe_reservation(monkeypatch):
|
||||
# MI325X DCP8 verify at 128K context: 128 gathered heads, 16K local tokens.
|
||||
sm_count, heads, local_seq_len, kv_lora_rank = 304, 128, 16384, 512
|
||||
max_rows = 4096
|
||||
reserved = []
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla, "is_workspace_manager_initialized", lambda: True
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"current_workspace_manager",
|
||||
lambda: SimpleNamespace(get_simultaneous=lambda *specs: reserved.extend(specs)),
|
||||
)
|
||||
rocm_aiter_mla._reserve_triton_dcp_verify_workspace(
|
||||
max_rows, heads, local_seq_len, kv_lora_rank, sm_count
|
||||
)
|
||||
((shape, _),) = reserved
|
||||
# The kernel's attn_logits offsets are int32; seq-only splits overflow them.
|
||||
seq_only = rocm_aiter_mla._compute_num_kv_splits(local_seq_len, sm_count)
|
||||
assert max_rows * heads * seq_only * (kv_lora_rank + 1) >= 2**31
|
||||
assert math.prod(shape) < 2**31
|
||||
|
||||
counts = [
|
||||
rocm_aiter_mla._triton_dcp_verify_num_kv_splits(
|
||||
local_seq_len, rows, heads, sm_count
|
||||
)
|
||||
for rows in range(1, max_rows + 1)
|
||||
]
|
||||
# Small batches keep the seq-only split count.
|
||||
assert counts[:4] == [seq_only] * 4
|
||||
assert counts == sorted(counts, reverse=True)
|
||||
for rows, splits in enumerate(counts, start=1):
|
||||
assert rows * heads * splits * (kv_lora_rank + 1) <= math.prod(shape)
|
||||
|
||||
|
||||
def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
|
||||
"""Exercise the target qlen>1 path and merge two rank-local results."""
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_segmented_mla_decode_supported", lambda: True)
|
||||
@@ -740,26 +840,34 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
|
||||
torch.testing.assert_close(output, reference, rtol=3e-2, atol=3e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_heads", [8, 16, 24, 32, 64, 128])
|
||||
@pytest.mark.parametrize(
|
||||
"spec_method, parallel_drafting",
|
||||
"num_heads,spec_method,parallel_drafting,group_cache_dtype,needs_row_map",
|
||||
[
|
||||
("deepseek_mtp", False),
|
||||
# A drafter that is not one of the historically recognized MTP methods,
|
||||
# and a parallel one, so the threshold is 1 + 2 * num_spec rather than
|
||||
# 1 + num_spec. Sizing the metadata off a method name instead leaves
|
||||
# these at qlen=1 while the router still admits the full range.
|
||||
("custom", True),
|
||||
("eagle", False),
|
||||
*[
|
||||
(num_heads, method, parallel, None, True)
|
||||
for num_heads in [8, 16, 24, 32, 64, 128]
|
||||
for method, parallel in [
|
||||
("deepseek_mtp", False),
|
||||
# Parallel drafting needs qlen=1+2*num_spec, regardless of name.
|
||||
("custom", True),
|
||||
("eagle", False),
|
||||
]
|
||||
],
|
||||
pytest.param(16, "deepseek_mtp", False, "auto", False, id="bf16-group"),
|
||||
pytest.param(16, "deepseek_mtp", False, "fp8_e5m2", False, id="e5m2-group"),
|
||||
],
|
||||
)
|
||||
def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
monkeypatch, num_heads, spec_method, parallel_drafting
|
||||
def test_builder_init_sizes_decode_metadata_and_filters_dcp_maps(
|
||||
monkeypatch,
|
||||
num_heads,
|
||||
spec_method,
|
||||
parallel_drafting,
|
||||
group_cache_dtype,
|
||||
needs_row_map,
|
||||
):
|
||||
"""Aiter init sizes the metadata for every query length decode can be handed.
|
||||
"""Size decode metadata for padded heads/qlen; skip unsupported DCP maps.
|
||||
|
||||
Sweeping num_heads asserts metadata is sized for the padded decode shape,
|
||||
covering Kimi-K3 TP4's 24 -> 32 head path and native fp8 nhead=32 folding.
|
||||
The group's raw cache format must override the global FP8 configuration.
|
||||
"""
|
||||
dtypes = SimpleNamespace(fp8="fp8", fp16="fp16", bf16="bf16")
|
||||
info_calls = []
|
||||
@@ -789,6 +897,13 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
|
||||
def init_common_builder(self, *args, **kwargs):
|
||||
self.num_heads = num_heads
|
||||
self.q_data_type = config.model_config.dtype
|
||||
self.mla_dims = SimpleNamespace(
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
)
|
||||
self.dcp_world_size = 1
|
||||
self.cp_kv_cache_interleave_size = 1
|
||||
# Mirror what _init_reorder_batch_threshold would have left behind: the
|
||||
@@ -813,6 +928,7 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
init_common_builder,
|
||||
)
|
||||
monkeypatch.setattr(rocm_aiter_mla, "_fp8_mla_prefill_supported", lambda: False)
|
||||
monkeypatch.setattr(rocm_aiter_mla, "is_aiter_found_and_supported", lambda: True)
|
||||
|
||||
config = SimpleNamespace(
|
||||
speculative_config=SimpleNamespace(
|
||||
@@ -840,7 +956,9 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
),
|
||||
)
|
||||
builder = AiterMLAMetadataBuilder(
|
||||
kv_cache_spec=SimpleNamespace(block_size=1, dtype=torch.bfloat16),
|
||||
kv_cache_spec=SimpleNamespace(
|
||||
block_size=1, dtype=torch.bfloat16, cache_dtype_str=group_cache_dtype
|
||||
),
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=config,
|
||||
device=torch.device("cpu"),
|
||||
@@ -860,6 +978,35 @@ def test_mtp_builder_init_sizes_native_fp8_metadata(
|
||||
assert builder._mla_q_dtype == dtypes.fp8
|
||||
assert builder._mla_kv_dtype == dtypes.fp8
|
||||
|
||||
builder.dcp_world_size = 2
|
||||
chunk = object()
|
||||
rows = torch.tensor([0], dtype=torch.int64)
|
||||
build_rows = mock.Mock(return_value=rows)
|
||||
monkeypatch.setattr(rocm_aiter_mla, "context_row_indices", build_rows)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla,
|
||||
"_supports_fused_dcp_prefill_config",
|
||||
mock.Mock(side_effect=AssertionError("Support must be cached at init")),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_mla.MLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args: SimpleNamespace(
|
||||
prefill=SimpleNamespace(
|
||||
chunked_context=SimpleNamespace(chunks=[chunk]),
|
||||
block_table=torch.empty(0),
|
||||
),
|
||||
decode=None,
|
||||
dcp_context_row_indices=None,
|
||||
),
|
||||
)
|
||||
for _ in range(2):
|
||||
metadata = builder.build(0, SimpleNamespace(causal=True))
|
||||
assert metadata.dcp_context_row_indices == ([rows] if needs_row_map else None)
|
||||
assert build_rows.call_args_list == (
|
||||
[mock.call(chunk, torch.device("cpu"))] * 2 if needs_row_map else []
|
||||
)
|
||||
|
||||
|
||||
def test_mtp_decode_qlen4_keeps_uniform_rows_with_metadata(monkeypatch):
|
||||
get_mla_metadata_v1 = mock.MagicMock()
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm() or not is_aiter_found_and_supported():
|
||||
pytest.skip(
|
||||
"ROCm AITER sparse indexer test requires ROCm with AITER.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as sparse_mod
|
||||
|
||||
|
||||
def test_rocm_fp8_mqa_logits_disables_clean_logits(monkeypatch):
|
||||
captured = {}
|
||||
expected = torch.empty((1, 1))
|
||||
|
||||
def fake_fp8_mqa_logits(*args, clean_logits):
|
||||
captured["clean_logits"] = clean_logits
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr(sparse_mod, "_ON_GFX942", False)
|
||||
monkeypatch.setattr(rocm_aiter_ops, "is_enabled", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod,
|
||||
"mqa_logits_module",
|
||||
lambda: SimpleNamespace(fp8_mqa_logits=fake_fp8_mqa_logits),
|
||||
)
|
||||
|
||||
q = torch.empty((1, 1, 1))
|
||||
k_fp8 = torch.empty((1, 1))
|
||||
scale = torch.empty((1,))
|
||||
weights = torch.empty((1, 1))
|
||||
cu_seqlen_ks = torch.zeros((1,), dtype=torch.int32)
|
||||
cu_seqlen_ke = torch.ones((1,), dtype=torch.int32)
|
||||
|
||||
result = sparse_mod.rocm_fp8_mqa_logits(
|
||||
q,
|
||||
(k_fp8, scale),
|
||||
weights,
|
||||
cu_seqlen_ks,
|
||||
cu_seqlen_ke,
|
||||
)
|
||||
|
||||
assert result is expected
|
||||
assert captured["clean_logits"] is False
|
||||
@@ -96,6 +96,8 @@ def test_forward_mqa_prepares_triton_sparse_mla_inputs(
|
||||
impl.sinks = torch.randn(num_heads, generator=gen).to(device)
|
||||
impl.kv_cache_dtype = kv_cache_dtype
|
||||
impl.topk_indices_buffer = topk_indices.to(device)
|
||||
impl.owns_indexer = True
|
||||
impl.index_epoch = 0
|
||||
impl.use_aiter_sparse_mla = True
|
||||
layer = SimpleNamespace(
|
||||
_q_scale=torch.tensor([0.02], device=device),
|
||||
|
||||
@@ -61,6 +61,31 @@ def test_aiter_unified_attention_uses_dedicated_metadata_builder():
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_backends_do_not_support_mm_prefix():
|
||||
"""Regression test: ROCm backends must not claim mm_prefix support.
|
||||
|
||||
If they do, prefix-LM multimodal models that attend vision tokens
|
||||
bidirectionally (e.g. Gemma3 4B) will use bidirectional attention
|
||||
via the ROCm backend instead of falling back to Triton, producing
|
||||
incorrect results. Remove this test only if support is properly added.
|
||||
"""
|
||||
from vllm.v1.attention.backends.rocm_aiter_unified_attn import (
|
||||
RocmAiterUnifiedAttentionBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend
|
||||
|
||||
assert not RocmAttentionBackend.supports_mm_prefix(), (
|
||||
"RocmAttentionBackend does not support mm_prefix — "
|
||||
"defer to Triton unified attention instead for now"
|
||||
"See https://github.com/vllm-project/vllm/issues/48172"
|
||||
)
|
||||
assert not RocmAiterUnifiedAttentionBackend.supports_mm_prefix(), (
|
||||
"RocmAiterUnifiedAttentionBackend does not support mm_prefix — "
|
||||
"defer to Triton unified attention instead for now"
|
||||
"See https://github.com/ROCm/aiter/issues/4168"
|
||||
)
|
||||
|
||||
|
||||
def test_segmented_attention_uses_dedicated_backend_components():
|
||||
from vllm.v1.attention.backends.rocm_segmented_attn import (
|
||||
RocmSegmentedAttentionBackend,
|
||||
@@ -181,7 +206,9 @@ def test_segmented_triton_cache_survives_standalone_compilation(
|
||||
RocmSegmentedAttentionImpl(8, 128, 128**-0.5, 2, None, None, "auto")
|
||||
assert os.environ["TRITON_CACHE_DIR"] == str(expected)
|
||||
assert expected.is_dir()
|
||||
InductorStandaloneAdaptor("binary").initialize_cache(str(tmp_path / model))
|
||||
InductorStandaloneAdaptor(
|
||||
use_aot_compile=True, save_format="binary"
|
||||
).initialize_cache(str(tmp_path / model))
|
||||
assert os.environ["TRITON_CACHE_DIR"] == str(expected)
|
||||
|
||||
|
||||
@@ -250,7 +277,6 @@ def test_segmented_attention_forward_uses_dedicated_dispatch(monkeypatch):
|
||||
output = torch.empty_like(query)
|
||||
kv_cache = torch.empty(2, 2, 16, 256, dtype=torch.bfloat16)
|
||||
metadata = SimpleNamespace(
|
||||
use_cascade=False,
|
||||
num_actual_tokens=3,
|
||||
block_table=torch.zeros(1, 1, dtype=torch.int32),
|
||||
query_start_loc=torch.tensor([0, 3], dtype=torch.int32),
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
ROCMAiterMLASparseBackend,
|
||||
_use_rocm_sparse_triton,
|
||||
fit_kpool_indices_to_aiter,
|
||||
)
|
||||
@@ -19,6 +20,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_validate_dsv4_sparse_dims,
|
||||
_validate_sparse_dims,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheLayout
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -104,6 +106,10 @@ def test_fit_kpool_indices_rejects_narrow_input():
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_sparse_mla_supports_glm_packed_layout():
|
||||
assert KVCacheLayout.BLHNC in ROCMAiterMLASparseBackend.supported_kv_cache_layouts()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"kv_cache_dtype",
|
||||
@@ -149,8 +155,86 @@ def test_rocm_sparse_triton_route(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("gfx942", "fp8", "num_prefills", "num_decodes", "prefill_max_seq_len", "route"),
|
||||
[
|
||||
pytest.param(True, False, 1, 0, 129, "triton", id="above-topk"),
|
||||
pytest.param(False, False, 1, 0, 129, "aiter", id="gfx950"),
|
||||
pytest.param(True, True, 1, 0, 129, "aiter", id="fp8"),
|
||||
pytest.param(True, False, 0, 2, 129, "aiter", id="decode-only"),
|
||||
pytest.param(True, False, 1, 1, 129, "aiter", id="mixed"),
|
||||
pytest.param(True, False, 1, 0, 128, "aiter", id="equal-topk"),
|
||||
pytest.param(True, False, 1, 0, 127, "aiter", id="below-topk"),
|
||||
pytest.param(True, False, 0, 0, 129, "aiter", id="empty"),
|
||||
pytest.param(
|
||||
True, False, 1, 0, None, "missing", id="required-prefill-metadata"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_rocm_long_bf16_prefill_dispatch(
|
||||
monkeypatch, gfx942, fp8, num_prefills, num_decodes, prefill_max_seq_len, route
|
||||
):
|
||||
"""Check the new dispatch gate using CPU tensors and intercepted kernels."""
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: gfx942)
|
||||
|
||||
class KernelSelected(Exception):
|
||||
pass
|
||||
|
||||
def triton_kernel(**kwargs):
|
||||
raise KernelSelected("triton")
|
||||
|
||||
def aiter_kernel(*args, **kwargs):
|
||||
raise KernelSelected("aiter")
|
||||
|
||||
monkeypatch.setattr(sparse_mod, "rocm_sparse_attn_prefill", triton_kernel)
|
||||
monkeypatch.setattr(sparse_mod.rocm_aiter_ops, "mla_decode_fwd", aiter_kernel)
|
||||
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
|
||||
impl.num_heads = 64
|
||||
impl.kv_lora_rank = 512
|
||||
impl.kv_cache_dtype = "fp8" if fp8 else "auto"
|
||||
impl.scale = 576**-0.5
|
||||
impl.sinks = None
|
||||
|
||||
dtype = torch.float8_e4m3fn if fp8 else torch.bfloat16
|
||||
q = torch.empty(2, 64, 576, dtype=dtype)
|
||||
kv = torch.empty(4, 1, 576, dtype=dtype)
|
||||
metadata = SimpleNamespace(
|
||||
attn_out_dtype=torch.bfloat16,
|
||||
num_prefills=num_prefills,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decodes,
|
||||
max_query_len=2,
|
||||
prefill_max_seq_len=prefill_max_seq_len,
|
||||
topk_tokens=128,
|
||||
work_meta_data=None,
|
||||
qo_indptr=torch.arange(3, dtype=torch.int32),
|
||||
paged_kv_indices=torch.empty(0, dtype=torch.int32),
|
||||
paged_kv_indptr=torch.zeros(3, dtype=torch.int32),
|
||||
paged_kv_last_page_len=torch.ones(2, dtype=torch.int32),
|
||||
)
|
||||
# Irrelevant routes must not inspect prefill-only metadata.
|
||||
if (
|
||||
not gfx942
|
||||
or fp8
|
||||
or num_decodes
|
||||
or not num_prefills
|
||||
or prefill_max_seq_len is None
|
||||
):
|
||||
del metadata.prefill_max_seq_len
|
||||
del metadata.topk_tokens
|
||||
layer = SimpleNamespace(_q_scale=1.0, _k_scale=1.0)
|
||||
error = AttributeError if route == "missing" else KernelSelected
|
||||
match = "prefill_max_seq_len" if route == "missing" else f"^{route}$"
|
||||
with pytest.raises(error, match=match):
|
||||
impl._forward_mla(layer, q, kv, metadata)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gfx942", [False, True])
|
||||
@pytest.mark.parametrize("num_heads", [8, 12])
|
||||
def test_rocm_sparse_triton_route_preserves_padded_sinks(monkeypatch, num_heads):
|
||||
def test_rocm_sparse_triton_route_preserves_padded_sinks(
|
||||
monkeypatch, num_heads, gfx942
|
||||
):
|
||||
monkeypatch.setattr("vllm.platforms.rocm.on_gfx942", lambda: gfx942)
|
||||
captured = {}
|
||||
|
||||
def fake_rocm_sparse_attn_prefill(**kwargs):
|
||||
|
||||
@@ -0,0 +1,445 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import multi_gpu_only
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_prefill import context_row_indices
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"padded,lengths,starts,total,expected",
|
||||
[
|
||||
([3, 2], [[7, 6], [2, 1]], [5, 0], 6, [0, 1, 5, 3, 4, 8]),
|
||||
([3], [[2, 9]], [4], 3, [3, 4, 5]),
|
||||
([3], [[1, 1]], [0], 3, None),
|
||||
],
|
||||
ids=["padding_and_chunk_starts", "empty_rank", "inconsistent_total"],
|
||||
)
|
||||
def test_dcp_context_indices(padded, lengths, starts, total, expected):
|
||||
chunk = SimpleNamespace(
|
||||
padded_local_seq_lens=padded,
|
||||
local_context_lens_allranks=lengths,
|
||||
local_starts=starts,
|
||||
num_local_context_tokens=sum(padded),
|
||||
num_context_tokens=total,
|
||||
)
|
||||
if expected is None:
|
||||
with pytest.raises(AssertionError):
|
||||
context_row_indices(chunk, torch.device("cpu"))
|
||||
else:
|
||||
rows = context_row_indices(chunk, torch.device("cpu"))
|
||||
assert rows.tolist() == expected
|
||||
assert rows.dtype == torch.int32
|
||||
|
||||
|
||||
def test_compressed_gather_preserves_bytes_and_workspace_partition(monkeypatch):
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_prefill as prefill
|
||||
|
||||
workspace = torch.full((12, 4), -1, dtype=torch.bfloat16)
|
||||
cache = torch.arange(8, dtype=torch.uint8).reshape(2, 4)
|
||||
chunk = SimpleNamespace(
|
||||
num_local_context_tokens=2,
|
||||
local_context_lens_allranks=[[2, 2]],
|
||||
padded_local_cu_seq_lens=torch.tensor([0, 2]),
|
||||
num_requests=1,
|
||||
starts=torch.tensor([0]),
|
||||
)
|
||||
|
||||
def local_gather(src, dst, *args):
|
||||
assert src.dtype == dst.dtype == torch.uint8
|
||||
dst.copy_(src)
|
||||
|
||||
def allgather(dst, src):
|
||||
assert dst.dtype == src.dtype == torch.uint8
|
||||
assert dst.shape == (4, 4)
|
||||
assert src.shape == (2, 4)
|
||||
dst[:2].copy_(src)
|
||||
dst[2:].copy_(src + 16)
|
||||
|
||||
monkeypatch.setattr(prefill.ops, "cp_gather_cache", local_gather)
|
||||
gathered = prefill.gather_compressed_context(
|
||||
cache, workspace, torch.empty(1, 1), chunk, allgather, torch.float8_e4m3fn
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
gathered.view(torch.uint8), torch.cat((cache, cache + 16))
|
||||
)
|
||||
# Retain the original logical capacity even though the byte view is larger.
|
||||
assert gathered.data_ptr() - workspace.data_ptr() == 4 * 4
|
||||
torch.testing.assert_close(workspace.view(torch.uint8).reshape(-1, 4)[:2], cache)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"unsupported",
|
||||
[
|
||||
"packed_cache",
|
||||
"e5m2",
|
||||
"bf16_cache",
|
||||
"quantized_weight",
|
||||
"bias",
|
||||
"no_indices",
|
||||
"query_dtype",
|
||||
"prefill_dtype",
|
||||
"weight_dtype",
|
||||
"weight_layout",
|
||||
"cache_layout",
|
||||
"dimensions",
|
||||
"no_aiter",
|
||||
],
|
||||
)
|
||||
def test_dcp_prefill_unsupported_formats_use_original_path(monkeypatch, unsupported):
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm MLA backend")
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
AiterMLAImpl,
|
||||
MLACommonImpl,
|
||||
)
|
||||
|
||||
if unsupported == "no_aiter":
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.mla.rocm_aiter_mla."
|
||||
"is_aiter_found_and_supported",
|
||||
lambda: False,
|
||||
)
|
||||
|
||||
impl = object.__new__(AiterMLAImpl)
|
||||
impl.kv_cache_dtype = {
|
||||
"packed_cache": "fp8_ds_mla",
|
||||
"e5m2": "fp8_e5m2",
|
||||
"bf16_cache": "auto",
|
||||
}.get(unsupported, "fp8")
|
||||
impl.kv_lora_rank = 512
|
||||
if unsupported == "dimensions":
|
||||
impl.kv_lora_rank = 256
|
||||
impl.qk_nope_head_dim = impl.v_head_dim = 128
|
||||
impl.qk_rope_head_dim = 64
|
||||
impl.num_heads = 1
|
||||
impl.kv_b_proj = SimpleNamespace(
|
||||
weight=torch.empty(256, 512, dtype=torch.bfloat16),
|
||||
weight_scale=torch.ones(1) if unsupported == "quantized_weight" else None,
|
||||
bias=torch.ones(256) if unsupported == "bias" else None,
|
||||
)
|
||||
if unsupported == "weight_dtype":
|
||||
impl.kv_b_proj.weight = impl.kv_b_proj.weight.half()
|
||||
elif unsupported == "weight_layout":
|
||||
impl.kv_b_proj.weight = torch.empty(512, 256, dtype=torch.bfloat16).T
|
||||
metadata = SimpleNamespace(
|
||||
prefill=SimpleNamespace(
|
||||
q_data_type=torch.float16
|
||||
if unsupported == "prefill_dtype"
|
||||
else torch.bfloat16
|
||||
),
|
||||
dcp_context_row_indices=None if unsupported == "no_indices" else [],
|
||||
)
|
||||
expected = (torch.ones(1), torch.zeros(1))
|
||||
|
||||
def fused_mla_kv_concat(kv_nope, k_pe, use_fp8_prefill):
|
||||
raise AssertionError("The fallback should receive the callback")
|
||||
|
||||
def original_path(*args):
|
||||
assert args[-1] is fused_mla_kv_concat
|
||||
return expected
|
||||
|
||||
monkeypatch.setattr(
|
||||
MLACommonImpl,
|
||||
"_context_parallel_compute_prefill_context",
|
||||
original_path,
|
||||
)
|
||||
q = torch.empty(1, 1, 192, dtype=torch.bfloat16)
|
||||
cache = torch.empty(1, 1536, 576, dtype=torch.uint8)
|
||||
if unsupported == "query_dtype":
|
||||
q = q.half()
|
||||
elif unsupported == "cache_layout":
|
||||
cache = torch.empty(1, 1536, 1152, dtype=torch.uint8)[..., ::2]
|
||||
assert (
|
||||
impl._context_parallel_compute_prefill_context(
|
||||
q,
|
||||
cache,
|
||||
metadata,
|
||||
torch.ones(1),
|
||||
8,
|
||||
fused_mla_kv_concat_fn=fused_mla_kv_concat,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm MLA backend")
|
||||
@multi_gpu_only(num_gpus=2)
|
||||
@pytest.mark.parametrize("interleave", [1, 64])
|
||||
@pytest.mark.parametrize(
|
||||
"context_lens,workspace_size",
|
||||
[([145, 37], 1024), ([545, 0, 3, 0], 256)],
|
||||
ids=["single_chunk", "continuations_and_empty_context"],
|
||||
)
|
||||
def test_dcp_prefill_matches_full_attention(
|
||||
interleave, context_lens, workspace_size, tmp_path
|
||||
):
|
||||
"""Real DCP gather, fused expansion and suffix/chunk merges match dense SDPA."""
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
if not is_aiter_found_and_supported():
|
||||
pytest.skip("AITER on CDNA 3 or newer is required")
|
||||
mp.spawn(
|
||||
_dcp_prefill_worker,
|
||||
args=(
|
||||
f"file://{tmp_path / 'dist_init'}",
|
||||
interleave,
|
||||
context_lens,
|
||||
workspace_size,
|
||||
),
|
||||
nprocs=2,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def _dcp_prefill_worker(rank, init_method, interleave, context_lens, workspace_size):
|
||||
import torch.nn.functional as F
|
||||
|
||||
from tests.v1.attention.test_mla_backends import MockMLAAttentionLayer
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.v1.attention.backends.mla.prefill.aiter_flash_attn import (
|
||||
AiterFlashAttnPrefillBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import (
|
||||
AiterMLAImpl,
|
||||
AiterMLAMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.ops.dcp import MLADCPManager
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
torch.accelerator.set_device_index(rank)
|
||||
torch.set_num_threads(1)
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
init_workspace_manager(device)
|
||||
world_size, num_heads, block_size = 2, 12, 64
|
||||
config = create_vllm_config(
|
||||
model_name="deepseek-ai/DeepSeek-V2-Lite",
|
||||
dtype=torch.bfloat16,
|
||||
tensor_parallel_size=world_size,
|
||||
max_model_len=2048,
|
||||
block_size=block_size,
|
||||
max_num_seqs=8,
|
||||
max_num_batched_tokens=2048,
|
||||
)
|
||||
config.cache_config.cache_dtype = "fp8"
|
||||
config.parallel_config.decode_context_parallel_size = world_size
|
||||
config.parallel_config.cp_kv_cache_interleave_size = interleave
|
||||
config.attention_config.use_prefill_query_quantization = False
|
||||
config.kernel_config.enable_jit_warmup = False
|
||||
with set_current_vllm_config(config):
|
||||
init_distributed_environment(
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
local_rank=rank,
|
||||
distributed_init_method=init_method,
|
||||
)
|
||||
initialize_model_parallel(
|
||||
tensor_model_parallel_size=world_size,
|
||||
decode_context_model_parallel_size=world_size,
|
||||
)
|
||||
try:
|
||||
projection = ColumnParallelLinear(
|
||||
512,
|
||||
num_heads * world_size * 256,
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
).to(device)
|
||||
torch.manual_seed(101 + rank)
|
||||
projection.weight.copy_(torch.randn_like(projection.weight) * 0.02)
|
||||
dims = dict(
|
||||
kv_lora_rank=512,
|
||||
qk_nope_head_dim=128,
|
||||
qk_rope_head_dim=64,
|
||||
v_head_dim=128,
|
||||
)
|
||||
impl = AiterMLAImpl(
|
||||
num_heads=num_heads,
|
||||
head_size=576,
|
||||
scale=192**-0.5,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype="fp8",
|
||||
logits_soft_cap=None,
|
||||
attn_type="decoder",
|
||||
kv_sharing_target_layer_name=None,
|
||||
q_lora_rank=None,
|
||||
qk_head_dim=192,
|
||||
kv_b_proj=projection,
|
||||
**dims,
|
||||
)
|
||||
layer = MockMLAAttentionLayer(
|
||||
impl=impl,
|
||||
num_heads=num_heads,
|
||||
device=device,
|
||||
kv_b_proj=projection,
|
||||
q_scale=1.0,
|
||||
k_scale=0.07,
|
||||
**dims,
|
||||
)
|
||||
layer.prefill_backend = AiterFlashAttnPrefillBackend(
|
||||
num_heads=num_heads,
|
||||
scale=impl.scale,
|
||||
vllm_config=config,
|
||||
**dims,
|
||||
)
|
||||
layer.dcp_manager = MLADCPManager(
|
||||
config,
|
||||
device,
|
||||
num_heads,
|
||||
576,
|
||||
512,
|
||||
torch.bfloat16,
|
||||
torch.bfloat16,
|
||||
None,
|
||||
True,
|
||||
False,
|
||||
)
|
||||
config.compilation_config.static_forward_context["mla"] = layer
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=current_platform.fp8_dtype(),
|
||||
cache_dtype_str="fp8",
|
||||
)
|
||||
builder = AiterMLAMetadataBuilder(spec, ["mla"], config, device)
|
||||
builder.chunked_prefill_workspace_size = workspace_size
|
||||
builder.chunked_prefill_workspace = torch.empty(
|
||||
(workspace_size + workspace_size // world_size, 576),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
layer.dcp_manager.init_kv_gather(
|
||||
builder.chunked_prefill_workspace, workspace_size
|
||||
)
|
||||
query_lens = [i + 2 for i in range(len(context_lens))]
|
||||
batch = BatchSpec(
|
||||
seq_lens=[c + q for c, q in zip(context_lens, query_lens)],
|
||||
query_lens=query_lens,
|
||||
)
|
||||
common = create_common_attn_metadata(
|
||||
batch, block_size, device, arange_block_indices=True
|
||||
)
|
||||
metadata = builder.build(0, common)
|
||||
assert metadata.prefill is not None
|
||||
chunked = metadata.prefill.chunked_context
|
||||
assert chunked is not None and metadata.dcp_context_row_indices is not None
|
||||
if 0 in context_lens:
|
||||
assert chunked.empty_token_slices
|
||||
assert any(c.is_continuation for c in chunked.chunks)
|
||||
else:
|
||||
assert len(chunked.chunks) == 1
|
||||
blocks_per_req = common.block_table_tensor.shape[1]
|
||||
cache = torch.zeros(
|
||||
(len(context_lens) * blocks_per_req, block_size, 576),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
).to(current_platform.fp8_dtype())
|
||||
queries, suffixes, references = [], [], []
|
||||
# All ranks reconstruct the same global cache before sharding it.
|
||||
torch.manual_seed(42)
|
||||
for request, (context_len, query_len) in enumerate(
|
||||
zip(context_lens, query_lens)
|
||||
):
|
||||
latent = torch.randn(
|
||||
context_len, 576, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
quantized = (latent.float() / layer._k_scale).to(cache.dtype)
|
||||
positions = torch.arange(context_len, device=device)
|
||||
shard = quantized.view(torch.uint8)[
|
||||
(positions // interleave) % world_size == rank
|
||||
]
|
||||
cache.view(len(context_lens), -1, 576).view(torch.uint8)[
|
||||
request, : len(shard)
|
||||
].copy_(shard)
|
||||
suffix = torch.randn(
|
||||
query_len, 576, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
q = (
|
||||
torch.randn(
|
||||
query_len, num_heads, 192, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
* 0.5
|
||||
)
|
||||
full_latent = torch.cat(
|
||||
(
|
||||
(quantized.float() * layer._k_scale).to(torch.bfloat16),
|
||||
suffix,
|
||||
)
|
||||
)
|
||||
kv = F.linear(full_latent[:, :512], projection.weight).view(
|
||||
-1, num_heads, 256
|
||||
)
|
||||
k_nope, v = kv.split(128, dim=-1)
|
||||
k = torch.cat(
|
||||
(k_nope, full_latent[:, None, 512:].expand(-1, num_heads, -1)),
|
||||
dim=-1,
|
||||
)
|
||||
mask = torch.arange(context_len + query_len, device=device)[
|
||||
None, :
|
||||
] <= (context_len + torch.arange(query_len, device=device)[:, None])
|
||||
references.append(
|
||||
F.scaled_dot_product_attention(
|
||||
q.transpose(0, 1).float(),
|
||||
k.transpose(0, 1).float(),
|
||||
v.transpose(0, 1).float(),
|
||||
attn_mask=mask,
|
||||
scale=impl.scale,
|
||||
).transpose(0, 1)
|
||||
)
|
||||
queries.append(q)
|
||||
suffixes.append(suffix)
|
||||
q, suffix = torch.cat(queries), torch.cat(suffixes)
|
||||
output = torch.empty(
|
||||
q.shape[0], num_heads * 128, device=device, dtype=torch.bfloat16
|
||||
)
|
||||
# Ensure this test cannot silently succeed through the fallback.
|
||||
with pytest.MonkeyPatch.context() as patch:
|
||||
|
||||
def unexpected_fallback(*args, **kwargs):
|
||||
raise AssertionError("Expected the compressed DCP path")
|
||||
|
||||
patch.setattr(
|
||||
"vllm.model_executor.layers.attention.mla_attention."
|
||||
"MLACommonImpl._context_parallel_compute_prefill_context",
|
||||
unexpected_fallback,
|
||||
)
|
||||
impl.forward_mha(
|
||||
q,
|
||||
suffix[:, :512],
|
||||
suffix[:, None, 512:],
|
||||
cache,
|
||||
metadata,
|
||||
layer._k_scale,
|
||||
output,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
output.view(-1, num_heads, 128).float(),
|
||||
torch.cat(references),
|
||||
rtol=0.03,
|
||||
atol=0.003,
|
||||
)
|
||||
finally:
|
||||
cleanup_dist_env_and_memory()
|
||||
@@ -36,6 +36,7 @@ def _chunk(block_table, token_start, token_end, row_ends):
|
||||
cu_seq_lens=torch.zeros(1, dtype=torch.int32),
|
||||
token_to_seq=torch.zeros(1, dtype=torch.int32),
|
||||
total_seq_lens=1,
|
||||
max_seqlen_k=int(ends.max()),
|
||||
token_start=token_start,
|
||||
token_end=token_end,
|
||||
num_reqs=block_table.shape[0],
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,20 +12,18 @@ from vllm.config import CacheConfig
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
|
||||
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import FlashMLASparseBackend
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend
|
||||
from vllm.v1.attention.backends.mla.sparse_utils import (
|
||||
_CONVERT_REQ_INDEX_TO_GLOBAL_INDEX_KERNEL,
|
||||
flat_kv_row_view,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
|
||||
from vllm.v1.core.kv_cache_utils import get_kv_cache_config_from_groups
|
||||
from vllm.v1.core.kv_cache_planning import get_kv_cache_config_from_groups
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, UniformTypeKVCacheSpecs
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
from vllm.v1.worker.utils import allocate_kv_cache
|
||||
from vllm.v1.worker.utils import allocate_kv_cache, customize_attention_spec
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
@@ -48,17 +46,27 @@ pytestmark = pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
"backend,cache_dtype,sm100,stride_alignment",
|
||||
[
|
||||
(FlashInferMLASparseTRTLLMBackend, "auto", True, 1152),
|
||||
(FlashInferMLASparseTRTLLMBackend, "fp8", True, 576),
|
||||
(FlashMLASparseBackend, "auto", True, 1152),
|
||||
(FlashMLASparseBackend, "fp8_ds_mla", False, None),
|
||||
(FlashMLASparseBackend, "fp8_ds_mla", True, 656),
|
||||
(FlashMLASparseBackend, "nvfp4_ds_mla", True, 352),
|
||||
(AttentionBackendEnum.FLASHINFER_MLA_SPARSE, "auto", True, 32 * 1152),
|
||||
(AttentionBackendEnum.FLASHINFER_MLA_SPARSE, "fp8", True, 32 * 576),
|
||||
pytest.param(
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE_FA4,
|
||||
"auto",
|
||||
True,
|
||||
64 * 1152,
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUDA only"
|
||||
),
|
||||
),
|
||||
(AttentionBackendEnum.FLASHMLA_SPARSE, "auto", True, 1152),
|
||||
(AttentionBackendEnum.FLASHMLA_SPARSE, "fp8_ds_mla", False, MultipleOf(64)),
|
||||
(AttentionBackendEnum.FLASHMLA_SPARSE, "fp8_ds_mla", True, 656),
|
||||
(AttentionBackendEnum.FLASHMLA_SPARSE, "nvfp4_ds_mla", True, 352),
|
||||
],
|
||||
)
|
||||
def test_allocation_and_warmup_follow_addressing_mode(
|
||||
monkeypatch, layout, backend, cache_dtype, sm100, stride_alignment, device
|
||||
):
|
||||
backend = backend.get_class()
|
||||
monkeypatch.setattr(
|
||||
current_platform, "is_device_capability_family", lambda family: sm100
|
||||
)
|
||||
@@ -79,7 +87,9 @@ def test_allocation_and_warmup_follow_addressing_mode(
|
||||
non_causal_multi_token_decode=False,
|
||||
)
|
||||
layer._uses_flat_kv_cache = MethodType(MLAAttention._uses_flat_kv_cache, layer)
|
||||
spec = MLAAttention.get_kv_cache_spec(layer, config)
|
||||
spec = customize_attention_spec(
|
||||
backend, MLAAttention.get_kv_cache_spec(layer, config)
|
||||
)
|
||||
indexer = SimpleNamespace(
|
||||
cache_config=config.cache_config, head_dim=132, dtype=torch.uint8
|
||||
)
|
||||
@@ -102,10 +112,8 @@ def test_allocation_and_warmup_follow_addressing_mode(
|
||||
MLAAttention.bind_kv_cache(layer, views["mla"])
|
||||
DeepseekV32IndexerCache.bind_kv_cache(indexer, views["indexer"])
|
||||
assert indexer.kv_cache.stride(0) == views["indexer"].stride(0)
|
||||
if stride_alignment is not None:
|
||||
assert (
|
||||
views["mla"].stride(0) * views["mla"].element_size() % stride_alignment == 0
|
||||
)
|
||||
alignment = spec.get_block_stride_alignment()
|
||||
assert views["mla"].stride(0) * views["mla"].element_size() % alignment == 0
|
||||
if not layer._uses_flat_kv_cache():
|
||||
register.assert_not_called()
|
||||
else:
|
||||
@@ -113,7 +121,7 @@ def test_allocation_and_warmup_follow_addressing_mode(
|
||||
register.assert_called_once_with(config, block_stride_rows=stride)
|
||||
layer.kv_cache[1, 0].fill_(3)
|
||||
torch.testing.assert_close(rows[stride], layer.kv_cache[1, 0])
|
||||
if stride_alignment is None:
|
||||
if alignment == 1:
|
||||
assert cache.kv_cache_tensors[0].size == cache.num_blocks * sum(
|
||||
item.page_size_bytes for item in specs.values()
|
||||
)
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.triton_attn_diffkv import (
|
||||
TritonAttentionDiffKVMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
|
||||
# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and
|
||||
# the 64-segment scratch holds every query token (e.g. multi-query verify).
|
||||
@pytest.mark.parametrize(
|
||||
"builder_cls",
|
||||
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"capability,num_kv_heads,num_seqs,query_len,segments",
|
||||
[
|
||||
((12, 0), 1, 11, 1, 64),
|
||||
((12, 0), 1, 12, 1, 16),
|
||||
((12, 0), 8, 1, 1, 64),
|
||||
((12, 0), 8, 2, 1, 16),
|
||||
((9, 0), 1, 1, 1, 16),
|
||||
((12, 0), 1, 2, 4, 64),
|
||||
((12, 0), 1, 10, 4, 16),
|
||||
],
|
||||
)
|
||||
def test_split_k_segments_follow_sm_occupancy(
|
||||
monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
current_platform, "is_device_capability", lambda cap: cap == capability
|
||||
)
|
||||
monkeypatch.setattr(current_platform, "num_compute_units", lambda: 188)
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_num_attention_heads=lambda _: 8,
|
||||
get_num_kv_heads=lambda _: num_kv_heads,
|
||||
get_head_size=lambda: 128,
|
||||
rswa_window=None,
|
||||
),
|
||||
parallel_config=None,
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=16),
|
||||
speculative_config=None,
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
|
||||
),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16
|
||||
)
|
||||
builder = builder_cls(spec, ["layer.0"], config, "cpu")
|
||||
batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs)
|
||||
metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu"))
|
||||
assert metadata.num_par_softmax_segments == segments
|
||||
assert metadata.softmax_segm_output.shape[2] == segments
|
||||
assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len
|
||||
# DiffKV re-allocates the output; it must stay sized like the base buffers.
|
||||
assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0]
|
||||
assert (
|
||||
metadata.softmax_segm_output.data_ptr()
|
||||
== builder.softmax_segm_output.data_ptr()
|
||||
)
|
||||
|
||||
|
||||
def _draft_builder(builder_cls, rswa_window: int | None):
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
get_num_attention_heads=lambda _: 8,
|
||||
get_num_kv_heads=lambda _: 1,
|
||||
get_head_size=lambda: 128,
|
||||
rswa_window=rswa_window,
|
||||
),
|
||||
parallel_config=None,
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=16),
|
||||
speculative_config=None,
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
|
||||
),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
|
||||
)
|
||||
return builder_cls(spec, ["layer.0"], config, "cpu")
|
||||
|
||||
|
||||
def _draft_decode_batch(seq_lens: list[int]):
|
||||
batch = BatchSpec(seq_lens=seq_lens, query_lens=[1] * len(seq_lens))
|
||||
return create_common_attn_metadata(batch, 16, "cpu")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"builder_cls",
|
||||
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
|
||||
)
|
||||
@pytest.mark.parametrize("rswa_window", [None, 64])
|
||||
def test_draft_decode_update_flag_matches_build_writes(builder_cls, rswa_window):
|
||||
"""The update is a no-op, so the builder may claim support only when
|
||||
build() writes no builder-owned tensor that a captured graph reads."""
|
||||
builder = _draft_builder(builder_cls, rswa_window)
|
||||
common_metadata = _draft_decode_batch([5, 9])
|
||||
common_metadata.rswa_prefix_lens = torch.tensor([3, 7], dtype=torch.int32)
|
||||
|
||||
def owned_bytes():
|
||||
# Bytes, not values: uninitialized scratch may hold NaNs.
|
||||
return {
|
||||
name: value.view(torch.uint8).clone()
|
||||
for name, value in vars(builder).items()
|
||||
if isinstance(value, torch.Tensor)
|
||||
}
|
||||
|
||||
before = owned_bytes()
|
||||
builder.build(0, common_metadata)
|
||||
after = owned_bytes()
|
||||
|
||||
written = [name for name in before if not torch.equal(before[name], after[name])]
|
||||
assert builder.supports_draft_decode_metadata_update is (not written)
|
||||
assert builder.supports_draft_decode_metadata_update is (rswa_window is None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"builder_cls",
|
||||
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
|
||||
)
|
||||
def test_draft_decode_update_matches_fresh_build(builder_cls):
|
||||
"""A replayed graph reads metadata built at capture; after an in-place
|
||||
update it must match a fresh build of the replayed batch."""
|
||||
torch.manual_seed(0)
|
||||
builder = _draft_builder(builder_cls, rswa_window=None)
|
||||
captured = _draft_decode_batch([100, 120])
|
||||
metadata = builder.build(0, captured)
|
||||
|
||||
replayed = _draft_decode_batch([128, 30])
|
||||
# The replayed batch arrives through the same persistent input buffers.
|
||||
captured.seq_lens.copy_(replayed.seq_lens)
|
||||
captured.block_table_tensor.copy_(replayed.block_table_tensor)
|
||||
captured.slot_mapping.copy_(replayed.slot_mapping)
|
||||
builder.update_draft_decode_metadata(metadata)
|
||||
|
||||
expected = _draft_builder(builder_cls, rswa_window=None).build(0, replayed)
|
||||
assert torch.equal(metadata.seq_lens, expected.seq_lens)
|
||||
assert torch.equal(metadata.block_table, expected.block_table)
|
||||
assert torch.equal(metadata.slot_mapping, expected.slot_mapping)
|
||||
assert torch.equal(metadata.query_start_loc, expected.query_start_loc)
|
||||
@@ -13,29 +13,19 @@ from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadataBuilde
|
||||
from vllm.v1.attention.backends.triton_attn_diffkv import (
|
||||
TritonAttentionDiffKVMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVQuantMode
|
||||
|
||||
|
||||
# 64 segments only while num_seqs * num_kv_heads * 16 < 188 SMs on SM12.0 and
|
||||
# the 64-segment scratch holds every query token (e.g. multi-query verify).
|
||||
@pytest.mark.parametrize(
|
||||
"builder_cls",
|
||||
[TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"capability,num_kv_heads,num_seqs,query_len,segments",
|
||||
[
|
||||
((12, 0), 1, 11, 1, 64),
|
||||
((12, 0), 1, 12, 1, 16),
|
||||
((12, 0), 8, 1, 1, 64),
|
||||
((12, 0), 8, 2, 1, 16),
|
||||
((9, 0), 1, 1, 1, 16),
|
||||
((12, 0), 1, 2, 4, 64),
|
||||
((12, 0), 1, 10, 4, 16),
|
||||
],
|
||||
)
|
||||
def test_split_k_segments_follow_sm_occupancy(
|
||||
monkeypatch, builder_cls, capability, num_kv_heads, num_seqs, query_len, segments
|
||||
def _build(
|
||||
monkeypatch,
|
||||
builder_cls,
|
||||
capability,
|
||||
num_kv_heads,
|
||||
query_lens,
|
||||
max_num_seqs=16,
|
||||
num_speculative_tokens=None,
|
||||
parallel_drafting=False,
|
||||
kv_quant_mode=KVQuantMode.NONE,
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
@@ -50,24 +40,119 @@ def test_split_k_segments_follow_sm_occupancy(
|
||||
rswa_window=None,
|
||||
),
|
||||
parallel_config=None,
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=16),
|
||||
speculative_config=None,
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=max_num_seqs),
|
||||
speculative_config=(
|
||||
None
|
||||
if num_speculative_tokens is None
|
||||
else SimpleNamespace(
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
parallel_drafting=parallel_drafting,
|
||||
)
|
||||
),
|
||||
compilation_config=SimpleNamespace(
|
||||
cudagraph_mode=CUDAGraphMode.NONE, static_forward_context={}
|
||||
),
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=num_kv_heads, head_size=128, dtype=torch.bfloat16
|
||||
block_size=16,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
)
|
||||
builder = builder_cls(spec, ["layer.0"], config, "cpu")
|
||||
batch = BatchSpec(seq_lens=[128] * num_seqs, query_lens=[query_len] * num_seqs)
|
||||
batch = BatchSpec(seq_lens=[128] * len(query_lens), query_lens=query_lens)
|
||||
metadata = builder.build(0, create_common_attn_metadata(batch, 16, "cpu"))
|
||||
assert metadata.num_par_softmax_segments == segments
|
||||
assert metadata.softmax_segm_output.shape[2] == segments
|
||||
assert metadata.softmax_segm_max.shape[0] >= num_seqs * query_len
|
||||
assert metadata.softmax_segm_output.shape[2] == metadata.num_par_softmax_segments
|
||||
assert metadata.softmax_segm_max.shape[0] >= sum(query_lens)
|
||||
# DiffKV re-allocates the output; it must stay sized like the base buffers.
|
||||
assert builder.softmax_segm_output.shape[0] == builder.softmax_segm_max.shape[0]
|
||||
assert (
|
||||
metadata.softmax_segm_output.data_ptr()
|
||||
== builder.softmax_segm_output.data_ptr()
|
||||
)
|
||||
return builder, metadata
|
||||
|
||||
|
||||
BUILDERS = [TritonAttentionMetadataBuilder, TritonAttentionDiffKVMetadataBuilder]
|
||||
|
||||
|
||||
# Verify batches: the 64-segment scratch holds min(11, max_num_seqs) * (1 + K)
|
||||
# rows, and its 16-segment view covers every 3D-eligible sequence. 64 segments
|
||||
# follow the request count, not Q blocks: on SM12.0 they stay faster for verify
|
||||
# batches up to 11 requests (22 Q blocks at K=3).
|
||||
@pytest.mark.parametrize("builder_cls", BUILDERS)
|
||||
@pytest.mark.parametrize(
|
||||
"max_num_seqs,num_speculative_tokens,num_seqs,rows,segments",
|
||||
[
|
||||
(16, 3, 10, 44, 64),
|
||||
(16, 3, 11, 44, 64),
|
||||
(16, 3, 12, 44, 16),
|
||||
(16, 3, 16, 44, 16),
|
||||
(1, 5, 1, 6, 64),
|
||||
],
|
||||
)
|
||||
def test_split_k_segment_rows_cover_verify_queries(
|
||||
monkeypatch,
|
||||
builder_cls,
|
||||
max_num_seqs,
|
||||
num_speculative_tokens,
|
||||
num_seqs,
|
||||
rows,
|
||||
segments,
|
||||
):
|
||||
builder, metadata = _build(
|
||||
monkeypatch,
|
||||
builder_cls,
|
||||
(12, 0),
|
||||
1,
|
||||
[1 + num_speculative_tokens] * num_seqs,
|
||||
max_num_seqs,
|
||||
num_speculative_tokens,
|
||||
)
|
||||
assert builder.softmax_segm_max.shape[:2] == (rows, 8)
|
||||
assert metadata.num_par_softmax_segments == segments
|
||||
|
||||
|
||||
# Without SM12.0 64-segment reuse, the scratch holds exactly
|
||||
# min(seq_threshold_3D, max_num_seqs) * query_len rows for split-K query_lens.
|
||||
@pytest.mark.parametrize(
|
||||
"num_speculative_tokens,parallel_drafting,max_num_seqs,rows",
|
||||
[
|
||||
(None, False, 8, 8),
|
||||
(4, False, 8, 40),
|
||||
(7, False, 8, 64),
|
||||
(8, False, 8, 64),
|
||||
(3, True, 8, 56),
|
||||
(4, True, 8, 64),
|
||||
(5, False, 1, 6),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"kv_quant_mode", [KVQuantMode.NONE, KVQuantMode.INT4_PER_TOKEN_HEAD]
|
||||
)
|
||||
def test_split_k_scratch_rows_follow_speculative_config(
|
||||
monkeypatch,
|
||||
num_speculative_tokens,
|
||||
parallel_drafting,
|
||||
max_num_seqs,
|
||||
rows,
|
||||
kv_quant_mode,
|
||||
):
|
||||
# The INT4 launcher never runs multi-query split-K.
|
||||
if kv_quant_mode == KVQuantMode.INT4_PER_TOKEN_HEAD:
|
||||
rows = max_num_seqs
|
||||
builder, _ = _build(
|
||||
monkeypatch,
|
||||
TritonAttentionMetadataBuilder,
|
||||
(9, 0),
|
||||
1,
|
||||
[1],
|
||||
max_num_seqs,
|
||||
num_speculative_tokens,
|
||||
parallel_drafting,
|
||||
kv_quant_mode,
|
||||
)
|
||||
assert builder.softmax_segm_output.shape == (rows, 8, 16, 128)
|
||||
assert builder.softmax_segm_max.shape == (rows, 8, 16)
|
||||
assert builder.softmax_segm_expsum.shape == (rows, 8, 16)
|
||||
|
||||
@@ -38,9 +38,7 @@ def _manager(
|
||||
num_prefill_lookahead=0,
|
||||
):
|
||||
init_none_hash(sha256)
|
||||
config = _make_hybrid_kv_cache_config(
|
||||
block_size, num_blocks, ["full", "mamba_align"]
|
||||
)
|
||||
config = _make_hybrid_kv_cache_config(block_size, num_blocks, ["full", "mamba"])
|
||||
if eagle_group is not None:
|
||||
groups = list(config.kv_cache_groups)
|
||||
groups[eagle_group] = replace(groups[eagle_group], is_eagle_group=True)
|
||||
|
||||
@@ -26,8 +26,8 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
)
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
@@ -2308,12 +2308,11 @@ def test_opted_out_scratch_group_keeps_partial_hash_hits():
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
KpoolTailSpec(
|
||||
CircularBufferSpec(
|
||||
block_size=mamba_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=mamba_block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
@@ -2366,12 +2365,11 @@ def test_kpool_tail_supports_128_token_partial_hash_hits():
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
KpoolTailSpec(
|
||||
CircularBufferSpec(
|
||||
block_size=kpool_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=kpool_block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
|
||||
@@ -340,7 +340,6 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
total_num_scheduled_tokens=1,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
|
||||
@@ -18,13 +18,15 @@ import torch
|
||||
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
from vllm.v1.core.kv_cache_planning import (
|
||||
_get_kv_cache_bytes_per_block,
|
||||
_get_packed_kv_cache_groups,
|
||||
_pool_bytes_per_block,
|
||||
generate_scheduler_kv_cache_config,
|
||||
get_kv_cache_config_from_groups,
|
||||
get_kv_cache_groups,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
generate_scheduler_kv_cache_config,
|
||||
resolve_kv_cache_block_sizes,
|
||||
)
|
||||
from vllm.v1.hisparse.layout import _build_hisparse_kv_cache_tensors
|
||||
@@ -88,7 +90,7 @@ def test_packed_alignment_preserves_hot_pages_and_generic_stride():
|
||||
hot = HiSparseHotSpec(block_size=16, page_size=5120, blocks_per_request=2)
|
||||
groups = [KVCacheGroupSpec(["attention"], spec), KVCacheGroupSpec(["hot"], hot)]
|
||||
alignment = lcm(1152, hot.page_size_bytes)
|
||||
stride = _pool_bytes_per_block(groups)
|
||||
stride = _pool_bytes_per_block(_mock_vllm_config("BLHNC"), groups)
|
||||
assert stride == alignment
|
||||
layout = KVCacheLayout.BLHNC
|
||||
tensors = _build_hisparse_kv_cache_tensors(groups, 3, 3 * stride, layout, stride)
|
||||
@@ -254,7 +256,7 @@ class TestCSALinearGrouping:
|
||||
def test_every_group_fits_one_packed_block(self):
|
||||
config = _shared_layout_config()
|
||||
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(groups)
|
||||
bytes_per_block = _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC)
|
||||
|
||||
pages = _pages(groups)
|
||||
for group in groups:
|
||||
@@ -276,7 +278,14 @@ class TestCSALinearGrouping:
|
||||
neither divides the hit length nor bounds the (empty) block list."""
|
||||
config = _shared_layout_config()
|
||||
config.cache_config.enable_prefix_caching = True
|
||||
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
|
||||
config.cache_config.mamba_cache_mode = "align"
|
||||
specs = {
|
||||
name: replace(spec, mamba_cache_mode="align")
|
||||
if isinstance(spec, MambaSpec)
|
||||
else spec
|
||||
for name, spec in _make_csa_linear_specs().items()
|
||||
}
|
||||
groups = get_kv_cache_groups(config, specs)
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=BYTES_PER_BLOCK * 64
|
||||
)
|
||||
@@ -418,7 +427,7 @@ class TestCSALinearGrouping:
|
||||
groups = _get_packed_kv_cache_groups(config, specs)
|
||||
gdn = [g for g in groups if g.layer_names[0].startswith("gdn.")]
|
||||
|
||||
assert _get_kv_cache_bytes_per_block(groups) == sum(
|
||||
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == sum(
|
||||
specs[name].page_size_bytes for name in specs if name.startswith("wide.")
|
||||
)
|
||||
# That block holds every GDN state at once, so the repeat pattern alone
|
||||
@@ -465,7 +474,7 @@ class TestSlidingWindowBucketCap:
|
||||
pages = _pages(groups)
|
||||
main = next(g for g in groups if "layers.2.attn" in g.layer_names)
|
||||
main_bytes = sum(pages[n] for n in main.layer_names)
|
||||
assert _get_kv_cache_bytes_per_block(groups) == main_bytes
|
||||
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == main_bytes
|
||||
swa_groups = [g for g in groups if g.layer_names[0].endswith(".swa")]
|
||||
per_group = main_bytes // pages["layers.0.swa"]
|
||||
assert len(swa_groups) == -(-43 // per_group)
|
||||
@@ -479,9 +488,9 @@ class TestSlidingWindowBucketCap:
|
||||
class TestDensePacking:
|
||||
def test_bytes_per_block_is_largest_group(self):
|
||||
groups, g1, g2 = _mixed_page_groups()
|
||||
assert _get_kv_cache_bytes_per_block(groups) == _expected_bytes_per_block(
|
||||
groups
|
||||
)
|
||||
assert _get_kv_cache_bytes_per_block(
|
||||
groups, KVCacheLayout.BLHNC
|
||||
) == _expected_bytes_per_block(groups)
|
||||
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config("BLHNC"), groups, MEMORY
|
||||
@@ -554,6 +563,20 @@ class TestDensePacking:
|
||||
assert idx_tensor.offset == 2 * _mla(512).page_size_bytes
|
||||
assert mla_tensor.block_stride == block_stride
|
||||
|
||||
def test_layer_outer_pads_only_unaligned_pages(self):
|
||||
"""Layer-outer block strides are pages, so only an unaligned page pads."""
|
||||
mla_page = _mla(512).page_size_bytes
|
||||
idx = replace(_mla(128), block_stride_alignment=12288)
|
||||
groups = [_uniform_group({"mla.0": _mla(512), "idx.0": idx})]
|
||||
config = get_kv_cache_config_from_groups(
|
||||
_mock_vllm_config("LBNHC"), groups, MEMORY
|
||||
)
|
||||
mla_tensor, idx_tensor = config.kv_cache_tensors
|
||||
assert config.num_blocks == MEMORY // (mla_page + 12288)
|
||||
assert (mla_tensor.block_stride, idx_tensor.block_stride) == (mla_page, 12288)
|
||||
assert idx_tensor.offset == mla_page * config.num_blocks
|
||||
_bind(config, "LBNHC")
|
||||
|
||||
def test_overlaid_groups_alias_and_stay_isolated(self):
|
||||
groups, g1, g2 = _mixed_page_groups()
|
||||
# Overlay models resolve to a block-outer layout at backend selection (the
|
||||
@@ -562,7 +585,9 @@ class TestDensePacking:
|
||||
_mock_vllm_config("BLNHC"), groups, MEMORY
|
||||
)
|
||||
assert config.num_blocks == MEMORY // _expected_bytes_per_block(groups)
|
||||
assert _pool_bytes_per_block(groups) == _expected_bytes_per_block(groups)
|
||||
assert _pool_bytes_per_block(
|
||||
_mock_vllm_config("BLNHC"), groups
|
||||
) == _expected_bytes_per_block(groups)
|
||||
|
||||
views = _bind(config, "BLNHC")
|
||||
assert set(views) == set(g1) | set(g2)
|
||||
@@ -667,7 +692,10 @@ class TestCompressorRingGroup:
|
||||
assert len(ring_groups) == 1
|
||||
assert not ring_groups[0].kv_cache_spec.prefix_cacheable
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
|
||||
config,
|
||||
groups,
|
||||
available_memory=64
|
||||
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
|
||||
)
|
||||
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (128, 64)
|
||||
manager = KVCacheManager(
|
||||
@@ -716,7 +744,10 @@ class TestSWABoundedReplayGrouping:
|
||||
for g in groups
|
||||
) == [(False, 128)] * 4 + [(True, 0)]
|
||||
kv_cache_config = get_kv_cache_config_from_groups(
|
||||
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
|
||||
config,
|
||||
groups,
|
||||
available_memory=64
|
||||
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
generate_scheduler_kv_cache_config([kv_cache_config]),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -279,6 +279,12 @@ def _allocate_scheduled(
|
||||
return blocks
|
||||
|
||||
|
||||
def _resident_block_ids(manager: KVCacheManager, request_id: str) -> list[int]:
|
||||
"""The GPU block HiSparse reads each resident page from; 0 if host-only."""
|
||||
resident = get_hisparse_coordinator(manager).resident_managers[0]
|
||||
return [block.block_id for block in resident.get_residency_row(request_id)]
|
||||
|
||||
|
||||
def test_hisparse_does_not_write_back_reprefillable_tokens():
|
||||
"""Multi-module MTP re-prefills the last tokens, so they are not final."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, num_prefill_lookahead=3)
|
||||
@@ -319,7 +325,7 @@ def test_hisparse_builds_dma_row_mirrors_across_pages():
|
||||
def test_hisparse_async_speculation_mirrors_uncertain_position_range():
|
||||
"""Unresolved drafts must not leave gaps in the eager host mirror."""
|
||||
coordinator = MagicMock(host_group_id=0)
|
||||
coordinator.take_block_table_updates.return_value = {}
|
||||
coordinator.take_residency_updates.return_value = {}
|
||||
coordinator.build_offload_command.return_value = None
|
||||
coordinator.build_row_mirrors.return_value = ()
|
||||
scheduler = HiSparseConnectorScheduler(
|
||||
@@ -332,7 +338,6 @@ def test_hisparse_async_speculation_mirrors_uncertain_position_range():
|
||||
connector.connector_scheduler = scheduler
|
||||
connector.update_state_after_alloc(request, None, 0)
|
||||
scheduler_output = SimpleNamespace(
|
||||
block_table_updates=None,
|
||||
kv_cache_block_copies=None,
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
@@ -370,11 +375,66 @@ def test_hisparse_reports_when_context_is_fully_resident():
|
||||
hot_manager.require_hot(request.request_id)
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens)) is not None
|
||||
_publish_hisparse_pages(manager)
|
||||
coordinator.take_residency_updates([request.request_id])
|
||||
core_row = manager.get_block_ids(request.request_id)[2]
|
||||
pool = manager.block_pool
|
||||
pool.get_new_blocks(pool.get_num_free_blocks())
|
||||
|
||||
assert not coordinator.all_context_pages_resident(scheduled)
|
||||
assert coordinator.take_block_table_updates().keys() == {request.request_id}
|
||||
# The block-table row stays append-only; the worker learns of the lost
|
||||
# pages from the residency update instead.
|
||||
assert manager.get_block_ids(request.request_id)[2] == core_row
|
||||
resident_ids = _resident_block_ids(manager, request.request_id)
|
||||
lost = [page for page, block_id in enumerate(resident_ids) if block_id == 0]
|
||||
assert lost
|
||||
update = coordinator.take_residency_updates([request.request_id])
|
||||
assert update[request.request_id].pages == lost
|
||||
assert update[request.request_id].block_ids[0] == [0] * len(lost)
|
||||
|
||||
|
||||
def test_hisparse_residency_update_sends_only_changed_pages():
|
||||
"""Losing one early page must not resend the pages after it: per-step
|
||||
residency updates would otherwise grow with the context length."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16)
|
||||
tokens = list(range(6 * HISPARSE_BLOCK_SIZE))
|
||||
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens))
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
for hot_manager in coordinator.hot_managers:
|
||||
hot_manager.require_hot(request.request_id)
|
||||
assert _allocate_scheduled(manager, request, num_new_tokens=len(tokens))
|
||||
coordinator.take_residency_updates([request.request_id])
|
||||
pool = manager.block_pool
|
||||
num_never_used = pool.get_num_free_blocks()
|
||||
# Pages unpin as their host copies complete, so page 0 is reused first.
|
||||
_publish_hisparse_pages(manager)
|
||||
pool.get_new_blocks(num_never_used + 1)
|
||||
|
||||
update = coordinator.take_residency_updates([request.request_id])
|
||||
|
||||
assert update[request.request_id].pages == [0]
|
||||
assert update[request.request_id].block_ids == ([0],)
|
||||
|
||||
|
||||
def test_hisparse_readmitted_request_resends_its_full_residency():
|
||||
"""A preempted request may resume in a worker state row holding another
|
||||
request's residency, so its first update must cover every page again."""
|
||||
manager = make_hisparse_kv_cache_manager(32, 16)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
tokens = list(range(2 * HISPARSE_BLOCK_SIZE))
|
||||
request = make_request("request", tokens, HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert _allocate_scheduled(manager, request, len(tokens)) is not None
|
||||
assert request.request_id in coordinator.take_residency_updates(
|
||||
[request.request_id]
|
||||
)
|
||||
manager.free(request)
|
||||
|
||||
assert _allocate_scheduled(manager, request, len(tokens)) is not None
|
||||
update = coordinator.take_residency_updates([request.request_id])
|
||||
assert update[request.request_id].pages == [0, 1]
|
||||
assert update[request.request_id].block_ids[0] == _resident_block_ids(
|
||||
manager, request.request_id
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
|
||||
@@ -416,9 +476,7 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
|
||||
source, indexer, resident, hot = manager.get_blocks(resumed.request_id).blocks
|
||||
assert len(source) == len(indexer) == len(resident) == 4
|
||||
assert len(hot) == 2
|
||||
assert not any(block.is_null for block in resident[:2])
|
||||
assert not resident[2].is_null
|
||||
assert not resident[3].is_null
|
||||
assert 0 not in _resident_block_ids(manager, resumed.request_id)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
assert not coordinator.build_offload_command().page_transfers
|
||||
coordinator.finish_host_import(resumed.request_id, failed=False)
|
||||
@@ -666,9 +724,10 @@ def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_cach
|
||||
pool.free_blocks(held)
|
||||
assert _allocate_scheduled(manager, first, num_new_tokens=1) is not None
|
||||
assert all(m.has_hot("first") for m in coordinator.hot_managers)
|
||||
coordinator.take_residency_updates(["first"])
|
||||
pool.get_new_blocks(pool.get_num_free_blocks())
|
||||
assert not coordinator.all_context_pages_resident(scheduled)
|
||||
assert "first" in coordinator.take_block_table_updates()
|
||||
assert "first" in coordinator.take_residency_updates(["first"])
|
||||
|
||||
|
||||
def test_hisparse_full_pool_keeps_pages_pinned_until_preemption():
|
||||
@@ -1204,11 +1263,7 @@ def test_hisparse_prefix_hit_adopts_gpu_shadow_pages():
|
||||
num_new_computed_tokens=num_computed,
|
||||
new_computed_blocks=computed,
|
||||
)
|
||||
resident_blocks = manager.get_blocks("resumed").blocks[2]
|
||||
assert [block.block_id for block in resident_blocks[:3]] == (
|
||||
original_resident_ids[:3]
|
||||
)
|
||||
assert not any(block.is_null for block in resident_blocks[:3])
|
||||
assert _resident_block_ids(manager, "resumed")[:3] == original_resident_ids[:3]
|
||||
assert get_hisparse_coordinator(manager).all_context_pages_resident(
|
||||
((resumed.request_id, num_computed, len(tokens) - num_computed),)
|
||||
)
|
||||
@@ -1249,17 +1304,13 @@ def test_hisparse_prefix_hit_under_pressure_adopts_surviving_copies(
|
||||
new_computed_blocks=computed,
|
||||
)
|
||||
|
||||
resumed_blocks = manager.get_blocks("resumed").blocks
|
||||
assert [block.block_id for block in resumed_blocks[2][:3]] == [
|
||||
resident_ids = _resident_block_ids(manager, "resumed")
|
||||
assert resident_ids[:3] == [
|
||||
copy_ids[page] if page in adopted_pages else pool.null_block.block_id
|
||||
for page in range(3)
|
||||
]
|
||||
gpu_ids = [
|
||||
block.block_id
|
||||
for group in resumed_blocks[1:]
|
||||
for block in group
|
||||
if not block.is_null
|
||||
]
|
||||
_, indexer, _, hot = manager.get_block_ids("resumed")
|
||||
gpu_ids = [block_id for block_id in indexer + resident_ids + hot if block_id]
|
||||
assert len(gpu_ids) == len(set(gpu_ids))
|
||||
|
||||
|
||||
@@ -1488,8 +1539,7 @@ def test_hisparse_reused_copy_is_not_adopted():
|
||||
num_new_computed_tokens=num_computed,
|
||||
new_computed_blocks=computed,
|
||||
)
|
||||
resident_blocks = manager.get_blocks("resumed").blocks[2]
|
||||
assert all(block.is_null for block in resident_blocks[:3])
|
||||
assert _resident_block_ids(manager, "resumed")[:3] == [0, 0, 0]
|
||||
|
||||
|
||||
def test_hisparse_external_import_uses_hard_gpu_footprint():
|
||||
@@ -1567,6 +1617,7 @@ def make_kv_cache_config_hybrid_model(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
|
||||
return KVCacheConfig(
|
||||
@@ -1602,6 +1653,7 @@ def make_kv_cache_config_three_types(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
elif third_spec_type == "sliding_window":
|
||||
third_spec = SlidingWindowSpec(
|
||||
@@ -1682,6 +1734,7 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
@@ -1707,10 +1760,13 @@ def test_prefix_cache_hit_uses_per_group_dcp_geometry(draft_sharded):
|
||||
req0 = make_request("0", common_token_ids + [99] * 7, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req0)
|
||||
assert num_computed_tokens == 0
|
||||
blocks = manager.allocate_slots(
|
||||
req0, len(req0.prompt_token_ids), 0, computed_blocks
|
||||
)
|
||||
assert blocks is not None
|
||||
# Align-mode Mamba keeps a state only where a chunk ends, so end the first
|
||||
# chunk on the shared prefix.
|
||||
for num_new_tokens in (len(common_token_ids), 7):
|
||||
blocks = manager.allocate_slots(req0, num_new_tokens, 0, computed_blocks)
|
||||
assert blocks is not None
|
||||
req0.num_computed_tokens += num_new_tokens
|
||||
computed_blocks = None
|
||||
|
||||
req1 = make_request("1", common_token_ids + [100] * 5, block_size, hash_fn)
|
||||
computed_blocks, num_computed_tokens, _ = manager.get_computed_blocks(req1)
|
||||
@@ -2235,7 +2291,7 @@ def _make_hybrid_kv_cache_config(
|
||||
- "full": FullAttentionSpec
|
||||
- "sliding_window": SlidingWindowSpec with window=2*block_size
|
||||
- "sliding_window_large": SlidingWindowSpec with window=4*block_size
|
||||
- "mamba": MambaSpec
|
||||
- "mamba": MambaSpec in prefix-caching ("align") mode
|
||||
|
||||
"""
|
||||
spec_map = {
|
||||
@@ -2263,11 +2319,6 @@ def _make_hybrid_kv_cache_config(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
),
|
||||
"mamba_align": lambda: MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
}
|
||||
@@ -2377,14 +2428,16 @@ def test_prefill_hybrid_model_combinations(spec_types: list[str]):
|
||||
assert not computed_blocks.blocks[0] # No cache hit initially
|
||||
assert num_computed_tokens == 0
|
||||
|
||||
blocks = manager.allocate_slots(
|
||||
req0, 55, len(computed_blocks.blocks[0]) * block_size, computed_blocks
|
||||
)
|
||||
assert blocks is not None
|
||||
# Should have blocks for all groups
|
||||
assert len(blocks.get_block_ids()) == num_groups
|
||||
|
||||
manager.new_step_starts()
|
||||
# Align-mode Mamba keeps a state only where a chunk ends, so end the first
|
||||
# chunk on the common prefix.
|
||||
for num_new_tokens in (len(common_token_ids), len(unique_token_ids)):
|
||||
blocks = manager.allocate_slots(req0, num_new_tokens, 0, computed_blocks)
|
||||
assert blocks is not None
|
||||
# Should have blocks for all groups
|
||||
assert len(blocks.get_block_ids()) == num_groups
|
||||
req0.num_computed_tokens += num_new_tokens
|
||||
computed_blocks = None
|
||||
manager.new_step_starts()
|
||||
|
||||
# Second request: should hit cached blocks for common prefix
|
||||
req1 = make_request("1", common_token_ids + [4] * 5, block_size, hash_fn)
|
||||
@@ -2499,7 +2552,7 @@ def test_prefill_hybrid_model_mamba_align():
|
||||
num_blocks = 30
|
||||
|
||||
kv_cache_config = _make_hybrid_kv_cache_config(
|
||||
block_size, num_blocks, ["full", "mamba_align"]
|
||||
block_size, num_blocks, ["full", "mamba"]
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config,
|
||||
@@ -2535,7 +2588,7 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
|
||||
"""
|
||||
block_size = 16
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 30, ["full", "mamba_align"]),
|
||||
_make_hybrid_kv_cache_config(block_size, 30, ["full", "mamba"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
@@ -2610,7 +2663,7 @@ def test_hybrid_model_mamba_align_with_dynamic_draft_tokens():
|
||||
num_blocks = 30
|
||||
|
||||
kv_cache_config = _make_hybrid_kv_cache_config(
|
||||
block_size, num_blocks, ["full", "mamba_align"]
|
||||
block_size, num_blocks, ["full", "mamba"]
|
||||
)
|
||||
manager = KVCacheManager(
|
||||
kv_cache_config,
|
||||
@@ -5846,49 +5899,6 @@ def test_mamba_reachable_block_mask_pins_shared_prefix():
|
||||
assert retained(0, None) == {14}
|
||||
|
||||
|
||||
def test_mamba_shared_prefix_survives_zero_retention():
|
||||
"""Manager-level check of the full wiring: a pinned shared-prefix boundary
|
||||
(``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style
|
||||
detection) keeps its Mamba state block cached under
|
||||
``prefix_cache_retention_interval=0``, which otherwise retains only the
|
||||
end-of-prompt replay boundary. Without this, a shared prefix (junction
|
||||
before ``num_prompt``) would be recomputed by every sharing request."""
|
||||
block_size = 16
|
||||
|
||||
# 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14.
|
||||
token_ids = [i for i in range(16) for _ in range(block_size)]
|
||||
|
||||
def cached_mamba_blocks(shared_prefix_boundary):
|
||||
# Fresh manager per scenario so cached blocks don't leak between runs.
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 100, ["full", "mamba"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
retention_interval=0,
|
||||
)
|
||||
req = make_request("r", token_ids, block_size, sha256)
|
||||
req.shared_prefix_boundary = shared_prefix_boundary
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req)
|
||||
blocks = manager.allocate_slots(
|
||||
req, len(token_ids), num_computed, computed_blocks
|
||||
)
|
||||
assert blocks is not None
|
||||
pool = manager.block_pool
|
||||
return {
|
||||
i
|
||||
for i in range(16)
|
||||
if pool.get_cached_block(req.block_hashes[i], kv_cache_group_ids=[1])
|
||||
is not None
|
||||
}
|
||||
|
||||
# Without a pinned boundary, retention=0 keeps only the replay boundary (14).
|
||||
assert cached_mamba_blocks(0) == {14}
|
||||
# Pinning the shared prefix at token 96 (state block 5) retains it too, so a
|
||||
# later request sharing that prefix can hit the Mamba state.
|
||||
assert cached_mamba_blocks(96) == {5, 14}
|
||||
|
||||
|
||||
def test_mamba_shared_prefix_reuse_under_zero_retention():
|
||||
"""Full cross-request Marconi flow: a partial shared prefix cached by the
|
||||
detecting request must stay reusable by a later request under
|
||||
@@ -5899,7 +5909,7 @@ def test_mamba_shared_prefix_reuse_under_zero_retention():
|
||||
|
||||
def last_req_hit(retention, pin):
|
||||
manager = make_kv_cache_manager(
|
||||
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]),
|
||||
_make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba"]),
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
|
||||
@@ -352,6 +352,12 @@ def test_async_scheduling_pp_allows_rescheduling_with_output_placeholders():
|
||||
_ = scheduler.schedule()
|
||||
assert req.num_output_placeholders > 0
|
||||
|
||||
if scheduler.use_v2_model_runner:
|
||||
# MRV2 staggers same-request decodes by pp_size steps to match the
|
||||
# worker-side sampled-token broadcast cadence.
|
||||
output = scheduler.schedule()
|
||||
assert req.request_id not in output.num_scheduled_tokens
|
||||
|
||||
# before any update_from_output, we still expect the request can be
|
||||
# scheduled again (multi-step in-flight).
|
||||
output = scheduler.schedule()
|
||||
@@ -735,7 +741,8 @@ def test_same_step_duplicate_encoder_input_stays_cached():
|
||||
assert "image" not in cache.freeable
|
||||
|
||||
|
||||
def test_no_mm_input_chunking():
|
||||
@pytest.mark.parametrize("warm_encoder_cache", [False, True])
|
||||
def test_no_mm_input_chunking(warm_encoder_cache: bool):
|
||||
# Disable multimodal input chunking.
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
@@ -747,6 +754,16 @@ def test_no_mm_input_chunking():
|
||||
requests = create_requests(
|
||||
num_requests=1, num_tokens=1200, mm_positions=mm_positions
|
||||
)
|
||||
if warm_encoder_cache:
|
||||
cached_request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=1200,
|
||||
mm_positions=mm_positions,
|
||||
req_ids=["cache-primer"],
|
||||
)[0]
|
||||
scheduler.encoder_cache_manager.allocate(cached_request, 0)
|
||||
scheduler.encoder_cache_manager.free(cached_request)
|
||||
|
||||
for request in requests:
|
||||
scheduler.add_request(request)
|
||||
|
||||
@@ -774,6 +791,8 @@ def test_no_mm_input_chunking():
|
||||
assert output.scheduled_cached_reqs.num_reqs == 1
|
||||
assert len(output.finished_req_ids) == 0
|
||||
assert output.num_scheduled_tokens[requests[0].request_id] == 800
|
||||
if warm_encoder_cache:
|
||||
assert not output.scheduled_encoder_inputs
|
||||
|
||||
# Test that we fail if we disable chunked mm input and use too small
|
||||
# of a max_num_batched_tokens for the mm input.
|
||||
@@ -785,6 +804,66 @@ def test_no_mm_input_chunking():
|
||||
)
|
||||
|
||||
|
||||
def test_no_mm_input_chunking_for_same_step_duplicate():
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
max_num_batched_tokens=1024,
|
||||
disable_chunked_mm_input=True,
|
||||
max_model_len=2048,
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=1352,
|
||||
mm_hashes_list=[["image", "image"]],
|
||||
mm_positions=[
|
||||
[
|
||||
PlaceholderRange(offset=100, length=576),
|
||||
PlaceholderRange(offset=776, length=576),
|
||||
]
|
||||
],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
output = scheduler.schedule()
|
||||
|
||||
assert output.num_scheduled_tokens[request.request_id] == 776
|
||||
assert output.scheduled_encoder_inputs == {request.request_id: [0]}
|
||||
|
||||
|
||||
def test_no_mm_input_chunking_at_item_boundary():
|
||||
"""A chunk starting exactly at a multimodal item must not split it.
|
||||
|
||||
Regression test for issue #52306: the rollback guard only fired while
|
||||
``num_computed_tokens < start_pos``, so a chunk beginning exactly at the
|
||||
item skipped the check and could cover only part of it - silently breaking
|
||||
models with bidirectional attention over the item.
|
||||
"""
|
||||
mm_positions = [[PlaceholderRange(offset=400, length=800)]]
|
||||
|
||||
def ask(num_computed_tokens: int, num_new_tokens: int) -> int:
|
||||
# A fresh scheduler per call: the encoder cache is stateful.
|
||||
scheduler = create_scheduler(
|
||||
model="llava-hf/llava-1.5-7b-hf",
|
||||
max_num_batched_tokens=1024,
|
||||
disable_chunked_mm_input=True,
|
||||
max_model_len=2048,
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1, num_tokens=1200, mm_positions=mm_positions
|
||||
)[0]
|
||||
return scheduler._try_schedule_encoder_inputs(
|
||||
request, num_computed_tokens, num_new_tokens, encoder_compute_budget=100_000
|
||||
)[1]
|
||||
|
||||
# Starts before the item and cannot reach past it: stop just before it.
|
||||
assert ask(0, 500) == 400
|
||||
# Starts exactly at the item and cannot cover it: schedule nothing rather
|
||||
# than half an item.
|
||||
assert ask(400, 500) == 0
|
||||
# Starts exactly at the item and covers it: schedule it.
|
||||
assert ask(400, 800) == 800
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_prefix_caching", [True, False])
|
||||
def test_schedule_concurrent_partial_requests(enable_prefix_caching: bool):
|
||||
"""Test scheduling behavior with concurrent partial requests.
|
||||
@@ -929,7 +1008,6 @@ def test_update_from_output_routes_sampling_masks_by_request():
|
||||
total_num_scheduled_tokens=3,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
@@ -973,7 +1051,6 @@ def test_update_from_output_routes_multi_position_sampling_masks():
|
||||
total_num_scheduled_tokens=2,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
@@ -1021,7 +1098,6 @@ def test_stop_via_update_from_output():
|
||||
requests[0].request_id: [],
|
||||
requests[1].request_id: [10],
|
||||
},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
@@ -1067,7 +1143,6 @@ def test_stop_via_update_from_output():
|
||||
requests[0].request_id: [10, 42],
|
||||
requests[1].request_id: [13],
|
||||
},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
@@ -1111,7 +1186,6 @@ def test_stop_via_update_from_output():
|
||||
requests[0].request_id: [10, 11],
|
||||
requests[1].request_id: [],
|
||||
},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
@@ -1149,7 +1223,6 @@ def test_stop_via_update_from_output():
|
||||
total_num_scheduled_tokens=3,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={requests[0].request_id: [EOS_TOKEN_ID, 10]},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
@@ -4200,7 +4273,6 @@ def test_abort_request_when_structured_output_fsm_cannot_advance():
|
||||
total_num_scheduled_tokens=1,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
@@ -5675,7 +5747,6 @@ def test_delayed_kv_connector_free_keeps_scheduler_active():
|
||||
total_num_scheduled_tokens=0,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
@@ -5757,7 +5828,6 @@ def test_scheduler_kv_connector_stats():
|
||||
total_num_scheduled_tokens=1,
|
||||
scheduled_spec_decode_tokens={},
|
||||
scheduled_encoder_inputs={},
|
||||
num_common_prefix_blocks=[0],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
@@ -5838,7 +5908,6 @@ def test_scheduler_ec_connector_stats():
|
||||
total_num_scheduled_tokens=1,
|
||||
scheduled_spec_decode_tokens={},
|
||||
scheduled_encoder_inputs={},
|
||||
num_common_prefix_blocks=[0],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
@@ -5870,7 +5939,6 @@ def test_ec_connector_update_connector_output_called():
|
||||
total_num_scheduled_tokens=0,
|
||||
scheduled_encoder_inputs={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
)
|
||||
|
||||
@@ -20,7 +20,6 @@ from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
ChunkedLocalAttentionManager,
|
||||
CircularBufferManager,
|
||||
FullAttentionManager,
|
||||
KpoolTailManager,
|
||||
MambaManager,
|
||||
RSWAManager,
|
||||
SlidingWindowManager,
|
||||
@@ -29,7 +28,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
MambaSpec,
|
||||
RSWASpec,
|
||||
SlidingWindowSpec,
|
||||
@@ -399,22 +397,21 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
|
||||
|
||||
manager.cache_blocks(request_id, 1024, replay_boundaries=(0,))
|
||||
manager.remove_skipped_blocks(request_id, 1024)
|
||||
assert manager.get_num_common_prefix_blocks(request_id) == 0
|
||||
assert manager.get_num_skipped_tokens(1024) == 0
|
||||
assert manager.req_to_blocks[request_id] == blocks
|
||||
|
||||
|
||||
@pytest.mark.parametrize("record_for_zeroing", [True, False])
|
||||
def test_external_kpool_tail_zeroing(record_for_zeroing):
|
||||
"""A transferred ring at index zero skips zeroing even with a local hit."""
|
||||
spec = KpoolTailSpec(
|
||||
"""The kpool tail ring is never zeroed, transferred or not: a pool only
|
||||
reads ring slots its own request wrote."""
|
||||
spec = CircularBufferSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=8,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
)
|
||||
manager = KpoolTailManager(
|
||||
manager = CircularBufferManager(
|
||||
spec,
|
||||
block_pool=BlockPool(10, enable_caching=True, hash_block_size=4),
|
||||
enable_caching=True,
|
||||
@@ -428,9 +425,7 @@ def test_external_kpool_tail_zeroing(record_for_zeroing):
|
||||
)
|
||||
blocks = manager.req_to_blocks["request"]
|
||||
assert len(blocks) == 1
|
||||
assert manager.take_new_block_ids() == (
|
||||
[blocks[0].block_id] if record_for_zeroing else []
|
||||
)
|
||||
assert manager.take_new_block_ids() == []
|
||||
assert manager.allocate_new_blocks("request", 20, 20) == []
|
||||
assert manager.take_new_block_ids() == []
|
||||
|
||||
|
||||
+75
-1
@@ -36,6 +36,10 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
KVQuantMode,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
@@ -238,7 +242,7 @@ def create_scheduler(
|
||||
structured_output_manager=StructuredOutputManager(vllm_config),
|
||||
)
|
||||
if use_v2_model_runner is None:
|
||||
use_v2_model_runner = bool(envs.VLLM_USE_V2_MODEL_RUNNER)
|
||||
use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER is not False
|
||||
scheduler.use_v2_model_runner = use_v2_model_runner
|
||||
return scheduler
|
||||
|
||||
@@ -333,3 +337,73 @@ def create_requests(
|
||||
)
|
||||
requests.append(request)
|
||||
return requests
|
||||
|
||||
|
||||
def new_kv_cache_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
page_size_padded=None,
|
||||
sliding_window=None,
|
||||
attention_chunk_size=None,
|
||||
kv_quant_mode=KVQuantMode.NONE,
|
||||
):
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
dtype=dtype,
|
||||
page_size_padded=page_size_padded,
|
||||
sliding_window=sliding_window,
|
||||
attention_chunk_size=attention_chunk_size,
|
||||
kv_quant_mode=kv_quant_mode,
|
||||
)
|
||||
|
||||
|
||||
def new_sliding_window_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
page_size_padded=None,
|
||||
sliding_window=1,
|
||||
):
|
||||
return SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
dtype=dtype,
|
||||
page_size_padded=page_size_padded,
|
||||
sliding_window=sliding_window,
|
||||
)
|
||||
|
||||
|
||||
def new_mamba_spec(
|
||||
block_size=16,
|
||||
shapes=((2, 512), (3, 32, 32)),
|
||||
dtypes=(torch.float32, torch.float32),
|
||||
num_speculative_blocks=2,
|
||||
mamba_cache_mode="none",
|
||||
page_size_padded=None,
|
||||
):
|
||||
return MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=shapes,
|
||||
dtypes=dtypes,
|
||||
page_size_padded=page_size_padded,
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
num_speculative_blocks=num_speculative_blocks,
|
||||
)
|
||||
|
||||
|
||||
def new_mla_spec(cache_dtype_str=None, block_size: int = 16):
|
||||
# head_size = kv_lora_rank(512) + qk_rope_head_dim(64) = 576
|
||||
return MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
max_tp_shards=1,
|
||||
head_size=576,
|
||||
dtype=torch.float32,
|
||||
cache_dtype_str=cache_dtype_str,
|
||||
)
|
||||
|
||||
@@ -11,6 +11,18 @@ from unittest.mock import MagicMock, patch
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
HAS_DEVICE_GRAPH = current_platform.is_cuda_alike() or current_platform.is_xpu()
|
||||
|
||||
if current_platform.is_xpu():
|
||||
from vllm.v1.worker.xpu_model_runner import _torch_cuda_wrapper
|
||||
|
||||
# Route torch.cuda graph/stream APIs to torch.xpu, as XPUModelRunner does.
|
||||
with _torch_cuda_wrapper():
|
||||
pass
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _enable_breakable_cudagraph(monkeypatch: pytest.MonkeyPatch):
|
||||
@@ -96,8 +108,8 @@ def cuda_capture_stream():
|
||||
stream, so all capture-using tests need to run under
|
||||
``torch.cuda.stream(...)`` for a separate stream.
|
||||
"""
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required")
|
||||
if not HAS_DEVICE_GRAPH:
|
||||
pytest.skip("CUDA or XPU graph required")
|
||||
from vllm.utils.torch_utils import _current_stream_tls
|
||||
|
||||
prev_stream = getattr(_current_stream_tls, "value", None)
|
||||
@@ -200,7 +212,7 @@ def test_active_state_isolated_across_threads(cuda_capture_stream):
|
||||
def test_capture_with_no_eager_break_records_one_graph(cuda_capture_stream):
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
|
||||
x = torch.zeros(4, device="cuda")
|
||||
x = torch.zeros(4, device=DEVICE_TYPE)
|
||||
cap = BreakableCUDAGraphCapture()
|
||||
with cap:
|
||||
x.add_(1.0)
|
||||
@@ -212,7 +224,7 @@ def test_capture_with_no_eager_break_records_one_graph(cuda_capture_stream):
|
||||
def test_add_eager_creates_alternating_graph_eager_graph(cuda_capture_stream):
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
|
||||
x = torch.zeros(4, device="cuda")
|
||||
x = torch.zeros(4, device=DEVICE_TYPE)
|
||||
counter = {"eager_calls": 0}
|
||||
|
||||
def eager_step():
|
||||
@@ -253,7 +265,7 @@ def test_capture_replay_matches_eager_simple(cuda_capture_stream):
|
||||
"""
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
|
||||
x = torch.zeros(8, device="cuda")
|
||||
x = torch.zeros(8, device=DEVICE_TYPE)
|
||||
log: list[str] = []
|
||||
|
||||
def eager_break_op():
|
||||
@@ -269,21 +281,21 @@ def test_capture_replay_matches_eager_simple(cuda_capture_stream):
|
||||
# Capture-time: graph kernels were recorded only; eager segment ran
|
||||
# once on x == 0, leaving x == 0.
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(x, torch.zeros(8, device="cuda"))
|
||||
assert torch.equal(x, torch.zeros(8, device=DEVICE_TYPE))
|
||||
assert log == ["eager"]
|
||||
|
||||
# Replay with a fresh input: 10 -> 11 -> 22 -> 27.
|
||||
x.fill_(10.0)
|
||||
cap.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(x, torch.full((8,), 27.0, device="cuda"))
|
||||
assert torch.equal(x, torch.full((8,), 27.0, device=DEVICE_TYPE))
|
||||
assert log == ["eager", "eager"]
|
||||
|
||||
# Replay again with another input: 100 -> 101 -> 202 -> 207.
|
||||
x.fill_(100.0)
|
||||
cap.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(x, torch.full((8,), 207.0, device="cuda"))
|
||||
assert torch.equal(x, torch.full((8,), 207.0, device=DEVICE_TYPE))
|
||||
assert log == ["eager", "eager", "eager"]
|
||||
|
||||
|
||||
@@ -301,13 +313,13 @@ def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
|
||||
# In-place double; stands in for "real" attention work.
|
||||
t.mul_(2.0)
|
||||
|
||||
x = torch.zeros(4, device="cuda")
|
||||
x = torch.zeros(4, device=DEVICE_TYPE)
|
||||
|
||||
# Outside capture: decorator should just call through.
|
||||
x.fill_(3.0)
|
||||
attention_like(x)
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(x, torch.full((4,), 6.0, device="cuda"))
|
||||
assert torch.equal(x, torch.full((4,), 6.0, device=DEVICE_TYPE))
|
||||
|
||||
# Inside capture: decorator should split the graph. Only the eager
|
||||
# segment actually mutates state during capture.
|
||||
@@ -318,7 +330,7 @@ def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
|
||||
attention_like(x) # eager: x *= 2 (on x == 0, no-op)
|
||||
x.add_(1.0) # recorded
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(x, torch.zeros(4, device="cuda"))
|
||||
assert torch.equal(x, torch.zeros(4, device=DEVICE_TYPE))
|
||||
# 2 graph segments + 1 eager segment, ordered G E G; the arithmetic
|
||||
# equivalence check below verifies the ordering.
|
||||
assert len(cap.segments) == 3
|
||||
@@ -329,7 +341,7 @@ def test_decorator_breaks_when_invoked_inside_capture(cuda_capture_stream):
|
||||
x.fill_(2.0)
|
||||
cap.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.equal(x, torch.full((4,), 15.0, device="cuda"))
|
||||
assert torch.equal(x, torch.full((4,), 15.0, device=DEVICE_TYPE))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("keyword", [False, True])
|
||||
@@ -352,7 +364,7 @@ def test_quantized_activation_replay_does_not_retain_tensors(
|
||||
data_out.copy_(x.data)
|
||||
scale_out.copy_(x.scale)
|
||||
|
||||
source = torch.ones((128, 128), device="cuda", dtype=torch.bfloat16)
|
||||
source = torch.ones((128, 128), device=DEVICE_TYPE, dtype=torch.bfloat16)
|
||||
data_out = source.to(torch.float8_e4m3fn)
|
||||
scale_out = source[:, ::32].to(torch.uint8)
|
||||
output = torch.empty_like(source)
|
||||
@@ -391,7 +403,7 @@ def test_eager_attention_inside_multistream_overlap(cuda_capture_stream):
|
||||
)
|
||||
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
|
||||
|
||||
x = torch.zeros(1024, device="cuda")
|
||||
x = torch.zeros(1024, device=DEVICE_TYPE)
|
||||
output = torch.empty_like(x)
|
||||
aux_stream = torch.cuda.Stream()
|
||||
main_event = torch.cuda.Event()
|
||||
@@ -440,7 +452,7 @@ def test_replay_invokes_eager_segments_in_order(cuda_capture_stream):
|
||||
from vllm.compilation.breakable_cudagraph import BreakableCUDAGraphCapture
|
||||
|
||||
log: list[str] = []
|
||||
x = torch.zeros(1, device="cuda")
|
||||
x = torch.zeros(1, device=DEVICE_TYPE)
|
||||
|
||||
def make_eager(name):
|
||||
def step():
|
||||
@@ -502,7 +514,7 @@ def test_nested_decorated_op_runs_inline(cuda_capture_stream):
|
||||
eager_break_during_capture,
|
||||
)
|
||||
|
||||
x = torch.zeros(4, device="cuda")
|
||||
x = torch.zeros(4, device=DEVICE_TYPE)
|
||||
inner_calls = 0
|
||||
|
||||
@eager_break_during_capture
|
||||
@@ -533,7 +545,7 @@ def test_nested_decorated_op_runs_inline(cuda_capture_stream):
|
||||
cap.replay()
|
||||
torch.accelerator.synchronize()
|
||||
# 0 -> +2 -> +1 (inner) -> +10 (outer) -> +100 = 113
|
||||
assert torch.equal(x, torch.full((4,), 113.0, device="cuda"))
|
||||
assert torch.equal(x, torch.full((4,), 113.0, device=DEVICE_TYPE))
|
||||
assert inner_calls == 2 # replay invokes the outer's lambda again
|
||||
|
||||
|
||||
|
||||
@@ -24,6 +24,14 @@ from vllm.platforms import current_platform
|
||||
from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
HAS_DEVICE_GRAPH = current_platform.is_cuda() or current_platform.is_xpu()
|
||||
|
||||
if current_platform.is_xpu():
|
||||
from vllm.v1.worker.xpu_model_runner import _torch_cuda_wrapper
|
||||
|
||||
# Route torch.cuda graph/stream APIs to torch.xpu, as XPUModelRunner does.
|
||||
with _torch_cuda_wrapper():
|
||||
pass
|
||||
|
||||
|
||||
# Helper MLP for testing
|
||||
@@ -50,6 +58,7 @@ def _create_vllm_config(
|
||||
mock_config.parallel_config = ParallelConfig()
|
||||
mock_config.speculative_config = None # No speculative decoding
|
||||
mock_config.num_speculative_tokens = 0
|
||||
mock_config.use_cumem_cudagraph_pool = False
|
||||
if not lora_config:
|
||||
mock_config.lora_config = None
|
||||
else:
|
||||
@@ -180,7 +189,6 @@ class TestCudagraphDispatcher:
|
||||
assert key == BatchDescriptor(num_tokens=15)
|
||||
|
||||
# 4. invalid_modes={FULL} should have a fall back mode
|
||||
# (e.g., cascade attention)
|
||||
desc_full_exact = BatchDescriptor(num_tokens=8, uniform=False)
|
||||
rt_mode, key = dispatcher.dispatch(
|
||||
num_tokens=8,
|
||||
@@ -268,7 +276,7 @@ class TestCudagraphDispatcher:
|
||||
assert dispatcher.get_capture_descs() == []
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
@pytest.mark.skipif(not HAS_DEVICE_GRAPH, reason="Requires CUDA or XPU graph")
|
||||
class TestCUDAGraphWrapper:
|
||||
def setup_method(self):
|
||||
self.vllm_config = _create_vllm_config(CompilationConfig())
|
||||
@@ -415,7 +423,7 @@ def _run_and_monitor_call(
|
||||
|
||||
|
||||
@create_new_process_for_each_test("spawn")
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
@pytest.mark.skipif(not HAS_DEVICE_GRAPH, reason="Requires CUDA or XPU graph")
|
||||
def test_capture_replay_bypass_logic():
|
||||
comp_config = CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
@@ -484,7 +492,7 @@ def test_capture_replay_bypass_logic():
|
||||
|
||||
|
||||
@create_new_process_for_each_test("spawn")
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
@pytest.mark.skipif(not HAS_DEVICE_GRAPH, reason="Requires CUDA or XPU graph")
|
||||
def test_nested_wrappers():
|
||||
"""Tests a scenario with a PIECEWISE wrapper inside a FULL one."""
|
||||
comp_config = CompilationConfig(
|
||||
|
||||
@@ -132,6 +132,7 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings():
|
||||
pcp_block_tables = SimpleNamespace(
|
||||
num_kv_cache_groups=1,
|
||||
input_block_tables=(torch.zeros(num_reqs * 2, 1, dtype=torch.int32),),
|
||||
cp_interleave=1,
|
||||
)
|
||||
pcp_manager = PCPManager(
|
||||
pcp_world_size=pcp_world_size,
|
||||
@@ -174,6 +175,52 @@ def test_piecewise_capture_uses_pcp_dummy_slot_mappings():
|
||||
block_tables.get_dummy_slot_mappings.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("full_cudagraph,expected_context_len", [(True, 2), (False, 0)])
|
||||
def test_dcp_full_capture_has_kv_context(
|
||||
monkeypatch, full_cudagraph, expected_context_len
|
||||
):
|
||||
"""DCP skips context attention on the host when no request has context, so a
|
||||
FULL graph captured without context would ignore the KV cache on replay."""
|
||||
monkeypatch.setattr(
|
||||
gpu_cudagraph_utils, "prepare_dcp_local_seq_lens", lambda *a, **kw: None
|
||||
)
|
||||
num_tokens = num_reqs = 8
|
||||
buffers = InputBuffers(num_reqs, num_tokens, torch.device("cpu"))
|
||||
block_tables = MagicMock()
|
||||
block_tables.cp_size = 2
|
||||
block_tables.cp_interleave = 1
|
||||
block_tables.block_sizes = [16]
|
||||
block_tables.get_dummy_block_tables.return_value = (
|
||||
torch.zeros(num_reqs, 4, dtype=torch.int32),
|
||||
)
|
||||
block_tables.get_dummy_slot_mappings.return_value = torch.empty(
|
||||
1, num_tokens, dtype=torch.int64
|
||||
)
|
||||
model_state = MagicMock()
|
||||
model_state.max_model_len = 1024
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=8, kv_cache_tensors=[], kv_cache_groups=[]
|
||||
)
|
||||
|
||||
gpu_cudagraph_utils.prepare_inputs_to_capture(
|
||||
num_reqs,
|
||||
num_tokens,
|
||||
model_state,
|
||||
buffers,
|
||||
block_tables,
|
||||
[],
|
||||
kv_cache_config,
|
||||
full_cudagraph=full_cudagraph,
|
||||
max_query_len=1,
|
||||
)
|
||||
|
||||
batch = model_state.prepare_attn.call_args.args[0]
|
||||
context_lens = batch.seq_lens_cpu_upper_bound[:num_reqs] - torch.from_numpy(
|
||||
batch.num_scheduled_tokens[:num_reqs]
|
||||
)
|
||||
assert (context_lens == expected_context_len).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"full_cudagraph,max_query_len,expected_max_query_len",
|
||||
[(False, None, None), (True, None, 8), (True, 1, 1), (True, 3, 3)],
|
||||
|
||||
@@ -19,6 +19,13 @@ if current_platform.is_rocm():
|
||||
("TritonAttn", "FULL", True),
|
||||
("TritonAttn", "FULL_AND_PIECEWISE", True),
|
||||
]
|
||||
elif current_platform.is_xpu():
|
||||
combo_cases_1 = [
|
||||
("FA2", "FULL", True),
|
||||
("FA2", "FULL_AND_PIECEWISE", True),
|
||||
("TritonAttn", "FULL", True),
|
||||
("TritonAttn", "FULL_AND_PIECEWISE", True),
|
||||
]
|
||||
else:
|
||||
combo_cases_1 = [
|
||||
("FA3", "FULL", True),
|
||||
|
||||
@@ -32,6 +32,15 @@ from vllm.v1.worker.encoder_cudagraph_defs import (
|
||||
EncoderItemSpec,
|
||||
)
|
||||
|
||||
HAS_DEVICE_GRAPH = current_platform.is_cuda_alike() or current_platform.is_xpu()
|
||||
|
||||
if current_platform.is_xpu():
|
||||
from vllm.v1.worker.xpu_model_runner import _torch_cuda_wrapper
|
||||
|
||||
# Route torch.cuda graph/stream APIs to torch.xpu, as XPUModelRunner does.
|
||||
with _torch_cuda_wrapper():
|
||||
pass
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -662,12 +671,10 @@ def _make_video_mm_kwargs(
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
|
||||
)
|
||||
@pytest.mark.skipif(not HAS_DEVICE_GRAPH, reason="Requires CUDA, ROCm or XPU graph")
|
||||
class TestEncoderCudaGraphCaptureReplay:
|
||||
def setup_method(self):
|
||||
self.device = torch.device("cuda:0")
|
||||
self.device = torch.device(current_platform.device_type, 0)
|
||||
self.dtype = torch.float16
|
||||
self.model = SimpleMockViTModel().to(self.device).half()
|
||||
self.mgr = _make_manager_for_gpu(
|
||||
@@ -758,9 +765,7 @@ class TestEncoderCudaGraphCaptureReplay:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
|
||||
)
|
||||
@pytest.mark.skipif(not HAS_DEVICE_GRAPH, reason="Requires CUDA, ROCm or XPU graph")
|
||||
@pytest.mark.usefixtures("dist_init", "workspace_init")
|
||||
@pytest.mark.parametrize("profile_only", [False, True])
|
||||
@torch.inference_mode()
|
||||
@@ -776,7 +781,7 @@ def test_eonly_capture_preserves_outputs_across_replay_and_fallback(profile_only
|
||||
from vllm.v1.worker.mm_encoder_model_runner import MMEncoderModelRunner
|
||||
from vllm.v1.worker.workspace import lock_workspace
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
device = torch.device(current_platform.device_type, 0)
|
||||
dtype = torch.float16
|
||||
model = SimpleMockViTModel().to(device).half()
|
||||
manager = _make_manager_for_gpu(model, _BUDGETS, _MAX_BATCH, device, dtype)
|
||||
@@ -1047,12 +1052,10 @@ _VIDEO_MAX_BATCH = 4
|
||||
_VIDEO_MAX_FRAMES = 8 # 2 frames per item at max_batch_size=4
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Skip if not cuda or rocm"
|
||||
)
|
||||
@pytest.mark.skipif(not HAS_DEVICE_GRAPH, reason="Requires CUDA, ROCm or XPU graph")
|
||||
class TestEncoderCudaGraphVideoReplay:
|
||||
def setup_method(self):
|
||||
self.device = torch.device("cuda:0")
|
||||
self.device = torch.device(current_platform.device_type, 0)
|
||||
self.dtype = torch.float16
|
||||
self.model = SimpleMockViTVideoModel().to(self.device).half()
|
||||
self.mgr = _make_manager_for_gpu(
|
||||
|
||||
@@ -180,6 +180,7 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
|
||||
llm = LLM(
|
||||
model=TEST_MODEL,
|
||||
tensor_parallel_size=tp_size,
|
||||
disable_custom_all_reduce=os.getenv("VLLM_TEST_DISABLE_CUSTOM_AR") == "1",
|
||||
max_num_seqs=128,
|
||||
max_model_len=8192,
|
||||
dtype="auto", # not everything is supported
|
||||
|
||||
@@ -9,7 +9,7 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from utils import _extract_step_logprobs, skip_if_not_cuda
|
||||
from utils import _extract_step_logprobs, skip_if_not_cuda_alike
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.inputs import TextPrompt
|
||||
@@ -89,7 +89,7 @@ def _assert_batch_invariant(llm: LLM, inputs: list[TextPrompt], sampling) -> Non
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("input_type", ["image", "video"])
|
||||
@pytest.mark.parametrize("mm_encoder_attn_backend", ["FLASH_ATTN", "TORCH_SDPA"])
|
||||
def test_vlm_batch_invariance_bs1_vs_bsN(
|
||||
@@ -101,7 +101,7 @@ def test_vlm_batch_invariance_bs1_vs_bsN(
|
||||
_run_vlm_batch_invariance(input_type, mm_encoder_attn_backend, enforce_eager=True)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
def test_vlm_batch_invariance_default_execution(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
|
||||
_run_vlm_batch_invariance("image", "FLASH_ATTN", enforce_eager=False)
|
||||
|
||||
@@ -19,8 +19,10 @@ from tests.kernels.moe.utils import make_dummy_moe_config, make_test_quant_confi
|
||||
from tests.kernels.quantization.nvfp4_utils import get_nvfp4_global_scale
|
||||
from tests.utils import TestFP8Layer, requires_fp8
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
|
||||
from vllm.model_executor.kernels.linear import fp8_scaled_mm_is_batch_invariant
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
@@ -31,8 +33,11 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsFp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kFp8StaticTokenSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
@@ -87,16 +92,42 @@ _NVFP4_MOE_BATCH_INVARIANT_CASES = (
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup_cuda():
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("CUTLASS FP8 kernels require CUDA.")
|
||||
if not current_platform.is_cuda_alike():
|
||||
pytest.skip("Requires CUDA or ROCm.")
|
||||
torch.set_default_device("cuda")
|
||||
|
||||
|
||||
@requires_fp8
|
||||
@pytest.mark.parametrize(
|
||||
"kernel",
|
||||
[
|
||||
pytest.param(
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUTLASS requires CUDA."
|
||||
),
|
||||
),
|
||||
TritonFP8ScaledMMLinearKernel,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"act_key,weight_key",
|
||||
[
|
||||
(kFp8DynamicTokenSym, kFp8StaticTensorSym),
|
||||
(kFp8DynamicTokenSym, kFp8StaticChannelSym),
|
||||
(kFp8DynamicTokenSym, kFp8StaticTokenSym),
|
||||
(kFp8StaticTensorSym, kFp8StaticTensorSym),
|
||||
# An amax over the whole batch: the GEMM is invariant, the layer is not.
|
||||
(kFp8DynamicTensorSym, kFp8StaticTensorSym),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("weight_shape", [(1024, 2048), (4608, 4096)])
|
||||
@pytest.mark.parametrize("batch_size", [1, 16, 17, 32, 64, 65, 256, 257])
|
||||
@torch.inference_mode()
|
||||
def test_cutlass_fp8_batch_invariant_fixed_config(
|
||||
def test_fp8_scaled_mm_batch_invariant_fixed_config(
|
||||
kernel: type,
|
||||
act_key,
|
||||
weight_key,
|
||||
weight_shape: tuple[int, int],
|
||||
batch_size: int,
|
||||
default_vllm_config,
|
||||
@@ -108,14 +139,16 @@ def test_cutlass_fp8_batch_invariant_fixed_config(
|
||||
torch.manual_seed(0)
|
||||
layer = TestFP8Layer(
|
||||
weight_shape=weight_shape,
|
||||
activation_quant_key=kFp8DynamicTokenSym,
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
activation_quant_key=act_key,
|
||||
weight_quant_key=weight_key,
|
||||
input_dtype=torch.bfloat16,
|
||||
out_dtype=torch.bfloat16,
|
||||
device=torch.device("cuda"),
|
||||
force_kernel=CutlassFP8ScaledMMLinearKernel,
|
||||
force_kernel=kernel,
|
||||
)
|
||||
assert isinstance(layer.kernel, CutlassFP8ScaledMMLinearKernel)
|
||||
assert isinstance(layer.kernel, kernel)
|
||||
dynamic_per_tensor = act_key == kFp8DynamicTensorSym
|
||||
assert fp8_scaled_mm_is_batch_invariant(layer.kernel) != dynamic_per_tensor
|
||||
|
||||
in_features = weight_shape[1]
|
||||
needle = torch.randn((1, in_features), device="cuda", dtype=torch.bfloat16)
|
||||
@@ -124,6 +157,12 @@ def test_cutlass_fp8_batch_invariant_fixed_config(
|
||||
filler = torch.randn(
|
||||
(max(batch_size - 1, 0), in_features), device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
if dynamic_per_tensor:
|
||||
if batch_size == 1:
|
||||
pytest.skip("no other rows to move the scale")
|
||||
front_output = layer(torch.cat([needle, 100 * filler], dim=0))[0]
|
||||
assert not torch.equal(front_output, baseline)
|
||||
return
|
||||
|
||||
front_batch = torch.cat([needle, filler], dim=0)
|
||||
back_batch = torch.cat([filler, needle], dim=0)
|
||||
@@ -135,6 +174,69 @@ def test_cutlass_fp8_batch_invariant_fixed_config(
|
||||
torch.testing.assert_close(back_output, baseline, rtol=0, atol=0)
|
||||
|
||||
|
||||
@requires_fp8
|
||||
@pytest.mark.parametrize("scheme", ["fp8_per_tensor", "fp8_per_channel"])
|
||||
@torch.inference_mode()
|
||||
def test_online_fp8_linear_batch_invariant(
|
||||
scheme, monkeypatch, default_vllm_config, dist_init
|
||||
):
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config.quantization import resolve_quantization_config
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
|
||||
default_vllm_config.model_config = ModelConfig(dtype="bfloat16")
|
||||
quant_config = OnlineQuantizationConfig(resolve_quantization_config(scheme, None))
|
||||
set_random_seed(0)
|
||||
with set_default_torch_dtype(torch.bfloat16), torch.device("cuda"):
|
||||
layer = ReplicatedLinear(
|
||||
1024, 768, bias=False, quant_config=quant_config, prefix="mlp.down_proj"
|
||||
)
|
||||
layer.weight.weight_loader(layer.weight, torch.randn(768, 1024))
|
||||
|
||||
needle = torch.randn(1, 1024)
|
||||
filler = 100 * torch.randn(63, 1024)
|
||||
baseline = layer(needle)[0]
|
||||
assert torch.equal(layer(torch.cat([needle, filler]))[0][:1], baseline)
|
||||
assert torch.equal(layer(torch.cat([filler, needle]))[0][-1:], baseline)
|
||||
|
||||
|
||||
@requires_fp8
|
||||
@torch.inference_mode()
|
||||
def test_block_fp8_batch_invariant_across_tuned_block_size_k(monkeypatch):
|
||||
"""The block-FP8 kernel applies the block scales once per K tile, so a tuned
|
||||
BLOCK_SIZE_K below the scale group size reorders the sum."""
|
||||
from vllm.model_executor.layers.quantization.utils import fp8_utils
|
||||
|
||||
base = dict(
|
||||
BLOCK_SIZE_M=64, BLOCK_SIZE_N=128, GROUP_SIZE_M=1, num_warps=4, num_stages=2
|
||||
)
|
||||
table = {256: {**base, "BLOCK_SIZE_K": 128}, 512: {**base, "BLOCK_SIZE_K": 64}}
|
||||
monkeypatch.setattr(fp8_utils, "get_w8a8_block_fp8_configs", lambda *_: table)
|
||||
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
|
||||
N, K, block = 12288, 2048, [128, 128]
|
||||
|
||||
set_random_seed(0)
|
||||
fp8 = current_platform.fp8_dtype()
|
||||
A = torch.randn(512, K).to(fp8)
|
||||
B = torch.randn(N, K).to(fp8)
|
||||
As = torch.rand(512, K // 128) / 100
|
||||
Bs = torch.rand(N // 128, K // 128) / 100
|
||||
|
||||
def run(m):
|
||||
return fp8_utils.w8a8_triton_block_scaled_mm(
|
||||
A[:m], B, As[:m], Bs, block, torch.bfloat16
|
||||
)[:256]
|
||||
|
||||
assert torch.equal(run(256), run(512))
|
||||
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", False)
|
||||
assert not torch.equal(run(256), run(512))
|
||||
|
||||
|
||||
@_NVFP4_REQUIRES_SM100
|
||||
@pytest.mark.parametrize("dtype", _NVFP4_LINEAR_DTYPES)
|
||||
@pytest.mark.parametrize("shape", _NVFP4_LINEAR_CONSISTENCY_SHAPES)
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from utils import skip_if_not_cuda
|
||||
from utils import skip_if_not_cuda_alike
|
||||
|
||||
import vllm.lora.ops.triton_ops.lora_shrink_op as shrink_op
|
||||
import vllm.lora.ops.triton_ops.utils as lora_utils
|
||||
@@ -39,7 +39,7 @@ def _shrink(inputs, weights, mapping, scaling):
|
||||
return out
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_size,rank,nslices,dtype",
|
||||
[
|
||||
@@ -68,7 +68,7 @@ def test_lora_shrink_matches_reference(hidden_size, rank, nslices, dtype):
|
||||
torch.testing.assert_close(out, ref, rtol=5e-3, atol=5e-3)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("nslices", [1, 3])
|
||||
def test_lora_shrink_invariant_across_batch_sizes(monkeypatch, nslices):
|
||||
reduce_kernel, reduce_grids = shrink_op._lora_shrink_reduce_kernel, []
|
||||
|
||||
@@ -10,7 +10,11 @@ import pytest
|
||||
import torch
|
||||
from utils import skip_unsupported
|
||||
|
||||
from vllm.model_executor.determinism.batch_invariant import matmul_batch_invariant
|
||||
import vllm.model_executor.determinism.batch_invariant as bi
|
||||
from vllm.model_executor.determinism.batch_invariant import (
|
||||
enable_batch_invariant_mode,
|
||||
matmul_batch_invariant,
|
||||
)
|
||||
from vllm.model_executor.determinism.batch_invariant_configs import (
|
||||
_BATCH_INVARIANT_MATMUL_TUNED_CONFIGS,
|
||||
_get_tuned_matmul_arch_family,
|
||||
@@ -129,3 +133,49 @@ def test_matmul_batch_invariance_across_tuned_m_buckets(m, transpose_b):
|
||||
batch_output = matmul_batch_invariant(a, b)
|
||||
|
||||
assert torch.equal(single_output[0], batch_output[0])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def batch_invariant_mode():
|
||||
# The overrides are process-global: remove them so later tests compare
|
||||
# against the stock aten kernels again.
|
||||
enable_batch_invariant_mode()
|
||||
yield
|
||||
assert bi._batch_invariant_LIB is not None
|
||||
bi._batch_invariant_LIB._destroy()
|
||||
bi._batch_invariant_MODE = False
|
||||
|
||||
|
||||
@skip_unsupported
|
||||
@pytest.mark.usefixtures("batch_invariant_mode")
|
||||
@pytest.mark.parametrize("op", ["mm", "addmm", "bmm"])
|
||||
def test_compiled_matmul_reaches_batch_invariant_out_overloads(op):
|
||||
"""Inductor lowers mm/addmm/bmm to ``extern_kernels.<op>(..., out=buf)``,
|
||||
i.e. the ``.out`` overload, which eager never dispatches."""
|
||||
if (
|
||||
op != "bmm"
|
||||
and current_platform.is_cuda()
|
||||
and not current_platform.is_device_capability_family(80)
|
||||
):
|
||||
pytest.skip("mm/addmm are not overridden on SM90+")
|
||||
torch.manual_seed(0)
|
||||
dtype, k, n, needle = torch.bfloat16, 1024, 768, 37
|
||||
w = torch.randn(4, k, n, device=DEVICE_TYPE, dtype=dtype)
|
||||
bias = torch.randn(n, device=DEVICE_TYPE, dtype=dtype)
|
||||
fn = {
|
||||
"mm": lambda x: torch.mm(x[0], w[0]),
|
||||
# Bare addmm: a following pointwise op would make Inductor unfuse the bias.
|
||||
"addmm": lambda x: torch.addmm(bias, x[0], w[0]),
|
||||
"bmm": lambda x: torch.bmm(x, w),
|
||||
}[op]
|
||||
compiled = torch.compile(fn, fullgraph=True)
|
||||
x = torch.randn(4, 256, k, device=DEVICE_TYPE, dtype=dtype)
|
||||
row = None
|
||||
for m in (64, 256, 128):
|
||||
xs = x[:, :m].clone()
|
||||
# With dynamic=True, a specialization on M would silently recompile.
|
||||
torch._dynamo.mark_dynamic(xs, 1)
|
||||
out = compiled(xs)
|
||||
assert torch.equal(out, fn(xs)), f"compiled {op} bypassed the BI kernel (M={m})"
|
||||
row = out[..., needle, :] if row is None else row
|
||||
assert torch.equal(out[..., needle, :], row)
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ROCm native MX linear kernels must not depend on the number of rows."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from utils import requires_mx
|
||||
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
SEED = 0
|
||||
|
||||
# E8M0 exponent half-width. With 4-bit operands and a narrow spread the fp32
|
||||
# partial sums are exact, so a reordering of them would be invisible.
|
||||
SCALE_SPREAD = 15
|
||||
|
||||
# (N, K, use_asm_gemm)
|
||||
MXFP4_CASES = [
|
||||
(4096, 2048, False),
|
||||
(2048, 6144, False),
|
||||
(4608, 7168, False),
|
||||
(8192, 2048, True),
|
||||
(8192, 1024, True),
|
||||
(8192, 14336, True),
|
||||
]
|
||||
|
||||
MXFP8_CASES = [(4096, 2048), (2048, 6144), (1024, 768), (1536, 1024)]
|
||||
|
||||
# The untuned AITER MXFP4 configs switch at M = 9/33/65/129/257/513.
|
||||
MXFP4_TOKEN_COUNTS = [
|
||||
1,
|
||||
2,
|
||||
8,
|
||||
9,
|
||||
16,
|
||||
32,
|
||||
33,
|
||||
64,
|
||||
65,
|
||||
128,
|
||||
129,
|
||||
256,
|
||||
257,
|
||||
512,
|
||||
513,
|
||||
1024,
|
||||
2048,
|
||||
]
|
||||
|
||||
MXFP8_TOKEN_COUNTS = [1, 32, 64, 65, 128, 129, 256, 257, 512, 1024, 1025, 2048]
|
||||
|
||||
# Row 0 lands in the first tile of every decomposition, so it can stay
|
||||
# invariant when the rest of the output does not.
|
||||
CHECK_ROWS = [0, 1, 2, 3, 7, 15, 31]
|
||||
|
||||
|
||||
def _probe(out: torch.Tensor) -> torch.Tensor:
|
||||
"""The leading rows of an output, kept so the sweep does not hold every GEMM."""
|
||||
return out[: CHECK_ROWS[-1] + 1].clone()
|
||||
|
||||
|
||||
def _variant_rows(probes: dict[int, torch.Tensor]) -> list[str]:
|
||||
"""Rows whose contents changed with the number of rows in the launch."""
|
||||
failures = []
|
||||
for row in CHECK_ROWS:
|
||||
counts = [n for n in sorted(probes) if n > row]
|
||||
if not counts:
|
||||
continue
|
||||
reference = probes[counts[0]][row]
|
||||
variant = [n for n in counts if not torch.equal(probes[n][row], reference)]
|
||||
if variant:
|
||||
failures.append(f"row {row} changed at row counts {variant}")
|
||||
return failures
|
||||
|
||||
|
||||
def _mxfp4_weights(n: int, k: int, use_asm_gemm: bool):
|
||||
"""Weights laid out as AiterMxfp4LinearKernel.process_weights_after_loading."""
|
||||
weight = torch.randint(0, 255, (n, k // 2), dtype=torch.uint8, device="cuda")
|
||||
weight_scale = torch.randint(
|
||||
127 - SCALE_SPREAD,
|
||||
128 + SCALE_SPREAD,
|
||||
(n, k // 32),
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
if not use_asm_gemm:
|
||||
return weight, weight_scale.T.contiguous()
|
||||
|
||||
from aiter.ops.shuffle import shuffle_weight
|
||||
|
||||
sm, sn = weight_scale.shape
|
||||
weight_scale = (
|
||||
weight_scale.view(sm // 32, 2, 16, sn // 8, 2, 4, 1)
|
||||
.permute(0, 3, 5, 2, 4, 1, 6)
|
||||
.contiguous()
|
||||
.view(sm, sn)
|
||||
)
|
||||
return shuffle_weight(weight, layout=(16, 16)), weight_scale
|
||||
|
||||
|
||||
def _mxfp8_weights(n: int, k: int):
|
||||
"""E4M3 weights with E8M0 block scales, as the MXFP8 layer stores them."""
|
||||
weight = (torch.randn(n, k, device="cuda") / 8).to(torch.float8_e4m3fn)
|
||||
weight_scale = torch.randint(
|
||||
127 - SCALE_SPREAD,
|
||||
128 + SCALE_SPREAD,
|
||||
(n, k // 32),
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
return weight, weight_scale
|
||||
|
||||
|
||||
@requires_mx
|
||||
@pytest.mark.parametrize("n,k,use_asm_gemm", MXFP4_CASES)
|
||||
def test_mxfp4_linear_is_batch_invariant(n: int, k: int, use_asm_gemm: bool):
|
||||
pytest.importorskip("aiter")
|
||||
|
||||
# Importing the module registers torch.ops.vllm.gemm_with_dynamic_quant.
|
||||
import vllm.model_executor.kernels.linear.mxfp4.aiter # noqa: F401
|
||||
|
||||
set_random_seed(SEED)
|
||||
weight, weight_scale = _mxfp4_weights(n, k, use_asm_gemm)
|
||||
x = torch.randn(max(MXFP4_TOKEN_COUNTS), k, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
probes = {
|
||||
num_tokens: _probe(
|
||||
torch.ops.vllm.gemm_with_dynamic_quant(
|
||||
x[:num_tokens], weight, weight_scale, use_asm_gemm, torch.bfloat16
|
||||
)
|
||||
)
|
||||
for num_tokens in MXFP4_TOKEN_COUNTS
|
||||
}
|
||||
|
||||
failures = _variant_rows(probes)
|
||||
assert not failures, (
|
||||
f"MXFP4 linear depends on the row count "
|
||||
f"(N={n}, K={k}, asm={use_asm_gemm}):\n " + "\n ".join(failures)
|
||||
)
|
||||
|
||||
|
||||
@requires_mx
|
||||
@pytest.mark.parametrize("n,k", MXFP8_CASES)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
def test_mxfp8_linear_is_batch_invariant(n: int, k: int, dtype: torch.dtype):
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
_mxfp8_dot_scaled_linear,
|
||||
)
|
||||
|
||||
set_random_seed(SEED)
|
||||
weight, weight_scale = _mxfp8_weights(n, k)
|
||||
x = torch.randn(max(MXFP8_TOKEN_COUNTS), k, device="cuda", dtype=dtype)
|
||||
|
||||
probes = {
|
||||
num_tokens: _probe(
|
||||
_mxfp8_dot_scaled_linear(x[:num_tokens], weight, weight_scale)
|
||||
)
|
||||
for num_tokens in MXFP8_TOKEN_COUNTS
|
||||
}
|
||||
|
||||
failures = _variant_rows(probes)
|
||||
assert not failures, (
|
||||
f"MXFP8 linear depends on the row count (N={n}, K={k}, {dtype}):\n "
|
||||
+ "\n ".join(failures)
|
||||
)
|
||||
@@ -9,10 +9,14 @@ import torch
|
||||
from utils import (
|
||||
_extract_step_logprobs,
|
||||
_random_prompt,
|
||||
skip_if_not_cuda,
|
||||
skip_if_not_cuda_alike,
|
||||
)
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# FLASH_ATTN is CUDA-only.
|
||||
E2E_BACKENDS = ["TRITON_ATTN"] if current_platform.is_rocm() else ["FLASH_ATTN"]
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not hasattr(torch, "float8_e4m3fn"),
|
||||
@@ -40,8 +44,8 @@ def _make_llm(max_num_seqs: int, backend: str) -> LLM:
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@pytest.mark.parametrize("backend", ["FLASH_ATTN"])
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("backend", E2E_BACKENDS)
|
||||
def test_dense_nvfp4_generation_is_deterministic_across_batch_sizes_e2e(backend):
|
||||
seed = int(os.getenv("VLLM_TEST_SEED", "12345"))
|
||||
random.seed(seed)
|
||||
|
||||
@@ -6,6 +6,7 @@ vLLM server and compare BS=1 vs BS=N results (tokens and per-step logprobs).
|
||||
Environment variables:
|
||||
- VLLM_TEST_MODEL: served model name (e.g., Qwen/Qwen3-1.7B / DeepSeek-R1)
|
||||
- VLLM_TP_SIZE: tensor parallelism size (e.g., 4)
|
||||
- VLLM_TEST_SERVER_ARGS: extra server args (e.g., "--data-parallel-size 4")
|
||||
|
||||
"""
|
||||
|
||||
@@ -16,7 +17,7 @@ from typing import Any
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
from utils import BACKENDS, TEST_MODEL, _random_prompt, skip_if_not_cuda
|
||||
from utils import BACKENDS, TEST_MODEL, _random_prompt, skip_if_not_cuda_alike
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
@@ -132,7 +133,7 @@ def _compare_bs1_vs_bsn_single_process(
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("backend", BACKENDS)
|
||||
def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
|
||||
backend: str,
|
||||
@@ -156,6 +157,7 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
|
||||
]
|
||||
if tp_size:
|
||||
server_args += ["-tp", tp_size]
|
||||
server_args += os.getenv("VLLM_TEST_SERVER_ARGS", "").split()
|
||||
|
||||
with RemoteOpenAIServer(TEST_MODEL, server_args) as server:
|
||||
client = server.get_client()
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from utils import skip_if_not_cuda, skip_unsupported
|
||||
from utils import skip_if_not_cuda_alike, skip_unsupported
|
||||
|
||||
from vllm.model_executor.determinism.batch_invariant import (
|
||||
rms_norm_batch_invariant,
|
||||
@@ -27,7 +27,7 @@ def _rms_norm_reference(
|
||||
return (output * weight.float()).to(input_tensor.dtype)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("batch_size", [1, 4, 64, 300])
|
||||
@pytest.mark.parametrize("hidden_size", [512, 2048, 4096, 8192])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@@ -76,7 +76,7 @@ def test_rms_norm_batch_invariant_vs_reference(
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("hidden_size", [512, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("eps", [1e-6])
|
||||
@@ -189,7 +189,7 @@ def _assert_rows_bit_identical(small, large, msg):
|
||||
torch.testing.assert_close(small, large, rtol=0.0, atol=0.0, msg=msg)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("hidden_size", [512, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("seed", list(range(4)))
|
||||
@@ -221,7 +221,7 @@ def test_rms_norm_batch_invariant_nonresidual_kernel(
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("hidden_size", [512, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("seed", list(range(4)))
|
||||
@@ -267,7 +267,7 @@ def test_rms_norm_static_fp8_quant_batch_invariant(
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("hidden_size", [512, 4096])
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("seed", list(range(4)))
|
||||
@@ -303,7 +303,7 @@ def test_rms_norm_per_block_quant_batch_invariant(
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("batch_size", [1, 16, 128])
|
||||
@pytest.mark.parametrize("seq_len", [1, 32, 512])
|
||||
@pytest.mark.parametrize("hidden_size", [2048, 4096])
|
||||
@@ -343,7 +343,7 @@ def test_rms_norm_3d_input(
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
def test_rms_norm_numerical_stability(default_vllm_config):
|
||||
"""Test RMS norm numerical stability with extreme values.
|
||||
|
||||
@@ -431,7 +431,7 @@ def test_rms_norm_formula(default_vllm_config):
|
||||
)
|
||||
|
||||
|
||||
@skip_if_not_cuda
|
||||
@skip_if_not_cuda_alike
|
||||
@pytest.mark.parametrize("hidden_size", [128, 1024, 4096, 16384])
|
||||
def test_rms_norm_different_hidden_sizes(default_vllm_config, hidden_size: int):
|
||||
"""Test RMS norm with various hidden sizes to ensure block size handling.
|
||||
|
||||
@@ -34,12 +34,16 @@ DEVICE_BACKENDS: dict[str, DeviceConfig] = {
|
||||
available=current_platform.is_xpu() and HAS_TRITON,
|
||||
backends=["TRITON_ATTN"],
|
||||
),
|
||||
"rocm": DeviceConfig(
|
||||
available=current_platform.is_rocm() and HAS_TRITON,
|
||||
backends=["TRITON_ATTN"],
|
||||
),
|
||||
}
|
||||
|
||||
DEFAULT_MODEL = "Qwen/Qwen3-1.7B"
|
||||
TEST_MODEL = os.getenv("VLLM_TEST_MODEL", DEFAULT_MODEL)
|
||||
|
||||
# Override backends for MLA models (MLA only supported on CUDA).
|
||||
# Override backends for MLA models.
|
||||
if os.getenv("VLLM_TEST_MODEL"):
|
||||
config = get_config(TEST_MODEL, trust_remote_code=False)
|
||||
if ModelArchConfigConvertorBase(config, config.get_text_config()).is_deepseek_mla():
|
||||
@@ -52,6 +56,10 @@ if os.getenv("VLLM_TEST_MODEL"):
|
||||
available=DEVICE_BACKENDS["xpu"].available,
|
||||
backends=[],
|
||||
)
|
||||
DEVICE_BACKENDS["rocm"] = DeviceConfig(
|
||||
available=DEVICE_BACKENDS["rocm"].available,
|
||||
backends=["TRITON_MLA"],
|
||||
)
|
||||
|
||||
# Only include backends for devices that are actually available.
|
||||
BACKENDS: list[str] = sorted(
|
||||
@@ -60,7 +68,7 @@ BACKENDS: list[str] = sorted(
|
||||
|
||||
skip_unsupported = pytest.mark.skipif(
|
||||
not any(cfg.available for cfg in DEVICE_BACKENDS.values()),
|
||||
reason="Requires CUDA >= Ampere (SM80) or Intel XPU with Triton",
|
||||
reason="Requires CUDA >= Ampere (SM80), ROCm, or Intel XPU with Triton",
|
||||
)
|
||||
|
||||
skip_if_not_cuda = pytest.mark.skipif(
|
||||
@@ -68,6 +76,16 @@ skip_if_not_cuda = pytest.mark.skipif(
|
||||
reason="Requires CUDA >= Ampere (SM80)",
|
||||
)
|
||||
|
||||
skip_if_not_cuda_alike = pytest.mark.skipif(
|
||||
not (DEVICE_BACKENDS["cuda"].available or DEVICE_BACKENDS["rocm"].available),
|
||||
reason="Requires CUDA >= Ampere (SM80) or ROCm",
|
||||
)
|
||||
|
||||
requires_mx = pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and current_platform.supports_mx()),
|
||||
reason="requires a ROCm device with native MX support (gfx95x)",
|
||||
)
|
||||
|
||||
|
||||
def _random_prompt(min_words: int = 1024, max_words: int = 1024 * 2) -> str:
|
||||
# Generate more realistic prompts that will actually produce varied tokens
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ....utils import create_new_process_for_each_test
|
||||
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"Cascade attention backends FLASH_ATTN and FLASHINFER are notsupported on ROCm",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
@pytest.mark.parametrize("attn_backend", ["FLASH_ATTN", "FLASHINFER"])
|
||||
def test_cascade_attention(example_system_message, attn_backend):
|
||||
prompt = "\n<User>: Implement fibonacci sequence in Python.\n<Claude>:"
|
||||
|
||||
if attn_backend == "FLASHINFER":
|
||||
pytest.skip(
|
||||
"This test is failing with FlashInfer backend and "
|
||||
"needs investigation. See issue #25679."
|
||||
)
|
||||
|
||||
llm = LLM(
|
||||
model="Qwen/Qwen2-1.5B-Instruct", attention_config={"backend": attn_backend}
|
||||
)
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
|
||||
|
||||
# No cascade attention.
|
||||
single_prompt = [example_system_message + prompt]
|
||||
responses = llm.generate(single_prompt, sampling_params)
|
||||
ref_output = responses[0].outputs[0].text
|
||||
|
||||
# (Probably) Use cascade attention.
|
||||
prompts = [example_system_message + prompt] * 64
|
||||
responses = llm.generate(prompts, sampling_params)
|
||||
for response in responses:
|
||||
assert response.outputs[0].text == ref_output
|
||||
@@ -83,13 +83,20 @@ def _get_hisparse_worker(runner: VllmRunner) -> HiSparseConnectorWorker:
|
||||
not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"with_offloading", [False, True], ids=["standalone", "offload"]
|
||||
"with_offloading,attention_backend",
|
||||
[
|
||||
pytest.param(False, None, id="standalone"),
|
||||
pytest.param(True, None, id="offload"),
|
||||
# Offloading does not touch the attention kernels: FA4 runs standalone.
|
||||
pytest.param(False, "FLASH_ATTN_MLA_SPARSE_FA4", id="standalone-fa4"),
|
||||
],
|
||||
)
|
||||
@fork_new_process_for_each_test
|
||||
def test_hisparse_spill_and_prefix_restore(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
vllm_runner: type[VllmRunner],
|
||||
with_offloading: bool,
|
||||
attention_backend: str | None,
|
||||
):
|
||||
"""Spilled prefixes restore and FULL-graph decode writes reach host KV.
|
||||
|
||||
@@ -101,6 +108,8 @@ def test_hisparse_spill_and_prefix_restore(
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability.major < 9:
|
||||
pytest.skip("Sparse MLA requires Hopper or newer")
|
||||
if attention_backend and not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("FA4 sparse MLA requires SM 10.x")
|
||||
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip")
|
||||
@@ -142,6 +151,7 @@ def test_hisparse_spill_and_prefix_restore(
|
||||
)
|
||||
),
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
attention_backend=attention_backend,
|
||||
block_size=64,
|
||||
max_model_len=320,
|
||||
max_num_batched_tokens=1024,
|
||||
|
||||
@@ -74,6 +74,12 @@ def test_kv_sharing_fast_prefill(
|
||||
"ROCm: torch.compile produces incorrect output for gemma-3n's GELU "
|
||||
"with tanh approximation. Use enforce_eager=True instead."
|
||||
)
|
||||
if not enforce_eager and not use_v2_model_runner and current_platform.is_cuda():
|
||||
pytest.skip(
|
||||
"cublasCreate fails with CUBLAS_STATUS_ALLOC_FAILED during memory "
|
||||
"profiling with torch 2.14: "
|
||||
"https://github.com/pytorch/pytorch/issues/199944"
|
||||
)
|
||||
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
|
||||
compilation_config = CompilationConfig(
|
||||
|
||||
@@ -16,7 +16,10 @@ from tests.utils import create_new_process_for_each_test
|
||||
from vllm import LLM, SamplingParams, TokensPrompt
|
||||
from vllm.config import CacheConfig
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
is_conv_state_dim_first,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -59,6 +62,10 @@ MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8"
|
||||
# where block boundaries fall. XPU rounds GDN block sizes up to a multiple of 64
|
||||
# (XPUPlatform.update_block_size_for_backend), so use one.
|
||||
BLOCK_SIZE = 576
|
||||
# The step actions also encode the split at the prompt's last block boundary,
|
||||
# which a GDN prefill backend with internal checkpoints (FlashInfer) skips; see
|
||||
# get_internal_checkpoint_step_configs.
|
||||
GDN_PREFILL_CONFIG = {"gdn_prefill_backend": "triton"}
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
NUM_HIDDEN_LAYERS = 1
|
||||
cur_step_action_idx = 0
|
||||
@@ -129,6 +136,10 @@ def get_fake_propose_draft_token_ids_fn():
|
||||
first_token_id_index = (
|
||||
num_computed_tokens + 1
|
||||
) # bonus token isn't considered as computed
|
||||
# num_accepted_tokens_cpu is filled by a non-blocking D2H copy in the
|
||||
# same step; wait for it or a lagging GPU yields stale counts.
|
||||
if self.num_accepted_tokens_event is not None:
|
||||
self.num_accepted_tokens_event.synchronize()
|
||||
first_token_id_index += self.input_batch.num_accepted_tokens_cpu[0].item()
|
||||
proposed_draft_token_ids = [
|
||||
prompt_token_ids[
|
||||
@@ -878,6 +889,73 @@ def get_mamba_prefix_cache_step_configs(
|
||||
return tests
|
||||
|
||||
|
||||
def get_internal_checkpoint_step_configs() -> dict[str, TestConfig]:
|
||||
"""Async-scheduling steps when the GDN prefill exports the checkpoint at
|
||||
the prompt's last block boundary itself, so the final chunk runs whole.
|
||||
|
||||
The checkpoint goes in the second-to-last block the chunk reaches, keyed to
|
||||
the boundary: column 1 holds the state at B for a 2B + 10 prompt.
|
||||
"""
|
||||
B = BLOCK_SIZE
|
||||
return {
|
||||
"prompt_2_block_size": TestConfig(
|
||||
num_prompt_tokens=B * 2,
|
||||
num_generated_tokens=10,
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, B * 2, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(B * 2, 4, [1, 1, 1, 1, 1, 1], (1, 2), (-1, -1)),
|
||||
],
|
||||
),
|
||||
"prompt_2_block_size_10": TestConfig(
|
||||
num_prompt_tokens=B * 2 + 10,
|
||||
num_generated_tokens=10,
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, B * 2 + 10, [0, 1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(B * 2 + 10, 4, [0, 1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
],
|
||||
),
|
||||
"prompt_10_block_size": TestConfig(
|
||||
num_prompt_tokens=B * 10,
|
||||
num_generated_tokens=10,
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
B * 5,
|
||||
B * 5,
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
(4, 9),
|
||||
(-1, -1),
|
||||
),
|
||||
StepAction(
|
||||
B * 10,
|
||||
4,
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1],
|
||||
(9, 10),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
"prompt_10_block_size_10": TestConfig(
|
||||
num_prompt_tokens=B * 10 + 10,
|
||||
num_generated_tokens=10,
|
||||
num_accepted_tokens=4,
|
||||
step_actions=[
|
||||
StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
|
||||
StepAction(
|
||||
B * 5,
|
||||
B * 5 + 10,
|
||||
[0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1],
|
||||
(4, 10),
|
||||
(-1, -1),
|
||||
),
|
||||
],
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _run_mamba_prefix_cache_mrv1(
|
||||
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
|
||||
):
|
||||
@@ -898,7 +976,7 @@ def _run_mamba_prefix_cache_mrv1(
|
||||
load_format="dummy",
|
||||
enable_prefix_caching=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
mamba_cache_mode="align",
|
||||
additional_config=GDN_PREFILL_CONFIG,
|
||||
speculative_config={
|
||||
"method": "qwen3_next_mtp",
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
@@ -1050,8 +1128,11 @@ def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
|
||||
def _run_mamba_prefix_cache_mrv2(
|
||||
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
async_scheduling: bool,
|
||||
additional_config: dict[str, Any] = GDN_PREFILL_CONFIG,
|
||||
):
|
||||
internal_checkpoints = additional_config["gdn_prefill_backend"] == "flashinfer"
|
||||
global async_scheduling_mode
|
||||
async_scheduling_mode = async_scheduling
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
@@ -1191,8 +1272,24 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
with gpu_sync_allowed():
|
||||
start_pos = input_batch.positions[input_batch.query_start_loc[0]].item()
|
||||
assert cur_step_action.num_computed_tokens_start == start_pos
|
||||
if scheduler_output.scheduled_new_reqs and "snapshot_col" in captured:
|
||||
captured["snapshot"] = block_states(self, captured.pop("snapshot_col"))
|
||||
return ret
|
||||
|
||||
def block_states(runner: MRV2GPUModelRunner, col: int) -> list[list[torch.Tensor]]:
|
||||
# Each Mamba layer's (conv, temporal) states in block column col.
|
||||
kv_cache_config = captured["kv_cache_config"]
|
||||
forward_context = runner.vllm_config.compilation_config.static_forward_context
|
||||
return [
|
||||
[
|
||||
temporal_block(state, captured["block_tables"][group_id], col).clone()
|
||||
for state in forward_context[layer_name].kv_cache
|
||||
]
|
||||
for group_ids in get_mamba_groups(kv_cache_config).values()
|
||||
for group_id in group_ids
|
||||
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names
|
||||
]
|
||||
|
||||
def fake_sample(
|
||||
self: MRV2GPUModelRunner,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -1254,7 +1351,7 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
skip_tokenizer_init=True,
|
||||
enable_prefix_caching=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
mamba_cache_mode="align",
|
||||
additional_config=additional_config,
|
||||
speculative_config={
|
||||
"method": "qwen3_next_mtp",
|
||||
"num_speculative_tokens": num_speculative_tokens,
|
||||
@@ -1267,7 +1364,11 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
)
|
||||
|
||||
try:
|
||||
tests = get_mamba_prefix_cache_step_configs(async_scheduling)
|
||||
tests = (
|
||||
get_internal_checkpoint_step_configs()
|
||||
if internal_checkpoints
|
||||
else get_mamba_prefix_cache_step_configs(async_scheduling)
|
||||
)
|
||||
|
||||
global step_actions
|
||||
global cur_step_action_idx
|
||||
@@ -1313,6 +1414,35 @@ def _run_mamba_prefix_cache_mrv2(
|
||||
f"got events={second_events!r}"
|
||||
)
|
||||
assert engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache()
|
||||
|
||||
if internal_checkpoints:
|
||||
# The checkpoint a 2B + 10 prefill exports at B (column 1) is the
|
||||
# final state of a prefill that stops at B (column 0). Distinct
|
||||
# tokens make a shifted conv window visible.
|
||||
tokens = [(i * 7919) % 30000 + 1 for i in range(BLOCK_SIZE * 2 + 10)]
|
||||
scheduler = engine.llm_engine.engine_core.engine_core.scheduler
|
||||
states = []
|
||||
for num_tokens, col in ((BLOCK_SIZE, 0), (len(tokens), 1)):
|
||||
captured["snapshot_col"] = col
|
||||
_ = engine.generate(
|
||||
[TokensPrompt(prompt_token_ids=tokens[:num_tokens])],
|
||||
sampling_params=sampling_params,
|
||||
)
|
||||
states.append(captured.pop("snapshot"))
|
||||
assert scheduler.reset_prefix_cache()
|
||||
hf_config = engine.llm_engine.model_config.hf_text_config
|
||||
state_len = hf_config.linear_conv_kernel_dim - 1
|
||||
for (ref_conv, ref_ssm), (conv, ssm) in zip(*states):
|
||||
if not is_conv_state_dim_first():
|
||||
ref_conv, conv = ref_conv.mT, conv.mT
|
||||
# Under MTP the conv state has num_spec scratch columns after
|
||||
# the window.
|
||||
for new, ref in (
|
||||
(conv[..., :state_len], ref_conv[..., :state_len]),
|
||||
(ssm, ref_ssm),
|
||||
):
|
||||
rel_err = (new.float() - ref.float()).norm() / ref.float().norm()
|
||||
assert rel_err < 1e-2, rel_err
|
||||
finally:
|
||||
del engine
|
||||
torch.accelerator.empty_cache()
|
||||
@@ -1327,3 +1457,26 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2_async(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv2(monkeypatch, async_scheduling=True)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda()
|
||||
or not (
|
||||
current_platform.is_device_capability(90)
|
||||
or (
|
||||
(
|
||||
current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
)
|
||||
and current_platform.get_cuda_runtime_major() >= 13
|
||||
)
|
||||
),
|
||||
reason="FlashInfer GDN prefill needs SM90, or SM100/SM120 with CUDA 13.",
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
def test_mamba_prefix_cache_mrv2_async_flashinfer(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_mamba_prefix_cache_mrv2(
|
||||
monkeypatch,
|
||||
async_scheduling=True,
|
||||
additional_config={"gdn_prefill_backend": "flashinfer"},
|
||||
)
|
||||
|
||||
@@ -8,23 +8,26 @@ request is queued before the first step, so the first prefill has the same
|
||||
shape in both runs. Give the engine its own process and the comparison turns
|
||||
into a coin toss: prefill starts before the last requests arrive over IPC, so
|
||||
one boot batches two requests where the other batches one. The batch shape
|
||||
sets the GEMM reduction order, logprobs shift by up to ~0.25, and near-ties
|
||||
flip — measured at up to 5 of 16 completions, with sharding disabled on both
|
||||
sides. A real sharding bug — rows routed to the wrong request, misaligned
|
||||
gathers — produces many-nats divergence and scrambled top-k sets."""
|
||||
sets the GEMM reduction order and can make near-ties flip. A real sharding
|
||||
bug — rows routed to the wrong request, misaligned gathers — produces
|
||||
many-nats divergence and scrambled top-k sets."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.utils import multi_gpu_test
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
from vllm.distributed import (
|
||||
cleanup_dist_env_and_memory,
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from vllm.logprobs import Logprob, LogprobsOnePosition
|
||||
|
||||
MODEL = "meta-llama/Llama-3.2-1B-Instruct"
|
||||
GPT_OSS_MODEL = "trl-internal-testing/tiny-GptOssForCausalLM"
|
||||
GPT_OSS_REVISION = "e82685101216af1953707faa42567d232a5e4d45"
|
||||
|
||||
# Misrouted rows move logprobs by nats; anything under this is float noise.
|
||||
# With matched batches the two boots agreed bitwise over 320 positions, so the
|
||||
# bound is not expected to be reached.
|
||||
LOGPROB_TOL = 0.5
|
||||
|
||||
PROMPTS = [
|
||||
@@ -58,18 +61,52 @@ def _is_sharded_sampling_active(worker) -> bool:
|
||||
return worker.model_runner.batch_sharder is not None
|
||||
|
||||
|
||||
def _sampling_params() -> list[SamplingParams]:
|
||||
return [SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(len(PROMPTS))]
|
||||
def _local_logits_match_full(worker) -> dict[str, bool]:
|
||||
"""Reassemble rank-local logits and compare with the full path."""
|
||||
model = worker.model_runner.model
|
||||
hidden_size = model.config.hidden_size
|
||||
dtype = model.lm_head.weight.dtype
|
||||
hidden_states = torch.linspace(
|
||||
-1,
|
||||
1,
|
||||
steps=2 * hidden_size,
|
||||
device=worker.model_runner.device,
|
||||
dtype=dtype,
|
||||
).view(2, hidden_size)
|
||||
|
||||
full_logits = model.compute_logits(hidden_states)
|
||||
local_logits = model.compute_logits_local(hidden_states)
|
||||
assert local_logits.shape[-1] == model.lm_head.num_embeddings_per_partition
|
||||
gathered_logits = tensor_model_parallel_all_gather(local_logits)
|
||||
gathered_logits = gathered_logits[..., : model.config.vocab_size]
|
||||
|
||||
has_full_logits = full_logits is not None
|
||||
return {
|
||||
"has_full_logits": has_full_logits,
|
||||
"matches": not has_full_logits or torch.equal(full_logits, gathered_logits),
|
||||
}
|
||||
|
||||
|
||||
def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
|
||||
def _sampling_params(num_prompts: int) -> list[SamplingParams]:
|
||||
return [SAMPLING_PARAMS[i % len(SAMPLING_PARAMS)] for i in range(num_prompts)]
|
||||
|
||||
|
||||
def _generate(
|
||||
disable_sharding: bool,
|
||||
model: str,
|
||||
prompts: list[str],
|
||||
sampling_params: list[SamplingParams],
|
||||
revision: str | None,
|
||||
):
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
model=model,
|
||||
revision=revision,
|
||||
tokenizer_revision=revision,
|
||||
tensor_parallel_size=2,
|
||||
enable_batch_sharded_sampling=not disable_sharding,
|
||||
max_model_len=1024,
|
||||
# More requests than ranks, so round-robin slot ownership splits the
|
||||
# batch across both ranks.
|
||||
# Keep capacity at least as large as the TP size. The actual request
|
||||
# count may be smaller to exercise an empty rank.
|
||||
max_num_seqs=len(PROMPTS),
|
||||
enforce_eager=True,
|
||||
enable_prefix_caching=False,
|
||||
@@ -83,12 +120,14 @@ def _generate(monkeypatch: pytest.MonkeyPatch, disable_sharding: bool):
|
||||
# intended sampling mode.
|
||||
modes = llm.llm_engine.collective_rpc(_is_sharded_sampling_active)
|
||||
assert all(mode == (not disable_sharding) for mode in modes), modes
|
||||
params = _sampling_params()
|
||||
# Two waves: the second reuses the request slots freed by the first
|
||||
# (in whatever order requests finished), covering the cross-rank
|
||||
# slot-recycling path that request ownership derives from.
|
||||
wave1 = llm.generate(PROMPTS, params)
|
||||
wave2 = llm.generate(PROMPTS[::-1], params[::-1])
|
||||
if not disable_sharding:
|
||||
logits_checks = llm.llm_engine.collective_rpc(_local_logits_match_full)
|
||||
assert any(check["has_full_logits"] for check in logits_checks)
|
||||
assert all(check["matches"] for check in logits_checks)
|
||||
# The second wave reuses request slots freed by the first,
|
||||
# covering the cross-rank slot-recycling path that ownership uses.
|
||||
wave1 = llm.generate(prompts, sampling_params)
|
||||
wave2 = llm.generate(prompts[::-1], sampling_params[::-1])
|
||||
return wave1 + wave2
|
||||
finally:
|
||||
del llm
|
||||
@@ -178,25 +217,11 @@ def test_logprob_comparison_rejects_missing_top_k_rank(rank: int | None):
|
||||
_assert_logprob_dicts_close(lps, lps, "missing rank", num_logprobs=1)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Generation, logprobs, and prompt logprobs match between batch-sharded
|
||||
sampling and the replicated fallback."""
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
# Required for the collective_rpc mode check below.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
# Queue every request before the first step, so both boots schedule the
|
||||
# same batches -- see the module docstring.
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
ref_outputs = _generate(monkeypatch, disable_sharding=True)
|
||||
shard_outputs = _generate(monkeypatch, disable_sharding=False)
|
||||
|
||||
assert len(ref_outputs) == len(shard_outputs) == 2 * len(PROMPTS)
|
||||
params = _sampling_params()
|
||||
def _assert_outputs_match(ref_outputs, shard_outputs, params) -> None:
|
||||
assert len(ref_outputs) == len(shard_outputs) == len(params)
|
||||
divergent = []
|
||||
for i, (ref, out, sampling_params) in enumerate(
|
||||
zip(ref_outputs, shard_outputs, params + params[::-1])
|
||||
zip(ref_outputs, shard_outputs, params)
|
||||
):
|
||||
ref_completion = ref.outputs[0]
|
||||
out_completion = out.outputs[0]
|
||||
@@ -244,7 +269,60 @@ def test_sharded_sampling_outputs_match(monkeypatch: pytest.MonkeyPatch):
|
||||
)
|
||||
|
||||
assert not divergent, (
|
||||
f"{len(divergent)}/{2 * len(PROMPTS)} completions diverged at "
|
||||
f"{len(divergent)}/{len(params)} completions diverged at "
|
||||
f"(prompt, position) {divergent}; the sampling modes did not agree "
|
||||
"token for token with in-process scheduling"
|
||||
)
|
||||
|
||||
|
||||
def _configure_test_environment(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
# Required for the collective_rpc mode check below.
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
# Queue every request before the first step, so both boots schedule the
|
||||
# same batches -- see the module docstring.
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
("model", "revision", "prompts"),
|
||||
[
|
||||
pytest.param(MODEL, None, PROMPTS, id="llama"),
|
||||
pytest.param(MODEL, None, PROMPTS[:1], id="llama-empty-rank"),
|
||||
pytest.param(
|
||||
GPT_OSS_MODEL,
|
||||
GPT_OSS_REVISION,
|
||||
PROMPTS,
|
||||
id="gpt-oss",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sharded_sampling_outputs_match(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model: str,
|
||||
revision: str | None,
|
||||
prompts: list[str],
|
||||
):
|
||||
"""Generation, logprobs, and prompt logprobs match between batch-sharded
|
||||
sampling and the replicated fallback."""
|
||||
_configure_test_environment(monkeypatch)
|
||||
sampling_params = _sampling_params(len(prompts))
|
||||
|
||||
ref_outputs = _generate(
|
||||
disable_sharding=True,
|
||||
model=model,
|
||||
prompts=prompts,
|
||||
sampling_params=sampling_params,
|
||||
revision=revision,
|
||||
)
|
||||
shard_outputs = _generate(
|
||||
disable_sharding=False,
|
||||
model=model,
|
||||
prompts=prompts,
|
||||
sampling_params=sampling_params,
|
||||
revision=revision,
|
||||
)
|
||||
|
||||
expected_params = sampling_params + sampling_params[::-1]
|
||||
_assert_outputs_match(ref_outputs, shard_outputs, expected_params)
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from tests.evals.gsm8k.gsm8k_eval import _build_gsm8k_prompts
|
||||
from vllm import SamplingParams
|
||||
from vllm.config import CompilationConfig
|
||||
|
||||
from ...utils import compute_acceptance_rate
|
||||
|
||||
|
||||
def test_medusa_acceptance_rate(
|
||||
sampling_config: SamplingParams,
|
||||
vllm_runner,
|
||||
):
|
||||
"""Verify a trained Medusa checkpoint achieves nonzero acceptance rate.
|
||||
|
||||
Uses the canonical FasterDecoding vicuna-7b checkpoint to confirm the
|
||||
speculation path actually accepts tokens — unlike test_medusa_correctness,
|
||||
which uses a random head and only validates output correctness.
|
||||
"""
|
||||
target_model = "lmsys/vicuna-7b-v1.3"
|
||||
medusa_model = "FasterDecoding/medusa-vicuna-7b-v1.3"
|
||||
prompts = _build_gsm8k_prompts(num_questions=10, num_shots=1)[0]
|
||||
|
||||
with vllm_runner(
|
||||
target_model,
|
||||
block_size=None,
|
||||
trust_remote_code=False,
|
||||
speculative_config={
|
||||
"method": "medusa",
|
||||
"model": medusa_model,
|
||||
"num_speculative_tokens": 3,
|
||||
},
|
||||
max_model_len=1024,
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=None,
|
||||
disable_log_stats=False,
|
||||
compilation_config=CompilationConfig(),
|
||||
) as spec_runner:
|
||||
spec_runner.llm.generate(prompts, sampling_config)
|
||||
metrics = spec_runner.llm.get_metrics()
|
||||
acceptance_rate = compute_acceptance_rate(metrics)
|
||||
|
||||
min_acceptance_rate = 0.198
|
||||
print(f"Medusa acceptance rate: {acceptance_rate:.4f} (min {min_acceptance_rate})")
|
||||
|
||||
# Regression guard at 90% of the measured baseline.
|
||||
assert acceptance_rate >= min_acceptance_rate, (
|
||||
f"Medusa acceptance rate {acceptance_rate:.4f} below min {min_acceptance_rate}"
|
||||
)
|
||||
@@ -18,17 +18,18 @@ from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
from .utils import _skip_if_insufficient_gpus_for_tp, get_test_prompts, greedy_sampling
|
||||
|
||||
# Even with in-process scheduling, near-tie tokens still flip on up to 3 of 20
|
||||
# prompts in CI, greedy and seeded. A sharding bug diverges wholesale instead.
|
||||
MAX_DIVERGENT_PROMPTS = 3
|
||||
|
||||
|
||||
def _is_sharded_sampling_active(worker) -> bool:
|
||||
return worker.model_runner.batch_sharder is not None
|
||||
|
||||
|
||||
def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Batch-sharded sampling must be a bit-exact drop-in for replicated
|
||||
sampling under MTP spec decoding: the collectives move the same logits
|
||||
bytes, Gumbel keys derive from (request slot, position, seed), and slot
|
||||
assignment is rank-deterministic. Both runs here are spec decode with
|
||||
identical math, so outputs must match exactly."""
|
||||
"""Batch-sharded sampling must match replicated sampling under MTP spec
|
||||
decoding, up to measured near-tie noise."""
|
||||
tp_size = 2
|
||||
_skip_if_insufficient_gpus_for_tp(tp_size)
|
||||
model_name = "Qwen/Qwen3.5-0.8B-Base"
|
||||
@@ -96,8 +97,7 @@ def test_mtp_sharded_sampling_equivalence(monkeypatch: pytest.MonkeyPatch):
|
||||
f" replicated: {ref.outputs[0].text!r}\n"
|
||||
f" sharded: {out.outputs[0].text!r}"
|
||||
)
|
||||
assert not divergent, (
|
||||
assert len(divergent) <= MAX_DIVERGENT_PROMPTS, (
|
||||
f"{name}: {len(divergent)}/{len(ref_outputs)} prompts diverged "
|
||||
f"at {divergent}; the sampling modes did not agree token for "
|
||||
"token with in-process scheduling"
|
||||
f"at {divergent}, beyond near-tie noise"
|
||||
)
|
||||
|
||||
@@ -88,7 +88,6 @@ def test_mtp_speculative_mixed_batch_short_prefill(
|
||||
trust_remote_code=True,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=enable_prefix_caching,
|
||||
mamba_cache_mode="align" if enable_prefix_caching else "none",
|
||||
) as llm:
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
@@ -161,7 +160,6 @@ def test_qwen_mtp_mamba_prefix_cache_hit_is_bounded(vllm_runner, monkeypatch):
|
||||
enforce_eager=True,
|
||||
enable_chunked_prefill=True,
|
||||
enable_prefix_caching=True,
|
||||
mamba_cache_mode="align",
|
||||
speculative_config={"method": "mtp", "num_speculative_tokens": 2},
|
||||
) as runner:
|
||||
llm = runner.get_llm()
|
||||
|
||||
@@ -60,7 +60,6 @@ def _check_replayssm_parity(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
mamba_backend=mamba_backend,
|
||||
)
|
||||
@@ -122,32 +121,48 @@ def test_replayssm_flashinfer_decode_matches_baseline(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", FLASHINFER_MODELS)
|
||||
def test_replayssm_flashinfer_spec_decode_matches_baseline(vllm_runner, model_name):
|
||||
@pytest.mark.parametrize("use_v2_model_runner", [False, True], ids=["v1", "v2"])
|
||||
def test_replayssm_flashinfer_spec_decode_matches_baseline(
|
||||
vllm_runner, model_name, monkeypatch, use_v2_model_runner
|
||||
):
|
||||
common = dict(
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
speculative_config={
|
||||
"method": "ngram",
|
||||
"num_speculative_tokens": 3,
|
||||
"prompt_lookup_max": 3,
|
||||
},
|
||||
)
|
||||
with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm:
|
||||
baseline = llm.generate_greedy_logprobs(
|
||||
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
use_replayssm=True,
|
||||
replayssm_buffer_len=16,
|
||||
mamba_backend="flashinfer",
|
||||
**common,
|
||||
) as llm:
|
||||
replay = llm.generate_greedy_logprobs(
|
||||
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
try:
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setenv("VLLM_USE_V2_MODEL_RUNNER", str(int(use_v2_model_runner)))
|
||||
envs.disable_envs_cache()
|
||||
with vllm_runner(model_name, mamba_backend="flashinfer", **common) as llm:
|
||||
assert (
|
||||
llm.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
is use_v2_model_runner
|
||||
)
|
||||
baseline = llm.generate_greedy_logprobs(
|
||||
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
use_replayssm=True,
|
||||
replayssm_buffer_len=16,
|
||||
mamba_backend="flashinfer",
|
||||
**common,
|
||||
) as llm:
|
||||
assert (
|
||||
llm.llm.llm_engine.vllm_config.use_v2_model_runner
|
||||
is use_v2_model_runner
|
||||
)
|
||||
replay = llm.generate_greedy_logprobs(
|
||||
FLASHINFER_PROMPTS, max_tokens=32, num_logprobs=5
|
||||
)
|
||||
finally:
|
||||
envs.disable_envs_cache()
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=baseline,
|
||||
@@ -165,7 +180,6 @@ def test_replayssm_flashinfer_mtp(vllm_runner, monkeypatch, use_v2_model_runner)
|
||||
max_model_len=1024,
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=False,
|
||||
mamba_cache_mode="none",
|
||||
mamba_backend="flashinfer",
|
||||
tensor_parallel_size=2,
|
||||
disable_log_stats=False,
|
||||
@@ -233,7 +247,6 @@ def _check_replayssm_prefix_caching_parity(
|
||||
trust_remote_code=True,
|
||||
enable_prefix_caching=True,
|
||||
enable_chunked_prefill=True,
|
||||
mamba_cache_mode="align",
|
||||
disable_log_stats=False, # required for llm.get_metrics()
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
)
|
||||
|
||||
@@ -22,12 +22,16 @@ class _Pos:
|
||||
self.offset = offset
|
||||
self.length = length
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1, identifier=None):
|
||||
self.mm_hash = mm_hash
|
||||
self.identifier = identifier if identifier is not None else mm_hash
|
||||
self.mm_position = _Pos(0, length)
|
||||
self.modality = "image"
|
||||
|
||||
|
||||
class _Request:
|
||||
@@ -80,7 +84,7 @@ def _make_scheduler(
|
||||
|
||||
def _load_ids(meta) -> list[int]:
|
||||
"""Transfer ids the scheduler dispatched in this step's metadata."""
|
||||
return [transfer_id for transfer_id, _ in meta.loads.values()]
|
||||
return [transfer_id for transfer_id, _, _ in meta.loads.values()]
|
||||
|
||||
|
||||
def _load_blocks(meta, mm_hash: str) -> list[int]:
|
||||
@@ -90,7 +94,7 @@ def _load_blocks(meta, mm_hash: str) -> list[int]:
|
||||
|
||||
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
|
||||
"""Pre-populate a ready cache entry backed by real blocks."""
|
||||
s._cache.alloc(mm_hash, n_blocks)
|
||||
s._cache.alloc(mm_hash, n_blocks, (n_blocks, _BS // s._dtype.itemsize))
|
||||
s._cache.mark_ready(mm_hash)
|
||||
|
||||
|
||||
|
||||
@@ -97,10 +97,9 @@ def test_second_instance_opens_existing_file_and_shares_memory():
|
||||
r1.cleanup()
|
||||
|
||||
|
||||
def test_dense_dp_engines_get_distinct_regions(monkeypatch):
|
||||
def test_dense_dp_engines_get_distinct_regions():
|
||||
"""Dense DP engines reset data_parallel_rank to 0 but share instance_id,
|
||||
so the region must be keyed by data_parallel_index."""
|
||||
monkeypatch.setattr(common, "_get_encoder_cache_hidden_dim", lambda cfg: 8)
|
||||
instance_id = str(uuid.uuid4())
|
||||
regions = []
|
||||
try:
|
||||
@@ -113,6 +112,7 @@ def test_dense_dp_engines_get_distinct_regions(monkeypatch):
|
||||
cfg.instance_id = instance_id
|
||||
cfg.parallel_config = parallel_config
|
||||
cfg.model_config.dtype = torch.float16
|
||||
cfg.model_config.get_inputs_embeds_size.return_value = 8
|
||||
cfg.ec_transfer_config.ec_connector_extra_config = {"ec_cpu_bytes": 64}
|
||||
regions.append(common.create_ec_shared_region(cfg))
|
||||
|
||||
|
||||
@@ -91,7 +91,9 @@ def _make_region() -> ECSharedRegion:
|
||||
|
||||
|
||||
def _vllm_config(rank: int = 0) -> Mock:
|
||||
return create_ec_vllm_config(rank=rank, dtype=_DTYPE)
|
||||
config = create_ec_vllm_config(rank=rank, dtype=_DTYPE)
|
||||
config.model_config.get_inputs_embeds_size.return_value = _HIDDEN_DIM
|
||||
return config
|
||||
|
||||
|
||||
def _meta(
|
||||
@@ -105,7 +107,7 @@ def _meta(
|
||||
return ECCPUConnectorMetadata(
|
||||
saves=saves or {},
|
||||
loads={
|
||||
mm_hash: (idx, block_ids)
|
||||
mm_hash: (idx, block_ids, (len(block_ids), _HIDDEN_DIM))
|
||||
for idx, (mm_hash, block_ids) in enumerate((loads or {}).items())
|
||||
},
|
||||
)
|
||||
@@ -904,21 +906,29 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
|
||||
# Build scheduler sharing the same region.
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
|
||||
|
||||
scheduler = ECCPUScheduler(_vllm_config())
|
||||
config = _vllm_config()
|
||||
# DeepStack-wide output: each embedding spans four blocks.
|
||||
width = 4 * _HIDDEN_DIM
|
||||
config.ec_transfer_config.mm_encoder_output_widths = {"image": width}
|
||||
scheduler = ECCPUScheduler(config)
|
||||
|
||||
# -- Step 1: scheduler allocates, worker saves --
|
||||
n_blocks = 3
|
||||
src = torch.arange(
|
||||
n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device=DEVICE_TYPE
|
||||
).reshape(n_blocks, _HIDDEN_DIM)
|
||||
n_tokens = 2
|
||||
src = torch.arange(n_tokens * width, dtype=_DTYPE, device=DEVICE_TYPE).reshape(
|
||||
n_tokens, width
|
||||
)
|
||||
|
||||
class _Pos:
|
||||
offset = 0
|
||||
length = n_blocks
|
||||
length = n_tokens
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
class _Feature:
|
||||
identifier = "img_001"
|
||||
mm_position = _Pos()
|
||||
modality = "image"
|
||||
|
||||
class _Request:
|
||||
request_id = "req_e2e"
|
||||
@@ -948,8 +958,9 @@ def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
|
||||
scheduler.update_state_after_alloc(_Request(), 0)
|
||||
meta_load = scheduler.build_connector_meta(scheduler_output=None)
|
||||
assert "img_001" in meta_load.loads
|
||||
_, load_blocks = meta_load.loads["img_001"]
|
||||
_, load_blocks, shape = meta_load.loads["img_001"]
|
||||
assert load_blocks == meta_save.saves["img_001"]
|
||||
assert shape == tuple(src.shape)
|
||||
|
||||
load_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(load_cache, meta_load)
|
||||
|
||||
@@ -45,6 +45,34 @@ KEY = (
|
||||
SPEC = TensorSpec((2, 2), "torch.float32", 16)
|
||||
|
||||
|
||||
def test_push_shapes_follow_each_modality(monkeypatch):
|
||||
"""Each push is sized at its modality's measured encoder output width."""
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake import scheduler
|
||||
|
||||
monkeypatch.setattr(scheduler, "ensure_mooncake_available", lambda: None)
|
||||
widths = {"image": 8192, "audio": 2048}
|
||||
config = create_ec_vllm_config(ec_role="ec_producer", encoder_output_widths=widths)
|
||||
request = SimpleNamespace(
|
||||
request_id="mixed",
|
||||
mm_features=[SimpleNamespace(identifier=m, modality=m) for m in widths],
|
||||
get_num_encoder_embeds=lambda index: 2,
|
||||
ec_transfer_params={
|
||||
"consumer_zmq": "tcp://consumer:1234",
|
||||
"ec_items": [{"mm_hash": m, "transfer_id": m} for m in widths],
|
||||
},
|
||||
)
|
||||
instance = scheduler.ECMooncakeScheduler(config)
|
||||
try:
|
||||
for index in range(len(widths)):
|
||||
instance.update_state_after_alloc(request, index)
|
||||
metadata = instance.build_connector_meta(
|
||||
SimpleNamespace(free_encoder_mm_hashes=[], preempted_req_ids=set())
|
||||
)
|
||||
assert [spec.shape for spec in metadata.pushes] == [(2, 8192), (2, 2048)]
|
||||
finally:
|
||||
instance.close()
|
||||
|
||||
|
||||
def test_embedding_key_combines_namespace_and_identifier():
|
||||
config = create_ec_vllm_config(ec_role="ec_producer")
|
||||
config.ec_transfer_config.ec_connector_extra_config.update(
|
||||
|
||||
@@ -246,6 +246,18 @@ class TestCacheExistence:
|
||||
assert result[1] # Second exists
|
||||
assert not result[2] # Third doesn't exist
|
||||
|
||||
def test_has_cache_item_miss_creates_no_folder(
|
||||
self, mock_vllm_config_consumer, temp_storage
|
||||
):
|
||||
"""Test a cache miss leaves no empty folder in the shared storage."""
|
||||
connector = ECExampleConnector(
|
||||
vllm_config=mock_vllm_config_consumer,
|
||||
role=ECConnectorRole.SCHEDULER,
|
||||
)
|
||||
|
||||
assert not connector.has_cache_item("missing_hash")
|
||||
assert os.listdir(temp_storage) == []
|
||||
|
||||
|
||||
class TestStateManagement:
|
||||
"""Test connector state management."""
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Startup measurement of the encoder output width EC producers size by."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
|
||||
from vllm.distributed.ec_transfer.ec_connector import encoder_output_width as eow
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
class _DeepStackModel:
|
||||
"""Emits each modality at its own width, as Qwen3-Omni does."""
|
||||
|
||||
widths = {"image": 4 * 16, "audio": 16}
|
||||
|
||||
def embed_multimodal(self, *, modality):
|
||||
return [torch.zeros(3, self.widths[modality])]
|
||||
|
||||
|
||||
class _Budget:
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.mm_max_toks_per_item = dict.fromkeys(_DeepStackModel.widths, 8)
|
||||
|
||||
def get_dummy_encoder_profile_inputs(self, modality, max_items_per_batch):
|
||||
assert max_items_per_batch == 1
|
||||
return [(modality, None)]
|
||||
|
||||
|
||||
def _worker(first_pp_rank=True, supports_mm=True):
|
||||
runner = SimpleNamespace(
|
||||
supports_mm_inputs=supports_mm,
|
||||
get_model=_DeepStackModel,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
pp_group = SimpleNamespace(is_first_rank=first_pp_rank)
|
||||
return SimpleNamespace(model_runner=runner, vllm_config=None), pp_group
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def patched(monkeypatch):
|
||||
import vllm.distributed.parallel_state as ps
|
||||
import vllm.multimodal.encoder_budget as budget_mod
|
||||
import vllm.multimodal.utils as mm_utils
|
||||
|
||||
monkeypatch.setattr(budget_mod, "MultiModalBudget", _Budget)
|
||||
monkeypatch.setattr(
|
||||
mm_utils,
|
||||
"group_and_batch_mm_kwargs",
|
||||
lambda items, **_: iter([(items[0][0], 1, {"modality": items[0][0]})]),
|
||||
)
|
||||
return lambda group: monkeypatch.setattr(ps, "get_pp_group", lambda: group)
|
||||
|
||||
|
||||
def test_worker_reports_the_encoder_output_width_per_modality(patched):
|
||||
"""The width comes from the tensor the encoder emits, not from config."""
|
||||
worker, group = _worker()
|
||||
patched(group)
|
||||
assert eow._measure_on_worker(worker) == {"image": 64, "audio": 16}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_pp_rank, supports_mm", [(False, True), (True, False)])
|
||||
def test_ranks_without_an_encoder_report_nothing(patched, first_pp_rank, supports_mm):
|
||||
worker, group = _worker(first_pp_rank, supports_mm)
|
||||
patched(group)
|
||||
assert eow._measure_on_worker(worker) == {}
|
||||
|
||||
|
||||
def test_producer_records_the_widths_its_encoder_ranks_agree_on():
|
||||
config = create_ec_vllm_config(ec_role="ec_producer", encoder_output_widths={})
|
||||
reports = [{"image": 64}, {}, {"image": 64}]
|
||||
eow.measure_encoder_output_widths(config, lambda fn: reports)
|
||||
assert config.ec_transfer_config.mm_encoder_output_widths == {"image": 64}
|
||||
assert eow.get_encoder_output_width(config, "image") == 64
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"reports, match",
|
||||
[
|
||||
([{}, {}], "could not measure"),
|
||||
([{"image": 64}, {"image": 16}], "disagree"),
|
||||
],
|
||||
ids=["no-encoder", "disagree"],
|
||||
)
|
||||
def test_producer_refuses_to_start_without_a_trustworthy_width(reports, match):
|
||||
config = create_ec_vllm_config(ec_role="ec_producer", encoder_output_widths={})
|
||||
with pytest.raises(ValueError, match=match):
|
||||
eow.measure_encoder_output_widths(config, lambda fn: reports)
|
||||
@@ -25,6 +25,9 @@ class _Pos:
|
||||
def __init__(self, offset, length):
|
||||
self.offset, self.length = offset, length
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1, data=None):
|
||||
@@ -85,6 +88,9 @@ class _FakeTransport:
|
||||
pass
|
||||
|
||||
|
||||
_MISSING = object()
|
||||
|
||||
|
||||
def _consumer_sched(monkeypatch):
|
||||
def _region(cfg):
|
||||
return ECSharedRegion(
|
||||
@@ -95,16 +101,15 @@ def _consumer_sched(monkeypatch):
|
||||
|
||||
# ec_enable_nixl defaults to False, so this builds gate-off; the fields
|
||||
# below flip on the NIXL consumer state directly.
|
||||
s = ECCPUScheduler(create_ec_vllm_config(ec_role="ec_consumer"))
|
||||
# No measured widths: a consumer takes each shape from the announcement.
|
||||
s = ECCPUScheduler(
|
||||
create_ec_vllm_config(ec_role="ec_consumer", encoder_output_widths={})
|
||||
)
|
||||
# Turn on NIXL consumer state without constructing real transports.
|
||||
s._nixl_enabled = True
|
||||
s._transport = _FakeTransport()
|
||||
s._data = None
|
||||
s._compat_hash = "c"
|
||||
# _setup_nixl normally computes these from model_config; set them
|
||||
# directly since this helper builds gate-off then flips fields on.
|
||||
s._hidden_dim = _HID
|
||||
s._element_size = _ES
|
||||
s._metadata_resolver._cache["image"] = {"image_grid_thw"}
|
||||
return s
|
||||
|
||||
@@ -115,24 +120,32 @@ def _params(mm_hash, length):
|
||||
"peer_host": "h",
|
||||
"peer_port": 1,
|
||||
"size_bytes": length * _HID * _ES,
|
||||
"shape": [length, _HID],
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def test_new_remote_read_defers_then_completes(monkeypatch):
|
||||
"""The read is sized from the announced shape, not a local width."""
|
||||
width = 4 * _HID
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
# Route _start_xfer to our fake session instead of real ZMQ/NIXL, but
|
||||
# still reserve a real not-ready cache entry so mark_ready works.
|
||||
def _fake_start(mm_hash, info, size):
|
||||
entry = s._cache.alloc(mm_hash, 1)
|
||||
def _fake_start(mm_hash, info, size, shape):
|
||||
assert size == width * _ES
|
||||
assert shape == (1, width)
|
||||
entry = s._cache.alloc(mm_hash, size // _BS, shape)
|
||||
assert entry is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", _fake_start)
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
params = _params("h1", 1)
|
||||
params["h1"]["size_bytes"] = width * _ES
|
||||
params["h1"]["shape"] = [1, width]
|
||||
req = _Request([_Feature("h1", 1)], params=params)
|
||||
|
||||
# Step 1: unseen remote item -> read started, request deferred.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
@@ -155,6 +168,7 @@ def test_new_remote_read_defers_then_completes(monkeypatch):
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta.loads
|
||||
assert meta.loads["h1"][2] == (1, width)
|
||||
assert "h1" not in s._step_completed # cleared by promote
|
||||
|
||||
# Step 3: still cached -> admitted directly, no new transfer.
|
||||
@@ -176,8 +190,8 @@ def test_failed_read_falls_back_only_with_local_input(monkeypatch, payload):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
def _fake_start(mm_hash, info, size):
|
||||
entry = s._cache.alloc(mm_hash, 1)
|
||||
def _fake_start(mm_hash, info, size, shape):
|
||||
entry = s._cache.alloc(mm_hash, 1, shape)
|
||||
assert entry is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
@@ -230,8 +244,8 @@ def test_remote_wait_budget_survives_retries_and_long_steps(monkeypatch, retry):
|
||||
fake = _FakeSession()
|
||||
s._sessions[("h", 1)] = fake
|
||||
|
||||
def start(mm_hash, info, size):
|
||||
assert s._cache.alloc(mm_hash, 1) is not None
|
||||
def start(mm_hash, info, size, shape):
|
||||
assert s._cache.alloc(mm_hash, 1, shape) is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
|
||||
@@ -261,8 +275,8 @@ def test_retryable_read_re_requests_without_admitting(monkeypatch):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
def _fake_start(mm_hash, info, size):
|
||||
entry = s._cache.alloc(mm_hash, 1)
|
||||
def _fake_start(mm_hash, info, size, shape):
|
||||
entry = s._cache.alloc(mm_hash, 1, shape)
|
||||
assert entry is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
@@ -287,14 +301,29 @@ def test_retryable_read_re_requests_without_admitting(monkeypatch):
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"size_fields",
|
||||
[{"size_bytes": 999}, {"size_bytes": None}, {"size_bytes": "big"}, {}],
|
||||
ids=["mismatch", "null-size", "text-size", "no-size"],
|
||||
"fields",
|
||||
[
|
||||
{"size_bytes": 999},
|
||||
{"size_bytes": None},
|
||||
{"size_bytes": "big"},
|
||||
{"size_bytes": _MISSING},
|
||||
{"shape": _MISSING},
|
||||
{"shape": [2, _HID]},
|
||||
],
|
||||
ids=[
|
||||
"size-mismatch",
|
||||
"null-size",
|
||||
"text-size",
|
||||
"no-size",
|
||||
"no-shape",
|
||||
"wrong-rows",
|
||||
],
|
||||
)
|
||||
def test_invalid_remote_size_fails_the_request(monkeypatch, size_fields):
|
||||
"""Invalid remote sizes fail the request without raising in the scheduler."""
|
||||
def test_invalid_announcement_fails_the_request(monkeypatch, fields):
|
||||
"""An unusable announced size or shape fails the request, not the engine."""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
announced = {"peer_host": "h", "peer_port": 1, **size_fields}
|
||||
announced = {**_params("h1", 1)["h1"], **fields}
|
||||
announced = {k: v for k, v in announced.items() if v is not _MISSING}
|
||||
req = _Request([_Feature("h1", 1)], params={"h1": announced})
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert s.take_unavailable_requests() == {"r1"}
|
||||
@@ -306,7 +335,7 @@ def test_invalid_remote_size_fails_the_request(monkeypatch, size_fields):
|
||||
def test_deferral_budget_is_per_request(monkeypatch):
|
||||
"""Shared hashes have independent wait budgets and cancellation cleanup."""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
|
||||
monkeypatch.setattr(s._cache, "alloc", lambda key, n, shape=None: None)
|
||||
params = _params("h1", 1)
|
||||
old = _Request([_Feature("h1", 1)], params=params, req_id="old")
|
||||
new = _Request([_Feature("h1", 1)], params=params, req_id="new")
|
||||
@@ -419,7 +448,7 @@ def test_in_flight_hash_defers_without_second_transfer(monkeypatch):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
started: list[str] = []
|
||||
|
||||
def _spy_start_xfer(mm_hash, info, size):
|
||||
def _spy_start_xfer(mm_hash, info, size, shape):
|
||||
started.append(mm_hash)
|
||||
return True
|
||||
|
||||
|
||||
@@ -16,6 +16,9 @@ class _Pos:
|
||||
def __init__(self, offset, length):
|
||||
self.offset, self.length = offset, length
|
||||
|
||||
def get_num_embeds(self):
|
||||
return self.length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1):
|
||||
@@ -53,11 +56,8 @@ def test_request_finished_producer_emits_params(monkeypatch):
|
||||
# Simulate NIXL-enabled producer bookkeeping without building real NIXL.
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
# _setup_nixl normally computes these from model_config; set them
|
||||
# directly since this test builds gate-off then flips fields on.
|
||||
s._hidden_dim, s._element_size = 32, 2
|
||||
# feature length=2, hidden_dim=32, element_size=2 -> 128 bytes -> 2 blocks.
|
||||
entry = s._cache.alloc("h1", 2)
|
||||
entry = s._cache.alloc("h1", 2, (2, 32))
|
||||
assert entry is not None
|
||||
s._cache.mark_ready("h1")
|
||||
|
||||
@@ -71,6 +71,7 @@ def test_request_finished_producer_emits_params(monkeypatch):
|
||||
"peer_host": "1.2.3.4",
|
||||
"peer_port": 5601,
|
||||
"size_bytes": 2 * 32 * 2,
|
||||
"shape": [2, 32],
|
||||
}
|
||||
}
|
||||
s.shutdown()
|
||||
@@ -84,8 +85,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
s._hidden_dim, s._element_size = 32, 2
|
||||
s._cache.alloc("h1", 2) # allocated but not marked ready
|
||||
s._cache.alloc("h1", 2, (2, 32)) # allocated but not marked ready
|
||||
|
||||
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
|
||||
assert delay is False
|
||||
@@ -96,6 +96,7 @@ def test_request_finished_announces_not_ready_entry(monkeypatch):
|
||||
"peer_host": "1.2.3.4",
|
||||
"peer_port": 5601,
|
||||
"size_bytes": 2 * 32 * 2,
|
||||
"shape": [2, 32],
|
||||
}
|
||||
}
|
||||
s.shutdown()
|
||||
@@ -105,7 +106,6 @@ def test_request_finished_skips_unallocated_entry(monkeypatch):
|
||||
s = _sched_gate_off(monkeypatch)
|
||||
s._nixl_enabled = True
|
||||
s._peer_host, s._peer_port = "1.2.3.4", 5601
|
||||
s._hidden_dim, s._element_size = 32, 2
|
||||
# No alloc() for "h1" at all — e.g. the cache was full at save time.
|
||||
|
||||
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
|
||||
|
||||
@@ -18,12 +18,16 @@ _EXECUTOR_BACKEND = "mp"
|
||||
_PCP_SIZE = 1
|
||||
|
||||
|
||||
_EMBED_WIDTH = 32
|
||||
|
||||
|
||||
def create_ec_vllm_config(
|
||||
*,
|
||||
ec_role: ECRole = "ec_both",
|
||||
tensor_parallel_size: int = 1,
|
||||
rank: int = 0,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
encoder_output_widths: dict[str, int] | None = None,
|
||||
) -> Mock:
|
||||
"""Build a `VllmConfig` stand-in for EC connector unit tests.
|
||||
|
||||
@@ -31,10 +35,8 @@ def create_ec_vllm_config(
|
||||
role derivation and the rank arithmetic the connector depends on behave as
|
||||
they do in production.
|
||||
|
||||
`model_config` is a stub because nothing under test needs a real one: the
|
||||
connector reads only `dtype` here, and the paths that inspect the model
|
||||
itself are not exercised. Building a real `ModelConfig` would resolve an HF
|
||||
model and inspect its architecture.
|
||||
`model_config` supplies a dtype and embedding width without resolving an
|
||||
HF model or inspecting its architecture.
|
||||
|
||||
Args:
|
||||
ec_role: EC role this instance plays. `is_ec_producer` and
|
||||
@@ -42,6 +44,8 @@ def create_ec_vllm_config(
|
||||
tensor_parallel_size: TP degree.
|
||||
rank: Global rank of this worker.
|
||||
dtype: Encoder cache dtype.
|
||||
encoder_output_widths: Per-modality encoder output width, as a
|
||||
producer measures it at startup. Defaults to the embedding width.
|
||||
|
||||
Returns:
|
||||
A `VllmConfig`-specced mock carrying the configs above.
|
||||
@@ -59,9 +63,15 @@ def create_ec_vllm_config(
|
||||
ec_role=ec_role,
|
||||
engine_id=str(uuid.uuid4()),
|
||||
)
|
||||
ec_transfer_config.mm_encoder_output_widths = (
|
||||
dict.fromkeys(("image", "video", "audio"), _EMBED_WIDTH)
|
||||
if encoder_output_widths is None
|
||||
else encoder_output_widths
|
||||
)
|
||||
|
||||
model_config = Mock(spec=ModelConfig)
|
||||
model_config.dtype = dtype
|
||||
model_config.get_inputs_embeds_size.return_value = _EMBED_WIDTH
|
||||
|
||||
vllm_config = Mock(spec=VllmConfig)
|
||||
vllm_config.ec_transfer_config = ec_transfer_config
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
These tests cover:
|
||||
- OutputProcessor.get_num_queued_tokens() token counting
|
||||
- AsyncLLM.check_admission() admission control logic
|
||||
- Admission rejection Prometheus samples
|
||||
- Exception classes (GracefulHTTPError, QueueOverflowError, MaxQueuedTokensError)
|
||||
- create_error_response() mapping GracefulHTTPError to HTTP 503
|
||||
- SchedulerConfig field defaults and validation
|
||||
@@ -14,6 +15,7 @@ These tests cover:
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import logging
|
||||
import multiprocessing
|
||||
from collections.abc import Awaitable, Callable
|
||||
from http import HTTPStatus
|
||||
@@ -21,6 +23,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from prometheus_client import REGISTRY
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.config.scheduler import SchedulerConfig
|
||||
@@ -40,6 +43,10 @@ from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.admission_control import SharedAdmissionStats
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
from vllm.v1.engine.output_processor import OutputProcessor
|
||||
from vllm.v1.metrics.loggers import StatLoggerManager
|
||||
|
||||
_ADMISSION_METRIC_MODEL = "admission-metric-test"
|
||||
_ADMISSION_LOGGER = "vllm.v1.engine.async_llm"
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
@@ -70,10 +77,43 @@ def _make_async_llm(
|
||||
llm.output_processor = MagicMock()
|
||||
llm.output_processor.get_num_unfinished_requests.return_value = num_unfinished
|
||||
llm.output_processor.get_num_queued_tokens.return_value = num_queued_tokens
|
||||
llm.logger_manager = None
|
||||
llm.admission_stats = None
|
||||
return llm
|
||||
|
||||
|
||||
def _attach_prometheus_logger(
|
||||
llm: AsyncLLM, engine_indexes: list[int] | None = None
|
||||
) -> None:
|
||||
"""Attach a real Prometheus logger manager to a lightweight AsyncLLM."""
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.observability_config.show_hidden_metrics = False
|
||||
vllm_config.observability_config.kv_cache_metrics = False
|
||||
vllm_config.observability_config.custom_histogram_buckets = None
|
||||
vllm_config.model_config.served_model_name = _ADMISSION_METRIC_MODEL
|
||||
vllm_config.model_config.max_model_len = 2048
|
||||
vllm_config.model_config.is_diffusion = False
|
||||
vllm_config.speculative_config = None
|
||||
vllm_config.kv_transfer_config = None
|
||||
vllm_config.ec_transfer_config = None
|
||||
vllm_config.lora_config = None
|
||||
llm.logger_manager = StatLoggerManager(
|
||||
vllm_config=vllm_config,
|
||||
engine_idxs=engine_indexes,
|
||||
enable_default_loggers=False,
|
||||
)
|
||||
|
||||
|
||||
def _admission_rejection_count(reason: str) -> float:
|
||||
"""Read the Prometheus sample for one admission rejection reason."""
|
||||
value = REGISTRY.get_sample_value(
|
||||
"vllm:admission_rejections_total",
|
||||
{"model_name": _ADMISSION_METRIC_MODEL, "reason": reason},
|
||||
)
|
||||
assert value is not None
|
||||
return value
|
||||
|
||||
|
||||
def _make_output_processor(**request_states) -> OutputProcessor:
|
||||
op = OutputProcessor.__new__(OutputProcessor)
|
||||
op.request_states = request_states
|
||||
@@ -157,11 +197,13 @@ def test_shared_request_snapshot_enforces_global_limit():
|
||||
server_0.set_num_requests(2)
|
||||
llm = _make_async_llm(max_num_queued_reqs=2)
|
||||
llm.admission_stats = server_1
|
||||
_attach_prometheus_logger(llm)
|
||||
|
||||
with pytest.raises(QueueOverflowError):
|
||||
llm.check_admission()
|
||||
|
||||
assert server_0.get_num_requests() == 2
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -247,19 +289,35 @@ def test_admission_no_limits_allows_everything():
|
||||
|
||||
def test_admission_reqs_allows_when_under_limit():
|
||||
llm = _make_async_llm(max_num_queued_reqs=10, num_unfinished=5)
|
||||
_attach_prometheus_logger(llm)
|
||||
llm.check_admission()
|
||||
llm.check_admission()
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 0
|
||||
assert _admission_rejection_count("max_num_queued_tokens") == 0
|
||||
|
||||
|
||||
def test_admission_reqs_rejects_at_limit():
|
||||
def test_admission_reqs_rejects_at_limit(caplog):
|
||||
llm = _make_async_llm(max_num_queued_reqs=10, num_unfinished=10)
|
||||
with pytest.raises(QueueOverflowError):
|
||||
_attach_prometheus_logger(llm, engine_indexes=[0, 1, 2])
|
||||
with (
|
||||
caplog.at_level(logging.INFO, logger=_ADMISSION_LOGGER),
|
||||
pytest.raises(QueueOverflowError),
|
||||
):
|
||||
llm.check_admission()
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 1
|
||||
assert _admission_rejection_count("max_num_queued_tokens") == 0
|
||||
assert not any(
|
||||
"Request queue full" in record.getMessage() for record in caplog.records
|
||||
)
|
||||
|
||||
|
||||
def test_admission_reqs_rejects_with_n():
|
||||
llm = _make_async_llm(max_num_queued_reqs=10, num_unfinished=8)
|
||||
_attach_prometheus_logger(llm)
|
||||
with pytest.raises(QueueOverflowError):
|
||||
llm.check_admission(3)
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 1
|
||||
assert _admission_rejection_count("max_num_queued_tokens") == 0
|
||||
|
||||
|
||||
def test_admission_reqs_allows_n_at_boundary():
|
||||
@@ -283,8 +341,11 @@ def test_admission_tokens_allows_when_under_limit():
|
||||
|
||||
def test_admission_tokens_rejects_at_limit():
|
||||
llm = _make_async_llm(max_num_queued_tokens=1000, num_queued_tokens=1000)
|
||||
_attach_prometheus_logger(llm)
|
||||
with pytest.raises(MaxQueuedTokensError):
|
||||
llm.check_admission()
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 0
|
||||
assert _admission_rejection_count("max_num_queued_tokens") == 1
|
||||
|
||||
|
||||
def test_admission_tokens_rejects_over_limit():
|
||||
@@ -320,8 +381,11 @@ def test_admission_req_limit_checked_before_token_limit():
|
||||
num_unfinished=10,
|
||||
num_queued_tokens=1000,
|
||||
)
|
||||
_attach_prometheus_logger(llm)
|
||||
with pytest.raises(QueueOverflowError):
|
||||
llm.check_admission()
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 1
|
||||
assert _admission_rejection_count("max_num_queued_tokens") == 0
|
||||
|
||||
|
||||
# -- n derived from params at the add_request call site ---------------------
|
||||
@@ -381,6 +445,17 @@ async def test_concurrent_single_request_admission_respects_limit():
|
||||
assert len(request_states) == 1
|
||||
|
||||
|
||||
def test_admission_preflight_pass_then_reject_records_once():
|
||||
llm = _make_async_llm(max_num_queued_reqs=1, num_unfinished=0)
|
||||
_attach_prometheus_logger(llm)
|
||||
llm.check_admission()
|
||||
llm.output_processor.get_num_unfinished_requests.return_value = 1
|
||||
with pytest.raises(QueueOverflowError):
|
||||
llm.check_admission()
|
||||
assert _admission_rejection_count("max_num_queued_reqs") == 1
|
||||
assert _admission_rejection_count("max_num_queued_tokens") == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("first_n", [1, 3])
|
||||
async def test_parallel_admission_is_all_or_nothing(first_n: int):
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.core.kv_cache_utils import check_enough_kv_cache_memory
|
||||
from vllm.v1.core.kv_cache_planning import check_enough_kv_cache_memory
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ def test_kv_cache_oom_insufficient_memory(monkeypatch):
|
||||
config.parallel_config.decode_context_parallel_size = 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.core.kv_cache_utils.max_memory_usage_bytes",
|
||||
"vllm.v1.core.kv_cache_planning.max_memory_usage_bytes",
|
||||
lambda c, s: 100 * 1024**3, # 100 GiB
|
||||
)
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ import time
|
||||
from unittest.mock import MagicMock, Mock
|
||||
|
||||
import numpy as np
|
||||
import prometheus_client
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -17,6 +18,7 @@ from tests.v1.engine.utils import (
|
||||
MockEngineCore,
|
||||
)
|
||||
from vllm import PoolingParams
|
||||
from vllm.config.watermarking import WatermarkConfig
|
||||
from vllm.logprobs import FlatLogprobs, Logprob, PromptLogprobs, SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import (
|
||||
@@ -24,6 +26,7 @@ from vllm.outputs import (
|
||||
PoolingRequestOutput,
|
||||
RequestOutput,
|
||||
SamplingMask,
|
||||
WeightVersionSpan,
|
||||
)
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
@@ -40,6 +43,9 @@ from vllm.v1.engine.output_processor import (
|
||||
RequestOutputCollector,
|
||||
RequestState,
|
||||
)
|
||||
from vllm.v1.engine.parallel_sampling import ParentRequest
|
||||
from vllm.v1.metrics.loggers import PrometheusStatLogger
|
||||
from vllm.v1.metrics.prometheus import unregister_vllm_metrics
|
||||
from vllm.v1.metrics.stats import IterationStats, PrefillStats, SchedulerStats
|
||||
from vllm.v1.outputs import SamplingMaskLists
|
||||
|
||||
@@ -87,6 +93,7 @@ def test_completion_output_preserves_each_sampling_mask_position() -> None:
|
||||
]
|
||||
state.routed_experts_chunks = []
|
||||
state.spec_decode_metrics = None
|
||||
state.weight_version_starts = []
|
||||
|
||||
output = state._new_completion_output([1, 2, 3], FinishReason.LENGTH, None)
|
||||
|
||||
@@ -1167,6 +1174,95 @@ def test_iteration_stats(dummy_test_vectors):
|
||||
assert iteration_stats.num_generation_tokens == num_active
|
||||
|
||||
|
||||
def test_watermark_status_partitions_finished_requests(dummy_test_vectors):
|
||||
"""Each finished request gets exactly one watermark status."""
|
||||
output_processor = OutputProcessor(dummy_test_vectors.tokenizer, log_stats=True)
|
||||
prompt_token_ids = dummy_test_vectors.prompt_tokens[0]
|
||||
|
||||
def make_request(
|
||||
request_id: str,
|
||||
sampling_params: SamplingParams | None = None,
|
||||
pooling_params: PoolingParams | None = None,
|
||||
) -> EngineCoreRequest:
|
||||
return EngineCoreRequest(
|
||||
request_id=request_id,
|
||||
external_req_id=f"{request_id}-ext",
|
||||
prompt_token_ids=prompt_token_ids,
|
||||
mm_features=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=pooling_params,
|
||||
)
|
||||
|
||||
# The frontend resolves the flags before the request reaches the
|
||||
# OutputProcessor; set them as it does.
|
||||
def resolved(watermarking: bool, skipped: bool, n: int = 1) -> SamplingParams:
|
||||
params = SamplingParams(n=n, watermarking=watermarking)
|
||||
params._watermarking_skipped = skipped
|
||||
return params
|
||||
|
||||
finish_reasons = {
|
||||
"wm": FinishReason.ABORT,
|
||||
"skip": FinishReason.STOP,
|
||||
# A re-resolved request can carry both flags; it is watermarked.
|
||||
"sticky": FinishReason.STOP,
|
||||
"off": FinishReason.ERROR,
|
||||
"pool": FinishReason.STOP,
|
||||
}
|
||||
output_processor.add_request(make_request("wm", resolved(True, False)), None)
|
||||
output_processor.add_request(make_request("skip", resolved(False, True)), None)
|
||||
output_processor.add_request(make_request("sticky", resolved(True, True)), None)
|
||||
output_processor.add_request(make_request("off", resolved(False, False)), None)
|
||||
output_processor.add_request(
|
||||
make_request("pool", pooling_params=PoolingParams(task="embed")), None
|
||||
)
|
||||
parent_req = ParentRequest(make_request("parent", resolved(True, False, n=2)))
|
||||
for idx in range(2):
|
||||
child_id, child_params = parent_req.get_child_info(idx)
|
||||
output_processor.add_request(
|
||||
make_request(child_id, child_params), None, parent_req, idx
|
||||
)
|
||||
finish_reasons[child_id] = FinishReason.LENGTH
|
||||
|
||||
outputs = [
|
||||
EngineCoreOutput(
|
||||
request_id=request_id,
|
||||
new_token_ids=[] if request_id == "pool" else [prompt_token_ids[0]],
|
||||
pooling_output=torch.tensor([1.0]) if request_id == "pool" else None,
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
for request_id, finish_reason in finish_reasons.items()
|
||||
]
|
||||
iteration_stats = IterationStats()
|
||||
output_processor.process_outputs(outputs, time.monotonic(), iteration_stats)
|
||||
|
||||
vllm_config = dummy_test_vectors.vllm_config
|
||||
vllm_config.watermark_config = WatermarkConfig(key=42)
|
||||
try:
|
||||
PrometheusStatLogger(vllm_config).record(
|
||||
scheduler_stats=None, iteration_stats=iteration_stats
|
||||
)
|
||||
totals: dict[str, dict[str, float]] = {}
|
||||
for metric in prometheus_client.REGISTRY.collect():
|
||||
for sample in metric.samples:
|
||||
if sample.name in (
|
||||
"vllm:watermark_requests_total",
|
||||
"vllm:request_success_total",
|
||||
):
|
||||
label = sample.labels.get("status", "all")
|
||||
by_label = totals.setdefault(sample.name, {})
|
||||
by_label[label] = by_label.get(label, 0) + sample.value
|
||||
finally:
|
||||
unregister_vllm_metrics()
|
||||
|
||||
status = totals["vllm:watermark_requests_total"]
|
||||
assert status == {"watermarked": 4, "skipped": 1, "not_watermarked": 2}
|
||||
assert sum(status.values()) == totals["vllm:request_success_total"]["all"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("log_stats", [True, False])
|
||||
def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
|
||||
"""Test LoRA request lifecycle tracking through waiting -> running -> finished."""
|
||||
@@ -1596,6 +1692,7 @@ def test_sampling_masks_follow_output_kind(output_kind):
|
||||
state.sampling_mask_chunks = []
|
||||
state.routed_experts_chunks = []
|
||||
state.spec_decode_metrics = None
|
||||
state.weight_version_starts = []
|
||||
|
||||
supports = [[10, 11], [20], [30, 31]]
|
||||
masks = []
|
||||
@@ -1725,3 +1822,53 @@ def test_async_pooling_cache_miss_does_not_fail_output_processor():
|
||||
assert output.error is not None
|
||||
assert output.error.retryable
|
||||
assert not output_processor.has_unfinished_requests()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"output_kind", [RequestOutputKind.DELTA, RequestOutputKind.FINAL_ONLY]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"steps,expected",
|
||||
[
|
||||
# Merge adjacent equal labels; A -> B -> A remains three spans.
|
||||
(
|
||||
[("v7", [10, 11]), ("v7", [12]), ("v8", [13, 14]), ("v7", [15])],
|
||||
[("v7", 0, 3), ("v8", 3, 5), ("v7", 5, 6)],
|
||||
),
|
||||
# An output without tokens has no spans.
|
||||
([], []),
|
||||
],
|
||||
)
|
||||
def test_weight_version_spans(output_kind: RequestOutputKind, steps, expected):
|
||||
output_processor = OutputProcessor(tokenizer=None, log_stats=False)
|
||||
queue = RequestOutputCollector(output_kind, request_id="request-0-int")
|
||||
request = EngineCoreRequest(
|
||||
request_id="request-0-int",
|
||||
external_req_id="request-0",
|
||||
prompt_token_ids=[1, 2, 3],
|
||||
mm_features=None,
|
||||
arrival_time=0,
|
||||
lora_request=None,
|
||||
cache_salt=None,
|
||||
data_parallel_rank=None,
|
||||
sampling_params=SamplingParams(detokenize=False, output_kind=output_kind),
|
||||
pooling_params=None,
|
||||
)
|
||||
output_processor.add_request(request, prompt=None, queue=queue)
|
||||
|
||||
for i, (weight_version, token_ids) in enumerate(steps):
|
||||
# Copy: merging deltas extends token lists in place.
|
||||
output = EngineCoreOutput(
|
||||
request_id="request-0-int", new_token_ids=list(token_ids)
|
||||
)
|
||||
output_processor.process_outputs([output], weight_version=weight_version)
|
||||
if i == 0 and output_kind == RequestOutputKind.DELTA:
|
||||
intermediate = queue.get_nowait()
|
||||
assert intermediate is not None
|
||||
assert intermediate.outputs[0].weight_versions is None
|
||||
|
||||
# Unread deltas merge into the abort output without losing spans.
|
||||
output_processor.abort_requests(["request-0"], internal=False)
|
||||
final = queue.get_nowait()
|
||||
assert final is not None and final.finished
|
||||
assert final.outputs[0].weight_versions == [WeightVersionSpan(*s) for s in expected]
|
||||
|
||||
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.aux_output_connector.connector import (
|
||||
AuxRequestOutput,
|
||||
)
|
||||
from vllm.v1.executor.ray_utils import detach_zero_copy_from_model_runner_output
|
||||
from vllm.v1.outputs import (
|
||||
LogprobsLists,
|
||||
LogprobsTensors,
|
||||
ModelRunnerOutput,
|
||||
)
|
||||
|
||||
|
||||
def _make_readonly(arr: np.ndarray) -> np.ndarray:
|
||||
arr.setflags(write=False)
|
||||
return arr
|
||||
|
||||
|
||||
def test_detach_zero_copy_from_model_runner_output_copies_only_numpy_views():
|
||||
cu_num_generated_tokens = [0, 2]
|
||||
prompt_logprobs = LogprobsTensors.empty_cpu(1, 2)
|
||||
output = ModelRunnerOutput(
|
||||
req_ids=["req-0"],
|
||||
req_id_to_index={"req-0": 0},
|
||||
logprobs=LogprobsLists(
|
||||
logprob_token_ids=_make_readonly(
|
||||
np.array([[1, 2], [3, 4]], dtype=np.int32)
|
||||
),
|
||||
logprobs=_make_readonly(
|
||||
np.array([[0.1, 0.2], [0.3, 0.4]], dtype=np.float32)
|
||||
),
|
||||
sampled_token_ranks=_make_readonly(np.array([1, 2], dtype=np.int32)),
|
||||
cu_num_generated_tokens=cu_num_generated_tokens,
|
||||
),
|
||||
prompt_logprobs_dict={"req-0": prompt_logprobs},
|
||||
)
|
||||
|
||||
original_logprobs = output.logprobs
|
||||
assert original_logprobs is not None
|
||||
|
||||
detach_zero_copy_from_model_runner_output(output)
|
||||
|
||||
detached_logprobs = output.logprobs
|
||||
assert detached_logprobs is not None
|
||||
assert detached_logprobs is not original_logprobs
|
||||
assert (
|
||||
detached_logprobs.logprob_token_ids is not original_logprobs.logprob_token_ids
|
||||
)
|
||||
assert detached_logprobs.logprobs is not original_logprobs.logprobs
|
||||
assert (
|
||||
detached_logprobs.sampled_token_ranks
|
||||
is not original_logprobs.sampled_token_ranks
|
||||
)
|
||||
assert detached_logprobs.logprob_token_ids.flags.writeable
|
||||
assert detached_logprobs.logprobs.flags.writeable
|
||||
assert detached_logprobs.sampled_token_ranks.flags.writeable
|
||||
assert detached_logprobs.cu_num_generated_tokens is cu_num_generated_tokens
|
||||
assert output.prompt_logprobs_dict["req-0"] is prompt_logprobs
|
||||
|
||||
|
||||
def test_detach_zero_copy_aux_output_without_logprobs():
|
||||
rows = _make_readonly(np.arange(12, dtype=np.uint8).reshape(2, 3, 2))
|
||||
output = ModelRunnerOutput(
|
||||
req_ids=["req-0"],
|
||||
req_id_to_index={"req-0": 0},
|
||||
aux_output_connector_output={"req-0": AuxRequestOutput(0, rows)},
|
||||
)
|
||||
|
||||
detach_zero_copy_from_model_runner_output(output)
|
||||
|
||||
aux_output = output.aux_output_connector_output
|
||||
assert aux_output is not None
|
||||
detached = aux_output["req-0"].rows
|
||||
assert detached is not rows
|
||||
assert detached.flags.writeable
|
||||
np.testing.assert_array_equal(detached, rows)
|
||||
@@ -1,8 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.v1.executor import vllm_net_devices
|
||||
from vllm.v1.executor.vllm_net_devices import normalize_pci
|
||||
|
||||
|
||||
@@ -74,3 +78,24 @@ def test_normalize_pci_device_out_of_range_raises():
|
||||
def test_normalize_pci_empty_string_raises():
|
||||
with pytest.raises(ValueError):
|
||||
normalize_pci("")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pcp,pp", [(1, 1), (2, 1), (2, 2)])
|
||||
def test_set_worker_net_device_uses_pcp_aware_gpu_index(monkeypatch, pcp, pp):
|
||||
"""NIC lookup uses the same DP-offset GPU index as device selection."""
|
||||
monkeypatch.setenv("VLLM_GPU_NIC_PCIE_MAPPING", "unused")
|
||||
monkeypatch.setenv("VLLM_NIC_SELECTION_VARS", "unused")
|
||||
seen: list[int] = []
|
||||
monkeypatch.setattr(vllm_net_devices, "set_worker_gpu_nic_mapping", seen.append)
|
||||
parallel_config = ParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
prefill_context_parallel_size=pcp,
|
||||
pipeline_parallel_size=pp,
|
||||
data_parallel_size=2,
|
||||
data_parallel_rank_local=1,
|
||||
distributed_executor_backend="mp",
|
||||
)
|
||||
vllm_net_devices.set_worker_net_device(
|
||||
1, SimpleNamespace(parallel_config=parallel_config)
|
||||
)
|
||||
assert seen == [1 + 2 * pcp * pp]
|
||||
|
||||
@@ -13,6 +13,7 @@ python3 -m pytest -s -x "${IMPORT_CANARY}"
|
||||
tp_configs=(
|
||||
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=1"
|
||||
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2"
|
||||
"GPU_MEMORY_UTILIZATION=0.6 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=1 SLEEP_WAKE=1"
|
||||
)
|
||||
|
||||
configs=("${tp_configs[@]}")
|
||||
|
||||
@@ -59,6 +59,12 @@ GPU_MEMORY_UTILIZATION=${GPU_MEMORY_UTILIZATION:-0.2}
|
||||
BLOCK_SIZE=${BLOCK_SIZE:-128}
|
||||
# Comma-separated extra args for vllm serve (e.g. --max-model-len,2048)
|
||||
VLLM_SERVE_EXTRA_ARGS=${VLLM_SERVE_EXTRA_ARGS:-}
|
||||
# Set to 1 to put every instance to sleep and wake it up before the eval.
|
||||
SLEEP_WAKE=${SLEEP_WAKE:-0}
|
||||
if [[ "$SLEEP_WAKE" == "1" ]]; then
|
||||
export VLLM_SERVER_DEV_MODE=1
|
||||
VLLM_SERVE_EXTRA_ARGS="${VLLM_SERVE_EXTRA_ARGS:+${VLLM_SERVE_EXTRA_ARGS},}--enable-sleep-mode"
|
||||
fi
|
||||
|
||||
# Resolve the repository root from the script location instead of `.git`.
|
||||
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
@@ -226,9 +232,17 @@ run_tests_for_model() {
|
||||
# Wait for the proxy to start
|
||||
sleep 5
|
||||
|
||||
SLEEP_WAKE_URLS=""
|
||||
if [[ "$SLEEP_WAKE" == "1" ]]; then
|
||||
for PORT in "${PREFILL_PORTS[@]}" "${DECODE_PORTS[@]}"; do
|
||||
SLEEP_WAKE_URLS="${SLEEP_WAKE_URLS:+${SLEEP_WAKE_URLS},}http://localhost:${PORT}"
|
||||
done
|
||||
fi
|
||||
|
||||
# Run lm eval for this model
|
||||
echo "Running tests for $model_name"
|
||||
TEST_MODEL=$model_name PROXY_PORT=$PROXY_PORT python3 -m pytest -s -x \
|
||||
TEST_MODEL=$model_name PROXY_PORT=$PROXY_PORT SLEEP_WAKE_URLS=$SLEEP_WAKE_URLS \
|
||||
python3 -m pytest -s -x \
|
||||
"${GIT_ROOT}/tests/v1/kv_connector/mooncake_integration/test_accuracy.py"
|
||||
|
||||
# Clean up before running next model
|
||||
|
||||
@@ -4,8 +4,13 @@ import os
|
||||
|
||||
import lm_eval
|
||||
import openai
|
||||
import requests
|
||||
|
||||
BASE_URL = f"http://localhost:{os.environ.get('PROXY_PORT', 8192)}/v1"
|
||||
# Comma-separated server URLs to sleep and wake up before the eval.
|
||||
SLEEP_WAKE_URLS = [
|
||||
url for url in os.environ.get("SLEEP_WAKE_URLS", "").split(",") if url
|
||||
]
|
||||
NUM_CONCURRENT = 100
|
||||
TASK = "gsm8k"
|
||||
FILTER = "exact_match,strict-match"
|
||||
@@ -35,9 +40,18 @@ def run_simple_prompt():
|
||||
print("-" * 50)
|
||||
|
||||
|
||||
def sleep_and_wake_up(urls: list[str]):
|
||||
for url in urls:
|
||||
requests.post(f"{url}/sleep", params={"level": 1}).raise_for_status()
|
||||
for url in urls:
|
||||
requests.post(f"{url}/wake_up").raise_for_status()
|
||||
|
||||
|
||||
def test_accuracy():
|
||||
"""Run the end to end accuracy test."""
|
||||
run_simple_prompt()
|
||||
if SLEEP_WAKE_URLS:
|
||||
sleep_and_wake_up(SLEEP_WAKE_URLS)
|
||||
|
||||
model_args = (
|
||||
f"model={MODEL_NAME},"
|
||||
|
||||
@@ -60,7 +60,6 @@ vllm serve "$MODEL" \
|
||||
--block-size 128 \
|
||||
--trust-remote-code \
|
||||
--enable-prefix-caching \
|
||||
--mamba-cache-mode align \
|
||||
--kv-transfer-config "$KV_CONFIG" \
|
||||
"${EXTRA_ARGS[@]}" &
|
||||
|
||||
@@ -78,7 +77,6 @@ vllm serve "$MODEL" \
|
||||
--block-size 128 \
|
||||
--trust-remote-code \
|
||||
--enable-prefix-caching \
|
||||
--mamba-cache-mode align \
|
||||
--kv-transfer-config "$KV_CONFIG" \
|
||||
"${EXTRA_ARGS[@]}" &
|
||||
|
||||
|
||||
@@ -23,5 +23,5 @@ def _mock_cpu_backend_block_sizes(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
CPUAttentionBackend,
|
||||
"get_supported_kernel_block_sizes",
|
||||
staticmethod(lambda: [MultipleOf(16)]),
|
||||
staticmethod(lambda kv_cache_spec=None: [MultipleOf(16)]),
|
||||
)
|
||||
|
||||
@@ -17,10 +17,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
SchedulerOffloadConfig,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
generate_scheduler_kv_cache_config,
|
||||
kv_cache_groups_tp_replicas,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_planning import kv_cache_groups_tp_replicas
|
||||
from vllm.v1.core.kv_cache_utils import generate_scheduler_kv_cache_config
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
@@ -71,7 +69,7 @@ def _make_vllm_config(
|
||||
decode_context_parallel_size=decode_context_parallel_size,
|
||||
)
|
||||
config.kv_events_config = None
|
||||
config.use_v2_model_runner = False
|
||||
config.use_v2_model_runner = True
|
||||
config.kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
@@ -284,7 +282,7 @@ def _parallelism_agnostic(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
*,
|
||||
canonical: bool = False,
|
||||
v2: bool = False,
|
||||
v2: bool = True,
|
||||
) -> bool:
|
||||
config = _make_vllm_config(
|
||||
extra_config={"canonical_layout": True} if canonical else None
|
||||
@@ -308,7 +306,7 @@ def _replicated_layout(
|
||||
prefill_context_parallel_size: int = 1,
|
||||
decode_context_parallel_size: int = 1,
|
||||
use_mla: bool = True,
|
||||
use_v2_model_runner: bool = False,
|
||||
use_v2_model_runner: bool = True,
|
||||
distributed_executor_backend: Any = "mp",
|
||||
nnodes: int = 1,
|
||||
world_size: int | None = None,
|
||||
@@ -1105,7 +1103,10 @@ def test_replicated_layout_parallel_gate(kwargs: dict[str, Any], case: str):
|
||||
|
||||
|
||||
def test_parallelism_agnostic_for_single_full_attention_group():
|
||||
assert _parallelism_agnostic([KVCacheGroupSpec(["l0"], _full_attention_spec())])
|
||||
# The direct (non-canonical) parallelism-agnostic layout is MRV1-only.
|
||||
assert _parallelism_agnostic(
|
||||
[KVCacheGroupSpec(["l0"], _full_attention_spec())], v2=False
|
||||
)
|
||||
|
||||
|
||||
_SWA_SPEC = SlidingWindowSpec(
|
||||
@@ -1147,7 +1148,9 @@ def _uniform_group(*specs: KVCacheSpec) -> KVCacheGroupSpec:
|
||||
],
|
||||
)
|
||||
def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
|
||||
assert not _parallelism_agnostic(kv_cache_groups)
|
||||
# The direct layout is never parallelism-agnostic on MRV2, so check the
|
||||
# MRV1 gate.
|
||||
assert not _parallelism_agnostic(kv_cache_groups, v2=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -1178,7 +1181,7 @@ def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
|
||||
def test_canonical_layout_gate(kv_cache_groups, certified):
|
||||
"""The canonical layout certifies portability group by group; none of
|
||||
these shapes are portable in the direct layout."""
|
||||
assert not _parallelism_agnostic(kv_cache_groups)
|
||||
assert not _parallelism_agnostic(kv_cache_groups, v2=False)
|
||||
assert _parallelism_agnostic(kv_cache_groups, canonical=True) is certified
|
||||
|
||||
|
||||
@@ -1233,7 +1236,7 @@ def test_canonical_layout_certifies_v2_model_runner():
|
||||
registration, so the static gate must not depend on the model-runner
|
||||
version — the v2 runner is the case the canonical layout exists for."""
|
||||
groups = _groups(_full_attention_spec())
|
||||
assert _parallelism_agnostic(groups, canonical=True, v2=True)
|
||||
assert _parallelism_agnostic(groups, canonical=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [1, 2, 4])
|
||||
@@ -1275,7 +1278,6 @@ def test_canonical_mla_dsa_rows_are_tp_independent(tp_size, scheduler):
|
||||
|
||||
def test_parallelism_agnostic_disabled_on_v2_model_runner():
|
||||
config = _make_vllm_config()
|
||||
config.use_v2_model_runner = True
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=0,
|
||||
kv_cache_tensors=[],
|
||||
|
||||
@@ -2,12 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from prometheus_client import Counter, Gauge, Histogram
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorLogging
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
KV_OFFLOAD_CONFIG_INFO,
|
||||
OffloadingConnectorStats,
|
||||
OffloadPromMetrics,
|
||||
_ConnectorMetricName,
|
||||
@@ -115,15 +117,21 @@ class _FakeVllmConfig:
|
||||
|
||||
def _spec_cls_with_metric_definitions(
|
||||
metric_definitions: dict[str, Any],
|
||||
info_keys: tuple[str, ...] = (),
|
||||
) -> type:
|
||||
"""Build a fake offloading spec class reporting the given metric
|
||||
definitions, so tests don't need to patch the real CPU spec."""
|
||||
definitions and info label names, so tests don't need to patch the real
|
||||
CPU spec."""
|
||||
|
||||
class _FakeOffloadingSpec:
|
||||
@staticmethod
|
||||
def build_metric_definitions(extra_config):
|
||||
return metric_definitions
|
||||
|
||||
@staticmethod
|
||||
def config_info_keys(extra_config):
|
||||
return info_keys
|
||||
|
||||
return _FakeOffloadingSpec
|
||||
|
||||
|
||||
@@ -784,3 +792,333 @@ def test_prom_metrics_rejects_undeclared_metric():
|
||||
_StatsKey.DATA: {"unknown:metric": {(): 1}},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _prom_metrics(
|
||||
metric_definitions: dict[str, Any] | None = None,
|
||||
info_keys: tuple[str, ...] = (),
|
||||
):
|
||||
"""Build OffloadPromMetrics over fake metric classes, with one engine."""
|
||||
spec_cls = _spec_cls_with_metric_definitions(metric_definitions or {}, info_keys)
|
||||
with patch.object(OffloadingSpecFactory, "get_spec_cls", return_value=spec_cls):
|
||||
return OffloadPromMetrics(
|
||||
vllm_config=_FakeVllmConfig(store_threshold=0), # type: ignore[arg-type]
|
||||
metric_types={
|
||||
Gauge: _FakeMetric,
|
||||
Counter: _FakeMetric,
|
||||
Histogram: _FakeMetric,
|
||||
},
|
||||
labelnames=["model_name", "engine"],
|
||||
per_engine_labelvalues={0: ["model", "0"], 1: ["model", "1"]},
|
||||
)
|
||||
|
||||
|
||||
def test_prom_metrics_keeps_a_gauge_declared_multiprocess_mode():
|
||||
"""A gauge that declares its own multiprocess_mode keeps it.
|
||||
|
||||
The value on OffloadingGaugeMetadata must stay a default. A hardcoded
|
||||
"mostrecent" in _create_metric passes the info-gauge test below and
|
||||
silently overrides a gauge that needs another merge, for example a real
|
||||
cross-frontend sum.
|
||||
"""
|
||||
prom_metrics = _prom_metrics(
|
||||
{
|
||||
PENDING_STORES: OffloadingGaugeMetadata(
|
||||
documentation="gauge declaring a non-default multiprocess mode.",
|
||||
multiprocess_mode="sum",
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
gauge_def = prom_metrics._offloading_metric_defs[PENDING_STORES]
|
||||
assert gauge_def.kwargs["multiprocess_mode"] == "sum"
|
||||
|
||||
|
||||
def test_prom_metrics_omits_multiprocess_mode_outside_a_gauge():
|
||||
"""multiprocess_mode reaches a gauge only.
|
||||
|
||||
prometheus_client accepts the argument on Gauge alone, so hoisting the
|
||||
kwarg out of the gauge branch breaks construction of a real Counter or
|
||||
Histogram.
|
||||
"""
|
||||
prom_metrics = _prom_metrics(
|
||||
{
|
||||
MY_COUNTER: OffloadingCounterMetadata(documentation="a counter."),
|
||||
LOOKUP_LATENCY: OffloadingHistogramMetadata(
|
||||
documentation="a histogram.",
|
||||
buckets=(0.1, 1),
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
counter_def = prom_metrics._offloading_metric_defs[MY_COUNTER]
|
||||
histogram_def = prom_metrics._offloading_metric_defs[LOOKUP_LATENCY]
|
||||
assert "multiprocess_mode" not in counter_def.kwargs
|
||||
assert "multiprocess_mode" not in histogram_def.kwargs
|
||||
|
||||
|
||||
def test_prom_metrics_declares_the_info_gauge_from_the_spec_keys():
|
||||
"""The API-server process holds no manager, but it does hold the spec, so
|
||||
it declares the gauge at startup, before any payload."""
|
||||
prom_metrics = _prom_metrics(info_keys=("cpu_num_chunks", "root_dir_name"))
|
||||
|
||||
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
|
||||
assert gauge_def.kwargs["name"] == KV_OFFLOAD_CONFIG_INFO
|
||||
assert gauge_def.kwargs["labelnames"] == [
|
||||
"model_name",
|
||||
"engine",
|
||||
"cpu_num_chunks",
|
||||
"root_dir_name",
|
||||
]
|
||||
# The frontend declares the metric and binds no label values, so an engine
|
||||
# that never reports exposes no series.
|
||||
assert gauge_def.children == []
|
||||
|
||||
prom_metrics.observe(
|
||||
{_StatsKey.INFO: [{"cpu_num_chunks": 512, "root_dir_name": "a"}]}
|
||||
)
|
||||
|
||||
# Every value renders as a label value, and the metric itself is pinned
|
||||
# to 1.
|
||||
(gauge,) = gauge_def.children
|
||||
assert gauge.labelvalues == ("model", "0", "512", "a")
|
||||
assert gauge.set_values == [1]
|
||||
|
||||
|
||||
def test_prom_metrics_reads_the_info_payload_by_label_name():
|
||||
"""A manager orders config_info() as it likes, and msgpack keeps that
|
||||
order, so the frontend must key on the name rather than on the position."""
|
||||
prom_metrics = _prom_metrics(info_keys=("first", "second"))
|
||||
|
||||
prom_metrics.observe({_StatsKey.INFO: [{"second": 2, "first": 1}]})
|
||||
|
||||
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
|
||||
(gauge,) = gauge_def.children
|
||||
assert gauge.labelvalues == ("model", "0", "1", "2")
|
||||
|
||||
|
||||
def test_prom_metrics_reports_one_info_series_for_each_mapping():
|
||||
"""A manager reports one mapping for each series, and a label of its own
|
||||
tells the series apart. Two mappings of one kind agree on every other value,
|
||||
so without that label they collapse into one series."""
|
||||
prom_metrics = _prom_metrics(info_keys=("source", "root_dir_name"))
|
||||
|
||||
prom_metrics.observe(
|
||||
{
|
||||
_StatsKey.INFO: [
|
||||
{"source": "1", "root_dir_name": "a"},
|
||||
{"source": "2", "root_dir_name": "b"},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
|
||||
assert [child.labelvalues for child in gauge_def.children] == [
|
||||
("model", "0", "1", "a"),
|
||||
("model", "0", "2", "b"),
|
||||
]
|
||||
assert [child.set_values for child in gauge_def.children] == [[1], [1]]
|
||||
|
||||
|
||||
def test_prom_metrics_declares_the_info_gauge_as_most_recent():
|
||||
"""The info gauge must merge across frontends by freshest write.
|
||||
|
||||
Offloading stats reach one frontend per step as complete per-engine
|
||||
snapshots, so summing would report the number of participating API-server
|
||||
processes instead of the 1 an info gauge is pinned to -- wrong only under
|
||||
multiprocess deployment, and silently, since the facts ride the labels.
|
||||
"""
|
||||
prom_metrics = _prom_metrics(info_keys=("cpu_num_chunks",))
|
||||
|
||||
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
|
||||
assert gauge_def.kwargs["multiprocess_mode"] == "mostrecent"
|
||||
|
||||
|
||||
def test_prom_metrics_reports_the_info_gauge_without_manager_labels():
|
||||
"""A spec that declares no fact still yields the metric, so its presence
|
||||
alone answers whether offloading runs."""
|
||||
prom_metrics = _prom_metrics()
|
||||
|
||||
prom_metrics.observe({_StatsKey.INFO: [{}]})
|
||||
|
||||
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
|
||||
assert gauge_def.kwargs["labelnames"] == ["model_name", "engine"]
|
||||
(gauge,) = gauge_def.children
|
||||
assert gauge.labelvalues == ("model", "0")
|
||||
|
||||
|
||||
def test_prom_metrics_empties_a_declared_info_label_no_manager_fills():
|
||||
"""A name the spec declares and the manager skips must publish empty.
|
||||
|
||||
PromQL reads an empty label value as an absent label, so the series still
|
||||
matches both shapes. This code runs on the engine output path of the API
|
||||
server, where an exception ends the output handler, so it warns and goes
|
||||
on rather than drop the whole series.
|
||||
"""
|
||||
prom_metrics = _prom_metrics(info_keys=("filled", "skipped"))
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics.logger"
|
||||
) as mock_logger:
|
||||
prom_metrics.observe({_StatsKey.INFO: [{"filled": 512}]})
|
||||
|
||||
assert mock_logger.warning_once.call_count == 1
|
||||
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
|
||||
(gauge,) = gauge_def.children
|
||||
assert gauge.labelvalues == ("model", "0", "512", "")
|
||||
|
||||
|
||||
def test_prom_metrics_keeps_quiet_when_a_name_belongs_to_another_series():
|
||||
"""A name that one series owns reads empty on the other series by design.
|
||||
|
||||
The declaration holds the names of every series, so the frontend checks the
|
||||
payload as a whole. A per-series check would read this normal shape as a
|
||||
disagreement, and warn at every start of a server that reports several
|
||||
series.
|
||||
"""
|
||||
prom_metrics = _prom_metrics(info_keys=("source", "root_dir_name", "bucket"))
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics.logger"
|
||||
) as mock_logger:
|
||||
prom_metrics.observe(
|
||||
{
|
||||
_StatsKey.INFO: [
|
||||
{"source": "1", "root_dir_name": "kv_a"},
|
||||
{"source": "2", "bucket": "kv_bucket"},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
mock_logger.warning_once.assert_not_called()
|
||||
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
|
||||
assert [child.labelvalues for child in gauge_def.children] == [
|
||||
("model", "0", "1", "kv_a", ""),
|
||||
("model", "0", "2", "", "kv_bucket"),
|
||||
]
|
||||
|
||||
|
||||
def test_prom_metrics_drops_an_info_label_the_spec_did_not_declare():
|
||||
"""A Prometheus metric binds its label names once, so a name that the spec
|
||||
did not declare cannot join the gauge. It warns and publishes the rest."""
|
||||
prom_metrics = _prom_metrics(info_keys=("declared",))
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics.logger"
|
||||
) as mock_logger:
|
||||
prom_metrics.observe({_StatsKey.INFO: [{"declared": 512, "undeclared": 1}]})
|
||||
|
||||
assert mock_logger.warning_once.call_count == 1
|
||||
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
|
||||
assert gauge_def.kwargs["labelnames"] == ["model_name", "engine", "declared"]
|
||||
(gauge,) = gauge_def.children
|
||||
assert gauge.labelvalues == ("model", "0", "512")
|
||||
|
||||
|
||||
def test_prom_metrics_reports_the_info_of_every_engine():
|
||||
"""Every engine of one instance runs one offloading config, so they share
|
||||
the gauge and differ by the engine label alone."""
|
||||
prom_metrics = _prom_metrics(info_keys=("cpu_num_chunks",))
|
||||
|
||||
prom_metrics.observe({_StatsKey.INFO: [{"cpu_num_chunks": 512}]}, engine_idx=0)
|
||||
prom_metrics.observe({_StatsKey.INFO: [{"cpu_num_chunks": 512}]}, engine_idx=1)
|
||||
|
||||
gauge_def = prom_metrics._offloading_metric_defs[KV_OFFLOAD_CONFIG_INFO]
|
||||
assert [child.labelvalues for child in gauge_def.children] == [
|
||||
("model", "0", "512"),
|
||||
("model", "1", "512"),
|
||||
]
|
||||
|
||||
|
||||
def test_stats_hold_the_config_info_without_an_observation():
|
||||
"""An info-only payload must survive the scheduler gate, which drops an
|
||||
empty payload, and must add no line to the interval log."""
|
||||
stats = OffloadingConnectorStats()
|
||||
assert stats.is_empty()
|
||||
|
||||
stats.set_info([{"cpu_num_chunks": 512}])
|
||||
|
||||
assert not stats.is_empty()
|
||||
assert stats.reduce() == {}
|
||||
|
||||
|
||||
def test_aggregate_keeps_the_config_info_of_the_other_payload():
|
||||
"""The scheduler payload holds the info and merges into the worker payload
|
||||
(v1/core/sched/scheduler.py), so a dropped info would hide the metric on
|
||||
every step that also transfers, which is every step that matters."""
|
||||
worker_stats = OffloadingConnectorStats()
|
||||
worker_stats.increase_counter(LOAD_BYTES, 100)
|
||||
scheduler_stats = OffloadingConnectorStats()
|
||||
scheduler_stats.set_info([{"cpu_num_chunks": 512}])
|
||||
|
||||
worker_stats.aggregate(scheduler_stats)
|
||||
|
||||
assert worker_stats.data[_StatsKey.INFO] == [{"cpu_num_chunks": 512}]
|
||||
assert worker_stats.data[_StatsKey.DATA][LOAD_BYTES] == {(): 100}
|
||||
|
||||
|
||||
def test_aggregate_keeps_the_config_info_of_a_later_payload():
|
||||
"""The scheduler sends the info once, so a later payload must not clear
|
||||
it."""
|
||||
stats = OffloadingConnectorStats()
|
||||
stats.set_info([{"cpu_num_chunks": 512}])
|
||||
later_stats = OffloadingConnectorStats()
|
||||
later_stats.increase_counter(LOAD_BYTES, 100)
|
||||
|
||||
stats.aggregate(later_stats)
|
||||
|
||||
assert stats.data[_StatsKey.INFO] == [{"cpu_num_chunks": 512}]
|
||||
|
||||
|
||||
def test_metric_sections_leave_out_the_config_info():
|
||||
"""A manager accesses its own metrics by a walk over the metric sections, as
|
||||
KVCRSecondaryTierManager does in v1/kv_offload/tiering/kvcr/manager.py. The
|
||||
info section holds label names, not metric names, so it stays out of that
|
||||
walk."""
|
||||
stats = OffloadingConnectorStats()
|
||||
stats.increase_counter(MY_COUNTER, 2, ("remote_deliver",))
|
||||
stats.set_info([{"info_label": "local"}])
|
||||
|
||||
info = stats.data[_StatsKey.INFO]
|
||||
sections = stats.metric_sections()
|
||||
|
||||
assert not any(section is info for section in sections)
|
||||
assert sections == (stats.data[_StatsKey.TYPES], stats.data[_StatsKey.DATA])
|
||||
|
||||
|
||||
def test_logging_writes_no_line_for_an_info_only_payload():
|
||||
"""An info-only payload reaches the logging path, and reduce() drops the info.
|
||||
|
||||
The info belongs to the Prometheus frontend only. Without the guard in
|
||||
KVConnectorLogging.log(), such a payload writes "KV Transfer metrics: "
|
||||
with an empty body.
|
||||
"""
|
||||
stats = OffloadingConnectorStats()
|
||||
stats.set_info([{"info_label": "local"}])
|
||||
assert not stats.is_empty()
|
||||
assert stats.reduce() == {}
|
||||
|
||||
kv_logging = KVConnectorLogging(None)
|
||||
kv_logging.transfer_stats_accumulator = stats
|
||||
log_fn = MagicMock()
|
||||
|
||||
kv_logging.log(log_fn=log_fn)
|
||||
|
||||
log_fn.assert_not_called()
|
||||
assert kv_logging.transfer_stats_accumulator is None
|
||||
|
||||
|
||||
def test_scheduler_sends_the_config_info_once(request_runner):
|
||||
"""MockOffloadingSpec publishes no fact, so the payload holds one empty
|
||||
mapping, which still yields the metric."""
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
)
|
||||
|
||||
stats = runner.connector_scheduler.get_stats()
|
||||
|
||||
assert stats is not None
|
||||
assert stats.data[_StatsKey.INFO] == [{}]
|
||||
assert runner.connector_scheduler.get_stats() is None
|
||||
|
||||
@@ -1017,6 +1017,129 @@ def test_scheduler_reports_lookup_sync_delay(request_runner):
|
||||
assert reduced[f"{_ConnectorMetricName.LOOKUP_SYNC_DELAY}_sum"] > 0
|
||||
|
||||
|
||||
def test_on_schedule_end_is_the_last_manager_call_of_a_step(request_runner):
|
||||
"""on_schedule_end must run after every other manager call of the step.
|
||||
|
||||
Callers bound a step's exclusive access to the manager by releasing at
|
||||
on_schedule_end, so anything issued after it — notably prepare_store, which
|
||||
evicts — would fall outside that window.
|
||||
"""
|
||||
block_size = 4
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=8,
|
||||
async_scheduling=False,
|
||||
)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
|
||||
connector_scheduler = runner.connector_scheduler
|
||||
build_meta = connector_scheduler.build_connector_meta
|
||||
windows: list[list[str]] = []
|
||||
|
||||
def spy(scheduler_output):
|
||||
first = len(runner.manager.mock_calls)
|
||||
meta = build_meta(scheduler_output)
|
||||
# Entering and exiting manager.lock brackets the step; only the state
|
||||
# operations it guards are ordered relative to on_schedule_end.
|
||||
windows.append(
|
||||
[
|
||||
name
|
||||
for name, _, _ in runner.manager.mock_calls[first:]
|
||||
if not name.startswith("lock.")
|
||||
]
|
||||
)
|
||||
return meta
|
||||
|
||||
connector_scheduler.build_connector_meta = spy
|
||||
|
||||
runner.new_request(token_ids=[0] * (block_size * 2))
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
|
||||
assert windows, "build_connector_meta was never called"
|
||||
assert any("prepare_store" in w for w in windows), (
|
||||
"step must exercise the calls that follow on_schedule_end today"
|
||||
)
|
||||
assert any("on_request_finished" in w for w in windows)
|
||||
for names in windows:
|
||||
assert names[-1] == "on_schedule_end", (
|
||||
f"on_schedule_end must be the last manager call of a step, got {names}"
|
||||
)
|
||||
|
||||
|
||||
def test_step_releases_the_manager_before_the_model_runs(request_runner):
|
||||
"""The step-scoped region must close when build_connector_meta returns.
|
||||
|
||||
Everything after it -- the model future above all -- runs with the manager
|
||||
free, which is the only window another thread has to use it.
|
||||
"""
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[1] * 4)
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
|
||||
scheduler = runner.connector_scheduler
|
||||
assert scheduler._step_region is None
|
||||
lock = runner.manager.lock
|
||||
assert lock.__enter__.call_count > 0
|
||||
assert lock.__enter__.call_count == lock.__exit__.call_count
|
||||
|
||||
|
||||
def test_has_pending_push_work_does_not_open_the_step_region(request_runner):
|
||||
"""It must take a short region, never the step-scoped one.
|
||||
|
||||
The engine asks this before schedule() and returns early when the answer is
|
||||
False, so a step-scoped acquire here would never reach the release in
|
||||
build_connector_meta and would hold the manager for good.
|
||||
"""
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
)
|
||||
scheduler = runner.connector_scheduler
|
||||
runner.manager.has_pending_work.return_value = False
|
||||
|
||||
assert scheduler.has_pending_push_work() is False
|
||||
assert scheduler._step_region is None
|
||||
lock = runner.manager.lock
|
||||
assert lock.__enter__.call_count == lock.__exit__.call_count
|
||||
|
||||
|
||||
def test_take_events_drains_the_manager_inside_the_region(request_runner):
|
||||
"""take_events must not hold the manager across its own iteration.
|
||||
|
||||
manager.take_events() is a generator whose backing list is cleared once
|
||||
exhausted, so draining it lazily would both pin the manager for as long as
|
||||
the consumer takes and lose events appended in the meantime.
|
||||
"""
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
)
|
||||
scheduler = runner.connector_scheduler
|
||||
runner.manager.take_events.return_value = iter(())
|
||||
runner.manager.reset_mock()
|
||||
|
||||
events = list(scheduler.take_events())
|
||||
|
||||
assert events == []
|
||||
assert runner.manager.take_events.call_count == 1
|
||||
assert scheduler._step_region is None
|
||||
lock = runner.manager.lock
|
||||
assert lock.__enter__.call_count == 1
|
||||
assert lock.__exit__.call_count == 1
|
||||
|
||||
|
||||
def test_scheduler_reports_lookup_async_delay_on_resolve(request_runner):
|
||||
"""A deferred lookup reports its async delay once it resolves."""
|
||||
runner = request_runner(
|
||||
|
||||
@@ -132,6 +132,7 @@ class MockOffloadingSpec(OffloadingSpec):
|
||||
self.manager.lookup.return_value = LookupResult.MISS
|
||||
self.manager.get_load_source.return_value = CacheHitSource.EXTERNAL_UNSPECIFIED
|
||||
self.manager.get_stats.return_value = None
|
||||
self.manager.config_info.return_value = [{}]
|
||||
self.manager.on_new_request.return_value = RequestOffloadingContext()
|
||||
self.handler = MockOffloadingWorker()
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@ from .utils import (
|
||||
create_scheduler,
|
||||
create_vllm_config,
|
||||
make_kv_cache_config,
|
||||
maybe_update_block_size,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
@@ -87,10 +88,11 @@ def _make_connector_with_fake_worker(
|
||||
hand_shake_latency=0, cycles_before_done=0, do_handshake=True
|
||||
):
|
||||
"""Create a NixlConnector with FakeNixlConnectorWorker."""
|
||||
block_size = maybe_update_block_size(32)
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=32
|
||||
kv_connector_extra_config=BIDIR_KV_EXTRA_CONFIG, block_size=block_size
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(block_size=32, num_blocks=2)
|
||||
kv_cache_config = make_kv_cache_config(block_size=block_size, num_blocks=2)
|
||||
connector = NixlConnector(vllm_config, KVConnectorRole.WORKER, kv_cache_config)
|
||||
connector.connector_worker = FakeNixlConnectorWorker(
|
||||
vllm_config,
|
||||
|
||||
@@ -101,6 +101,7 @@ def test_kv_connector(
|
||||
def _build_config(
|
||||
*,
|
||||
kv_connector: str | None,
|
||||
kv_connector_extra_config: dict | None = None,
|
||||
enable_sleep_mode: bool = False,
|
||||
enable_cumem_allocator: bool = False,
|
||||
) -> VllmConfig:
|
||||
@@ -109,7 +110,11 @@ def _build_config(
|
||||
from types import SimpleNamespace
|
||||
|
||||
kv_transfer_config = (
|
||||
KVTransferConfig(kv_connector=kv_connector, kv_role="kv_both")
|
||||
KVTransferConfig(
|
||||
kv_connector=kv_connector,
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config=kv_connector_extra_config or {},
|
||||
)
|
||||
if kv_connector is not None
|
||||
else None
|
||||
)
|
||||
@@ -141,7 +146,7 @@ def test_kv_connector_allows_expandable_segments_with_sleep_mode(monkeypatch):
|
||||
"""Sleep mode routes KV allocations through CuMemAllocator's pool, which
|
||||
auto-disables expandable_segments (see #40812)."""
|
||||
monkeypatch.setenv("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
|
||||
_build_config(kv_connector="NixlConnector", enable_sleep_mode=True)
|
||||
_build_config(kv_connector="MooncakeConnector", enable_sleep_mode=True)
|
||||
|
||||
|
||||
def test_kv_connector_allows_expandable_segments_with_cumem_allocator(
|
||||
@@ -152,6 +157,27 @@ def test_kv_connector_allows_expandable_segments_with_cumem_allocator(
|
||||
_build_config(kv_connector="NixlConnector", enable_cumem_allocator=True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kv_connector", "extra_config", "env"),
|
||||
[
|
||||
("NixlConnector", {}, None),
|
||||
("MooncakeConnector", {"mooncake_protocol": "tcp"}, None),
|
||||
("MooncakeConnector", {}, "MC_USE_TENT"),
|
||||
],
|
||||
)
|
||||
def test_sleep_mode_refuses_unsupported_kv_connector(
|
||||
monkeypatch, kv_connector, extra_config, env
|
||||
):
|
||||
if env:
|
||||
monkeypatch.setenv(env, "1")
|
||||
with pytest.raises(ValueError, match=f"^{kv_connector} does not support sleep"):
|
||||
_build_config(
|
||||
kv_connector=kv_connector,
|
||||
kv_connector_extra_config=extra_config,
|
||||
enable_sleep_mode=True,
|
||||
)
|
||||
|
||||
|
||||
def test_kv_connector_allows_other_alloc_conf(monkeypatch):
|
||||
"""Other PYTORCH_CUDA_ALLOC_CONF values must not be rejected."""
|
||||
monkeypatch.setenv(
|
||||
|
||||
@@ -10,7 +10,7 @@ import torch
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.example_hidden_states_connector import ( # noqa: E501
|
||||
ExampleHiddenStatesConnector,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import get_kv_cache_groups
|
||||
from vllm.v1.core.kv_cache_planning import get_kv_cache_groups
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import contextlib
|
||||
import ctypes
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -25,14 +25,20 @@ from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
|
||||
HiSparseConnectorWorker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
|
||||
from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
|
||||
from vllm.v1.hisparse import runtime as runtime_module
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.hisparse.runtime import HiSparseCacheHandle
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
HiSparseCacheHandle,
|
||||
update_hisparse_residency,
|
||||
)
|
||||
from vllm.v1.hisparse.types import (
|
||||
SparseKVOffloadCommand,
|
||||
SparseKVPageTransfer,
|
||||
SparseKVResidencyUpdate,
|
||||
SparseKVRowMirror,
|
||||
)
|
||||
from vllm.v1.metrics.stats import KVCacheEvictionEvent
|
||||
from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector
|
||||
|
||||
|
||||
@@ -100,6 +106,46 @@ def test_scheduler_stats_report_host_pool_usage():
|
||||
assert stats.data["host_cache_usage_perc"] == [expected]
|
||||
|
||||
|
||||
def test_hisparse_host_residency_reported_as_hisparse_metrics():
|
||||
"""Host blocks sample into HiSparse's own residency metrics."""
|
||||
from tests.v1.core.test_prefix_caching import (
|
||||
make_hisparse_kv_cache_config,
|
||||
make_kv_cache_manager,
|
||||
)
|
||||
from tests.v1.core.utils import create_requests
|
||||
|
||||
collector = KVCacheMetricsCollector(sample_rate=1.0)
|
||||
manager = make_kv_cache_manager(
|
||||
make_hisparse_kv_cache_config(2, 2),
|
||||
max_model_len=128,
|
||||
enable_caching=True,
|
||||
hash_block_size=16,
|
||||
metrics_collector=collector,
|
||||
)
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
assert coordinator.host_manager is not None
|
||||
scheduler = HiSparseConnectorScheduler(async_speculative=False)
|
||||
scheduler.bind_coordinator(coordinator)
|
||||
device = manager.block_pool
|
||||
host = coordinator.host_manager.block_pool
|
||||
request = create_requests(1, num_tokens=16)[0]
|
||||
for pool, birth in ((device, 1), (host, 2)):
|
||||
with patch("time.monotonic_ns", return_value=birth * 10**9):
|
||||
block = pool.get_new_blocks(1)[0]
|
||||
assert block.block_id == 1
|
||||
pool.cache_full_blocks(request, [block], 0, 1, 16, 0)
|
||||
with patch("time.monotonic_ns", return_value=4_000_000_000):
|
||||
host.free_blocks([host.blocks[1]])
|
||||
host.evict_blocks({1})
|
||||
with patch("time.monotonic_ns", return_value=6_000_000_000):
|
||||
device.evict_blocks({1})
|
||||
|
||||
assert collector.drain_events() == [KVCacheEvictionEvent(5.0, 5.0, ())]
|
||||
stats = scheduler.get_kv_connector_stats()
|
||||
assert stats.data["host_block_lifetime_seconds"] == [2.0]
|
||||
assert stats.data["host_block_idle_before_evict_seconds"] == [2.0]
|
||||
|
||||
|
||||
def test_no_forward_enqueues_deferred_hisparse_transfers():
|
||||
"""A zero-token step must still enqueue deferred post-forward transfers."""
|
||||
connector = object.__new__(ActiveKVConnector)
|
||||
@@ -141,7 +187,7 @@ def test_full_graph_step_prepares_host_mirror_outside_model():
|
||||
connector = object.__new__(HiSparseConnector)
|
||||
connector.connector_worker = worker
|
||||
connector._get_connector_metadata = MagicMock(
|
||||
return_value=HiSparseConnectorMetadata(None, (), (), {}, True)
|
||||
return_value=HiSparseConnectorMetadata(None, (), (), {}, True, {})
|
||||
)
|
||||
req_id_per_token = torch.tensor([0, 1], dtype=torch.int32)
|
||||
attn_metadata = SimpleNamespace(
|
||||
@@ -279,6 +325,7 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
|
||||
)
|
||||
},
|
||||
True,
|
||||
{},
|
||||
)
|
||||
)
|
||||
# A verification step: four query tokens of one request fill the page.
|
||||
@@ -312,7 +359,7 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
|
||||
# The next step's start mirrors the drafter's rows, then hands the page over.
|
||||
compute_stream.wait_event.reset_mock()
|
||||
connector._get_connector_metadata.return_value = HiSparseConnectorMetadata(
|
||||
None, (), (), {}, True
|
||||
None, (), (), {}, True, {}
|
||||
)
|
||||
connector.start_load_kv(
|
||||
SimpleNamespace(),
|
||||
@@ -354,8 +401,56 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
|
||||
assert worker_meta.completed_transfer_counts == {transfer_id: 1}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shared", [True, False])
|
||||
@pytest.mark.parametrize("num_tokens", [0, 4])
|
||||
def test_zero_token_step_holds_tp_ranks_after_host_write_wait(
|
||||
monkeypatch, shared, num_tokens
|
||||
):
|
||||
"""Steps without tokens hold every TP rank after its shared event wait."""
|
||||
calls: list[tuple[str, object]] = []
|
||||
compute_stream = MagicMock()
|
||||
compute_stream.wait_event.side_effect = lambda event: calls.append(("wait", event))
|
||||
tp_group = MagicMock()
|
||||
tp_group.barrier.side_effect = lambda: calls.append(("barrier", None))
|
||||
monkeypatch.setattr(worker_module, "current_stream", lambda: compute_stream)
|
||||
monkeypatch.setattr(worker_module, "get_tp_group", lambda: tp_group)
|
||||
|
||||
worker = object.__new__(HiSparseConnectorWorker)
|
||||
worker.shared_host_region = MagicMock() if shared else None
|
||||
worker.host_write_events = (MagicMock(), MagicMock())
|
||||
worker.host_write_event = worker.host_write_events[1]
|
||||
worker._next_host_write_event = 0
|
||||
worker._slot_mapping_staging = None
|
||||
worker.cache_handles = []
|
||||
worker._pending_invalid_block_ids = []
|
||||
for name in (
|
||||
"_stage_row_mirror_mapping",
|
||||
"_finish_previous_step",
|
||||
"_release_completed_dma_descriptors",
|
||||
"_set_row_mirrors",
|
||||
"_clear_forward_mirror_state",
|
||||
"_copy_host_blocks",
|
||||
"_restore_pages",
|
||||
"_submit_transfers",
|
||||
):
|
||||
setattr(worker, name, MagicMock())
|
||||
|
||||
worker.start_step(
|
||||
HiSparseConnectorMetadata(None, (), (), {}, True, {}),
|
||||
None,
|
||||
[],
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
|
||||
expected = [("wait", worker.host_write_events[1])]
|
||||
if shared and not num_tokens:
|
||||
expected.append(("barrier", None))
|
||||
assert calls == expected
|
||||
|
||||
|
||||
def test_scheduled_prefix_hit_publishes_adopted_copies():
|
||||
"""Copies adopted after scheduling must reach the worker's block table."""
|
||||
"""Copies adopted after scheduling reach the worker as a residency update,
|
||||
leaving the block-table row the scheduler output carries untouched."""
|
||||
from tests.v1.core.test_prefix_caching import (
|
||||
HISPARSE_BLOCK_SIZE,
|
||||
_allocate_scheduled,
|
||||
@@ -401,7 +496,39 @@ def test_scheduled_prefix_hit_publishes_adopted_copies():
|
||||
num_scheduled_tokens={resumed.request_id: num_new_tokens},
|
||||
)
|
||||
|
||||
scheduler.build_connector_meta(scheduler_output)
|
||||
core_row = list(scheduler_output.scheduled_new_reqs[0].block_ids[2])
|
||||
|
||||
resident_ids = scheduler_output.block_table_updates[resumed.request_id][2]
|
||||
assert resident_ids[:3] == copy_ids[:3]
|
||||
metadata = scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
update = metadata.residency_updates[resumed.request_id]
|
||||
assert update.pages == [0, 1, 2, 3]
|
||||
assert update.block_ids[0][:3] == copy_ids[:3]
|
||||
assert manager.get_block_ids(resumed.request_id)[2] == core_row
|
||||
assert core_row[:3] == [0, 0, 0]
|
||||
|
||||
|
||||
def test_residency_updates_persist_by_state_row():
|
||||
"""Updates name requests by batch row but are stored by state row.
|
||||
|
||||
An update for a lost page and an appended page must leave the other pages
|
||||
and the other request's row intact across steps that reorder the batch.
|
||||
"""
|
||||
table = torch.zeros((4, 2, 4), dtype=torch.int32)
|
||||
update_hisparse_residency(
|
||||
table,
|
||||
{
|
||||
"a": SparseKVResidencyUpdate([0, 1, 2], ([1, 2, 3], [11, 12, 13])),
|
||||
"b": SparseKVResidencyUpdate([0, 1], ([4, 5], [14, 15])),
|
||||
},
|
||||
["a", "b"],
|
||||
torch.tensor([2, 0], dtype=torch.int32),
|
||||
)
|
||||
update_hisparse_residency(
|
||||
table,
|
||||
{"a": SparseKVResidencyUpdate([1, 3], ([0, 6], [0, 16]))},
|
||||
["b", "a"],
|
||||
torch.tensor([0, 2], dtype=torch.int32),
|
||||
)
|
||||
|
||||
assert table[[0, 2], 0].tolist() == [[4, 5, 0, 0], [1, 0, 3, 6]]
|
||||
assert table[[0, 2], 1].tolist() == [[14, 15, 0, 0], [11, 0, 13, 16]]
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnector,
|
||||
)
|
||||
@@ -82,7 +84,8 @@ def test_build_kv_connector_stats_round_trip():
|
||||
assert rebuilt.reduce() == stats.reduce()
|
||||
|
||||
|
||||
def test_prom_metrics_observe_host_usage_gauges():
|
||||
@pytest.mark.parametrize("kv_cache_metrics", [True, False])
|
||||
def test_prom_metrics_observe_host_metrics(kv_cache_metrics: bool):
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
@@ -91,12 +94,14 @@ def test_prom_metrics_observe_host_usage_gauges():
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.stats import (
|
||||
HiSparsePromMetrics,
|
||||
)
|
||||
from vllm.v1.metrics.stats import KVCacheEvictionEvent
|
||||
|
||||
class _FakeMetric:
|
||||
def __init__(self, **kwargs: Any):
|
||||
self.kwargs = kwargs
|
||||
self.increments: list[int | float] = []
|
||||
self.set_values: list[int | float] = []
|
||||
self.observed: list[float] = []
|
||||
|
||||
def labels(self, *labelvalues: object) -> "_FakeMetric":
|
||||
return self
|
||||
@@ -107,6 +112,9 @@ def test_prom_metrics_observe_host_usage_gauges():
|
||||
def set(self, value: int | float) -> None:
|
||||
self.set_values.append(value)
|
||||
|
||||
def observe(self, value: float) -> None:
|
||||
self.observed.append(value)
|
||||
|
||||
created: dict[str, _FakeMetric] = {}
|
||||
|
||||
class _NamedFake(_FakeMetric):
|
||||
@@ -115,7 +123,12 @@ def test_prom_metrics_observe_host_usage_gauges():
|
||||
created[kwargs["name"]] = self
|
||||
|
||||
prom = HiSparsePromMetrics(
|
||||
vllm_config=SimpleNamespace(kv_transfer_config=None),
|
||||
vllm_config=SimpleNamespace(
|
||||
kv_transfer_config=None,
|
||||
observability_config=SimpleNamespace(
|
||||
kv_cache_metrics=kv_cache_metrics, custom_histogram_buckets=None
|
||||
),
|
||||
),
|
||||
metric_types={Gauge: _NamedFake, Counter: _NamedFake, Histogram: _NamedFake},
|
||||
labelnames=["model_name"],
|
||||
per_engine_labelvalues={0: ["model"]},
|
||||
@@ -125,9 +138,21 @@ def test_prom_metrics_observe_host_usage_gauges():
|
||||
stats.record_snapshot(hits=3, misses=2, host_to_device_bytes=32)
|
||||
stats.record_host_usage(usage=0.5, pending_page_transfers=1)
|
||||
stats.record_host_usage(usage=0.75, pending_page_transfers=3)
|
||||
prom.observe(stats.to_dict())
|
||||
stats.record_host_evictions([KVCacheEvictionEvent(5.0, 2.0, (1.0, 2.0))])
|
||||
later = HiSparseKVConnectorStats()
|
||||
later.record_host_evictions([KVCacheEvictionEvent(3.0, 3.0, ())])
|
||||
prom.observe(stats.aggregate(later).to_dict())
|
||||
|
||||
assert created["vllm:hisparse_cache_hits"].increments == [3]
|
||||
assert created["vllm:hisparse_host_to_device_bytes"].increments == [32]
|
||||
assert created["vllm:hisparse_host_cache_usage_perc"].set_values == [0.75]
|
||||
assert created["vllm:hisparse_pending_page_transfers"].set_values == [3]
|
||||
if not kv_cache_metrics:
|
||||
assert not any("host_block" in name for name in created)
|
||||
return
|
||||
assert created["vllm:hisparse_host_block_lifetime_seconds"].observed == [5.0, 3.0]
|
||||
assert created["vllm:hisparse_host_block_idle_before_evict_seconds"].observed == [
|
||||
2.0,
|
||||
3.0,
|
||||
]
|
||||
assert created["vllm:hisparse_host_block_reuse_gap_seconds"].observed == [1.0, 2.0]
|
||||
|
||||
@@ -27,7 +27,6 @@ def _make_empty_scheduler_output():
|
||||
total_num_scheduled_tokens=0,
|
||||
scheduled_spec_decode_tokens={},
|
||||
scheduled_encoder_inputs={},
|
||||
num_common_prefix_blocks=[],
|
||||
finished_req_ids=set(),
|
||||
free_encoder_mm_hashes=[],
|
||||
kv_connector_metadata=ExampleConnectorMetadata(),
|
||||
|
||||
@@ -11,6 +11,7 @@ from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import msgspec
|
||||
import pytest
|
||||
import torch
|
||||
import zmq.asyncio
|
||||
@@ -57,7 +58,12 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
from .utils import create_request, create_scheduler, create_vllm_config
|
||||
from .utils import (
|
||||
create_request,
|
||||
create_scheduler,
|
||||
create_vllm_config,
|
||||
maybe_update_block_size,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
@@ -148,6 +154,7 @@ def test_region_length_validation_checks_replicated_gqa_heads(
|
||||
|
||||
|
||||
def _make_test_kv_cache_config() -> KVCacheConfig:
|
||||
block_size = maybe_update_block_size(16)
|
||||
return KVCacheConfig(
|
||||
num_blocks=0,
|
||||
kv_cache_tensors=[],
|
||||
@@ -160,7 +167,7 @@ def _make_test_kv_cache_config() -> KVCacheConfig:
|
||||
"model.layers.1.eagle_attn",
|
||||
],
|
||||
FullAttentionSpec(
|
||||
block_size=16,
|
||||
block_size=block_size,
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
@@ -170,6 +177,14 @@ def _make_test_kv_cache_config() -> KVCacheConfig:
|
||||
)
|
||||
|
||||
|
||||
def _make_worker_vllm_config(kv_role: str):
|
||||
return create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role=kv_role,
|
||||
block_size=maybe_update_block_size(16),
|
||||
)
|
||||
|
||||
|
||||
def _make_packed_mla_kv_cache_config(
|
||||
num_blocks: int, *, num_groups: int = 1
|
||||
) -> KVCacheConfig:
|
||||
@@ -218,6 +233,9 @@ class FakeMooncakeWrapper:
|
||||
def batch_register_memory(self, buffer_addresses, capacities) -> int:
|
||||
return 0
|
||||
|
||||
def batch_unregister_memory(self, buffer_addresses) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def test_align_transfer_regions_uses_layer_name_occurrences():
|
||||
"""Repeated layer names should align by occurrence order."""
|
||||
@@ -429,9 +447,7 @@ async def test_send_kv_to_decode_aligns_consumer_regions_by_layer_metadata(
|
||||
):
|
||||
"""Producer sends its PP layer shard to the matching consumer layer address."""
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
vllm_config = _make_worker_vllm_config("kv_producer")
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
prefill_connector = MooncakeConnector(
|
||||
@@ -1205,7 +1221,7 @@ def _xfer_meta(
|
||||
@contextlib.contextmanager
|
||||
def mooncake_register_worker(kv_cache_config, *, kv_role: str = "kv_consumer"):
|
||||
"""Build a worker whose engine registration and sender thread are patched."""
|
||||
vllm_config = create_vllm_config(kv_connector="MooncakeConnector", kv_role=kv_role)
|
||||
vllm_config = _make_worker_vllm_config(kv_role)
|
||||
connector_mod = (
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector"
|
||||
)
|
||||
@@ -1241,9 +1257,7 @@ async def test_receive_kv_selects_remote_pp_workers(
|
||||
expected_addrs: list[str],
|
||||
):
|
||||
"""Decode workers should not hard-code producer pp_rank 0."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_consumer"
|
||||
)
|
||||
vllm_config = _make_worker_vllm_config("kv_consumer")
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
decode_connector = MooncakeConnector(
|
||||
@@ -1314,7 +1328,7 @@ def test_resolve_need_send_accounts_for_remote_tp_fanout():
|
||||
@pytest.mark.asyncio
|
||||
async def test_heterogeneous_pp_waits_for_consumer_without_shared_layers():
|
||||
"""P4/D2 must retain KV until both D stages finish, including a no-op pull."""
|
||||
config = create_vllm_config(kv_connector="MooncakeConnector", kv_role="kv_producer")
|
||||
config = _make_worker_vllm_config("kv_producer")
|
||||
with (
|
||||
set_current_vllm_config(config),
|
||||
patch_worker_dependencies(),
|
||||
@@ -1386,9 +1400,7 @@ async def test_kv_producer(monkeypatch):
|
||||
Verifies memory offset calculation: ptr = base_addr + block_id * block_len.
|
||||
"""
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
vllm_config = _make_worker_vllm_config("kv_producer")
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
prefill_connector = MooncakeConnector(
|
||||
@@ -1589,9 +1601,7 @@ async def test_kv_consumuer(monkeypatch):
|
||||
|
||||
Verifies that MooncakeXferMetadata is correctly serialized and sent via ZMQ.
|
||||
"""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_consumer"
|
||||
)
|
||||
vllm_config = _make_worker_vllm_config("kv_consumer")
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies() as mocks:
|
||||
decode_connector = MooncakeConnector(
|
||||
@@ -1661,9 +1671,7 @@ async def test_kv_consumuer(monkeypatch):
|
||||
@pytest.mark.asyncio
|
||||
async def test_worker_get_finished_timeout(monkeypatch):
|
||||
"""Tests the cleanup mechanism for requests."""
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
vllm_config = _make_worker_vllm_config("kv_producer")
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
prefill_connector = MooncakeConnector(
|
||||
vllm_config,
|
||||
@@ -1713,7 +1721,7 @@ async def test_worker_get_finished_timeout(monkeypatch):
|
||||
def test_register_kv_caches(layout: KVCacheLayout, separate_kv_head_groups: bool):
|
||||
"""Tests the memory registration logic with the underlying Mooncake engine."""
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
block_size=maybe_update_block_size(16),
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
@@ -1794,7 +1802,10 @@ def test_register_noncontiguous_packed_allows_matching_pp():
|
||||
known. Rejecting every local PP>1 layout at startup is too strict.
|
||||
"""
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=64, dtype=torch.float16
|
||||
block_size=maybe_update_block_size(16),
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
layer_names = ["model.layers.0.self_attn", "model.layers.1.self_attn"]
|
||||
with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, _reg):
|
||||
@@ -1814,8 +1825,9 @@ def test_register_kv_caches_supports_mixed_mla_and_eagle_shapes():
|
||||
with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, reg):
|
||||
worker.use_mla = True
|
||||
worker.transfer_topo.is_mla = True
|
||||
mla_cache = torch.zeros((2, 16, 512), dtype=torch.float16)
|
||||
eagle_cache = torch.zeros((2, 16, 8, 64), dtype=torch.float16)
|
||||
block_size = maybe_update_block_size(16)
|
||||
mla_cache = torch.zeros((2, block_size, 512), dtype=torch.float16)
|
||||
eagle_cache = torch.zeros((2, block_size, 8, 64), dtype=torch.float16)
|
||||
kv_caches = {
|
||||
"model.layers.0.mla_attn": mla_cache,
|
||||
"model.layers.1.eagle_attn": eagle_cache,
|
||||
@@ -2218,6 +2230,215 @@ def test_prepare_transfer_regions_reuses_success_and_error():
|
||||
assert calls == 6
|
||||
|
||||
|
||||
def _short_xfer_meta(worker, layout_id: int, req_blocks: dict) -> MooncakeXferMetadata:
|
||||
return MooncakeXferMetadata(
|
||||
remote_hostname="peer",
|
||||
remote_port=9,
|
||||
remote_tp_size=worker.tp_size,
|
||||
remote_tp_rank=0,
|
||||
remote_pp_size=worker.pp_size,
|
||||
layout_id=layout_id,
|
||||
req_blocks=req_blocks,
|
||||
kv_caches_base_addr=[],
|
||||
block_lens=[],
|
||||
kv_block_lens=[],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_steady_pull_omits_layout_and_unknown_id_asks_for_resend():
|
||||
"""A later pull is much smaller, and an unseen layout_id is not guessed."""
|
||||
n = 146
|
||||
names = [f"model.layers.{i}.self_attn" for i in range(n)]
|
||||
req_blocks = {"d": ("xfer", [[1, 2, 3]])}
|
||||
full = MooncakeXferMetadata(
|
||||
remote_hostname="h",
|
||||
remote_port=1,
|
||||
remote_tp_size=4,
|
||||
remote_tp_rank=0,
|
||||
layout_id=1,
|
||||
req_blocks=req_blocks,
|
||||
kv_caches_base_addr=list(range(n)),
|
||||
block_lens=[4096] * n,
|
||||
kv_block_lens=[3000] * n,
|
||||
registered_layer_names=names,
|
||||
registered_layer_indices=list(range(n)),
|
||||
registered_group_indices=[i % 5 for i in range(n)],
|
||||
registered_shared_group_ids=[[] for _ in range(n)],
|
||||
registered_row_offsets=[i * 100 for i in range(n)],
|
||||
)
|
||||
steady = MooncakeXferMetadata(
|
||||
remote_hostname="h",
|
||||
remote_port=1,
|
||||
remote_tp_size=4,
|
||||
remote_tp_rank=0,
|
||||
layout_id=1,
|
||||
req_blocks=req_blocks,
|
||||
kv_caches_base_addr=[],
|
||||
block_lens=[],
|
||||
kv_block_lens=[],
|
||||
)
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
full_bytes = len(encoder.encode(full))
|
||||
steady_bytes = len(encoder.encode(steady))
|
||||
assert steady_bytes < full_bytes * 0.2
|
||||
|
||||
worker, _, regions, _ = _register_sliced_packed_mla([0, 1])
|
||||
calls = 0
|
||||
original = worker._get_transfer_regions
|
||||
|
||||
def counting(*args, **kwargs):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return original(*args, **kwargs)
|
||||
|
||||
worker._get_transfer_regions = counting
|
||||
|
||||
def handshake(layout_id: int) -> MooncakeXferMetadata:
|
||||
return _xfer_meta(
|
||||
regions,
|
||||
req_blocks,
|
||||
remote_hostname="peer",
|
||||
remote_port=9,
|
||||
remote_tp_size=worker.tp_size,
|
||||
remote_pp_size=worker.pp_size,
|
||||
layout_id=layout_id,
|
||||
)
|
||||
|
||||
prepared = worker._prepare_transfer_regions(handshake(1))
|
||||
assert prepared[2] is None
|
||||
assert calls == 2
|
||||
stored = worker._layout_by_peer[worker._peer_layout_key(handshake(1))]
|
||||
assert stored == (prepared[0], prepared[1])
|
||||
|
||||
# Same layout under a new id hits the tuple cache and must still be stored.
|
||||
worker._prepare_transfer_regions(handshake(3))
|
||||
assert calls == 2
|
||||
assert worker._peer_layout_key(handshake(3)) in worker._layout_by_peer
|
||||
|
||||
sock = MagicMock()
|
||||
sock.send_multipart = AsyncMock()
|
||||
await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 5, req_blocks))
|
||||
_, payload = sock.send_multipart.call_args[0][0]
|
||||
response = worker._xfer_resp_decoder.decode(payload)
|
||||
assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS
|
||||
assert calls == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_peer_layouts_evict_least_recently_used(monkeypatch):
|
||||
"""Stale layout ids are bounded, and an evicted peer is asked to resend."""
|
||||
monkeypatch.setattr(mooncake_connector, "_MAX_PEER_LAYOUTS", 2)
|
||||
worker, _, regions, _ = _register_sliced_packed_mla([0])
|
||||
req_blocks = {"d": ("xfer", [[1]])}
|
||||
|
||||
def handshake(layout_id: int) -> MooncakeXferMetadata:
|
||||
return _xfer_meta(
|
||||
regions,
|
||||
req_blocks,
|
||||
remote_hostname="peer",
|
||||
remote_port=9,
|
||||
remote_tp_size=worker.tp_size,
|
||||
remote_pp_size=worker.pp_size,
|
||||
layout_id=layout_id,
|
||||
)
|
||||
|
||||
worker._prepare_transfer_regions(handshake(1))
|
||||
worker._prepare_transfer_regions(handshake(3))
|
||||
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks))
|
||||
worker._prepare_transfer_regions(handshake(5))
|
||||
|
||||
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 1, req_blocks))
|
||||
assert worker._lookup_peer_layout(_short_xfer_meta(worker, 5, req_blocks))
|
||||
sock = MagicMock()
|
||||
sock.send_multipart = AsyncMock()
|
||||
await worker.send_kv_to_decode(b"d", sock, _short_xfer_meta(worker, 3, req_blocks))
|
||||
_, payload = sock.send_multipart.call_args[0][0]
|
||||
response = worker._xfer_resp_decoder.decode(payload)
|
||||
assert response.status == MooncakeXferResponseStatus.LAYOUT_MISS
|
||||
|
||||
|
||||
class _ScriptedSock:
|
||||
def __init__(self, responses: list[bytes], sent: list[bytes]):
|
||||
self._responses = responses
|
||||
self._sent = sent
|
||||
|
||||
def setsockopt(self, *_args, **_kwargs):
|
||||
return None
|
||||
|
||||
async def send(self, data: bytes):
|
||||
self._sent.append(data)
|
||||
|
||||
async def recv(self):
|
||||
return self._responses.pop(0)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
|
||||
def _single_pull() -> dict[str, PullReqMeta]:
|
||||
return {
|
||||
"d": PullReqMeta(
|
||||
d_req_id="d",
|
||||
transfer_id="t",
|
||||
local_block_ids=[[1]],
|
||||
remote_engine_id="p",
|
||||
remote_bootstrap_addr="http://bootstrap",
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
async def _pull_once(worker, addr: str, responses: list[MooncakeXferResponse]):
|
||||
sent: list[bytes] = []
|
||||
encoded = [worker._encoder.encode(response) for response in responses]
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake."
|
||||
"mooncake_connector.make_zmq_socket",
|
||||
return_value=_ScriptedSock(encoded, sent),
|
||||
):
|
||||
await worker.receive_kv_from_single_worker(addr, _single_pull())
|
||||
return sent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pull_resends_layout_when_prefill_forgot_it():
|
||||
worker, _, _, _ = _register_sliced_packed_mla([0])
|
||||
addr = "tcp://peer:1"
|
||||
worker._acked_layout_peers.add(addr)
|
||||
sent = await _pull_once(
|
||||
worker,
|
||||
addr,
|
||||
[
|
||||
MooncakeXferResponse(status=MooncakeXferResponseStatus.LAYOUT_MISS),
|
||||
MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.FINISH,
|
||||
ok_reqs=["d"],
|
||||
layout_id=worker._kv_layout_id,
|
||||
),
|
||||
],
|
||||
)
|
||||
assert len(sent) == 2
|
||||
assert len(sent[1]) > len(sent[0])
|
||||
assert addr in worker._acked_layout_peers
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pull_keeps_layout_for_prefill_that_does_not_echo_it():
|
||||
"""A prefill without layout_id support never gets a layout-less pull."""
|
||||
worker, _, _, _ = _register_sliced_packed_mla([0])
|
||||
addr = "tcp://old-peer:1"
|
||||
finish = MooncakeXferResponse(
|
||||
status=MooncakeXferResponseStatus.FINISH, ok_reqs=["d"]
|
||||
)
|
||||
first = await _pull_once(worker, addr, [finish])
|
||||
assert addr not in worker._acked_layout_peers
|
||||
second = await _pull_once(worker, addr, [finish])
|
||||
assert len(second[0]) == len(first[0])
|
||||
|
||||
|
||||
def test_coalesce_keeps_a_run_inside_its_starting_row():
|
||||
"""The row bound is the run start, not the previous slice."""
|
||||
row = 2500
|
||||
@@ -2240,7 +2461,7 @@ def _register_sliced_packed_mla(
|
||||
):
|
||||
"""Register column slices of one MLA packed row. Returns worker fields."""
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=16,
|
||||
block_size=maybe_update_block_size(16),
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.uint8,
|
||||
@@ -2286,7 +2507,7 @@ def test_sliced_packed_mla_registers_each_layer_view():
|
||||
KVCacheGroupSpec(
|
||||
[_layer_name(i) for i in layer_idxs],
|
||||
MLAAttentionSpec(
|
||||
block_size=16,
|
||||
block_size=maybe_update_block_size(16),
|
||||
num_kv_heads=1,
|
||||
head_size=64,
|
||||
dtype=torch.uint8,
|
||||
@@ -2436,9 +2657,7 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
|
||||
remote_block_len = LOCAL_BLOCK_LEN * P_TP_SIZE // d_tp_size
|
||||
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
vllm_config = _make_worker_vllm_config("kv_producer")
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
prefill_connector = MooncakeConnector(
|
||||
@@ -2568,3 +2787,29 @@ async def test_kv_producer_heterogeneous_tp(monkeypatch, d_tp_size):
|
||||
|
||||
prefill_worker.sender_loop = origin_sender_loop
|
||||
prefill_worker.shutdown()
|
||||
|
||||
|
||||
def test_sleep_unregisters_and_restores_kv_caches():
|
||||
spec = FullAttentionSpec(
|
||||
block_size=maybe_update_block_size(16),
|
||||
num_kv_heads=4,
|
||||
head_size=64,
|
||||
dtype=torch.float16,
|
||||
)
|
||||
layer_names = ["model.layers.0.self_attn", "model.layers.1.self_attn"]
|
||||
with mooncake_register_worker(_make_test_kv_cache_config()) as (worker, reg):
|
||||
for layer_name in layer_names:
|
||||
worker._layer_specs[layer_name] = spec
|
||||
raw = torch.zeros(2 * 2 * spec.page_size_bytes, dtype=torch.int8)
|
||||
views = dense_kv_cache_views(raw, spec, 2, 2, KVCacheLayout.LBHNC)
|
||||
worker.register_kv_caches(dict(zip(layer_names, views)))
|
||||
reg.reset_mock()
|
||||
with patch.object(
|
||||
worker.engine, "batch_unregister_memory", return_value=0
|
||||
) as unreg:
|
||||
worker.unregister_kv_caches()
|
||||
worker.unregister_kv_caches()
|
||||
unreg.assert_called_once_with([raw.data_ptr()])
|
||||
worker.restore_kv_caches()
|
||||
worker.restore_kv_caches()
|
||||
reg.assert_called_once_with([raw.data_ptr()], [raw.nbytes])
|
||||
|
||||
@@ -30,7 +30,12 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
|
||||
from .test_mooncake_connector import FakeMooncakeWrapper, patch_worker_dependencies
|
||||
from .utils import create_request, create_vllm_config, make_kv_cache_config
|
||||
from .utils import (
|
||||
create_request,
|
||||
create_vllm_config,
|
||||
make_kv_cache_config,
|
||||
maybe_update_block_size,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -224,12 +229,11 @@ def test_metadata_hma_block_ids():
|
||||
async def test_build_transfer_params_multi_group_trimming(monkeypatch):
|
||||
"""_build_transfer_params trims per-group blocks when local > remote."""
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=vllm_config.cache_config.block_size, swa_enabled=True
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer", block_size=block_size
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(block_size=block_size, swa_enabled=True)
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
connector = MooncakeConnector(
|
||||
@@ -320,12 +324,11 @@ async def test_build_transfer_params_multi_group_trimming(monkeypatch):
|
||||
async def test_build_transfer_params_group_count_mismatch(monkeypatch):
|
||||
"""_build_transfer_params reports an error when group counts differ."""
|
||||
monkeypatch.setenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "5")
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer"
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(
|
||||
block_size=vllm_config.cache_config.block_size, swa_enabled=True
|
||||
kv_connector="MooncakeConnector", kv_role="kv_producer", block_size=block_size
|
||||
)
|
||||
kv_cache_config = make_kv_cache_config(block_size=block_size, swa_enabled=True)
|
||||
|
||||
with set_current_vllm_config(vllm_config), patch_worker_dependencies():
|
||||
connector = MooncakeConnector(
|
||||
@@ -449,7 +452,7 @@ def test_request_finished_with_hma_groups():
|
||||
def _make_kv_consumer_worker(
|
||||
swa_enabled: bool = False, kv_cache_config: KVCacheConfig | None = None
|
||||
):
|
||||
block_size = 16
|
||||
block_size = maybe_update_block_size(16)
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_consumer",
|
||||
@@ -495,6 +498,7 @@ def test_worker_is_hma_required(swa_enabled, expected_is_hma):
|
||||
def test_worker_is_hma_required_multiple_full_attention_groups():
|
||||
"""Two full-attention groups with different block sizes also select
|
||||
request-level failure reporting, matching the scheduler."""
|
||||
block_size = maybe_update_block_size(16)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
kv_cache_tensors=[],
|
||||
@@ -502,13 +506,19 @@ def test_worker_is_hma_required_multiple_full_attention_groups():
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
FullAttentionSpec(
|
||||
block_size=16, num_kv_heads=4, head_size=16, dtype=torch.float16
|
||||
block_size=block_size,
|
||||
num_kv_heads=4,
|
||||
head_size=16,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer1"],
|
||||
FullAttentionSpec(
|
||||
block_size=32, num_kv_heads=4, head_size=16, dtype=torch.float16
|
||||
block_size=2 * block_size,
|
||||
num_kv_heads=4,
|
||||
head_size=16,
|
||||
dtype=torch.float16,
|
||||
),
|
||||
),
|
||||
],
|
||||
|
||||
@@ -8,6 +8,7 @@ validated by this test module.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from collections import OrderedDict
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -36,7 +37,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
|
||||
from .test_mooncake_connector import patch_worker_dependencies
|
||||
from .utils import create_request, create_vllm_config
|
||||
from .utils import create_request, create_vllm_config, maybe_update_block_size
|
||||
|
||||
|
||||
def noop_shutdown():
|
||||
@@ -147,6 +148,7 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch):
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_consumer",
|
||||
block_size=maybe_update_block_size(16),
|
||||
)
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(
|
||||
vllm_config.cache_config.block_size
|
||||
@@ -164,13 +166,12 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch):
|
||||
fa_spec = kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
gdn_spec = kv_cache_config.kv_cache_groups[1].kv_cache_spec
|
||||
fa_raw = torch.empty(num_blocks * fa_spec.page_size_bytes, dtype=torch.int8)
|
||||
gdn_raw = torch.empty(num_blocks * gdn_spec.page_size_bytes, dtype=torch.int8)
|
||||
gdn_block_stride = gdn_spec.page_size_bytes + 64
|
||||
gdn_raw = torch.empty(num_blocks, gdn_block_stride, dtype=torch.int8)
|
||||
(fa_cache,) = dense_kv_cache_views(
|
||||
fa_raw, fa_spec, num_blocks, 1, KVCacheLayout.LBHNC
|
||||
)
|
||||
(gdn_cache,) = dense_kv_cache_views(
|
||||
gdn_raw, gdn_spec, num_blocks, 1, KVCacheLayout.LBHNC
|
||||
)
|
||||
gdn_cache = gdn_raw[:, : gdn_spec.page_size_bytes]
|
||||
|
||||
worker.register_kv_caches(
|
||||
{
|
||||
@@ -184,7 +185,7 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch):
|
||||
"model.layers.0.self_attn",
|
||||
"model.layers.1.linear_attn",
|
||||
]
|
||||
assert worker.block_len_per_layer == [
|
||||
assert worker.kv_block_len_per_layer == [
|
||||
fa_spec.page_size_bytes,
|
||||
gdn_spec.page_size_bytes,
|
||||
]
|
||||
@@ -204,6 +205,7 @@ def test_register_kv_caches_scales_attention_len_to_kernel_block(monkeypatch):
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_consumer",
|
||||
block_size=maybe_update_block_size(16),
|
||||
)
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(
|
||||
vllm_config.cache_config.block_size
|
||||
@@ -242,8 +244,10 @@ async def test_build_transfer_params_uses_non_overlapping_physical_pages():
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(block_size=16)
|
||||
worker = object.__new__(MooncakeConnectorWorker)
|
||||
worker.shutdown = noop_shutdown
|
||||
# __new__ skips __init__, where this cache is created. register_kv_caches clears it.
|
||||
# __new__ skips __init__, where these caches are created.
|
||||
worker._prepared_transfer_regions = {}
|
||||
worker._layout_by_peer = OrderedDict()
|
||||
worker._acked_layout_peers = set()
|
||||
worker.use_mla = False
|
||||
worker.engine = SimpleNamespace(batch_register_memory=lambda *_: 0)
|
||||
worker.is_kv_consumer = True
|
||||
@@ -393,6 +397,7 @@ def test_register_kv_caches_deduplicates_shared_backing_memory(monkeypatch):
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_consumer",
|
||||
block_size=maybe_update_block_size(16),
|
||||
)
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(
|
||||
vllm_config.cache_config.block_size
|
||||
@@ -406,8 +411,13 @@ def test_register_kv_caches_deduplicates_shared_backing_memory(monkeypatch):
|
||||
)
|
||||
worker = connector.connector_worker
|
||||
|
||||
backing = torch.empty((4, 64), dtype=torch.float16)
|
||||
fa_cache = backing[:2, :16]
|
||||
fa_spec = kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
fa_cache_width = (
|
||||
fa_spec.page_size_bytes
|
||||
// torch.tensor([], dtype=torch.float16).element_size()
|
||||
)
|
||||
backing = torch.empty((4, fa_cache_width), dtype=torch.float16)
|
||||
fa_cache = backing[:2]
|
||||
gdn_cache = backing[:3]
|
||||
|
||||
with patch.object(
|
||||
@@ -439,6 +449,7 @@ def test_hybrid_gdn_transfer_params_preserve_group_identity(monkeypatch):
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_producer",
|
||||
block_size=maybe_update_block_size(16),
|
||||
)
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(
|
||||
vllm_config.cache_config.block_size
|
||||
@@ -568,6 +579,7 @@ def test_hybrid_gdn_keeps_packed_fa_and_gdn_regions_whole(
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeConnector",
|
||||
kv_role="kv_producer",
|
||||
block_size=maybe_update_block_size(16),
|
||||
)
|
||||
kv_cache_config = make_hybrid_gdn_kv_cache_config(
|
||||
vllm_config.cache_config.block_size
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
|
||||
MooncakeStoreConnectorStats,
|
||||
)
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
@@ -141,18 +142,74 @@ def test_validation_rejects_mamba_mode_directly():
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_requires_align_mode_for_mamba():
|
||||
vllm_config = _make_vllm_config()
|
||||
mamba_spec = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="none",
|
||||
def _make_mamba_none_kv_cache_config(
|
||||
attention_block_size: int | None, mamba_block_size: int
|
||||
) -> KVCacheConfig:
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="none",
|
||||
),
|
||||
)
|
||||
]
|
||||
if attention_block_size is not None:
|
||||
# Kimi K3 layout without prefix caching: the LCM hash unit exceeds both
|
||||
# group block sizes, so no lookup coordinator can be built.
|
||||
groups.insert(
|
||||
0,
|
||||
KVCacheGroupSpec(
|
||||
["attention"],
|
||||
FullAttentionSpec(
|
||||
block_size=attention_block_size,
|
||||
num_kv_heads=8,
|
||||
head_size=64,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
)
|
||||
return KVCacheConfig(num_blocks=4, kv_cache_tensors=[], kv_cache_groups=groups)
|
||||
|
||||
|
||||
def _make_role_vllm_config(kv_role: str, enable_lookup: bool, save_decode_cache: bool):
|
||||
vllm_config = create_vllm_config(
|
||||
kv_connector="MooncakeStoreConnector",
|
||||
kv_role=kv_role,
|
||||
kv_connector_extra_config={
|
||||
"enable_lookup": enable_lookup,
|
||||
"save_decode_cache": save_decode_cache,
|
||||
},
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[KVCacheGroupSpec(["mamba"], mamba_spec)],
|
||||
vllm_config.cache_config.enable_prefix_caching = False
|
||||
return vllm_config
|
||||
|
||||
|
||||
_MAMBA_NONE_LAYOUTS = pytest.mark.parametrize(
|
||||
"attention_block_size,mamba_block_size",
|
||||
[(None, 16), (1_048_576, 6_400)],
|
||||
ids=["mamba-only", "hybrid-no-prefix"],
|
||||
)
|
||||
|
||||
|
||||
@_MAMBA_NONE_LAYOUTS
|
||||
@pytest.mark.parametrize(
|
||||
"kv_role,enable_lookup,save_decode_cache",
|
||||
[
|
||||
("kv_consumer", True, False),
|
||||
("kv_consumer", False, True),
|
||||
("kv_producer", False, False),
|
||||
("kv_both", False, False),
|
||||
],
|
||||
)
|
||||
def test_scheduler_requires_align_mode_for_mamba_kv_transfer(
|
||||
kv_role, enable_lookup, save_decode_cache, attention_block_size, mamba_block_size
|
||||
):
|
||||
vllm_config = _make_role_vllm_config(kv_role, enable_lookup, save_decode_cache)
|
||||
kv_cache_config = _make_mamba_none_kv_cache_config(
|
||||
attention_block_size, mamba_block_size
|
||||
)
|
||||
|
||||
with (
|
||||
@@ -165,6 +222,30 @@ def test_scheduler_requires_align_mode_for_mamba():
|
||||
scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
|
||||
|
||||
|
||||
@_MAMBA_NONE_LAYOUTS
|
||||
def test_capacity_only_scheduler_allows_mamba_none_mode(
|
||||
attention_block_size, mamba_block_size
|
||||
):
|
||||
"""Capacity-only consumers transfer no KV, so any Mamba mode is allowed."""
|
||||
vllm_config = _make_role_vllm_config(
|
||||
"kv_consumer", enable_lookup=False, save_decode_cache=False
|
||||
)
|
||||
kv_cache_config = _make_mamba_none_kv_cache_config(
|
||||
attention_block_size, mamba_block_size
|
||||
)
|
||||
|
||||
with patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"scheduler.LookupKeyClient"
|
||||
) as mock_client:
|
||||
store_scheduler = scheduler.MooncakeStoreScheduler(vllm_config, kv_cache_config)
|
||||
assert store_scheduler.get_num_new_matched_tokens(MagicMock(), 0) == (0, False)
|
||||
meta = store_scheduler.build_connector_meta(SchedulerOutput.make_empty())
|
||||
|
||||
mock_client.return_value.lookup.assert_not_called()
|
||||
assert meta.requests == []
|
||||
|
||||
|
||||
def _make_block_stored(
|
||||
block_hash: bytes = b"hash", group_idx: int | None = None
|
||||
) -> BlockStored:
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
from math import lcm
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
|
||||
@@ -30,7 +31,13 @@ def _mamba_align(block_size=32):
|
||||
)
|
||||
|
||||
|
||||
def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=None):
|
||||
def _make_coord(
|
||||
groups,
|
||||
hash_block_size,
|
||||
use_eagle=False,
|
||||
retention_interval=None,
|
||||
enable_partial_hash_hits=False,
|
||||
):
|
||||
"""Construct a coordinator using the natural LCM of group block sizes as
|
||||
the scheduler block size — mirrors ``resolve_kv_cache_block_sizes`` for
|
||||
the test fixtures."""
|
||||
@@ -42,6 +49,7 @@ def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=Non
|
||||
hash_block_size=hash_block_size,
|
||||
use_eagle=use_eagle,
|
||||
retention_interval=retention_interval,
|
||||
enable_partial_hash_hits=enable_partial_hash_hits,
|
||||
)
|
||||
|
||||
|
||||
@@ -219,8 +227,7 @@ def test_coordinator_fine_grained_partial_tail_hit():
|
||||
KVCacheGroupSpec(["L0"], _full(32)),
|
||||
KVCacheGroupSpec(["L1"], _mamba_align(32)),
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
assert coord.enable_partial_hash_hits
|
||||
coord = _make_coord(groups, hash_block_size=16, enable_partial_hash_hits=True)
|
||||
hs = _hashes(4) # 4 hash units of 16 = 64 tokens; block 0 = [0,32), etc.
|
||||
# Both groups: full block 0 (key = last sub-hash hs[1]) + partial boundary
|
||||
# at token 48 (key = hs[2]). No hs[3] -> block 1 is not full.
|
||||
@@ -287,38 +294,6 @@ def test_store_mask_swa_only_window_around_each_lcm_boundary():
|
||||
assert masks[1] == [False, False, False, True, False, False, False, True]
|
||||
|
||||
|
||||
def test_store_mask_swa_does_not_double_scale_under_dcp():
|
||||
"""The coordinator receives DCP-resolved specs (worker.py applies
|
||||
resolve_dcp_kv_cache_spec) and indexes chunks in that already-scaled block
|
||||
size. So dcp_world_size must NOT further scale reachable_block_mask; the
|
||||
mask must be identical for any dcp_world_size given the same specs.
|
||||
|
||||
Regression: previously the coordinator forwarded its dcp_world_size, which
|
||||
double-scaled the SWA block size and produced a wrong mask when dcp > 1.
|
||||
"""
|
||||
full = _full(32)
|
||||
swa = _swa(block_size=8, sliding_window=8)
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
scheduler_block_size = lcm(32, 8)
|
||||
|
||||
def make(dcp_world_size):
|
||||
return MooncakeStoreCoordinator(
|
||||
groups,
|
||||
scheduler_block_size=scheduler_block_size,
|
||||
hash_block_size=8,
|
||||
dcp_world_size=dcp_world_size,
|
||||
)
|
||||
|
||||
mask_dcp1 = make(1).store_mask(64)
|
||||
mask_dcp2 = make(2).store_mask(64)
|
||||
|
||||
# dcp must not change the mask; both match the correct single-block-size
|
||||
# result (chunks ending at 32 and 64 -> blocks 3 and 7).
|
||||
expected = [False, False, False, True, False, False, False, True]
|
||||
assert mask_dcp1[1] == expected
|
||||
assert mask_dcp2[1] == expected
|
||||
|
||||
|
||||
def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
|
||||
"""Same hybrid layout but sliding_window=16 (= 2 SWA blocks). Each lcm
|
||||
boundary should now span two SWA tail blocks."""
|
||||
@@ -448,6 +423,49 @@ def test_store_mask_retention_interval_zero_keeps_only_replay_boundary():
|
||||
assert masks[1] == [i == 11 for i in range(16)]
|
||||
|
||||
|
||||
def test_store_mask_retention_interval_zero_eagle_keeps_lookup_tail():
|
||||
"""With EAGLE the replay boundary sits one scheduler block lower, as in
|
||||
core's ``replay_boundaries``: the lookup matches that block and drops
|
||||
back from it. The retained SWA tail must span the ``need`` contiguous
|
||||
blocks the lookup requires (cdiv(8 - 1, 8) + 1 = 2 here); the non-EAGLE
|
||||
boundary leaves only one of them inside the saved prefix."""
|
||||
coord = _make_coord(
|
||||
_retention_groups(), hash_block_size=8, use_eagle=True, retention_interval=0
|
||||
)
|
||||
# num_prompt=100 -> boundary 64 -> chunks 7, 8 (8 is the EAGLE peek).
|
||||
masks = coord.store_mask(96, num_prompt_tokens=100)
|
||||
assert masks[0] is None
|
||||
assert masks[1] == [i in (7, 8) for i in range(12)]
|
||||
# Aligned num_prompt=96 -> boundaries 32 (resend) and 64 (longer sibling).
|
||||
masks = coord.store_mask(96, num_prompt_tokens=96)
|
||||
assert masks[1] == [i in (3, 4, 7, 8) for i in range(12)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_prompt_tokens", [100, 96])
|
||||
def test_store_mask_retention_interval_zero_eagle_round_trip(num_prompt_tokens):
|
||||
"""Everything store_mask keeps is in the store; the EAGLE lookup of the
|
||||
same prompt must then reach the full block below the one it drops (64 of
|
||||
96). The unaligned prompt hit 0 before the fix; the aligned one already
|
||||
hit."""
|
||||
groups = _retention_groups()
|
||||
coord = _make_coord(groups, hash_block_size=8, use_eagle=True, retention_interval=0)
|
||||
hashes = _hashes(96 // coord.hash_block_size)
|
||||
exists: set[tuple[int, bytes]] = set()
|
||||
store_masks = coord.store_mask(96, num_prompt_tokens=num_prompt_tokens)
|
||||
for gid, (group, mask) in enumerate(zip(groups, store_masks, strict=True)):
|
||||
group_hashes = coord.block_hashes_for_spec(hashes, group.kv_cache_spec)
|
||||
for chunk_id, block_hash in enumerate(group_hashes):
|
||||
if mask is None or mask[chunk_id]:
|
||||
exists.add((gid, bytes(block_hash)))
|
||||
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hashes,
|
||||
max_length=96,
|
||||
cached_block_pool=ExternalCachedBlockPool(coord.hash_block_size, exists),
|
||||
)
|
||||
assert hit == 64
|
||||
|
||||
|
||||
def test_store_mask_retention_interval_keeps_segment_and_replay_tails():
|
||||
"""Sparse segment tails (interval=64 -> chunks 7,15) plus the replay
|
||||
boundary tail (num_prompt=100 -> chunk 11) coexist."""
|
||||
|
||||
@@ -18,11 +18,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator imp
|
||||
MooncakeStoreCoordinator,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
BoundaryPut,
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
LoadSpec,
|
||||
ReqMeta,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import ( # noqa: E501
|
||||
_partial_tail_non_mamba_puts,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( # noqa: E501
|
||||
KVCacheStoreRecvingThread,
|
||||
KVCacheStoreSendingThread,
|
||||
@@ -31,7 +35,6 @@ from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
@@ -379,6 +382,15 @@ def test_chunked_token_database_hash_block_size_smaller_than_block_size():
|
||||
assert out[1][2].hex() == fine_hashes[6].hex()
|
||||
|
||||
|
||||
def _resolve_partial_tail(thread, req: ReqMeta) -> ReqMeta:
|
||||
"""Add the tail's non-Mamba puts as the scheduler does before the worker."""
|
||||
req.boundary_puts = [BoundaryPut(*put) for put in req.boundary_puts or []]
|
||||
req.boundary_puts[:0] = _partial_tail_non_mamba_puts(
|
||||
thread.coord, req, [db.block_size for db in thread.token_databases]
|
||||
)
|
||||
return req
|
||||
|
||||
|
||||
def test_sub_block_partial_tail_offload_reads_cow_block():
|
||||
"""Sub-block prompt (the 900/128/1536 shape, scaled to 12/4/16): the
|
||||
partial tail is offloaded for both groups under the boundary sub-hash. The
|
||||
@@ -395,7 +407,12 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], mamba),
|
||||
]
|
||||
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
|
||||
coord = MooncakeStoreCoordinator(
|
||||
groups,
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=4,
|
||||
enable_partial_hash_hits=True,
|
||||
)
|
||||
assert coord.enable_partial_hash_hits
|
||||
|
||||
class _RecordingStore(_DictStore):
|
||||
@@ -442,11 +459,13 @@ def test_sub_block_partial_tail_offload_reads_cow_block():
|
||||
block_ids=([1], [2]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=20,
|
||||
boundary_state_offloads=[(1, mamba_cow_block, 12)],
|
||||
num_prompt_tokens=13,
|
||||
prefill_end_tokens=20,
|
||||
completed_token_len=20,
|
||||
boundary_puts=[(1, mamba_cow_block, 12)],
|
||||
)
|
||||
|
||||
send._maybe_offload_boundary_states(req)
|
||||
send._maybe_offload_boundary_states(_resolve_partial_tail(send, req))
|
||||
|
||||
# boundary = 12 // 4 * 4 = 12 -> keyed by hs[12 // 4 - 1] = hs[2].
|
||||
partial_hash = hs[2]
|
||||
@@ -474,7 +493,12 @@ def test_offload_syncs_event_before_put():
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], mamba),
|
||||
]
|
||||
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
|
||||
coord = MooncakeStoreCoordinator(
|
||||
groups,
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=4,
|
||||
enable_partial_hash_hits=True,
|
||||
)
|
||||
event = MagicMock()
|
||||
|
||||
class _FencedStore(_DictStore):
|
||||
@@ -514,12 +538,13 @@ def test_offload_syncs_event_before_put():
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=12,
|
||||
completed_token_len=12,
|
||||
store_job_id=1,
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
boundary_puts=[(1, 7, 8)],
|
||||
)
|
||||
req.current_event = event
|
||||
|
||||
send.add_request(req)
|
||||
send.add_request(_resolve_partial_tail(send, req))
|
||||
send._handle_request(send.request_queue.get())
|
||||
assert send.request_queue.qsize() == 0
|
||||
assert store._data
|
||||
@@ -533,7 +558,8 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
|
||||
offload must persist every FA block up to the boundary — the normal save
|
||||
floors to the lcm, so those blocks are otherwise never written and the
|
||||
consumer's per-group lookup would miss. The mamba boundary block still
|
||||
reads the core-provided CoW target."""
|
||||
reads the core-provided CoW target. Without EAGLE, the 12-token prompt's
|
||||
resend-safe checkpoint is 8, leaving two attention blocks to save."""
|
||||
full = FullAttentionSpec(block_size=4, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
@@ -545,7 +571,12 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], mamba),
|
||||
]
|
||||
coord = MooncakeStoreCoordinator(groups, scheduler_block_size=16, hash_block_size=4)
|
||||
coord = MooncakeStoreCoordinator(
|
||||
groups,
|
||||
scheduler_block_size=16,
|
||||
hash_block_size=4,
|
||||
enable_partial_hash_hits=True,
|
||||
)
|
||||
assert coord.enable_partial_hash_hits
|
||||
|
||||
class _RecordingStore(_DictStore):
|
||||
@@ -591,19 +622,19 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=12,
|
||||
boundary_state_offloads=[(1, mamba_cow_block, 12)],
|
||||
completed_token_len=12,
|
||||
boundary_puts=[(1, mamba_cow_block, 8)],
|
||||
)
|
||||
|
||||
send._maybe_offload_boundary_states(req)
|
||||
send._maybe_offload_boundary_states(_resolve_partial_tail(send, req))
|
||||
|
||||
# FA (block 4): full blocks ending at 4, 8 and 12, keyed by their normal
|
||||
# FA (block 4): full blocks ending at 4 and 8, keyed by their normal
|
||||
# block-end hashes; mamba (block 16): the partial boundary block under
|
||||
# the boundary sub-hash, read from the CoW target.
|
||||
expected = {
|
||||
token_dbs[0].key_for(hs[0]): [1 * 512],
|
||||
token_dbs[0].key_for(hs[1]): [2 * 512],
|
||||
token_dbs[0].key_for(hs[2]): [3 * 512],
|
||||
token_dbs[1].key_for(hs[2]): [10_000 + mamba_cow_block * 512],
|
||||
token_dbs[1].key_for(hs[1]): [10_000 + mamba_cow_block * 512],
|
||||
}
|
||||
assert store.puts == expected
|
||||
|
||||
@@ -612,11 +643,10 @@ def test_worker_lookup_hits_sub_block_partial_tail():
|
||||
"""worker.lookup must query sub-block keys when partial hash hits are on.
|
||||
|
||||
Regression test: ``lookup`` hard-coded ``fine_grained = False``, so the
|
||||
sub-block keys persisted by ``_sub_block_tail_puts`` were never probed and
|
||||
partial prefix hits silently returned 0 while the store side kept writing
|
||||
them. Here the mamba block (16) exceeds the hash unit (4), so
|
||||
``enable_partial_hash_hits`` is on and the lookup must find the stored
|
||||
boundary at 12.
|
||||
partial tail's sub-block keys were never probed and partial prefix hits
|
||||
silently returned 0 while the store side kept writing them. Here the
|
||||
mamba block (16) exceeds the hash unit (4), so ``enable_partial_hash_hits``
|
||||
is on and the lookup must find the stored boundary at 8.
|
||||
"""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
@@ -645,6 +675,8 @@ def test_worker_lookup_hits_sub_block_partial_tail():
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], mamba),
|
||||
],
|
||||
# The hit alignment the engine core resolves for this config.
|
||||
cache_hit_alignment_tokens=4,
|
||||
)
|
||||
vllm_config = _minimal_vllm_config(cache_block_size=16)
|
||||
# Hash unit 4 < mamba block 16 -> partial hash hits are enabled.
|
||||
@@ -673,7 +705,7 @@ def test_worker_lookup_hits_sub_block_partial_tail():
|
||||
replicate_config=MagicMock(),
|
||||
)
|
||||
|
||||
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//4-1]).
|
||||
# A 12-token prompt leaves a resend-safe sub-block checkpoint at 8.
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(5)]
|
||||
req = ReqMeta(
|
||||
req_id="r0",
|
||||
@@ -681,15 +713,16 @@ def test_worker_lookup_hits_sub_block_partial_tail():
|
||||
block_ids=([1], [2]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=20,
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
num_prompt_tokens=12,
|
||||
completed_token_len=12,
|
||||
boundary_puts=[(1, 7, 8)],
|
||||
)
|
||||
send_thread._maybe_offload_boundary_states(req)
|
||||
send_thread._maybe_offload_boundary_states(_resolve_partial_tail(send_thread, req))
|
||||
|
||||
worker.store = store
|
||||
|
||||
# A 13-token prompt sharing the prefix must hit the stored boundary at 12.
|
||||
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 12
|
||||
# A 13-token prompt sharing the prefix must hit the stored boundary at 8.
|
||||
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 8
|
||||
|
||||
|
||||
def test_worker_setup_tolerates_finer_scratch_group():
|
||||
@@ -711,13 +744,12 @@ def test_worker_setup_tolerates_finer_scratch_group():
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
# Scratch block 4 is not divisible by the hash unit 8 below.
|
||||
scratch = KpoolTailSpec(
|
||||
scratch = CircularBufferSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
)
|
||||
cfg = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
@@ -746,6 +778,8 @@ def test_worker_setup_tolerates_finer_scratch_group():
|
||||
KVCacheGroupSpec(["L1"], mamba),
|
||||
KVCacheGroupSpec(["L2"], scratch),
|
||||
],
|
||||
# The hit alignment the engine core resolves for this config.
|
||||
cache_hit_alignment_tokens=8,
|
||||
)
|
||||
vllm_config = _minimal_vllm_config(cache_block_size=16)
|
||||
# Hash unit 8 divides the participating groups (16) but not the scratch
|
||||
@@ -779,7 +813,7 @@ def test_worker_setup_tolerates_finer_scratch_group():
|
||||
group_participates=[True, True],
|
||||
)
|
||||
|
||||
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//8-1]).
|
||||
# Persist the sub-block partial tail at boundary 8 (keyed by hs[8//8-1]).
|
||||
hs = [BlockHash(bytes([i + 1]) * 8) for i in range(3)]
|
||||
req = ReqMeta(
|
||||
req_id="r0",
|
||||
@@ -787,10 +821,11 @@ def test_worker_setup_tolerates_finer_scratch_group():
|
||||
block_ids=([1], [2]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=20,
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
num_prompt_tokens=12,
|
||||
completed_token_len=12,
|
||||
boundary_puts=[(1, 7, 8)],
|
||||
)
|
||||
send_thread._maybe_offload_boundary_states(req)
|
||||
send_thread._maybe_offload_boundary_states(_resolve_partial_tail(send_thread, req))
|
||||
|
||||
worker.store = store
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user