Merge branch 'main' into patches

This commit is contained in:
2026-10-17 08:06:56 +00:00
222 changed files with 14823 additions and 3606 deletions
@@ -51,6 +51,7 @@ def _run(block_table, cu_num_tokens, kernel_block_size, out_len, device):
block_table.stride(0),
block_table.stride(1),
cu_num_tokens,
cu_num_tokens[1:] - cu_num_tokens[:-1],
KERNEL_BLOCK_SIZE=kernel_block_size,
BLOCK_SIZE=1024,
)
@@ -109,6 +110,7 @@ def test_ragged_batch_leaves_other_rows_untouched():
bt.stride(0),
bt.stride(1),
cu,
cu[1:] - cu[:-1],
KERNEL_BLOCK_SIZE=kbs,
BLOCK_SIZE=1024,
)
@@ -153,6 +155,7 @@ def test_over_provisioned_chunks_do_not_change_result():
bt.stride(0),
bt.stride(1),
cu,
cu[1:] - cu[:-1],
KERNEL_BLOCK_SIZE=kbs,
BLOCK_SIZE=1024,
)
+144 -7
View File
@@ -10,6 +10,7 @@ Known Issues:
import sys
from types import MethodType, SimpleNamespace
from typing import Literal
import pytest
import torch
@@ -47,6 +48,9 @@ from vllm.v1.attention.backends.mla.prefill.base import MLADimensions
from vllm.v1.attention.backends.mla.prefill.selector import (
MLAPrefillSelectorConfig,
)
from vllm.v1.attention.backends.mla.prefill.trtllm_ragged import (
is_cute_dsl_mla_prefill_supported,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.ops.flashmla import is_flashmla_dense_supported
from vllm.v1.kv_cache_interface import (
@@ -418,6 +422,11 @@ for backend in BACKENDS_TO_TEST:
if query_len_support != QueryLenSupport.SINGLE_ONLY:
SPEC_DECODE_BACKENDS.append(backend)
# Backends whose decode pipeline replays device-trimmed verify batches.
ADAPTIVE_VARLEN_BACKENDS = [
b for b in BACKENDS_TO_TEST if b == AttentionBackendEnum.ROCM_AITER_MLA
]
BACKEND_BLOCK_SIZES = {}
for backend in BACKENDS_TO_TEST:
supported_sizes = backend.get_class().get_supported_kernel_block_sizes()
@@ -470,6 +479,10 @@ BATCH_SPECS = {
"spec_decode_small": BatchSpec(
seq_lens=[128, 256, 512, 1024], query_lens=[4, 4, 4, 4]
),
# Adaptive verification: verify requests trimmed to 1..k+1 tokens on device.
"spec_decode_ragged_small": BatchSpec(
seq_lens=[128, 256, 512, 1024], query_lens=[8, 3, 1, 5]
),
"spec_decode_medium": BatchSpec(
seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[8, 8, 8, 8, 8, 8]
),
@@ -1474,6 +1487,7 @@ def run_attention_backend(
# Force the prefill backend selection (None means auto-select).
vllm_config.attention_config.mla_prefill_backend = prefill_backend
mla_attention_module.backend_supports_prefill_query_quantization.cache_clear()
# Set the current vllm config so that get_current_vllm_config() works
# in the backend implementations
@@ -1581,6 +1595,42 @@ def run_attention_backend(
return output
def _with_even_cpu_query_split(
common_attn_metadata: CommonAttentionMetadata,
) -> CommonAttentionMetadata:
"""Host query lengths as adaptive verification leaves them.
The device keeps the trimmed per-request lengths; the host sees the same
total spread evenly, the first requests taking the remainder
(AdaptiveVerificationManager.compact_batch).
"""
num_reqs = common_attn_metadata.num_reqs
total = int(common_attn_metadata.query_start_loc_cpu[num_reqs])
cpu_lens = torch.full((num_reqs,), total // num_reqs, dtype=torch.int32)
cpu_lens[: total % num_reqs] += 1
query_start_loc_cpu = torch.zeros_like(common_attn_metadata.query_start_loc_cpu)
torch.cumsum(cpu_lens, dim=0, out=query_start_loc_cpu[1 : num_reqs + 1])
return common_attn_metadata.replace(
query_start_loc_cpu=query_start_loc_cpu,
max_query_len=int(
(
common_attn_metadata.query_start_loc[1:]
- common_attn_metadata.query_start_loc[:-1]
)
.max()
.item()
),
)
def _fp8_cache_roundtrip(tensor: torch.Tensor, scale: float) -> torch.Tensor:
"""Reproduce cache storage rounding for the SDPA reference."""
fp8_dtype = current_platform.fp8_dtype()
limits = torch.finfo(fp8_dtype)
quantized = (tensor.float() / scale).clamp(limits.min, limits.max).to(fp8_dtype)
return (quantized.float() * scale).to(tensor.dtype)
def _run_backend_correctness(
default_vllm_config,
dist_init,
@@ -1595,6 +1645,9 @@ def _run_backend_correctness(
qk_nope_head_dim: int,
v_head_dim: int,
chunked_prefill_workspace_size: int | None = None,
adaptive_verification: bool = False,
trtllm_ragged_backend: Literal["trtllm-gen", "cute-dsl"] = "trtllm-gen",
use_prefill_query_quantization: bool = False,
):
"""Test that all backends produce similar outputs to a reference implementation
using torch.nn.functional.scaled_dot_product_attention.
@@ -1626,6 +1679,10 @@ def _run_backend_correctness(
) and AttentionBackendEnum.CUTLASS_MLA in backends_to_test:
# CUTLASS_MLA does not support non-1 Q/K scales
backends_to_test.remove(AttentionBackendEnum.CUTLASS_MLA)
if adaptive_verification:
backends_to_test = [
b for b in backends_to_test if b in ADAPTIVE_VARLEN_BACKENDS
]
if not backends_to_test:
pytest.skip(f"No backends support kv_cache_dtype={kv_cache_dtype}")
@@ -1670,18 +1727,25 @@ def _run_backend_correctness(
hf_config_override["num_attention_heads"]
)
vllm_config.cache_config.cache_dtype = kv_cache_dtype
vllm_config.attention_config.trtllm_ragged_backend = trtllm_ragged_backend
vllm_config.attention_config.use_prefill_query_quantization = (
use_prefill_query_quantization
)
# For spec decode tests, add a speculative_config to set the reorder_batch_threshold
if is_spec_decode_test:
from vllm.config import SpeculativeConfig
# Get the query length from the batch spec (they should all be uniform)
query_len = batch_spec.query_lens[0]
# Get the query length from the batch spec (they should all be uniform,
# except under adaptive verification, where the longest is k+1)
query_len = max(batch_spec.query_lens)
# Set num_speculative_tokens to query_len - 1
# (since threshold is 1 + num_spec_tokens)
# Use ngram method which doesn't require a draft model
vllm_config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=query_len - 1
method="ngram",
num_speculative_tokens=query_len - 1,
enable_adaptive_verification=adaptive_verification,
)
device = torch.device(f"{DEVICE_TYPE}:0")
@@ -1756,6 +1820,19 @@ def _run_backend_correctness(
# K_PE (rope component): [s_len, 1, qk_rope_head_dim]
k_pe_full = torch.randn(s_len, 1, qk_rope_head_dim, dtype=dtype, device=device)
kv_c_cached, k_pe_cached = kv_c_full, k_pe_full
kv_c_prefill, k_pe_prefill = kv_c_full, k_pe_full
if kv_cache_dtype.startswith("fp8"):
kv_c_cached = _fp8_cache_roundtrip(kv_c_full, k_scale)
k_pe_cached = _fp8_cache_roundtrip(k_pe_full, k_scale)
# Prefill reads cached context and unquantized new tokens.
kv_c_prefill = torch.cat(
[kv_c_cached[:context_len], kv_c_full[context_len:]]
)
k_pe_prefill = torch.cat(
[k_pe_cached[:context_len], k_pe_full[context_len:]]
)
# Determine if this sequence uses the decode pipeline or prefill
# pipeline for each backend
# NOTE: For spec decode tests with uniform query_len > 1, backends that
@@ -1805,10 +1882,10 @@ def _run_backend_correctness(
q_mqa = torch.cat([ql_nope, q_pe], dim=-1)
# K: [s_len, kv_lora_rank + qk_rope_head_dim]
# (broadcasted to all heads)
k_mqa = torch.cat([kv_c_full, k_pe_full.squeeze(1)], dim=-1)
k_mqa = torch.cat([kv_c_cached, k_pe_cached.squeeze(1)], dim=-1)
k_mqa = k_mqa.unsqueeze(1).expand(-1, num_q_heads, -1)
# V: [s_len, kv_lora_rank] (broadcasted to all heads)
v_mqa = kv_c_full.unsqueeze(1).expand(-1, num_q_heads, -1)
v_mqa = kv_c_cached.unsqueeze(1).expand(-1, num_q_heads, -1)
# Create custom attention mask for decode path:
# - Query tokens can attend to all context tokens
@@ -1837,13 +1914,18 @@ def _run_backend_correctness(
#######################################################
# Prefill path: MHA-style attention with full sequence
# Apply kv_b_proj to the full kv_c tensor
kv_nope_full = torch.einsum("sl,lnh->snh", kv_c_full, kv_b_proj_weight)
kv_nope_full = torch.einsum("sl,lnh->snh", kv_c_prefill, kv_b_proj_weight)
k_nope_full, v_full = kv_nope_full.split([qk_nope_head_dim, v_head_dim], dim=-1)
# Build attention inputs for full sequence
q_mha = torch.cat([q_nope, q_pe], dim=-1) # [q_len, num_heads, total_dim]
k_pe_full_expanded = k_pe_full.expand(-1, num_q_heads, -1)
k_pe_full_expanded = k_pe_prefill.expand(-1, num_q_heads, -1)
k_full = torch.cat([k_nope_full, k_pe_full_expanded], dim=-1)
if use_prefill_query_quantization:
fp8_dtype = current_platform.fp8_dtype()
q_mha = q_mha.to(fp8_dtype).to(dtype)
k_full = k_full.to(fp8_dtype).to(dtype)
v_full = v_full.to(fp8_dtype).to(dtype)
# Create custom attention mask:
# - Query tokens can attend to all context tokens
@@ -1951,6 +2033,13 @@ def _run_backend_correctness(
)
kv_cache_per_block_size[block_size] = kv_cache
if adaptive_verification:
# Only after the cache is filled: prepopulation reads the context
# lengths off the host query lengths.
metadata_per_block_size[block_size] = _with_even_cpu_query_split(
common_attn_metadata
)
# 4. Run vLLM backends and compare
rtol = 1e-2
atol = {
@@ -2111,6 +2200,8 @@ def test_backend_correctness(
_prefill_backend_dimension_params(),
)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("trtllm_ragged_backend", ["trtllm-gen", "cute-dsl"])
@pytest.mark.parametrize("use_prefill_query_quantization", [False, True])
def test_chunked_context_backend_correctness(
default_vllm_config,
dist_init,
@@ -2119,8 +2210,26 @@ def test_chunked_context_backend_correctness(
qk_nope_head_dim: int,
v_head_dim: int,
kv_cache_dtype: str,
trtllm_ragged_backend: Literal["trtllm-gen", "cute-dsl"],
use_prefill_query_quantization: bool,
):
"""Split, packed, and context-free requests match the SDPA reference."""
if prefill_backend != MLAPrefillBackendEnum.TRTLLM_RAGGED and (
trtllm_ragged_backend == "cute-dsl" or use_prefill_query_quantization
):
pytest.skip("TRTLLM ragged-specific test configuration")
if use_prefill_query_quantization and kv_cache_dtype == "auto":
pytest.skip("FP8 prefill queries require an FP8 KV cache")
if trtllm_ragged_backend == "cute-dsl":
query_dtype = (
current_platform.fp8_dtype()
if use_prefill_query_quantization
else torch.bfloat16
)
if not is_cute_dsl_mla_prefill_supported(
qk_nope_head_dim + 64, v_head_dim, query_dtype
):
pytest.skip("Unsupported CuTe DSL prefill dimensions or query dtype")
_run_backend_correctness(
default_vllm_config,
dist_init,
@@ -2135,4 +2244,32 @@ def test_chunked_context_backend_correctness(
qk_nope_head_dim=qk_nope_head_dim,
v_head_dim=v_head_dim,
chunked_prefill_workspace_size=1024,
trtllm_ragged_backend=trtllm_ragged_backend,
use_prefill_query_quantization=use_prefill_query_quantization,
)
@pytest.mark.skipif(
not ADAPTIVE_VARLEN_BACKENDS, reason="No adaptive varlen decode backend"
)
def test_adaptive_varlen_decode_correctness(
default_vllm_config, dist_init, workspace_init
):
"""Device query lengths 1..k+1 against an even host split, as adaptive
verification hands them to the target model's attention. TP 8 gives 16
heads, the padded Kimi-K3 count."""
_run_backend_correctness(
default_vllm_config,
dist_init,
workspace_init,
"spec_decode_ragged_small",
"deepseek-ai/DeepSeek-R1",
8,
"fp8",
1.0,
1.0,
None,
128,
128,
adaptive_verification=True,
)
@@ -63,9 +63,11 @@ class _ExpandPageIndicesKernel:
stride_0,
stride_1,
paged_kv_indptr,
num_real_tokens,
*,
KERNEL_BLOCK_SIZE,
BLOCK_SIZE,
CLAMP_TAIL=False,
):
# Both block-table strides are passed explicitly: the kernel no longer
# assumes the column stride is 1. This stand-in indexes the tensor
@@ -145,6 +147,7 @@ def _builder(
)
),
_mtp_decode_qlen=mtp_decode_qlen,
_varlen_decode_qlen=0,
_uniform_padded_mtp_qo_len=(AiterMLAMetadataBuilder._uniform_padded_mtp_qo_len),
_use_persistent_metadata=False,
kernel_block_size=kernel_block_size,
@@ -500,6 +503,7 @@ def test_non_causal_dcp_block_bypasses_cprr_in_forward(monkeypatch):
impl.qk_rope_head_dim = 64
impl.scale = 576**-0.5
decode = SimpleNamespace(
varlen_q_rows=None,
max_qo_len=4,
qo_indptr=torch.tensor([0, 4], dtype=torch.int32),
paged_kv_indptr=torch.tensor([0, 1], dtype=torch.int32),
@@ -563,6 +567,7 @@ def test_single_token_dcp_decode_returns_unpadded_lse(monkeypatch):
impl.qk_rope_head_dim = 64
impl.scale = head_dim**-0.5
decode = SimpleNamespace(
varlen_q_rows=None,
max_qo_len=1,
qo_indptr=torch.arange(num_tokens + 1, dtype=torch.int32),
paged_kv_indptr=torch.zeros(num_tokens + 1, dtype=torch.int32),
@@ -787,6 +792,7 @@ def test_segmented_dcp_verify_matches_causal_attention(monkeypatch):
torch.tensor([global_seq_len], dtype=torch.int32, device=device),
)
decode = SimpleNamespace(
varlen_q_rows=None,
max_qo_len=qlen,
paged_kv_indptr=torch.tensor(
[0, local_kv.shape[0]], dtype=torch.int32, device=device
@@ -1452,3 +1458,29 @@ def test_a_two_token_bf16_block_is_allowed(monkeypatch):
monkeypatch, num_heads=12, kv_cache_dtype="auto", qlen=2, mtp_qlen=8
)
assert metadata.has_persistent_metadata
def _adaptive_config(
*, adaptive: bool = True, dcp: int = 1, num_speculative_tokens: int = 7
) -> SimpleNamespace:
return SimpleNamespace(
speculative_config=SimpleNamespace(
num_speculative_tokens=num_speculative_tokens,
enable_adaptive_verification=adaptive,
parallel_drafting=False,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp),
use_v2_model_runner=True,
)
def test_varlen_cudagraph_bound_follows_adaptive_verification():
bound = AiterMLAMetadataBuilder.get_varlen_cudagraph_max_query_len
assert bound(_adaptive_config(), SimpleNamespace()) == 8
assert bound(_adaptive_config(adaptive=False), SimpleNamespace()) is None
# DCP routes plan per-row windows from host lengths.
assert bound(_adaptive_config(dcp=2), SimpleNamespace()) is None
# Kimi-K3 DSpark draft layers share the target's group; causality is
# decided per batch, so the merged non-causal spec keeps the bound.
shared_draft_group = SimpleNamespace(non_causal_multi_token_decode=True)
assert bound(_adaptive_config(), shared_draft_group) == 8
@@ -0,0 +1,294 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for KVCacheConfigBuilder resolution."""
from unittest.mock import MagicMock, patch
import pytest
from vllm.platforms import Platform
from vllm.v1.core.kv_cache_config_builder import (
KVCacheConfigBuilder,
get_kv_cache_config_builder,
)
from vllm.v1.core.kv_cache_planning import (
DefaultKVCacheConfigBuilder,
get_kv_cache_groups_from_builder,
)
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheGroupSpec
def _make_vllm_config(supports_grouping: bool = False) -> MagicMock:
"""Create a minimal mock VllmConfig for builder resolution tests.
``supports_grouping`` selects whether the resolved model class declares
``SupportsKVCacheGrouping`` (exposing a ``get_kv_cache_groups`` classmethod).
"""
cfg = MagicMock()
cfg.model_config.supports_kv_cache_grouping = supports_grouping
cfg.attention_config.hisparse_config = None
if supports_grouping:
cfg.model_config.registry.resolve_model_cls.return_value = (
_GroupedModel,
"arch",
)
return cfg
CUSTOM_PATH = "tests.v1.core.test_kv_cache_config_builder.CustomBuilder"
DEFAULT_PATH = "vllm.v1.core.kv_cache_planning.DefaultKVCacheConfigBuilder"
class CustomBuilder(DefaultKVCacheConfigBuilder):
"""A test builder subclass."""
pass
class _GroupedModel:
"""A test model class with a ``get_kv_cache_groups`` classmethod."""
@classmethod
def get_kv_cache_groups(cls, vllm_config, kv_cache_spec):
return []
class _PartialGroupedModel:
"""A test model class whose grouping omits layers."""
@classmethod
def get_kv_cache_groups(cls, vllm_config, kv_cache_spec):
from vllm.v1.kv_cache_interface import KVCacheGroupSpec
return [KVCacheGroupSpec(["a"], kv_cache_spec["a"])]
class TestPlatformHookResolution:
"""The platform hook selects only the placement builder."""
def test_default_hook_returns_default_builder(self):
cfg = _make_vllm_config()
assert Platform.get_kv_cache_config_builder_cls(cfg) == DEFAULT_PATH
class TestBuilderResolution:
@patch("vllm.platforms.current_platform")
@pytest.mark.parametrize(
("builder_cls_path", "expected_cls"),
[
(None, DefaultKVCacheConfigBuilder),
(CUSTOM_PATH, CustomBuilder),
],
)
def test_resolves_hook_selected_builder(
self, mock_platform, builder_cls_path, expected_cls
):
mock_platform.get_kv_cache_config_builder_cls.return_value = (
builder_cls_path or DEFAULT_PATH
)
cfg = _make_vllm_config()
assert type(get_kv_cache_config_builder(cfg)) is expected_cls
@patch("vllm.platforms.current_platform")
def test_resolves_fresh_instance_per_call(self, mock_platform):
mock_platform.get_kv_cache_config_builder_cls.return_value = CUSTOM_PATH
cfg = _make_vllm_config()
assert get_kv_cache_config_builder(cfg) is not get_kv_cache_config_builder(cfg)
def test_public_interface_declares_two_placement_hooks(self):
# get_kv_cache_configs lives in kv_cache_planning and is Core-owned;
# the builder only declares the placement hooks.
assert KVCacheConfigBuilder.__abstractmethods__ == {
"get_kv_cache_groups",
"get_kv_cache_config_from_groups",
}
assert issubclass(DefaultKVCacheConfigBuilder, KVCacheConfigBuilder)
assert (
"pool_bytes_per_block"
not in DefaultKVCacheConfigBuilder.__abstractmethods__
)
assert (
"max_memory_usage_bytes"
not in DefaultKVCacheConfigBuilder.__abstractmethods__
)
class TestModelGroupingPriority:
"""A model class with a get_kv_cache_groups classmethod owns grouping."""
def test_model_classmethod_owns_grouping(self):
cfg = _make_vllm_config(supports_grouping=True)
assert DefaultKVCacheConfigBuilder().get_kv_cache_groups(cfg, {}) == []
def test_model_grouping_rejects_hisparse(self):
cfg = _make_vllm_config(supports_grouping=True)
cfg.attention_config.hisparse_config = object()
with pytest.raises(ValueError, match="HiSparse"):
DefaultKVCacheConfigBuilder().get_kv_cache_groups(cfg, {})
class TestCoreChecksGrouping:
"""Core checks the grouping whether a model or a platform produced it."""
_SPEC = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=128, dtype="float16"
)
def test_model_grouping_is_checked(self):
cfg = _make_vllm_config(supports_grouping=True)
cfg.model_config.registry.resolve_model_cls.return_value = (
_PartialGroupedModel,
"arch",
)
with pytest.raises(AssertionError, match="miss layers"):
get_kv_cache_groups_from_builder(
DefaultKVCacheConfigBuilder(), cfg, {"a": self._SPEC, "b": self._SPEC}
)
@pytest.mark.parametrize(
("groups", "match"),
[
([KVCacheGroupSpec(["a"], _SPEC)], "miss layers"),
(
[KVCacheGroupSpec(["a", "b"], _SPEC), KVCacheGroupSpec(["b"], _SPEC)],
"overlaps",
),
],
)
def test_platform_grouping_is_checked(self, groups, match):
class FixedGroupsBuilder(DefaultKVCacheConfigBuilder):
def get_kv_cache_groups(self, vllm_config, kv_cache_spec):
return groups
with pytest.raises(AssertionError, match=match):
get_kv_cache_groups_from_builder(
FixedGroupsBuilder(),
_make_vllm_config(),
{"a": self._SPEC, "b": self._SPEC},
)
class TestPlatformCustomPriority:
"""A vendor platform can override the hook to customize priority."""
def test_platform_builder_wins_over_model_declaration(self):
class PlatformFirstPlatform(Platform):
@classmethod
def get_kv_cache_config_builder_cls(cls, vllm_config):
return CUSTOM_PATH
cfg = _make_vllm_config()
assert PlatformFirstPlatform.get_kv_cache_config_builder_cls(cfg) == CUSTOM_PATH
with patch("vllm.platforms.current_platform", PlatformFirstPlatform):
assert isinstance(get_kv_cache_config_builder(cfg), CustomBuilder)
class TestProjectionPreservesGroupMetadata:
"""PP projection must carry host_resident/role/enable_kv_transfer."""
def test_partial_projection_preserves_metadata(self):
from vllm.v1.core.kv_cache_planning import _project_kv_cache_groups_to_worker
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupRole,
)
spec = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=128, dtype="float16"
)
group = MagicMock()
group.layer_names = ["layer1", "layer2", "layer3"]
group.kv_cache_spec = spec
group.is_eagle_group = False
group.host_resident = True
group.enable_kv_transfer = False
group.role = KVCacheGroupRole.HISPARSE_SOURCE
projected = _project_kv_cache_groups_to_worker(
[group], {"layer1": spec, "layer2": spec}
)
assert len(projected) == 1
p = projected[0]
assert p.host_resident is True
assert p.enable_kv_transfer is False
assert p.role is KVCacheGroupRole.HISPARSE_SOURCE
assert p.layer_names == ["layer1", "layer2"]
class TestBytesPerBlockProbe:
"""The single-block probe must not be polluted by num_gpu_blocks_override."""
def test_probe_suppresses_override(self):
import torch
from vllm.config import CacheConfig, ModelConfig, VllmConfig
from vllm.v1.attention.backends.utils import resolve_kv_cache_layout
from vllm.v1.core.kv_cache_planning import (
DefaultKVCacheConfigBuilder,
_pool_bytes_per_block,
_pool_bytes_per_block_from_builder,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
)
spec = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=128, dtype=torch.float16
)
vc = VllmConfig(
model_config=ModelConfig(max_model_len=100),
cache_config=CacheConfig(block_size=16),
)
resolve_kv_cache_layout(vc, [["BLHNC"]], [spec])
groups = [
KVCacheGroupSpec(["a"], spec),
KVCacheGroupSpec(["b"], spec),
]
pure_c = _pool_bytes_per_block(vc, groups)
vc.cache_config.num_gpu_blocks_override = 500
probed_c = _pool_bytes_per_block_from_builder(
DefaultKVCacheConfigBuilder(), vc, groups
)
assert probed_c == pure_c
assert vc.cache_config.num_gpu_blocks_override == 500
@pytest.mark.parametrize("declares_pool_bytes", [False, True])
def test_several_device_allocations_need_declared_pool_bytes(
self, declares_pool_bytes
):
"""A placement with one allocation per group cannot be sized from one
backing; the builder must declare its per-block bytes."""
from vllm.v1.core.kv_cache_planning import _pool_bytes_per_block_from_builder
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
)
class PerGroupAllocationBuilder(DefaultKVCacheConfigBuilder):
def get_kv_cache_config_from_groups(self, vllm_config, groups, num_blocks):
tensors = [
KVCacheTensor(
size=page * num_blocks,
layers=group.layer_names,
layer_stride=page * num_blocks,
block_stride=page,
)
for group, page in zip(groups, (64, 32))
]
return KVCacheConfig(num_blocks, tensors, groups)
def _get_pool_bytes_per_block(self, vllm_config, groups):
return 64 + 32 if declares_pool_bytes else None
vc = _make_vllm_config()
vc.cache_config.num_gpu_blocks_override = None
groups = [KVCacheGroupSpec([name], MagicMock()) for name in "ab"]
builder = PerGroupAllocationBuilder()
if declares_pool_bytes:
assert _pool_bytes_per_block_from_builder(builder, vc, groups) == 96
else:
with pytest.raises(ValueError, match="_get_pool_bytes_per_block"):
_pool_bytes_per_block_from_builder(builder, vc, groups)
+53 -33
View File
@@ -18,6 +18,7 @@ from vllm.inputs import PromptType
from vllm.outputs import RequestOutput
from vllm.platforms import current_platform
from vllm.sampling_params import RequestOutputKind
from vllm.v1.engine import EngineCoreRequestType
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.core_client import DPLBAsyncMPClient
from vllm.v1.metrics.loggers import StatLoggerBase
@@ -336,12 +337,12 @@ async def _poll_flag(engine: AsyncLLM, want: bool, timeout: float) -> bool:
@pytest.mark.asyncio
async def test_dp_pause_late_request_does_not_block_drain():
"""A request arriving after pause must not leave the coordinator believing
"""A request arriving after pause must not make the coordinator believe
the engines are running.
Paused engines discard START_DP_WAVE, so nothing can report wave
completion afterwards; if forwarding the wake also marks the engines as
running, drain has no way back and can only end in a timeout.
Paused engines start no wave for it, so nothing could report wave
completion afterwards; a running flag set on its behalf would leave drain
no way back but a timeout.
MoE only: wave coordination is enabled iff the model is MoE, so a dense
model never reaches the coordinator state this exercises.
@@ -365,11 +366,9 @@ async def test_dp_pause_late_request_does_not_block_drain():
# A design that never reports the engines as running would satisfy
# every drain assertion below while destroying the signal, so pin it
# down first. The front-end sets its own copy optimistically when it
# forwards the wake, so sample only after several coordinator
# publishes (every 100ms while stats change) have overwritten it.
# Rank 0 reports the wave only after its first step, which can outlast
# the 2s on a cold device, so wait for the report rather than sample it.
# down first. Rank 0 reports the wave only after its first step, which
# can outlast the 2s on a cold device, so wait for the report rather
# than sample it.
await asyncio.sleep(2)
assert not long_request.done(), "the warmup request was too short to sample"
assert await _poll_flag(engine, True, timeout=30), (
@@ -382,24 +381,12 @@ async def test_dp_pause_late_request_does_not_block_drain():
await engine.pause_generation(mode="abort")
await engine.wait_for_requests_to_drain(drain_timeout=30)
# Awaiting add_request guarantees the new-request notification has been
# sent to the coordinator - the message that used to latch the flag.
collector = await engine.add_request(
request_id="late",
prompt=DP_PAUSE_PROMPT,
params=SamplingParams(max_tokens=5),
)
# The front-end marks the engines running off the back of that
# notification. This is what makes the test non-vacuous: it is the
# path that used to leave the coordinator stuck.
assert await _poll_flag(engine, True, timeout=5), (
"the late request did not notify the coordinator"
)
# It must settle back by itself. Unfixed it never does, because the
# paused engines discard the wake and so never report wave completion.
assert await _poll_flag(engine, False, timeout=60)
assert not await _poll_flag(engine, True, timeout=5)
# The late request was held rather than dropped: it completes on resume.
await engine.resume_generation()
@@ -409,6 +396,46 @@ async def test_dp_pause_late_request_does_not_block_drain():
break
@pytest.mark.asyncio
async def test_dp_request_reaching_rank_after_resume_wakes_peers():
"""A request can reach its rank only after resume (e.g. from another API
server), with no other work anywhere. That rank must wake its peers itself,
or it steps alone. Modelled here by holding the request's ADD on the client
side across pause and resume."""
with ExitStack() as after:
engine = AsyncLLM.from_engine_args(
_get_dp_pause_engine_args(expert_parallel=True)
)
after.callback(engine.shutdown)
client = engine.engine_core
params = SamplingParams(max_tokens=5)
await _consume(engine.generate(DP_PAUSE_PROMPT, params, "warmup"))
await engine.pause_generation(mode="abort")
assert await _poll_flag(engine, False, timeout=30)
held: list[tuple] = []
send_input = client._send_input
def hold_add(request_type, request, engine_id=None):
if request_type == EngineCoreRequestType.ADD:
held.append((request_type, request, engine_id))
return asyncio.sleep(0)
return send_input(request_type, request, engine_id)
client._send_input = hold_add
task = asyncio.create_task(
_consume(engine.generate(DP_PAUSE_PROMPT, params, "late"))
)
await asyncio.sleep(2)
assert held
await engine.resume_generation()
client._send_input = send_input
await send_input(*held[0])
await asyncio.wait_for(task, timeout=60)
@pytest.mark.asyncio
async def test_dp_sleep_late_request_does_not_block_drain():
"""The same latch, reached through sleep rather than pause.
@@ -441,14 +468,7 @@ async def test_dp_sleep_late_request_does_not_block_drain():
prompt=DP_PAUSE_PROMPT,
params=SamplingParams(max_tokens=5),
)
assert await _poll_flag(engine, True, timeout=5), (
"the request did not notify the coordinator"
)
# Sleeping engines cannot report wave completion, so a coordinator that
# marked them running when it forwarded the wake never hears otherwise.
assert await _poll_flag(engine, False, timeout=60)
assert not await _poll_flag(engine, True, timeout=5)
await engine.wake_up()
assert not await engine.is_sleeping()
@@ -628,7 +648,7 @@ async def test_dp_pause_keep_race_staggered_engines():
@pytest.mark.asyncio
async def test_dp_pause_barrier_request_deadlock():
"""Test that start_dp_wave is ignored while paused.
"""Test that a request sent while paused does not wake any engine.
Sequence:
1. Pause all engines (PAUSED_ALL).
@@ -697,7 +717,7 @@ async def test_dp_pause_barrier_request_deadlock():
# Yield so generate() preprocessing completes and
# add_request_async is called (which, in buggy code,
# would send FIRST_REQ and wake engine 1).
# would wake engine 1).
for _ in range(200):
await asyncio.sleep(0)
@@ -725,7 +745,7 @@ async def test_dp_pause_barrier_request_deadlock():
for t in mid_barrier_tasks:
t.cancel()
pytest.fail(
"Staggered barrier deadlocked — FIRST_REQ sent while "
"Staggered barrier deadlocked — a request sent while "
"paused caused collective-op mismatch between engines"
)
@@ -12,6 +12,7 @@ from typing import Any
import pytest
import torch
import vllm.envs as envs
from tests.conftest import HfRunner, VllmRunner
from tests.utils import (
large_gpu_mark,
@@ -168,9 +169,16 @@ def test_with_eagle3_spec_decoding(
reason=("XPU matmul/attention kernels are not batch-invariant"),
)
def test_with_ngram_gpu_spec_decoding(
vllm_runner: type[VllmRunner], hf_runner: type[HfRunner]
vllm_runner: type[VllmRunner],
hf_runner: type[HfRunner],
monkeypatch: pytest.MonkeyPatch,
):
"""Check ngram accuracy and acceptance across scheduling configurations."""
if current_platform.is_rocm():
# hipBLASLt picks GEMM solutions by batch shape, so sync and async runs
# can flip near-tie greedy tokens and shift ngram acceptance.
monkeypatch.setenv("VLLM_BATCH_INVARIANT", "1")
monkeypatch.setattr(envs, "VLLM_BATCH_INVARIANT", True)
ngram_gpu_config = {
"method": "ngram_gpu",
"num_speculative_tokens": 3,
@@ -8,6 +8,7 @@ import pytest
from tests.models.utils import check_logprobs_close
from vllm import LLM, SamplingParams
from vllm.inputs import TokensPrompt
from vllm.platforms import current_platform
pytestmark = pytest.mark.cpu_model
@@ -112,3 +113,24 @@ def test_prefix_cache_hit_matches_cold_cache(full_prefill_refs):
name_0="cold_cache",
name_1="warm_cache",
)
def test_prefix_cache_hit_reaches_last_full_block():
"""A repeated prompt must hit every full block of the hybrid block size.
The workers raise the block size to fit the GDN state pages, and the
engine core reruns its config checks after its handshake; the raised size
must survive them, or the prefill's state checkpoint misses the block
boundaries and nothing is reused. One chunk holds the whole prompt.
"""
llm = _make_llm(enable_prefix_caching=True, max_num_batched_tokens=2048)
block_size = llm.llm_engine.vllm_config.cache_config.block_size
# Ends mid-block: only the last full block's state can be reused.
num_tokens = block_size + block_size // 2
assert num_tokens <= 2048
prompt = TokensPrompt(prompt_token_ids=[1000 + i for i in range(num_tokens)])
sp = SamplingParams(max_tokens=1)
hits = [llm.generate(prompt, sp)[0].num_cached_tokens for _ in range(2)]
del llm
assert hits == [0, block_size]
+49 -1
View File
@@ -25,7 +25,7 @@ from vllm.engine.arg_utils import EngineArgs
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_default_torch_num_threads
from vllm.v1.core.sched.interface import PauseState
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.engine import EngineCoreRequest, EngineCoreRequestType
from vllm.v1.engine.core import DPEngineCoreProc, EngineCore, EngineCoreProc
from vllm.v1.executor.abstract import Executor
from vllm.v1.executor.uniproc_executor import UniProcExecutor
@@ -689,6 +689,54 @@ def test_dp_sync_interval_idle_pause_consensus_on_first_step(monkeypatch):
assert not core.pending_pause
@pytest.mark.parametrize(
"request_wave,engines_running,pause_state,announces",
[
pytest.param(3, False, PauseState.UNPAUSED, True, id="current-wave"),
pytest.param(2, False, PauseState.UNPAUSED, True, id="stale-wave"),
pytest.param(4, False, PauseState.UNPAUSED, True, id="newer-wave"),
pytest.param(3, True, PauseState.UNPAUSED, False, id="running"),
pytest.param(3, False, PauseState.PAUSED_ALL, False, id="paused"),
],
)
def test_idle_dp_rank_announces_wave_for_new_work(
request_wave, engines_running, pause_state, announces
):
"""An idle rank handed work announces the wave itself, since only engines
start waves; a paused rank waits for resume."""
core = object.__new__(DPEngineCoreProc)
core.has_coordinator = True
core.current_wave = 3
core.engines_running = engines_running
core.scheduler = MagicMock(pause_state=pause_state)
core.output_queue = MagicMock()
with patch.object(EngineCore, "add_request"):
core.add_request(MagicMock(), request_wave)
if announces:
assert core.engines_running
_, outputs = core.output_queue.put_nowait.call_args.args[0]
assert outputs.start_wave == max(request_wave, 3)
else:
core.output_queue.put_nowait.assert_not_called()
@pytest.mark.parametrize("paused", [False, True])
def test_paused_dp_rank_ignores_start_wave(paused):
"""A START_DP_WAVE still in flight when the pause completes must not wake
the paused rank."""
core = object.__new__(DPEngineCoreProc)
core.ignore_start_dp_wave = paused
core.engine_index = 0
core.current_wave = 3
core.engines_running = False
core._handle_client_request(EngineCoreRequestType.START_DP_WAVE, (3, 1))
assert core.engines_running != paused
def _pausable_engine_core_proc() -> EngineCoreProc:
"""A bare EngineCoreProc holding just the state pause_scheduler touches."""
core = object.__new__(EngineCoreProc)
+3 -3
View File
@@ -314,7 +314,7 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
client.current_wave = 0
client.resources = SimpleNamespace(stats_update_task=None)
client.stats_update_address = "inproc://scale-down-stats"
client.first_req_sock_addr = "inproc://scale-down-first-request"
client.scale_sock_addr = "inproc://scale-down-notify"
pause_started = asyncio.Event()
async def pause_scheduler(*args, **kwargs):
@@ -326,11 +326,11 @@ async def test_dplb_scale_down_routes_after_stale_stats_snapshot():
with (
zmq.asyncio.Context() as ctx,
ctx.socket(zmq.XPUB) as coordinator,
ctx.socket(zmq.PAIR) as first_req_socket,
ctx.socket(zmq.PAIR) as scale_socket,
):
client.ctx = ctx
coordinator.bind(client.stats_update_address)
first_req_socket.bind(client.first_req_sock_addr)
scale_socket.bind(client.scale_sock_addr)
client._ensure_stats_update_task()
scale_task = None
try:
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,55 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Widen the dummy weight init for the MoRIIO dummy-weight P/D test.
The dummy loader fills float weights in +-1e-3, which keeps every activation so
small that outputs barely depend on the input, so a wrong KV prefix would go
unnoticed. The P/D test names ``WideDummyWeights`` as each server's
``--worker-extension-cls``. Workers import it before they load the model, which
re-registers the ``dummy`` load format in every worker process with a loader
that fills 16/32-bit float weights in +-``MORIIO_DUMMY_WEIGHT_RANGE``; 8-bit
weights and scales keep the default init. The format keeps the name ``dummy``
so every config path that special-cases dummy weights still applies.
The test process itself never imports this module.
"""
import os
import torch
from torch import nn
from vllm.config import ModelConfig
from vllm.model_executor.model_loader import register_model_loader
from vllm.model_executor.model_loader.dummy_loader import DummyModelLoader
from vllm.model_executor.model_loader.reload.layerwise import get_layerwise_info
from vllm.model_executor.model_loader.reload.utils import get_layer_tensors
from vllm.model_executor.model_loader.weight_utils import (
initialize_single_dummy_weight,
)
_RANGE = float(os.environ["MORIIO_DUMMY_WEIGHT_RANGE"])
@register_model_loader("dummy")
class WideDummyModelLoader(DummyModelLoader):
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
# DummyModelLoader.load_weights with a wider init for 16/32-bit floats.
for layer in model.modules():
info = get_layerwise_info(layer)
if info.can_load():
self._process_online_quant_layer(layer, info)
continue
for name, tensor in get_layer_tensors(layer).items():
if name in layer._non_persistent_buffers_set:
continue
if tensor.is_floating_point() and torch.finfo(tensor.dtype).bits >= 16:
initialize_single_dummy_weight(
tensor, low=-_RANGE, high=_RANGE, name=name
)
else:
initialize_single_dummy_weight(tensor, name=name)
class WideDummyWeights:
"""Worker extension that only exists to import this module in workers."""
@@ -0,0 +1,736 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Single-node MoRIIO WRITE P/D checks on dummy weights.
Each case serves a real model architecture with ``--load-format dummy``, widened
by ``moriio_wide_dummy`` (see ``DUMMY_WEIGHT_RANGE``): the
config keeps the real hidden size, head geometry, MLA ranks and recurrent-state
shapes, so the connector registers and transfers exactly the production cache
layout, but the layer stack is cut to a few layers that still contain every
cache-group kind and the MoE is cut to a handful of experts. Dummy weights are
seeded, so the prefill, decode and reference servers hold identical weights.
Cases marked ``HMA_off`` run every server with
``--disable-hybrid-kv-cache-manager``, so all layers, sliding-window ones
included, share a single cache group.
Prefill runs on one GPU, decode on another, and MoRI moves the KV over xGMI.
Either way decode computes the last prompt token itself. For hybrid models
prefill computes only the first ``n - 1`` tokens and transfers the KV and the
recurrent state after them. For attention-only models prefill computes and
transfers all ``n``, and decode's scheduler, seeing the whole prompt cached,
recomputes the last token. A direct serve splits the same prompt the same way
when the first ``n - 1`` tokens end on a chunk boundary that both servers share,
so the prompts are chosen to make that happen: one is a single chunk plus one
token, the others are a whole number of hybrid cache blocks plus one token
(prefix caching ends chunks on block boundaries) or, for attention-only models,
a whole number of chunks plus one token. With the split matched, a correct
transfer reproduces the direct serve's first-token log-probabilities exactly,
so the comparison needs no noise allowance. The
servers compile with Inductor's deterministic mode, since otherwise each one
benchmarks its own reduction launch configs and separately started servers
can drift apart by a few bf16 logit steps. Guards:
* decode must report an external prefix-cache hit for every transferred token
of every prompt, so the match is not a local recompute;
* changing the first half of a prompt must move the direct serve's log-probs by
at least ``MIN_CONTEXT_EFFECT``, so a wrong prefix would be visible;
* ``test_corrupted_prefill_state_is_detected`` makes prefill compute its KV with
a different norm epsilon (same shapes, different values) and requires every
prompt to fail the comparison.
"""
import atexit
import contextlib
import json
import os
import random
import shutil
import signal
import subprocess
import sys
import tempfile
import time
from collections.abc import Callable, Iterator
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import pytest
import requests
from tests.utils import RemoteOpenAIServer
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
_ROOT = Path(__file__).resolve().parents[4]
_PROXY = (
_ROOT / "examples/disaggregated/disaggregated_serving/moriio_toy_proxy_server.py"
)
# Max allowed difference between centered first-token log-probabilities of the
# P/D and direct outputs, over the direct serve's top-k tokens. The split is
# matched, so correct transfers produce identical logits; this only absorbs the
# fp32 rounding of the log-softmax (~1e-7), far below one bf16 logit step.
TOLERANCE = float(os.getenv("MORIIO_DUMMY_E2E_TOL", "1e-5"))
# Several bf16 steps of the logits at DUMMY_WEIGHT_RANGE.
MIN_CONTEXT_EFFECT = 0.05
# Half-width of the uniform dummy init for 16/32-bit float weights, applied in
# every server worker by moriio_wide_dummy.py. The loader default (1e-3) shrinks
# every activation so far that a context change moves the first-token log-probs
# by ~1e-4; at 0.1 it moves them by >0.1 and a corrupted prefill changes the
# top-k set entirely.
DUMMY_WEIGHT_RANGE = 0.1
_WIDE_DUMMY_DIR = Path(__file__).resolve().parent
TOP_K = 20
MAX_TOKENS = 4
MAX_MODEL_LEN = 4096
# Prefill token budget of every server. Prompts longer than it are chunked, so
# the final-chunk save path runs; the short prompt fills exactly one chunk.
PREFILL_CHUNK = 299
SHORT_PROMPT_LEN = PREFILL_CHUNK + 1
# Hybrid blocks per aligned prompt, before the trailing token.
ALIGNED_BLOCKS = (1, 2)
# Prefill chunks per long attention-only prompt, before the trailing token.
ALIGNED_CHUNKS = (2, 4)
def _mori_available() -> bool:
try:
import mori.io # noqa: F401
except ImportError:
return False
return True
pytestmark = [
pytest.mark.skipif(
not current_platform.is_rocm() or not _mori_available(),
reason="MoRIIO P/D needs ROCm and MoRI",
),
pytest.mark.skipif(
current_platform.is_rocm() and current_platform.device_count() < 2,
reason="needs one GPU for prefill and one for decode",
),
]
# ---------------------------------------------------------------------------
# Model architectures
def _kimi_linear_overrides(cfg) -> dict[str, Any]:
# 3 KDA layers then 1 MLA layer; layer 0 stays dense (first_k_dense_replace).
linear = dict(cfg.linear_attn_config)
linear["kda_layers"] = [1, 2, 3]
linear["full_attn_layers"] = [4]
groups = max(1, cfg.num_expert_group)
return {
"num_hidden_layers": 4,
"linear_attn_config": linear,
"num_experts": 2 * groups,
"num_experts_per_token": 2,
"topk_group": min(cfg.topk_group, groups),
}
def _qwen3_5_moe_overrides(cfg) -> dict[str, Any]:
# 3 Gated DeltaNet layers then 1 full-attention layer.
return {
"text_config": {
"num_hidden_layers": 4,
"layer_types": ["linear_attention"] * 3 + ["full_attention"],
"num_experts": 4,
"num_experts_per_tok": 2,
}
}
def _qwen3_dense_overrides(cfg) -> dict[str, Any]:
# Attention-only: decode gets the KV for the whole prompt.
return {"num_hidden_layers": 4}
def _deepseek_v3_overrides(cfg) -> dict[str, Any]:
# MLA attention-only: layer 0 dense, layer 1 MoE, no MTP layer.
return {
"num_hidden_layers": 2,
"first_k_dense_replace": 1,
"n_routed_experts": 8,
"num_experts_per_tok": 2,
"n_group": 1,
"topk_group": 1,
"num_nextn_predict_layers": 0,
}
def _gemma3_overrides(cfg) -> dict[str, Any]:
# Five sliding-window layers then one global layer, as in the real
# 5:1 pattern. Without the hybrid KV cache manager vLLM gives the sliding
# layers full-attention cache specs, so all six share one group.
return {
"num_hidden_layers": 6,
"layer_types": ["sliding_attention"] * 5 + ["full_attention"],
}
@dataclass(frozen=True)
class ArchCase:
model: str
overrides: Callable[[Any], dict[str, Any]]
# Norm-epsilon override that perturbs every computed KV/state value
# without changing a single shape. Used by the corruption control.
corrupt_overrides: dict[str, Any]
env: dict[str, str] = field(default_factory=dict)
args: tuple[str, ...] = ()
# Has recurrent-state (mamba-style) layers, so prefill stops at n - 1.
hybrid: bool = True
ARCHS: dict[str, ArchCase] = {
"kimi_linear": ArchCase(
model="moonshotai/Kimi-Linear-48B-A3B-Instruct",
overrides=_kimi_linear_overrides,
corrupt_overrides={"rms_norm_eps": 1.0},
env={"VLLM_ROCM_USE_AITER": "1"},
args=("--trust-remote-code", "--mamba-ssm-cache-dtype", "float32"),
),
"qwen3_5_moe": ArchCase(
model="Qwen/Qwen3.5-35B-A3B",
overrides=_qwen3_5_moe_overrides,
corrupt_overrides={"text_config": {"rms_norm_eps": 1.0}},
# AITER's GDN conv-update kernel rejects the DS conv-state layout that
# MoRIIO's recurrent-state transfer requires.
env={"VLLM_ROCM_USE_AITER": "0"},
args=("--language-model-only",),
),
"qwen3_dense": ArchCase(
model="Qwen/Qwen3-32B",
overrides=_qwen3_dense_overrides,
corrupt_overrides={"rms_norm_eps": 1.0},
env={"VLLM_ROCM_USE_AITER": "0"},
hybrid=False,
),
"deepseek_v3": ArchCase(
model="deepseek-ai/DeepSeek-V3",
overrides=_deepseek_v3_overrides,
corrupt_overrides={"rms_norm_eps": 1.0},
env={"VLLM_ROCM_USE_AITER": "1"},
hybrid=False,
),
# Text-only Gemma 3. The multimodal Gemma3ForConditionalGeneration takes
# the inputs_embeds path on KV consumers (enable_mm_embeds is forced on
# there) and computes differently from a direct serve without any transfer.
"gemma3": ArchCase(
model="google/gemma-3-1b-it",
overrides=_gemma3_overrides,
corrupt_overrides={"rms_norm_eps": 1.0},
env={"VLLM_ROCM_USE_AITER": "0"},
hybrid=False,
),
}
def _arch_overrides(arch: ArchCase) -> dict[str, Any]:
from vllm.transformers_utils.config import get_config
return arch.overrides(get_config(arch.model, trust_remote_code=True))
def _merge(base: dict[str, Any], extra: dict[str, Any]) -> dict[str, Any]:
merged = dict(base)
for key, value in extra.items():
if isinstance(value, dict) and isinstance(merged.get(key), dict):
merged[key] = _merge(merged[key], value)
else:
merged[key] = value
return merged
# ---------------------------------------------------------------------------
# Serving
@dataclass(frozen=True)
class Case:
arch: str
attention_backend: str
prefill_graph: str
decode_graph: str
# Run every server of the case without the hybrid KV cache manager.
hma_off: bool = False
@property
def extra_args(self) -> tuple[str, ...]:
return ("--disable-hybrid-kv-cache-manager",) if self.hma_off else ()
@property
def id(self) -> str:
return (
f"{self.arch}-{self.attention_backend}-"
f"P_{self.prefill_graph}-D_{self.decode_graph}"
+ ("-HMA_off" if self.hma_off else "")
)
_DEFAULT = "FULL_AND_PIECEWISE"
CASES = [
# Kimi-Linear: KDA + MLA groups.
Case("kimi_linear", "ROCM_AITER_MLA", _DEFAULT, _DEFAULT),
Case("kimi_linear", "ROCM_AITER_MLA", "NONE", "NONE"),
Case("kimi_linear", "ROCM_AITER_MLA", "PIECEWISE", "PIECEWISE"),
Case("kimi_linear", "ROCM_AITER_MLA", _DEFAULT, "FULL_DECODE_ONLY"),
Case("kimi_linear", "TRITON_MLA", _DEFAULT, _DEFAULT),
# Qwen3.5: Gated DeltaNet + full-attention groups.
Case("qwen3_5_moe", "TRITON_ATTN", _DEFAULT, _DEFAULT),
Case("qwen3_5_moe", "TRITON_ATTN", "NONE", "NONE"),
Case("qwen3_5_moe", "TRITON_ATTN", "PIECEWISE", "PIECEWISE"),
Case("qwen3_5_moe", "TRITON_ATTN", _DEFAULT, "FULL_DECODE_ONLY"),
# ROCM_ATTN keeps a (2, num_blocks, ...) KV layout and opts out of KV
# connectors, so the second ROCm attention backend is the unified one.
Case("qwen3_5_moe", "ROCM_AITER_UNIFIED_ATTN", _DEFAULT, _DEFAULT),
# Attention-only: prefill transfers the whole prompt.
Case("qwen3_dense", "TRITON_ATTN", _DEFAULT, _DEFAULT),
Case("qwen3_dense", "ROCM_AITER_UNIFIED_ATTN", _DEFAULT, _DEFAULT),
# Without the hybrid KV cache manager: one cache group per model.
Case("qwen3_dense", "TRITON_ATTN", _DEFAULT, _DEFAULT, hma_off=True),
Case("deepseek_v3", "ROCM_AITER_MLA", _DEFAULT, _DEFAULT, hma_off=True),
Case("gemma3", "TRITON_ATTN", _DEFAULT, _DEFAULT, hma_off=True),
]
def _graph_args(mode: str) -> list[str]:
if mode == "NONE":
return ["--enforce-eager"]
# Inductor otherwise picks reduction launch configs by timing them, so each
# separately compiled server may sum in a different order and drift by a
# few bf16 logit steps; deterministic mode fixes the choice.
config = {
"cudagraph_mode": mode,
"inductor_compile_config": {"deterministic": True},
}
return ["--compilation-config", json.dumps(config)]
def _visible_devices() -> tuple[str, list[str]]:
selectors = ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")
selector = next((k for k in selectors if k in os.environ), selectors[0])
visible = os.getenv(selector)
devices = (
visible.split(",")
if visible
else [str(i) for i in range(current_platform.device_count())]
)
return selector, devices
def _device_env(selector: str, device: str) -> dict[str, str]:
# vLLM mirrors HIP_VISIBLE_DEVICES into CUDA_VISIBLE_DEVICES at import, so
# the test process carries both; pin every selector or the server refuses
# the mismatch. ROCR narrows the set first, so HIP/CUDA then index into it.
env = {selector: device}
for key in ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
if key != selector:
env[key] = "0" if selector == "ROCR_VISIBLE_DEVICES" else device
return env
# KV consumers force enable_mm_embeds on, which moves multimodal-capable models
# (Qwen3.5) from input_ids to inputs_embeds, but the AOT compile-cache key does
# not cover it, so a consumer can load a producer's graph and crash in
# profile_run. Keep each role's compile cache apart until
# vllm-project/vllm#54642 lands (vllm-project/vllm#50891).
_CACHE_ROOT = Path(tempfile.mkdtemp(prefix="moriio_dummy_e2e_cache_"))
atexit.register(shutil.rmtree, _CACHE_ROOT, ignore_errors=True)
def _server(
arch: ArchCase,
overrides: dict[str, Any],
backend: str,
graph: str,
device: str,
selector: str,
*,
port: int | None = None,
extra_args: tuple[str, ...] = (),
kv_transfer_config: dict[str, Any] | None = None,
) -> RemoteOpenAIServer:
port = port or get_open_port()
args = [
*arch.args,
*extra_args,
"--port",
str(port),
"--load-format",
"dummy",
"--worker-extension-cls",
"moriio_wide_dummy.WideDummyWeights",
"--hf-overrides",
json.dumps(overrides),
"--dtype",
"bfloat16",
"--attention-backend",
backend,
"--max-model-len",
str(MAX_MODEL_LEN),
"--max-num-batched-tokens",
str(PREFILL_CHUNK),
"--max-num-seqs",
"16",
"--kv-cache-memory-bytes",
str(2 * 1024**3),
"--gpu-memory-utilization",
"0.5",
"--enable-prefix-caching",
*_graph_args(graph),
]
role = "direct"
if kv_transfer_config is not None:
args += ["--kv-transfer-config", json.dumps(kv_transfer_config)]
role = kv_transfer_config["kv_role"]
env = {
"PATH": str(Path(sys.executable).parent) + os.pathsep + os.environ["PATH"],
**_device_env(selector, device),
"VLLM_HOST_IP": "127.0.0.1",
"VLLM_PORT": str(get_open_port()),
"VLLM_WORKER_MULTIPROC_METHOD": "spawn",
"VLLM_SSM_CONV_STATE_LAYOUT": "DS",
"VLLM_CACHE_ROOT": str(_CACHE_ROOT / role),
# Only this directory, so the source tree cannot shadow installed vllm.
"PYTHONPATH": os.pathsep.join(
filter(None, [str(_WIDE_DUMMY_DIR), os.environ.get("PYTHONPATH")])
),
"MORIIO_DUMMY_WEIGHT_RANGE": str(DUMMY_WEIGHT_RANGE),
**arch.env,
}
return RemoteOpenAIServer(
arch.model, args, env_dict=env, auto_port=False, max_wait_seconds=900
)
def _moriio_config(role: str, http_port: int, ping_port: int) -> dict[str, Any]:
return {
"kv_connector": "MoRIIOConnector",
"kv_role": role,
"kv_connector_extra_config": {
"backend": "xgmi",
"read_mode": False,
"host_ip": "127.0.0.1",
"proxy_ip": "127.0.0.1",
"proxy_ping_port": str(ping_port),
"http_port": str(http_port),
"handshake_port": str(get_open_port()),
"notify_port": str(get_open_port()),
},
}
@contextlib.contextmanager
def _proxy(ping_port: int) -> Iterator[str]:
# Free ports for both sockets, so cases from parallel shards on one host
# do not collide.
port = get_open_port()
process = subprocess.Popen(
[
sys.executable,
str(_PROXY),
"--port",
str(port),
"--service-discovery-port",
str(ping_port),
],
start_new_session=True,
)
try:
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
assert process.poll() is None, "MoRIIO proxy exited during startup"
try:
requests.get(f"http://127.0.0.1:{port}/", timeout=1)
break
except requests.ConnectionError:
time.sleep(0.2)
else:
raise TimeoutError("MoRIIO proxy did not start")
yield f"http://127.0.0.1:{port}"
finally:
if process.poll() is None:
os.killpg(process.pid, signal.SIGTERM)
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.wait(timeout=10)
@contextlib.contextmanager
def _pd_pair(
arch: ArchCase,
prefill_overrides: dict[str, Any],
decode_overrides: dict[str, Any],
case: Case,
) -> Iterator[tuple[str, RemoteOpenAIServer]]:
selector, devices = _visible_devices()
assert len(devices) >= 2, "needs one GPU for prefill and one for decode"
prefill_port, decode_port, ping_port = (get_open_port() for _ in range(3))
with _proxy(ping_port) as proxy_url, contextlib.ExitStack() as stack:
servers: list[RemoteOpenAIServer] = []
stack.callback(RemoteOpenAIServer.shutdown_many, servers)
with ThreadPoolExecutor(2) as pool:
futures = [
pool.submit(
_server,
arch,
prefill_overrides,
case.attention_backend,
case.prefill_graph,
devices[0],
selector,
port=prefill_port,
extra_args=case.extra_args,
kv_transfer_config=_moriio_config(
"kv_producer", prefill_port, ping_port
),
),
pool.submit(
_server,
arch,
decode_overrides,
case.attention_backend,
case.decode_graph,
devices[1],
selector,
port=decode_port,
extra_args=case.extra_args,
kv_transfer_config=_moriio_config(
"kv_consumer", decode_port, ping_port
),
),
]
for future in futures:
servers.append(future.result())
_wait_for_registration(proxy_url, arch.model)
yield proxy_url, servers[1]
def _wait_for_registration(proxy_url: str, model: str) -> None:
deadline = time.monotonic() + 120
while time.monotonic() < deadline:
# The proxy hands decode max_tokens - 1 (prefill emits the first
# token), so the probe needs at least 2 for decode to accept it.
try:
response = requests.post(
f"{proxy_url}/v1/completions",
json={"model": model, "prompt": [1000, 1001], "max_tokens": 2},
timeout=300,
)
# The proxy streams the decode body, so a decode-side error can
# surface as a truncated response rather than a status code.
if response.status_code == 200 and response.json()["choices"]:
return
except requests.RequestException:
pass
time.sleep(1)
raise TimeoutError("prefill/decode never registered with the MoRIIO proxy")
# ---------------------------------------------------------------------------
# Measurement
def _metric(server: RemoteOpenAIServer, name: str) -> list[Any]:
from prometheus_client.parser import text_string_to_metric_families
response = requests.get(server.url_for("metrics"), timeout=10)
response.raise_for_status()
return [
sample
for family in text_string_to_metric_families(response.text)
for sample in family.samples
if sample.name == name
]
def _block_size(server: RemoteOpenAIServer) -> int:
# Hybrid models pad the attention block to the recurrent-state page, so the
# effective size is only known once the server has started.
sizes = {
int(sample.labels["block_size"])
for sample in _metric(server, "vllm:cache_config_info")
}
assert len(sizes) == 1, f"expected one cache block size, got {sizes}"
return sizes.pop()
def _prompt_lengths(arch: ArchCase, server: RemoteOpenAIServer) -> list[int]:
# Hybrid chunks end on cache-block boundaries; attention-only chunks end on
# the prefill budget, so the last token is computed on its own either way.
if arch.hybrid:
unit, counts = _block_size(server), ALIGNED_BLOCKS
else:
unit, counts = PREFILL_CHUNK, ALIGNED_CHUNKS
aligned = [
k * unit + 1 for k in counts if k * unit + 1 + MAX_TOKENS <= MAX_MODEL_LEN
]
assert aligned, f"alignment unit {unit} leaves no aligned prompt"
return [SHORT_PROMPT_LEN, *aligned]
def _prompts(lengths: list[int]) -> list[list[int]]:
rng = random.Random(0)
return [[rng.randrange(1000, 30000) for _ in range(n)] for n in lengths]
def _first_step(url: str, model: str, prompt: list[int]) -> dict[str, float]:
response = requests.post(
f"{url}/v1/completions",
json={
"model": model,
"prompt": prompt,
"max_tokens": MAX_TOKENS,
"temperature": 0,
"logprobs": TOP_K,
"return_tokens_as_token_ids": True,
},
timeout=600,
)
response.raise_for_status()
top = response.json()["choices"][0]["logprobs"]["top_logprobs"]
assert top, "server returned no logprobs"
return top[0]
def _distance(ref: dict[str, float], got: dict[str, float]) -> float:
"""Max difference of centered log-probs over the shared top-k tokens.
Centering removes the log-sum-exp offset, leaving the logit differences the
KV actually determines.
"""
common = sorted(ref.keys() & got.keys())
if len(common) < len(ref) // 2:
return float("inf")
ref_mean = sum(ref[t] for t in common) / len(common)
got_mean = sum(got[t] for t in common) / len(common)
return max(abs((ref[t] - ref_mean) - (got[t] - got_mean)) for t in common)
def _external_hits(server: RemoteOpenAIServer) -> float:
samples = _metric(server, "vllm:external_prefix_cache_hits_total")
return sum(sample.value for sample in samples)
@dataclass
class Reference:
prompts: list[list[int]]
logprobs: list[dict[str, float]]
hybrid: bool
@property
def expected_external_hits(self) -> int:
# Hybrid prefill stops one token short; attention-only prefill hands
# decode the whole prompt.
return sum(len(p) - int(self.hybrid) for p in self.prompts)
def _reference(arch: ArchCase, overrides: dict[str, Any], case: Case) -> Reference:
"""Direct serve; also checks that the comparison can see a wrong prefix."""
selector, devices = _visible_devices()
# Prefill graphs run the chunks and the decode-only last step runs the same
# captured graph in every decode mode used here, so the prefill mode
# reproduces both halves of the P/D computation.
with _server(
arch,
overrides,
case.attention_backend,
case.prefill_graph,
devices[0],
selector,
extra_args=case.extra_args,
) as server:
url = server.url_root
lengths = _prompt_lengths(arch, server)
prompts = _prompts(lengths)
refs = [_first_step(url, arch.model, p) for p in prompts]
# Same tail, different early context: the comparison has to see it.
for prompt, ref in zip(prompts, refs):
altered = [t + 1 for t in prompt[: len(prompt) // 2]]
altered += prompt[len(prompt) // 2 :]
moved = _distance(ref, _first_step(url, arch.model, altered))
print(f"[sensitivity] len={len(prompt)} context-change distance={moved}")
assert moved >= MIN_CONTEXT_EFFECT, (
f"first-token log-probs barely depend on the context (moved "
f"{moved:.4g} for len {len(prompt)}); a wrong prefix would go "
"unnoticed"
)
# Determinism of the reference itself (served from its prefix cache).
again = _first_step(url, arch.model, prompts[-1])
print(f"[noise] repeat distance={_distance(refs[-1], again)}")
return Reference(prompts, refs, arch.hybrid)
def _pd_distances(
arch: ArchCase,
prefill_overrides: dict[str, Any],
decode_overrides: dict[str, Any],
case: Case,
ref: Reference,
) -> tuple[list[float], float]:
with _pd_pair(arch, prefill_overrides, decode_overrides, case) as (
proxy_url,
decode,
):
hits_before = _external_hits(decode)
distances = [
_distance(expected, _first_step(proxy_url, arch.model, prompt))
for prompt, expected in zip(ref.prompts, ref.logprobs)
]
hits = _external_hits(decode) - hits_before
lengths = [len(p) for p in ref.prompts]
print(f"[pd] {case.id} lens={lengths} distances={distances} external_hits={hits}")
return distances, hits
# ---------------------------------------------------------------------------
# Tests
@pytest.mark.parametrize("case", CASES, ids=lambda c: c.id)
def test_write_matches_direct_serve(case: Case):
arch = ARCHS[case.arch]
overrides = _arch_overrides(arch)
ref = _reference(arch, overrides, case)
distances, hits = _pd_distances(arch, overrides, overrides, case, ref)
# Every transferred token of every prompt must come from prefill.
assert hits == ref.expected_external_hits, (
f"decode reported {hits} external prefix-cache hits, expected "
f"{ref.expected_external_hits}; it did not use all the transferred KV"
)
lengths = [len(p) for p in ref.prompts]
bad = {n: d for n, d in zip(lengths, distances) if not d <= TOLERANCE}
assert not bad, f"P/D first-token log-probs differ from direct serve: {bad}"
@pytest.mark.parametrize("arch_name", sorted(ARCHS))
def test_corrupted_prefill_state_is_detected(arch_name: str):
"""Negative control: KV computed with different values must fail."""
arch = ARCHS[arch_name]
default = next(c for c in CASES if c.arch == arch_name)
case = Case(
arch_name, default.attention_backend, _DEFAULT, _DEFAULT, default.hma_off
)
overrides = _arch_overrides(arch)
ref = _reference(arch, overrides, case)
corrupted = _merge(overrides, arch.corrupt_overrides)
distances, hits = _pd_distances(arch, corrupted, overrides, case, ref)
assert hits == ref.expected_external_hits, (
"decode did not use the transferred KV; the control proves nothing"
)
assert all(d > TOLERANCE for d in distances), (
f"P/D outputs with wrongly computed prefill KV still matched the "
f"reference (distances {distances}); the comparison cannot detect "
"transfer corruption"
)
@@ -0,0 +1,47 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import itertools
import runpy
from pathlib import Path
from unittest.mock import AsyncMock
import httpx
import pytest
pytestmark = pytest.mark.cpu_test
@pytest.mark.asyncio
@pytest.mark.parametrize("endpoint", ["/v1/completions", "/v1/chat/completions"])
@pytest.mark.parametrize("stream", [False, True])
async def test_response_media_type_matches_payload(monkeypatch, endpoint, stream):
"""Clients must select SSE parsing for streaming completion responses."""
path = (
Path(__file__).parents[4]
/ "examples/disaggregated/mooncake_connector/mooncake_connector_proxy.py"
)
proxy = runpy.run_path(str(path))["_handle_completions"].__globals__
app = proxy["app"]
app.state.ready = asyncio.Event()
app.state.ready.set()
app.state.prefill_iterator = itertools.cycle([({}, 0)])
app.state.decode_iterator = itertools.cycle([0])
app.state.decode_clients = [{}]
payload = b'{"choices":[{"text":"hello"}]}'
body = b"data: " + payload + b"\n\ndata: [DONE]\n\n" if stream else payload
async def decode_response(*args, **kwargs):
yield body
monkeypatch.setitem(proxy, "send_request_to_service", AsyncMock())
monkeypatch.setitem(proxy, "stream_service_response", decode_response)
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="http://proxy"
) as client:
response = await client.post(endpoint, json={"stream": stream})
expected_type = "text/event-stream" if stream else "application/json"
assert response.headers["content-type"].split(";")[0] == expected_type
assert response.content == body
@@ -146,6 +146,7 @@ def _write_consumer_scheduler_for_finished_request(tp_size: int = 2):
scheduler.is_producer = False
scheduler.mode = MoRIIOMode.WRITE
scheduler.tp_size = tp_size
scheduler._has_mamba = False
scheduler._reqs_need_recv = {}
scheduler.unmap_request_id = MagicMock()
return scheduler
@@ -158,6 +159,10 @@ def _write_producer_scheduler(block_size: int = 1) -> Any:
scheduler.mode = MoRIIOMode.WRITE
scheduler.is_producer = True
scheduler.block_size = block_size
scheduler._has_mamba = False
scheduler.kv_cache_config = SimpleNamespace(
kv_cache_groups=[SimpleNamespace(kv_cache_spec=object())]
)
scheduler.blocks_per_sw = [0]
scheduler._group_block_sizes = [block_size]
scheduler.transfer_id_to_request_id = {}
@@ -944,6 +949,7 @@ def test_clamp_to_prompt_blocks_clamps_each_group_by_its_own_block_size():
# their own prompt-block count; a sliding-window group is left untouched.
scheduler = MoRIIOConnectorScheduler.__new__(MoRIIOConnectorScheduler)
scheduler.blocks_per_sw = [0, 0, 3]
scheduler._has_mamba = False
scheduler._group_block_sizes = [16, 32, 16]
req = SimpleNamespace(num_prompt_tokens=40)
@@ -12,12 +12,13 @@ wiring.
import importlib
from dataclasses import asdict
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.core.sched.output import KVConnectorBlockState, SchedulerOutput
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
from vllm.v1.kv_cache_interface import FullAttentionSpec, UniformTypeKVCacheSpecs
@@ -52,6 +53,7 @@ class _FakeScheduler(moriio_connector.MoRIIOConnectorScheduler): # type: ignore
attributes read by the method under test are set."""
def __init__(self, **attrs):
self.mode = MoRIIOMode.READ
self._mamba_group_ids: list[int] = []
self._attn_group_ids: list[int] = [0]
self._num_ssm_scratch_blocks = 0
@@ -287,7 +289,8 @@ def test_scheduler_rejects_mamba_group_without_two_states():
moriio_connector.MoRIIOConnectorScheduler(_gate_vllm_config(), "engine", config)
def test_scheduler_rejects_non_dspark_speculative_hybrid_read():
@pytest.mark.parametrize("read_mode", [True, False])
def test_scheduler_rejects_non_dspark_speculative_hybrid(read_mode):
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
@@ -298,7 +301,9 @@ def test_scheduler_rejects_non_dspark_speculative_hybrid_read():
with pytest.raises(moriio_common.MoRIIOError, match="speculative decoding"):
moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(speculative_config=_spec_config("ngram")),
_gate_vllm_config(
read_mode=read_mode, speculative_config=_spec_config("ngram")
),
"engine",
config,
)
@@ -336,14 +341,16 @@ def _dspark_wrapped_attention_config(*, dcp_sharded: bool = True):
@pytest.mark.usefixtures("restore_moriio_role")
@pytest.mark.parametrize("read_mode", [True, False])
@pytest.mark.parametrize(
"dcp_size,dcp_sharded,expected_tail", [(1, True, 4), (4, True, 1), (4, False, 4)]
)
def test_scheduler_accepts_dspark_with_wrapped_full_attention(
dcp_size, dcp_sharded, expected_tail
dcp_size, dcp_sharded, expected_tail, read_mode
):
config = _dspark_wrapped_attention_config(dcp_sharded=dcp_sharded)
vllm_config = _gate_vllm_config(
read_mode=read_mode,
speculative_config=_spec_config("dspark"),
block_size=4,
num_lookahead_tokens=7,
@@ -384,19 +391,234 @@ def test_scheduler_accepts_dspark_with_wrapped_full_attention(
)
def test_scheduler_rejects_hybrid_write():
config = SimpleNamespace(
kv_cache_groups=[
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=object()),
SimpleNamespace(enable_kv_transfer=True, kv_cache_spec=_mamba_spec()),
],
@pytest.mark.usefixtures("restore_moriio_role")
def test_hybrid_write_clamps_attention_and_excludes_scratch_slots():
config = _dspark_wrapped_attention_config()
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
sched = moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
)
config.transfer_groups = tuple(config.kv_cache_groups)
assert sched._clamp_to_prompt_blocks(
SimpleNamespace(num_prompt_tokens=9), [[10, 11, 12, 13], [1, 2, 3, 4]]
) == [[11], [1, 2]]
with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"):
moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False), "engine", config
@pytest.mark.usefixtures("restore_moriio_role")
def test_hybrid_write_resolves_current_state_instead_of_cached_block_ids():
config = _dspark_wrapped_attention_config()
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
sched = moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
)
state = KVConnectorBlockState(
req_ids={"req"},
resolve_block_ids=lambda _: ([0, 51, 52, 53], [31, 32, 33]),
boundary_state_offloads={},
)
assert sched._get_write_blocks(
SimpleNamespace(request_id="req", num_prompt_tokens=9),
[[10, 11, 12], [1]],
SimpleNamespace(kv_connector_block_state=state),
) == [[51], [31, 32]]
@pytest.mark.usefixtures("restore_moriio_role")
@pytest.mark.parametrize("external_tokens", [0, 9])
def test_hybrid_write_notifies_grouped_slots_or_releases_a_complete_local_hit(
external_tokens,
):
config = _dspark_wrapped_attention_config()
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
sched = moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False), "engine", config
)
sched.send_notify_block = Mock()
sched._send_transfer_release = Mock()
req = _mk_request(
list(range(10)),
params={
"do_remote_prefill": True,
"transfer_id": "tx",
"remote_host": "127.0.0.2",
"remote_notify_port": 7000,
},
)
req.request_id = "req"
sched.update_state_after_alloc(
req, _FakeBlocks(([10, 11, 12], [1, 2, 3])), external_tokens
)
if external_tokens:
sched.send_notify_block.assert_called_once_with(
req_id="req",
transfer_id="tx",
block_notify_list=[[10], [1, 2, 3]],
host="127.0.0.2",
port=7000,
)
sched._send_transfer_release.assert_not_called()
else:
sched.send_notify_block.assert_not_called()
sched._send_transfer_release.assert_called_once_with("tx", "127.0.0.2", 7000)
assert not req.kv_transfer_params["do_remote_prefill"]
@pytest.mark.usefixtures("restore_moriio_role")
def test_hybrid_write_keeps_separate_draft_groups_and_their_block_spans():
config = _dspark_wrapped_attention_config()
# A disabled cache is absent from the wire. Target and draft may have
# different token spans, and neither can borrow the other's block IDs.
draft = SimpleNamespace(
enable_kv_transfer=True,
kv_cache_spec=FullAttentionSpec(
block_size=4,
num_kv_heads=1,
head_size=2,
dtype=torch.float32,
dcp_sharded=False,
),
)
disabled = SimpleNamespace(enable_kv_transfer=False, kv_cache_spec=object())
config.kv_cache_groups.insert(1, disabled)
config.kv_cache_groups.append(draft)
config.transfer_groups = tuple(
g for g in config.kv_cache_groups if g.enable_kv_transfer
)
config.select_transfer_block_ids = lambda blocks: (blocks[0], blocks[2], blocks[3])
sched = moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(
read_mode=False, dcp_size=2, speculative_config=_spec_config("dspark")
),
"engine",
config,
)
req = _mk_request(
list(range(9)),
params={
"do_remote_decode": True,
"transfer_id": "tx",
},
)
req.request_id = "req"
req.status = moriio_connector.RequestStatus.FINISHED_LENGTH_CAPPED
blocks = ([10, 11, 12, 13], [99], [20, 21, 22], [30, 31, 32, 33])
expected = [[11], [20, 21], [30, 31, 32]]
assert sched._clamp_to_prompt_blocks(req, blocks) == expected
# Exercise the connector entry point too: completion must not flatten
# a multi-group WRITE payload through READ's single-attention helper.
delay_free, params = _FakeConnector(sched).request_finished_all_groups(req, blocks)
assert delay_free
assert params["remote_block_ids"] == expected
@pytest.mark.usefixtures("restore_moriio_role")
@pytest.mark.parametrize(
"blocks,delay_free",
[(([0, 51, 52, 53], [31, 32, 33]), True), (([], []), False)],
)
def test_hybrid_write_request_finished_defers_free_only_with_blocks_to_send(
blocks, delay_free
):
config = _dspark_wrapped_attention_config()
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
sched = moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
)
sched.is_producer = True
req = _mk_request(
list(range(9)), params={"do_remote_decode": True, "transfer_id": "tx"}
)
req.request_id = "req"
req.status = moriio_connector.RequestStatus.FINISHED_LENGTH_CAPPED
assert sched.request_finished(req, blocks)[0] is delay_free
# The WRITE ACK, not the READ watchdog, releases the deferred blocks.
if delay_free:
assert sched._deferred_send_deadlines["req"][1] == "tx"
else:
assert "req" not in sched._deferred_send_deadlines
@pytest.mark.usefixtures("restore_moriio_role")
def test_hybrid_write_build_meta_saves_current_state_on_the_final_chunk():
"""Chunked prefill must not ship block ids cached at admission: align-mode
state slots move between chunks without appending a block."""
config = _dspark_wrapped_attention_config()
config.select_transfer_block_ids = lambda blocks: tuple(blocks)
sched = moriio_connector.MoRIIOConnectorScheduler(
_gate_vllm_config(read_mode=False, dcp_size=2), "engine", config
)
sched.is_producer = True
set_role(moriio_common.ROLE.PRODUCER)
params = {
"do_remote_decode": True,
"transfer_id": "tx",
"remote_engine_id": "decode",
"remote_block_ids": [],
"remote_host": "127.0.0.1",
"remote_handshake_port": 6000,
"remote_notify_port": 6001,
}
req = _mk_request(list(range(9)), params=params)
req.request_id = "req"
req.num_computed_tokens = 0
sched._reqs_need_save["req"] = (req, [[10, 11, 12], [1]])
sched._req_kv_params["req"] = dict(params)
def _output(cached_req_ids, new_block_ids, num_scheduled):
return SimpleNamespace(
scheduled_cached_reqs=SimpleNamespace(
req_ids=cached_req_ids, new_block_ids=new_block_ids
),
num_scheduled_tokens={"req": num_scheduled},
new_block_ids_to_zero=None,
kv_connector_block_state=KVConnectorBlockState(
req_ids={"req"},
resolve_block_ids=lambda _: ([0, 51, 52, 53], [31, 32, 33]),
boundary_state_offloads={},
),
)
# First chunk: not final, so nothing is saved yet.
meta = sched.build_connector_meta(_output([], [], 4))
assert not meta.reqs_to_save
assert "req" in sched._reqs_need_pending_save
# Final chunk: stale cached ids plus new ids are ignored in favour of
# the scheduler's current table, clamped to the prompt.
req.num_computed_tokens = 4
meta = sched.build_connector_meta(_output(["req"], [([77], [78])], 5))
assert meta.reqs_to_save["req"].local_block_ids == [[51], [31, 32]]
assert "req" not in sched._reqs_need_pending_save
@pytest.mark.parametrize(
"mode,deadlines,expected",
[
(MoRIIOMode.WRITE, {"req": (0.0, "tx")}, True),
(MoRIIOMode.WRITE, {}, False),
(MoRIIOMode.READ, {"req": (0.0, "tx")}, False),
],
)
def test_has_pending_push_work_tracks_write_deferred_frees(mode, deadlines, expected):
connector = _FakeConnector(_FakeScheduler(_deferred_send_deadlines=deadlines))
connector.mode = mode
assert connector.has_pending_push_work() is expected
def test_has_pending_push_work_is_false_on_worker_role():
connector = _FakeConnector(None)
connector.mode = MoRIIOMode.WRITE
assert connector.has_pending_push_work() is False
def test_loaded_kv_cache_groups_are_every_transfer_group():
# The connector restores the request's own blocks, not a hash-addressed
# prefix, so a transfer-enabled Mamba group counts as loaded too.
connector = _FakeConnector(None)
connector._kv_cache_config = SimpleNamespace(
transfer_group_ids=(0, 2), prefix_cacheable_group_ids=(0,)
)
assert connector.get_loaded_kv_cache_group_ids(SimpleNamespace()) == (0, 2)
def test_split_block_groups_rejects_non_hybrid_use():
@@ -728,11 +950,15 @@ def test_session_build_rejects_per_layer_region_count_mismatch():
worker._get_built_session("prefill")
def test_register_kv_caches_rejects_hybrid_write_before_registration():
worker = _FakeWorker(mode=MoRIIOMode.WRITE, _transfer_layer_names={"kda.0"})
def test_register_kv_caches_rejects_invalid_hybrid_write_state_layout():
worker = _FakeWorker(
mode=MoRIIOMode.WRITE,
_transfer_layer_names={"kda.0"},
layer_to_spec={"kda.0": _mamba_spec(1)},
)
worker._is_mamba_layer = lambda _layer_name: True
with pytest.raises(moriio_common.MoRIIOError, match="READ mode only"):
with pytest.raises(moriio_common.MoRIIOError, match="exactly two Mamba states"):
worker.register_kv_caches({"kda.0": object()})
@@ -933,6 +1159,40 @@ def test_read_matches_only_requests_with_pending_remote_prefill(params, expected
assert scheduler.get_num_new_matched_tokens(request, 0) == expected
@pytest.mark.parametrize(
"has_mamba,expected", [(False, (32, True)), (True, (31, True))]
)
def test_write_consumer_matches_full_prompt_unless_hybrid(has_mamba, expected):
"""Without a mamba group (attention-only, or HMA disabled) prefill pushed
KV for all N tokens; with one it pushed h(N-1)."""
scheduler = _FakeScheduler(
is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=has_mamba
)
request = SimpleNamespace(
num_prompt_tokens=32, kv_transfer_params={"do_remote_prefill": True}
)
assert scheduler.get_num_new_matched_tokens(request, 0) == expected
@pytest.mark.parametrize(
"is_producer,has_mamba,expected_prompt_len",
[(True, True, 4), (True, False, 5), (False, True, 5)],
)
def test_on_new_request_truncates_only_hybrid_producers(
is_producer, has_mamba, expected_prompt_len
):
"""Attention-only prefill (including models run with HMA disabled)
computes all N prompt tokens; only a hybrid producer drops the last."""
scheduler = _FakeScheduler(is_producer=is_producer, _has_mamba=has_mamba)
request = _mk_request([10, 11, 12, 13, 14], params={"do_remote_decode": True})
scheduler.on_new_request(request)
assert request.num_prompt_tokens == expected_prompt_len
assert len(request.prompt_token_ids) == expected_prompt_len
def test_read_rematch_after_alloc_keeps_queued_transfer():
"""Clearing the flag blocks another external hit, not the queued READ."""
sched = _make_read_scheduler()
@@ -981,12 +1241,37 @@ def test_get_num_new_matched_tokens_write_plain_keeps_all_tokens():
assert is_async is True
@pytest.mark.parametrize("params", [None, {}, {"do_remote_prefill": False}])
def test_write_does_not_wait_again_after_receive_or_for_local_requests(params):
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True)
req = _mk_request(list(range(10)))
req.kv_transfer_params = params
assert sched.get_num_new_matched_tokens(req, 0) == (0, False)
def test_write_never_declares_an_empty_async_load():
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=False)
req = _mk_request(list(range(10)), params={"do_remote_prefill": True})
assert sched.get_num_new_matched_tokens(req, 10) == (0, False)
@pytest.mark.parametrize(
"prompt_len,computed,expected", [(16, 0, 15), (17, 16, 0), (1, 0, 0)]
)
def test_hybrid_write_advertises_only_the_state_computed_by_prefill(
prompt_len, computed, expected
):
sched = _FakeScheduler(is_producer=False, mode=MoRIIOMode.WRITE, _has_mamba=True)
req = _mk_request(list(range(prompt_len)), params={"do_remote_prefill": True})
assert sched.get_num_new_matched_tokens(req, computed) == (expected, expected > 0)
@pytest.mark.parametrize(
("mode", "num_computed_tokens", "expected", "is_async"),
[
(MoRIIOMode.READ, 0, 9, False),
(MoRIIOMode.READ, 10, 0, False),
(MoRIIOMode.WRITE, 2, 8, True),
(MoRIIOMode.WRITE, 2, 7, True),
],
)
def test_get_num_new_matched_tokens_supports_embeds_only_prompts(
@@ -7,6 +7,7 @@ import threading
from collections import OrderedDict
from types import SimpleNamespace
from typing import Any
from unittest.mock import Mock
import pytest
import torch
@@ -106,7 +107,7 @@ def _write_task(layer_name: str, transfer_id: str = "xfer") -> Any:
request_id="req",
transfer_id=transfer_id,
dst_engine_id="remote-engine",
local_block_ids=[1, 3],
local_block_ids=[[1, 3]],
remote_block_ids_hint=None,
layer_name=layer_name,
event=None,
@@ -345,6 +346,8 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
calls: list[str] = []
class FakeWorker:
_has_mamba: bool
layer_to_group: dict[str, int]
kv_caches: dict[str, torch.Tensor]
layer_name_to_local_kv_cache_metadata: dict[str, list[Any]]
@@ -364,11 +367,13 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
return ([call_id], [call_id + 10], [call_id + 20])
fake_worker = FakeWorker()
fake_worker._has_mamba = False
fake_worker.layer_to_group = dict.fromkeys(kv_caches, 0)
fake_worker.kv_caches = kv_caches
fake_worker.layer_name_to_local_kv_cache_metadata = {name: [] for name in kv_caches}
writer = MoRIIOWriter.__new__(MoRIIOWriter)
writer._worker_ref = lambda: fake_worker
request_info = RemoteAllocInfo(block_ids=[4, 5])
request_info = RemoteAllocInfo(block_ids=[[4, 5]])
remote_meta = _remote_meta()
dense0_plan = writer._prepare_transfer_plan(
@@ -395,7 +400,7 @@ def test_write_transfer_plan_caches_offsets_per_layer_geometry():
def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
request_info = RemoteAllocInfo(block_ids=[4, 5])
request_info = RemoteAllocInfo(block_ids=[[4, 5]])
wrapper = _wrapper_for_messages()
wrapper.done_remote_allocate_req_dict["xfer"] = request_info
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
@@ -411,6 +416,80 @@ def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
assert writer._sealed_writes["xfer"] == 2
def test_hybrid_write_attention_uses_its_own_group_not_the_first_group():
worker = SimpleNamespace(
_has_mamba=True,
kv_caches={"draft": torch.empty((8, 1, 4, 3))},
layer_to_group={"draft": 2},
_region_session_indices=lambda _: [5],
_compute_block_transfer_offsets=Mock(return_value=([1], [2], [3])),
)
writer = make_moriio_writer(worker)
task = _write_task("draft")
task.local_block_ids = [[10], [20], [30, 31]]
remote = RemoteAllocInfo(block_ids=[[40], [50], [60, 61, 62]])
meta = _remote_meta()
plan = writer._prepare_transfer_plan(task, remote, meta)
worker._compute_block_transfer_offsets.assert_called_once_with(
"draft", [30, 31], [60, 61], meta, remote_engine_id=task.dst_engine_id
)
assert plan.sess_idx == 5
def test_hybrid_write_posts_both_regions_before_layer_completion():
from unittest.mock import Mock
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
MoRIIOConnectorWorker,
)
wrapper = _wrapper_for_messages()
info = RemoteAllocInfo(block_ids=[[], [7], [9]])
wrapper.done_remote_allocate_req_dict["xfer"] = info
wrapper.write_remote_data = Mock(side_effect=["conv-status", "ssm-status"])
worker = SimpleNamespace(
moriio_wrapper=wrapper,
tp_rank=0,
get_engine_name_with_dp=lambda name, rank: name,
_get_built_session=lambda name: (["conv-session", "ssm-session"], None),
_is_mamba_layer=lambda name: True,
_mamba_payload_index_by_layer={"kda": 2},
_region_session_indices=lambda name: [0, 1],
_compute_mamba_transfer_offsets=Mock(
return_value=([10, 20, 30, 40], [50, 60, 70, 80], [2, 4, 8, 16], 3)
),
)
worker._mamba_blocks_for_layer = lambda name, blocks: (
MoRIIOConnectorWorker._mamba_blocks_for_layer(worker, name, blocks)
)
writer = make_moriio_writer(worker)
task = _write_task("kda")
task.local_block_ids = [[], [1], [3]]
task.remote_tp_size = 2
task.event = Mock()
writer._execute_write_task(task)
task.event.synchronize.assert_called_once()
worker._compute_mamba_transfer_offsets.assert_called_once_with(
"kda", [3], [9], remote_tp_size=2
)
assert wrapper.write_remote_data.call_args_list[0].args == (
[2, 4, 8],
[10, 20, 30],
[50, 60, 70],
"conv-session",
)
assert wrapper.write_remote_data.call_args_list[1].args == (
[16],
[40],
[80],
"ssm-session",
)
assert info.transfer_statuses == ["conv-status", "ssm-status"]
assert info.writes_done == 1
assert not info.completion_notified
def test_write_completion_notifies_once_after_all_sealed_writes_finish():
class FakeWrapper:
def __init__(self):
@@ -445,7 +524,7 @@ def test_write_completion_notifies_once_after_all_sealed_writes_finish():
)
wrapper = FakeWrapper()
request_info = RemoteAllocInfo(block_ids=[4, 5], writes_expected=2)
request_info = RemoteAllocInfo(block_ids=[[4, 5]], writes_expected=2)
request_info.transfer_statuses.extend(["status-a", "status-b"])
request_info.completion_request_id = "req"
# Final notify port is resolved in _execute_write_task; finalize reads it
@@ -487,6 +566,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain():
wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper)
wrapper.lock = threading.Lock()
wrapper._transfer_timeout = 1
wrapper.moriio_engine = object()
global_status = FakeStatus()
scoped_status = FakeStatus()
wrapper.transfer_status = [global_status]
@@ -500,7 +580,7 @@ def test_moriio_wrapper_waits_scoped_statuses_without_global_drain():
def test_write_failure_marks_terminal_and_clears_scheduled_state():
wrapper = _wrapper_for_messages()
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[[4, 5]])
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
writer._scheduled_writes["xfer"] = 2
writer._scheduled_layers["xfer"] = {"dense0", "indexer"}
@@ -518,7 +598,7 @@ def test_write_failure_marks_terminal_and_clears_scheduled_state():
def test_schedule_write_rejects_terminal_transfer_without_recreating_state():
wrapper = _wrapper_for_messages()
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[[4, 5]])
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
writer._scheduled_writes["xfer"] = 1
writer._scheduled_layers["xfer"] = {"dense0"}
@@ -610,7 +690,7 @@ def test_moriio_wrapper_routes_valid_messages(role, payload, expected):
if expected == "remote_blocks":
request_info = wrapper.done_remote_allocate_req_dict["xfer"]
assert request_info.block_ids == [4, 5]
assert request_info.block_ids == [[4, 5]]
assert request_info.decode_dp_rank == 3
elif expected == "write_done":
assert wrapper.done_write_cache_req_ids == ["xfer"]
@@ -11,6 +11,8 @@ import importlib
import importlib.util
import threading
import time
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
@@ -127,3 +129,62 @@ def test_wait_blocks_until_terminal():
assert time.monotonic() - start >= 0.15
finally:
timer.cancel()
@pytest.mark.parametrize("timeout,expected_ms", [(0.05, 50), (0.0001, 1)])
def test_native_wait_receives_all_statuses_and_a_positive_deadline(
timeout, expected_ms
):
statuses = [_pending(), _ok()]
engine = SimpleNamespace(
wait_all=Mock(return_value=moriio_engine.StatusCode.SUCCESS)
)
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=timeout)
wrapper.waiting_for_transfer_complete(statuses)
engine.wait_all.assert_called_once_with(statuses, timeout_ms=expected_ms)
@pytest.mark.parametrize(
"result,statuses,message",
[
("ERR_RDMA_OP", [_pending(), _bad()], "SQ full"),
("IN_PROGRESS", [_pending(), _ok()], "timed out"),
("ERR_INVALID_ARGS", [_pending()], "ERR_INVALID_ARGS"),
],
)
def test_native_wait_reports_failure_or_timeout(result, statuses, message):
engine = SimpleNamespace(
wait_all=Mock(return_value=getattr(moriio_engine.StatusCode, result))
)
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
with pytest.raises(TransferError, match=message):
wrapper.waiting_for_transfer_complete(statuses)
def test_wait_falls_back_without_native_status_enum(monkeypatch):
monkeypatch.setattr(moriio_engine, "StatusCode", None)
engine = SimpleNamespace(wait_all=Mock())
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
with pytest.raises(TransferError, match="SQ full"):
wrapper.waiting_for_transfer_complete([_bad(), _ok()])
engine.wait_all.assert_not_called()
def test_nonblocking_poll_does_not_call_native_wait():
engine = SimpleNamespace(wait_all=Mock())
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
assert wrapper.poll_transfer_batch([_pending()]) is TransferBatchState.PENDING
wrapper.waiting_for_transfer_complete([])
engine.wait_all.assert_not_called()
def test_native_wait_drains_only_the_unscoped_status_snapshot():
statuses = [_pending(), _ok()]
engine = SimpleNamespace(
wait_all=Mock(return_value=moriio_engine.StatusCode.SUCCESS)
)
wrapper = MoRIIOWrapper(moriio_engine=engine, transfer_timeout=0.05)
wrapper.transfer_status.extend(statuses)
wrapper.waiting_for_transfer_complete()
engine.wait_all.assert_called_once_with(statuses, timeout_ms=50)
assert wrapper.transfer_status == []
@@ -14,6 +14,8 @@ import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.moriio import moriio_engine
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
ROLE,
LayerTransferPlan,
MoRIIOError,
MoRIIOMode,
RemoteAllocInfo,
ReqMeta,
@@ -96,7 +98,7 @@ def _write_task(
request_id=request_id,
transfer_id=transfer_id,
dst_engine_id="remote-engine",
local_block_ids=[1, 3],
local_block_ids=[[1, 3]],
remote_block_ids_hint=None,
layer_name="layer0",
event=None,
@@ -249,6 +251,7 @@ def test_finalize_single_pod_port_uses_global_rank():
def test_write_blocks_for_req_forwards_routing_to_schedule_write():
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = False
captured: dict[str, Any] = {}
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
@@ -266,13 +269,14 @@ def test_write_blocks_for_req_forwards_routing_to_schedule_write():
assert captured["multi_pod_hosts"] == ["10.0.0.1", "10.0.0.2"]
assert captured["remote_dp_size_local"] == 8
assert captured["remote_ip"] == "10.0.0.1"
assert captured["local_block_ids"] == [1]
assert captured["remote_block_ids"] == [2]
assert captured["local_block_ids"] == [[1]]
assert captured["remote_block_ids"] == [[2]]
assert not hasattr(worker, "multi_pod_hosts")
def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = False
captured: dict[str, Any] = {}
def _capture_schedule_write_blocks(**kwargs: Any) -> None:
@@ -290,8 +294,8 @@ def test_write_blocks_for_req_falls_back_when_multi_pod_hosts_empty():
assert captured["multi_pod_hosts"] == ["10.0.0.9"]
assert captured["remote_dp_size_local"] == 4
assert captured["local_block_ids"] == [1]
assert captured["remote_block_ids"] == [2]
assert captured["local_block_ids"] == [[1]]
assert captured["remote_block_ids"] == [[2]]
def test_deferred_write_timeout_fails_transfer_on_every_decode_rank():
@@ -373,3 +377,73 @@ def test_write_failed_is_reported_as_failed_recving_on_owning_rank():
assert results.failed_recving == {"req-1"}
assert wrapper.pop_finished_write_req_ids() == set()
assert list(worker._unmatched_write_failures) == ["tOtherRank"]
def test_hybrid_write_rejects_unequal_tp_before_scheduling_any_layer():
from unittest.mock import Mock
worker = MoRIIOConnectorWorker.__new__(MoRIIOConnectorWorker)
worker._has_mamba = True
worker.world_size = 2
worker.schedule_write_blocks = Mock()
with pytest.raises(NotImplementedError, match="heterogeneous-TP"):
worker._write_blocks_for_req("req", _req_meta(), "attention", kv_layer=None)
worker.schedule_write_blocks.assert_not_called()
@pytest.mark.parametrize(
"notify,expected",
[
# Plain WRITE peers still send one flat attention list.
([7, 8], [[7, 8]]),
# Hybrid WRITE sends one list per transfer group, kept as-is even
# when the first group is empty (full attention hit, state only).
([[7], [9]], [[7], [9]]),
([[], [9]], [[], [9]]),
],
)
def test_remote_blocks_message_normalizes_to_grouped_block_ids(notify, expected):
wrapper = _real_wrapper()
with patch.object(moriio_engine, "get_role", return_value=ROLE.PRODUCER):
wrapper._handle_remote_blocks_message(
{"transfer_id": "tA", "block_notify_list": notify, "decode_rank": 2}
)
info = wrapper.done_remote_allocate_req_dict["tA"]
assert info.block_ids == expected
assert info.decode_dp_rank == 2
def test_empty_layer_plan_posts_no_rdma_write():
from unittest.mock import Mock
wrapper = SimpleNamespace(write_remote_data=Mock(), write_remote_data_single=Mock())
writer = make_moriio_writer(SimpleNamespace(moriio_wrapper=wrapper))
plan = LayerTransferPlan(
request_id="req",
transfer_id="tA",
layer_name="layer0",
sess_idx=0,
transfer_local_offsets=[],
transfer_remote_offsets=[],
transfer_sizes=[],
)
assert writer._do_layer_write(plan, ["session"]) == []
wrapper.write_remote_data.assert_not_called()
wrapper.write_remote_data_single.assert_not_called()
@pytest.mark.parametrize("local_state", [[], [1, 2]])
def test_hybrid_write_requires_exactly_one_running_state_slot(local_state):
worker = SimpleNamespace(
_mamba_blocks_for_layer=lambda layer_name, block_ids: block_ids[1],
)
writer = make_moriio_writer(worker)
task = _write_task("tA", "10.0.0.1")
task.layer_name = "kda"
task.local_block_ids = [[], local_state]
with pytest.raises(MoRIIOError, match="one running-state slot"):
writer._prepare_mamba_transfer_plans(task, RemoteAllocInfo(block_ids=[[], [9]]))
@@ -805,6 +805,23 @@ class TestMultiConnectorStats:
child_metrics.observe.assert_called_once_with({"transfer_duration": [1.5]}, 2)
@pytest.mark.parametrize("with_exporter", [False, True])
def test_prom_metrics_skips_stats_only_connectors(self, with_exporter):
"""Stats-only children must not prevent supported metrics from exporting."""
child_metrics = MagicMock()
metrics = object.__new__(MultiKVConnectorPromMetrics)
metrics._prom_metrics = (
{"MooncakeStoreConnector": child_metrics} if with_exporter else {}
)
payload = {"MooncakeConnector": {"bytes_transferred": [1024]}}
if with_exporter:
payload["MooncakeStoreConnector"] = {"load_get_bytes": [2048]}
metrics.observe(payload, engine_idx=2)
if with_exporter:
child_metrics.observe.assert_called_once_with({"load_get_bytes": [2048]}, 2)
def test_aggregate_same_connector(self):
"""Test aggregating stats from the same connector type."""
stats1 = MultiKVConnectorStats(
+82 -2
View File
@@ -293,7 +293,7 @@ class TestTritonTopkTopp:
self._compare_results(logits, k, p=None)
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
def test_topp_only(self, batch_size: int, vocab_size: int):
"""Test top-p only (k=None)."""
@@ -307,7 +307,7 @@ class TestTritonTopkTopp:
self._compare_results(logits, k=None, p=p)
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128, 512, 1024])
@pytest.mark.parametrize("batch_size", [1, 8, 32, 48, 128, 512, 1024])
@pytest.mark.parametrize("vocab_size", [1024, 32000, 128256])
def test_topk_and_topp(self, batch_size: int, vocab_size: int):
"""Test combined top-k and top-p."""
@@ -328,6 +328,24 @@ class TestTritonTopkTopp:
self._compare_results(logits, k, p)
@pytest.mark.parametrize("batch_size", [8, 48, 64])
@pytest.mark.parametrize("vocab_size", [151936, 200064, 262144])
def test_topk_and_topp_large_vocab(self, batch_size: int, vocab_size: int):
"""Mixed batches at vocab sizes where the split top-p pipeline uses
its largest split counts (up to _SPLIT_MAX_SPLITS)."""
logits = torch.randn(
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
)
k = torch.randint(1, 100, (batch_size,), generator=self.generator)
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
# Rows cycle through top-k + top-p, top-p only, top-k only, neither.
k[1::4] = vocab_size
p[2::4] = 1.0
k[3::4] = vocab_size
p[3::4] = 1.0
self._compare_results(logits, k, p)
def test_both_disabled(self):
"""Test when both k and p are None (should be no-op)."""
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
@@ -788,6 +806,38 @@ class TestTritonTopkTopp:
f"{kept_topk_topp.tolist()}"
)
@pytest.mark.parametrize("batch_size", [8, 48])
def test_skip_split_without_topp_only_rows(
self, batch_size: int, monkeypatch: pytest.MonkeyPatch
):
"""has_topp_only_rows=False skips the split pipeline; the result must
not change when every top-p row also has a top-k, grammar rows
(k >= #finite logits) included."""
import vllm.v1.sample.ops.topk_topp_triton as tt
vocab_size = 32000
logits = torch.randn(
batch_size, vocab_size, generator=self.generator, dtype=torch.float32
)
logits[1] = float("-inf")
idx = torch.randperm(vocab_size, generator=self.generator)[:30]
logits[1, idx] = torch.randn(30, generator=self.generator)
k = torch.randint(
50, 100, (batch_size,), generator=self.generator, dtype=torch.int32
)
p = torch.rand(batch_size, generator=self.generator) * 0.9 + 0.1
k[::4] = vocab_size
p[::4] = 1.0
expected = tt.apply_top_k_top_p_triton(logits.clone(), k, p)
monkeypatch.setattr(
tt, "_apply_topp_split", lambda *args: pytest.fail("split launched")
)
actual = tt.apply_top_k_top_p_triton(
logits.clone(), k, p, has_topp_only_rows=False
)
assert torch.equal(actual, expected)
def test_topp_tie_break_deterministic(self):
"""Boundary-duplicate trimming must be deterministic across calls.
@@ -821,6 +871,36 @@ class TestTritonTopkTopp:
)
assert len(kept_idx) == pytorch_kept
def test_topp_ties_across_blocks(self):
"""The split top-p tie budget must carry across a slice's blocks.
Regression: the split top-p mask kernel restarted its count of
boundary-value copies at every 8192-entry block, so each block of a
slice could keep up to `numkeep` copies of its own.
"""
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
batch_size, vocab_size, num_ties = 64, 151936, 4096
# 4096 tied tokens spread over many blocks, between 64 tokens above
# and a low tail; the p=0.75 boundary lands mid-copy in the tie group.
tie_idx = torch.arange(num_ties) * 37
logits = torch.full((batch_size, vocab_size), -8.0)
logits[:, tie_idx] = 0.0
logits[:, -64:] = 4.0
p = torch.full((batch_size,), 0.75, dtype=torch.float32)
kept = apply_top_k_top_p_triton(logits.clone(), None, p) != float("-inf")
expected = apply_top_k_top_p_pytorch(logits.clone(), None, p) != float("-inf")
num_kept, num_expected = kept.sum(dim=-1), expected.sum(dim=-1)
assert torch.equal(num_kept, num_expected), (
f"kept {num_kept.unique().tolist()}, "
f"expected {num_expected.unique().tolist()}"
)
# Kept copies of the boundary value are the first ones in index order.
kept_ties = kept[:, tie_idx]
num_kept_ties = kept_ties.sum(dim=-1, keepdim=True)
assert torch.equal(kept_ties, torch.arange(num_ties) < num_kept_ties)
# =============================================================================
# FlashInfer top-k/top-p robustness tests
@@ -245,6 +245,55 @@ def test_connector_reports_configured_cache_source(
assert connector.get_external_cache_hit_sources(None, 0) == {} # type: ignore[arg-type]
@pytest.mark.parametrize("num_layers", [1, 2])
@pytest.mark.parametrize("backend", ["cpu", "disk"])
def test_connector_explicit_cpu_blocks_align_uneven_stages(
tmp_path, num_layers: int, backend: str
):
"""The same configured pool size must fit lighter and heavier PP stages."""
config = _make_kv_cache_config(num_blocks=16)
names = [f"layer_{i}" for i in range(num_layers)]
config.kv_cache_groups[0].layer_names = names
config.kv_cache_tensors[0].layers = names
config.kv_cache_tensors[0].size *= num_layers
vllm_config = _make_vllm_config()
extra = {
"cpu_bytes_to_use_per_rank": _BYTES_PER_BLOCK * 8,
"num_cpu_blocks": 4,
"kv_offload_backend": backend,
}
if backend == "disk":
extra.update(
disk_path=str(tmp_path / "offload.bin"),
disk_capacity_bytes=_BYTES_PER_BLOCK * 8,
)
assert vllm_config.kv_transfer_config is not None
vllm_config.kv_transfer_config.kv_connector_extra_config = extra
connector = SimpleCPUOffloadConnector(
vllm_config, KVConnectorRole.SCHEDULER, config
)
assert connector.scheduler_manager is not None
assert connector.scheduler_manager.num_cpu_blocks == 4
assert len(connector.scheduler_manager.cpu_block_pool.blocks) == 4
@pytest.mark.parametrize(
"count, capacity_bytes",
[(count, _BYTES_PER_BLOCK * 8) for count in [0, -1, True, 1.5, "4", 9]]
+ [(1, _BYTES_PER_BLOCK // 2)],
)
def test_connector_rejects_invalid_explicit_cpu_blocks(count, capacity_bytes):
config = _make_kv_cache_config(num_blocks=16)
vllm_config = _make_vllm_config()
assert vllm_config.kv_transfer_config is not None
vllm_config.kv_transfer_config.kv_connector_extra_config = {
"cpu_bytes_to_use_per_rank": capacity_bytes,
"num_cpu_blocks": count,
}
with pytest.raises(ValueError, match="num_cpu_blocks"):
SimpleCPUOffloadConnector(vllm_config, KVConnectorRole.SCHEDULER, config)
def make_scheduler(
num_cpu_blocks: int = 8,
num_gpu_blocks: int = 16,
+63
View File
@@ -371,6 +371,69 @@ def test_register_shared_kv_cache_storage(monkeypatch, layout: KVCacheLayout):
}
@pytest.mark.parametrize("num_layers", [1, 2])
def test_explicit_cpu_blocks_bound_live_worker_allocation(monkeypatch, num_layers):
"""Different live KV strides must honor the same explicit CPU pool size."""
num_blocks = 4
names = [f"layer.{i}" for i in range(num_layers)]
raw = torch.zeros(
num_layers * num_blocks * BLOCK_BYTES, dtype=torch.int8, device="cuda"
)
config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=raw.nbytes,
layers=names,
layer_stride=num_blocks * BLOCK_BYTES,
block_stride=BLOCK_BYTES,
)
],
kv_cache_groups=[],
)
caches = dict(zip(names, raw.view(num_layers, num_blocks, BLOCK_BYTES)))
worker = SimpleCPUOffloadWorker(None, config, 8 * BLOCK_BYTES, num_cpu_blocks=4)
monkeypatch.setattr("vllm.v1.simple_kv_offload.worker.PIN_MEMORY", False)
try:
worker.register_kv_caches(caches)
assert worker.num_cpu_blocks == 4
assert worker.cpu_kv_caches is not None
assert all(t.shape[0] == 4 for t in worker.cpu_kv_caches.values())
assert sum(t.nbytes for t in worker.cpu_kv_caches.values()) <= 8 * BLOCK_BYTES
finally:
if worker._backend is not None:
worker._backend.shutdown()
@pytest.mark.parametrize(
"capacity_bytes, count, local_capacity",
[(8 * BLOCK_BYTES, 5, 4), (BLOCK_BYTES, 1, 0)],
)
def test_explicit_cpu_blocks_reject_heavier_worker_before_allocation(
capacity_bytes, count, local_capacity
):
"""A count that fits the scheduler's stage must fail on a smaller worker."""
names = ["layer.0", "layer.1"]
raw = torch.zeros(2 * 4 * BLOCK_BYTES, dtype=torch.int8, device="cuda")
config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[
KVCacheTensor(
size=raw.nbytes,
layers=names,
layer_stride=4 * BLOCK_BYTES,
block_stride=BLOCK_BYTES,
)
],
kv_cache_groups=[],
)
worker = SimpleCPUOffloadWorker(None, config, capacity_bytes, num_cpu_blocks=count)
caches = dict(zip(names, raw.view(2, 4, BLOCK_BYTES)))
with pytest.raises(ValueError, match=f"local offload capacity {local_capacity}"):
worker.register_kv_caches(caches)
assert worker.cpu_kv_caches is None
def test_register_kv_cache_storage_with_trailing_padding(monkeypatch):
num_blocks = 4
block_bytes = 32
@@ -1157,6 +1157,91 @@ def test_chunked_requests_match_full_batch(has_draft_logits: bool):
assert torch.equal(chunked_sampled[valid], sampled[valid])
@pytest.mark.parametrize("use_block_verification", [False, True])
@pytest.mark.parametrize("has_draft_logits", [False, True])
def test_adaptive_draft_lengths_match_and_bonus_oracle(
use_block_verification: bool, has_draft_logits: bool
):
torch.manual_seed(7)
device = "cuda"
K = 3
num_drafts = [3, 1, 0, 2, 3, 2]
temperatures = [1.0, 0.0, 0.0, 0.6, 0.0, 1.0]
greedy_num_matches = {1: 1, 2: 0, 4: 2}
num_reqs = len(num_drafts)
num_logits_per_req = torch.tensor(num_drafts, device=device) + 1
cu_num_logits = torch.zeros(num_reqs + 1, dtype=torch.int32, device=device)
cu_num_logits[1:] = num_logits_per_req.cumsum(0)
num_logits = int(cu_num_logits[-1])
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
expanded_idx_mapping = idx_mapping.repeat_interleave(num_logits_per_req)
expanded_local_pos = (
torch.arange(num_logits, dtype=torch.int32, device=device)
- cu_num_logits[expanded_idx_mapping]
)
draft_logits = torch.randn(num_reqs, K, VOCAB_SIZE, device=device)
target_logits = torch.randn(num_logits, VOCAB_SIZE, device=device)
# Allocate at exactly num_logits so a lookahead past the batch is OOB.
draft_sampled = torch.randint(VOCAB_SIZE, (num_logits,), device=device)
target_argmax = target_logits.argmax(dim=-1)
for req_idx, num_matches in greedy_num_matches.items():
lo = int(cu_num_logits[req_idx])
for step in range(num_drafts[req_idx]):
token = target_argmax[lo + step]
if step >= num_matches:
token = (token + 1) % VOCAB_SIZE
draft_sampled[lo + step + 1] = token
inputs = dict(
target_logits=target_logits,
draft_logits=draft_logits if has_draft_logits else None,
draft_sampled=draft_sampled,
cu_num_logits=cu_num_logits,
pos=torch.arange(num_logits, dtype=torch.int32, device=device),
idx_mapping=idx_mapping,
expanded_idx_mapping=expanded_idx_mapping,
expanded_local_pos=expanded_local_pos,
temperature=torch.tensor(temperatures, device=device),
seed=torch.arange(num_reqs, dtype=torch.int64, device=device),
num_speculative_steps=K,
use_block_verification=use_block_verification,
)
sampled, num_sampled = rejection_sample(**inputs)
for req_idx, num_matches in greedy_num_matches.items():
lo = int(cu_num_logits[req_idx])
n = num_matches + 1
assert int(num_sampled[req_idx]) == n, f"greedy request {req_idx}"
assert torch.equal(sampled[req_idx, :n], target_argmax[lo : lo + n]), (
f"greedy request {req_idx}"
)
for req_idx in range(num_reqs):
lo, hi = int(cu_num_logits[req_idx]), int(cu_num_logits[req_idx + 1])
req_inputs = dict(inputs)
for name in (
"target_logits",
"draft_sampled",
"pos",
"expanded_idx_mapping",
"expanded_local_pos",
):
req_inputs[name] = inputs[name][lo:hi]
req_inputs["cu_num_logits"] = cu_num_logits[req_idx : req_idx + 2] - lo
req_inputs["idx_mapping"] = idx_mapping[req_idx : req_idx + 1]
req_sampled, req_num_sampled = rejection_sample(**req_inputs)
n = int(req_num_sampled[0])
assert n <= num_drafts[req_idx] + 1
assert int(num_sampled[req_idx]) == n, f"request {req_idx} mismatch"
assert torch.equal(sampled[req_idx, :n], req_sampled[0, :n]), (
f"request {req_idx} mismatch"
)
def _residual_recovery_logits(
inputs: dict,
num_sampled: torch.Tensor,
+24
View File
@@ -109,6 +109,30 @@ def test_custom_processor_add_request_gates_pipeline_flag(admitted: bool):
assert sampler.needs_logits_processing[3] == admitted
def test_has_topp_only_rows():
"""A row is top-p-only when it has top-p and no effective top-k; top_k
of 0 (the default) or at least the logits width disables top-k."""
sampler = _make_sampler()
sampler.add_request(0, SamplingParams(top_k=10, top_p=0.9))
sampler.add_request(1, SamplingParams(top_k=10))
sampler.add_request(2, SamplingParams(top_p=0.9))
sampler.add_request(3, SamplingParams(top_k=VOCAB_SIZE + 1, top_p=0.9))
states = sampler.sampling_states
states.apply_staged_writes()
def has_topp_only_rows(idx: list[int], vocab_size: int = VOCAB_SIZE) -> bool:
idx_np = np.array(idx, dtype=np.int32)
idx_gpu = torch.from_numpy(idx_np).to(DEVICE)
return states.get_top_k_top_p(idx_gpu, idx_np, vocab_size)[2]
assert not has_topp_only_rows([0, 1])
assert has_topp_only_rows([0, 2])
assert has_topp_only_rows([3])
# The kernel compares top_k against the logits width, so a top_k at or
# above a narrower logits width counts as no top-k.
assert has_topp_only_rows([0], vocab_size=10)
def test_logits_processing_cache_only_checks_active_requests():
sampler = _make_sampler()
sampler.add_request(0, SamplingParams(temperature=0.0))