[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
+7
View File
@@ -364,6 +364,12 @@ def main() -> None:
type=int,
help="Maximum number of concurrent requests",
)
parser.add_argument(
"--request-timeout-seconds",
type=float,
default=600,
help="Timeout for each request, including time waiting for a connection",
)
parser.add_argument("--save-results", type=str, help="Save results to JSON file")
args = parser.parse_args()
@@ -377,6 +383,7 @@ def main() -> None:
temperature=args.temperature,
seed=args.seed,
max_concurrency=args.max_concurrency,
request_timeout_seconds=args.request_timeout_seconds,
)
# Print results to terminal
@@ -217,7 +217,8 @@ def test_flashinfer_mla_dense_decode_unified_slot_view():
assert (out_ref - out_view).abs().max().item() == 0.0
def test_flashmla_fp8_sparse_decode_unified_slot_view():
@pytest.mark.parametrize("next_n", [1, 2, 4])
def test_flashmla_fp8_sparse_decode_unified_slot_view(next_n: int):
"""FlashMLA fp8 sparse decode (DeepSeek V3.2/V4 DSA path) must read a
unified-slot block-major view bit-identically to a contiguous fp8_ds_mla
cache, with finite nonzero output."""
@@ -240,7 +241,10 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view():
n_layers = 3
layer = 1
q = torch.randn(batch, 1, h_q, head_dim, device=dev, dtype=torch.bfloat16) * 0.1
q = (
torch.randn(batch, next_n, h_q, head_dim, device=dev, dtype=torch.bfloat16)
* 0.1
)
# Structurally valid fp8 ds_mla payload: 512B fp8 + 16B f32 scales + 128B
# bf16 rope (random bytes corrupt the scale region and yield NaNs).
@@ -273,13 +277,13 @@ def test_flashmla_fp8_sparse_decode_unified_slot_view():
# Sparse indices: each batch uses its own disjoint blocks.
blocks_per_batch = num_blocks // batch
idx = torch.full((batch, 1, topk), -1, device=dev, dtype=torch.int32)
idx = torch.full((batch, next_n, topk), -1, device=dev, dtype=torch.int32)
for b in range(batch):
slots: list[int] = []
for blk in range(b * blocks_per_batch, (b + 1) * blocks_per_batch):
slots.extend(blk * page + off for off in range(page))
slots_t = torch.tensor(slots[:topk], device=dev, dtype=torch.int32)
idx[b, 0, : slots_t.numel()] = slots_t
idx[b, :, : slots_t.numel()] = slots_t
def run(kc):
meta, num_splits = fm.get_mla_metadata()
+42
View File
@@ -316,6 +316,48 @@ def test_cp_gather_fp8_shuffled_blocks():
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
def test_cp_gather_fp8_mixes_resident_and_pinned_host_rows():
seq_lens = [7, 5]
block_size = 4
(
cache,
block_table,
workspace_starts,
num_reqs,
total_tokens,
expected,
) = _build_test_case(seq_lens, block_size)
device_cache = cache.clone()
host_cache = cache.cpu().pin_memory().view(-1, ENTRY_BYTES)
device_rows = torch.arange(host_cache.shape[0], dtype=torch.int32, device="cuda")
use_device = device_rows % 2 == 0
host_rows = device_rows.clone()
device_rows.masked_fill_(~use_device, -1)
host_cache[use_device.cpu()] = 0
device_cache.view(-1, ENTRY_BYTES)[~use_device] = 0
dst = torch.empty(
total_tokens, NOPE_DIM + ROPE_DIM, dtype=torch.bfloat16, device="cuda"
)
ops.cp_gather_and_upconvert_fp8_kv_cache(
device_cache,
dst,
block_table,
workspace_starts,
num_reqs,
host_cache=host_cache,
host_row_ids=host_rows,
device_row_ids=device_rows,
)
torch.testing.assert_close(
dst[:, :NOPE_DIM], expected[:, :NOPE_DIM], atol=1e-3, rtol=1e-2
)
assert torch.equal(dst[:, NOPE_DIM:], expected[:, NOPE_DIM:])
@pytest.mark.parametrize(
"gather_seq_lens,seq_starts",
[
@@ -234,6 +234,76 @@ def test_fused_norm_rope(num_tokens: int, index_interleave: bool, mla_dtype: str
assert (topk == -1).all(), "topk buffer not cleared on indexer layer"
def test_fused_norm_rope_packed_indexer_block_stride():
"""Indexer writes use their HMA block stride and independent slot map."""
torch.manual_seed(7)
dev = "cuda"
num_tokens = block_size = 2
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64)
q_c = torch.randn(num_tokens, Q_LORA, device=dev, dtype=torch.bfloat16)
kv_c = torch.randn(num_tokens, KV_LORA, device=dev, dtype=torch.bfloat16)
k_pe = torch.randn(num_tokens, ROPE_DIM, device=dev, dtype=torch.bfloat16)
qw = torch.randn(Q_LORA, device=dev, dtype=torch.bfloat16)
kvw = torch.randn(KV_LORA, device=dev, dtype=torch.bfloat16)
ik = torch.randn(num_tokens, INDEX_HEAD_DIM, device=dev, dtype=torch.bfloat16)
ikw = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
ikb = torch.randn(INDEX_HEAD_DIM, device=dev, dtype=torch.float32)
cos_sin = make_cos_sin(32, ROPE_DIM, dev)
idx_row = INDEX_HEAD_DIM + INDEX_HEAD_DIM // 128 * 4
packed_block_stride = block_size * idx_row + 64
backing = torch.zeros(2 * packed_block_stride, device=dev, dtype=torch.uint8)
idx_cache = torch.as_strided(
backing,
(2, block_size, idx_row),
(packed_block_stride, idx_row, 1),
)
mla_cache = torch.zeros(
1,
block_size,
KV_LORA + ROPE_DIM,
device=dev,
dtype=torch.bfloat16,
)
topk = torch.zeros(num_tokens, 8, device=dev, dtype=torch.int32)
K.fused_norm_rope(
pos,
q_c,
qw,
EPS,
kv_c,
kvw,
EPS,
k_pe,
cos_sin,
ik,
ikw,
ikb,
EPS,
cos_sin,
topk,
slot_mapping=torch.arange(num_tokens, device=dev, dtype=torch.int64),
indexer_slot_mapping=torch.arange(
block_size, block_size + num_tokens, device=dev, dtype=torch.int64
),
indexer_k_cache=idx_cache,
mla_kv_cache=mla_cache,
has_indexer=True,
)
ik_ref = rope(layer_norm(ik, ikw, ikb), pos, cos_sin, interleave=False)
q_ref, s_ref = ue8m0_quant(ik_ref)
packed = idx_cache[1].reshape(-1)
values = (
packed[: block_size * INDEX_HEAD_DIM].view(FP8).view(block_size, INDEX_HEAD_DIM)
)
scales = packed[block_size * INDEX_HEAD_DIM :].view(torch.float32)
assert_fp8(values, q_ref, "packed indexer-K fp8")
torch.testing.assert_close(scales, s_ref, rtol=0, atol=0)
assert (backing[block_size * idx_row : packed_block_stride] == 0).all()
def test_fused_norm_rope_materializes_qk_without_local_cache_slots():
"""DCP non-owner ranks still need valid query shards for query AllGather."""
torch.manual_seed(7)
@@ -7,6 +7,7 @@ import pytest
import torch
import vllm.model_executor.layers.sparse_attn_indexer as sparse_indexer
import vllm.models.deepseek_v32.attention as deepseek_attention
from vllm.config import CUDAGraphMode
from vllm.models.deepseek_v32 import attention as deepseek_v32_attention
from vllm.models.deepseek_v32.attention import DeepseekV32Attention
@@ -16,6 +17,51 @@ INDEXER_LAYER = "model.layers.0.self_attn.indexer.k_cache"
MLA_LAYER = "model.layers.0.self_attn.attn"
def test_sparse_attention_refreshes_batch_state_inside_eager_segment(
monkeypatch: pytest.MonkeyPatch,
) -> None:
metadata = SimpleNamespace(num_actual_tokens=1)
class BatchStateRefreshed(Exception):
pass
def refresh_batch_state(attn_metadata):
assert attn_metadata is metadata
raise BatchStateRefreshed
layer = SimpleNamespace(
indexer=None,
skip_topk=True,
layer_name=MLA_LAYER,
impl=SimpleNamespace(
prepare_for_batch=refresh_batch_state,
record_logical_topk_ready=lambda: None,
),
)
monkeypatch.setattr(
deepseek_attention,
"get_attention_context",
lambda layer_name: (metadata, None, torch.empty(1), None),
)
with pytest.raises(BatchStateRefreshed):
DeepseekV32Attention._sparse_indexer_and_attn(
layer,
torch.empty(1, dtype=torch.long),
torch.empty(1, 1),
torch.empty(1, 1, 1),
torch.empty(1, 1, 1),
None,
None,
None,
None,
None,
torch.empty(1, 1, 1),
torch.empty(1, 1, 1),
torch.empty(1, 1),
)
def make_indexer_metadata(
*,
num_decodes: int = 0,
@@ -246,6 +292,10 @@ def test_deepseek_v32_dispatches_selected_mha(
_fp8_query=fp8_query,
_use_sparse_mha=lambda _: True,
rotary_emb=lambda _positions, q: (q + 1, None),
impl=SimpleNamespace(
record_logical_topk_ready=lambda: None,
prepare_for_batch=lambda _: None,
),
forward_impl=record_forward_impl,
)
q_nope = torch.randn(2, 1, 2)
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from inspect import signature
from types import SimpleNamespace
from typing import Any
from unittest.mock import Mock, call, patch
@@ -76,8 +77,16 @@ def test_flashinfer_autotune_token_counts_are_bounded_and_deduplicated():
assert token_counts == (32,)
def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
@pytest.mark.parametrize("skip_attn", [False, True])
def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run(skip_attn):
from vllm.v1.worker.gpu.model_runner import GPUModelRunner as V2Runner
from vllm.v1.worker.gpu_model_runner import GPUModelRunner as V1Runner
runner = _make_runner([])
dummy_run_signature = signature((V2Runner if skip_attn else V1Runner)._dummy_run)
runner._dummy_run.side_effect = lambda **kwargs: dummy_run_signature.bind(
None, **kwargs
)
max_buckets = (1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 8192)
deferred_buckets = (1, 2, 4, 8, 16, 32, 64, 128)
@@ -93,7 +102,7 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
) as get_buckets,
patch("vllm.utils.flashinfer.autotune") as autotune,
):
_run_flashinfer_autotune_dummy_runs(runner)
_run_flashinfer_autotune_dummy_runs(runner, skip_attn=skip_attn)
assert get_buckets.call_args_list == [call(8192), call(128)]
assert autotune.call_args_list == [
@@ -106,11 +115,13 @@ def test_flashinfer_autotune_uses_token_buckets_for_each_dummy_run():
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
**({"skip_attn": True} if skip_attn else {}),
),
call(
num_tokens=128,
skip_eplb=True,
is_profile=True,
randomize_inputs=True,
**({"skip_attn": True} if skip_attn else {}),
),
]
+66 -2
View File
@@ -19,10 +19,12 @@ import vllm.config.vllm as vllm_config_module
import vllm.envs as envs
from vllm.compilation.backends import VllmBackend
from vllm.config import (
AttentionConfig,
CacheConfig,
CompilationConfig,
DeviceConfig,
EngramConfig,
HiSparseConfig,
KernelConfig,
KVTransferConfig,
ModelConfig,
@@ -279,6 +281,62 @@ def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
assert envs.VLLM_USE_V2_MODEL_RUNNER is expected
def test_hisparse_requires_v2_model_runner():
config = object.__new__(VllmConfig)
config.attention_config = AttentionConfig(hisparse_config=HiSparseConfig())
with patch.object(envs, "VLLM_USE_V2_MODEL_RUNNER", None):
assert config.use_v2_model_runner
with (
patch.object(envs, "VLLM_USE_V2_MODEL_RUNNER", False),
pytest.raises(ValueError, match="requires Model Runner V2"),
):
_ = config.use_v2_model_runner
def test_hisparse_rejects_decode_context_parallelism(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(current_platform, "device_count", lambda: 2)
with pytest.raises(ValueError, match="decode context parallelism"):
VllmConfig(
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
parallel_config=ParallelConfig(
tensor_parallel_size=2,
decode_context_parallel_size=2,
),
)
def test_hisparse_rejects_pipeline_parallelism(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr(current_platform, "device_count", lambda: 2)
with pytest.raises(ValueError, match="pipeline parallelism"):
VllmConfig(
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
parallel_config=ParallelConfig(pipeline_parallel_size=2),
)
def test_hisparse_rejects_disabled_hybrid_kv_cache_manager(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: True)
monkeypatch.setattr("vllm.config.vllm.HAS_TRITON", True)
with pytest.raises(ValueError, match="requires the hybrid KV cache manager"):
VllmConfig(
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
scheduler_config=SchedulerConfig(
max_model_len=2048,
is_encoder_decoder=False,
disable_hybrid_kv_cache_manager=True,
),
)
def test_hisparse_rejects_non_cuda(monkeypatch):
monkeypatch.setattr(current_platform, "is_cuda", lambda: False)
with pytest.raises(ValueError, match="requires NVIDIA CUDA"):
VllmConfig(attention_config=AttentionConfig(hisparse_config=HiSparseConfig()))
def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
"""ROCm keeps the DSA models (DeepSeek V3.2/V4, GLM-5.2) on their compiled
MRV1 paths and off breakable cudagraphs by default."""
@@ -305,7 +363,8 @@ def test_rocm_keeps_compiled_deepseek_defaults(monkeypatch):
# (warning_once args must be hashable for its lru_cache).
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
config = SimpleNamespace(
model_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"])
model_config=SimpleNamespace(architectures=["DeepseekV32ForCausalLM"]),
attention_config=AttentionConfig(),
)
assert VllmConfig.use_v2_model_runner.fget(config) is False
finally:
@@ -385,6 +444,7 @@ def test_dsa_models_default_to_mrv2_and_breakable_cudagraph(
)
config = SimpleNamespace(
model_config=model_config,
attention_config=AttentionConfig(),
speculative_config=SimpleNamespace(method="mtp") if with_mtp else None,
parallel_config=SimpleNamespace(prefill_context_parallel_size=1),
compilation_config=CompilationConfig(
@@ -897,7 +957,10 @@ def test_models_default_to_v2_model_runner(model_config, expected, monkeypatch):
monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
monkeypatch.setattr(vllm_config_module, "HAS_TRITON", True)
monkeypatch.setattr(current_platform, "is_rocm", lambda: False)
config = SimpleNamespace(model_config=model_config)
config = SimpleNamespace(
model_config=model_config,
attention_config=AttentionConfig(),
)
config._get_v2_model_runner_unsupported_features = lambda: []
assert VllmConfig.use_v2_model_runner.fget(config) is expected
@@ -2787,6 +2850,7 @@ def test_draft_sample_method_gumbel_is_rejected():
def _watermarked_vllm_config() -> VllmConfig:
config = object.__new__(VllmConfig)
config.watermark_config = WatermarkConfig(key=42)
config.attention_config = AttentionConfig()
config.speculative_config = None
return config
+54 -2
View File
@@ -1,16 +1,19 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for per-KV-group attention backend selection (backend_per_kind)."""
"""Tests for attention configuration and per-KV-group backend selection."""
from types import SimpleNamespace
import pytest
from vllm.config.attention import AttentionConfig
from vllm.config.attention import AttentionConfig, HiSparseConfig
from vllm.model_executor.layers.attention.attention import (
_largest_kernel_block_within,
)
from vllm.v1.attention.backend import AttentionType, MultipleOf
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.selector import get_attn_spec_kind
from vllm.v1.hisparse.runtime import ResolvedHiSparseConfig
from vllm.v1.kv_cache_interface import KVCacheSpecKind
@@ -86,3 +89,52 @@ def test_backend_per_kind_rejects_unknown_kind():
def test_backend_per_kind_defaults_empty():
assert AttentionConfig().backend_per_kind == {}
def test_hisparse_device_buffer_size_boundaries():
vllm_config = SimpleNamespace(
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
speculative_config=None,
)
resolved = ResolvedHiSparseConfig.from_vllm_config(vllm_config, model_top_k=128)
assert resolved is not None
assert resolved.device_buffer_size == 256
vllm_config.attention_config.hisparse_config = HiSparseConfig(
device_buffer_size=127
)
with pytest.raises(ValueError, match="expected at least 128"):
ResolvedHiSparseConfig.from_vllm_config(vllm_config, model_top_k=128)
vllm_config.attention_config.hisparse_config = HiSparseConfig(
device_buffer_size=32768
)
resolved = ResolvedHiSparseConfig.from_vllm_config(vllm_config, model_top_k=128)
assert resolved is not None
assert resolved.device_buffer_size == 32768
vllm_config.attention_config.hisparse_config = HiSparseConfig(
device_buffer_size=32769
)
with pytest.raises(ValueError, match="int16 slot-index limit"):
ResolvedHiSparseConfig.from_vllm_config(vllm_config, model_top_k=128)
def test_hisparse_device_buffer_covers_speculative_window():
vllm_config = SimpleNamespace(
attention_config=AttentionConfig(hisparse_config=HiSparseConfig()),
speculative_config=SimpleNamespace(
num_speculative_tokens=3,
parallel_drafting=False,
),
)
resolved = ResolvedHiSparseConfig.from_vllm_config(vllm_config, model_top_k=128)
assert resolved is not None
assert resolved.device_buffer_size == 5 * 128
vllm_config.attention_config.hisparse_config = HiSparseConfig(
device_buffer_size=4 * 128 - 1
)
with pytest.raises(ValueError, match="expected at least 512"):
ResolvedHiSparseConfig.from_vllm_config(vllm_config, model_top_k=128)
+1
View File
@@ -200,6 +200,7 @@ def test_mla_kv_cache_spec_uses_layer_cache_dtype(
layer = SimpleNamespace(
kv_cache_dtype=cache_dtype,
head_size=576,
indexer=None,
non_causal_multi_token_decode=False,
sliding_window=None,
)
@@ -55,6 +55,7 @@ def build_chunked_context(
def test_sparse_context_lengths_do_not_force_gpu_sync(monkeypatch):
"""Sparse MLA must pin computed context lengths before the H2D copy."""
builder = SimpleNamespace(
_context_lens_cpu=torch.empty(2, dtype=torch.int32, pin_memory=True),
chunked_prefill_workspace=torch.empty((2048, 1)),
chunked_prefill_workspace_size=1024,
kv_cache_spec=SimpleNamespace(block_size=BLOCK_SIZE),
@@ -113,6 +114,7 @@ def test_chunks_gather_every_context_row_exactly_once(context_lens, workspace_si
query_lens = [4] * len(context_lens)
metadata = build_chunked_context(context_lens, query_lens, workspace_size)
assert metadata is not None
assert metadata.context_lens.tolist() == context_lens
gathered: dict[int, list[tuple[int, int]]] = {}
previous_request_start = -1
@@ -253,6 +255,7 @@ def test_dcp_chunks_fit_the_per_rank_row_budget():
dcp_local_block_size=interleave,
)
assert metadata is not None
assert metadata.context_lens.tolist() == context_lens
virtual_block_size = interleave * dcp_world_size
local_cursor: dict[int, int] = {}
@@ -330,3 +333,23 @@ def test_dcp_reorg_uses_each_chunks_local_starts():
)
torch.testing.assert_close(reorganized, torch.cat(expected))
def test_chunked_context_metadata_owns_context_lengths():
"""Reusing CPU scratch must not overwrite an earlier batch's context lengths."""
context_lens = torch.tensor([16, 32], dtype=torch.int32)
metadata = build_mla_chunked_context_metadata(
context_lens_cpu=context_lens,
prefill_query_start_loc_cpu=torch.tensor([0, 4, 8], dtype=torch.int32),
chunked_prefill_workspace=torch.empty((64, 1)),
chunked_prefill_workspace_size=64,
block_size=BLOCK_SIZE,
align_chunk_to_block=True,
device=torch.device("cpu"),
dcp_world_size=1,
dcp_local_block_size=1,
dcp_virtual_block_size=1,
)
context_lens.zero_()
assert metadata is not None
assert metadata.context_lens.tolist() == [16, 32]
File diff suppressed because it is too large Load Diff
@@ -41,7 +41,9 @@ def test_connector_without_divergent_hit_support_uses_common_lookup():
manager = MagicMock()
manager.get_computed_blocks.return_value = (common_blocks, 0, 0)
scheduler = SimpleNamespace(
connector=SimpleNamespace(supports_divergent_local_hybrid_hits=False),
connector=SimpleNamespace(
supports_divergent_local_hybrid_hits=False,
),
kv_cache_manager=manager,
)
@@ -61,7 +63,9 @@ def test_capable_connector_uses_divergent_partial_hit_lookup():
True,
)
scheduler = SimpleNamespace(
connector=SimpleNamespace(supports_divergent_local_hybrid_hits=True),
connector=SimpleNamespace(
supports_divergent_local_hybrid_hits=True,
),
kv_cache_manager=manager,
)
@@ -922,71 +926,6 @@ def test_connector_finish_registers_partial_tail_before_cleanup():
scheduler.kv_cache_manager.remove_skipped_blocks.assert_called_once()
def test_block_pool_touch_pins_released_cow_target():
"""The connector can rescue an offered CoW target after its step-scoped
retention is released by using the bound BlockPool's touch method."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 6
req0.append_output_token_ids([3])
assert manager.allocate_slots(req0, 1) is not None
# Retention released before the drain (defer_block_free=False ordering).
_copies, retained = manager.take_kv_cache_block_copies()
manager.block_pool.free_blocks(retained)
offloads = drain_boundary_state_offloads(manager)
((_group_id, block_id, boundary_tokens),) = offloads["0"]
assert boundary_tokens == 6
cow_block = manager.block_pool.blocks[block_id]
assert cow_block.ref_cnt == 0
assert block_id in _free_block_ids(manager)
manager.block_pool.touch([cow_block])
assert cow_block.ref_cnt == 1
assert block_id not in _free_block_ids(manager)
# The connector-pinned block is out of the free queue: draining every free block
# neither trips the allocator's ref_cnt assert nor hands it out.
new_blocks = manager.block_pool.get_new_blocks(
manager.block_pool.get_num_free_blocks()
)
assert block_id not in {b.block_id for b in new_blocks}
def test_boundary_state_offload_dropped_when_request_freed_before_drain():
"""A hand-off recorded in the same scheduling pass as the request's death
must not be drained: its release hook has already run, so draining would
+27 -1
View File
@@ -13,7 +13,7 @@ from vllm.v1.request import RequestStatus
from vllm.v1.structured_output import StructuredOutputGrammar
from vllm.v1.utils import ConstantList
from .utils import create_requests, create_scheduler
from .utils import create_requests, create_scheduler, mock_kv
pytestmark = pytest.mark.cpu_test
@@ -145,6 +145,32 @@ def test_abort():
assert req.num_output_tokens == abort_order_copy.index(i)
def test_connector_metadata_precedes_async_placeholder_advance(monkeypatch):
"""Mirroring must see earlier unresolved outputs, excluding the current step."""
scheduler = create_scheduler(
async_scheduling=True,
use_kv_connector=mock_kv(matched_tokens=0, is_async=False),
)
(request,) = create_requests(num_requests=1, num_tokens=4, max_tokens=4)
scheduler.add_request(request)
assert scheduler.connector is not None
build_connector_meta = scheduler.connector.build_connector_meta
observed_placeholders = []
def record_placeholders(scheduler_output):
observed_placeholders.append(request.num_output_placeholders)
return build_connector_meta(scheduler_output)
monkeypatch.setattr(
scheduler.connector, "build_connector_meta", record_placeholders
)
scheduler.schedule()
scheduler.schedule()
assert observed_placeholders == [0, 1]
def test_preempt():
scheduler = create_scheduler(async_scheduling=True)
requests = create_requests(num_requests=10, max_tokens=20)
@@ -75,6 +75,7 @@ def _mock_vllm_config(layout: str | None):
config.cache_config = CacheConfig()
config.cache_config.num_gpu_blocks_override = None
config.cache_config.kv_cache_layout = layout
config.attention_config.hisparse_config = None
return config
+144
View File
@@ -22,6 +22,7 @@ from vllm.config import (
SchedulerConfig,
VllmConfig,
)
from vllm.config.attention import HiSparseConfig
from vllm.config.kv_events import KVEventsConfig
from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import (
@@ -52,10 +53,16 @@ from vllm.v1.core.kv_cache_utils import (
make_block_hash_with_group_id,
tensor_data,
)
from vllm.v1.hisparse.layout import (
create_hisparse_layout,
get_hisparse_gpu_memory_usage,
)
from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
HiddenStateCacheSpec,
HiSparseHotSpec,
HiSparseResidentSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
@@ -68,18 +75,139 @@ from vllm.v1.kv_cache_interface import (
SinkFullAttentionSpec,
SlidingWindowMLASpec,
SlidingWindowSpec,
SparseCacheRole,
UniformTypeKVCacheSpecs,
get_kv_cache_spec_kind,
get_kv_cache_spec_sliding_window,
is_full_attention_spec,
iter_layer_specs,
)
from vllm.v1.kv_cache_layout import KVCacheLayout
from vllm.v1.metrics.stats import CachingMetrics, PrefixCacheStats
from vllm.v1.request import Request
pytestmark = pytest.mark.cpu_test
@pytest.mark.parametrize("gpu_block_size", [32, 64])
def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
specs = {
"model.layers.0.self_attn": MLAAttentionSpec(
block_size=gpu_block_size,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
is_index_group_leader=True,
),
"model.layers.0.self_attn.indexer": MLAAttentionSpec(
block_size=gpu_block_size,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
cache_role=SparseCacheRole.INDEXER,
),
}
group_spec = UniformTypeKVCacheSpecs.from_specs(specs)
assert group_spec is not None
group = KVCacheGroupSpec(list(specs), group_spec)
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
model_config=SimpleNamespace(
hf_config=SimpleNamespace(index_topk=128),
max_model_len=gpu_block_size,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
cache_config=SimpleNamespace(
num_gpu_blocks_override=7,
prefix_cache_retention_interval=None,
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC,
),
)
indexer_spec = specs["model.layers.0.self_attn.indexer"]
assert get_hisparse_gpu_memory_usage(config, [group]) == (
indexer_spec.max_memory_usage_bytes(config)
)
monkeypatch.setattr(kv_cache_utils, "get_hisparse_host_pool_bytes", lambda _: 2**30)
cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
config, [group], available_memory=2**30
)
assert cache_config.num_blocks == 7
assert cache_config.hisparse_host_num_blocks is not None
assert cache_config.hisparse_host_num_blocks > 7
host_group, indexer_group, *auxiliary_groups = cache_config.kv_cache_groups
assert host_group.host_resident
assert not any(group.host_resident for group in [indexer_group, *auxiliary_groups])
host_layers = set(host_group.layer_names)
for tensor in cache_config.kv_cache_tensors:
assert all(
(name in host_layers) == tensor.host_resident for name in tensor.layers
)
host_tensor = next(t for t in cache_config.kv_cache_tensors if t.host_resident)
host_spec = host_group.kv_cache_spec.kv_cache_specs[host_tensor.layers[0]]
assert host_tensor.block_stride == host_spec.page_size_bytes
assert host_tensor.layer_stride == (
host_spec.page_size_bytes * cache_config.hisparse_host_num_blocks
)
assert host_group.kv_cache_spec.block_size == gpu_block_size
assert indexer_group.kv_cache_spec.block_size == gpu_block_size
host_specs = host_group.kv_cache_spec.kv_cache_specs
gpu_indexer_specs = indexer_group.kv_cache_spec.kv_cache_specs
assert set(host_specs) == {"model.layers.0.self_attn"}
assert set(gpu_indexer_specs) == {"model.layers.0.self_attn.indexer"}
assert indexer_group.kv_cache_spec.prefix_cacheable
assert host_group.enable_kv_transfer
auxiliary_specs = [group.kv_cache_spec for group in auxiliary_groups]
assert any(isinstance(spec, HiSparseResidentSpec) for spec in auxiliary_specs)
assert any(isinstance(spec, HiSparseHotSpec) for spec in auxiliary_specs)
assert all(
spec.block_size == gpu_block_size
for spec in auxiliary_specs
if isinstance(spec, (HiSparseResidentSpec, HiSparseHotSpec))
)
scheduler_block_size, hash_block_size = kv_cache_utils.resolve_kv_cache_block_sizes(
cache_config,
SimpleNamespace(
cache_config=SimpleNamespace(
block_size=16,
enable_prefix_caching=True,
prefix_match_unit=None,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
kv_transfer_config=object(),
),
)
assert scheduler_block_size == hash_block_size == gpu_block_size
def test_hisparse_rejects_deepseek_v4():
full_specs = {
"model.layers.0.attn": MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=512,
dtype=torch.uint8,
model_version="deepseek_v4",
)
}
full_uniform = UniformTypeKVCacheSpecs.from_specs(full_specs)
assert full_uniform is not None
group = KVCacheGroupSpec(list(full_specs), full_uniform)
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=HiSparseConfig()),
model_config=SimpleNamespace(hf_config=SimpleNamespace(index_topk=512)),
cache_config=SimpleNamespace(num_gpu_blocks_override=7),
)
with pytest.raises(ValueError, match="does not support DeepSeek V4"):
create_hisparse_layout(
config,
[group],
host_budget=2**30,
)
@pytest.fixture(autouse=True)
def _auto_init_hash_fn(request):
hash_fn: Callable
@@ -3848,6 +3976,22 @@ def test_iter_layer_specs_returns_group_members():
assert list(iter_layer_specs(wrapped)) == [full, mla]
def test_wrapped_mamba_group_requires_block_zeroing():
mamba = MambaSpec(
block_size=4,
shapes=((4, 1),),
dtypes=(torch.float32,),
)
wrapped = UniformTypeKVCacheSpecs(block_size=4, kv_cache_specs={"mamba": mamba})
config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[],
kv_cache_groups=[KVCacheGroupSpec(["mamba"], wrapped)],
)
assert config.needs_kv_cache_zeroing
def _spec_decode_grouping_config(method="dspark", model_type=None):
"""Grouping config with an EAGLE-family speculative method enabled."""
return SimpleNamespace(
File diff suppressed because it is too large Load Diff
+69
View File
@@ -19,7 +19,12 @@ from vllm.config import (
SpeculativeConfig,
VllmConfig,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
HiSparseConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalKwargsItem,
@@ -34,6 +39,7 @@ from vllm.v1.core.sched.output import CachedRequestData, SchedulerOutput
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.core.single_type_kv_cache_manager import register_all_kvcache_specs
from vllm.v1.engine import FinishReason
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
@@ -148,6 +154,30 @@ def test_get_num_unfinished_requests():
assert scheduler.get_num_unfinished_requests() == len(requests) - i - 1
def _bind_hisparse_connector(scheduler):
connector = object.__new__(HiSparseConnector)
connector.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
from tests.v1.core.test_prefix_caching import make_hisparse_kv_cache_manager
coordinator = get_hisparse_coordinator(make_hisparse_kv_cache_manager(16, 16))
connector.connector_scheduler.bind_coordinator(coordinator)
composite = object.__new__(MultiConnector)
composite._connectors = (connector,)
scheduler.connector = composite
return coordinator
def test_pending_hisparse_spill_keeps_scheduler_alive():
"""A final host spill must complete after the last request finishes."""
scheduler = create_scheduler()
coordinator = _bind_hisparse_connector(scheduler)
pending = coordinator.pending_spills
pending[0] = Mock()
assert scheduler.has_requests()
pending.clear()
assert not scheduler.has_requests()
@pytest.mark.parametrize(
"enable_prefix_caching, prompt_logprobs",
[
@@ -1181,6 +1211,45 @@ def test_preempt_during_execution():
assert requests[1].output_token_ids[0] == 42
def test_pending_hisparse_reclamation_defers_preemption(monkeypatch):
"""Pending reclamation must stall rather than preempt its owning request."""
scheduler = create_scheduler(enable_prefix_caching=False)
request = create_requests(num_requests=1, num_tokens=16, block_size=16)[0]
scheduler.add_request(request)
scheduler_output = scheduler.schedule()
scheduler.update_from_output(
scheduler_output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[0]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
monkeypatch.setattr(
scheduler.kv_cache_manager, "allocate_slots", Mock(return_value=None)
)
coordinator = _bind_hisparse_connector(scheduler)
monkeypatch.setattr(
scheduler.connector, "build_connector_meta", Mock(return_value=None)
)
monkeypatch.setattr(
coordinator,
"has_pending_reclamation",
Mock(return_value=True),
)
deferred_output = scheduler.schedule()
assert not deferred_output.num_scheduled_tokens
assert scheduler.running == [request]
assert request.status == RequestStatus.RUNNING
assert request.num_preemptions == 0
def test_prefix_cache_query_not_inflated_by_connector_defer():
"""The GPU prefix-cache query is recorded at admission, so a request the
connector defers several times is counted once, not once per retry."""
+225
View File
@@ -0,0 +1,225 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import prometheus_client
import pytest
import torch
from tests.conftest import VllmRunner
from tests.utils import fork_new_process_for_each_test
from vllm.config import AttentionConfig, HiSparseConfig, KVTransferConfig
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import (
MultiConnector,
)
from vllm.platforms import current_platform
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
MODEL = "deepseek-ai/DeepSeek-V3.2"
HF_OVERRIDES = {
"num_hidden_layers": 8,
"hidden_size": 256,
"intermediate_size": 512,
"num_attention_heads": 8,
"num_key_value_heads": 1,
"n_routed_experts": 8,
"num_experts_per_tok": 2,
"index_topk": 128,
}
def _shrink_config(config):
overrides = HF_OVERRIDES.copy()
if any("MTP" in architecture for architecture in config.architectures):
overrides.pop("num_hidden_layers")
config.update(overrides)
return config
def _num_hisparse_spills(runner: VllmRunner) -> int:
client = runner.llm.llm_engine.engine_core
coordinator = get_hisparse_coordinator(
client.engine_core.scheduler.kv_cache_manager
)
return coordinator.next_spill_id
def _offload_load_bytes() -> float:
return sum(
sample.value
for metric in prometheus_client.REGISTRY.collect()
if metric.name == "vllm:kv_offload_load_bytes"
for sample in metric.samples
if sample.name == "vllm:kv_offload_load_bytes_total"
)
def _get_hisparse_worker(runner: VllmRunner) -> HiSparseConnectorWorker:
engine_core = runner.llm.llm_engine.engine_core.engine_core
model_runner = engine_core.model_executor.driver_worker.worker.model_runner
connector = model_runner.kv_connector.kv_connector
if isinstance(connector, MultiConnector):
connector = next(
child
for child in connector.sub_connectors
if isinstance(child, HiSparseConnector)
)
assert isinstance(connector, HiSparseConnector)
assert connector.connector_worker is not None
return connector.connector_worker
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="HiSparse requires NVIDIA CUDA"
)
@pytest.mark.parametrize(
"with_offloading", [False, True], ids=["standalone", "offload"]
)
@fork_new_process_for_each_test
def test_hisparse_spill_and_prefix_restore(
monkeypatch: pytest.MonkeyPatch,
vllm_runner: type[VllmRunner],
with_offloading: bool,
):
"""Spilled prefixes restore and FULL-graph decode writes reach host KV.
A regression omitted attention metadata before FULL graph replay, so decode
completed normally while its newly written KV rows were never copied to host.
"""
capability = current_platform.get_device_capability()
if capability is None or capability.major < 9:
pytest.skip("Sparse MLA requires Hopper or newer")
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_DEEP_GEMM_WARMUP", "skip")
target = [1000 + i % 64 for i in range(257)]
pressure = [
[2000 + request_idx * 128 + i % 64 for i in range(257)]
for request_idx in range(4)
]
hisparse_connector = {
"kv_connector": "HiSparseConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"host_pool_gib": 1},
}
kv_transfer_config = KVTransferConfig(**hisparse_connector)
if with_offloading:
kv_transfer_config = KVTransferConfig(
kv_connector="MultiConnector",
kv_role="kv_both",
kv_connector_extra_config={
"connectors": [
hisparse_connector,
{
"kv_connector": "OffloadingConnector",
"kv_role": "kv_both",
"kv_connector_extra_config": {"cpu_bytes_to_use": 1 << 30},
},
]
},
)
with vllm_runner(
MODEL,
load_format="dummy",
hf_overrides=_shrink_config,
attention_config=AttentionConfig(
hisparse_config=HiSparseConfig(
device_buffer_size=512,
)
),
kv_transfer_config=kv_transfer_config,
block_size=64,
max_model_len=320,
max_num_batched_tokens=1024,
max_num_seqs=4,
num_gpu_blocks_override=128,
disable_log_stats=False,
enable_chunked_prefill=True,
enable_prefix_caching=True,
speculative_config={"method": "mtp", "num_speculative_tokens": 3},
compilation_config={
"cudagraph_mode": "FULL_AND_PIECEWISE",
"cudagraph_capture_sizes": [1, 4],
},
) as runner:
worker = _get_hisparse_worker(runner)
engine_core = runner.llm.llm_engine.engine_core.engine_core
model_runner = engine_core.model_executor.driver_worker.worker.model_runner
full_graph_calls = 0
full_replay_pending = False
original_run_fullgraph = model_runner.cudagraph_manager.run_fullgraph
def record_fullgraph(batch_desc):
nonlocal full_graph_calls, full_replay_pending
full_graph_calls += 1
full_replay_pending = True
return original_run_fullgraph(batch_desc)
monkeypatch.setattr(
model_runner.cudagraph_manager, "run_fullgraph", record_fullgraph
)
original_finish_forward = worker.finish_forward
verified_full_replay_rows = 0
copied_nonzero_kv = False
def finish_forward():
nonlocal full_replay_pending, verified_full_replay_rows
nonlocal copied_nonzero_kv
try:
original_finish_forward()
if not full_replay_pending:
return
# Synchronize only the test's inspection, not finish_forward itself.
with gpu_sync_allowed():
torch.accelerator.synchronize()
assert worker._row_mirrors
for layer_index, cache in enumerate(worker.cache_handles):
source_index = cache.runtime.resident_source_index
for mirror in worker._row_mirrors:
source_row = mirror.source_starts[source_index]
destination_row = mirror.destination_start
num_rows = mirror.num_rows
source_block, source_offset = divmod(
source_row, worker.kernel_block_size
)
with gpu_sync_allowed():
gpu_rows = worker.resident_caches[layer_index][
source_block, source_offset : source_offset + num_rows
].cpu()
host_rows = worker.host_caches[layer_index][
destination_row : destination_row + num_rows
]
assert torch.equal(host_rows, gpu_rows)
copied_nonzero_kv |= bool(torch.count_nonzero(gpu_rows).item())
verified_full_replay_rows += num_rows
finally:
full_replay_pending = False
monkeypatch.setattr(worker, "finish_forward", finish_forward)
expected = runner.generate_greedy([target], max_tokens=8)
assert full_graph_calls > 0
assert verified_full_replay_rows > 0
assert copied_nonzero_kv
runner.generate_greedy(pressure, max_tokens=8)
assert _num_hisparse_spills(runner) > 0
load_bytes = _offload_load_bytes()
actual = runner.generate_greedy([target], max_tokens=8)
runner.generate_greedy([[42]], max_tokens=1)
assert actual == expected
if with_offloading:
assert _offload_load_bytes() > load_bytes
@@ -21,6 +21,7 @@ from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
HiddenStateCacheSpec,
KVCacheConfig,
KVCacheGroupRole,
KVCacheGroupSpec,
KVCacheSpec,
KVCacheTensor,
@@ -336,6 +337,36 @@ def test_worker_kv_bytes_preserves_tensor_layout(packed: bool):
assert offloading_config.cache.blocks_per_chunk == 2
def test_hisparse_offloads_only_indexer_group():
source = KVCacheGroupSpec(
["source"],
_full_attention_spec(),
host_resident=True,
)
indexer = KVCacheGroupSpec(
["indexer"],
_full_attention_spec(),
role=KVCacheGroupRole.HISPARSE_INDEXER,
)
kv_cache_config = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[],
kv_cache_groups=[source, indexer],
hisparse_host_num_blocks=4,
)
config = _make_vllm_config()
offloading_config = build_offloading_config(config, kv_cache_config)
scheduler_config = SchedulerOffloadConfig.from_spec(
MockOffloadingSpec(offloading_config), config, kv_cache_config
)
assert [
(group.group_id, group.layer_names) for group in offloading_config.groups
] == [(1, ("indexer",))]
assert [group.group_idx for group in scheduler_config.kv_group_configs] == [1]
def test_zero_blocks_skips_tensor_layout_validation():
kv_cache_config = _make_sizing_kv_cache_config(packed=False)
kv_cache_config.num_blocks = 0
@@ -792,7 +823,6 @@ def test_parallelism_agnostic_excluded(kv_cache_groups: list[KVCacheGroupSpec]):
False,
id="uniform-uncertifiable-inner",
),
pytest.param([], False, id="attention-free"),
],
)
def test_canonical_layout_gate(kv_cache_groups, certified):
@@ -98,6 +98,14 @@ def test_swa_offload_window_covers_unaligned_hit(boundary, eagle, left_state):
)
manager = CPUOffloadingManager(num_chunks=100)
spec = SimpleNamespace(
config=SimpleNamespace(
groups=tuple(
SimpleNamespace(
group_id=i, tokens_per_block=group.kv_cache_spec.block_size
)
for i, group in enumerate(groups)
)
),
tokens_per_block=(256, 64, 8),
tokens_per_hash=8,
blocks_per_chunk=4,
@@ -303,7 +311,24 @@ def test_aligned_boundary_store_uses_exact_source_with_partial_tail():
assert src_spec.block_indices == [0, 0]
def test_aligned_boundary_store_flushes_before_cow_destination_reuse():
def test_aligned_boundary_store_maps_sparse_group_id_to_dense_transfer_slot():
scheduler = _make_partial_tail_scheduler()
indexer_config = scheduler.config.kv_group_configs[1]
scheduler.config = scheduler.config._replace(kv_group_configs=(indexer_config,))
_make_partial_tail_request(scheduler)
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
jobs = scheduler._build_aligned_boundary_store_jobs({"req": [(1, 98, 16)]})
[job] = jobs.values()
assert job.src_spec.group_sizes == [1]
assert job.src_spec.block_indices == [0]
@pytest.mark.parametrize("cow_reuse", [False, True])
def test_aligned_boundary_store_flushes_before_block_reuse(cow_reuse):
scheduler = _make_partial_tail_scheduler()
_make_partial_tail_request(scheduler)
scheduler.manager.prepare_store.side_effect = lambda keys, req_context: (
@@ -320,7 +345,11 @@ def test_aligned_boundary_store_flushes_before_cow_destination_reuse():
[job_id] = meta.store_jobs
output = SchedulerOutput.make_empty()
output.kv_cache_block_copies = [KVCacheBlockCopy(98, 99)]
if cow_reuse:
output.kv_cache_block_copies = [KVCacheBlockCopy(98, 99)]
else:
output.scheduled_cached_reqs.req_ids = ["req"]
output.scheduled_cached_reqs.new_block_ids = [([], [99])]
meta = scheduler.build_connector_meta(output)
assert meta.jobs_to_flush == {job_id}
@@ -384,6 +413,20 @@ def test_partial_lookup_returns_exact_boundary_and_group_load_keys():
assert req_status.partial_tail_boundary is None
def test_lookup_cap_stops_at_authoritative_prefix_boundary():
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(scheduler)
request.skip_reading_prefix_cache = False
scheduler.manager.lookup.return_value = LookupResult.HIT
tokens, load_async = scheduler.get_num_new_matched_tokens(
request, 0, max_num_new_tokens=20
)
assert (tokens, load_async) == (20, True)
assert scheduler._req_status["req"].partial_tail_boundary == 20
def test_recurrent_group_unhashed_block_does_not_truncate_load_boundary():
"""A recurrent group may hold unhashed blocks inside the computed prefix.
@@ -0,0 +1,116 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole
from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import (
ExampleConnector,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
HiSparseConnectorMetadata,
HiSparseConnectorScheduler,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.hisparse.runtime import HiSparseCacheHandle
from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector
@pytest.mark.parametrize("nested", [False, True])
def test_cache_manager_binding_preserves_hisparse_and_legacy_pool_hooks(nested):
"""Composite binding must reach HiSparse and existing pool-only connectors."""
hisparse = object.__new__(HiSparseConnector)
hisparse._role = KVConnectorRole.SCHEDULER
hisparse.connector_scheduler = HiSparseConnectorScheduler(async_speculative=False)
legacy = object.__new__(ExampleConnector)
legacy.bind_gpu_block_pool = MagicMock()
connector = object.__new__(MultiConnector)
connector._connectors = [hisparse, legacy]
if nested:
parent = object.__new__(MultiConnector)
parent._connectors = [connector]
connector = parent
from tests.v1.core.test_prefix_caching import make_hisparse_kv_cache_manager
manager = make_hisparse_kv_cache_manager(16, 16)
connector.bind_kv_cache_manager(manager)
assert hisparse.connector_scheduler.coordinator is get_hisparse_coordinator(manager)
legacy.bind_gpu_block_pool.assert_called_once_with(manager.block_pool)
def test_hisparse_requires_block_outermost_device_layout():
assert HiSparseConnector.get_required_kvcache_layout(MagicMock()) == "BLHNC"
def test_no_forward_enqueues_deferred_hisparse_transfers():
"""A zero-token step must still enqueue deferred post-forward transfers."""
connector = object.__new__(ActiveKVConnector)
connector._disabled = False
connector.pre_forward = MagicMock()
connector.finish_forward = MagicMock()
connector.post_forward = MagicMock(return_value=None)
scheduler_output = SimpleNamespace(finished_req_ids=set())
connector.no_forward(scheduler_output)
connector.pre_forward.assert_called_once_with(scheduler_output)
connector.finish_forward.assert_called_once_with()
def test_full_graph_step_prepares_host_mirror_outside_model():
"""Graph replay must restore host-mirror state cleared at step start."""
runtime = SimpleNamespace(
is_group_leader=True,
eager_host_mirror=True,
begin_forward=MagicMock(),
invalidate_written_slots=MagicMock(),
)
handle = HiSparseCacheHandle(runtime)
handle.mirror_slot_mapping = torch.tensor([4, 5])
worker = object.__new__(HiSparseConnectorWorker)
worker.cache_layer_names = ["layer"]
worker.cache_handles = [handle]
worker._group_leaders = (("layer", handle),)
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker.is_host_writer = True
worker._enqueue_row_dma = MagicMock()
worker.start_step = MagicMock(
side_effect=lambda *_args, **_kwargs: worker._clear_forward_mirror_state()
)
connector = object.__new__(HiSparseConnector)
connector.connector_worker = worker
connector._get_connector_metadata = MagicMock(
return_value=HiSparseConnectorMetadata(None, (), (), {}, True)
)
req_id_per_token = torch.tensor([0, 1], dtype=torch.int32)
attn_metadata = SimpleNamespace(
num_actual_tokens=2,
num_decode_tokens=2,
num_reqs=2,
max_query_len=1,
req_id_per_token=req_id_per_token,
)
connector.start_load_kv(
SimpleNamespace(),
request_state_indices=None,
request_ids=[],
attn_metadata={"layer": attn_metadata},
)
worker._enqueue_host_mirror()
worker._enqueue_row_dma.assert_called_once_with((0,), ready_event=None)
runtime.invalidate_written_slots.assert_called_once()
@@ -3297,7 +3297,9 @@ def _make_bare_worker(
# path; everything flows through the coordinator).
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
)
worker.disk_offload_buffer_budget_bytes = None
@@ -3311,6 +3313,18 @@ def _make_bare_worker(
block_size=block_size, num_kv_heads=8, head_size=64, dtype=None
)
group = KVCacheGroupSpec(["layer0", "__cross_layer__"], spec)
worker._kv_cache_config = KVCacheConfig(
num_blocks=num_gpu_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=0,
layers=["layer0", "__cross_layer__"],
layer_stride=0,
block_stride=0,
)
],
kv_cache_groups=[group],
)
worker._kv_cache_groups = [group]
worker.pcp_size = 1
worker.dcp_size = 1
@@ -4059,6 +4073,77 @@ def test_register_kv_caches_shared_storage(layout: KVCacheLayout):
worker.store.register_buffer.assert_called_once_with(raw.data_ptr(), raw.nbytes)
def test_register_kv_caches_uses_transfer_group_memory_domain():
"""Derived GPU caches must not change host-source transfer addresses."""
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupRole,
KVCacheGroupSpec,
KVCacheTensor,
)
host_num_blocks = 4
gpu_num_blocks = 2
page_size = 32
source_spec = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float16
)
indexer_spec = FullAttentionSpec(
block_size=16, num_kv_heads=1, head_size=1, dtype=torch.float16
)
source_group = KVCacheGroupSpec(
["source"],
source_spec,
host_resident=True,
)
indexer_group = KVCacheGroupSpec(
["indexer"],
indexer_spec,
enable_kv_transfer=False,
role=KVCacheGroupRole.HISPARSE_INDEXER,
)
config = KVCacheConfig(
num_blocks=gpu_num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=host_num_blocks * page_size,
layers=["source"],
layer_stride=host_num_blocks * page_size,
block_stride=page_size,
host_resident=True,
),
KVCacheTensor(
size=gpu_num_blocks * page_size,
layers=["indexer"],
layer_stride=gpu_num_blocks * page_size,
block_stride=page_size,
),
],
kv_cache_groups=[source_group, indexer_group],
hisparse_host_num_blocks=host_num_blocks,
)
worker = _make_bare_worker(num_gpu_blocks=gpu_num_blocks)
worker._kv_cache_config = config
worker._kv_cache_groups = list(config.transfer_groups)
worker.token_dbs = [
ChunkedTokenDatabase(KeyMetadata("test-model", 0, 0, 0, 0), block_size=16)
]
source = torch.zeros(host_num_blocks, page_size, dtype=torch.uint8)
indexer = torch.zeros(gpu_num_blocks, page_size, dtype=torch.uint8)
_register_with_mocked_threads(
worker,
{"source": source, "indexer": indexer},
)
db = worker.token_dbs[0]
assert db.kv_caches_base_addr == [source.data_ptr()]
assert db.block_len == [page_size]
worker.store.register_buffer.assert_called_once_with(
source.data_ptr(), source.nbytes
)
def test_register_kv_caches_separate_head_groups():
# LHBNC gives each head group its own region; the K/V split doubles heads.
layout = KVCacheLayout.LHBNC
@@ -304,10 +304,10 @@ def test_multi_example_connector_consistency():
events = get_connector_events()
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
# Initial events bind the block pool, query completion counts, and exchange
# Initial events bind the cache manager, query completion counts, and exchange
# handshake metadata before the request is enqueued.
assert storage1_scheduler_events[:7] == [
"bind_gpu_block_pool",
"bind_kv_cache_manager",
"get_finished_count",
"set_xfer_handshake_metadata_pp_aware",
"on_new_request",
@@ -315,8 +315,8 @@ def test_multi_example_connector_consistency():
"update_state_after_alloc num_blocks=[7] 0",
"build_connector_meta",
]
# First three events are from initialization. During generate(), layer hooks
# run before the deferred load is started after the forward pass.
# First three events are from initialization. Layer hooks run before the
# deferred load starts after the forward pass.
expected_worker_prefix = [
"register_kv_caches",
"set_host_xfer_buffer_ops",
@@ -328,12 +328,12 @@ def test_multi_example_connector_consistency():
]
for connector_name in ("storage1-WORKER", "storage2-WORKER"):
worker_events = events[connector_name]
assert worker_events[:7] == expected_worker_prefix
assert worker_events[: len(expected_worker_prefix)] == expected_worker_prefix
assert worker_events.index("start_load_kv") > worker_events.index(
"save_kv_layer"
)
assert storage2_scheduler_events[:7] == [
"bind_gpu_block_pool",
"bind_kv_cache_manager",
"get_finished_count",
"set_xfer_handshake_metadata_pp_aware",
"on_new_request",
+140 -38
View File
@@ -29,7 +29,10 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
get_current_attn_backend,
)
from vllm.distributed.kv_transfer.kv_connector.v1 import nixl
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorRole,
KVConnectorTransferResults,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import (
MultiKVConnectorStats,
@@ -518,6 +521,7 @@ class FakeNixlConnectorWorker(NixlConnectorWorker):
slot_size_bytes = 4096
self.slot_size_per_layer = [slot_size_bytes]
self.block_len_per_layer = [slot_size_bytes * self.block_size]
self.num_blocks = self.kv_cache_config.num_blocks
self.num_regions = 1
self.block_stride_per_layer = list(self.block_len_per_layer)
self.region_num_blocks = [self.num_blocks]
@@ -635,6 +639,12 @@ class TestNixlHandshake:
if pcp_rank > 0:
assert connector.get_finished(set()) == (set(), set())
worker.get_transfer_results = MagicMock(
return_value=KVConnectorTransferResults(finished_sending={"sent"})
)
results = connector.get_transfer_results(set())
assert results.finished_sending == ({"sent"} if pcp_rank == 0 else set())
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
@@ -1195,6 +1205,7 @@ class TestNixlHandshake:
ssm_sizes=(0, 0),
attn_backend_name=worker.backend_name,
physical_blocks_per_logical_kv_block=1,
region_num_blocks=None,
)
assert worker._build_fa_remote(plan, meta, block_size_ratio=1).tolist() == [
@@ -1435,9 +1446,9 @@ def test_kv_connector_stats(default_vllm_config, dist_init):
metadata = NixlConnectorMetadata()
metadata.add_new_req_to_recv(
request_id=request_id,
local_block_ids=([1, 2, 3],),
local_block_ids=([0],),
kv_transfer_params={
"remote_block_ids": ([4, 5, 6],),
"remote_block_ids": ([0],),
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
"remote_request_id": f"prefill-{request_id}",
"remote_host": "localhost",
@@ -1482,6 +1493,10 @@ def test_kv_connector_stats(default_vllm_config, dist_init):
assert stats_after_reset is None
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
)
def test_reqs_to_send_deadline_rebased_to_worker_clock(default_vllm_config, dist_init):
"""reqs_to_send deadlines are stamped with the scheduler process's
perf_counter, whose epoch differs across processes and (by boot-time
@@ -1906,9 +1921,10 @@ def test_mixed_memory_local_descriptors_split_by_memory_type():
assert handle == 22
assert worker._dram_src_handles_by_block_size[worker.block_size] == 11
assert [block[2] for block in blocks] == [0, 0, 3, 3]
assert [
memory_types = [
call.args[1] for call in worker.nixl_wrapper.get_xfer_descs.call_args_list
] == ["DRAM", "VRAM"]
]
assert memory_types == ["DRAM", "VRAM"]
def test_mixed_memory_read_notifies_after_both_transfers_finish():
@@ -1919,6 +1935,7 @@ def test_mixed_memory_read_notifies_after_both_transfers_finish():
worker._recving_metadata = {"request": MagicMock()}
worker._recving_transfers = defaultdict(list)
worker._pending_recv_notifs = {}
worker._failed_inflight_recvs = set()
worker.xfer_stats = MagicMock()
worker.nixl_wrapper = MagicMock()
worker.nixl_wrapper.make_prepped_xfer.side_effect = [101, 102]
@@ -1959,6 +1976,7 @@ def test_mixed_memory_read_failure_does_not_notify_producer():
worker._recving_metadata = {"request": MagicMock()}
worker._recving_transfers = {"request": [101, 102]}
worker._pending_recv_notifs = {"request": [("prefill", b"request:1")]}
worker._failed_inflight_recvs = set()
worker._is_hma_required = True
worker._failed_recv_reqs = MagicMock()
worker._log_failure = MagicMock() # type: ignore[method-assign]
@@ -2871,9 +2889,11 @@ def test_handshake_failure_returns_finished(default_vllm_config, dist_init):
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FailingNixlWrapper,
)
def test_transfer_setup_failure_returns_finished(default_vllm_config, dist_init):
"""Test that transfer setup failures mark blocks invalid
and return via get_finished."""
@pytest.mark.parametrize("is_hma", [False, True])
def test_transfer_setup_failure_returns_finished(
default_vllm_config, dist_init, is_hma
):
"""Setup failures report the request; only non-HMA reports block IDs."""
vllm_config = create_vllm_config()
connector = NixlConnector(
@@ -2883,6 +2903,7 @@ def test_transfer_setup_failure_returns_finished(default_vllm_config, dist_init)
vllm_config, connector.engine_id, hand_shake_latency=0
)
connector.connector_worker.nixl_wrapper.fail_transfer_setup = True
connector.connector_worker._is_hma_required = is_hma
request_id = "test_transfer_fail"
metadata = NixlConnectorMetadata()
@@ -2912,13 +2933,106 @@ def test_transfer_setup_failure_returns_finished(default_vllm_config, dist_init)
time.sleep(0.1)
connector.start_load_kv(dummy_ctx)
# check that blocks were marked invalid
results = connector.get_transfer_results(finished_req_ids=set())
assert request_id in results.finished_recving
assert results.failed_recving == {request_id}
invalid_blocks = connector.get_block_ids_with_load_errors()
assert invalid_blocks == {7, 8, 9}
assert invalid_blocks == (set() if is_hma else {7, 8, 9})
# ensure request appears in get_finished
class _ScriptedXferWrapper(FakeNixlWrapper):
"""Scripts per-handle xfer states; forbids releasing in-flight handles."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
# handle -> successive states; the last one repeats.
self.xfer_states: dict[int, list[str]] = {}
self.released: list[int] = []
def check_xfer_state(self, handle: int) -> str:
states = self.xfer_states[handle]
return states.pop(0) if len(states) > 1 else states[0]
def release_xfer_handle(self, handle: int) -> None:
# A posted-but-unfinished transfer cannot be aborted: releasing it
# leaves the RDMA READ armed. Production code must never do this.
assert self.xfer_states.get(handle, ["DONE"])[0] != "PROC", (
f"released in-flight handle {handle}"
)
self.released.append(handle)
def _make_split_read_connector(vllm_config, request_id, states):
"""Seed a request with scripted in-flight xfer handles + recv metadata."""
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
)
connector.connector_worker = FakeNixlConnectorWorker(
vllm_config, connector.engine_id, hand_shake_latency=0
)
worker = connector.connector_worker
wrapper = _ScriptedXferWrapper("agent")
worker.nixl_wrapper = wrapper
metadata = NixlConnectorMetadata()
metadata.add_new_req_to_recv(
request_id=request_id,
local_block_ids=([7, 8, 9],),
kv_transfer_params={
"remote_block_ids": ([10, 11, 12],),
"remote_engine_id": FakeNixlConnectorWorker.REMOTE_ENGINE_ID,
"remote_request_id": f"prefill-{request_id}",
"remote_host": "localhost",
"remote_port": 1234,
"remote_tp_size": 1,
},
)
worker._recving_metadata[request_id] = metadata.reqs_to_recv[request_id]
wrapper.xfer_states = dict(states)
worker._recving_transfers[request_id] = list(states)
return connector, worker, wrapper
@patch(
"vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker.NixlWrapper",
FakeNixlWrapper,
)
def test_split_read_failure_defers_report_until_last_handle(
default_vllm_config, dist_init
):
"""One half of a split (mixed DRAM/VRAM) read failing must not report the
request — nor invalidate its blocks — while the sibling xfer is still in
flight: a posted READ cannot be aborted and would DMA into blocks the
scheduler could free and reuse. The report happens exactly once, when the
last handle is terminal."""
request_id = "split_read_partial_failure"
err_handle, live_handle = 11, 22
connector, worker, wrapper = _make_split_read_connector(
create_vllm_config(),
request_id,
{err_handle: ["ERR"], live_handle: ["PROC", "PROC", "DONE"]},
)
# Poll 1: one half fails; the sibling is in flight -> nothing reported.
_, done_recving = connector.get_finished(finished_req_ids=set())
assert request_id in done_recving
assert done_recving == set()
assert connector.get_block_ids_with_load_errors() == set()
assert request_id in worker._recving_metadata
assert wrapper.released == [err_handle]
# Poll 2: sibling still in flight.
_, done_recving = connector.get_finished(finished_req_ids=set())
assert done_recving == set()
# Poll 3: sibling terminal -> reported exactly once, blocks invalidated.
_, done_recving = connector.get_finished(finished_req_ids=set())
assert done_recving == {request_id}
assert connector.get_block_ids_with_load_errors() == {7, 8, 9}
assert request_id not in worker._recving_metadata
assert wrapper.released == [err_handle, live_handle]
# Poll 4: nothing left; no double report.
_, done_recving = connector.get_finished(finished_req_ids=set())
assert done_recving == set()
@patch(
@@ -2938,7 +3052,7 @@ def test_failed_request_skips_kv_postprocessing(
default_vllm_config, dist_init, failure_mode
):
"""Test that failed requests skip KV sync and post-processing in
get_finished().
get_transfer_results().
This is the core safety behavior: when a KV transfer fails at any stage,
the request must still appear in done_recving (so the scheduler can apply
@@ -3023,11 +3137,10 @@ def test_failed_request_skips_kv_postprocessing(
patch.object(worker, "sync_recved_kv_to_device") as mock_sync,
patch.object(worker, "post_process_device_kv_on_receive") as mock_postprocess,
):
_, done_recving = connector.get_finished(finished_req_ids=set())
results = connector.get_transfer_results(finished_req_ids=set())
# The failed request must appear in done_recving so the scheduler
# can handle it (e.g., trigger recompute via kv_load_failure_policy).
assert request_id in done_recving
assert request_id in results.finished_recving
assert results.failed_recving == {request_id}
# Critical: KV sync and post-processing must NOT have been called
# since no valid KV data was received for the failed request.
@@ -3049,19 +3162,7 @@ def test_failed_request_skips_kv_postprocessing(
def test_handles_failing_in_separate_polls_do_not_kill_the_engine(
default_vllm_config, dist_init
):
"""A request whose handles fail in different polls must not crash the engine.
One transfer handle is created per remote rank, and when a peer goes away
they do not all fail in the same poll: one errors while another is still
PROC. The first failure reports the request via _failed_recv_reqs and
get_finished() pops its metadata; when the remaining handles fail in a
later poll, the request must be cleaned up without being reported again.
Reporting twice kills the EngineCore: the scheduler's assert in
_update_from_kv_xfer_finished only expects a finished recv for a request
still waiting for KVs, having moved this one out of
WAITING_FOR_REMOTE_KVS to recompute locally after the first report.
"""
"""Failures across polls report once, after the last writer is terminal."""
vllm_config = create_vllm_config()
connector = NixlConnector(
vllm_config, KVConnectorRole.WORKER, make_kv_cache_config(block_size=16)
@@ -3107,20 +3208,21 @@ def test_handles_failing_in_separate_polls_do_not_kill_the_engine(
):
_, done_recving = connector.get_finished(finished_req_ids=set())
assert request_id in done_recving
assert request_id not in worker._recving_metadata
assert request_id not in done_recving
assert request_id in worker._recving_metadata
assert worker._recving_transfers[request_id] == [second]
assert connector.get_block_ids_with_load_errors() == set()
# Poll 2: the second handle fails too. The request was already reported,
# so it must not be reported a second time: the scheduler has since moved
# it out of WAITING_FOR_REMOTE_KVS to recompute locally, and asserts on a
# finished recv for a request in that state. Its remaining handles are
# still released and the transfer entry removed.
# Only the last terminal handle permits the scheduler to reuse the blocks.
with patch.object(worker.nixl_wrapper, "check_xfer_state", return_value="ERR"):
_, done_recving = connector.get_finished(finished_req_ids=set())
assert request_id not in done_recving
assert done_recving == {request_id}
assert request_id not in worker._recving_transfers
assert request_id not in worker._recving_metadata
assert connector.get_block_ids_with_load_errors() == {1, 2, 3}
_, done_recving = connector.get_finished(finished_req_ids=set())
assert done_recving == set()
def _set_test_speculative_config(
@@ -5,16 +5,31 @@
import gc
from unittest.mock import MagicMock, patch
import msgspec
import pytest
import torch
from tests.v1.attention.utils import MockMambaBuilder
from vllm import LLM, SamplingParams
from vllm.config import KVTransferConfig
from vllm.config import KVTransferConfig, set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker as bw
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
NixlAgentMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
from vllm.v1.core.single_type_kv_cache_manager import (
FullAttentionManager,
SlidingWindowManager,
)
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
MLAAttentionSpec,
)
from .utils import (
create_request,
@@ -272,26 +287,24 @@ def test_logical_to_kernel_block_ids_with_hma():
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"is_rocm,has_mamba,use_host_buffer,done_recving,failed_recving,expected_syncs",
"is_rocm,has_mamba,use_host_buffer,done_recving,expected_syncs",
[
(True, True, False, {"req"}, set(), 1),
(False, True, False, {"req"}, set(), 0),
(True, False, False, {"req"}, set(), 0),
(True, True, True, {"req"}, set(), 0),
(True, True, False, set(), set(), 0),
(True, True, False, {"req"}, {"req"}, 0),
(True, True, False, {"req"}, 1),
(False, False, False, {"req"}, 0),
(False, True, False, {"req"}, 0),
(True, True, True, {"req"}, 0),
(True, True, False, set(), 0),
],
)
def test_sync_device_after_mamba_recv_gates(
def test_sync_device_after_direct_recv_gates(
monkeypatch,
is_rocm,
has_mamba,
use_host_buffer,
done_recving,
failed_recving,
expected_syncs,
):
"""Only direct-GPU Mamba receives on ROCm need a device fence."""
"""Only backends requiring visibility fences synchronize direct receives."""
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import base_worker
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
@@ -309,7 +322,7 @@ def test_sync_device_after_mamba_recv_gates(
lambda: sync_calls.append(True),
)
worker._sync_device_after_mamba_recv(done_recving, failed_recving)
worker._sync_device_after_direct_recv(done_recving)
assert len(sync_calls) == expected_syncs
@@ -399,6 +412,8 @@ def test_read_blocks_for_req_expands_remote_ids(
)
remote_engine_id = "remote-engine"
worker.region_group_ids = list(range(len(resolved_types)))
worker.dst_region_group_ids = {remote_engine_id: list(worker.region_group_ids)}
worker.transfer_topo = MagicMock()
# tp_ratio not exercised (all_source_ranks is empty so no reads run),
@@ -888,6 +903,45 @@ def _make_mock_worker_for_desc_ids(
return worker
@pytest.mark.cpu_test
def test_shared_nixl_region_broadcasts_every_group_block_table():
"""A shared region must receive every group's block table exactly once."""
worker = _make_mock_worker_for_desc_ids(
num_regions=3,
has_mamba=False,
group_spec_types=(FullAttentionSpec,) * 3,
block_len_per_layer=[100] * 3,
)
worker.region_group_ids = [0, bw._SHARED_REGION_GROUP_ID, 2]
worker._uses_region_group_mapping = True
result = worker._compute_desc_ids(
block_ids=([1], [2], [3]),
dst_num_blocks=10,
block_size_ratio=None,
physical_blocks_per_logical=1,
region_num_blocks=[10, 10, 10],
)
assert result.tolist() == [1, 11, 12, 13, 23]
@pytest.mark.cpu_test
def test_nixl_region_sort_preserves_pipeline_layer_order():
"""Numeric layer order prevents PP peers from pairing different regions."""
names = [
"model.layers.10.self_attn",
"model.layers.2.self_attn",
"model.layers.1.self_attn",
]
assert sorted(names, key=bw._region_sort_key) == [
"model.layers.1.self_attn",
"model.layers.2.self_attn",
"model.layers.10.self_attn",
]
@pytest.mark.cpu_test
def test_get_block_descs_ids_hybrid_ssm():
"""Test _compute_desc_ids uses per-group strides for hybrid
@@ -1097,10 +1151,7 @@ def test_post_process_zeroes_untransferred_tail():
worker.device_kv_caches = {"attn.0": attn_cache, "mamba.0": mamba_cache}
fa_group = MagicMock(layer_names=["attn.0"])
ssm_group = MagicMock(layer_names=["mamba.0"])
worker.kv_cache_config = MagicMock(
kv_cache_groups=[fa_group, ssm_group],
transfer_groups=[fa_group, ssm_group],
)
worker.kv_cache_config = MagicMock(transfer_groups=[fa_group, ssm_group])
# The cached property filters mamba layers out of the permuted caches.
attn_caches = NixlConnectorWorker._attention_kv_caches.func(worker)
assert len(attn_caches) == 1 and attn_caches[0] is attn_cache
@@ -1188,6 +1239,11 @@ def _make_fake_kv_cache_manager():
manager = object.__new__(KVCacheManager)
manager.coordinator = MagicMock()
manager.kv_cache_config = MagicMock()
manager.kv_cache_config.kv_cache_groups = (
MagicMock(host_resident=False),
MagicMock(host_resident=False),
)
manager.coordinator.single_type_managers = (
_FakeSingleTypeManager(True, 16, [10, 11, 12, 13, 14, 15]), # attention
_FakeSingleTypeManager(False, 16, [20, 21, 22, 23, 24, 25]), # mamba
@@ -1723,16 +1779,6 @@ def test_exchange_clipped_blocks_ssm_positional_states():
assert clipped == ([1, 2, 3], [0, 5, 6, 7])
@pytest.mark.cpu_test
def test_exchange_clipped_blocks_excludes_nontransfer_groups():
"""Scheduler block tables must match the worker's registered regions."""
sched = make_nixl_scheduler()
sched.kv_cache_config = make_kv_cache_config(block_size=16, swa_enabled=True)
sched.kv_cache_config.kv_cache_groups[1].enable_kv_transfer = False
assert sched.get_exchange_clipped_blocks(([1, 2], [9])) == ([1, 2],)
# ── Hybrid MLA+SSM (KimiLinear-shaped KDA+MLA) tests ─────────────────────
@@ -1786,6 +1832,95 @@ def _make_hybrid_mla_kv_cache_config(num_blocks: int = 4):
)
@pytest.mark.cpu_test
@pytest.mark.parametrize("kernel_block_size", [16, 8])
def test_nixl_keeps_device_block_count_with_hisparse_host_pool(kernel_block_size):
host_num_blocks = 4
gpu_num_blocks = 9
spec = MLAAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=8,
dtype=torch.float16,
)
kv_cache_config = KVCacheConfig(
num_blocks=gpu_num_blocks,
hisparse_host_num_blocks=host_num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=host_num_blocks * spec.page_size_bytes,
layers=["mla.host"],
layer_stride=host_num_blocks * spec.page_size_bytes,
block_stride=spec.page_size_bytes,
host_resident=True,
),
KVCacheTensor(
size=gpu_num_blocks * spec.page_size_bytes,
layers=["mla.device"],
layer_stride=gpu_num_blocks * spec.page_size_bytes,
block_stride=spec.page_size_bytes,
),
],
kv_cache_groups=[
KVCacheGroupSpec(
["mla.host"],
spec,
host_resident=True,
),
KVCacheGroupSpec(["mla.device"], spec),
],
)
vllm_config = create_vllm_config(block_size=16)
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
fake_backend = MagicMock()
fake_backend.get_supported_kernel_block_sizes.return_value = [kernel_block_size]
fake_backend.get_name.return_value = "FLASHMLA"
fake_backend.full_cls_name.return_value = "fake.FLASHMLA"
fake_backend.get_kv_cache_shape.return_value = (1, kernel_block_size, 1, 1)
fake_platform = MagicMock()
fake_platform.device_type = "cuda"
fake_platform.get_nixl_memory_type.return_value = "VRAM"
physical_per_logical = spec.block_size // kernel_block_size
physical_page_bytes = spec.page_size_bytes // physical_per_logical
host_cache = torch.zeros(
host_num_blocks * physical_per_logical, physical_page_bytes, dtype=torch.uint8
)
device_cache = torch.zeros(
gpu_num_blocks * physical_per_logical, physical_page_bytes, dtype=torch.uint8
)
with (
patch.object(bw, "NixlWrapper"),
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
patch.object(bw, "current_platform", fake_platform),
set_current_vllm_config(vllm_config),
):
worker = NixlConnectorWorker(vllm_config, "test-engine", kv_cache_config)
worker.use_mla = True
worker.nixl_wrapper.get_agent_metadata.return_value = b"metadata"
worker.register_kv_caches({"mla.host": host_cache, "mla.device": device_cache})
assert worker.num_blocks == gpu_num_blocks * physical_per_logical
assert dict(zip(worker.region_group_ids, worker.region_num_blocks)) == {
0: host_num_blocks * physical_per_logical,
1: gpu_num_blocks * physical_per_logical,
}
caches = {"mla.host": host_cache, "mla.device": device_cache}
expected_addrs = [
caches[name].data_ptr() + block * stride
for name, stride, count in zip(
worker.region_names,
worker.block_stride_per_layer,
worker.region_num_blocks,
strict=True,
)
for block in range(count)
]
assert worker.src_blocks_data[:, 0].tolist() == expected_addrs
@pytest.mark.cpu_test
def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
"""Hybrid MLA+KDA registration: HMA tensors shared by both layer types
@@ -1803,6 +1938,9 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
kv_cache_config = _make_hybrid_mla_kv_cache_config()
unified_page = kv_cache_config.kv_cache_groups[0].kv_cache_spec.page_size_bytes
vllm_config = create_vllm_config(block_size=12)
# The engine rewrites this global value to the smallest scheduler-group
# block size. NIXL must retain the transferable groups' 12-token geometry.
vllm_config.cache_config.block_size = 4
# kv_buffer_device defaults to the *real* platform's device type, which on
# a CPU-only test host would make this a host-buffer worker: host xfer
# buffers are per-layer, so the HMA shared tensors would not be
@@ -1833,7 +1971,13 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
worker.use_mla = True # opt-125m test config is not MLA; force the flag
worker.nixl_wrapper.get_agent_metadata.return_value = b"fake-agent-metadata"
tensors = [torch.zeros(4 * unified_page, dtype=torch.uint8) for _ in range(2)]
kernel_page = unified_page // 3
block_stride = 2 * kernel_page
backing = torch.zeros(12, block_stride, dtype=torch.uint8)
tensors = [
backing[:, :kernel_page],
backing[:, kernel_page:],
]
# KDA layer first per tensor: exercises the dual-purpose flag merge.
worker.register_kv_caches(
{
@@ -1849,6 +1993,8 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
# 12-token logical blocks over the 4-token MLA kernel block.
assert worker._physical_blocks_per_logical_kv_block == 3
assert worker.block_size == 4 and worker.num_blocks == 12
assert worker.region_group_ids == [bw._SHARED_REGION_GROUP_ID] * 2
assert worker.block_stride_per_layer == [block_stride, block_stride]
# Both shared tensors are dual-purpose: their FA view is MLA even though
# a KDA layer registered them first.
assert worker._region_is_mla == [True, True]
@@ -1861,8 +2007,20 @@ def test_register_kv_caches_hybrid_mla_dual_purpose_regions():
# Mamba descs: 2 regions x (3 conv sub-projections + 1 ssm) x 4 blocks.
assert worker.src_blocks_data.shape == (24 + 32, 3)
fa_descs = worker.src_blocks_data[:24]
assert fa_descs[1][0] - fa_descs[0][0] == unified_page // 3
expected_addrs = [
tensor.data_ptr() + block * block_stride
for tensor in tensors
for block in range(12)
]
assert fa_descs[:, 0].tolist() == expected_addrs
assert all(size == unified_page // 3 for size in fa_descs[:, 1])
worker.nixl_wrapper.register_memory.assert_called_once()
metadata = msgspec.msgpack.decode(
worker.xfer_handshake_metadata.agent_metadata_bytes,
type=NixlAgentMetadata,
)
assert metadata.block_strides == [block_stride, block_stride]
assert metadata.region_num_blocks == [12, 12]
@pytest.mark.cpu_test
@@ -1909,6 +2067,8 @@ def test_push_write_hybrid_mla_replicates_attention():
worker.dst_xfer_side_handles = {engine_id: {0: 100, 1: 101}}
worker.src_xfer_handles_by_tp_ratio = {(-2, 4): [200, 201]}
worker.src_xfer_handles_by_block_size = {4: 300}
worker.region_group_ids = [0, 1]
worker.dst_region_group_ids = {engine_id: [0, 1]}
worker._sending_transfers = defaultdict(list)
worker._sending_transfers_lock = threading.Lock()
worker.kv_cache_config = _make_hybrid_mla_kv_cache_config()
@@ -186,11 +186,16 @@ def test_overlaid_transfer_groups_share_region_geometry():
worker.host_buffer_kv_cache_layout = "NHD"
worker._physical_blocks_per_logical_kv_block = 1
worker._logical_num_blocks = num_blocks
worker.region_mem_types = []
worker.region_group_ids = []
worker.region_mem_types = []
worker._mixed_mem_types = False
worker.region_names = []
worker.region_num_blocks = []
worker._mixed_mem_types = False
worker._desc_is_dram_by_block_size = {}
worker._desc_pos_by_block_size = {}
worker._dram_src_handles_by_block_size = {}
worker._region_is_mla = []
worker.block_len_per_layer = []
worker.block_stride_per_layer = []
@@ -304,8 +309,6 @@ def _make_mla_hybrid_worker(local_block_size, kernel_block_size, num_logical_blo
# would not be deduplicated. Pin it to the faked device type.
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
from unittest.mock import MagicMock
fake_backend = MagicMock()
fake_backend.get_supported_kernel_block_sizes.return_value = [kernel_block_size]
fake_backend.get_name.return_value = "FLASHMLA"
@@ -552,6 +555,16 @@ def _make_remote_meta(
)
def _register_remote_agents(worker, metadata, tp_size):
"""Mirror the async handshake callback that publishes prepared agents."""
worker._remote_agents[metadata.engine_id] = {
(0, rank): worker.add_remote_agent(
metadata, remote_tp_rank=rank, remote_tp_size=tp_size
)
for rank in range(tp_size)
}
def _owned_byte_ranges(worker, group_logical_ids):
"""Byte ranges owned by a request: for each HMA region tensor, every
logical block id of every group maps to one unified page."""
@@ -609,8 +622,7 @@ def test_hetero_ppl_multi_read_writes_stay_within_request_blocks():
remote_num_logical=12,
remote_ssm_sizes=(24, 32),
)
for rank in (0, 1):
worker.add_remote_agent(meta_r, remote_tp_rank=rank, remote_tp_size=2)
_register_remote_agents(worker, meta_r, 2)
# Request B: 17 matched tokens. Local: 2 logical blocks (24 tok
# capacity); remote: 16 prefilled tokens -> 2 remote logical blocks.
@@ -626,7 +638,7 @@ def test_hetero_ppl_multi_read_writes_stay_within_request_blocks():
"remote_block_ids": remote_ids,
"remote_engine_id": "remote-engine",
"remote_request_id": "prefill-req-b",
"remote_host": "localhost",
"remote_host": "remote-host",
"remote_port": 1234,
"tp_size": 2,
},
@@ -709,8 +721,7 @@ def _run_hetero_case(
remote_num_logical=max(2 * n_remote + 4, 8),
remote_ssm_sizes=(48 // tp_size, 64 // tp_size),
)
for rank in range(tp_size):
worker.add_remote_agent(meta_r, remote_tp_rank=rank, remote_tp_size=tp_size)
_register_remote_agents(worker, meta_r, tp_size)
# Sparse ids so neighbors exist between the request's blocks.
local_attn = [2 * i + 1 for i in range(n_local)]
@@ -726,7 +737,7 @@ def _run_hetero_case(
"remote_block_ids": remote_ids,
"remote_engine_id": "remote-engine",
"remote_request_id": "prefill-req-b",
"remote_host": "localhost",
"remote_host": "remote-host",
"remote_port": 1234,
"tp_size": tp_size,
},
@@ -15,7 +15,7 @@ requiring a real NIXL agent or network:
* The worker matches D registrations against P finished blocks (both
scenario directions) and forwards non-PUSH_REG NIXL notifs to the main
thread's ``_get_new_notifs``.
* ``get_finished`` enqueues evictions for the writer.
* ``get_transfer_results`` enqueues evictions for the writer.
"""
from __future__ import annotations
@@ -32,6 +32,9 @@ from unittest.mock import MagicMock, patch
import msgspec
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorTransferResults,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
PUSH_REG_NOTIF_PREFIX,
NixlAgentMetadata,
@@ -347,6 +350,8 @@ class _StubWriterWorker(NixlPushConnectorWorker):
# Base worker fields touched by start_load_kv / _get_new_notifs.
w._recving_metadata = {}
w._pending_recv_notifs = {}
w._failed_inflight_recvs = set()
w._recving_transfers = defaultdict(list)
w._reqs_to_process = set()
w._reqs_to_send = {}
@@ -742,21 +747,19 @@ class TestPushWriterNotifs:
assert notified == set()
assert request_id in w._recving_transfers
def test_get_finished_evicts_completed_state(self):
"""``get_finished`` should enqueue evictions and wake the writer."""
def test_get_transfer_results_evicts_completed_state(self):
"""Transfer completion should enqueue evictions and wake the writer."""
w = _StubWriterWorker.fresh()
# Stub the base ``get_finished`` to return one done_sending entry.
# Patch via the MRO's parent class.
with patch.object(
NixlPushConnectorWorker.__mro__[1],
"get_finished",
return_value=({"req-done"}, set()),
"get_transfer_results",
return_value=KVConnectorTransferResults(finished_sending={"req-done"}),
):
done_sending, done_recving = w.get_finished()
results = w.get_transfer_results()
assert "req-done" in done_sending
assert done_recving == set()
assert results.finished_sending == {"req-done"}
assert results.finished_recving == set()
# Eviction enqueued for the writer.
evicted = []
while True:
@@ -928,19 +931,21 @@ class TestPushWriterNegative:
assert len(w._pending_d_registrations) == 1
assert w.start_push_calls == []
def test_get_finished_enqueues_eviction_for_each_done_request(self):
"""``get_finished`` must enqueue an eviction for every request
def test_get_transfer_results_enqueues_each_completed_request(self):
"""``get_transfer_results`` must enqueue every completed request
in ``done_sending`` so the writer can drop stale matching state.
Unlike the happy-path test, this verifies the *cardinality*: N
completed requests -> N evictions, in order."""
w = _StubWriterWorker.fresh()
with patch.object(
NixlPushConnectorWorker.__mro__[1],
"get_finished",
return_value=({"req-1", "req-2", "req-3"}, set()),
"get_transfer_results",
return_value=KVConnectorTransferResults(
finished_sending={"req-1", "req-2", "req-3"}
),
):
done_sending, _ = w.get_finished()
assert done_sending == {"req-1", "req-2", "req-3"}
results = w.get_transfer_results()
assert results.finished_sending == {"req-1", "req-2", "req-3"}
evicted: list[str] = []
while True:
@@ -950,23 +955,47 @@ class TestPushWriterNegative:
break
assert sorted(evicted) == ["req-1", "req-2", "req-3"]
def test_get_finished_with_no_completions_does_not_enqueue_eviction(self):
def test_empty_transfer_results_do_not_enqueue_eviction(self):
"""If there's nothing newly done, no eviction should be enqueued.
The wake event IS still set because ``get_finished`` always wakes
the writer to drain notifs."""
w = _StubWriterWorker.fresh()
with patch.object(
NixlPushConnectorWorker.__mro__[1],
"get_finished",
return_value=(set(), set()),
"get_transfer_results",
return_value=KVConnectorTransferResults(),
):
done_sending, done_recving = w.get_finished()
assert done_sending == set()
assert done_recving == set()
results = w.get_transfer_results()
assert results.finished_sending == set()
assert results.finished_recving == set()
assert w._evict_finished_inbox.qsize() == 0
# Wake set so the writer drains NIXL notifs even when idle.
assert w._push_writer_wake.is_set()
def test_failed_push_transfer_finishes_sending(self):
w = _StubWriterWorker.fresh()
w.nixl_wrapper = MagicMock()
w.nixl_wrapper.check_xfer_state.return_value = "ERR"
w.xfer_stats = MagicMock()
w._log_failure = MagicMock()
w._sending_transfers["req-failed"] = [17]
w._reqs_to_send["req-failed"] = time.perf_counter() + 30
w._reqs_to_process.add("req-failed")
with patch.object(
NixlPushConnectorWorker.__mro__[1],
"get_transfer_results",
return_value=KVConnectorTransferResults(),
):
results = w.get_transfer_results()
assert results.finished_sending == {"req-failed"}
assert results.finished_recving == set()
assert "req-failed" not in w._sending_transfers
assert "req-failed" not in w._reqs_to_send
assert "req-failed" not in w._reqs_to_process
w.nixl_wrapper.release_xfer_handle.assert_called_once_with(17)
def test_get_new_notifs_unknown_request_is_logged_and_skipped(self, caplog):
"""A completion notif for a request the worker doesn't know
about should be logged but not crash."""
@@ -1160,6 +1189,7 @@ class TestPushWriterMlaReplication:
)
}
w._logical_to_kernel_block_ids = lambda block_ids, ratio: block_ids
w.dst_region_group_ids[engine_id] = [0]
w.dst_xfer_side_handles = {engine_id: {r: 1000 + r for r in d_ranks}}
w.src_xfer_handles_by_block_size = {16: 2000}
w._remote_agents = {engine_id: {(0, r): f"agent-{r}" for r in d_ranks}}
@@ -1240,6 +1270,8 @@ class TestPushWriterMlaReplication:
)
}
w._logical_to_kernel_block_ids = lambda block_ids, ratio: block_ids
w.region_group_ids = [0, 1, 2, 3]
w.dst_region_group_ids[engine_id] = [0, 1, 2, 3]
w.dst_xfer_side_handles = {engine_id: {0: 1000, 1: 1001}}
w.src_xfer_handles_by_tp_ratio = {(-2, 16): [2000, 2001]}
@@ -15,12 +15,14 @@ class DummyModelRunnerOutput(ModelRunnerOutput):
finished_sending: set[str] | None = None,
finished_recving: set[str] | None = None,
invalid_block_ids: set[int] | None = None,
failed_recving: set[str] | None = None,
expected_finished_count: int = 0,
):
self.kv_connector_output = KVConnectorOutput(
finished_sending=finished_sending,
finished_recving=finished_recving,
invalid_block_ids=invalid_block_ids or set(),
failed_recving=failed_recving or set(),
expected_finished_count=expected_finished_count,
)
@@ -73,7 +75,9 @@ def test_aggregate_workers_output():
output1 = DummyModelRunnerOutput(invalid_block_ids={3, 4})
output2 = DummyModelRunnerOutput(
finished_recving={"req2"}, invalid_block_ids={4, 5}
finished_recving={"req2"},
invalid_block_ids={4, 5},
failed_recving={"req3"},
)
aggregated = aggregator.aggregate([output1, output2])
@@ -83,6 +87,12 @@ def test_aggregate_workers_output():
assert aggregated.finished_sending is None
assert aggregated.finished_recving == {"req2"}
assert aggregated.invalid_block_ids == {3, 4, 5}
assert not aggregated.failed_recving
output1 = DummyModelRunnerOutput(finished_recving={"req3"})
output2 = DummyModelRunnerOutput(finished_recving={"req3"})
aggregated = aggregator.aggregate([output1, output2])
assert aggregated.kv_connector_output.failed_recving == {"req3"}
def test_aggregate_workers_output_with_expected_finished_count():
+1
View File
@@ -539,6 +539,7 @@ def make_nixl_scheduler(
sched._heartbeat_interval = kv_lease_duration // 6
# Fields touched by build_connector_meta / request_finished:
sched._reqs_need_recv = {}
sched._hisparse_host_blocks_to_recv = {}
sched._reqs_need_send = {}
sched._reqs_in_batch = set()
sched._reqs_not_processed = set()
+3 -3
View File
@@ -62,7 +62,7 @@ def _make_offloading_config(
normalized_extra_config["cpu_bytes_to_use"] = cpu_bytes_to_use
if groups is None:
groups = (OffloadingGroupConfig(16, ("layer",)),)
groups = (OffloadingGroupConfig(16, ("layer",), 0),)
return OffloadingConfig(
groups=groups,
@@ -493,8 +493,8 @@ def test_offloading_spec_has_replicated_layout_default():
def test_offloading_spec_uses_normalized_chunk_geometry():
groups = (
OffloadingGroupConfig(12, ("full_layer",)),
OffloadingGroupConfig(16, ("mla_layer",)),
OffloadingGroupConfig(12, ("full_layer",), 0),
OffloadingGroupConfig(16, ("mla_layer",), 1),
)
spec = _create_spec(
groups=groups,
+4 -1
View File
@@ -26,8 +26,11 @@ def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
OffloadingGroupConfig(
tokens_per_block=tokens_per_block,
layer_names=(layer_name,),
group_id=group_id,
)
for group_id, (tokens_per_block, layer_name) in enumerate(
kwargs.get("groups", ())
)
for tokens_per_block, layer_name in kwargs.get("groups", ())
),
worker_kv_bytes_per_block=0,
enable_kv_cache_events=False,
@@ -400,6 +400,8 @@ def test_register_mixed_page_sizes_in_one_cache_group(monkeypatch):
vllm_config.cache_config = CacheConfig()
vllm_config.cache_config.kv_cache_layout = layout.name
vllm_config.cache_config.num_gpu_blocks_override = None
vllm_config.attention_config.hisparse_config = None
vllm_config.kv_transfer_config = None
pages = [spec.page_size_bytes for spec in specs.values()]
num_blocks = 4
@@ -458,6 +460,8 @@ def test_register_mixed_page_sizes_odd_block_counts(monkeypatch, rank_blocks):
vllm_config.cache_config = CacheConfig()
vllm_config.cache_config.kv_cache_layout = layout.name
vllm_config.cache_config.num_gpu_blocks_override = None
vllm_config.attention_config.hisparse_config = None
vllm_config.kv_transfer_config = None
pages = [spec.page_size_bytes for spec in specs.values()]
kv_cache_config = get_kv_cache_config_from_groups(
@@ -507,6 +511,8 @@ def test_mixed_page_byte_placement_is_dcp_invariant():
vllm_config.cache_config.kv_cache_layout = KVCacheLayout.LBNHC.name
vllm_config.cache_config.num_gpu_blocks_override = None
vllm_config.parallel_config.decode_context_parallel_size = dcp_size
vllm_config.attention_config.hisparse_config = None
vllm_config.kv_transfer_config = None
page_bytes = sum(spec.page_size_bytes for spec in specs.values())
config = get_kv_cache_config_from_groups(vllm_config, [group], page_bytes * 4)
+15
View File
@@ -16,6 +16,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
ChunkedLocalAttentionManager,
CrossAttentionManager,
FullAttentionManager,
HiSparseSourceManager,
MambaManager,
SingleTypeKVCacheManager,
SinkFullAttentionManager,
@@ -27,6 +28,7 @@ from vllm.v1.kv_cache_interface import (
CrossAttentionSpec,
FullAttentionSpec,
HiddenStateCacheSpec,
KVCacheGroupRole,
KVCacheSpec,
KVCacheSpecKind,
MambaSpec,
@@ -176,6 +178,19 @@ class TestKVCacheSpecRegistry:
is spec_uniform_base_map[spec_cls]
)
@pytest.mark.parametrize("role", list(KVCacheGroupRole))
def test_mla_manager_selection_by_role(self, role):
"""Only the source role overrides ordinary MLA manager selection."""
expected = (
HiSparseSourceManager
if role == KVCacheGroupRole.HISPARSE_SOURCE
else FullAttentionManager
)
assert (
KVCacheSpecRegistry.get_manager_class(make_spec(MLAAttentionSpec), role)
is expected
)
@pytest.mark.parametrize("spec_cls", list(spec_manager_map))
def test_custom_spec_register(self, spec_cls):
"""A decorated custom spec resolves to the declared manager."""
+126 -1
View File
@@ -13,15 +13,18 @@ import pytest
import torch
from tests.v1.attention.utils import dense_kv_cache_views
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.hisparse.binding import allocate_hisparse_kv_caches
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
HiSparseResidentSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheLayout,
KVCacheTensor,
MLAAttentionSpec,
SparseCacheRole,
compute_layout_strides,
)
from vllm.v1.worker.gpu import attn_utils
@@ -36,6 +39,60 @@ from vllm.v1.worker.utils import (
)
@pytest.mark.parametrize(
("enabled", "block_size", "main_sizes", "indexer_sizes", "expected"),
[
(True, 256, [64], [64], 64),
(True, 64, [32, 64], [16, 32], 32),
(True, 64, [MultipleOf(16)], [32], 32),
(True, 64, [64], [32], None),
(False, 256, [64], [64], 256),
],
)
def test_get_kv_cache_spec_resolves_hisparse_block_size(
monkeypatch, enabled, block_size, main_sizes, indexer_sizes, expected
):
"""Resolve shared MLA geometry before planning; leave other specs alone."""
specs = {
"main": MLAAttentionSpec(
block_size=block_size, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
),
"indexer": MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
cache_role=SparseCacheRole.INDEXER,
),
"dense": FullAttentionSpec(
block_size=block_size, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
),
}
layers = {}
for name, sizes in zip(specs, [main_sizes, indexer_sizes, [block_size]]):
backend = SimpleNamespace(
customize_spec=AttentionBackend.customize_spec,
get_supported_kernel_block_sizes=lambda sizes=sizes: sizes,
)
layers[name] = SimpleNamespace(
get_kv_cache_spec=lambda _, spec=specs[name]: spec,
get_attn_backend=lambda backend=backend: backend,
)
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=object() if enabled else None)
)
if expected is None:
with pytest.raises(ValueError, match="supported by every sparse"):
attn_utils.get_kv_cache_spec(config)
return
resolved = attn_utils.get_kv_cache_spec(config)
assert resolved["main"].block_size == resolved["indexer"].block_size == expected
assert resolved["dense"] is specs["dense"]
assert all(spec.block_size == block_size for spec in specs.values())
class _FakeMetadataBuilder:
def __init__(self, support: AttentionCGSupport):
self.support = support
@@ -390,3 +447,71 @@ def test_copy_kv_cache_blocks_with_virtual_block_splitting(
torch.testing.assert_close(
cache[dst_start + physical_idx], expected[layer_idx][physical_idx]
)
def test_allocate_hisparse_kv_caches_host_pool_and_view_less_specs():
"""Host tensors get their own backing; view-less specs keep the raw one."""
spec = FullAttentionSpec(
block_size=2, num_kv_heads=1, head_size=4, dtype=torch.float32
)
page = spec.page_size_bytes
resident_spec = HiSparseResidentSpec(block_size=2, page_size=page)
device_size = 4 * page
config = KVCacheConfig(
num_blocks=4,
hisparse_host_num_blocks=3,
kv_cache_tensors=[
KVCacheTensor(
size=3 * page,
layers=["source"],
layer_stride=3 * page,
block_stride=page,
host_resident=True,
),
KVCacheTensor(
size=device_size,
layers=["indexer"],
layer_stride=device_size,
block_stride=page,
),
KVCacheTensor(
size=device_size,
layers=["resident"],
layer_stride=device_size,
block_stride=page,
),
],
kv_cache_groups=[
KVCacheGroupSpec(["source"], spec, host_resident=True),
KVCacheGroupSpec(["indexer"], spec),
KVCacheGroupSpec(["resident"], resident_spec),
],
)
host_buffers: list[torch.Tensor] = []
def host_allocator(size: int) -> torch.Tensor:
host_buffers.append(torch.zeros(size, dtype=torch.int8))
return host_buffers[-1]
caches = allocate_hisparse_kv_caches(
config,
torch.device("cpu"),
KVCacheLayout.LBHNC,
[2, 2, 2],
SimpleNamespace(allocate=host_allocator),
)
assert len(config.kv_cache_tensors) == 3
assert [buf.numel() for buf in host_buffers] == [3 * page]
assert caches["source"].shape[0] == 3
assert (
caches["source"].untyped_storage().data_ptr()
== host_buffers[0].untyped_storage().data_ptr()
)
assert caches["indexer"].shape[0] == 4
backing = caches["resident"]
assert backing.dtype == torch.int8 and backing.numel() >= device_size
assert (
backing.untyped_storage().data_ptr()
== caches["indexer"].untyped_storage().data_ptr()
)
@@ -134,6 +134,7 @@ def test_overlaid_zeroer_dedups_segments_with_max_span():
vllm_config.cache_config = CacheConfig()
vllm_config.cache_config.num_gpu_blocks_override = None
vllm_config.cache_config.kv_cache_layout = "BLHNC"
vllm_config.attention_config.hisparse_config = None
config = get_kv_cache_config_from_groups(vllm_config, groups, 8 * 1024 * 1024)
views = allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout.BLHNC, None)
buf_ptr = views["g1.big"].data_ptr()
@@ -351,6 +351,8 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
speculator = object.__new__(_TestSpeculator)
speculator.num_speculative_steps = 4
speculator.current_draft_step = torch.tensor(0)
speculator.slot_mapping_observer = None
speculator.host_mirror_forward_observer = None
speculator.input_buffers = SimpleNamespace(
positions=torch.arange(2),
query_start_loc=torch.arange(3),
+23 -3
View File
@@ -5,9 +5,11 @@ from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
import vllm.v1.worker.gpu.kv_connector as kv_connector_module
from vllm.config import KVTransferConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorTransferResults
from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector
@@ -18,9 +20,9 @@ def _make_connector(
backend = Mock()
backend.handle_preemptions.side_effect = lambda _: events.append("handle")
backend.bind_connector_metadata.side_effect = lambda _: events.append("bind")
backend.start_load_kv.side_effect = lambda _: events.append("start")
backend.start_load_kv.side_effect = lambda *_args, **_kwargs: events.append("start")
backend.wait_for_save.side_effect = lambda: events.append("wait")
backend.get_finished.side_effect = lambda _: (set(), set())
backend.get_transfer_results.return_value = KVConnectorTransferResults()
backend.get_block_ids_with_load_errors.return_value = set()
backend.get_kv_connector_stats.return_value = None
backend.get_kv_connector_kv_cache_events.return_value = None
@@ -61,7 +63,15 @@ def test_load_start_phase(
connector = _make_connector(monkeypatch, events)
output = _scheduler_output(has_sync_kv_loads)
connector.pre_forward(output) # type: ignore[arg-type]
request_indices = torch.tensor([3, 1])
request_ids = ["first", "second"]
attn_metadata = {"layer": object()}
connector.pre_forward( # type: ignore[arg-type]
output,
request_state_indices=request_indices,
request_ids=request_ids,
attn_metadata=attn_metadata,
)
assert events == (
["handle", "bind", "start"] if has_sync_kv_loads else ["handle", "bind"]
)
@@ -69,6 +79,16 @@ def test_load_start_phase(
connector.post_forward(set())
assert events == ["handle", "bind", "start", "wait", "clear"]
kwargs = connector.kv_connector.start_load_kv.call_args.kwargs
assert kwargs["request_state_indices"] is request_indices
assert kwargs["request_ids"] is request_ids
assert kwargs["attn_metadata"] is attn_metadata
# A subsequent step without a forward must not reuse the prior batch.
connector.no_forward(_scheduler_output(False)) # type: ignore[arg-type]
assert connector.kv_connector.start_load_kv.call_count == 2
assert connector.kv_connector.start_load_kv.call_args.kwargs == {}
def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
events: list[str] = []
+4
View File
@@ -312,6 +312,10 @@ def test_select_common_block_size_uses_largest_shared_int():
assert selected_size == 64
def test_select_common_block_size_without_active_backends_uses_manager_size():
assert select_common_block_size(256, []) == 256
def test_select_common_block_size_accepts_rocm_sparse_block_size_16(monkeypatch):
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
@@ -237,6 +237,7 @@ def _make_capture_runner(captured: bool) -> GPUModelRunner:
runner.attn_groups = None
runner.kv_cache_config = None
runner.use_aux_hidden_state_outputs = False
runner.kv_connector = model_runner_module.NO_OP_KV_CONNECTOR
return runner
@@ -89,6 +89,7 @@ def _make_runner(
return SimpleNamespace(
num_speculative_steps=num_spec_steps,
adaptive_verification=None,
rejection_sampler=None,
decode_query_len=num_spec_steps + 1,
is_pooling_model=False,
is_encoder_decoder=False,
@@ -5,8 +5,10 @@ from contextlib import AbstractContextManager
from types import SimpleNamespace
from typing import Any, cast
import pytest
import torch
import vllm.v1.hisparse.binding as hisparse_binding
import vllm.v1.worker.gpu.attn_utils as attn_utils
import vllm.v1.worker.gpu_model_runner as gpu_model_runner
from vllm.v1.worker.gpu_worker import Worker
@@ -26,7 +28,8 @@ class _AllocationScope(AbstractContextManager):
self.active = False
def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch) -> None:
@pytest.mark.parametrize("hisparse", [False, True])
def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch, hisparse) -> None:
scope = _AllocationScope()
kv_caches = {"layer": torch.empty(0)}
@@ -38,23 +41,48 @@ def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch) -> None:
assert not scope.active
monkeypatch.setattr(attn_utils, "allocate_kv_cache", allocate)
monkeypatch.setattr(attn_utils, "bind_kv_cache", bind)
monkeypatch.setattr(attn_utils, "bind_kv_cache_to_layers", bind)
monkeypatch.setattr(attn_utils, "get_shared_kv_cache_layers", lambda config: {})
hisparse_bindings = []
if hisparse:
host_pool = SimpleNamespace()
def bind_hisparse(**kwargs):
hisparse_bindings.append(kwargs)
assert scope.active
assert kwargs["kv_caches"] is kv_caches
assert kwargs["host_pool"] is host_pool
return [
SimpleNamespace(
view=SimpleNamespace(cache=torch.empty(1, 1, 1), block_size=1),
runtime=SimpleNamespace(),
)
]
monkeypatch.setattr(hisparse_binding, "HiSparseHostPool", lambda: host_pool)
monkeypatch.setattr(hisparse_binding, "allocate_hisparse_kv_caches", allocate)
monkeypatch.setattr(hisparse_binding, "bind_hisparse_kv_caches", bind_hisparse)
config = SimpleNamespace(
attention_config=SimpleNamespace(
hisparse_config=object() if hisparse else None
),
scheduler_config=SimpleNamespace(max_num_seqs=1, max_num_batched_tokens=1),
cache_config=SimpleNamespace(get_resolved_kv_cache_layout=lambda: None),
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type="test")),
)
result = attn_utils.init_kv_cache(
[],
{},
SimpleNamespace(kv_cache_groups=[]),
torch.device("cpu"),
[],
config,
kv_cache_allocation_context=scope,
block_tables=SimpleNamespace(),
)
assert len(hisparse_bindings) == int(hisparse)
assert result is kv_caches
assert not scope.active
+7 -1
View File
@@ -35,10 +35,15 @@ def test_mixed_warmup_skipped_for_single_seq(max_num_reqs):
def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_warmup):
"""Startup must warm fixed drafts before calibration and retain its manager."""
manager = SimpleNamespace(cost_tables=None)
runner = SimpleNamespace(adaptive_verification=manager)
rejection_sampler = SimpleNamespace(enable_adaptive_verification=True)
runner = SimpleNamespace(
adaptive_verification=manager,
rejection_sampler=rejection_sampler,
)
def run_steps(model_runner, execute, sample):
assert model_runner.adaptive_verification is None
assert not model_runner.rejection_sampler.enable_adaptive_verification
if fail_warmup:
raise RuntimeError("warmup failed")
@@ -50,3 +55,4 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
warmup.warmup_kernels(runner, _fail, _fail)
assert runner.adaptive_verification is manager
assert manager.cost_tables is None
assert rejection_sampler.enable_adaptive_verification
+891 -3
View File
@@ -1,13 +1,897 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import deque
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock, call
import numpy as np
import pytest
import torch
import vllm.v1.attention.backends.mla.index_group as index_group_module
import vllm.v1.hisparse.runtime as hisparse_runtime_module
from vllm.config import CUDAGraphMode
from vllm.config.mamba import MambaBackendEnum, MambaConfig
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse import (
worker as hisparse_worker_module,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
_flatten_row_mirrors,
_select_written_row_mirrors,
_SlotMappingStaging,
)
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
from vllm.v1.worker.utils import bind_kv_cache
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror
from vllm.v1.worker.utils import (
bind_kv_cache,
bind_kv_cache_to_layers,
copy_kv_cache_blocks_inplace,
)
def _make_hisparse_worker() -> HiSparseConnectorWorker:
worker = object.__new__(HiSparseConnectorWorker)
worker.refines_row_mirrors = False
worker._slot_mapping_staging = None
worker._row_mirror_num_rows = 0
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._pending_dma_descriptors = deque()
worker._dma_free_descriptors = []
worker.host_write_events = (MagicMock(), MagicMock())
worker.host_write_event = worker.host_write_events[1]
worker._next_host_write_event = 0
worker.dma_stream = None
worker.shared_host_region = None
worker._completed_host_copy_dst_ids = []
worker._slot_mapping_staging = None
return worker
def test_hisparse_row_mirrors_follow_runner_request_order():
first = SparseKVRowMirror((1,), 10, 1)
second = SparseKVRowMirror((2,), 20, 1)
row_mirrors = {"first": (first,), "second": (second,)}
assert _flatten_row_mirrors(row_mirrors, ["second", "first"]) == (
second,
first,
)
def test_hisparse_gpu_slots_select_written_rows_from_async_envelope():
"""Rejected speculative rows must never become durable host KV."""
candidates = (
SparseKVRowMirror((100, 500), 1000, 4),
SparseKVRowMirror((20, 300), 2000, 4),
)
source_slots = np.array([102, 103, -1, 21, 22, 23], dtype=np.int64)
assert _select_written_row_mirrors(candidates, source_slots, 0) == (
SparseKVRowMirror((102, 502), 1002, 2),
SparseKVRowMirror((21, 301), 2001, 3),
)
def test_hisparse_written_rows_preserve_scheduler_page_boundaries():
candidates = (
SparseKVRowMirror((0,), 100, 4),
SparseKVRowMirror((4,), 104, 4),
)
assert _select_written_row_mirrors(
candidates, np.array([2, 3, 4, 5], dtype=np.int64), 0
) == (
SparseKVRowMirror((2,), 102, 2),
SparseKVRowMirror((4,), 104, 2),
)
def test_hisparse_written_rows_without_host_destinations_are_skipped():
assert (
_select_written_row_mirrors((), np.array([100, 101], dtype=np.int64), 0) == ()
)
def test_hisparse_appends_reference_slots_within_a_mirror_phase(monkeypatch):
"""Separate context and query writes must share one mirror phase."""
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.refines_row_mirrors = True
state = _SlotMappingStaging(
MagicMock(), MagicMock(), torch.empty(4, dtype=torch.int64)
)
worker._slot_mapping_staging = state
handle = SimpleNamespace(runtime=SimpleNamespace(resident_source_index=1))
worker.cache_layer_names = ["layer"]
worker.cache_handles = [handle]
worker._layer_mirror_callbacks = (MagicMock(),)
worker._row_mirror_num_rows = 1
worker._submitted_mirror_layers = set()
main_stream = MagicMock()
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: main_stream)
monkeypatch.setattr(torch.cuda, "stream", lambda stream: nullcontext())
handle.slot_mapping = torch.tensor([7, 8], dtype=torch.int64)
worker._stage_row_mirror_mapping(2)
handle.slot_mapping = torch.tensor([9, 10], dtype=torch.int64)
worker._stage_row_mirror_mapping(2)
torch.testing.assert_close(state.slots, torch.tensor([7, 8, 9, 10]))
assert state.stream.wait_stream.call_args_list == [call(main_stream)] * 2
assert state.event.record.call_args_list == [call(state.stream)] * 2
assert state.num_tokens == 4
assert state.source_index == 1
@pytest.mark.parametrize(
("mode", "expected_calls"),
[(CUDAGraphMode.PIECEWISE, 1), (CUDAGraphMode.FULL, 0)],
)
def test_hisparse_submits_layer_mirror_at_replayed_attention_boundary(
monkeypatch, mode, expected_calls
):
"""Piecewise replay must pipeline DMA; FULL replay uses the batch fallback."""
handle = object.__new__(hisparse_runtime_module.HiSparseCacheHandle)
handle.dummy_batch = False
handle.decode_batch = False
handle.submit_layer_mirror = MagicMock()
monkeypatch.setattr(
hisparse_runtime_module,
"get_forward_context",
lambda: SimpleNamespace(cudagraph_runtime_mode=mode),
)
handle.finish_kv_update()
assert handle.submit_layer_mirror.call_count == expected_calls
def test_copy_cpu_kv_cache_logical_blocks_ignores_storage_padding():
backing = torch.full((10, 2, 3), -1, dtype=torch.float32)
cache = backing[1:9]
cache[2:4] = 7
cache[6:8] = 11
copy_kv_cache_blocks_inplace(
[cache],
num_blocks=4,
kv_cache_block_copies=[
KVCacheBlockCopy(1, 0),
KVCacheBlockCopy(3, 2),
],
)
torch.testing.assert_close(cache[0:2], torch.full_like(cache[0:2], 7))
torch.testing.assert_close(cache[4:6], torch.full_like(cache[4:6], 11))
assert (backing[0] == -1).all()
assert (backing[9] == -1).all()
@pytest.mark.parametrize("shared,rank", [(False, 0), (True, 0), (True, 1)])
@pytest.mark.parametrize("has_copies", [False, True])
def test_hisparse_host_copy_waits_for_writes_on_writer_only(
monkeypatch, shared, rank, has_copies
):
"""Host CoW must read completed writes; shared non-writers only barrier."""
worker = _make_hisparse_worker()
cache = torch.zeros((2, 1, 1))
worker.host_caches = [cache]
worker.host_num_blocks = 2
worker.shared_host_region = object() if shared else None
event = MagicMock()
event.synchronize.side_effect = lambda: cache[0].fill_(7)
barrier = MagicMock()
monkeypatch.setattr(
hisparse_worker_module, "get_tensor_model_parallel_rank", lambda: rank
)
monkeypatch.setattr(
hisparse_worker_module, "get_tp_group", lambda: SimpleNamespace(barrier=barrier)
)
copies = [KVCacheBlockCopy(0, 1)] if has_copies else []
worker._copy_host_blocks(copies, event)
writes = has_copies and (not shared or rank == 0)
assert cache[1].item() == (7 if writes else 0)
assert event.synchronize.call_count == int(writes)
assert barrier.call_count == int(shared and has_copies)
assert worker._completed_host_copy_dst_ids == ([1] if has_copies else [])
def test_hisparse_worker_updates_request_state_mapping_in_place(monkeypatch):
worker = _make_hisparse_worker()
worker.request_state_indices = torch.arange(4, dtype=torch.int32)
worker._pending_invalid_block_ids = [5]
invalidations = []
worker.invalidate_blocks = lambda blocks, states: invalidations.append(
(blocks.copy(), states.clone())
)
original_ptr = worker.request_state_indices.data_ptr()
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
worker.set_request_state_indices(torch.tensor([3, 1], dtype=torch.int32))
assert worker.request_state_indices.data_ptr() == original_ptr
assert worker.request_state_indices.tolist() == [3, 1, -1, -1]
assert len(invalidations) == 1
assert invalidations[0][0] == [5]
torch.testing.assert_close(
invalidations[0][1], torch.tensor([3, 1], dtype=torch.int32)
)
assert worker._pending_invalid_block_ids == []
def test_hisparse_pre_forward_transfer_builds_page_descriptors():
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.kernel_block_size = 2
source = torch.empty((3, 2, 4), dtype=torch.uint8)
destination = torch.empty((6, 4), dtype=torch.uint8)
worker.resident_caches = (source,)
worker.host_caches = (destination,)
worker.cache_handles = [
SimpleNamespace(runtime=SimpleNamespace(resident_source_index=0))
]
worker._dma_free_descriptors = []
worker._submit_dma_descriptors = MagicMock()
worker._enqueue_transfers([SparseKVPageTransfer(7, 2, (1,), after_forward=False)])
descriptors, count = worker._submit_dma_descriptors.call_args.args
assert count == 1
assert descriptors.src[:count].tolist() == [
source.data_ptr() + source.stride(0) * source.element_size()
]
assert descriptors.dst[:count].tolist() == [destination.data_ptr() + 4 * 4]
assert descriptors.sizes[:count].tolist() == [2 * 4]
assert worker._submit_dma_descriptors.call_args.kwargs["transfer_ids"] == (7,)
def test_hisparse_eager_mirror_records_transfer_without_page_copy():
worker = _make_hisparse_worker()
worker.cache_handles = [
SimpleNamespace(runtime=SimpleNamespace(eager_host_mirror=True))
]
worker._record_transfer_completion = MagicMock()
worker._enqueue_transfers = MagicMock()
transfers = [SparseKVPageTransfer(7, 2, (1,), after_forward=False)]
worker._submit_transfers(transfers)
worker._record_transfer_completion.assert_called_once_with(transfers)
worker._enqueue_transfers.assert_not_called()
def test_hisparse_lazy_mirror_copies_transfer_pages():
worker = _make_hisparse_worker()
worker.cache_handles = [
SimpleNamespace(runtime=SimpleNamespace(eager_host_mirror=False))
]
worker._record_transfer_completion = MagicMock()
worker._enqueue_transfers = MagicMock()
transfers = [SparseKVPageTransfer(7, 2, (1,), after_forward=False)]
worker._submit_transfers(transfers)
worker._enqueue_transfers.assert_called_once_with(transfers)
worker._record_transfer_completion.assert_not_called()
def test_hisparse_dma_row_mirror_builds_descriptors(monkeypatch):
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.kernel_block_size = 2
source = torch.empty((3, 2, 4), dtype=torch.uint8)
destination = torch.empty((6, 4), dtype=torch.uint8)
worker.resident_caches = (source,)
worker.host_caches = (destination,)
worker.cache_handles = [
SimpleNamespace(
runtime=SimpleNamespace(resident_source_index=0), decode_batch=True
)
]
worker.hot_backing = SimpleNamespace(device=torch.device("cuda:0"))
worker.dma_stream = MagicMock()
worker.host_write_event = MagicMock()
worker._dma_free_descriptors = []
worker._pending_dma_descriptors = deque()
worker._pending_transfer_events = deque()
worker._enqueued_transfer_ids = []
worker._dma_submitted = False
current_stream = MagicMock()
event = MagicMock()
swap_blocks_batch = MagicMock()
monkeypatch.setattr(
hisparse_worker_module, "current_stream", lambda: current_stream
)
monkeypatch.setattr(torch.cuda, "stream", lambda stream: nullcontext())
monkeypatch.setattr(torch, "Event", lambda: event)
monkeypatch.setattr(
hisparse_worker_module.ops, "swap_blocks_batch", swap_blocks_batch
)
worker._set_row_mirrors((SparseKVRowMirror((3,), 4, 1),))
worker._enqueue_row_dma(range(1))
src_ptrs, dst_ptrs, sizes = swap_blocks_batch.call_args.args
assert src_ptrs.tolist() == [source.data_ptr() + 3 * 4]
assert dst_ptrs.tolist() == [destination.data_ptr() + 4 * 4]
assert sizes.tolist() == [4]
worker.dma_stream.wait_stream.assert_called_once_with(current_stream)
worker.host_write_event.record.assert_called_once_with(worker.dma_stream)
event.record.assert_called_once_with(worker.dma_stream)
assert worker._dma_submitted
def test_hisparse_row_dma_uses_resident_spans():
worker = _make_hisparse_worker()
source = torch.empty((2, 2, 4), dtype=torch.uint8)
destination = torch.empty((12, 4), dtype=torch.uint8)
worker.is_host_writer = True
worker.kernel_block_size = 2
worker.resident_caches = (source,)
worker.host_caches = (destination,)
worker.cache_handles = [
SimpleNamespace(
runtime=SimpleNamespace(resident_source_index=0), decode_batch=True
)
]
worker._set_row_mirrors(
(
SparseKVRowMirror((0,), 4, 2),
SparseKVRowMirror((2,), 9, 1),
)
)
worker._dma_free_descriptors = []
worker._submit_dma_descriptors = MagicMock()
worker._enqueue_row_dma(range(1))
descriptors, count = worker._submit_dma_descriptors.call_args.args
assert count == 2
assert descriptors.src[:count].tolist() == [
source.data_ptr(),
source.data_ptr() + 2 * 4,
]
assert descriptors.dst[:count].tolist() == [
destination.data_ptr() + 4 * 4,
destination.data_ptr() + 9 * 4,
]
assert descriptors.sizes[:count].tolist() == [2 * 4, 4]
def test_hisparse_finish_forward_mirrors_all_layers_once(monkeypatch):
dst_slots = torch.tensor([7, 8, 9], dtype=torch.int64)
req_ids = torch.tensor([0, 1, 2], dtype=torch.int32)
leader = SimpleNamespace(
eager_host_mirror=True,
is_group_leader=True,
invalidate_written_slots=MagicMock(),
)
follower = SimpleNamespace(
eager_host_mirror=True,
is_group_leader=False,
invalidate_written_slots=MagicMock(),
)
handles = [
SimpleNamespace(
runtime=runtime,
decode_batch=True,
host_mirror_required=True,
num_actual_tokens=3,
num_decode_tokens=2,
req_id_per_token=req_ids,
mirror_slot_mapping=dst_slots,
)
for runtime in (leader, follower)
]
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = handles
worker.hot_backing = SimpleNamespace(device=torch.device("cuda:0"))
worker._set_row_mirrors((SparseKVRowMirror((0, 0), 7, 3),))
worker._dma_submitted = False
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._post_forward_transfers = []
worker._enqueue_row_dma = MagicMock()
worker.host_write_event = MagicMock()
worker._forward_ready_event = MagicMock()
current_stream = MagicMock()
monkeypatch.setattr(
hisparse_worker_module, "current_stream", lambda: current_stream
)
worker.finish_forward()
worker._forward_ready_event.record.assert_called_once_with()
worker._enqueue_row_dma.assert_called_once_with(
(0, 1), ready_event=worker._forward_ready_event
)
leader.invalidate_written_slots.assert_called_once()
torch.testing.assert_close(
leader.invalidate_written_slots.call_args.args[0], dst_slots
)
follower.invalidate_written_slots.assert_not_called()
worker.host_write_event.record.assert_called_once_with(current_stream)
def test_hisparse_finish_forward_does_not_repeat_per_layer_mirrors():
slots = torch.tensor([7, 8], dtype=torch.int64)
runtime = SimpleNamespace(
eager_host_mirror=False,
is_group_leader=False,
invalidate_written_slots=MagicMock(),
)
handles = [
SimpleNamespace(
runtime=runtime,
decode_batch=False,
host_mirror_required=True,
num_actual_tokens=2,
num_decode_tokens=0,
req_id_per_token=torch.empty(0, dtype=torch.int32),
mirror_slot_mapping=slots,
)
for _ in range(2)
]
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = handles
worker._set_row_mirrors((SparseKVRowMirror((0, 0), 7, 2),))
worker._per_layer_mirrored = {0, 1}
worker._submitted_mirror_layers = {0, 1}
worker._enqueue_row_dma = MagicMock()
worker._enqueue_host_mirror()
worker._enqueue_row_dma.assert_not_called()
def test_hisparse_finish_forward_submits_lazy_post_forward_transfer(monkeypatch):
runtime = SimpleNamespace(eager_host_mirror=False)
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = [SimpleNamespace(runtime=runtime, num_actual_tokens=0)]
transfer = SparseKVPageTransfer(7, 2, (1,), after_forward=True)
worker._post_forward_transfers = [transfer]
worker._forward_ready_event = MagicMock()
worker._enqueue_host_mirror = MagicMock()
worker._submit_transfers = MagicMock()
worker._dma_submitted = False
worker._submitted_mirror_layers = set()
worker._finish_mirror_phase = MagicMock()
worker._release_completed_dma_descriptors = MagicMock()
stream = MagicMock()
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream)
worker.finish_forward()
worker._finish_mirror_phase.assert_called_once_with(worker._forward_ready_event)
worker._submit_transfers.assert_called_once_with([transfer])
worker.host_write_event.record.assert_called_once_with(stream)
def test_hisparse_prefill_mirrors_source_groups_and_flushes_partial_group():
slots = torch.tensor([7, 8], dtype=torch.int64)
source_indices = [0, 0, 1, 1, 1, 1, 2]
handles = [
SimpleNamespace(
runtime=SimpleNamespace(
eager_host_mirror=False,
is_group_leader=False,
resident_source_index=source_index,
),
decode_batch=False,
host_mirror_required=layer_index < 5,
num_actual_tokens=2 if layer_index < 5 else 0,
num_decode_tokens=0,
req_id_per_token=torch.empty(0, dtype=torch.int32),
mirror_slot_mapping=slots,
)
for layer_index, source_index in enumerate(source_indices)
]
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = handles
worker._set_row_mirrors((SparseKVRowMirror((0, 0, 0), 7, 2),))
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._layer_ready_events = tuple(MagicMock() for _ in handles)
worker._enqueue_row_dma = MagicMock()
for layer_index in range(5):
worker._enqueue_layer_mirror(layer_index)
worker._enqueue_row_dma.assert_called_once_with(
(0, 1), ready_event=worker._layer_ready_events[1]
)
worker._enqueue_host_mirror(ready_event=worker._layer_ready_events[-1])
assert worker._enqueue_row_dma.call_args_list[1].args == ((2, 3, 4),)
assert worker._enqueue_row_dma.call_args_list[1].kwargs == {
"ready_event": worker._layer_ready_events[-1]
}
def test_hisparse_finish_forward_rejects_partial_per_layer_mirror():
slots = torch.tensor([7, 8], dtype=torch.int64)
runtime = SimpleNamespace(
eager_host_mirror=False,
is_group_leader=False,
invalidate_written_slots=MagicMock(),
)
handles = [
SimpleNamespace(
runtime=runtime,
decode_batch=False,
host_mirror_required=True,
num_actual_tokens=2,
num_decode_tokens=0,
req_id_per_token=torch.empty(0, dtype=torch.int32),
mirror_slot_mapping=slots,
)
for _ in range(2)
]
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = handles
worker._set_row_mirrors((SparseKVRowMirror((0, 0), 7, 2),))
worker._per_layer_mirrored = {0}
worker._submitted_mirror_layers = set()
worker._enqueue_row_dma = MagicMock()
with pytest.raises(RuntimeError, match="did not mirror every active layer"):
worker._enqueue_host_mirror()
def test_hisparse_finish_forward_orders_next_forward_after_dma(monkeypatch):
current_stream = MagicMock()
worker = _make_hisparse_worker()
worker.hot_backing = SimpleNamespace(device=torch.device("cuda:0"))
worker.is_host_writer = True
worker._dma_submitted = True
worker.host_write_event = MagicMock()
worker.cache_handles = [
SimpleNamespace(
runtime=SimpleNamespace(eager_host_mirror=True), num_actual_tokens=0
)
]
worker._post_forward_transfers = []
worker._pending_dma_descriptors = deque()
worker._dma_free_descriptors = []
worker._forward_ready_event = MagicMock()
worker._finish_mirror_phase = MagicMock()
monkeypatch.setattr(
hisparse_worker_module, "current_stream", lambda: current_stream
)
worker.finish_forward()
current_stream.wait_event.assert_called_once_with(worker.host_write_event)
assert not worker._dma_submitted
def test_hisparse_finish_forward_mirrors_standalone_mtp_cache(monkeypatch):
dst_slots = torch.tensor([7, 8], dtype=torch.int64)
runtime = SimpleNamespace(
eager_host_mirror=True,
is_group_leader=False,
invalidate_written_slots=MagicMock(),
)
inactive = SimpleNamespace(
runtime=runtime,
decode_batch=False,
host_mirror_required=False,
num_actual_tokens=0,
num_decode_tokens=0,
req_id_per_token=None,
mirror_slot_mapping=dst_slots,
)
mtp = SimpleNamespace(
runtime=runtime,
decode_batch=True,
host_mirror_required=True,
num_actual_tokens=2,
num_decode_tokens=2,
req_id_per_token=torch.tensor([0, 1], dtype=torch.int32),
mirror_slot_mapping=dst_slots,
)
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = [inactive, mtp]
worker._set_row_mirrors((SparseKVRowMirror((0,), 7, 2),))
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._enqueue_row_dma = MagicMock()
worker._enqueue_host_mirror()
worker._enqueue_row_dma.assert_called_once_with((1,), ready_event=None)
def test_hisparse_shared_host_reader_skips_mirror(monkeypatch):
"""A non-writer TP rank must not mirror rows into the shared host pool."""
dst_slots = torch.tensor([7, 8], dtype=torch.int64)
leader = SimpleNamespace(
eager_host_mirror=True,
is_group_leader=True,
invalidate_written_slots=MagicMock(),
)
handle = SimpleNamespace(
runtime=leader,
decode_batch=True,
host_mirror_required=True,
num_actual_tokens=2,
num_decode_tokens=1,
req_id_per_token=torch.tensor([0], dtype=torch.int32),
mirror_slot_mapping=dst_slots,
)
worker = _make_hisparse_worker()
worker.is_host_writer = False
worker.cache_handles = [handle]
worker._set_row_mirrors((SparseKVRowMirror((0,), 7, 2),))
worker._per_layer_mirrored = set()
worker._enqueue_row_dma = MagicMock()
worker._enqueue_host_mirror()
worker._enqueue_row_dma.assert_not_called()
leader.invalidate_written_slots.assert_called_once()
def test_hisparse_shared_host_reader_skips_transfer_completion():
"""A non-writer TP rank must not acknowledge host writes."""
worker = _make_hisparse_worker()
worker.is_host_writer = False
worker.kernel_block_size = 1
worker._enqueued_transfer_ids = []
worker._pending_transfer_events = []
worker._record_transfer_completion([SparseKVPageTransfer(1, 2, (3,), True)])
assert worker._enqueued_transfer_ids == []
assert worker._pending_transfer_events == []
def test_hisparse_writer_records_transfer_completion_after_dma(monkeypatch):
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.dma_stream = MagicMock()
worker._enqueued_transfer_ids = []
worker._pending_transfer_events = deque()
event = MagicMock()
monkeypatch.setattr(torch, "Event", lambda: event)
worker._record_transfer_completion(
[
SparseKVPageTransfer(3, 2, (1,), False),
SparseKVPageTransfer(7, 4, (5,), True),
]
)
event.record.assert_called_once_with(worker.dma_stream)
assert worker._enqueued_transfer_ids == [3, 7]
assert worker._pending_transfer_events == deque([(event, (3, 7))])
@pytest.mark.parametrize("is_host_writer", [False, True])
def test_hisparse_step_waits_for_previous_host_write(monkeypatch, is_host_writer):
worker = _make_hisparse_worker()
worker.is_host_writer = is_host_writer
worker.hot_backing = SimpleNamespace(device=torch.device("cuda:1"))
host_write_events = (MagicMock(), MagicMock())
worker.host_write_events = host_write_events
worker.host_write_event = host_write_events[1]
worker._next_host_write_event = 0
worker.host_caches = ()
worker.host_num_blocks = 1
worker._post_forward_transfers = []
worker._pending_invalid_block_ids = []
worker.cache_handles = [
SimpleNamespace(
runtime=SimpleNamespace(eager_host_mirror=True),
decode_batch=False,
host_mirror_required=False,
num_actual_tokens=0,
num_decode_tokens=0,
req_id_per_token=None,
)
]
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._layer_mirror_callbacks = ()
stream = MagicMock()
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream)
worker.start_step(
SimpleNamespace(
host_block_copies=[],
command=None,
source_block_ids=[],
row_mirrors={},
all_context_pages_resident=True,
row_mirrors_from_resident=False,
),
None,
)
worker.start_step(
SimpleNamespace(
host_block_copies=[],
command=None,
source_block_ids=[],
row_mirrors={},
all_context_pages_resident=True,
row_mirrors_from_resident=False,
),
None,
)
assert stream.wait_event.call_args_list == [
call(host_write_events[1]),
call(host_write_events[0]),
]
assert worker.host_write_event is host_write_events[1]
@pytest.mark.parametrize("tp_rank, expected_copies", [(0, 1), (1, 0)])
def test_hisparse_shared_host_block_copy_has_one_writer(
monkeypatch, tp_rank, expected_copies
):
worker = _make_hisparse_worker()
worker.shared_host_region = object()
worker.host_caches = (torch.empty(4, 1),)
worker.host_num_blocks = 4
previous_event = MagicMock()
copies = (KVCacheBlockCopy(0, 1),)
copy_blocks = MagicMock()
tp_group = MagicMock()
monkeypatch.setattr(
hisparse_worker_module, "get_tensor_model_parallel_rank", lambda: tp_rank
)
monkeypatch.setattr(hisparse_worker_module, "get_tp_group", lambda: tp_group)
monkeypatch.setattr(
hisparse_worker_module, "copy_kv_cache_blocks_inplace", copy_blocks
)
worker._copy_host_blocks(copies, previous_event)
assert copy_blocks.call_count == expected_copies
tp_group.barrier.assert_called_once_with()
assert worker.take_completed_host_copies() == [1]
assert worker.take_completed_host_copies() == []
def test_hisparse_empty_step_does_not_replay_stale_host_mirror(monkeypatch):
handle = SimpleNamespace(
runtime=SimpleNamespace(eager_host_mirror=True),
decode_batch=True,
num_actual_tokens=2,
num_decode_tokens=2,
req_id_per_token=torch.tensor([0, 1]),
mirror_slot_mapping=torch.tensor([4, 5]),
)
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.hot_backing = SimpleNamespace(device=torch.device("cpu"))
worker.host_write_event = MagicMock()
worker._forward_ready_event = MagicMock()
worker.host_caches = ()
worker.host_num_blocks = 1
worker.cache_handles = [handle]
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._layer_mirror_callbacks = (MagicMock(),)
worker._post_forward_transfers = []
worker._pending_invalid_block_ids = []
worker._enqueue_host_mirror = MagicMock(wraps=worker._enqueue_host_mirror)
stream = MagicMock()
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream)
worker.start_step(
SimpleNamespace(
host_block_copies=[],
command=None,
source_block_ids=[],
row_mirrors={},
all_context_pages_resident=True,
row_mirrors_from_resident=False,
),
None,
)
worker.finish_forward()
worker._enqueue_host_mirror.assert_called_once_with(worker._forward_ready_event)
assert handle.num_actual_tokens == 0
torch.testing.assert_close(handle.mirror_slot_mapping, torch.tensor([4, 5]))
def test_hisparse_cache_handles_join_index_groups_during_construction(monkeypatch):
"""Followers must not allocate duplicate runtime state before profiling."""
config = SimpleNamespace(
scheduler_config=SimpleNamespace(
max_num_seqs=2,
max_num_batched_tokens=2,
async_scheduling=False,
),
speculative_config=None,
kv_transfer_config=None,
)
resolved = hisparse_runtime_module.ResolvedHiSparseConfig(
top_k=4,
device_buffer_size=8,
)
monkeypatch.setattr(hisparse_runtime_module, "_has_hisparse_ops", lambda: True)
monkeypatch.setattr(
hisparse_runtime_module.ResolvedHiSparseConfig,
"from_vllm_config",
classmethod(lambda cls, vllm_config, model_top_k: resolved),
)
shared_states: list[object] = []
streams: list[object] = []
def create_shared_state(_device, _max_rows, _top_k):
shared_states.append(object())
return shared_states[-1]
def create_stream(_device):
streams.append(object())
return streams[-1]
monkeypatch.setattr(
hisparse_runtime_module, "_create_shared_topk_state", create_shared_state
)
monkeypatch.setattr(hisparse_runtime_module, "_create_copy_stream", create_stream)
monkeypatch.setattr(index_group_module, "_create_side_stream", lambda _: object())
monkeypatch.setattr(index_group_module, "_create_event", lambda: object())
index_group_builder = index_group_module.SparseMLAIndexGroupBuilder(
torch.empty((2, 4), dtype=torch.int32)
)
def make_cache_handle(is_leader: bool):
index_group, _ = index_group_builder.register_layer(is_leader)
cache_handle = hisparse_runtime_module.create_hisparse_cache_handle(
config,
model_top_k=4,
is_index_group_leader=is_leader,
row_width=8,
kv_dtype=torch.float32,
index_group=index_group,
device="cpu",
)
assert cache_handle is not None
return cache_handle
first_leader = make_cache_handle(True)
first_index_group = index_group_builder.current_group
first_follower = make_cache_handle(False)
second_leader = make_cache_handle(True)
second_index_group = index_group_builder.current_group
second_follower = make_cache_handle(False)
assert first_index_group is not None
assert second_index_group is not None
assert first_follower.runtime.index_group is first_leader.runtime.index_group
assert second_follower.runtime.index_group is second_leader.runtime.index_group
assert first_leader.runtime.index_group is not second_leader.runtime.index_group
assert first_leader.runtime.index_group.copy_stream is first_index_group.side_stream
assert (
first_leader.runtime.index_group.logical_topk_ready
is first_index_group.logical_topk_ready
)
assert (
second_leader.runtime.index_group.copy_stream is second_index_group.side_stream
)
assert (
second_leader.runtime.index_group.logical_topk_ready
is second_index_group.logical_topk_ready
)
assert len(shared_states) == 2
assert streams == []
class _TestReplaySSMMixer(MambaMixer2):
@@ -30,7 +914,8 @@ def _packed_replayssm_cache(num_blocks: int) -> torch.Tensor:
return torch.full((num_blocks, 1, 1, 80), 0, dtype=torch.int8)
def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group():
@pytest.mark.parametrize("layers_only", [False, True])
def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group(layers_only):
mixers = [_TestReplaySSMMixer() for _ in range(3)]
layer_names = [f"layers.{i}.mixer" for i in range(3)]
ctx = dict(zip(layer_names, mixers))
@@ -45,7 +930,10 @@ def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group():
SimpleNamespace(layer_names=[layer_names[1]]),
]
bind_kv_cache(kv_cache, ctx, [], kv_cache_groups=kv_cache_groups)
if layers_only:
bind_kv_cache_to_layers(kv_cache, ctx, kv_cache_groups=kv_cache_groups)
else:
bind_kv_cache(kv_cache, ctx, [], kv_cache_groups=kv_cache_groups)
assert (
mixers[0]._replayssm_ring_start.data_ptr()