[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
+126 -1
View File
@@ -13,15 +13,18 @@ import pytest
import torch
from tests.v1.attention.utils import dense_kv_cache_views
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.hisparse.binding import allocate_hisparse_kv_caches
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
HiSparseResidentSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheLayout,
KVCacheTensor,
MLAAttentionSpec,
SparseCacheRole,
compute_layout_strides,
)
from vllm.v1.worker.gpu import attn_utils
@@ -36,6 +39,60 @@ from vllm.v1.worker.utils import (
)
@pytest.mark.parametrize(
("enabled", "block_size", "main_sizes", "indexer_sizes", "expected"),
[
(True, 256, [64], [64], 64),
(True, 64, [32, 64], [16, 32], 32),
(True, 64, [MultipleOf(16)], [32], 32),
(True, 64, [64], [32], None),
(False, 256, [64], [64], 256),
],
)
def test_get_kv_cache_spec_resolves_hisparse_block_size(
monkeypatch, enabled, block_size, main_sizes, indexer_sizes, expected
):
"""Resolve shared MLA geometry before planning; leave other specs alone."""
specs = {
"main": MLAAttentionSpec(
block_size=block_size, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
),
"indexer": MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
cache_role=SparseCacheRole.INDEXER,
),
"dense": FullAttentionSpec(
block_size=block_size, num_kv_heads=1, head_size=128, dtype=torch.bfloat16
),
}
layers = {}
for name, sizes in zip(specs, [main_sizes, indexer_sizes, [block_size]]):
backend = SimpleNamespace(
customize_spec=AttentionBackend.customize_spec,
get_supported_kernel_block_sizes=lambda sizes=sizes: sizes,
)
layers[name] = SimpleNamespace(
get_kv_cache_spec=lambda _, spec=specs[name]: spec,
get_attn_backend=lambda backend=backend: backend,
)
monkeypatch.setattr(attn_utils, "get_layers_from_vllm_config", lambda *_: layers)
config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=object() if enabled else None)
)
if expected is None:
with pytest.raises(ValueError, match="supported by every sparse"):
attn_utils.get_kv_cache_spec(config)
return
resolved = attn_utils.get_kv_cache_spec(config)
assert resolved["main"].block_size == resolved["indexer"].block_size == expected
assert resolved["dense"] is specs["dense"]
assert all(spec.block_size == block_size for spec in specs.values())
class _FakeMetadataBuilder:
def __init__(self, support: AttentionCGSupport):
self.support = support
@@ -390,3 +447,71 @@ def test_copy_kv_cache_blocks_with_virtual_block_splitting(
torch.testing.assert_close(
cache[dst_start + physical_idx], expected[layer_idx][physical_idx]
)
def test_allocate_hisparse_kv_caches_host_pool_and_view_less_specs():
"""Host tensors get their own backing; view-less specs keep the raw one."""
spec = FullAttentionSpec(
block_size=2, num_kv_heads=1, head_size=4, dtype=torch.float32
)
page = spec.page_size_bytes
resident_spec = HiSparseResidentSpec(block_size=2, page_size=page)
device_size = 4 * page
config = KVCacheConfig(
num_blocks=4,
hisparse_host_num_blocks=3,
kv_cache_tensors=[
KVCacheTensor(
size=3 * page,
layers=["source"],
layer_stride=3 * page,
block_stride=page,
host_resident=True,
),
KVCacheTensor(
size=device_size,
layers=["indexer"],
layer_stride=device_size,
block_stride=page,
),
KVCacheTensor(
size=device_size,
layers=["resident"],
layer_stride=device_size,
block_stride=page,
),
],
kv_cache_groups=[
KVCacheGroupSpec(["source"], spec, host_resident=True),
KVCacheGroupSpec(["indexer"], spec),
KVCacheGroupSpec(["resident"], resident_spec),
],
)
host_buffers: list[torch.Tensor] = []
def host_allocator(size: int) -> torch.Tensor:
host_buffers.append(torch.zeros(size, dtype=torch.int8))
return host_buffers[-1]
caches = allocate_hisparse_kv_caches(
config,
torch.device("cpu"),
KVCacheLayout.LBHNC,
[2, 2, 2],
SimpleNamespace(allocate=host_allocator),
)
assert len(config.kv_cache_tensors) == 3
assert [buf.numel() for buf in host_buffers] == [3 * page]
assert caches["source"].shape[0] == 3
assert (
caches["source"].untyped_storage().data_ptr()
== host_buffers[0].untyped_storage().data_ptr()
)
assert caches["indexer"].shape[0] == 4
backing = caches["resident"]
assert backing.dtype == torch.int8 and backing.numel() >= device_size
assert (
backing.untyped_storage().data_ptr()
== caches["indexer"].untyped_storage().data_ptr()
)
@@ -134,6 +134,7 @@ def test_overlaid_zeroer_dedups_segments_with_max_span():
vllm_config.cache_config = CacheConfig()
vllm_config.cache_config.num_gpu_blocks_override = None
vllm_config.cache_config.kv_cache_layout = "BLHNC"
vllm_config.attention_config.hisparse_config = None
config = get_kv_cache_config_from_groups(vllm_config, groups, 8 * 1024 * 1024)
views = allocate_kv_cache(config, torch.device("cpu"), KVCacheLayout.BLHNC, None)
buf_ptr = views["g1.big"].data_ptr()
@@ -351,6 +351,8 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
speculator = object.__new__(_TestSpeculator)
speculator.num_speculative_steps = 4
speculator.current_draft_step = torch.tensor(0)
speculator.slot_mapping_observer = None
speculator.host_mirror_forward_observer = None
speculator.input_buffers = SimpleNamespace(
positions=torch.arange(2),
query_start_loc=torch.arange(3),
+23 -3
View File
@@ -5,9 +5,11 @@ from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
import vllm.v1.worker.gpu.kv_connector as kv_connector_module
from vllm.config import KVTransferConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorTransferResults
from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector
@@ -18,9 +20,9 @@ def _make_connector(
backend = Mock()
backend.handle_preemptions.side_effect = lambda _: events.append("handle")
backend.bind_connector_metadata.side_effect = lambda _: events.append("bind")
backend.start_load_kv.side_effect = lambda _: events.append("start")
backend.start_load_kv.side_effect = lambda *_args, **_kwargs: events.append("start")
backend.wait_for_save.side_effect = lambda: events.append("wait")
backend.get_finished.side_effect = lambda _: (set(), set())
backend.get_transfer_results.return_value = KVConnectorTransferResults()
backend.get_block_ids_with_load_errors.return_value = set()
backend.get_kv_connector_stats.return_value = None
backend.get_kv_connector_kv_cache_events.return_value = None
@@ -61,7 +63,15 @@ def test_load_start_phase(
connector = _make_connector(monkeypatch, events)
output = _scheduler_output(has_sync_kv_loads)
connector.pre_forward(output) # type: ignore[arg-type]
request_indices = torch.tensor([3, 1])
request_ids = ["first", "second"]
attn_metadata = {"layer": object()}
connector.pre_forward( # type: ignore[arg-type]
output,
request_state_indices=request_indices,
request_ids=request_ids,
attn_metadata=attn_metadata,
)
assert events == (
["handle", "bind", "start"] if has_sync_kv_loads else ["handle", "bind"]
)
@@ -69,6 +79,16 @@ def test_load_start_phase(
connector.post_forward(set())
assert events == ["handle", "bind", "start", "wait", "clear"]
kwargs = connector.kv_connector.start_load_kv.call_args.kwargs
assert kwargs["request_state_indices"] is request_indices
assert kwargs["request_ids"] is request_ids
assert kwargs["attn_metadata"] is attn_metadata
# A subsequent step without a forward must not reuse the prior batch.
connector.no_forward(_scheduler_output(False)) # type: ignore[arg-type]
assert connector.kv_connector.start_load_kv.call_count == 2
assert connector.kv_connector.start_load_kv.call_args.kwargs == {}
def test_no_forward_starts_deferred_load_once(monkeypatch: pytest.MonkeyPatch):
events: list[str] = []
+4
View File
@@ -312,6 +312,10 @@ def test_select_common_block_size_uses_largest_shared_int():
assert selected_size == 64
def test_select_common_block_size_without_active_backends_uses_manager_size():
assert select_common_block_size(256, []) == 256
def test_select_common_block_size_accepts_rocm_sparse_block_size_16(monkeypatch):
monkeypatch.setattr(current_platform, "is_rocm", lambda: True)
@@ -237,6 +237,7 @@ def _make_capture_runner(captured: bool) -> GPUModelRunner:
runner.attn_groups = None
runner.kv_cache_config = None
runner.use_aux_hidden_state_outputs = False
runner.kv_connector = model_runner_module.NO_OP_KV_CONNECTOR
return runner
@@ -89,6 +89,7 @@ def _make_runner(
return SimpleNamespace(
num_speculative_steps=num_spec_steps,
adaptive_verification=None,
rejection_sampler=None,
decode_query_len=num_spec_steps + 1,
is_pooling_model=False,
is_encoder_decoder=False,
@@ -5,8 +5,10 @@ from contextlib import AbstractContextManager
from types import SimpleNamespace
from typing import Any, cast
import pytest
import torch
import vllm.v1.hisparse.binding as hisparse_binding
import vllm.v1.worker.gpu.attn_utils as attn_utils
import vllm.v1.worker.gpu_model_runner as gpu_model_runner
from vllm.v1.worker.gpu_worker import Worker
@@ -26,7 +28,8 @@ class _AllocationScope(AbstractContextManager):
self.active = False
def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch) -> None:
@pytest.mark.parametrize("hisparse", [False, True])
def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch, hisparse) -> None:
scope = _AllocationScope()
kv_caches = {"layer": torch.empty(0)}
@@ -38,23 +41,48 @@ def test_mrv2_kv_pool_only_wraps_backing_allocation(monkeypatch) -> None:
assert not scope.active
monkeypatch.setattr(attn_utils, "allocate_kv_cache", allocate)
monkeypatch.setattr(attn_utils, "bind_kv_cache", bind)
monkeypatch.setattr(attn_utils, "bind_kv_cache_to_layers", bind)
monkeypatch.setattr(attn_utils, "get_shared_kv_cache_layers", lambda config: {})
hisparse_bindings = []
if hisparse:
host_pool = SimpleNamespace()
def bind_hisparse(**kwargs):
hisparse_bindings.append(kwargs)
assert scope.active
assert kwargs["kv_caches"] is kv_caches
assert kwargs["host_pool"] is host_pool
return [
SimpleNamespace(
view=SimpleNamespace(cache=torch.empty(1, 1, 1), block_size=1),
runtime=SimpleNamespace(),
)
]
monkeypatch.setattr(hisparse_binding, "HiSparseHostPool", lambda: host_pool)
monkeypatch.setattr(hisparse_binding, "allocate_hisparse_kv_caches", allocate)
monkeypatch.setattr(hisparse_binding, "bind_hisparse_kv_caches", bind_hisparse)
config = SimpleNamespace(
attention_config=SimpleNamespace(
hisparse_config=object() if hisparse else None
),
scheduler_config=SimpleNamespace(max_num_seqs=1, max_num_batched_tokens=1),
cache_config=SimpleNamespace(get_resolved_kv_cache_layout=lambda: None),
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type="test")),
)
result = attn_utils.init_kv_cache(
[],
{},
SimpleNamespace(kv_cache_groups=[]),
torch.device("cpu"),
[],
config,
kv_cache_allocation_context=scope,
block_tables=SimpleNamespace(),
)
assert len(hisparse_bindings) == int(hisparse)
assert result is kv_caches
assert not scope.active
+7 -1
View File
@@ -35,10 +35,15 @@ def test_mixed_warmup_skipped_for_single_seq(max_num_reqs):
def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_warmup):
"""Startup must warm fixed drafts before calibration and retain its manager."""
manager = SimpleNamespace(cost_tables=None)
runner = SimpleNamespace(adaptive_verification=manager)
rejection_sampler = SimpleNamespace(enable_adaptive_verification=True)
runner = SimpleNamespace(
adaptive_verification=manager,
rejection_sampler=rejection_sampler,
)
def run_steps(model_runner, execute, sample):
assert model_runner.adaptive_verification is None
assert not model_runner.rejection_sampler.enable_adaptive_verification
if fail_warmup:
raise RuntimeError("warmup failed")
@@ -50,3 +55,4 @@ def test_kernel_warmup_restores_uncalibrated_adaptive_manager(monkeypatch, fail_
warmup.warmup_kernels(runner, _fail, _fail)
assert runner.adaptive_verification is manager
assert manager.cost_tables is None
assert rejection_sampler.enable_adaptive_verification
+891 -3
View File
@@ -1,13 +1,897 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import deque
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import MagicMock, call
import numpy as np
import pytest
import torch
import vllm.v1.attention.backends.mla.index_group as index_group_module
import vllm.v1.hisparse.runtime as hisparse_runtime_module
from vllm.config import CUDAGraphMode
from vllm.config.mamba import MambaBackendEnum, MambaConfig
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse import (
worker as hisparse_worker_module,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
_flatten_row_mirrors,
_select_written_row_mirrors,
_SlotMappingStaging,
)
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
from vllm.v1.worker.utils import bind_kv_cache
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror
from vllm.v1.worker.utils import (
bind_kv_cache,
bind_kv_cache_to_layers,
copy_kv_cache_blocks_inplace,
)
def _make_hisparse_worker() -> HiSparseConnectorWorker:
worker = object.__new__(HiSparseConnectorWorker)
worker.refines_row_mirrors = False
worker._slot_mapping_staging = None
worker._row_mirror_num_rows = 0
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._pending_dma_descriptors = deque()
worker._dma_free_descriptors = []
worker.host_write_events = (MagicMock(), MagicMock())
worker.host_write_event = worker.host_write_events[1]
worker._next_host_write_event = 0
worker.dma_stream = None
worker.shared_host_region = None
worker._completed_host_copy_dst_ids = []
worker._slot_mapping_staging = None
return worker
def test_hisparse_row_mirrors_follow_runner_request_order():
first = SparseKVRowMirror((1,), 10, 1)
second = SparseKVRowMirror((2,), 20, 1)
row_mirrors = {"first": (first,), "second": (second,)}
assert _flatten_row_mirrors(row_mirrors, ["second", "first"]) == (
second,
first,
)
def test_hisparse_gpu_slots_select_written_rows_from_async_envelope():
"""Rejected speculative rows must never become durable host KV."""
candidates = (
SparseKVRowMirror((100, 500), 1000, 4),
SparseKVRowMirror((20, 300), 2000, 4),
)
source_slots = np.array([102, 103, -1, 21, 22, 23], dtype=np.int64)
assert _select_written_row_mirrors(candidates, source_slots, 0) == (
SparseKVRowMirror((102, 502), 1002, 2),
SparseKVRowMirror((21, 301), 2001, 3),
)
def test_hisparse_written_rows_preserve_scheduler_page_boundaries():
candidates = (
SparseKVRowMirror((0,), 100, 4),
SparseKVRowMirror((4,), 104, 4),
)
assert _select_written_row_mirrors(
candidates, np.array([2, 3, 4, 5], dtype=np.int64), 0
) == (
SparseKVRowMirror((2,), 102, 2),
SparseKVRowMirror((4,), 104, 2),
)
def test_hisparse_written_rows_without_host_destinations_are_skipped():
assert (
_select_written_row_mirrors((), np.array([100, 101], dtype=np.int64), 0) == ()
)
def test_hisparse_appends_reference_slots_within_a_mirror_phase(monkeypatch):
"""Separate context and query writes must share one mirror phase."""
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.refines_row_mirrors = True
state = _SlotMappingStaging(
MagicMock(), MagicMock(), torch.empty(4, dtype=torch.int64)
)
worker._slot_mapping_staging = state
handle = SimpleNamespace(runtime=SimpleNamespace(resident_source_index=1))
worker.cache_layer_names = ["layer"]
worker.cache_handles = [handle]
worker._layer_mirror_callbacks = (MagicMock(),)
worker._row_mirror_num_rows = 1
worker._submitted_mirror_layers = set()
main_stream = MagicMock()
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: main_stream)
monkeypatch.setattr(torch.cuda, "stream", lambda stream: nullcontext())
handle.slot_mapping = torch.tensor([7, 8], dtype=torch.int64)
worker._stage_row_mirror_mapping(2)
handle.slot_mapping = torch.tensor([9, 10], dtype=torch.int64)
worker._stage_row_mirror_mapping(2)
torch.testing.assert_close(state.slots, torch.tensor([7, 8, 9, 10]))
assert state.stream.wait_stream.call_args_list == [call(main_stream)] * 2
assert state.event.record.call_args_list == [call(state.stream)] * 2
assert state.num_tokens == 4
assert state.source_index == 1
@pytest.mark.parametrize(
("mode", "expected_calls"),
[(CUDAGraphMode.PIECEWISE, 1), (CUDAGraphMode.FULL, 0)],
)
def test_hisparse_submits_layer_mirror_at_replayed_attention_boundary(
monkeypatch, mode, expected_calls
):
"""Piecewise replay must pipeline DMA; FULL replay uses the batch fallback."""
handle = object.__new__(hisparse_runtime_module.HiSparseCacheHandle)
handle.dummy_batch = False
handle.decode_batch = False
handle.submit_layer_mirror = MagicMock()
monkeypatch.setattr(
hisparse_runtime_module,
"get_forward_context",
lambda: SimpleNamespace(cudagraph_runtime_mode=mode),
)
handle.finish_kv_update()
assert handle.submit_layer_mirror.call_count == expected_calls
def test_copy_cpu_kv_cache_logical_blocks_ignores_storage_padding():
backing = torch.full((10, 2, 3), -1, dtype=torch.float32)
cache = backing[1:9]
cache[2:4] = 7
cache[6:8] = 11
copy_kv_cache_blocks_inplace(
[cache],
num_blocks=4,
kv_cache_block_copies=[
KVCacheBlockCopy(1, 0),
KVCacheBlockCopy(3, 2),
],
)
torch.testing.assert_close(cache[0:2], torch.full_like(cache[0:2], 7))
torch.testing.assert_close(cache[4:6], torch.full_like(cache[4:6], 11))
assert (backing[0] == -1).all()
assert (backing[9] == -1).all()
@pytest.mark.parametrize("shared,rank", [(False, 0), (True, 0), (True, 1)])
@pytest.mark.parametrize("has_copies", [False, True])
def test_hisparse_host_copy_waits_for_writes_on_writer_only(
monkeypatch, shared, rank, has_copies
):
"""Host CoW must read completed writes; shared non-writers only barrier."""
worker = _make_hisparse_worker()
cache = torch.zeros((2, 1, 1))
worker.host_caches = [cache]
worker.host_num_blocks = 2
worker.shared_host_region = object() if shared else None
event = MagicMock()
event.synchronize.side_effect = lambda: cache[0].fill_(7)
barrier = MagicMock()
monkeypatch.setattr(
hisparse_worker_module, "get_tensor_model_parallel_rank", lambda: rank
)
monkeypatch.setattr(
hisparse_worker_module, "get_tp_group", lambda: SimpleNamespace(barrier=barrier)
)
copies = [KVCacheBlockCopy(0, 1)] if has_copies else []
worker._copy_host_blocks(copies, event)
writes = has_copies and (not shared or rank == 0)
assert cache[1].item() == (7 if writes else 0)
assert event.synchronize.call_count == int(writes)
assert barrier.call_count == int(shared and has_copies)
assert worker._completed_host_copy_dst_ids == ([1] if has_copies else [])
def test_hisparse_worker_updates_request_state_mapping_in_place(monkeypatch):
worker = _make_hisparse_worker()
worker.request_state_indices = torch.arange(4, dtype=torch.int32)
worker._pending_invalid_block_ids = [5]
invalidations = []
worker.invalidate_blocks = lambda blocks, states: invalidations.append(
(blocks.copy(), states.clone())
)
original_ptr = worker.request_state_indices.data_ptr()
monkeypatch.setattr(torch.cuda, "is_current_stream_capturing", lambda: False)
worker.set_request_state_indices(torch.tensor([3, 1], dtype=torch.int32))
assert worker.request_state_indices.data_ptr() == original_ptr
assert worker.request_state_indices.tolist() == [3, 1, -1, -1]
assert len(invalidations) == 1
assert invalidations[0][0] == [5]
torch.testing.assert_close(
invalidations[0][1], torch.tensor([3, 1], dtype=torch.int32)
)
assert worker._pending_invalid_block_ids == []
def test_hisparse_pre_forward_transfer_builds_page_descriptors():
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.kernel_block_size = 2
source = torch.empty((3, 2, 4), dtype=torch.uint8)
destination = torch.empty((6, 4), dtype=torch.uint8)
worker.resident_caches = (source,)
worker.host_caches = (destination,)
worker.cache_handles = [
SimpleNamespace(runtime=SimpleNamespace(resident_source_index=0))
]
worker._dma_free_descriptors = []
worker._submit_dma_descriptors = MagicMock()
worker._enqueue_transfers([SparseKVPageTransfer(7, 2, (1,), after_forward=False)])
descriptors, count = worker._submit_dma_descriptors.call_args.args
assert count == 1
assert descriptors.src[:count].tolist() == [
source.data_ptr() + source.stride(0) * source.element_size()
]
assert descriptors.dst[:count].tolist() == [destination.data_ptr() + 4 * 4]
assert descriptors.sizes[:count].tolist() == [2 * 4]
assert worker._submit_dma_descriptors.call_args.kwargs["transfer_ids"] == (7,)
def test_hisparse_eager_mirror_records_transfer_without_page_copy():
worker = _make_hisparse_worker()
worker.cache_handles = [
SimpleNamespace(runtime=SimpleNamespace(eager_host_mirror=True))
]
worker._record_transfer_completion = MagicMock()
worker._enqueue_transfers = MagicMock()
transfers = [SparseKVPageTransfer(7, 2, (1,), after_forward=False)]
worker._submit_transfers(transfers)
worker._record_transfer_completion.assert_called_once_with(transfers)
worker._enqueue_transfers.assert_not_called()
def test_hisparse_lazy_mirror_copies_transfer_pages():
worker = _make_hisparse_worker()
worker.cache_handles = [
SimpleNamespace(runtime=SimpleNamespace(eager_host_mirror=False))
]
worker._record_transfer_completion = MagicMock()
worker._enqueue_transfers = MagicMock()
transfers = [SparseKVPageTransfer(7, 2, (1,), after_forward=False)]
worker._submit_transfers(transfers)
worker._enqueue_transfers.assert_called_once_with(transfers)
worker._record_transfer_completion.assert_not_called()
def test_hisparse_dma_row_mirror_builds_descriptors(monkeypatch):
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.kernel_block_size = 2
source = torch.empty((3, 2, 4), dtype=torch.uint8)
destination = torch.empty((6, 4), dtype=torch.uint8)
worker.resident_caches = (source,)
worker.host_caches = (destination,)
worker.cache_handles = [
SimpleNamespace(
runtime=SimpleNamespace(resident_source_index=0), decode_batch=True
)
]
worker.hot_backing = SimpleNamespace(device=torch.device("cuda:0"))
worker.dma_stream = MagicMock()
worker.host_write_event = MagicMock()
worker._dma_free_descriptors = []
worker._pending_dma_descriptors = deque()
worker._pending_transfer_events = deque()
worker._enqueued_transfer_ids = []
worker._dma_submitted = False
current_stream = MagicMock()
event = MagicMock()
swap_blocks_batch = MagicMock()
monkeypatch.setattr(
hisparse_worker_module, "current_stream", lambda: current_stream
)
monkeypatch.setattr(torch.cuda, "stream", lambda stream: nullcontext())
monkeypatch.setattr(torch, "Event", lambda: event)
monkeypatch.setattr(
hisparse_worker_module.ops, "swap_blocks_batch", swap_blocks_batch
)
worker._set_row_mirrors((SparseKVRowMirror((3,), 4, 1),))
worker._enqueue_row_dma(range(1))
src_ptrs, dst_ptrs, sizes = swap_blocks_batch.call_args.args
assert src_ptrs.tolist() == [source.data_ptr() + 3 * 4]
assert dst_ptrs.tolist() == [destination.data_ptr() + 4 * 4]
assert sizes.tolist() == [4]
worker.dma_stream.wait_stream.assert_called_once_with(current_stream)
worker.host_write_event.record.assert_called_once_with(worker.dma_stream)
event.record.assert_called_once_with(worker.dma_stream)
assert worker._dma_submitted
def test_hisparse_row_dma_uses_resident_spans():
worker = _make_hisparse_worker()
source = torch.empty((2, 2, 4), dtype=torch.uint8)
destination = torch.empty((12, 4), dtype=torch.uint8)
worker.is_host_writer = True
worker.kernel_block_size = 2
worker.resident_caches = (source,)
worker.host_caches = (destination,)
worker.cache_handles = [
SimpleNamespace(
runtime=SimpleNamespace(resident_source_index=0), decode_batch=True
)
]
worker._set_row_mirrors(
(
SparseKVRowMirror((0,), 4, 2),
SparseKVRowMirror((2,), 9, 1),
)
)
worker._dma_free_descriptors = []
worker._submit_dma_descriptors = MagicMock()
worker._enqueue_row_dma(range(1))
descriptors, count = worker._submit_dma_descriptors.call_args.args
assert count == 2
assert descriptors.src[:count].tolist() == [
source.data_ptr(),
source.data_ptr() + 2 * 4,
]
assert descriptors.dst[:count].tolist() == [
destination.data_ptr() + 4 * 4,
destination.data_ptr() + 9 * 4,
]
assert descriptors.sizes[:count].tolist() == [2 * 4, 4]
def test_hisparse_finish_forward_mirrors_all_layers_once(monkeypatch):
dst_slots = torch.tensor([7, 8, 9], dtype=torch.int64)
req_ids = torch.tensor([0, 1, 2], dtype=torch.int32)
leader = SimpleNamespace(
eager_host_mirror=True,
is_group_leader=True,
invalidate_written_slots=MagicMock(),
)
follower = SimpleNamespace(
eager_host_mirror=True,
is_group_leader=False,
invalidate_written_slots=MagicMock(),
)
handles = [
SimpleNamespace(
runtime=runtime,
decode_batch=True,
host_mirror_required=True,
num_actual_tokens=3,
num_decode_tokens=2,
req_id_per_token=req_ids,
mirror_slot_mapping=dst_slots,
)
for runtime in (leader, follower)
]
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = handles
worker.hot_backing = SimpleNamespace(device=torch.device("cuda:0"))
worker._set_row_mirrors((SparseKVRowMirror((0, 0), 7, 3),))
worker._dma_submitted = False
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._post_forward_transfers = []
worker._enqueue_row_dma = MagicMock()
worker.host_write_event = MagicMock()
worker._forward_ready_event = MagicMock()
current_stream = MagicMock()
monkeypatch.setattr(
hisparse_worker_module, "current_stream", lambda: current_stream
)
worker.finish_forward()
worker._forward_ready_event.record.assert_called_once_with()
worker._enqueue_row_dma.assert_called_once_with(
(0, 1), ready_event=worker._forward_ready_event
)
leader.invalidate_written_slots.assert_called_once()
torch.testing.assert_close(
leader.invalidate_written_slots.call_args.args[0], dst_slots
)
follower.invalidate_written_slots.assert_not_called()
worker.host_write_event.record.assert_called_once_with(current_stream)
def test_hisparse_finish_forward_does_not_repeat_per_layer_mirrors():
slots = torch.tensor([7, 8], dtype=torch.int64)
runtime = SimpleNamespace(
eager_host_mirror=False,
is_group_leader=False,
invalidate_written_slots=MagicMock(),
)
handles = [
SimpleNamespace(
runtime=runtime,
decode_batch=False,
host_mirror_required=True,
num_actual_tokens=2,
num_decode_tokens=0,
req_id_per_token=torch.empty(0, dtype=torch.int32),
mirror_slot_mapping=slots,
)
for _ in range(2)
]
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = handles
worker._set_row_mirrors((SparseKVRowMirror((0, 0), 7, 2),))
worker._per_layer_mirrored = {0, 1}
worker._submitted_mirror_layers = {0, 1}
worker._enqueue_row_dma = MagicMock()
worker._enqueue_host_mirror()
worker._enqueue_row_dma.assert_not_called()
def test_hisparse_finish_forward_submits_lazy_post_forward_transfer(monkeypatch):
runtime = SimpleNamespace(eager_host_mirror=False)
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = [SimpleNamespace(runtime=runtime, num_actual_tokens=0)]
transfer = SparseKVPageTransfer(7, 2, (1,), after_forward=True)
worker._post_forward_transfers = [transfer]
worker._forward_ready_event = MagicMock()
worker._enqueue_host_mirror = MagicMock()
worker._submit_transfers = MagicMock()
worker._dma_submitted = False
worker._submitted_mirror_layers = set()
worker._finish_mirror_phase = MagicMock()
worker._release_completed_dma_descriptors = MagicMock()
stream = MagicMock()
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream)
worker.finish_forward()
worker._finish_mirror_phase.assert_called_once_with(worker._forward_ready_event)
worker._submit_transfers.assert_called_once_with([transfer])
worker.host_write_event.record.assert_called_once_with(stream)
def test_hisparse_prefill_mirrors_source_groups_and_flushes_partial_group():
slots = torch.tensor([7, 8], dtype=torch.int64)
source_indices = [0, 0, 1, 1, 1, 1, 2]
handles = [
SimpleNamespace(
runtime=SimpleNamespace(
eager_host_mirror=False,
is_group_leader=False,
resident_source_index=source_index,
),
decode_batch=False,
host_mirror_required=layer_index < 5,
num_actual_tokens=2 if layer_index < 5 else 0,
num_decode_tokens=0,
req_id_per_token=torch.empty(0, dtype=torch.int32),
mirror_slot_mapping=slots,
)
for layer_index, source_index in enumerate(source_indices)
]
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = handles
worker._set_row_mirrors((SparseKVRowMirror((0, 0, 0), 7, 2),))
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._layer_ready_events = tuple(MagicMock() for _ in handles)
worker._enqueue_row_dma = MagicMock()
for layer_index in range(5):
worker._enqueue_layer_mirror(layer_index)
worker._enqueue_row_dma.assert_called_once_with(
(0, 1), ready_event=worker._layer_ready_events[1]
)
worker._enqueue_host_mirror(ready_event=worker._layer_ready_events[-1])
assert worker._enqueue_row_dma.call_args_list[1].args == ((2, 3, 4),)
assert worker._enqueue_row_dma.call_args_list[1].kwargs == {
"ready_event": worker._layer_ready_events[-1]
}
def test_hisparse_finish_forward_rejects_partial_per_layer_mirror():
slots = torch.tensor([7, 8], dtype=torch.int64)
runtime = SimpleNamespace(
eager_host_mirror=False,
is_group_leader=False,
invalidate_written_slots=MagicMock(),
)
handles = [
SimpleNamespace(
runtime=runtime,
decode_batch=False,
host_mirror_required=True,
num_actual_tokens=2,
num_decode_tokens=0,
req_id_per_token=torch.empty(0, dtype=torch.int32),
mirror_slot_mapping=slots,
)
for _ in range(2)
]
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = handles
worker._set_row_mirrors((SparseKVRowMirror((0, 0), 7, 2),))
worker._per_layer_mirrored = {0}
worker._submitted_mirror_layers = set()
worker._enqueue_row_dma = MagicMock()
with pytest.raises(RuntimeError, match="did not mirror every active layer"):
worker._enqueue_host_mirror()
def test_hisparse_finish_forward_orders_next_forward_after_dma(monkeypatch):
current_stream = MagicMock()
worker = _make_hisparse_worker()
worker.hot_backing = SimpleNamespace(device=torch.device("cuda:0"))
worker.is_host_writer = True
worker._dma_submitted = True
worker.host_write_event = MagicMock()
worker.cache_handles = [
SimpleNamespace(
runtime=SimpleNamespace(eager_host_mirror=True), num_actual_tokens=0
)
]
worker._post_forward_transfers = []
worker._pending_dma_descriptors = deque()
worker._dma_free_descriptors = []
worker._forward_ready_event = MagicMock()
worker._finish_mirror_phase = MagicMock()
monkeypatch.setattr(
hisparse_worker_module, "current_stream", lambda: current_stream
)
worker.finish_forward()
current_stream.wait_event.assert_called_once_with(worker.host_write_event)
assert not worker._dma_submitted
def test_hisparse_finish_forward_mirrors_standalone_mtp_cache(monkeypatch):
dst_slots = torch.tensor([7, 8], dtype=torch.int64)
runtime = SimpleNamespace(
eager_host_mirror=True,
is_group_leader=False,
invalidate_written_slots=MagicMock(),
)
inactive = SimpleNamespace(
runtime=runtime,
decode_batch=False,
host_mirror_required=False,
num_actual_tokens=0,
num_decode_tokens=0,
req_id_per_token=None,
mirror_slot_mapping=dst_slots,
)
mtp = SimpleNamespace(
runtime=runtime,
decode_batch=True,
host_mirror_required=True,
num_actual_tokens=2,
num_decode_tokens=2,
req_id_per_token=torch.tensor([0, 1], dtype=torch.int32),
mirror_slot_mapping=dst_slots,
)
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = [inactive, mtp]
worker._set_row_mirrors((SparseKVRowMirror((0,), 7, 2),))
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._enqueue_row_dma = MagicMock()
worker._enqueue_host_mirror()
worker._enqueue_row_dma.assert_called_once_with((1,), ready_event=None)
def test_hisparse_shared_host_reader_skips_mirror(monkeypatch):
"""A non-writer TP rank must not mirror rows into the shared host pool."""
dst_slots = torch.tensor([7, 8], dtype=torch.int64)
leader = SimpleNamespace(
eager_host_mirror=True,
is_group_leader=True,
invalidate_written_slots=MagicMock(),
)
handle = SimpleNamespace(
runtime=leader,
decode_batch=True,
host_mirror_required=True,
num_actual_tokens=2,
num_decode_tokens=1,
req_id_per_token=torch.tensor([0], dtype=torch.int32),
mirror_slot_mapping=dst_slots,
)
worker = _make_hisparse_worker()
worker.is_host_writer = False
worker.cache_handles = [handle]
worker._set_row_mirrors((SparseKVRowMirror((0,), 7, 2),))
worker._per_layer_mirrored = set()
worker._enqueue_row_dma = MagicMock()
worker._enqueue_host_mirror()
worker._enqueue_row_dma.assert_not_called()
leader.invalidate_written_slots.assert_called_once()
def test_hisparse_shared_host_reader_skips_transfer_completion():
"""A non-writer TP rank must not acknowledge host writes."""
worker = _make_hisparse_worker()
worker.is_host_writer = False
worker.kernel_block_size = 1
worker._enqueued_transfer_ids = []
worker._pending_transfer_events = []
worker._record_transfer_completion([SparseKVPageTransfer(1, 2, (3,), True)])
assert worker._enqueued_transfer_ids == []
assert worker._pending_transfer_events == []
def test_hisparse_writer_records_transfer_completion_after_dma(monkeypatch):
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.dma_stream = MagicMock()
worker._enqueued_transfer_ids = []
worker._pending_transfer_events = deque()
event = MagicMock()
monkeypatch.setattr(torch, "Event", lambda: event)
worker._record_transfer_completion(
[
SparseKVPageTransfer(3, 2, (1,), False),
SparseKVPageTransfer(7, 4, (5,), True),
]
)
event.record.assert_called_once_with(worker.dma_stream)
assert worker._enqueued_transfer_ids == [3, 7]
assert worker._pending_transfer_events == deque([(event, (3, 7))])
@pytest.mark.parametrize("is_host_writer", [False, True])
def test_hisparse_step_waits_for_previous_host_write(monkeypatch, is_host_writer):
worker = _make_hisparse_worker()
worker.is_host_writer = is_host_writer
worker.hot_backing = SimpleNamespace(device=torch.device("cuda:1"))
host_write_events = (MagicMock(), MagicMock())
worker.host_write_events = host_write_events
worker.host_write_event = host_write_events[1]
worker._next_host_write_event = 0
worker.host_caches = ()
worker.host_num_blocks = 1
worker._post_forward_transfers = []
worker._pending_invalid_block_ids = []
worker.cache_handles = [
SimpleNamespace(
runtime=SimpleNamespace(eager_host_mirror=True),
decode_batch=False,
host_mirror_required=False,
num_actual_tokens=0,
num_decode_tokens=0,
req_id_per_token=None,
)
]
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._layer_mirror_callbacks = ()
stream = MagicMock()
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream)
worker.start_step(
SimpleNamespace(
host_block_copies=[],
command=None,
source_block_ids=[],
row_mirrors={},
all_context_pages_resident=True,
row_mirrors_from_resident=False,
),
None,
)
worker.start_step(
SimpleNamespace(
host_block_copies=[],
command=None,
source_block_ids=[],
row_mirrors={},
all_context_pages_resident=True,
row_mirrors_from_resident=False,
),
None,
)
assert stream.wait_event.call_args_list == [
call(host_write_events[1]),
call(host_write_events[0]),
]
assert worker.host_write_event is host_write_events[1]
@pytest.mark.parametrize("tp_rank, expected_copies", [(0, 1), (1, 0)])
def test_hisparse_shared_host_block_copy_has_one_writer(
monkeypatch, tp_rank, expected_copies
):
worker = _make_hisparse_worker()
worker.shared_host_region = object()
worker.host_caches = (torch.empty(4, 1),)
worker.host_num_blocks = 4
previous_event = MagicMock()
copies = (KVCacheBlockCopy(0, 1),)
copy_blocks = MagicMock()
tp_group = MagicMock()
monkeypatch.setattr(
hisparse_worker_module, "get_tensor_model_parallel_rank", lambda: tp_rank
)
monkeypatch.setattr(hisparse_worker_module, "get_tp_group", lambda: tp_group)
monkeypatch.setattr(
hisparse_worker_module, "copy_kv_cache_blocks_inplace", copy_blocks
)
worker._copy_host_blocks(copies, previous_event)
assert copy_blocks.call_count == expected_copies
tp_group.barrier.assert_called_once_with()
assert worker.take_completed_host_copies() == [1]
assert worker.take_completed_host_copies() == []
def test_hisparse_empty_step_does_not_replay_stale_host_mirror(monkeypatch):
handle = SimpleNamespace(
runtime=SimpleNamespace(eager_host_mirror=True),
decode_batch=True,
num_actual_tokens=2,
num_decode_tokens=2,
req_id_per_token=torch.tensor([0, 1]),
mirror_slot_mapping=torch.tensor([4, 5]),
)
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.hot_backing = SimpleNamespace(device=torch.device("cpu"))
worker.host_write_event = MagicMock()
worker._forward_ready_event = MagicMock()
worker.host_caches = ()
worker.host_num_blocks = 1
worker.cache_handles = [handle]
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._layer_mirror_callbacks = (MagicMock(),)
worker._post_forward_transfers = []
worker._pending_invalid_block_ids = []
worker._enqueue_host_mirror = MagicMock(wraps=worker._enqueue_host_mirror)
stream = MagicMock()
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream)
worker.start_step(
SimpleNamespace(
host_block_copies=[],
command=None,
source_block_ids=[],
row_mirrors={},
all_context_pages_resident=True,
row_mirrors_from_resident=False,
),
None,
)
worker.finish_forward()
worker._enqueue_host_mirror.assert_called_once_with(worker._forward_ready_event)
assert handle.num_actual_tokens == 0
torch.testing.assert_close(handle.mirror_slot_mapping, torch.tensor([4, 5]))
def test_hisparse_cache_handles_join_index_groups_during_construction(monkeypatch):
"""Followers must not allocate duplicate runtime state before profiling."""
config = SimpleNamespace(
scheduler_config=SimpleNamespace(
max_num_seqs=2,
max_num_batched_tokens=2,
async_scheduling=False,
),
speculative_config=None,
kv_transfer_config=None,
)
resolved = hisparse_runtime_module.ResolvedHiSparseConfig(
top_k=4,
device_buffer_size=8,
)
monkeypatch.setattr(hisparse_runtime_module, "_has_hisparse_ops", lambda: True)
monkeypatch.setattr(
hisparse_runtime_module.ResolvedHiSparseConfig,
"from_vllm_config",
classmethod(lambda cls, vllm_config, model_top_k: resolved),
)
shared_states: list[object] = []
streams: list[object] = []
def create_shared_state(_device, _max_rows, _top_k):
shared_states.append(object())
return shared_states[-1]
def create_stream(_device):
streams.append(object())
return streams[-1]
monkeypatch.setattr(
hisparse_runtime_module, "_create_shared_topk_state", create_shared_state
)
monkeypatch.setattr(hisparse_runtime_module, "_create_copy_stream", create_stream)
monkeypatch.setattr(index_group_module, "_create_side_stream", lambda _: object())
monkeypatch.setattr(index_group_module, "_create_event", lambda: object())
index_group_builder = index_group_module.SparseMLAIndexGroupBuilder(
torch.empty((2, 4), dtype=torch.int32)
)
def make_cache_handle(is_leader: bool):
index_group, _ = index_group_builder.register_layer(is_leader)
cache_handle = hisparse_runtime_module.create_hisparse_cache_handle(
config,
model_top_k=4,
is_index_group_leader=is_leader,
row_width=8,
kv_dtype=torch.float32,
index_group=index_group,
device="cpu",
)
assert cache_handle is not None
return cache_handle
first_leader = make_cache_handle(True)
first_index_group = index_group_builder.current_group
first_follower = make_cache_handle(False)
second_leader = make_cache_handle(True)
second_index_group = index_group_builder.current_group
second_follower = make_cache_handle(False)
assert first_index_group is not None
assert second_index_group is not None
assert first_follower.runtime.index_group is first_leader.runtime.index_group
assert second_follower.runtime.index_group is second_leader.runtime.index_group
assert first_leader.runtime.index_group is not second_leader.runtime.index_group
assert first_leader.runtime.index_group.copy_stream is first_index_group.side_stream
assert (
first_leader.runtime.index_group.logical_topk_ready
is first_index_group.logical_topk_ready
)
assert (
second_leader.runtime.index_group.copy_stream is second_index_group.side_stream
)
assert (
second_leader.runtime.index_group.logical_topk_ready
is second_index_group.logical_topk_ready
)
assert len(shared_states) == 2
assert streams == []
class _TestReplaySSMMixer(MambaMixer2):
@@ -30,7 +914,8 @@ def _packed_replayssm_cache(num_blocks: int) -> torch.Tensor:
return torch.full((num_blocks, 1, 1, 80), 0, dtype=torch.int8)
def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group():
@pytest.mark.parametrize("layers_only", [False, True])
def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group(layers_only):
mixers = [_TestReplaySSMMixer() for _ in range(3)]
layer_names = [f"layers.{i}.mixer" for i in range(3)]
ctx = dict(zip(layer_names, mixers))
@@ -45,7 +930,10 @@ def test_bind_kv_cache_shares_replayssm_trackers_by_cache_group():
SimpleNamespace(layer_names=[layer_names[1]]),
]
bind_kv_cache(kv_cache, ctx, [], kv_cache_groups=kv_cache_groups)
if layers_only:
bind_kv_cache_to_layers(kv_cache, ctx, kv_cache_groups=kv_cache_groups)
else:
bind_kv_cache(kv_cache, ctx, [], kv_cache_groups=kv_cache_groups)
assert (
mixers[0]._replayssm_ring_start.data_ptr()