[Bugfix][HiSparse] Fix MTP acceptance collapse under FULL graphs with a saturated GPU pool (#59309)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-10-01 23:04:21 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Matthew Bonanni
parent 0cbac6cd13
commit 4056c8ac1f
11 changed files with 426 additions and 122 deletions
+102 -35
View File
@@ -1943,6 +1943,10 @@ def _make_hisparse_cache_handle(
return HiSparseCacheHandle(runtime)
def _valid_rows(num_rows: int) -> torch.Tensor:
return torch.tensor([num_rows], dtype=torch.int32, device=DEVICE_TYPE)
@requires_hisparse_ops
def test_hisparse_uses_graph_stable_request_state_mapping():
device = torch.device(DEVICE_TYPE)
@@ -1969,6 +1973,7 @@ def test_hisparse_uses_graph_stable_request_state_mapping():
block_table=torch.tensor([[1]], dtype=torch.int32, device=device),
logical_topk_indices=torch.tensor([[0]], dtype=torch.int32, device=device),
block_size=block_size,
num_valid_rows=_valid_rows(1),
)
torch.accelerator.synchronize()
@@ -2011,6 +2016,7 @@ def test_hisparse_maps_speculative_rows_through_request_state():
block_table=torch.zeros((1, 1), dtype=torch.int32, device=device),
logical_topk_indices=torch.zeros((4, 1), dtype=torch.int32, device=device),
block_size=block_size,
num_valid_rows=_valid_rows(4),
)
torch.accelerator.synchronize()
@@ -2072,6 +2078,7 @@ def test_hisparse_speculative_rows_resolve_host_misses_consistently(
block_table=block_table,
logical_topk_indices=topk.clone(),
block_size=block_size,
num_valid_rows=_valid_rows(num_rows),
)
torch.accelerator.synchronize()
@@ -2115,12 +2122,99 @@ def test_hisparse_speculative_rows_count_a_shared_load_once():
block_table=block_table,
logical_topk_indices=topk,
block_size=block_size,
num_valid_rows=_valid_rows(2),
)
torch.accelerator.synchronize()
assert runtime.index_group.swap_stats.cpu().tolist() == [top_k, top_k]
@requires_hisparse_ops
def test_hisparse_resolver_reads_current_request_ids():
"""The resolver must see this call's request ids, not the previous call's.
It runs on the copy stream after waiting only on ``logical_topk_ready``,
which the indexer records before attention writes the ids. An MTP drafter
alternates four rows per request with one, so stale ids resolve rows
against other requests' KV. Stalling the compute stream exposes the race.
"""
device = torch.device(DEVICE_TYPE)
block_size, row_width, top_k, blocks_per_req, num_reqs = 64, 16, 64, 4, 2
kv_pool = torch.randn(
(num_reqs * blocks_per_req + 1, block_size, row_width), dtype=torch.float32
).pin_memory()
runtime = _make_hisparse_runtime(
top_k=top_k,
device_buffer_size=4 * top_k,
max_num_reqs=num_reqs,
row_width=row_width,
block_size=block_size,
max_swap_rows=4 * num_reqs,
)
runtime.bind_source_cache(kv_pool)
ready = torch.Event()
runtime.index_group.logical_topk_ready = ready
cache = HiSparseCacheHandle(runtime)
cache.all_context_pages_resident = False
cache.source_block_table = torch.arange(
1, num_reqs * blocks_per_req + 1, dtype=torch.int32, device=device
).view(num_reqs, blocks_per_req)
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.caches = [cache]
index_group.physical_topk_indices = torch.empty(
(4 * num_reqs + 1, top_k), dtype=torch.int32, device=device
)
# Four rows per request, then one per request plus a CUDA-graph padding
# row, which maps to request 0 and must be skipped.
for rows_per_req, num_padding in ((4, 0), (1, 1)):
num_rows = num_reqs * rows_per_req
req_ids = torch.arange(num_reqs, dtype=torch.int32, device=device)
req_ids = torch.cat(
[
req_ids.repeat_interleave(rows_per_req),
torch.zeros(num_padding, dtype=torch.int32, device=device),
]
)
topk = torch.stack(
[
torch.randperm(blocks_per_req * block_size, device=device)[:top_k]
for _ in range(req_ids.numel())
]
).to(torch.int32)
query_start_loc = torch.arange(
0, num_rows + 1, rows_per_req, dtype=torch.int32, device=device
)
attn_metadata = SimpleNamespace(
req_id_per_token=req_ids,
query_start_loc=torch.cat([query_start_loc, query_start_loc[-1:]]),
block_size=block_size,
)
runtime.begin_forward()
ready.record(current_stream())
torch.cuda._sleep(200_000_000)
hot_indices = index_group.convert_logical_to_physical_topk(
0, topk, attn_metadata, block_stride_rows=None, return_valid_counts=False
)
torch.accelerator.synchronize()
assert (hot_indices[num_rows:] == -1).all()
global_ref = _triton_convert_reference_impl(
req_ids[:num_rows],
cache.source_block_table,
topk[:num_rows],
block_size,
top_k,
)
gathered = runtime.hot.attention_cache.reshape(-1, row_width)[
hot_indices[:num_rows].to(torch.long)
].cpu()
torch.testing.assert_close(
gathered, kv_pool.reshape(-1, row_width)[global_ref.cpu().to(torch.long)]
)
@requires_hisparse_ops
def test_hisparse_union_table_scales_with_union_bound():
"""Shared memory follows the per-request union bound, not the hot buffer.
@@ -2157,6 +2251,7 @@ def test_hisparse_union_table_scales_with_union_bound():
block_table=block_table,
logical_topk_indices=topk,
block_size=block_size,
num_valid_rows=_valid_rows(req_ids.numel()),
)
torch.accelerator.synchronize()
@@ -2217,6 +2312,7 @@ def test_hisparse_resident_rows_bypass_hot_lru():
block_table=source_table,
logical_topk_indices=topk,
block_size=block_size,
num_valid_rows=_valid_rows(request_ids.numel()),
return_valid_counts=True,
)
torch.accelerator.synchronize()
@@ -2280,6 +2376,7 @@ def test_hisparse_swap_in_preserves_rows_across_eviction():
block_table=block_table,
logical_topk_indices=topk.clone(),
block_size=block_size,
num_valid_rows=_valid_rows(req_ids.numel()),
return_valid_counts=True,
)
torch.accelerator.synchronize()
@@ -2361,6 +2458,7 @@ def test_hisparse_multi_step_swaps_match_independent():
req_id_per_token=req_ids,
block_table=block_table,
block_size=block_size,
num_valid_rows=_valid_rows(req_ids.numel()),
)
producer_indices = [
producer.swap_in(logical_topk_indices=topk.clone(), **kw) for topk in topks
@@ -2435,6 +2533,7 @@ def test_hisparse_multi_step_writes_request_major_output():
block_table,
logical[:, step],
block_size=block_size,
num_valid_rows=_valid_rows(request_ids.numel()),
return_valid_counts=True,
attention_indices_out=physical[:, step],
valid_counts_out=valid_counts[:, step],
@@ -2619,6 +2718,7 @@ def test_hisparse_remaps_strided_hma_rows_for_attention():
[[0, 1, 2, 3]], dtype=torch.int32, device=device
),
block_size=block_size,
num_valid_rows=_valid_rows(1),
)
assert runtime.hot.attention_cache is not None
@@ -2752,6 +2852,7 @@ def test_hisparse_newest_write_and_recycled_slot_invalidation():
block_table=block_table,
logical_topk_indices=topk,
block_size=block_size,
num_valid_rows=_valid_rows(req_ids.numel()),
)
torch.accelerator.synchronize()
stale_hot_slot = hot_indices[0, 0].item()
@@ -2767,6 +2868,7 @@ def test_hisparse_newest_write_and_recycled_slot_invalidation():
block_table=block_table,
logical_topk_indices=topk,
block_size=block_size,
num_valid_rows=_valid_rows(req_ids.numel()),
)
torch.accelerator.synchronize()
idx = hot_indices.cpu().tolist()[0][0]
@@ -3200,8 +3302,6 @@ def test_hisparse_fp8_decode_resolves_rows_once_then_runs_batched_attention():
index_group.physical_topk_indices = torch.empty(
(num_tokens + 1, 4), dtype=torch.int32, device=device
)
index_group.row_indices = torch.arange(num_tokens, dtype=torch.int32, device=device)
index_group.request_ids = torch.empty_like(index_group.row_indices)
impl = SimpleNamespace(
kv_lora_rank=1,
index_group=index_group,
@@ -3255,39 +3355,6 @@ def test_hisparse_keeps_speculative_rows_as_decodes():
assert builder.reorder_batch_threshold == 4
def test_hisparse_decode_skips_padding_rows():
"""Padding rows are masked, and the ids stay in the group's persistent buffer.
The resolver reads them on the HiSparse copy stream, where a compute-stream
temporary can already be recycled (CUDA graphs do), scrambling rows' requests.
"""
topk = torch.arange(20, dtype=torch.int32).view(10, 2)
cache = SimpleNamespace(
source_block_table=torch.zeros((2, 1), dtype=torch.int32),
swap_in=MagicMock(return_value=topk),
)
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.caches = [cache]
index_group.physical_topk_indices = torch.empty((11, 2), dtype=torch.int32)
index_group.row_indices = torch.arange(10, dtype=torch.int32)
index_group.request_ids = torch.empty_like(index_group.row_indices)
metadata = SimpleNamespace(
query_start_loc=torch.tensor([0, 8, 9], dtype=torch.int32),
req_id_per_token=torch.tensor([0] * 8 + [1, 0], dtype=torch.int32),
block_size=64,
)
index_group.convert_logical_to_physical_topk(
0, topk, metadata, block_stride_rows=None, return_valid_counts=False
)
request_ids = cache.swap_in.call_args.args[0]
assert request_ids.data_ptr() == index_group.request_ids.data_ptr()
torch.testing.assert_close(
request_ids, torch.tensor([0] * 8 + [1, -1], dtype=torch.int32)
)
def test_flashinfer_hisparse_decode_runs_batched_attention():
device = torch.device("cpu")
num_tokens = 6
+50 -27
View File
@@ -95,6 +95,8 @@ def test_hisparse_spill_and_prefix_restore(
A regression omitted attention metadata before FULL graph replay, so decode
completed normally while its newly written KV rows were never copied to host.
MTP layers write after the target forward; their rows used to be mirrored
before the drafter wrote them, so the drafter later read stale host rows.
"""
capability = current_platform.get_device_capability()
if capability is None or capability.major < 9:
@@ -171,49 +173,70 @@ def test_hisparse_spill_and_prefix_restore(
model_runner.cudagraph_manager, "run_fullgraph", record_fullgraph
)
# The MTP layer writes its rows after the target forward, so it is
# mirrored at the start of the next step instead of in finish_forward.
draft_layers = set(worker._draft_layers)
assert draft_layers
original_finish_forward = worker.finish_forward
verified_full_replay_rows = 0
original_finish_previous_step = worker._finish_previous_step
verified_rows = {"target": 0, "draft": 0}
copied_nonzero_kv = False
draft_check_pending = False
def check_host_rows(layer_indices: set[int], kind: str) -> None:
nonlocal copied_nonzero_kv
# Synchronize only the test's inspection, not the worker itself.
with gpu_sync_allowed():
torch.accelerator.synchronize()
assert worker._row_mirrors
for layer_index in layer_indices:
cache = worker.cache_handles[layer_index]
source_index = cache.runtime.resident_source_index
for mirror in worker._row_mirrors:
source_row = mirror.source_starts[source_index]
destination_row = mirror.destination_start
num_rows = mirror.num_rows
source_block, source_offset = divmod(
source_row, worker.kernel_block_size
)
with gpu_sync_allowed():
gpu_rows = worker.resident_caches[layer_index][
source_block, source_offset : source_offset + num_rows
].cpu()
host_rows = worker.host_caches[layer_index][
destination_row : destination_row + num_rows
]
assert torch.equal(host_rows, gpu_rows)
copied_nonzero_kv |= bool(torch.count_nonzero(gpu_rows).item())
verified_rows[kind] += num_rows
def finish_forward():
nonlocal full_replay_pending, verified_full_replay_rows
nonlocal copied_nonzero_kv
nonlocal full_replay_pending, draft_check_pending
try:
original_finish_forward()
if not full_replay_pending:
return
# Synchronize only the test's inspection, not finish_forward itself.
with gpu_sync_allowed():
torch.accelerator.synchronize()
assert worker._row_mirrors
for layer_index, cache in enumerate(worker.cache_handles):
source_index = cache.runtime.resident_source_index
for mirror in worker._row_mirrors:
source_row = mirror.source_starts[source_index]
destination_row = mirror.destination_start
num_rows = mirror.num_rows
source_block, source_offset = divmod(
source_row, worker.kernel_block_size
)
with gpu_sync_allowed():
gpu_rows = worker.resident_caches[layer_index][
source_block, source_offset : source_offset + num_rows
].cpu()
host_rows = worker.host_caches[layer_index][
destination_row : destination_row + num_rows
]
assert torch.equal(host_rows, gpu_rows)
copied_nonzero_kv |= bool(torch.count_nonzero(gpu_rows).item())
verified_full_replay_rows += num_rows
target_layers = set(range(len(worker.cache_handles))) - draft_layers
check_host_rows(target_layers, "target")
draft_check_pending = True
finally:
full_replay_pending = False
def finish_previous_step():
nonlocal draft_check_pending
original_finish_previous_step()
if draft_check_pending:
draft_check_pending = False
check_host_rows(draft_layers, "draft")
monkeypatch.setattr(worker, "finish_forward", finish_forward)
monkeypatch.setattr(worker, "_finish_previous_step", finish_previous_step)
expected = runner.generate_greedy([target], max_tokens=8)
assert full_graph_calls > 0
assert verified_full_replay_rows > 0
assert verified_rows["target"] > 0
assert verified_rows["draft"] > 0
assert copied_nonzero_kv
runner.generate_greedy(pressure, max_tokens=8)
@@ -1,15 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import ctypes
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from vllm.config.compilation import CUDAGraphMode
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorRole
from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import (
ExampleConnector,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse import (
worker as worker_module,
)
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
HiSparseConnector,
HiSparseConnectorMetadata,
@@ -19,8 +25,14 @@ from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
HiSparseConnectorWorker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.multi_connector import MultiConnector
from vllm.v1.hisparse import runtime as runtime_module
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.hisparse.runtime import HiSparseCacheHandle
from vllm.v1.hisparse.types import (
SparseKVOffloadCommand,
SparseKVPageTransfer,
SparseKVRowMirror,
)
from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector
@@ -83,6 +95,7 @@ def test_full_graph_step_prepares_host_mirror_outside_model():
worker._group_leaders = (("layer", handle),)
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._draft_layers = ()
worker.is_host_writer = True
worker._enqueue_row_dma = MagicMock()
worker.start_step = MagicMock(
@@ -116,6 +129,195 @@ def test_full_graph_step_prepares_host_mirror_outside_model():
runtime.invalidate_written_slots.assert_called_once()
class _FakeEvent:
log: list[tuple[str, object]] = []
def __init__(self, *args, **kwargs) -> None:
pass
def record(self, stream=None) -> None:
self.log.append(("event", self))
def query(self) -> bool:
return True
def synchronize(self) -> None:
pass
@pytest.mark.parametrize("cg_mode", [CUDAGraphMode.FULL, CUDAGraphMode.NONE])
def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
"""Draft-layer host rows must hold the drafter's writes before page release.
The drafter writes its layers' rows after the target forward. Mirroring
them at the end of the target forward (the only mirror a FULL graph gets)
leaves pre-draft rows on the host, and the page is still reported clean.
"""
block_size, width = 4, 8
resident_block, host_block, transfer_id = 2, 1, 7
log: list[tuple[str, object]] = []
_FakeEvent.log = log
def swap_blocks_batch(src, dst, sizes):
for src_ptr, dst_ptr, size in zip(src.tolist(), dst.tolist(), sizes.tolist()):
ctypes.memmove(dst_ptr, src_ptr, size)
log.append(("dma", dst_ptr))
monkeypatch.setattr(torch, "Event", _FakeEvent)
monkeypatch.setattr(torch, "Stream", lambda *args, **kwargs: MagicMock())
monkeypatch.setattr(torch.cuda, "Stream", lambda *args, **kwargs: MagicMock())
monkeypatch.setattr(torch.cuda, "stream", lambda _: contextlib.nullcontext())
compute_stream = MagicMock()
monkeypatch.setattr(worker_module, "current_stream", lambda: compute_stream)
monkeypatch.setattr(worker_module.ops, "swap_blocks_batch", swap_blocks_batch)
monkeypatch.setattr(
runtime_module,
"get_forward_context",
lambda: SimpleNamespace(cudagraph_runtime_mode=cg_mode),
)
slots = torch.arange(block_size) + resident_block * block_size
request_state_indices = torch.full((1,), -1, dtype=torch.int32)
handles = []
for draft_layer in (False, True):
runtime = SimpleNamespace(
is_group_leader=True,
eager_host_mirror=True,
resident_source_index=0,
host_cache=torch.zeros(4 * block_size, width),
request_state_indices=request_state_indices,
begin_forward=MagicMock(),
invalidate_written_slots=MagicMock(),
)
handle = HiSparseCacheHandle(runtime)
handle.view = SimpleNamespace(
block_size=block_size, cache=torch.zeros(4, block_size, width)
)
handle.slot_mapping = slots
handle.mirror_slot_mapping = slots
handle.draft_layer = draft_layer
handles.append(handle)
target, draft = handles
# Rows the draft layer held before this step's drafter ran.
draft.view.cache[resident_block] = -1.0
worker = HiSparseConnectorWorker(
SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=16),
num_lookahead_tokens=3,
),
MagicMock(),
)
worker.initialize(
handles,
["target", "draft"],
MagicMock(),
max_num_reqs=1,
host_num_blocks=4,
device=torch.device("cpu"),
pinned_host_pools=[],
)
connector = object.__new__(HiSparseConnector)
connector.connector_worker = worker
connector._get_connector_metadata = MagicMock(
return_value=HiSparseConnectorMetadata(
SparseKVOffloadCommand(
[
SparseKVPageTransfer(
transfer_id=transfer_id,
host_block_id=host_block,
resident_block_ids=(resident_block,),
after_forward=True,
)
]
),
(),
(),
{
"req": (
SparseKVRowMirror(
source_starts=(resident_block * block_size,),
destination_start=host_block * block_size,
num_rows=block_size,
),
)
},
True,
)
)
# A verification step: four query tokens of one request fill the page.
attn_metadata = SimpleNamespace(
num_actual_tokens=block_size,
num_decode_tokens=0,
num_reqs=1,
max_query_len=block_size,
req_id_per_token=torch.zeros(block_size, dtype=torch.int32),
)
connector.start_load_kv(
SimpleNamespace(),
request_state_indices=None,
request_ids=["req"],
num_tokens=block_size,
attn_metadata={"target": attn_metadata, "draft": attn_metadata},
)
target.view.cache[resident_block] = 1.0
if cg_mode != CUDAGraphMode.FULL:
target.finish_kv_update()
connector.finish_forward()
enqueued_before_draft = list(worker._enqueued_transfer_ids)
# The drafter writes the draft layer's rows for the same positions.
if cg_mode != CUDAGraphMode.FULL:
draft.prepare_group_for_batch(attn_metadata)
draft.view.cache[resident_block] = 2.0
if cg_mode != CUDAGraphMode.FULL:
draft.finish_kv_update()
# The next step's start mirrors the drafter's rows, then hands the page over.
compute_stream.wait_event.reset_mock()
connector._get_connector_metadata.return_value = HiSparseConnectorMetadata(
None, (), (), {}, True
)
connector.start_load_kv(
SimpleNamespace(),
request_state_indices=None,
request_ids=[],
num_tokens=0,
attn_metadata={},
)
((completion_event, _),) = worker._pending_transfer_events
worker_meta = connector.build_connector_worker_meta()
host_rows = slice(host_block * block_size, (host_block + 1) * block_size)
torch.testing.assert_close(target.runtime.host_cache[host_rows], torch.ones(4, 8))
torch.testing.assert_close(
draft.runtime.host_cache[host_rows], torch.full((4, 8), 2.0)
)
draft_host = draft.runtime.host_cache
draft_host_range = range(
draft_host.data_ptr(),
draft_host.data_ptr() + draft_host.numel() * draft_host.element_size(),
)
draft_copies = [
index
for index, (kind, ptr) in enumerate(log)
if kind == "dma" and ptr in draft_host_range
]
# One draft-layer copy, and the page is not handed back before it.
assert len(draft_copies) == 1
assert enqueued_before_draft == []
assert log.index(("event", completion_event)) > draft_copies[0]
# The next forward waits for the draft copy: the scheduler may already
# have handed the blocks it reads and writes to another request.
draft_copy_event = next(
entry[1] for entry in log[draft_copies[0] :] if entry[0] == "event"
)
assert compute_stream.wait_event.call_args_list[0].args == (draft_copy_event,)
assert worker_meta is not None
assert worker_meta.enqueued_transfer_counts == {transfer_id: 1}
assert worker_meta.completed_transfer_counts == {transfer_id: 1}
def test_scheduled_prefix_hit_publishes_adopted_copies():
"""Copies adopted after scheduling must reach the worker's block table."""
from tests.v1.core.test_prefix_caching import (
+2 -24
View File
@@ -44,6 +44,8 @@ def _make_hisparse_worker() -> HiSparseConnectorWorker:
worker._row_mirror_num_rows = 0
worker._per_layer_mirrored = set()
worker._submitted_mirror_layers = set()
worker._draft_layers = ()
worker._draft_mirror_pending = False
worker._pending_dma_descriptors = deque()
worker._dma_free_descriptors = []
worker.host_write_events = (MagicMock(), MagicMock())
@@ -785,30 +787,6 @@ def test_hisparse_finish_forward_does_not_repeat_per_layer_mirrors():
worker._enqueue_row_dma.assert_not_called()
def test_hisparse_finish_forward_submits_lazy_post_forward_transfer(monkeypatch):
runtime = SimpleNamespace(eager_host_mirror=False)
worker = _make_hisparse_worker()
worker.is_host_writer = True
worker.cache_handles = [SimpleNamespace(runtime=runtime, num_actual_tokens=0)]
transfer = SparseKVPageTransfer(7, 2, (1,), after_forward=True)
worker._post_forward_transfers = [transfer]
worker._forward_ready_event = MagicMock()
worker._enqueue_host_mirror = MagicMock()
worker._submit_transfers = MagicMock()
worker._dma_submitted = False
worker._submitted_mirror_layers = set()
worker._finish_mirror_phase = MagicMock()
worker._release_completed_dma_descriptors = MagicMock()
stream = MagicMock()
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream)
worker.finish_forward()
worker._finish_mirror_phase.assert_called_once_with(worker._forward_ready_event)
worker._submit_transfers.assert_called_once_with([transfer])
worker.host_write_event.record.assert_called_once_with(stream)
def test_hisparse_prefill_mirrors_source_groups_and_flushes_partial_group():
slots = torch.tensor([7, 8], dtype=torch.int64)
source_indices = [0, 0, 1, 1, 1, 1, 2]