[Bugfix][HiSparse] Resolve MTP verification rows with a union residency kernel (#59235)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-09-30 03:01:45 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent bec710fccf
commit 2798f66860
13 changed files with 705 additions and 442 deletions
+229 -107
View File
@@ -68,10 +68,10 @@ from vllm.utils.torch_utils import current_stream
from vllm.v1.attention.backends.mla import index_group as index_group_module
from vllm.v1.attention.backends.mla.flashattn_mla_sparse import (
FlashAttnMLASparseImpl,
FlashAttnMLASparseMetadataBuilder,
)
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseImpl,
FlashInferMLASparseMetadataBuilder,
FlashInferMLASparseTRTLLMBackend,
)
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm120 import (
@@ -1880,11 +1880,13 @@ def _make_hisparse_runtime(
block_size: int = 64,
max_swap_rows: int | None = None,
index_group: hisparse_runtime.HiSparseIndexGroup | None = None,
max_union_rows: int | None = None,
) -> HiSparseRuntime:
runtime = HiSparseRuntime(
config=ResolvedHiSparseConfig(
top_k=top_k,
device_buffer_size=device_buffer_size,
max_union_rows=max_union_rows or device_buffer_size,
),
max_num_reqs=max_num_reqs,
row_width=row_width,
@@ -2015,6 +2017,167 @@ def test_hisparse_maps_speculative_rows_through_request_state():
assert physical.tolist() == [[block_size]] * 4
@requires_hisparse_ops
@pytest.mark.parametrize(
"num_rows,row_stride,top_k",
[(8, 128, 128), (4, 32, 128), (8, 2048, 2048)],
ids=["disjoint", "overlap", "long_mtp"],
)
def test_hisparse_speculative_rows_resolve_host_misses_consistently(
num_rows, row_stride, top_k
):
"""Verification rows of one request must agree on its hot-buffer slots.
Each row resolved residency in its own thread block against the request's
shared slot ownership and LRU, so rows missing on different host rows could
claim the same free slot. One row's hot index then pointed at a slot holding
another row's KV. Overlapping rows must share one slot per host row, and
long_mtp's rows span several resolution passes.
"""
device = torch.device(DEVICE_TYPE)
block_size, row_width = 64, 64
num_blocks = 2 * num_rows * top_k // block_size
kv_pool = torch.randn(
(num_blocks + 1, block_size, row_width), dtype=torch.float32
).pin_memory()
flat_pool = kv_pool.reshape(-1, row_width)
runtime = _make_hisparse_runtime(
top_k=top_k,
device_buffer_size=num_rows * top_k,
max_num_reqs=1,
row_width=row_width,
block_size=block_size,
max_swap_rows=num_rows,
)
runtime.bind_source_cache(kv_pool)
cache = HiSparseCacheHandle(runtime)
cache.all_context_pages_resident = False
block_table = torch.arange(
1, num_blocks + 1, dtype=torch.int32, device=device
).view(1, num_blocks)
req_ids = torch.zeros(num_rows, dtype=torch.int32, device=device)
rows = torch.arange(num_rows, dtype=torch.int32, device=device)[:, None]
base = torch.arange(top_k, dtype=torch.int32, device=device)[None, :]
for step in range(4):
runtime.begin_forward()
# Rows start row_stride apart (disjoint at row_stride == top_k) and
# shift by half a top-k per step: all misses first, then a mix.
topk = (rows * row_stride + base + step * (top_k // 2)) % (
num_blocks * block_size
)
hot_indices = cache.swap_in(
req_id_per_token=req_ids,
block_table=block_table,
logical_topk_indices=topk.clone(),
block_size=block_size,
)
torch.accelerator.synchronize()
global_ref = _triton_convert_reference_impl(
req_ids, block_table, topk, block_size, top_k
)
gathered = runtime.hot.attention_cache.reshape(-1, row_width)[
hot_indices.to(torch.long)
].cpu()
torch.testing.assert_close(gathered, flat_pool[global_ref.cpu().to(torch.long)])
@requires_hisparse_ops
def test_hisparse_speculative_rows_count_a_shared_load_once():
"""A host row two verification rows select loads once; the reuse is a hit."""
device = torch.device(DEVICE_TYPE)
block_size, row_width, top_k = 64, 64, 128
num_blocks = top_k // block_size
kv_pool = torch.randn(
(num_blocks + 1, block_size, row_width), dtype=torch.float32
).pin_memory()
runtime = _make_hisparse_runtime(
top_k=top_k,
device_buffer_size=2 * top_k,
max_num_reqs=1,
row_width=row_width,
block_size=block_size,
max_swap_rows=2,
)
runtime.bind_source_cache(kv_pool)
cache = HiSparseCacheHandle(runtime)
cache.all_context_pages_resident = False
block_table = torch.arange(
1, num_blocks + 1, dtype=torch.int32, device=device
).view(1, num_blocks)
topk = torch.arange(top_k, dtype=torch.int32, device=device).repeat(2, 1)
runtime.begin_forward()
cache.swap_in(
req_id_per_token=torch.zeros(2, dtype=torch.int32, device=device),
block_table=block_table,
logical_topk_indices=topk,
block_size=block_size,
)
torch.accelerator.synchronize()
assert runtime.index_group.swap_stats.cpu().tolist() == [top_k, top_k]
@requires_hisparse_ops
def test_hisparse_union_table_scales_with_union_bound():
"""Shared memory follows the per-request union bound, not the hot buffer.
Sizing the table from device_buffer_size needs ~340 KB here and fails.
"""
device = torch.device(DEVICE_TYPE)
block_size, row_width, top_k = 64, 64, 128
num_blocks = top_k // block_size
kv_pool = torch.randn(
(num_blocks + 1, block_size, row_width), dtype=torch.float32
).pin_memory()
runtime = _make_hisparse_runtime(
top_k=top_k,
device_buffer_size=32768,
max_union_rows=top_k,
max_num_reqs=1,
row_width=row_width,
block_size=block_size,
max_swap_rows=1,
)
runtime.bind_source_cache(kv_pool)
cache = HiSparseCacheHandle(runtime)
cache.all_context_pages_resident = False
req_ids = torch.zeros(1, dtype=torch.int32, device=device)
block_table = torch.arange(
1, num_blocks + 1, dtype=torch.int32, device=device
).view(1, num_blocks)
topk = torch.arange(top_k, dtype=torch.int32, device=device).view(1, top_k)
runtime.begin_forward()
hot_indices = cache.swap_in(
req_id_per_token=req_ids,
block_table=block_table,
logical_topk_indices=topk,
block_size=block_size,
)
torch.accelerator.synchronize()
global_ref = _triton_convert_reference_impl(
req_ids, block_table, topk, block_size, top_k
)
gathered = runtime.hot.attention_cache.reshape(-1, row_width)[
hot_indices.to(torch.long)
].cpu()
torch.testing.assert_close(
gathered, kv_pool.reshape(-1, row_width)[global_ref.cpu().to(torch.long)]
)
@requires_hisparse_ops
def test_hisparse_rejects_union_bound_beyond_shared_memory():
"""An oversized union bound fails at construction, not on the first forward."""
with pytest.raises(ValueError, match="shared memory"):
_make_hisparse_runtime(top_k=2048, device_buffer_size=32768)
@requires_hisparse_ops
def test_hisparse_resident_rows_bypass_hot_lru():
device = torch.device(DEVICE_TYPE)
@@ -3001,8 +3164,8 @@ def test_hisparse_gather_prefill_cache_prefers_resident_rows():
torch.testing.assert_close(staged_flat[i], expected)
def test_hisparse_fp8_decode_resolves_steps_then_runs_batched_attention(monkeypatch):
"""FP8 decode must use active dimensions when common metadata is padded."""
def test_hisparse_fp8_decode_resolves_rows_once_then_runs_batched_attention():
"""Verification rows resolve in one swap_in; FP8 attention sees them batched."""
device = torch.device("cpu")
num_decodes = 2
query_len = 3
@@ -3013,48 +3176,31 @@ def test_hisparse_fp8_decode_resolves_steps_then_runs_batched_attention(monkeypa
.view(-1, 1)
.expand(-1, 4)
)
steps: list[int] = []
kernel_shapes: list[torch.Size] = []
def swap_in(req_ids, *, logical_topk_indices, **kwargs): # noqa: ARG001
steps.append(int(logical_topk_indices[0, 0]))
output = kwargs["attention_indices_out"]
if output is not None:
output.copy_(logical_topk_indices + 10)
return topk[:num_decodes]
return logical_topk_indices + 10
def run_kernel(self, *, q, **kwargs): # noqa: ARG001
kernel_shapes.append(q.shape)
return q[..., :1], None
runtime = SimpleNamespace(
hot=SimpleNamespace(attention_cache=torch.empty(1, device=device))
)
leader_cache = SimpleNamespace(
runtime=runtime,
source_block_table=torch.empty(
num_decodes, 1, dtype=torch.int32, device=device
cache = SimpleNamespace(
runtime=SimpleNamespace(
hot=SimpleNamespace(attention_cache=torch.empty(1, device=device))
),
swap_in=MagicMock(side_effect=swap_in),
)
follower_cache = SimpleNamespace(
runtime=runtime,
source_block_table=torch.empty(
num_decodes, 1, dtype=torch.int32, device=device
),
swap_in=MagicMock(side_effect=swap_in),
)
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.caches = [leader_cache, follower_cache]
index_group.caches = [cache]
index_group.physical_topk_indices = torch.empty(
(num_tokens + 1, 4), dtype=torch.int32, device=device
)
index_group.valid_topk_counts = torch.empty(
num_tokens + 1, dtype=torch.int32, device=device
)
index_group.request_ids = torch.arange(
num_decodes, dtype=torch.int32, device=device
)
index_group.row_indices = torch.arange(num_tokens, dtype=torch.int32, device=device)
index_group.request_ids = torch.empty_like(index_group.row_indices)
impl = SimpleNamespace(
kv_lora_rank=1,
index_group=index_group,
@@ -3062,84 +3208,83 @@ def test_hisparse_fp8_decode_resolves_steps_then_runs_batched_attention(monkeypa
)
impl._fp8_flash_mla_kernel = MethodType(run_kernel, impl)
metadata = SimpleNamespace(
num_decodes=num_tokens,
num_decode_tokens=num_tokens,
decode_max_query_len=1,
query_start_loc=torch.arange(
0,
num_tokens + 1,
query_len,
dtype=torch.int32,
device=device,
0, num_tokens + 1, query_len, dtype=torch.int32, device=device
),
block_table=torch.empty(num_decodes, 1, dtype=torch.int32, device=device),
req_id_per_token=torch.arange(
num_decodes, dtype=torch.int32, device=device
).repeat_interleave(query_len),
block_size=64,
)
output = FlashMLASparseImpl._host_backed_fp8_decode(
impl,
q,
topk,
metadata,
SimpleNamespace(),
num_decodes,
query_len,
impl, q, topk, metadata, SimpleNamespace(), num_decodes
)
assert steps == [0, 1, 2]
cache.swap_in.assert_called_once()
torch.testing.assert_close(
cache.swap_in.call_args.args[0], metadata.req_id_per_token
)
torch.testing.assert_close(
cache.swap_in.call_args.kwargs["logical_topk_indices"], topk
)
assert kernel_shapes == [(num_decodes, query_len, 2, 4)]
assert output.shape == (num_tokens, 2, 1)
torch.testing.assert_close(
index_group.physical_topk_indices[:num_tokens], topk + 10
def test_hisparse_keeps_speculative_rows_as_decodes():
"""HiSparse limits decodes to verification requests, not short extends.
FlashAttn sparse MLA used to classify MTP verification as prefill under
HiSparse, whose residency-dependent route a FULL CUDA graph freezes at
capture.
"""
builder = object.__new__(FlashAttnMLASparseMetadataBuilder)
builder.vllm_config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=object()),
speculative_config=SimpleNamespace(
num_speculative_tokens=3, parallel_drafting=False
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
use_v2_model_runner=True,
)
follower_indices = index_group.convert_decode_logical_to_physical_topk(
1,
topk,
metadata,
return_valid_counts=False,
num_decodes=num_decodes,
decode_query_len=query_len,
)
assert steps == [0, 1, 2, 0, 1, 2]
torch.testing.assert_close(follower_indices, topk + 10)
builder._init_reorder_batch_threshold(256, supports_spec_as_decode=True)
assert builder.reorder_batch_threshold == 4
def test_hisparse_varlen_decode_uses_canonical_request_rows():
"""Graph metadata's query upper bound must preserve packed token rows."""
num_decodes = 2
topk = torch.arange(18, dtype=torch.int32).view(9, 2)
def swap_in(_request_ids, *, logical_topk_indices, **_kwargs):
return logical_topk_indices + 10
def test_hisparse_decode_skips_padding_rows():
"""Padding rows are masked, and the ids stay in the group's persistent buffer.
The resolver reads them on the HiSparse copy stream, where a compute-stream
temporary can already be recycled (CUDA graphs do), scrambling rows' requests.
"""
topk = torch.arange(20, dtype=torch.int32).view(10, 2)
cache = SimpleNamespace(
source_block_table=torch.zeros((num_decodes, 1), dtype=torch.int32),
swap_in=MagicMock(side_effect=swap_in),
source_block_table=torch.zeros((2, 1), dtype=torch.int32),
swap_in=MagicMock(return_value=topk),
)
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.caches = [cache]
index_group.physical_topk_indices = torch.empty((10, 2), dtype=torch.int32)
index_group.valid_topk_counts = torch.empty(10, dtype=torch.int32)
index_group.request_ids = torch.arange(num_decodes, dtype=torch.int32)
index_group.physical_topk_indices = torch.empty((11, 2), dtype=torch.int32)
index_group.row_indices = torch.arange(10, dtype=torch.int32)
index_group.request_ids = torch.empty_like(index_group.row_indices)
metadata = SimpleNamespace(
num_decodes=num_decodes,
num_decode_tokens=topk.shape[0],
decode_max_query_len=15,
query_start_loc=torch.tensor([0, 8, 9], dtype=torch.int32),
req_id_per_token=torch.tensor([0] * 8 + [1, 0], dtype=torch.int32),
block_size=64,
)
physical_topk = index_group.convert_decode_logical_to_physical_topk(
0,
topk,
metadata,
return_valid_counts=False,
index_group.convert_logical_to_physical_topk(
0, topk, metadata, block_stride_rows=None, return_valid_counts=False
)
torch.testing.assert_close(physical_topk, topk + 10)
request_rows = cache.swap_in.call_args_list[0].args[0]
torch.testing.assert_close(request_rows, index_group.request_ids)
request_ids = cache.swap_in.call_args.args[0]
assert request_ids.data_ptr() == index_group.request_ids.data_ptr()
torch.testing.assert_close(
request_ids, torch.tensor([0] * 8 + [1, -1], dtype=torch.int32)
)
def test_flashinfer_hisparse_decode_runs_batched_attention():
@@ -3168,7 +3313,7 @@ def test_flashinfer_hisparse_decode_runs_batched_attention():
)
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.caches = [cache_handle]
index_group.convert_decode_logical_to_physical_topk = MethodType(
index_group.convert_logical_to_physical_topk = MethodType(
convert_decode, index_group
)
impl = object.__new__(FlashInferMLASparseImpl)
@@ -3200,7 +3345,7 @@ def test_flashattn_hisparse_decode_uses_index_group():
physical = topk.clone()
counts = torch.full((num_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE)
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.convert_decode_logical_to_physical_topk = MagicMock(
index_group.convert_logical_to_physical_topk = MagicMock(
return_value=(physical, counts)
)
index_group.physical_kv_cache = MagicMock(
@@ -3223,7 +3368,7 @@ def test_flashattn_hisparse_decode_uses_index_group():
assert output.shape == (num_tokens, 2, 1)
assert lse is None
index_group.convert_decode_logical_to_physical_topk.assert_called_once()
index_group.convert_logical_to_physical_topk.assert_called_once()
impl._run_mqa_kernel.assert_called_once()
@@ -3233,9 +3378,7 @@ def test_flashinfer_sm120_hisparse_decode_uses_index_group():
topk = torch.zeros(num_tokens, 4, dtype=torch.int32, device=DEVICE_TYPE)
physical = topk.clone()
index_group = object.__new__(HiSparseMLAIndexGroup)
index_group.convert_decode_logical_to_physical_topk = MagicMock(
return_value=physical
)
index_group.convert_logical_to_physical_topk = MagicMock(return_value=physical)
index_group.physical_kv_cache = MagicMock(
return_value=torch.empty(1, device=DEVICE_TYPE)
)
@@ -3259,7 +3402,7 @@ def test_flashinfer_sm120_hisparse_decode_uses_index_group():
assert output.shape == (num_tokens, 2, 1)
assert lse is None
index_group.convert_decode_logical_to_physical_topk.assert_called_once()
index_group.convert_logical_to_physical_topk.assert_called_once()
impl._run_mqa_kernel.assert_called_once()
@@ -3291,27 +3434,6 @@ def test_hisparse_resident_prefill_uses_attention_block_stride():
assert index_group._convert_once.call_args.kwargs["block_stride_rows"] == 832
def test_hisparse_shared_sparse_builder_routes_multi_token_chunks_to_prefill():
"""Default sparse backends must route multi-token chunks to prefill."""
builder = object.__new__(FlashInferMLASparseMetadataBuilder)
builder.vllm_config = SimpleNamespace(
attention_config=SimpleNamespace(hisparse_config=object()),
speculative_config=SimpleNamespace(
num_speculative_tokens=4,
parallel_drafting=False,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
)
builder._init_reorder_batch_threshold(
1024,
supports_spec_as_decode=True,
supports_dcp_with_varlen=True,
)
assert builder.reorder_batch_threshold == 1
def test_hisparse_flashmla_reorders_full_speculative_window_as_decode():
builder = object.__new__(FlashMLASparseMetadataBuilder)
builder.vllm_config = SimpleNamespace(
+1
View File
@@ -1156,6 +1156,7 @@ def test_hisparse_cache_handles_join_index_groups_during_construction(monkeypatc
resolved = hisparse_runtime_module.ResolvedHiSparseConfig(
top_k=4,
device_buffer_size=8,
max_union_rows=8,
)
monkeypatch.setattr(hisparse_runtime_module, "_has_hisparse_ops", lambda: True)
monkeypatch.setattr(