[HiSparse] Read GPU-resident KV directly for mixed and BF16 prefills (#60417)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c19c263ac8
commit
292da21f9d
@@ -1366,6 +1366,47 @@ def test_flashmla_forward_bf16_kv_slices_req_id_to_mqa_tokens():
|
|||||||
torch.testing.assert_close(captured["indices"], reference, rtol=0, atol=0)
|
torch.testing.assert_close(captured["indices"], reference, rtol=0, atol=0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_flashmla_hisparse_resident_mixed_batch_skips_host_staging():
|
||||||
|
"""A resident prefill batched with decodes reads resident KV in one kernel
|
||||||
|
instead of staging its whole history from host."""
|
||||||
|
num_tokens = 6
|
||||||
|
q = torch.zeros(num_tokens, 4, 576, dtype=torch.bfloat16, device=DEVICE_TYPE)
|
||||||
|
topk = torch.zeros(num_tokens, 4, dtype=torch.int32, device=DEVICE_TYPE)
|
||||||
|
counts = torch.full((num_tokens,), 4, dtype=torch.int32, device=DEVICE_TYPE)
|
||||||
|
kernel_rows: list[int] = []
|
||||||
|
|
||||||
|
def kernel(q, kv, indices, lengths, actual_num_heads): # noqa: ARG001
|
||||||
|
kernel_rows.append(q.shape[0])
|
||||||
|
return q[..., :512], None
|
||||||
|
|
||||||
|
index_group = object.__new__(HiSparseMLAIndexGroup)
|
||||||
|
index_group.caches = [SimpleNamespace(all_context_pages_resident=True)]
|
||||||
|
index_group.convert_logical_to_physical_topk = MagicMock(
|
||||||
|
return_value=(topk, counts)
|
||||||
|
)
|
||||||
|
index_group.physical_kv_cache = MagicMock(
|
||||||
|
return_value=torch.empty(1, device=DEVICE_TYPE)
|
||||||
|
)
|
||||||
|
index_group.stage_prefill_rows = MagicMock(side_effect=AssertionError)
|
||||||
|
impl = SimpleNamespace(
|
||||||
|
_bf16_flash_mla_kernel=kernel,
|
||||||
|
index_group=index_group,
|
||||||
|
index_group_index=0,
|
||||||
|
)
|
||||||
|
metadata = SimpleNamespace(
|
||||||
|
req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32),
|
||||||
|
block_table=torch.zeros((2, 1), dtype=torch.int32),
|
||||||
|
num_decode_tokens=2,
|
||||||
|
)
|
||||||
|
|
||||||
|
out, _ = FlashMLASparseImpl._forward_bf16_kv(
|
||||||
|
impl, q, torch.empty(1, device=DEVICE_TYPE), topk, metadata, q.shape[1]
|
||||||
|
)
|
||||||
|
|
||||||
|
assert kernel_rows == [num_tokens]
|
||||||
|
assert out.shape == (num_tokens, 4, 512)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"seq_lens,max_buf,expected",
|
"seq_lens,max_buf,expected",
|
||||||
[
|
[
|
||||||
@@ -3268,6 +3309,7 @@ def test_hisparse_mixed_batch_bf16_row_split(
|
|||||||
kv_pool = kv_cache.squeeze(1).cpu().pin_memory()
|
kv_pool = kv_cache.squeeze(1).cpu().pin_memory()
|
||||||
cache_handle.runtime.bind_source_cache(kv_pool)
|
cache_handle.runtime.bind_source_cache(kv_pool)
|
||||||
cache_handle.source_block_table = metadata.block_table
|
cache_handle.source_block_table = metadata.block_table
|
||||||
|
cache_handle.all_context_pages_resident = False
|
||||||
|
|
||||||
staging_calls = []
|
staging_calls = []
|
||||||
original_gather = cache_handle.runtime.gather_prefill_cache
|
original_gather = cache_handle.runtime.gather_prefill_cache
|
||||||
@@ -3519,6 +3561,9 @@ def test_hisparse_fp8_decode_resolves_rows_once_then_runs_batched_attention():
|
|||||||
num_decodes, 1, dtype=torch.int32, device=device
|
num_decodes, 1, dtype=torch.int32, device=device
|
||||||
),
|
),
|
||||||
swap_in=MagicMock(side_effect=swap_in),
|
swap_in=MagicMock(side_effect=swap_in),
|
||||||
|
all_context_pages_resident=True,
|
||||||
|
num_decode_tokens=num_tokens,
|
||||||
|
num_actual_tokens=num_tokens,
|
||||||
)
|
)
|
||||||
index_group = object.__new__(HiSparseMLAIndexGroup)
|
index_group = object.__new__(HiSparseMLAIndexGroup)
|
||||||
index_group.caches = [cache]
|
index_group.caches = [cache]
|
||||||
@@ -3780,14 +3825,16 @@ def test_sparse_mqa_plain_lane_wiring(monkeypatch):
|
|||||||
[
|
[
|
||||||
pytest.param(0, True, False, id="resident"),
|
pytest.param(0, True, False, id="resident"),
|
||||||
pytest.param(0, False, False, id="staged"),
|
pytest.param(0, False, False, id="staged"),
|
||||||
pytest.param(2, True, True, id="mixed"),
|
pytest.param(2, True, True, id="mixed_resident"),
|
||||||
|
pytest.param(2, False, True, id="mixed_staged"),
|
||||||
pytest.param(5, True, False, id="decode"),
|
pytest.param(5, True, False, id="decode"),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
def test_sparse_mqa_hisparse_lanes(
|
def test_sparse_mqa_hisparse_lanes(
|
||||||
monkeypatch, num_decode_tokens, all_resident, split_q
|
monkeypatch, num_decode_tokens, all_resident, split_q
|
||||||
):
|
):
|
||||||
"""HiSparse splits a batch into at most one decode and one prefill call."""
|
"""HiSparse runs all-resident and decode-only batches in one call against the
|
||||||
|
hot cache, and otherwise splits into one decode and one staged-prefill call."""
|
||||||
import vllm.model_executor.layers.attention.sparse_mla_attention as sparse_mla
|
import vllm.model_executor.layers.attention.sparse_mla_attention as sparse_mla
|
||||||
|
|
||||||
num_tokens, topk_tokens, block_size, head_dim = 5, 8, 4, 6
|
num_tokens, topk_tokens, block_size, head_dim = 5, 8, 4, 6
|
||||||
@@ -3814,9 +3861,9 @@ def test_sparse_mqa_hisparse_lanes(
|
|||||||
)
|
)
|
||||||
# The fill value names the converter that produced a call's indices.
|
# The fill value names the converter that produced a call's indices.
|
||||||
index_group.convert_logical_to_physical_topk = MagicMock(
|
index_group.convert_logical_to_physical_topk = MagicMock(
|
||||||
return_value=converted(num_decode_tokens, 1)
|
side_effect=lambda layer, topk, *args, **kwargs: converted(
|
||||||
if num_decode_tokens
|
topk.shape[0], 1 if topk.shape[0] == num_decode_tokens else 2
|
||||||
else converted(num_tokens, 2)
|
)
|
||||||
)
|
)
|
||||||
index_group.stage_prefill_rows = MagicMock(return_value=(staged, None, None))
|
index_group.stage_prefill_rows = MagicMock(return_value=(staged, None, None))
|
||||||
calls: list[tuple] = []
|
calls: list[tuple] = []
|
||||||
@@ -3850,36 +3897,41 @@ def test_sparse_mqa_hisparse_lanes(
|
|||||||
|
|
||||||
q_rows = q.flatten().tolist()
|
q_rows = q.flatten().tolist()
|
||||||
expected = []
|
expected = []
|
||||||
if num_decode_tokens:
|
if not num_prefill:
|
||||||
# Decode reads the hot cache through the group's conversion.
|
expected.append((q_rows, hot.shape[0], 1))
|
||||||
expected.append((q_rows[:num_decode_tokens], hot.shape[0], 1))
|
elif all_resident:
|
||||||
if num_decode_tokens == 0 and all_resident:
|
# Decode and prefill rows read the hot cache in one call.
|
||||||
expected.append((q_rows, hot.shape[0], 2))
|
expected.append((q_rows, hot.shape[0], 2))
|
||||||
elif num_prefill:
|
else:
|
||||||
|
if num_decode_tokens:
|
||||||
|
expected.append((q_rows[:num_decode_tokens], hot.shape[0], 1))
|
||||||
expected.append((q_rows[num_decode_tokens:], staged.shape[0], 3))
|
expected.append((q_rows[num_decode_tokens:], staged.shape[0], 3))
|
||||||
assert calls == expected
|
assert calls == expected
|
||||||
if not num_prefill:
|
if not num_prefill:
|
||||||
# A decode-only batch returns before it looks up the prefill cache.
|
# A decode-only batch returns before it looks up the prefill cache.
|
||||||
index_group.cache.assert_not_called()
|
index_group.cache.assert_not_called()
|
||||||
assert lse is None
|
assert lse is None
|
||||||
# A lane handed the whole batch would concatenate to more rows than this.
|
num_decode_rows = num_tokens if not num_prefill else 0
|
||||||
assert out.flatten().tolist() == [1.0] * num_decode_tokens + [2.0] * num_prefill
|
if num_prefill and not all_resident:
|
||||||
|
num_decode_rows = num_decode_tokens
|
||||||
|
assert out.flatten().tolist() == [1.0] * num_decode_rows + [2.0] * (
|
||||||
|
num_tokens - num_decode_rows
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_hisparse_resident_prefill_uses_attention_block_stride():
|
@pytest.mark.parametrize("max_swap_rows", [1, 4], ids=["over_capacity", "mixed"])
|
||||||
"""A batch beyond the swap-row capacity must bypass swap_in.
|
def test_hisparse_resident_prefill_uses_attention_block_stride(max_swap_rows):
|
||||||
|
"""Resident batches with prefill rows read resident pages directly, whether
|
||||||
The index-group workspace has one more row than the HiSparse swap rows, so
|
they exceed the swap-row capacity or fit it."""
|
||||||
routing on its size sent a 2-token batch with max_num_seqs=1 (a piecewise
|
|
||||||
CUDA-graph capture) into swap_in, which overflowed its capacity of 1.
|
|
||||||
"""
|
|
||||||
expected = torch.tensor([[19]], dtype=torch.int32)
|
expected = torch.tensor([[19]], dtype=torch.int32)
|
||||||
cache_handle = SimpleNamespace(
|
cache_handle = SimpleNamespace(
|
||||||
all_context_pages_resident=True,
|
all_context_pages_resident=True,
|
||||||
|
num_decode_tokens=1,
|
||||||
|
num_actual_tokens=2,
|
||||||
view=SimpleNamespace(block_size=64, attention_block_stride=832),
|
view=SimpleNamespace(block_size=64, attention_block_stride=832),
|
||||||
block_table=torch.tensor([[3]], dtype=torch.int32),
|
block_table=torch.tensor([[3]], dtype=torch.int32),
|
||||||
batch_block_table=lambda: torch.tensor([[3]], dtype=torch.int32),
|
batch_block_table=lambda: torch.tensor([[3]], dtype=torch.int32),
|
||||||
runtime=SimpleNamespace(max_swap_rows=1),
|
runtime=SimpleNamespace(max_swap_rows=max_swap_rows),
|
||||||
)
|
)
|
||||||
index_group = object.__new__(HiSparseMLAIndexGroup)
|
index_group = object.__new__(HiSparseMLAIndexGroup)
|
||||||
index_group.caches = [cache_handle]
|
index_group.caches = [cache_handle]
|
||||||
|
|||||||
@@ -802,6 +802,27 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
|||||||
kv_c_and_k_pe_cache.dtype
|
kv_c_and_k_pe_cache.dtype
|
||||||
)
|
)
|
||||||
assert hot_cache.is_contiguous(), "HiSparse hot buffer must be contiguous"
|
assert hot_cache.is_contiguous(), "HiSparse hot buffer must be contiguous"
|
||||||
|
if (
|
||||||
|
num_decode_tokens >= num_tokens
|
||||||
|
or index_group.cache(layer_index).all_context_pages_resident
|
||||||
|
):
|
||||||
|
physical_topk, valid_counts = index_group.convert_logical_to_physical_topk(
|
||||||
|
layer_index,
|
||||||
|
topk,
|
||||||
|
attn_metadata,
|
||||||
|
block_stride_rows=None,
|
||||||
|
return_valid_counts=True,
|
||||||
|
)
|
||||||
|
return self._forward_mqa_kernel(
|
||||||
|
q,
|
||||||
|
hot_cache,
|
||||||
|
physical_topk,
|
||||||
|
valid_counts,
|
||||||
|
layer=layer,
|
||||||
|
block_size=block_size,
|
||||||
|
is_decode=num_decode_tokens >= num_tokens,
|
||||||
|
)
|
||||||
|
|
||||||
decode_out = decode_lse = None
|
decode_out = decode_lse = None
|
||||||
if num_decode_tokens:
|
if num_decode_tokens:
|
||||||
physical_topk, valid_counts = index_group.convert_logical_to_physical_topk(
|
physical_topk, valid_counts = index_group.convert_logical_to_physical_topk(
|
||||||
@@ -825,31 +846,18 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
|||||||
block_size=block_size,
|
block_size=block_size,
|
||||||
is_decode=True,
|
is_decode=True,
|
||||||
)
|
)
|
||||||
if num_decode_tokens >= num_tokens:
|
|
||||||
return decode_out, decode_lse
|
|
||||||
|
|
||||||
cache = index_group.cache(layer_index)
|
prefill_cache, block_table, req_ids = index_group.stage_prefill_rows(
|
||||||
if num_decode_tokens == 0 and cache.all_context_pages_resident:
|
layer_index, kv_c_and_k_pe_cache, attn_metadata
|
||||||
physical_topk, valid_counts = index_group.convert_logical_to_physical_topk(
|
)
|
||||||
layer_index,
|
physical_topk, valid_counts = triton_convert_req_index_to_global_index(
|
||||||
topk,
|
req_ids,
|
||||||
attn_metadata,
|
block_table,
|
||||||
block_stride_rows=None,
|
topk[num_decode_tokens:],
|
||||||
return_valid_counts=True,
|
BLOCK_SIZE=block_size,
|
||||||
)
|
NUM_TOPK_TOKENS=topk.shape[1],
|
||||||
prefill_cache = hot_cache
|
return_valid_counts=True,
|
||||||
else:
|
)
|
||||||
prefill_cache, block_table, req_ids = index_group.stage_prefill_rows(
|
|
||||||
layer_index, kv_c_and_k_pe_cache, attn_metadata
|
|
||||||
)
|
|
||||||
physical_topk, valid_counts = triton_convert_req_index_to_global_index(
|
|
||||||
req_ids,
|
|
||||||
block_table,
|
|
||||||
topk[num_decode_tokens:],
|
|
||||||
BLOCK_SIZE=block_size,
|
|
||||||
NUM_TOPK_TOKENS=topk.shape[1],
|
|
||||||
return_valid_counts=True,
|
|
||||||
)
|
|
||||||
q_prefill = (
|
q_prefill = (
|
||||||
(q[0][num_decode_tokens:], q[1][num_decode_tokens:])
|
(q[0][num_decode_tokens:], q[1][num_decode_tokens:])
|
||||||
if isinstance(q, tuple)
|
if isinstance(q, tuple)
|
||||||
|
|||||||
@@ -122,6 +122,26 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada
|
|||||||
index_group = self.index_group
|
index_group = self.index_group
|
||||||
if isinstance(index_group, HiSparseMLAIndexGroup):
|
if isinstance(index_group, HiSparseMLAIndexGroup):
|
||||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||||
|
physical_kv_cache = index_group.physical_kv_cache(self.index_group_index)
|
||||||
|
if (
|
||||||
|
num_decode_tokens == num_actual_toks
|
||||||
|
or index_group.cache(self.index_group_index).all_context_pages_resident
|
||||||
|
):
|
||||||
|
topk_indices_physical = cast(
|
||||||
|
torch.Tensor,
|
||||||
|
index_group.convert_logical_to_physical_topk(
|
||||||
|
self.index_group_index,
|
||||||
|
topk_indices,
|
||||||
|
attn_metadata,
|
||||||
|
block_stride_rows=None,
|
||||||
|
return_valid_counts=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
self._run_mqa_kernel(q, physical_kv_cache, topk_indices_physical),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
outputs = []
|
outputs = []
|
||||||
if num_decode_tokens:
|
if num_decode_tokens:
|
||||||
topk_indices_physical = cast(
|
topk_indices_physical = cast(
|
||||||
@@ -137,51 +157,32 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada
|
|||||||
outputs.append(
|
outputs.append(
|
||||||
self._run_mqa_kernel(
|
self._run_mqa_kernel(
|
||||||
q[:num_decode_tokens],
|
q[:num_decode_tokens],
|
||||||
index_group.physical_kv_cache(self.index_group_index),
|
physical_kv_cache,
|
||||||
topk_indices_physical,
|
topk_indices_physical,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if num_decode_tokens < num_actual_toks:
|
prefill_cache, block_table, req_ids = index_group.stage_prefill_rows(
|
||||||
cache = index_group.cache(self.index_group_index)
|
self.index_group_index,
|
||||||
if num_decode_tokens == 0 and cache.all_context_pages_resident:
|
kv_c_and_k_pe_cache,
|
||||||
topk_indices_physical = cast(
|
attn_metadata,
|
||||||
torch.Tensor,
|
)
|
||||||
index_group.convert_logical_to_physical_topk(
|
topk_indices_physical = cast(
|
||||||
self.index_group_index,
|
torch.Tensor,
|
||||||
topk_indices,
|
triton_convert_req_index_to_global_index(
|
||||||
attn_metadata,
|
req_ids,
|
||||||
block_stride_rows=None,
|
block_table,
|
||||||
return_valid_counts=False,
|
topk_indices[num_decode_tokens:],
|
||||||
),
|
BLOCK_SIZE=attn_metadata.block_size,
|
||||||
)
|
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||||
prefill_cache = index_group.physical_kv_cache(
|
),
|
||||||
self.index_group_index
|
)
|
||||||
)
|
outputs.append(
|
||||||
else:
|
self._run_mqa_kernel(
|
||||||
prefill_cache, block_table, req_ids = (
|
q[num_decode_tokens:],
|
||||||
index_group.stage_prefill_rows(
|
prefill_cache,
|
||||||
self.index_group_index,
|
topk_indices_physical,
|
||||||
kv_c_and_k_pe_cache,
|
|
||||||
attn_metadata,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
topk_indices_physical = cast(
|
|
||||||
torch.Tensor,
|
|
||||||
triton_convert_req_index_to_global_index(
|
|
||||||
req_ids,
|
|
||||||
block_table,
|
|
||||||
topk_indices[num_decode_tokens:],
|
|
||||||
BLOCK_SIZE=attn_metadata.block_size,
|
|
||||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
|
||||||
),
|
|
||||||
)
|
|
||||||
outputs.append(
|
|
||||||
self._run_mqa_kernel(
|
|
||||||
q[num_decode_tokens:],
|
|
||||||
prefill_cache,
|
|
||||||
topk_indices_physical,
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
)
|
||||||
output = torch.cat(outputs) if len(outputs) > 1 else outputs[0]
|
output = torch.cat(outputs) if len(outputs) > 1 else outputs[0]
|
||||||
return output, None
|
return output, None
|
||||||
|
|
||||||
|
|||||||
@@ -852,6 +852,21 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
|||||||
if cache is not None:
|
if cache is not None:
|
||||||
assert isinstance(index_group, HiSparseMLAIndexGroup)
|
assert isinstance(index_group, HiSparseMLAIndexGroup)
|
||||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||||
|
if num_decode_tokens < q.shape[0] and cache.all_context_pages_resident:
|
||||||
|
physical_topk, lengths = index_group.convert_logical_to_physical_topk(
|
||||||
|
self.index_group_index,
|
||||||
|
topk_indices,
|
||||||
|
attn_metadata,
|
||||||
|
block_stride_rows=None,
|
||||||
|
return_valid_counts=True,
|
||||||
|
)
|
||||||
|
return self._bf16_flash_mla_kernel(
|
||||||
|
q,
|
||||||
|
index_group.physical_kv_cache(self.index_group_index),
|
||||||
|
physical_topk,
|
||||||
|
lengths,
|
||||||
|
actual_num_heads,
|
||||||
|
)
|
||||||
if num_decode_tokens > 0:
|
if num_decode_tokens > 0:
|
||||||
decode_topk, decode_lengths = (
|
decode_topk, decode_lengths = (
|
||||||
index_group.convert_logical_to_physical_topk(
|
index_group.convert_logical_to_physical_topk(
|
||||||
|
|||||||
@@ -233,7 +233,10 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup):
|
|||||||
cache = self.cache(layer_index)
|
cache = self.cache(layer_index)
|
||||||
num_tokens = logical_topk_indices.shape[0]
|
num_tokens = logical_topk_indices.shape[0]
|
||||||
req_id_per_token = attn_metadata.req_id_per_token[:num_tokens]
|
req_id_per_token = attn_metadata.req_id_per_token[:num_tokens]
|
||||||
if num_tokens > cache.runtime.max_swap_rows:
|
if num_tokens > cache.runtime.max_swap_rows or (
|
||||||
|
cache.all_context_pages_resident
|
||||||
|
and cache.num_decode_tokens < cache.num_actual_tokens
|
||||||
|
):
|
||||||
# Prefill-sized batches do not fit the decode residency workspace.
|
# Prefill-sized batches do not fit the decode residency workspace.
|
||||||
# Non-resident prefills are staged before reaching this path.
|
# Non-resident prefills are staged before reaching this path.
|
||||||
assert cache.all_context_pages_resident
|
assert cache.all_context_pages_resident
|
||||||
|
|||||||
Reference in New Issue
Block a user