[Perf][Attention] Remove D2H sync from FlashInfer SM90 sparse MLA plan under async scheduling (#58684)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-09-27 13:06:23 +02:00
committed by GitHub
parent 24c9772d19
commit c8d7a7dd13
2 changed files with 379 additions and 104 deletions
@@ -9,6 +9,7 @@ causality), ckv/kpe cache splitting, and the backend's model-shape gates.
"""
from types import SimpleNamespace
from typing import Any
import pytest
import torch
@@ -50,6 +51,8 @@ class FakeWrapper:
def __init__(self):
self.plan_args = None
self.run_args = None
self._plan_info: Any = None
self._planned_backend: Any = None
def plan(self, *args, **kwargs):
self.plan_args = (args, kwargs)
@@ -69,8 +72,11 @@ class FakeState:
self.kv_indptr = torch.zeros(max_tokens + 1, dtype=torch.int32)
self.wrapper = FakeWrapper()
self.plan_calls = []
self.index_topk = 2048
self.index_kpool = 4
self.max_valid = 2051
def plan(self, num_tokens, kv_lens):
def plan(self, num_tokens, kv_lens, cam, req_id_per_token):
self.plan_calls.append((num_tokens, kv_lens))
def pack_indices(self, slots):
@@ -168,13 +174,12 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_decodes):
"""MHA prefill rows must not make the MQA kernel read beyond its query."""
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
builder._async_scheduling = False
builder._adaptive_verification = False
builder.state = FakeState(TOPK)
builder._attention_layer = SimpleNamespace(_use_sparse_mha=lambda _: use_mha)
metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata)
metadata.state = None
metadata.req_id_per_token = None
metadata.num_prefills = 1
metadata.num_decode_tokens = num_decodes
monkeypatch.setattr(
@@ -206,12 +211,17 @@ def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_dec
def test_plan_uses_state_params(monkeypatch):
"""The NoPE/rope dims and scale live on the builder state, not the layer.
plan() takes exact per-row KV lengths; the schedule is rebuilt on every
call (contexts grow between steps) and the indptrs are always full-size
Without a device batch layout plan() takes exact per-row KV lengths and
rebuilds the schedule on every call; the indptrs are always full-size
with zero-query padding rows past num_tokens.
"""
impl, rows = make_impl(64, "auto")
wrapper = FakeWrapper()
plan_info = [0] * sm90_mod._PLAN_INFO_LEN
plan_info[sm90_mod._PI_KV_INDPTR] = 64 * 4
wrapper._plan_info = torch.tensor(plan_info)
wrapper._planned_backend = SimpleNamespace(
_int_workspace_buffer=torch.zeros(64, dtype=torch.int32)
)
state = sm90_mod._SM90State.__new__(sm90_mod._SM90State)
state.device = torch.device("cpu")
state.wrapper = wrapper
@@ -227,8 +237,9 @@ def test_plan_uses_state_params(monkeypatch):
state._qo_cpu = torch.empty(5, dtype=torch.int32)
state._kv_cpu = torch.empty(5, dtype=torch.int32)
state._lens_cpu = torch.full((4,), TOPK, dtype=torch.int32)
state._saved_kv_end = torch.empty(0, dtype=torch.int32)
state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32))
state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32), None, None)
assert wrapper.plan_args is not None
args, kwargs = wrapper.plan_args
(qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args
@@ -241,10 +252,10 @@ def test_plan_uses_state_params(monkeypatch):
assert kwargs["kv_data_type"] == torch.bfloat16
# Replanning a smaller batch must clear the previous rows' lengths.
state.plan(1, torch.tensor([TOPK], dtype=torch.int32))
state.plan(1, torch.tensor([TOPK], dtype=torch.int32), None, None)
assert state._kv_cpu.tolist() == [0, TOPK, TOPK, TOPK, TOPK]
assert state._lens_cpu.tolist() == [TOPK, 0, 0, 0]
state.plan(0, torch.empty(0, dtype=torch.int32))
state.plan(0, torch.empty(0, dtype=torch.int32), None, None)
assert state._kv_cpu.tolist() == [0, 0, 0, 0, 0]
assert state._lens_cpu.tolist() == [0, 0, 0, 0]
@@ -279,38 +290,49 @@ def test_pack_indices_replays_with_updated_offsets(width):
assert (state.kv_indices[expected.numel() :] == -99).all()
def test_kv_lens_host_formula():
@pytest.mark.parametrize("adaptive", [False, True])
def test_kv_lens_host_formula(adaptive):
"""Per-row host lengths: context == position + 1; capped at
index_topk + trailing-pool remainder past the sparse threshold."""
index_topk + trailing-pool remainder past the sparse threshold.
Exact (synced) without a host upper bound; with one, a sync-free bound
capped at the largest valid count, which the plan clamps on device.
"""
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
builder._async_scheduling = False
builder.state = FakeState(TOPK)
builder._adaptive_verification = adaptive
seq_lens = torch.tensor([100, 9, 3000], dtype=torch.int32)
cam = SimpleNamespace(
num_reqs=3,
query_start_loc_cpu=torch.tensor([0, 5, 7, 10], dtype=torch.int32),
seq_lens=torch.tensor([100, 9, 3000], dtype=torch.int32),
seq_lens_cpu_upper_bound=torch.tensor([100, 9, 3000], dtype=torch.int32),
positions=None,
seq_lens=seq_lens,
seq_lens_cpu_upper_bound=None,
)
num_rows, lens = builder._kv_lens_host(cam)
assert num_rows == 10
num_rows, lens, exact = builder._kv_lens_host(cam)
assert num_rows == 10 and exact
# req0: positions 95..99 -> ctx 96..100 (all <= 2048: full context)
# req1: positions 7,8 -> ctx 8,9
# req2: positions 2997..2999 -> ctx 2998..3000 (> 2048: topk + ctx%4)
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2050, 2051, 2048]
cam.seq_lens_cpu_upper_bound = seq_lens
num_rows, lens, exact = builder._kv_lens_host(cam)
assert num_rows == 10 and not exact
if adaptive:
# Host query split may differ from the device one: batch-wide bound.
assert lens.tolist() == [2051] * 10
else:
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2051, 2051, 2051]
def test_kv_lens_host_empty():
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
cam = SimpleNamespace(
num_reqs=0,
query_start_loc_cpu=torch.tensor([0], dtype=torch.int32),
seq_lens=torch.zeros(0, dtype=torch.int32),
)
num_rows, lens = builder._kv_lens_host(cam)
num_rows, lens, _ = builder._kv_lens_host(cam)
assert num_rows == 0 and lens.numel() == 0
@@ -334,7 +356,13 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
"__init__",
lambda *_args, **_kwargs: None,
)
monkeypatch.setattr(sm90_mod, "_SM90State", lambda *_args, **_kwargs: None)
class _RecordingState:
def __init__(self, *_args, **kwargs):
self.index_topk = kwargs["index_topk"]
self.index_kpool = kwargs["index_kpool"]
monkeypatch.setattr(sm90_mod, "_SM90State", _RecordingState)
impl, _ = make_impl(64)
spec = MLAAttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
@@ -351,6 +379,7 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
max_num_batched_tokens=64, async_scheduling=False
),
model_config=SimpleNamespace(hf_text_config=hf_config),
speculative_config=None,
)
builder = FlashInferMLASparseSM90Builder(
spec, ["attn"], vllm_config, torch.device("cpu")
@@ -359,14 +388,138 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
cam = SimpleNamespace(
num_reqs=2,
query_start_loc_cpu=torch.tensor([0, 4, 6], dtype=torch.int32),
seq_lens_cpu_upper_bound=torch.tensor([42295, 100], dtype=torch.int32),
positions=None,
seq_lens=torch.tensor([42295, 100], dtype=torch.int32),
seq_lens_cpu_upper_bound=None,
)
num_rows, lens = builder._kv_lens_host(cam)
assert num_rows == 6
num_rows, lens, exact = builder._kv_lens_host(cam)
assert num_rows == 6 and exact
assert lens.tolist() == prefill_lens + [99, 100]
def _sm90_flashinfer_available():
return (
torch.cuda.is_available()
and torch.cuda.get_device_capability()[0] == 9
and sm90_mod.has_flashinfer_sm90_nope_mla()
)
SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH = 16, 2048, 4, 2176
def _real_state(rows):
return sm90_mod._SM90State(
torch.device("cuda"),
SM90_HEADS,
torch.bfloat16,
rows,
SM90_WIDTH,
HEAD,
0,
HEAD**-0.5,
SM90_TOPK,
SM90_KPOOL,
)
def _clamp_batch(num_reqs, q_per_req, lo, hi, topk, kpool, width):
"""Random batch: per-row contexts, device layout, and top-k slots."""
last = torch.randint(lo + q_per_req, hi, (num_reqs,))
ctx = (last[:, None] + torch.arange(1 - q_per_req, 1)).reshape(-1)
valid = torch.where(ctx <= topk, ctx, topk + ctx % kpool)
cam = SimpleNamespace(
seq_lens=last.to(torch.int32).cuda(),
query_start_loc=(torch.arange(num_reqs + 1) * q_per_req).to(torch.int32).cuda(),
)
req_id = torch.arange(num_reqs, dtype=torch.int32).repeat_interleave(q_per_req)
slots = torch.randint(0, 1 << 16, (ctx.numel(), width), dtype=torch.int32)
slots[torch.arange(width) >= valid[:, None]] = -1
return ctx, valid.to(torch.int32), cam, req_id.cuda(), slots.cuda()
@pytest.mark.skipif(
not _sm90_flashinfer_available(), reason="Requires SM90 FlashInfer MLA"
)
@pytest.mark.parametrize(
"num_reqs,q_per_req,lo,hi",
[
(64, 1, 1, 3000),
(16, 6, 2030, 2070), # MTP-style rows straddling index_topk
(4096, 1, 1, 3000), # clamp spans several program blocks
],
)
def test_upper_bound_plan_matches_exact(num_reqs, q_per_req, lo, hi):
"""Sync-free planning from optimistic host bounds (async spec decode)
must match exact planning: the device clamp trims every work item back
to its row's valid count, so no -1 slot is read."""
torch.manual_seed(0)
heads, topk, kpool, width = SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH
ctx, valid, cam, req_id, slots = _clamp_batch(
num_reqs, q_per_req, lo, hi, topk, kpool, width
)
rows = ctx.numel()
kv = torch.randn(1 << 16, 1, HEAD, device="cuda", dtype=torch.bfloat16)
q = torch.randn(rows, heads, HEAD, device="cuda", dtype=torch.bfloat16)
q_pe = q.new_empty(rows, heads, 0)
def run(state):
state.pack_indices(slots)
return state.wrapper.run(q, q_pe, kv, kv[..., :0])
exact = _real_state(rows)
exact.plan(rows, valid, None, None)
ub = _real_state(rows)
host_bound = torch.clamp(ctx + torch.randint(0, 7, (rows,)), max=ub.max_valid)
ub.plan(rows, host_bound, cam, req_id)
out = run(ub)
assert not out.isnan().any()
torch.testing.assert_close(out, run(exact), atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not _sm90_flashinfer_available(), reason="Requires SM90 FlashInfer MLA"
)
def test_upper_bound_plan_reused_across_steps(monkeypatch):
"""Growing contexts (draft/decode steps) reuse the padded plan and only
re-clamp on device, matching an exact replan every step."""
torch.manual_seed(0)
heads, topk, kpool, width = SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH
rows = 48
ctx, _, _, req_id, _ = _clamp_batch(rows, 1, 1, 3000, topk, kpool, width)
exact, ub = _real_state(rows), _real_state(rows)
replans: list[int] = []
plan_fn = ub._plan
def counting_plan(num_tokens, kv_lens):
replans.append(num_tokens)
plan_fn(num_tokens, kv_lens)
monkeypatch.setattr(ub, "_plan", counting_plan)
kv = torch.randn(1 << 16, 1, HEAD, device="cuda", dtype=torch.bfloat16)
for step in range(5):
step_ctx = ctx + step
valid = torch.where(step_ctx <= topk, step_ctx, topk + step_ctx % kpool)
slots = torch.randint(0, 1 << 16, (rows, width), dtype=torch.int32)
slots[torch.arange(width) >= valid[:, None]] = -1
slots = slots.cuda()
cam = SimpleNamespace(
seq_lens=step_ctx.to(torch.int32).cuda(),
query_start_loc=torch.arange(rows + 1, dtype=torch.int32).cuda(),
)
num_plans = len(replans)
ub.plan(rows, torch.clamp(step_ctx + 5, max=ub.max_valid), cam, req_id)
assert len(replans) - num_plans == (1 if step == 0 else 0)
exact.plan(rows, valid.to(torch.int32), None, None)
q = torch.randn(rows, heads, HEAD, device="cuda", dtype=torch.bfloat16)
outs = []
for state in (ub, exact):
state.pack_indices(slots)
outs.append(state.wrapper.run(q, q[..., :0], kv, kv[..., :0]))
torch.testing.assert_close(outs[0], outs[1], atol=1e-2, rtol=1e-2)
def test_supports_combination_gates(monkeypatch, default_vllm_config):
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True)
call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination(