[Perf][Attention] Remove D2H sync from FlashInfer SM90 sparse MLA plan under async scheduling (#58684)

Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
Nicolò Lucchesi
2026-09-27 13:06:23 +02:00
committed by GitHub
parent 24c9772d19
commit c8d7a7dd13
2 changed files with 379 additions and 104 deletions
@@ -9,6 +9,7 @@ causality), ckv/kpe cache splitting, and the backend's model-shape gates.
""" """
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any
import pytest import pytest
import torch import torch
@@ -50,6 +51,8 @@ class FakeWrapper:
def __init__(self): def __init__(self):
self.plan_args = None self.plan_args = None
self.run_args = None self.run_args = None
self._plan_info: Any = None
self._planned_backend: Any = None
def plan(self, *args, **kwargs): def plan(self, *args, **kwargs):
self.plan_args = (args, kwargs) self.plan_args = (args, kwargs)
@@ -69,8 +72,11 @@ class FakeState:
self.kv_indptr = torch.zeros(max_tokens + 1, dtype=torch.int32) self.kv_indptr = torch.zeros(max_tokens + 1, dtype=torch.int32)
self.wrapper = FakeWrapper() self.wrapper = FakeWrapper()
self.plan_calls = [] self.plan_calls = []
self.index_topk = 2048
self.index_kpool = 4
self.max_valid = 2051
def plan(self, num_tokens, kv_lens): def plan(self, num_tokens, kv_lens, cam, req_id_per_token):
self.plan_calls.append((num_tokens, kv_lens)) self.plan_calls.append((num_tokens, kv_lens))
def pack_indices(self, slots): def pack_indices(self, slots):
@@ -168,13 +174,12 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_decodes): def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_decodes):
"""MHA prefill rows must not make the MQA kernel read beyond its query.""" """MHA prefill rows must not make the MQA kernel read beyond its query."""
builder = object.__new__(FlashInferMLASparseSM90Builder) builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048 builder._adaptive_verification = False
builder._index_kpool = 4
builder._async_scheduling = False
builder.state = FakeState(TOPK) builder.state = FakeState(TOPK)
builder._attention_layer = SimpleNamespace(_use_sparse_mha=lambda _: use_mha) builder._attention_layer = SimpleNamespace(_use_sparse_mha=lambda _: use_mha)
metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata) metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata)
metadata.state = None metadata.state = None
metadata.req_id_per_token = None
metadata.num_prefills = 1 metadata.num_prefills = 1
metadata.num_decode_tokens = num_decodes metadata.num_decode_tokens = num_decodes
monkeypatch.setattr( monkeypatch.setattr(
@@ -206,12 +211,17 @@ def test_builder_plans_only_rows_dispatched_to_mqa(monkeypatch, use_mha, num_dec
def test_plan_uses_state_params(monkeypatch): def test_plan_uses_state_params(monkeypatch):
"""The NoPE/rope dims and scale live on the builder state, not the layer. """The NoPE/rope dims and scale live on the builder state, not the layer.
plan() takes exact per-row KV lengths; the schedule is rebuilt on every Without a device batch layout plan() takes exact per-row KV lengths and
call (contexts grow between steps) and the indptrs are always full-size rebuilds the schedule on every call; the indptrs are always full-size
with zero-query padding rows past num_tokens. with zero-query padding rows past num_tokens.
""" """
impl, rows = make_impl(64, "auto")
wrapper = FakeWrapper() wrapper = FakeWrapper()
plan_info = [0] * sm90_mod._PLAN_INFO_LEN
plan_info[sm90_mod._PI_KV_INDPTR] = 64 * 4
wrapper._plan_info = torch.tensor(plan_info)
wrapper._planned_backend = SimpleNamespace(
_int_workspace_buffer=torch.zeros(64, dtype=torch.int32)
)
state = sm90_mod._SM90State.__new__(sm90_mod._SM90State) state = sm90_mod._SM90State.__new__(sm90_mod._SM90State)
state.device = torch.device("cpu") state.device = torch.device("cpu")
state.wrapper = wrapper state.wrapper = wrapper
@@ -227,8 +237,9 @@ def test_plan_uses_state_params(monkeypatch):
state._qo_cpu = torch.empty(5, dtype=torch.int32) state._qo_cpu = torch.empty(5, dtype=torch.int32)
state._kv_cpu = torch.empty(5, dtype=torch.int32) state._kv_cpu = torch.empty(5, dtype=torch.int32)
state._lens_cpu = torch.full((4,), TOPK, dtype=torch.int32) state._lens_cpu = torch.full((4,), TOPK, dtype=torch.int32)
state._saved_kv_end = torch.empty(0, dtype=torch.int32)
state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32)) state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32), None, None)
assert wrapper.plan_args is not None assert wrapper.plan_args is not None
args, kwargs = wrapper.plan_args args, kwargs = wrapper.plan_args
(qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args (qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args
@@ -241,10 +252,10 @@ def test_plan_uses_state_params(monkeypatch):
assert kwargs["kv_data_type"] == torch.bfloat16 assert kwargs["kv_data_type"] == torch.bfloat16
# Replanning a smaller batch must clear the previous rows' lengths. # Replanning a smaller batch must clear the previous rows' lengths.
state.plan(1, torch.tensor([TOPK], dtype=torch.int32)) state.plan(1, torch.tensor([TOPK], dtype=torch.int32), None, None)
assert state._kv_cpu.tolist() == [0, TOPK, TOPK, TOPK, TOPK] assert state._kv_cpu.tolist() == [0, TOPK, TOPK, TOPK, TOPK]
assert state._lens_cpu.tolist() == [TOPK, 0, 0, 0] assert state._lens_cpu.tolist() == [TOPK, 0, 0, 0]
state.plan(0, torch.empty(0, dtype=torch.int32)) state.plan(0, torch.empty(0, dtype=torch.int32), None, None)
assert state._kv_cpu.tolist() == [0, 0, 0, 0, 0] assert state._kv_cpu.tolist() == [0, 0, 0, 0, 0]
assert state._lens_cpu.tolist() == [0, 0, 0, 0] assert state._lens_cpu.tolist() == [0, 0, 0, 0]
@@ -279,38 +290,49 @@ def test_pack_indices_replays_with_updated_offsets(width):
assert (state.kv_indices[expected.numel() :] == -99).all() assert (state.kv_indices[expected.numel() :] == -99).all()
def test_kv_lens_host_formula(): @pytest.mark.parametrize("adaptive", [False, True])
def test_kv_lens_host_formula(adaptive):
"""Per-row host lengths: context == position + 1; capped at """Per-row host lengths: context == position + 1; capped at
index_topk + trailing-pool remainder past the sparse threshold.""" index_topk + trailing-pool remainder past the sparse threshold.
Exact (synced) without a host upper bound; with one, a sync-free bound
capped at the largest valid count, which the plan clamps on device.
"""
builder = object.__new__(FlashInferMLASparseSM90Builder) builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048 builder.state = FakeState(TOPK)
builder._index_kpool = 4 builder._adaptive_verification = adaptive
builder._async_scheduling = False seq_lens = torch.tensor([100, 9, 3000], dtype=torch.int32)
cam = SimpleNamespace( cam = SimpleNamespace(
num_reqs=3, num_reqs=3,
query_start_loc_cpu=torch.tensor([0, 5, 7, 10], dtype=torch.int32), query_start_loc_cpu=torch.tensor([0, 5, 7, 10], dtype=torch.int32),
seq_lens=torch.tensor([100, 9, 3000], dtype=torch.int32), seq_lens=seq_lens,
seq_lens_cpu_upper_bound=torch.tensor([100, 9, 3000], dtype=torch.int32), seq_lens_cpu_upper_bound=None,
positions=None,
) )
num_rows, lens = builder._kv_lens_host(cam) num_rows, lens, exact = builder._kv_lens_host(cam)
assert num_rows == 10 assert num_rows == 10 and exact
# req0: positions 95..99 -> ctx 96..100 (all <= 2048: full context) # req0: positions 95..99 -> ctx 96..100 (all <= 2048: full context)
# req1: positions 7,8 -> ctx 8,9 # req1: positions 7,8 -> ctx 8,9
# req2: positions 2997..2999 -> ctx 2998..3000 (> 2048: topk + ctx%4) # req2: positions 2997..2999 -> ctx 2998..3000 (> 2048: topk + ctx%4)
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2050, 2051, 2048] assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2050, 2051, 2048]
cam.seq_lens_cpu_upper_bound = seq_lens
num_rows, lens, exact = builder._kv_lens_host(cam)
assert num_rows == 10 and not exact
if adaptive:
# Host query split may differ from the device one: batch-wide bound.
assert lens.tolist() == [2051] * 10
else:
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2051, 2051, 2051]
def test_kv_lens_host_empty(): def test_kv_lens_host_empty():
builder = object.__new__(FlashInferMLASparseSM90Builder) builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
cam = SimpleNamespace( cam = SimpleNamespace(
num_reqs=0, num_reqs=0,
query_start_loc_cpu=torch.tensor([0], dtype=torch.int32), query_start_loc_cpu=torch.tensor([0], dtype=torch.int32),
seq_lens=torch.zeros(0, dtype=torch.int32), seq_lens=torch.zeros(0, dtype=torch.int32),
) )
num_rows, lens = builder._kv_lens_host(cam) num_rows, lens, _ = builder._kv_lens_host(cam)
assert num_rows == 0 and lens.numel() == 0 assert num_rows == 0 and lens.numel() == 0
@@ -334,7 +356,13 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
"__init__", "__init__",
lambda *_args, **_kwargs: None, lambda *_args, **_kwargs: None,
) )
monkeypatch.setattr(sm90_mod, "_SM90State", lambda *_args, **_kwargs: None)
class _RecordingState:
def __init__(self, *_args, **kwargs):
self.index_topk = kwargs["index_topk"]
self.index_kpool = kwargs["index_kpool"]
monkeypatch.setattr(sm90_mod, "_SM90State", _RecordingState)
impl, _ = make_impl(64) impl, _ = make_impl(64)
spec = MLAAttentionSpec( spec = MLAAttentionSpec(
block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16 block_size=BLOCK_SIZE, num_kv_heads=1, head_size=576, dtype=torch.bfloat16
@@ -351,6 +379,7 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
max_num_batched_tokens=64, async_scheduling=False max_num_batched_tokens=64, async_scheduling=False
), ),
model_config=SimpleNamespace(hf_text_config=hf_config), model_config=SimpleNamespace(hf_text_config=hf_config),
speculative_config=None,
) )
builder = FlashInferMLASparseSM90Builder( builder = FlashInferMLASparseSM90Builder(
spec, ["attn"], vllm_config, torch.device("cpu") spec, ["attn"], vllm_config, torch.device("cpu")
@@ -359,14 +388,138 @@ def test_builder_kpool_from_model_config(monkeypatch, index_kpool, prefill_lens)
cam = SimpleNamespace( cam = SimpleNamespace(
num_reqs=2, num_reqs=2,
query_start_loc_cpu=torch.tensor([0, 4, 6], dtype=torch.int32), query_start_loc_cpu=torch.tensor([0, 4, 6], dtype=torch.int32),
seq_lens_cpu_upper_bound=torch.tensor([42295, 100], dtype=torch.int32), seq_lens=torch.tensor([42295, 100], dtype=torch.int32),
positions=None, seq_lens_cpu_upper_bound=None,
) )
num_rows, lens = builder._kv_lens_host(cam) num_rows, lens, exact = builder._kv_lens_host(cam)
assert num_rows == 6 assert num_rows == 6 and exact
assert lens.tolist() == prefill_lens + [99, 100] assert lens.tolist() == prefill_lens + [99, 100]
def _sm90_flashinfer_available():
return (
torch.cuda.is_available()
and torch.cuda.get_device_capability()[0] == 9
and sm90_mod.has_flashinfer_sm90_nope_mla()
)
SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH = 16, 2048, 4, 2176
def _real_state(rows):
return sm90_mod._SM90State(
torch.device("cuda"),
SM90_HEADS,
torch.bfloat16,
rows,
SM90_WIDTH,
HEAD,
0,
HEAD**-0.5,
SM90_TOPK,
SM90_KPOOL,
)
def _clamp_batch(num_reqs, q_per_req, lo, hi, topk, kpool, width):
"""Random batch: per-row contexts, device layout, and top-k slots."""
last = torch.randint(lo + q_per_req, hi, (num_reqs,))
ctx = (last[:, None] + torch.arange(1 - q_per_req, 1)).reshape(-1)
valid = torch.where(ctx <= topk, ctx, topk + ctx % kpool)
cam = SimpleNamespace(
seq_lens=last.to(torch.int32).cuda(),
query_start_loc=(torch.arange(num_reqs + 1) * q_per_req).to(torch.int32).cuda(),
)
req_id = torch.arange(num_reqs, dtype=torch.int32).repeat_interleave(q_per_req)
slots = torch.randint(0, 1 << 16, (ctx.numel(), width), dtype=torch.int32)
slots[torch.arange(width) >= valid[:, None]] = -1
return ctx, valid.to(torch.int32), cam, req_id.cuda(), slots.cuda()
@pytest.mark.skipif(
not _sm90_flashinfer_available(), reason="Requires SM90 FlashInfer MLA"
)
@pytest.mark.parametrize(
"num_reqs,q_per_req,lo,hi",
[
(64, 1, 1, 3000),
(16, 6, 2030, 2070), # MTP-style rows straddling index_topk
(4096, 1, 1, 3000), # clamp spans several program blocks
],
)
def test_upper_bound_plan_matches_exact(num_reqs, q_per_req, lo, hi):
"""Sync-free planning from optimistic host bounds (async spec decode)
must match exact planning: the device clamp trims every work item back
to its row's valid count, so no -1 slot is read."""
torch.manual_seed(0)
heads, topk, kpool, width = SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH
ctx, valid, cam, req_id, slots = _clamp_batch(
num_reqs, q_per_req, lo, hi, topk, kpool, width
)
rows = ctx.numel()
kv = torch.randn(1 << 16, 1, HEAD, device="cuda", dtype=torch.bfloat16)
q = torch.randn(rows, heads, HEAD, device="cuda", dtype=torch.bfloat16)
q_pe = q.new_empty(rows, heads, 0)
def run(state):
state.pack_indices(slots)
return state.wrapper.run(q, q_pe, kv, kv[..., :0])
exact = _real_state(rows)
exact.plan(rows, valid, None, None)
ub = _real_state(rows)
host_bound = torch.clamp(ctx + torch.randint(0, 7, (rows,)), max=ub.max_valid)
ub.plan(rows, host_bound, cam, req_id)
out = run(ub)
assert not out.isnan().any()
torch.testing.assert_close(out, run(exact), atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not _sm90_flashinfer_available(), reason="Requires SM90 FlashInfer MLA"
)
def test_upper_bound_plan_reused_across_steps(monkeypatch):
"""Growing contexts (draft/decode steps) reuse the padded plan and only
re-clamp on device, matching an exact replan every step."""
torch.manual_seed(0)
heads, topk, kpool, width = SM90_HEADS, SM90_TOPK, SM90_KPOOL, SM90_WIDTH
rows = 48
ctx, _, _, req_id, _ = _clamp_batch(rows, 1, 1, 3000, topk, kpool, width)
exact, ub = _real_state(rows), _real_state(rows)
replans: list[int] = []
plan_fn = ub._plan
def counting_plan(num_tokens, kv_lens):
replans.append(num_tokens)
plan_fn(num_tokens, kv_lens)
monkeypatch.setattr(ub, "_plan", counting_plan)
kv = torch.randn(1 << 16, 1, HEAD, device="cuda", dtype=torch.bfloat16)
for step in range(5):
step_ctx = ctx + step
valid = torch.where(step_ctx <= topk, step_ctx, topk + step_ctx % kpool)
slots = torch.randint(0, 1 << 16, (rows, width), dtype=torch.int32)
slots[torch.arange(width) >= valid[:, None]] = -1
slots = slots.cuda()
cam = SimpleNamespace(
seq_lens=step_ctx.to(torch.int32).cuda(),
query_start_loc=torch.arange(rows + 1, dtype=torch.int32).cuda(),
)
num_plans = len(replans)
ub.plan(rows, torch.clamp(step_ctx + 5, max=ub.max_valid), cam, req_id)
assert len(replans) - num_plans == (1 if step == 0 else 0)
exact.plan(rows, valid.to(torch.int32), None, None)
q = torch.randn(rows, heads, HEAD, device="cuda", dtype=torch.bfloat16)
outs = []
for state in (ub, exact):
state.pack_indices(slots)
outs.append(state.wrapper.run(q, q[..., :0], kv, kv[..., :0]))
torch.testing.assert_close(outs[0], outs[1], atol=1e-2, rtol=1e-2)
def test_supports_combination_gates(monkeypatch, default_vllm_config): def test_supports_combination_gates(monkeypatch, default_vllm_config):
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True) monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True)
call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination( call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination(
@@ -17,14 +17,14 @@ encoded by the indexer's selection, so ``causal=False``.
CUDA-graph handling: ``plan()`` copies its inputs to host unconditionally, CUDA-graph handling: ``plan()`` copies its inputs to host unconditionally,
so it must stay outside graph capture. Each metadata builder owns a wrapper, so it must stay outside graph capture. Each metadata builder owns a wrapper,
reserved capture-stable device buffers, and the plan parameters. The wrapper reserved capture-stable device buffers, and the plan parameters. The wrapper
bakes the per-row ``kv_len`` into its int schedule bakes the per-row ``kv_len`` into its int schedule at plan() time — ``run()``
at plan() time — ``run()`` never reads the device-side buffer — so the never reads the device-side buffer. The builder plans (outside capture) from
metadata builder replans every step (outside capture) with exact host-side sync-free host upper bounds and clamps each scheduled work item's ``kv_end``
lengths derived from the batch's sequence lengths; a full-width schedule on device to the exact valid count, so the kernel never reads past a row's
would send the kernel past each row's valid count into the -1 tail of the valid prefix (the -1 tail of the converted index buffer) and async scheduling
converted index buffer (illegal address). Per-step content (top-k slots) needs no D2H sync. Per-step content (top-k slots) is written into the
is written into the reserved buffers by kernels inside the captured reserved buffers by kernels inside the captured forward, and captured runs
forward, and captured runs read the refreshed plan buffers on replay. read the refreshed plan buffers on replay.
KV cache format: plain contiguous E4M3 ``[num_blocks, block_size, 512]`` KV cache format: plain contiguous E4M3 ``[num_blocks, block_size, 512]``
(uint8 storage) with a per-tensor ``k_scale``; BF16 caches also work. The (uint8 storage) with a per-tensor ``k_scale``; BF16 caches also work. The
@@ -63,6 +63,26 @@ from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
_FP8_KV_DTYPES = ("fp8", "fp8_e4m3") _FP8_KV_DTYPES = ("fp8", "fp8_e4m3")
_WORKSPACE_BYTES = 128 * 1024 * 1024 _WORKSPACE_BYTES = 128 * 1024 * 1024
# TODO: FlashInfer bakes per-row kv_len into the plan on the host, which
# forces a D2H sync under async scheduling. The plan-info layout constants
# below and the int-workspace clamp in _SM90State work around this by
# planning from sync-free upper bounds; delete them once FlashInfer accepts
# an upper-bound plan() that clamps kv_end internally, or exposes a
# GPU-side plan API (upstream ask:
# https://github.com/flashinfer-ai/flashinfer/issues/5559, see
# vllm-project/vllm#58684).
# FlashInfer MLAPlanInfo layout (scheduler.cuh) field indices.
_PLAN_INFO_LEN = 18
_PI_NUM_BLKS_Y = 1
_PI_Q_INDPTR = 2
_PI_KV_INDPTR = 3
_PI_KV_START = 13
_PI_KV_END = 14
_PI_WORK_INDPTR = 15
# Planned lengths exceed the host upper bound by up to this many tokens so a
# plan stays reusable across draft steps and consecutive decode steps; the
# device-side clamp restores the exact lengths.
_PLAN_SLACK = 32
@triton.jit @triton.jit
@@ -82,6 +102,40 @@ def _pack_topk_indices(
tl.store(indices + start + cols, tl.maximum(values, 0), mask=mask) tl.store(indices + start + cols, tl.maximum(values, 0), mask=mask)
@triton.jit
def _clamp_work_kv_end(
int_ws,
saved_kv_end,
seq_lens,
query_start_loc,
req_id_per_token,
num_works_idx,
q_indptr_off,
kv_start_off,
kv_end_off,
TOPK: tl.constexpr,
KPOOL: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
# Clamp each planned work item to its row's exact valid count, derived
# from the device batch layout exactly as the convert kernel counts it.
works = tl.program_id(0) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask = works < tl.load(int_ws + num_works_idx)
row = tl.load(int_ws + q_indptr_off + works, mask=mask, other=0)
req = tl.load(req_id_per_token + row, mask=mask, other=0)
seq_len = tl.load(seq_lens + req, mask=mask, other=0)
query_end = tl.load(query_start_loc + req + 1, mask=mask, other=0)
ctx = seq_len - query_end + row + 1
valid = tl.where(ctx <= TOPK, ctx, TOPK + ctx % KPOOL)
start = tl.load(int_ws + kv_start_off + works, mask=mask, other=0)
end = tl.load(saved_kv_end + works, mask=mask, other=0)
tl.store(
int_ws + kv_end_off + works,
tl.maximum(tl.minimum(end, valid), start),
mask=mask,
)
class FlashInferMLASparseSM90Backend(AttentionBackend): class FlashInferMLASparseSM90Backend(AttentionBackend):
supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16]
supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [ supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = [
@@ -190,6 +244,8 @@ class _SM90State:
kv_lora_rank: int, kv_lora_rank: int,
qk_rope_head_dim: int, qk_rope_head_dim: int,
sm_scale: float, sm_scale: float,
index_topk: int,
index_kpool: int,
) -> None: ) -> None:
from flashinfer.mla import BatchMLAPagedAttentionWrapper from flashinfer.mla import BatchMLAPagedAttentionWrapper
@@ -202,6 +258,10 @@ class _SM90State:
self.kv_lora_rank = kv_lora_rank self.kv_lora_rank = kv_lora_rank
self.qk_rope_head_dim = qk_rope_head_dim self.qk_rope_head_dim = qk_rope_head_dim
self.sm_scale = sm_scale self.sm_scale = sm_scale
self.index_topk = index_topk
self.index_kpool = max(index_kpool, 1)
# Largest valid count the convert kernel can produce for any context.
self.max_valid = min(self.index_topk + self.index_kpool - 1, topk_width)
# User-reserved buffers: with use_cuda_graph=True plan() refreshes # User-reserved buffers: with use_cuda_graph=True plan() refreshes
# these in place, so run()'s captured kernels always read them. # these in place, so run()'s captured kernels always read them.
self.kv_indices = torch.zeros( self.kv_indices = torch.zeros(
@@ -226,24 +286,54 @@ class _SM90State:
self._lens_cpu = torch.full( self._lens_cpu = torch.full(
(self.max_tokens,), self.topk_width, dtype=torch.int32 (self.max_tokens,), self.topk_width, dtype=torch.int32
) )
# Planned per-work kv_end of the current plan, restored by each clamp;
# sized on first plan to the planner's work capacity.
self._saved_kv_end = torch.empty(0, dtype=torch.int32, device=device)
self._planned_num_tokens = -1
self._int_ws: torch.Tensor | None = None
self._plan_offsets: tuple[int, int, int, int] | None = None
def plan(self, num_tokens: int, kv_lens: torch.Tensor) -> None: def plan(
"""Replan with exact per-row KV lengths (CPU int32, ``[num_tokens]``). self,
num_tokens: int,
kv_lens: torch.Tensor,
cam: CommonAttentionMetadata | None,
req_id_per_token: torch.Tensor | None,
) -> None:
"""Plan per-row KV lengths (CPU int32, ``[num_tokens]``).
The wrapper bakes kv_len into its int schedule from host values; The wrapper bakes kv_len into its int schedule from host values, so
run() never reads the device kv_len_arr buffer. Scheduling with the rows past their valid count would send the kernel into the -1 tail of
full buffer width while kv_indices rows carry a -1 tail past each the converted index buffer. With ``cam`` and ``req_id_per_token``
row's valid count makes the kernel compute ``-1 * ckv_stride_page`` (device batch layout), ``kv_lens`` only needs to upper-bound the
(illegal address), so the lengths must be exact at plan time and valid counts: the plan is padded by ``_PLAN_SLACK`` and reused while
replanned every step as contexts grow. Must run outside CUDA graph the bound still fits, and every call clamps each work item's kv_end
capture: the in-place refreshed plan_info/indptr buffers are what on device to the exact count, so no D2H sync is needed. Without them
captured runs read. the lengths must be exact. Must run outside CUDA graph capture: the
in-place refreshed plan_info/indptr buffers are what captured runs
read.
""" """
if torch.cuda.is_current_stream_capturing(): if torch.cuda.is_current_stream_capturing():
raise RuntimeError( raise RuntimeError(
"FlashInferMLASparseSM90 plan() called inside CUDA graph " "FlashInferMLASparseSM90 plan() called inside CUDA graph "
"capture; lengths must be planned host-side before capture." "capture; lengths must be planned host-side before capture."
) )
kv_lens = kv_lens.to(torch.int32)
if cam is None or req_id_per_token is None:
self._plan(num_tokens, kv_lens)
self._planned_num_tokens = -1
return
planned = self._lens_cpu[:num_tokens]
if num_tokens != self._planned_num_tokens or not bool(
((planned >= kv_lens) & (planned <= kv_lens + _PLAN_SLACK)).all()
):
self._plan(
num_tokens, torch.clamp(kv_lens + _PLAN_SLACK, max=self.max_valid)
)
self._planned_num_tokens = num_tokens
self._clamp_kv_end(cam, req_id_per_token)
def _plan(self, num_tokens: int, kv_lens: torch.Tensor) -> None:
# CPU staging buffers are filled in place: plan() runs per step # CPU staging buffers are filled in place: plan() runs per step
# (once per draft/verify metadata build), so per-call allocations and # (once per draft/verify metadata build), so per-call allocations and
# device round trips are on the hot path. Passing CPU tensors lets # device round trips are on the hot path. Passing CPU tensors lets
@@ -257,13 +347,16 @@ class _SM90State:
# matching the exact lengths, including zero-length padded rows. # matching the exact lengths, including zero-length padded rows.
torch.clamp(self._arange_cpu, max=num_tokens, out=self._qo_cpu) torch.clamp(self._arange_cpu, max=num_tokens, out=self._qo_cpu)
self._lens_cpu.zero_() self._lens_cpu.zero_()
self._lens_cpu[:num_tokens] = kv_lens.to(torch.int32) self._lens_cpu[:num_tokens] = kv_lens
self._kv_cpu[0] = 0 self._kv_cpu[0] = 0
torch.cumsum(self._lens_cpu, dim=0, out=self._kv_cpu[1:]) torch.cumsum(self._lens_cpu, dim=0, out=self._kv_cpu[1:])
# Only the live prefix of kv_indices: the wrapper snapshots and
# copies whatever is passed.
num_indices = max(int(self._kv_cpu[-1]), 1)
self.wrapper.plan( self.wrapper.plan(
self._qo_cpu, self._qo_cpu,
self._kv_cpu, self._kv_cpu,
self.kv_indices, self.kv_indices[:num_indices],
self._lens_cpu, self._lens_cpu,
self.num_heads, self.num_heads,
self.kv_lora_rank, # head_dim_ckv self.kv_lora_rank, # head_dim_ckv
@@ -274,6 +367,42 @@ class _SM90State:
q_data_type=torch.bfloat16, q_data_type=torch.bfloat16,
kv_data_type=self.kv_dtype, kv_data_type=self.kv_dtype,
) )
plan_info = [int(x) for x in self.wrapper._plan_info]
assert len(plan_info) == _PLAN_INFO_LEN, plan_info
# Per-work arrays are laid out back to back at the planner's capacity.
max_works = (plan_info[_PI_KV_INDPTR] - plan_info[_PI_Q_INDPTR]) // 4
if self._saved_kv_end.numel() != max_works:
self._saved_kv_end = torch.empty(
max_works, dtype=torch.int32, device=self.device
)
self._int_ws = self.wrapper._planned_backend._int_workspace_buffer.view(
torch.int32
)
self._plan_offsets = (
plan_info[_PI_WORK_INDPTR] // 4 + plan_info[_PI_NUM_BLKS_Y],
plan_info[_PI_Q_INDPTR] // 4,
plan_info[_PI_KV_START] // 4,
plan_info[_PI_KV_END] // 4,
)
kv_end_off = self._plan_offsets[3]
self._saved_kv_end.copy_(self._int_ws[kv_end_off : kv_end_off + max_works])
def _clamp_kv_end(
self, cam: CommonAttentionMetadata, req_id_per_token: torch.Tensor
) -> None:
assert self._int_ws is not None and self._plan_offsets is not None
block = 1024
_clamp_work_kv_end[(triton.cdiv(self._saved_kv_end.numel(), block),)](
self._int_ws,
self._saved_kv_end,
cam.seq_lens,
cam.query_start_loc,
req_id_per_token,
*self._plan_offsets,
TOPK=self.index_topk,
KPOOL=self.index_kpool,
BLOCK_SIZE=block,
)
def pack_indices(self, topk_slots: torch.Tensor) -> None: def pack_indices(self, topk_slots: torch.Tensor) -> None:
num_tokens, width = topk_slots.shape num_tokens, width = topk_slots.shape
@@ -312,6 +441,8 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
) )
topk_indices_buffer = impl.topk_indices_buffer topk_indices_buffer = impl.topk_indices_buffer
assert topk_indices_buffer is not None assert topk_indices_buffer is not None
hf_config = vllm_config.model_config.hf_text_config
assert hf_config.index_topk is not None
self.state = _SM90State( self.state = _SM90State(
device, device,
impl.num_heads, impl.num_heads,
@@ -321,20 +452,18 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
kv_lora_rank=impl.kv_lora_rank, kv_lora_rank=impl.kv_lora_rank,
qk_rope_head_dim=impl.qk_rope_head_dim, qk_rope_head_dim=impl.qk_rope_head_dim,
sm_scale=impl.scale, sm_scale=impl.scale,
index_topk=int(hf_config.index_topk),
index_kpool=int(getattr(hf_config, "index_kpool", 1) or 1),
)
spec_config = vllm_config.speculative_config
self._adaptive_verification = bool(
spec_config is not None and spec_config.enable_adaptive_verification
) )
# seq_lens_cpu_upper_bound is optimistic on decode rows under async
# spec decode, so the fast sync-free path is only safe without it;
# under async scheduling the exact (device) positions are used at
# the cost of one D2H sync per metadata build.
self._async_scheduling = bool(vllm_config.scheduler_config.async_scheduling)
hf_config = vllm_config.model_config.hf_text_config
assert hf_config.index_topk is not None
self._index_topk = int(hf_config.index_topk)
self._index_kpool = int(getattr(hf_config, "index_kpool", 1) or 1)
def _kv_lens_host(self, cam: CommonAttentionMetadata) -> tuple[int, torch.Tensor]: def _kv_lens_host(
"""Exact per-row KV lengths, host-side (the flashinfer wrapper bakes self, cam: CommonAttentionMetadata
them into its schedule at plan time; there is no device-side path). ) -> tuple[int, torch.Tensor, bool]:
"""Host per-row KV lengths for the plan and whether they are exact.
A row for the j-th query token of request i attends A row for the j-th query token of request i attends
``seq_lens[i] - q_len[i] + j + 1`` tokens. The indexer's selection ``seq_lens[i] - q_len[i] + j + 1`` tokens. The indexer's selection
@@ -343,53 +472,41 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
``index_topk`` pool-expanded tokens plus the trailing incomplete ``index_topk`` pool-expanded tokens plus the trailing incomplete
pool (valid == ``index_topk + context % index_kpool``). Both match pool (valid == ``index_topk + context % index_kpool``). Both match
the count of non -1 entries the convert kernel produces. the count of non -1 entries the convert kernel produces.
With ``seq_lens_cpu_upper_bound`` (optimistic under async spec
decode) returns sync-free upper bounds, which the plan clamps on
device. Otherwise returns exact lengths via a D2H copy.
""" """
num_reqs = cam.num_reqs num_reqs = cam.num_reqs
qsl = cam.query_start_loc_cpu[: num_reqs + 1] qsl = cam.query_start_loc_cpu[: num_reqs + 1]
num_rows = int(qsl[-1]) num_rows = int(qsl[-1])
if num_rows == 0: if num_rows == 0:
return 0, torch.zeros(0, dtype=torch.int32) return 0, torch.zeros(0, dtype=torch.int32), True
# Row context == position + 1. Without async scheduling the state = self.state
# maintained host upper bound is exact, giving a sync-free path;
# under async scheduling it is optimistic on decode rows, so fall
# back to the exact device positions (one D2H sync per build). The
# seq_lens derivation equals position + 1 because positions are
# contiguous per request.
sl_host = cam.seq_lens_cpu_upper_bound sl_host = cam.seq_lens_cpu_upper_bound
positions = cam.positions if sl_host is None:
if not self._async_scheduling and sl_host is not None: ctx = self._row_contexts(qsl, cam.seq_lens[:num_reqs].cpu())
seq_lens = sl_host[:num_reqs].to(torch.int32) topk, kpool = state.index_topk, state.index_kpool
q_lens = qsl[1:] - qsl[:-1] lens = torch.where(ctx <= topk, ctx, topk + ctx % kpool)
first_pos = seq_lens - q_lens return num_rows, lens.to(torch.int32), True
req_of_row = torch.repeat_interleave( if self._adaptive_verification:
torch.arange(num_reqs, dtype=torch.int64), q_lens.to(torch.int64) # The host query split may not match the device one, so bound
) # every row by the longest request.
rows = torch.arange(num_rows, dtype=torch.int32) ctx = torch.full((num_rows,), int(sl_host[:num_reqs].max()))
ctx = (
first_pos.to(torch.int64)[req_of_row]
+ rows.to(torch.int64)
- qsl.to(torch.int64)[req_of_row]
+ 1
)
elif positions is not None and num_rows <= positions.shape[0]:
ctx = positions[:num_rows].cpu().to(torch.int64) + 1
else: else:
seq_lens = cam.seq_lens[:num_reqs].cpu().to(torch.int32) ctx = self._row_contexts(qsl, sl_host[:num_reqs])
q_lens = qsl[1:] - qsl[:-1] return num_rows, torch.clamp(ctx, max=state.max_valid), False
first_pos = seq_lens - q_lens
req_of_row = torch.repeat_interleave( @staticmethod
torch.arange(num_reqs, dtype=torch.int64), q_lens.to(torch.int64) def _row_contexts(qsl: torch.Tensor, seq_lens: torch.Tensor) -> torch.Tensor:
) num_reqs = qsl.shape[0] - 1
rows = torch.arange(num_rows, dtype=torch.int32) q_lens = (qsl[1:] - qsl[:-1]).to(torch.int64)
ctx = ( first_ctx = seq_lens.to(torch.int64) - q_lens + 1
first_pos.to(torch.int64)[req_of_row] req_of_row = torch.repeat_interleave(
+ rows.to(torch.int64) torch.arange(num_reqs, dtype=torch.int64), q_lens
- qsl.to(torch.int64)[req_of_row] )
+ 1 rows = torch.arange(int(qsl[-1]), dtype=torch.int64)
) return first_ctx[req_of_row] + rows - qsl.to(torch.int64)[req_of_row]
topk = self._index_topk
lens = torch.where(ctx <= topk, ctx, topk + ctx % self._index_kpool)
return num_rows, lens.to(torch.int32)
def build( def build(
self, self,
@@ -399,9 +516,9 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
) -> FlashInferMLASparseSM90Metadata: ) -> FlashInferMLASparseSM90Metadata:
metadata = super().build(common_prefix_len, common_attn_metadata, fast_build) metadata = super().build(common_prefix_len, common_attn_metadata, fast_build)
assert isinstance(metadata, FlashInferMLASparseSM90Metadata) assert isinstance(metadata, FlashInferMLASparseSM90Metadata)
# Replan every step outside any CUDA graph capture with this step's # Plan outside any CUDA graph capture; captured runs read the
# exact per-row lengths; captured runs read the refreshed buffers. # refreshed buffers.
num_rows, kv_lens = self._kv_lens_host(common_attn_metadata) num_rows, kv_lens, exact = self._kv_lens_host(common_attn_metadata)
# MHA prefills are removed from q before forward_mqa. Plan only the # MHA prefills are removed from q before forward_mqa. Plan only the
# decode prefix in that case, using the same routing decision as MLA. # decode prefix in that case, using the same routing decision as MLA.
if metadata.num_prefills > 0 and self._attention_layer._use_sparse_mha( if metadata.num_prefills > 0 and self._attention_layer._use_sparse_mha(
@@ -409,7 +526,12 @@ class FlashInferMLASparseSM90Builder(FlashInferMLASparseMetadataBuilder):
): ):
num_rows = metadata.num_decode_tokens num_rows = metadata.num_decode_tokens
kv_lens = kv_lens[:num_rows] kv_lens = kv_lens[:num_rows]
self.state.plan(num_rows, kv_lens) if exact:
self.state.plan(num_rows, kv_lens, None, None)
else:
self.state.plan(
num_rows, kv_lens, common_attn_metadata, metadata.req_id_per_token
)
metadata.state = self.state metadata.state = self.state
return metadata return metadata