[Perf][GDN] Internal prefill checkpoints for the FlashInfer GDN prefill backend (#60050)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jiangyun Zhu
2026-10-14 14:49:46 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent a2fb482732
commit badfcdbf91
12 changed files with 794 additions and 151 deletions
@@ -13,10 +13,19 @@ if current_platform.is_rocm():
)
import flashinfer.gdn_prefill # noqa: E402
import torch.nn.functional as F # noqa: E402
from vllm.model_executor.layers.mamba.checkpoint import ( # noqa: E402
MambaPrefillCheckpointExporter,
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
fi_chunk_gated_delta_rule,
) # noqa: E402
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
plan_flashinfer_checkpoints,
)
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID # noqa: E402
def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
@@ -51,3 +60,130 @@ def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
assert captured_cu_seqlens.dtype == torch.int64
assert output.shape == q.shape
assert final_state is None
@pytest.mark.skipif(
not (
current_platform.is_device_capability(90)
or current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
),
reason="FlashInfer GDN prefill checkpoints need SM90, SM100 or SM120.",
)
@pytest.mark.parametrize("use_cp", [True, False])
@pytest.mark.parametrize("ssm_dtype", [torch.float32, torch.bfloat16])
def test_flashinfer_checkpoints_export_the_state_at_each_offset(use_cp, ssm_dtype):
"""Each exported checkpoint is the state of the prefill stopped there.
The leading decode row is outside the prefill kernel. The two checkpointed
prefills put FlashInfer's stride (their offsets' gcd) below both offsets,
so the exporter must pick each state out of the shared emitted buffer;
the third prefill takes no checkpoint. Both FlashInfer routes are covered
because ``use_cp="auto"`` may take either. The conv cache has MTP's spec
slots after the window, which the conv kernels leave alone.
"""
torch.manual_seed(0)
device = "cuda"
num_qk_heads, num_v_heads, head_dim, conv_dim, state_len = 2, 4, 128, 16, 3
num_spec = 2
offsets = [0, 128, 0, 192]
query_lens = [200, 50, 300]
state_indices = [NULL_BLOCK_ID, 3, NULL_BLOCK_ID, 5]
num_blocks = 8
num_tokens = sum(query_lens)
q = F.normalize(
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
).bfloat16()
k = F.normalize(
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
).bfloat16()
v = torch.randn(1, num_tokens, num_v_heads, head_dim, device=device).bfloat16()
g = F.logsigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
beta = torch.sigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
initial_state = 0.1 * torch.randn(
len(query_lens), num_v_heads, head_dim, head_dim, device=device
)
prefill_cu = [0]
for query_len in query_lens:
prefill_cu.append(prefill_cu[-1] + query_len)
every_n_tokens, cu_starts, rows = plan_flashinfer_checkpoints(offsets, query_lens)
state_checkpoints = torch.empty(
cu_starts[-1], num_v_heads, head_dim, head_dim, device=device
)
fi_chunk_gated_delta_rule(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=torch.tensor(prefill_cu, device=device),
use_qk_l2norm_in_kernel=False,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=torch.tensor(cu_starts, device=device),
checkpoint_every_n_tokens=every_n_tokens,
use_cp=use_cp,
)
# The exporter sees the decode row's token in front of the prefills.
raw_qkv = torch.randn(1 + num_tokens, conv_dim, device=device).bfloat16()
cu_seqlens = torch.tensor([0] + [1 + n for n in prefill_cu], device=device)
conv_state = torch.zeros(
num_blocks,
state_len + num_spec,
conv_dim,
device=device,
dtype=torch.bfloat16,
).transpose(-1, -2)
ssm_state = torch.zeros(
num_blocks, num_v_heads, head_dim, head_dim, device=device, dtype=ssm_dtype
)
MambaPrefillCheckpointExporter(state_len=state_len).export(
MambaPrefillCheckpointMetadata(
checkpoint_offsets=torch.tensor(offsets, device=device, dtype=torch.int32),
state_indices=torch.tensor(state_indices, device=device),
),
raw_qkv=raw_qkv,
conv_state=conv_state,
recurrent_checkpoint=state_checkpoints,
recurrent_state=ssm_state,
cu_seqlens=cu_seqlens,
recurrent_rows=torch.tensor(rows, device=device),
)
written = []
for row, (offset, block) in enumerate(zip(offsets, state_indices)):
if not offset:
continue
written.append(block)
seq = row - 1
start = prefill_cu[seq]
_, expected_state = fi_chunk_gated_delta_rule(
q=q[:, start : start + offset],
k=k[:, start : start + offset],
v=v[:, start : start + offset],
g=g[:, start : start + offset],
beta=beta[:, start : start + offset],
initial_state=initial_state[seq : seq + 1],
output_final_state=True,
cu_seqlens=torch.tensor([0, offset], device=device),
use_qk_l2norm_in_kernel=False,
)
torch.testing.assert_close(
ssm_state[block].float(), expected_state[0], atol=2e-2, rtol=2e-2
)
end = cu_seqlens[row].item() + offset
torch.testing.assert_close(
conv_state[block, :, :state_len],
raw_qkv[end - state_len : end].T,
atol=0,
rtol=0,
)
assert not conv_state[block, :, state_len:].any()
untouched = [i for i in range(num_blocks) if i not in written]
assert not ssm_state[untouched].any()
assert not conv_state[untouched].any()
@@ -21,6 +21,7 @@ from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.gdn_attn import (
GDNAttentionMetadata,
GDNAttentionMetadataBuilder,
plan_flashinfer_checkpoints,
)
from vllm.v1.attention.backends.utils import mamba_get_block_table_tensor
from vllm.v1.kv_cache_interface import MambaSpec
@@ -488,3 +489,20 @@ def test_update_block_table_regathers_checkpoint(num_spec):
assert not torch.equal(
actual.checkpoint.state_indices, source.checkpoint.state_indices
)
def test_plan_flashinfer_checkpoints_selects_the_state_at_each_offset():
"""Each checkpoint row maps to the state FlashInfer emits at its offset.
The leading decode row is not in the prefill kernel and never checkpoints;
the 50-token prefill is shorter than the stride and emits no states.
"""
every_n_tokens, cu_starts, rows = plan_flashinfer_checkpoints(
offsets=[0, 128, 0, 192], query_lens=[200, 50, 300]
)
assert every_n_tokens == 64
assert cu_starts == [0, 3, 3, 7]
# Row 1: state after 128 tokens of the first prefill. Row 3: after 192
# tokens of the third prefill, whose states start at index 3.
assert rows == [0, 1, 0, 5]
+154 -3
View File
@@ -16,7 +16,10 @@ from tests.utils import create_new_process_for_each_test
from vllm import LLM, SamplingParams, TokensPrompt
from vllm.config import CacheConfig
from vllm.distributed import cleanup_dist_env_and_memory
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateCopyFunc
from vllm.model_executor.layers.mamba.mamba_utils import (
MambaStateCopyFunc,
is_conv_state_dim_first,
)
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -59,6 +62,10 @@ MODEL = "Qwen/Qwen3-Next-80B-A3B-Instruct-FP8"
# where block boundaries fall. XPU rounds GDN block sizes up to a multiple of 64
# (XPUPlatform.update_block_size_for_backend), so use one.
BLOCK_SIZE = 576
# The step actions also encode the split at the prompt's last block boundary,
# which a GDN prefill backend with internal checkpoints (FlashInfer) skips; see
# get_internal_checkpoint_step_configs.
GDN_PREFILL_CONFIG = {"gdn_prefill_backend": "triton"}
DEVICE_TYPE = current_platform.device_type
NUM_HIDDEN_LAYERS = 1
cur_step_action_idx = 0
@@ -882,6 +889,73 @@ def get_mamba_prefix_cache_step_configs(
return tests
def get_internal_checkpoint_step_configs() -> dict[str, TestConfig]:
"""Async-scheduling steps when the GDN prefill exports the checkpoint at
the prompt's last block boundary itself, so the final chunk runs whole.
The checkpoint goes in the second-to-last block the chunk reaches, keyed to
the boundary: column 1 holds the state at B for a 2B + 10 prompt.
"""
B = BLOCK_SIZE
return {
"prompt_2_block_size": TestConfig(
num_prompt_tokens=B * 2,
num_generated_tokens=10,
num_accepted_tokens=4,
step_actions=[
StepAction(0, B * 2, [1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(B * 2, 4, [1, 1, 1, 1, 1, 1], (1, 2), (-1, -1)),
],
),
"prompt_2_block_size_10": TestConfig(
num_prompt_tokens=B * 2 + 10,
num_generated_tokens=10,
num_accepted_tokens=4,
step_actions=[
StepAction(0, B * 2 + 10, [0, 1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(B * 2 + 10, 4, [0, 1, 1, 1, 1, 1], (-1, -1), (-1, -1)),
],
),
"prompt_10_block_size": TestConfig(
num_prompt_tokens=B * 10,
num_generated_tokens=10,
num_accepted_tokens=4,
step_actions=[
StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
B * 5,
B * 5,
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1],
(4, 9),
(-1, -1),
),
StepAction(
B * 10,
4,
[0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1],
(9, 10),
(-1, -1),
),
],
),
"prompt_10_block_size_10": TestConfig(
num_prompt_tokens=B * 10 + 10,
num_generated_tokens=10,
num_accepted_tokens=4,
step_actions=[
StepAction(0, B * 5, [0, 0, 0, 0, 1, 1, 1, 1], (-1, -1), (-1, -1)),
StepAction(
B * 5,
B * 5 + 10,
[0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1],
(4, 10),
(-1, -1),
),
],
),
}
def _run_mamba_prefix_cache_mrv1(
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
):
@@ -902,6 +976,7 @@ def _run_mamba_prefix_cache_mrv1(
load_format="dummy",
enable_prefix_caching=True,
block_size=BLOCK_SIZE,
additional_config=GDN_PREFILL_CONFIG,
speculative_config={
"method": "qwen3_next_mtp",
"num_speculative_tokens": num_speculative_tokens,
@@ -957,8 +1032,11 @@ def test_mamba_prefix_cache_mrv1_async(monkeypatch: pytest.MonkeyPatch):
def _run_mamba_prefix_cache_mrv2(
monkeypatch: pytest.MonkeyPatch, async_scheduling: bool
monkeypatch: pytest.MonkeyPatch,
async_scheduling: bool,
additional_config: dict[str, Any] = GDN_PREFILL_CONFIG,
):
internal_checkpoints = additional_config["gdn_prefill_backend"] == "flashinfer"
global async_scheduling_mode
async_scheduling_mode = async_scheduling
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
@@ -1098,8 +1176,24 @@ def _run_mamba_prefix_cache_mrv2(
with gpu_sync_allowed():
start_pos = input_batch.positions[input_batch.query_start_loc[0]].item()
assert cur_step_action.num_computed_tokens_start == start_pos
if scheduler_output.scheduled_new_reqs and "snapshot_col" in captured:
captured["snapshot"] = block_states(self, captured.pop("snapshot_col"))
return ret
def block_states(runner: MRV2GPUModelRunner, col: int) -> list[list[torch.Tensor]]:
# Each Mamba layer's (conv, temporal) states in block column col.
kv_cache_config = captured["kv_cache_config"]
forward_context = runner.vllm_config.compilation_config.static_forward_context
return [
[
temporal_block(state, captured["block_tables"][group_id], col).clone()
for state in forward_context[layer_name].kv_cache
]
for group_ids in get_mamba_groups(kv_cache_config).values()
for group_id in group_ids
for layer_name in kv_cache_config.kv_cache_groups[group_id].layer_names
]
def fake_sample(
self: MRV2GPUModelRunner,
hidden_states: torch.Tensor,
@@ -1161,6 +1255,7 @@ def _run_mamba_prefix_cache_mrv2(
skip_tokenizer_init=True,
enable_prefix_caching=True,
block_size=BLOCK_SIZE,
additional_config=additional_config,
speculative_config={
"method": "qwen3_next_mtp",
"num_speculative_tokens": num_speculative_tokens,
@@ -1173,7 +1268,11 @@ def _run_mamba_prefix_cache_mrv2(
)
try:
tests = get_mamba_prefix_cache_step_configs(async_scheduling)
tests = (
get_internal_checkpoint_step_configs()
if internal_checkpoints
else get_mamba_prefix_cache_step_configs(async_scheduling)
)
global step_actions
global cur_step_action_idx
@@ -1219,6 +1318,35 @@ def _run_mamba_prefix_cache_mrv2(
f"got events={second_events!r}"
)
assert engine.llm_engine.engine_core.engine_core.scheduler.reset_prefix_cache()
if internal_checkpoints:
# The checkpoint a 2B + 10 prefill exports at B (column 1) is the
# final state of a prefill that stops at B (column 0). Distinct
# tokens make a shifted conv window visible.
tokens = [(i * 7919) % 30000 + 1 for i in range(BLOCK_SIZE * 2 + 10)]
scheduler = engine.llm_engine.engine_core.engine_core.scheduler
states = []
for num_tokens, col in ((BLOCK_SIZE, 0), (len(tokens), 1)):
captured["snapshot_col"] = col
_ = engine.generate(
[TokensPrompt(prompt_token_ids=tokens[:num_tokens])],
sampling_params=sampling_params,
)
states.append(captured.pop("snapshot"))
assert scheduler.reset_prefix_cache()
hf_config = engine.llm_engine.model_config.hf_text_config
state_len = hf_config.linear_conv_kernel_dim - 1
for (ref_conv, ref_ssm), (conv, ssm) in zip(*states):
if not is_conv_state_dim_first():
ref_conv, conv = ref_conv.mT, conv.mT
# Under MTP the conv state has num_spec scratch columns after
# the window.
for new, ref in (
(conv[..., :state_len], ref_conv[..., :state_len]),
(ssm, ref_ssm),
):
rel_err = (new.float() - ref.float()).norm() / ref.float().norm()
assert rel_err < 1e-2, rel_err
finally:
del engine
torch.accelerator.empty_cache()
@@ -1233,3 +1361,26 @@ def test_mamba_prefix_cache_mrv2(monkeypatch: pytest.MonkeyPatch):
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv2_async(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv2(monkeypatch, async_scheduling=True)
@pytest.mark.skipif(
not current_platform.is_cuda()
or not (
current_platform.is_device_capability(90)
or (
(
current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
)
and current_platform.get_cuda_runtime_major() >= 13
)
),
reason="FlashInfer GDN prefill needs SM90, or SM100/SM120 with CUDA 13.",
)
@create_new_process_for_each_test()
def test_mamba_prefix_cache_mrv2_async_flashinfer(monkeypatch: pytest.MonkeyPatch):
_run_mamba_prefix_cache_mrv2(
monkeypatch,
async_scheduling=True,
additional_config={"gdn_prefill_backend": "flashinfer"},
)
+52 -1
View File
@@ -29,6 +29,7 @@ from vllm.distributed.parallel_state import (
from vllm.lora.layers import LoRAMappingType
from vllm.lora.request import LoRARequest
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.mamba.abstract import MambaBase
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
from vllm.platforms import current_platform
@@ -42,7 +43,10 @@ from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerBackend
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.backends.registry import (
AttentionBackendEnum,
MambaAttentionBackendEnum,
)
from vllm.v1.core.kv_cache_planning import (
estimate_max_model_len,
get_kv_cache_configs,
@@ -512,6 +516,53 @@ def test_alignment_rejected_by_a_sibling_backend_raises(monkeypatch, cpu):
platform.update_block_size_for_backend(vllm_config)
class _CheckpointingMambaLayer(MambaBase):
mamba_type = MambaAttentionBackendEnum.GDN_ATTN
def __init__(self, prefill_checkpoint_alignment: int | None):
self._prefill_checkpoint_alignment = prefill_checkpoint_alignment
def get_prefill_checkpoint_alignment(self) -> int | None:
return self._prefill_checkpoint_alignment
def get_state_shape(self):
return ()
def get_state_dtype(self):
return ()
def _align_qwen3_5_blocks(*prefill_checkpoint_alignments: int | None):
from tests.v1.attention.utils import create_vllm_config
vllm_config = create_vllm_config(model_name="Qwen/Qwen3.5-0.8B")
cache_config = vllm_config.cache_config
cache_config.mamba_cache_mode = "align"
forward_context = vllm_config.compilation_config.static_forward_context
for i, alignment in enumerate(prefill_checkpoint_alignments):
forward_context[f"mamba.{i}"] = _CheckpointingMambaLayer(alignment)
Platform._align_hybrid_block_size(vllm_config, _mock_backend([MultipleOf(16)]))
return cache_config
@pytest.mark.parametrize(
("prefill_checkpoint_alignments", "block_size"),
[((64,), 576), ((None,), 544), ((64, None), 544)],
)
def test_blocks_land_on_the_mamba_checkpoint_grid(
prefill_checkpoint_alignments, block_size
):
"""A layer that checkpoints every 64 tokens needs 64-aligned blocks.
Unaligned, Qwen3.5-0.8B's block (544) puts every other checkpoint off the
grid. A Mamba layer that cannot checkpoint (like Qwen4Exp's PLE short conv)
stops the scheduler from taking any, so the block stays.
"""
cache_config = _align_qwen3_5_blocks(*prefill_checkpoint_alignments)
assert cache_config.block_size == block_size
def test_xpu_gdn_rounding_rejected_by_a_sibling_backend_raises(monkeypatch):
# XPU's GDN rounding runs after super()'s check, so it must check again.
try: