[Perf][Spec Decode] Enable fused multi-step draft decode for FlashInfer trtllm-gen (#58371)

Signed-off-by: Bowen Wang <bowen@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bowen Wang
2026-09-29 06:06:47 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 28f6739576
commit 35d6fb3187
2 changed files with 58 additions and 0 deletions
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Integration tests for TRTLLM gen-full attention through FlashInfer."""
import dataclasses
import unittest.mock
from functools import partial
@@ -513,3 +514,50 @@ def test_trtllm_gen_nvfp4_kv_integration(batch_spec_name: str):
kv_cache_dtype="nvfp4",
model_name=MODEL_NVFP4,
)
@torch.inference_mode()
def test_trtllm_gen_draft_decode_metadata_tracks_in_place_seq_lens(
monkeypatch: pytest.MonkeyPatch,
):
"""Fused draft steps reuse the step-1 build after advancing seq_lens in
place, so its decode metadata must match the split path's step-2 rebuild."""
monkeypatch.setattr(
"vllm.utils.flashinfer.supports_trtllm_attention",
unittest.mock.Mock(return_value=True),
)
monkeypatch.setattr(
"vllm.v1.attention.backends.flashinfer.get_per_layer_parameters",
_mock_get_per_layer_parameters,
)
vllm_config = create_vllm_config(
model_name=MODEL, block_size=BLOCK_SIZE, num_gpu_blocks=NUM_GPU_BLOCKS
)
model_config = vllm_config.model_config
kv_cache_spec = FullAttentionSpec(
block_size=BLOCK_SIZE,
num_kv_heads=model_config.get_num_kv_heads(vllm_config.parallel_config),
head_size=model_config.get_head_size(),
dtype=model_config.dtype,
)
device = torch.device(f"{DEVICE_TYPE}:0")
with set_current_vllm_config(vllm_config):
builder = FlashInferMetadataBuilder(
kv_cache_spec, ["test_layer_0"], vllm_config, device
)
assert builder.supports_draft_decode_metadata_update
step1 = create_common_attn_metadata(BATCH_SPECS["decode_only"], BLOCK_SIZE, device)
reused = builder.build(common_prefix_len=0, common_attn_metadata=step1)
step1.seq_lens.add_(1)
step2 = dataclasses.replace(step1, seq_lens_cpu_upper_bound=step1.seq_lens.cpu())
rebuilt = builder.build(common_prefix_len=0, common_attn_metadata=step2)
assert isinstance(reused.decode, FlashInferTrtllmAPIDecode)
for field in dataclasses.fields(reused.decode):
value = getattr(reused.decode, field.name)
expected = getattr(rebuilt.decode, field.name)
if isinstance(value, torch.Tensor):
assert torch.equal(value, expected), field.name
else:
assert value == expected, field.name
+10
View File
@@ -860,6 +860,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
self.use_trtllm_gen_varlen_decode = self._uses_trtllm_gen_varlen_decode(
vllm_config, self.flashinfer_trtllm_api_decode_kernel, self.use_dcp
)
# Fused draft steps advance seq_lens in place and keep block tables fixed,
# so one trtllm-gen decode build serves every step (not DCP-local seq_lens).
self.supports_draft_decode_metadata_update = (
self.flashinfer_trtllm_api_decode_kernel
== FlashInferDecodeKernel.TRTLLM_GEN
and not self.use_dcp
)
self._init_reorder_batch_threshold(
1,
supports_spec_as_decode=(
@@ -1837,6 +1844,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
attn_metadata.decode = FIDecode(wrapper=decode_wrapper)
return attn_metadata
def update_draft_decode_metadata(self, _metadata: FlashInferMetadata) -> None:
pass
def use_cascade_attention(self, *args, **kwargs) -> bool:
if self.kv_cache_spec.dtype != self.vllm_config.model_config.dtype:
# TODO: The cascade wrapper currently does not support setting