From 35d6fb3187d0a5a70c43ee5b0c3e923a7cb96226 Mon Sep 17 00:00:00 2001 From: Bowen Wang Date: Tue, 29 Sep 2026 14:06:47 +0800 Subject: [PATCH] [Perf][Spec Decode] Enable fused multi-step draft decode for FlashInfer trtllm-gen (#58371) Signed-off-by: Bowen Wang Co-authored-by: Claude Opus 5.5 --- .../test_trtllm_attention_integration.py | 48 +++++++++++++++++++ vllm/v1/attention/backends/flashinfer.py | 10 ++++ 2 files changed, 58 insertions(+) diff --git a/tests/v1/attention/test_trtllm_attention_integration.py b/tests/v1/attention/test_trtllm_attention_integration.py index 42dd8f9f48..e0af9732a3 100644 --- a/tests/v1/attention/test_trtllm_attention_integration.py +++ b/tests/v1/attention/test_trtllm_attention_integration.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Integration tests for TRTLLM gen-full attention through FlashInfer.""" +import dataclasses import unittest.mock from functools import partial @@ -513,3 +514,50 @@ def test_trtllm_gen_nvfp4_kv_integration(batch_spec_name: str): kv_cache_dtype="nvfp4", model_name=MODEL_NVFP4, ) + + +@torch.inference_mode() +def test_trtllm_gen_draft_decode_metadata_tracks_in_place_seq_lens( + monkeypatch: pytest.MonkeyPatch, +): + """Fused draft steps reuse the step-1 build after advancing seq_lens in + place, so its decode metadata must match the split path's step-2 rebuild.""" + monkeypatch.setattr( + "vllm.utils.flashinfer.supports_trtllm_attention", + unittest.mock.Mock(return_value=True), + ) + monkeypatch.setattr( + "vllm.v1.attention.backends.flashinfer.get_per_layer_parameters", + _mock_get_per_layer_parameters, + ) + vllm_config = create_vllm_config( + model_name=MODEL, block_size=BLOCK_SIZE, num_gpu_blocks=NUM_GPU_BLOCKS + ) + model_config = vllm_config.model_config + kv_cache_spec = FullAttentionSpec( + block_size=BLOCK_SIZE, + num_kv_heads=model_config.get_num_kv_heads(vllm_config.parallel_config), + head_size=model_config.get_head_size(), + dtype=model_config.dtype, + ) + device = torch.device(f"{DEVICE_TYPE}:0") + with set_current_vllm_config(vllm_config): + builder = FlashInferMetadataBuilder( + kv_cache_spec, ["test_layer_0"], vllm_config, device + ) + assert builder.supports_draft_decode_metadata_update + + step1 = create_common_attn_metadata(BATCH_SPECS["decode_only"], BLOCK_SIZE, device) + reused = builder.build(common_prefix_len=0, common_attn_metadata=step1) + step1.seq_lens.add_(1) + step2 = dataclasses.replace(step1, seq_lens_cpu_upper_bound=step1.seq_lens.cpu()) + rebuilt = builder.build(common_prefix_len=0, common_attn_metadata=step2) + + assert isinstance(reused.decode, FlashInferTrtllmAPIDecode) + for field in dataclasses.fields(reused.decode): + value = getattr(reused.decode, field.name) + expected = getattr(rebuilt.decode, field.name) + if isinstance(value, torch.Tensor): + assert torch.equal(value, expected), field.name + else: + assert value == expected, field.name diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index ecc66a75ca..91ff6da3f0 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -860,6 +860,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): self.use_trtllm_gen_varlen_decode = self._uses_trtllm_gen_varlen_decode( vllm_config, self.flashinfer_trtllm_api_decode_kernel, self.use_dcp ) + # Fused draft steps advance seq_lens in place and keep block tables fixed, + # so one trtllm-gen decode build serves every step (not DCP-local seq_lens). + self.supports_draft_decode_metadata_update = ( + self.flashinfer_trtllm_api_decode_kernel + == FlashInferDecodeKernel.TRTLLM_GEN + and not self.use_dcp + ) self._init_reorder_batch_threshold( 1, supports_spec_as_decode=( @@ -1837,6 +1844,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): attn_metadata.decode = FIDecode(wrapper=decode_wrapper) return attn_metadata + def update_draft_decode_metadata(self, _metadata: FlashInferMetadata) -> None: + pass + def use_cascade_attention(self, *args, **kwargs) -> bool: if self.kv_cache_spec.dtype != self.vllm_config.model_config.dtype: # TODO: The cascade wrapper currently does not support setting