[Perf][Spec Decode] Enable fused multi-step draft decode for FlashInfer trtllm-gen (#58371)
Signed-off-by: Bowen Wang <bowen@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
28f6739576
commit
35d6fb3187
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Integration tests for TRTLLM gen-full attention through FlashInfer."""
|
||||
|
||||
import dataclasses
|
||||
import unittest.mock
|
||||
from functools import partial
|
||||
|
||||
@@ -513,3 +514,50 @@ def test_trtllm_gen_nvfp4_kv_integration(batch_spec_name: str):
|
||||
kv_cache_dtype="nvfp4",
|
||||
model_name=MODEL_NVFP4,
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_gen_draft_decode_metadata_tracks_in_place_seq_lens(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Fused draft steps reuse the step-1 build after advancing seq_lens in
|
||||
place, so its decode metadata must match the split path's step-2 rebuild."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.utils.flashinfer.supports_trtllm_attention",
|
||||
unittest.mock.Mock(return_value=True),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.v1.attention.backends.flashinfer.get_per_layer_parameters",
|
||||
_mock_get_per_layer_parameters,
|
||||
)
|
||||
vllm_config = create_vllm_config(
|
||||
model_name=MODEL, block_size=BLOCK_SIZE, num_gpu_blocks=NUM_GPU_BLOCKS
|
||||
)
|
||||
model_config = vllm_config.model_config
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=BLOCK_SIZE,
|
||||
num_kv_heads=model_config.get_num_kv_heads(vllm_config.parallel_config),
|
||||
head_size=model_config.get_head_size(),
|
||||
dtype=model_config.dtype,
|
||||
)
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
with set_current_vllm_config(vllm_config):
|
||||
builder = FlashInferMetadataBuilder(
|
||||
kv_cache_spec, ["test_layer_0"], vllm_config, device
|
||||
)
|
||||
assert builder.supports_draft_decode_metadata_update
|
||||
|
||||
step1 = create_common_attn_metadata(BATCH_SPECS["decode_only"], BLOCK_SIZE, device)
|
||||
reused = builder.build(common_prefix_len=0, common_attn_metadata=step1)
|
||||
step1.seq_lens.add_(1)
|
||||
step2 = dataclasses.replace(step1, seq_lens_cpu_upper_bound=step1.seq_lens.cpu())
|
||||
rebuilt = builder.build(common_prefix_len=0, common_attn_metadata=step2)
|
||||
|
||||
assert isinstance(reused.decode, FlashInferTrtllmAPIDecode)
|
||||
for field in dataclasses.fields(reused.decode):
|
||||
value = getattr(reused.decode, field.name)
|
||||
expected = getattr(rebuilt.decode, field.name)
|
||||
if isinstance(value, torch.Tensor):
|
||||
assert torch.equal(value, expected), field.name
|
||||
else:
|
||||
assert value == expected, field.name
|
||||
|
||||
@@ -860,6 +860,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
self.use_trtllm_gen_varlen_decode = self._uses_trtllm_gen_varlen_decode(
|
||||
vllm_config, self.flashinfer_trtllm_api_decode_kernel, self.use_dcp
|
||||
)
|
||||
# Fused draft steps advance seq_lens in place and keep block tables fixed,
|
||||
# so one trtllm-gen decode build serves every step (not DCP-local seq_lens).
|
||||
self.supports_draft_decode_metadata_update = (
|
||||
self.flashinfer_trtllm_api_decode_kernel
|
||||
== FlashInferDecodeKernel.TRTLLM_GEN
|
||||
and not self.use_dcp
|
||||
)
|
||||
self._init_reorder_batch_threshold(
|
||||
1,
|
||||
supports_spec_as_decode=(
|
||||
@@ -1837,6 +1844,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
attn_metadata.decode = FIDecode(wrapper=decode_wrapper)
|
||||
return attn_metadata
|
||||
|
||||
def update_draft_decode_metadata(self, _metadata: FlashInferMetadata) -> None:
|
||||
pass
|
||||
|
||||
def use_cascade_attention(self, *args, **kwargs) -> bool:
|
||||
if self.kv_cache_spec.dtype != self.vllm_config.model_config.dtype:
|
||||
# TODO: The cascade wrapper currently does not support setting
|
||||
|
||||
Reference in New Issue
Block a user