[Attention][Spec Decode] Support varlen trtllm-gen decode for adaptive verification (#52157)

Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
This commit is contained in:
Guan-Ming Chiu
2026-08-24 14:43:57 -04:00
committed by GitHub
parent 342b8ebd8b
commit 6a9c69fa85
2 changed files with 138 additions and 7 deletions
@@ -358,6 +358,99 @@ def test_flashinfer_trtllm_decode_with_baseline(
)
@pytest.mark.parametrize("quant_dtypes", [(None, None, None), (None, FP8_DTYPE, None)])
@pytest.mark.parametrize("q_lens_cfg", [[1, 3, 2, 4], [2, 2, 1, 5, 3]])
@torch.inference_mode
def test_flashinfer_trtllm_decode_varlen(
quant_dtypes: tuple[torch.dtype | None, torch.dtype | None, torch.dtype | None],
q_lens_cfg: list[int],
) -> None:
"""Ragged decode via cum_seq_lens_q must match the varlen baseline."""
dtype = torch.bfloat16
num_qo_heads, num_kv_heads = 40, 8
head_size = 128
block_size = 16
max_kv_len = 4096
torch.set_default_device("cuda")
set_random_seed(42)
_, kv_quant_dtype, _ = quant_dtypes
kv_quant_dtype = kv_quant_dtype or dtype
sm_scale = float(1.0 / (head_size**0.5))
batch_size = len(q_lens_cfg)
q_lens = torch.tensor(q_lens_cfg, dtype=torch.int32)
q_indptr = torch.cat(
[
torch.tensor([0], dtype=torch.int32),
torch.cumsum(q_lens, dim=0, dtype=torch.int32),
]
)
query = torch.randn(int(q_lens.sum()), num_qo_heads, head_size, dtype=dtype)
kv_lens = torch.randint(1, max_kv_len, (batch_size,), dtype=torch.int32)
kv_lens[-1] = max_kv_len
seq_lens = kv_lens + q_lens
max_seq_len = int(seq_lens.max())
kv_cache_shape = (NUM_BLOCKS, 2, num_kv_heads, block_size, head_size)
kv_cache = torch.randn(kv_cache_shape, dtype=dtype)
kv_cache, kv_cache_sf, kv_scale, ref_kv_cache, _ = make_quantized_kv_cache(
kv_cache, kv_quant_dtype, block_size, head_size
)
max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
block_tables = torch.randint(
0, NUM_BLOCKS, (batch_size, max_num_blocks_per_seq), dtype=torch.int32
)
kv_indptr, kv_indices, kv_last_page_lens = build_paged_kv_metadata(
seq_lens, block_tables, block_size
)
workspace_buffer = torch.zeros(128 * 1024 * 1024, dtype=torch.int8)
wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper(
float_workspace_buffer=workspace_buffer, kv_layout="HND", backend="fa2"
)
wrapper.plan(
qo_indptr=q_indptr,
paged_kv_indptr=kv_indptr,
paged_kv_indices=kv_indices,
paged_kv_last_page_len=kv_last_page_lens,
num_qo_heads=num_qo_heads,
num_kv_heads=num_kv_heads,
head_dim_qk=head_size,
page_size=block_size,
causal=True,
sm_scale=sm_scale,
window_left=-1,
q_data_type=dtype,
kv_data_type=dtype,
)
output = torch.empty(query.shape, dtype=dtype)
wrapper.run(query, ref_kv_cache, out=output)
output_trtllm = torch.empty(query.shape, dtype=dtype)
flashinfer.decode.trtllm_batch_decode_with_kv_cache(
query=query,
kv_cache=kv_cache,
workspace_buffer=workspace_buffer,
block_tables=block_tables,
seq_lens=seq_lens,
max_seq_len=max_seq_len,
bmm1_scale=kv_scale * sm_scale,
bmm2_scale=kv_scale,
window_left=-1,
out=output_trtllm,
q_len_per_req=None,
max_q_len=int(q_lens.max()),
cum_seq_lens_q=q_indptr,
kv_cache_sf=kv_cache_sf,
)
rtol, atol = (4e-2, 6e-2) if kv_quant_dtype == FP8_DTYPE else (1e-2, 1e-2)
torch.testing.assert_close(output, output_trtllm, atol=atol, rtol=rtol)
@pytest.mark.parametrize("dtype", DTYPE)
@pytest.mark.parametrize("quant_dtypes", QUANT_DTYPES)
@pytest.mark.parametrize("batch_size", BATCH_SIZE)
+45 -7
View File
@@ -450,9 +450,19 @@ class FlashInferBackend(AttentionBackend):
@classmethod
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
# The wrappers are planned from qo_indptr_cpu, so the CPU query offsets
# have to be the ones the kernel runs on.
return False
# Only trtllm-gen decode (SM100 family) reads query offsets from the
# device tensor; XQA and wrapper decode plan from qo_indptr_cpu, and
# under DCP spec queries are routed to the prefill path.
if not current_platform.is_device_capability_family(100):
return False
vllm_config = get_current_vllm_config_or_none()
if vllm_config is None or vllm_config.model_config is None:
return True
pc = vllm_config.parallel_config
mc = vllm_config.model_config
return pc.decode_context_parallel_size == 1 and can_use_trtllm_attention(
mc.get_num_attention_heads(pc), mc.get_num_kv_heads(pc), is_prefill=False
)
@classmethod
def supports_sliding_window(cls) -> bool:
@@ -834,6 +844,16 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
current_platform.is_device_capability_family(120)
and self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.XQA
)
# Adaptive verification trims drafts on device, so decode query lengths
# must come from the device qo_indptr; only trtllm-gen supports that
# (the selector already rejects the other configurations).
self.use_trtllm_gen_varlen_decode = (
speculative_config is not None
and speculative_config.enable_adaptive_verification
and self.flashinfer_trtllm_api_decode_kernel
== FlashInferDecodeKernel.TRTLLM_GEN
and not self.use_dcp
)
supports_spec_as_decode = (
self.flashinfer_trtllm_api_decode_kernel
== FlashInferDecodeKernel.TRTLLM_GEN
@@ -1619,7 +1639,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
if num_decodes > 0:
if decode_with_flashinfer_trtllm_api:
assert self.flashinfer_trtllm_api_decode_kernel is not None
if not self.use_dedicated_xqa:
if not self.use_dedicated_xqa and not self.use_trtllm_gen_varlen_decode:
assert num_decode_tokens % num_decodes == 0, (
"XQA/trtllm-gen decode requires uniform query lengths "
f"per request. Got {num_decode_tokens=} and {num_decodes=}."
@@ -1648,6 +1668,16 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
num_decodes,
bool(causal),
)
elif self.use_trtllm_gen_varlen_decode:
# CPU lens are only an upper bound, device qo_indptr is
# the truth.
decode_q_lens = (
qo_indptr_cpu[1 : num_decodes + 1] - qo_indptr_cpu[:num_decodes]
)
max_q_upper = int(decode_q_lens.max())
if max_q_upper > 1:
q_len_per_req = max_q_upper
q_cu_seq_lens = qo_indptr[: num_decodes + 1]
attn_metadata.decode = FlashInferTrtllmAPIDecode(
kernel=self.flashinfer_trtllm_api_decode_kernel,
block_tables=block_table_tensor[:num_decodes],
@@ -2382,7 +2412,7 @@ class FlashInferImpl(AttentionImpl):
bmm1_scale = self.get_xqa_bmm1_scale(
layer, attn_metadata.q_data_type_decode
)
q_len_per_req = attn_metadata.decode.q_len_per_req
q_len_per_req: int | None = attn_metadata.decode.q_len_per_req
flashinfer_xqa_batch_decode_with_kv_cache(
query=decode_query,
@@ -2421,14 +2451,20 @@ class FlashInferImpl(AttentionImpl):
if needs_fp8_out:
out = self._nvfp4_fp8_out[:num_decode_tokens]
if num_decode_tokens % attn_metadata.num_decodes != 0:
q_cu_seq_lens = attn_metadata.decode.q_cu_seq_lens
max_q_len = None
q_len_per_req = None
if q_cu_seq_lens is not None:
# q_len_per_req must stay None for the kernel's varlen path.
max_q_len = attn_metadata.decode.q_len_per_req
elif num_decode_tokens % attn_metadata.num_decodes != 0:
# This gets triggered when the dummy_run forces
# attention to be initialized with q_len = 0
q_len_per_req = 1
else:
q_len_per_req = num_decode_tokens // attn_metadata.num_decodes
if decode_with_xqa and q_len_per_req > 1:
if decode_with_xqa and q_len_per_req is not None and q_len_per_req > 1:
raise NotImplementedError(
"FlashInfer XQA speculative decode is not wired in vLLM yet."
)
@@ -2472,6 +2508,8 @@ class FlashInferImpl(AttentionImpl):
kv_layout=get_flashinfer_layout_string(self.kv_cache_layout),
backend=attn_metadata.decode.kernel.value,
q_len_per_req=q_len_per_req,
max_q_len=max_q_len,
cum_seq_lens_q=q_cu_seq_lens,
kv_cache_sf=(
nvfp4_kv_block_scales if self.is_kvcache_nvfp4 else None
),