[Attention][Spec Decode] Support varlen trtllm-gen decode for adaptive verification (#52157)
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
This commit is contained in:
@@ -358,6 +358,99 @@ def test_flashinfer_trtllm_decode_with_baseline(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quant_dtypes", [(None, None, None), (None, FP8_DTYPE, None)])
|
||||
@pytest.mark.parametrize("q_lens_cfg", [[1, 3, 2, 4], [2, 2, 1, 5, 3]])
|
||||
@torch.inference_mode
|
||||
def test_flashinfer_trtllm_decode_varlen(
|
||||
quant_dtypes: tuple[torch.dtype | None, torch.dtype | None, torch.dtype | None],
|
||||
q_lens_cfg: list[int],
|
||||
) -> None:
|
||||
"""Ragged decode via cum_seq_lens_q must match the varlen baseline."""
|
||||
dtype = torch.bfloat16
|
||||
num_qo_heads, num_kv_heads = 40, 8
|
||||
head_size = 128
|
||||
block_size = 16
|
||||
max_kv_len = 4096
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(42)
|
||||
|
||||
_, kv_quant_dtype, _ = quant_dtypes
|
||||
kv_quant_dtype = kv_quant_dtype or dtype
|
||||
sm_scale = float(1.0 / (head_size**0.5))
|
||||
batch_size = len(q_lens_cfg)
|
||||
|
||||
q_lens = torch.tensor(q_lens_cfg, dtype=torch.int32)
|
||||
q_indptr = torch.cat(
|
||||
[
|
||||
torch.tensor([0], dtype=torch.int32),
|
||||
torch.cumsum(q_lens, dim=0, dtype=torch.int32),
|
||||
]
|
||||
)
|
||||
query = torch.randn(int(q_lens.sum()), num_qo_heads, head_size, dtype=dtype)
|
||||
|
||||
kv_lens = torch.randint(1, max_kv_len, (batch_size,), dtype=torch.int32)
|
||||
kv_lens[-1] = max_kv_len
|
||||
seq_lens = kv_lens + q_lens
|
||||
max_seq_len = int(seq_lens.max())
|
||||
|
||||
kv_cache_shape = (NUM_BLOCKS, 2, num_kv_heads, block_size, head_size)
|
||||
kv_cache = torch.randn(kv_cache_shape, dtype=dtype)
|
||||
kv_cache, kv_cache_sf, kv_scale, ref_kv_cache, _ = make_quantized_kv_cache(
|
||||
kv_cache, kv_quant_dtype, block_size, head_size
|
||||
)
|
||||
|
||||
max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, NUM_BLOCKS, (batch_size, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
kv_indptr, kv_indices, kv_last_page_lens = build_paged_kv_metadata(
|
||||
seq_lens, block_tables, block_size
|
||||
)
|
||||
workspace_buffer = torch.zeros(128 * 1024 * 1024, dtype=torch.int8)
|
||||
|
||||
wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper(
|
||||
float_workspace_buffer=workspace_buffer, kv_layout="HND", backend="fa2"
|
||||
)
|
||||
wrapper.plan(
|
||||
qo_indptr=q_indptr,
|
||||
paged_kv_indptr=kv_indptr,
|
||||
paged_kv_indices=kv_indices,
|
||||
paged_kv_last_page_len=kv_last_page_lens,
|
||||
num_qo_heads=num_qo_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim_qk=head_size,
|
||||
page_size=block_size,
|
||||
causal=True,
|
||||
sm_scale=sm_scale,
|
||||
window_left=-1,
|
||||
q_data_type=dtype,
|
||||
kv_data_type=dtype,
|
||||
)
|
||||
output = torch.empty(query.shape, dtype=dtype)
|
||||
wrapper.run(query, ref_kv_cache, out=output)
|
||||
|
||||
output_trtllm = torch.empty(query.shape, dtype=dtype)
|
||||
flashinfer.decode.trtllm_batch_decode_with_kv_cache(
|
||||
query=query,
|
||||
kv_cache=kv_cache,
|
||||
workspace_buffer=workspace_buffer,
|
||||
block_tables=block_tables,
|
||||
seq_lens=seq_lens,
|
||||
max_seq_len=max_seq_len,
|
||||
bmm1_scale=kv_scale * sm_scale,
|
||||
bmm2_scale=kv_scale,
|
||||
window_left=-1,
|
||||
out=output_trtllm,
|
||||
q_len_per_req=None,
|
||||
max_q_len=int(q_lens.max()),
|
||||
cum_seq_lens_q=q_indptr,
|
||||
kv_cache_sf=kv_cache_sf,
|
||||
)
|
||||
|
||||
rtol, atol = (4e-2, 6e-2) if kv_quant_dtype == FP8_DTYPE else (1e-2, 1e-2)
|
||||
torch.testing.assert_close(output, output_trtllm, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPE)
|
||||
@pytest.mark.parametrize("quant_dtypes", QUANT_DTYPES)
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZE)
|
||||
|
||||
@@ -450,9 +450,19 @@ class FlashInferBackend(AttentionBackend):
|
||||
|
||||
@classmethod
|
||||
def supports_device_cpu_query_lens_mismatch(cls) -> bool:
|
||||
# The wrappers are planned from qo_indptr_cpu, so the CPU query offsets
|
||||
# have to be the ones the kernel runs on.
|
||||
return False
|
||||
# Only trtllm-gen decode (SM100 family) reads query offsets from the
|
||||
# device tensor; XQA and wrapper decode plan from qo_indptr_cpu, and
|
||||
# under DCP spec queries are routed to the prefill path.
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
return False
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
if vllm_config is None or vllm_config.model_config is None:
|
||||
return True
|
||||
pc = vllm_config.parallel_config
|
||||
mc = vllm_config.model_config
|
||||
return pc.decode_context_parallel_size == 1 and can_use_trtllm_attention(
|
||||
mc.get_num_attention_heads(pc), mc.get_num_kv_heads(pc), is_prefill=False
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def supports_sliding_window(cls) -> bool:
|
||||
@@ -834,6 +844,16 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
current_platform.is_device_capability_family(120)
|
||||
and self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.XQA
|
||||
)
|
||||
# Adaptive verification trims drafts on device, so decode query lengths
|
||||
# must come from the device qo_indptr; only trtllm-gen supports that
|
||||
# (the selector already rejects the other configurations).
|
||||
self.use_trtllm_gen_varlen_decode = (
|
||||
speculative_config is not None
|
||||
and speculative_config.enable_adaptive_verification
|
||||
and self.flashinfer_trtllm_api_decode_kernel
|
||||
== FlashInferDecodeKernel.TRTLLM_GEN
|
||||
and not self.use_dcp
|
||||
)
|
||||
supports_spec_as_decode = (
|
||||
self.flashinfer_trtllm_api_decode_kernel
|
||||
== FlashInferDecodeKernel.TRTLLM_GEN
|
||||
@@ -1619,7 +1639,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
if num_decodes > 0:
|
||||
if decode_with_flashinfer_trtllm_api:
|
||||
assert self.flashinfer_trtllm_api_decode_kernel is not None
|
||||
if not self.use_dedicated_xqa:
|
||||
if not self.use_dedicated_xqa and not self.use_trtllm_gen_varlen_decode:
|
||||
assert num_decode_tokens % num_decodes == 0, (
|
||||
"XQA/trtllm-gen decode requires uniform query lengths "
|
||||
f"per request. Got {num_decode_tokens=} and {num_decodes=}."
|
||||
@@ -1648,6 +1668,16 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
num_decodes,
|
||||
bool(causal),
|
||||
)
|
||||
elif self.use_trtllm_gen_varlen_decode:
|
||||
# CPU lens are only an upper bound, device qo_indptr is
|
||||
# the truth.
|
||||
decode_q_lens = (
|
||||
qo_indptr_cpu[1 : num_decodes + 1] - qo_indptr_cpu[:num_decodes]
|
||||
)
|
||||
max_q_upper = int(decode_q_lens.max())
|
||||
if max_q_upper > 1:
|
||||
q_len_per_req = max_q_upper
|
||||
q_cu_seq_lens = qo_indptr[: num_decodes + 1]
|
||||
attn_metadata.decode = FlashInferTrtllmAPIDecode(
|
||||
kernel=self.flashinfer_trtllm_api_decode_kernel,
|
||||
block_tables=block_table_tensor[:num_decodes],
|
||||
@@ -2382,7 +2412,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
bmm1_scale = self.get_xqa_bmm1_scale(
|
||||
layer, attn_metadata.q_data_type_decode
|
||||
)
|
||||
q_len_per_req = attn_metadata.decode.q_len_per_req
|
||||
q_len_per_req: int | None = attn_metadata.decode.q_len_per_req
|
||||
|
||||
flashinfer_xqa_batch_decode_with_kv_cache(
|
||||
query=decode_query,
|
||||
@@ -2421,14 +2451,20 @@ class FlashInferImpl(AttentionImpl):
|
||||
if needs_fp8_out:
|
||||
out = self._nvfp4_fp8_out[:num_decode_tokens]
|
||||
|
||||
if num_decode_tokens % attn_metadata.num_decodes != 0:
|
||||
q_cu_seq_lens = attn_metadata.decode.q_cu_seq_lens
|
||||
max_q_len = None
|
||||
q_len_per_req = None
|
||||
if q_cu_seq_lens is not None:
|
||||
# q_len_per_req must stay None for the kernel's varlen path.
|
||||
max_q_len = attn_metadata.decode.q_len_per_req
|
||||
elif num_decode_tokens % attn_metadata.num_decodes != 0:
|
||||
# This gets triggered when the dummy_run forces
|
||||
# attention to be initialized with q_len = 0
|
||||
q_len_per_req = 1
|
||||
else:
|
||||
q_len_per_req = num_decode_tokens // attn_metadata.num_decodes
|
||||
|
||||
if decode_with_xqa and q_len_per_req > 1:
|
||||
if decode_with_xqa and q_len_per_req is not None and q_len_per_req > 1:
|
||||
raise NotImplementedError(
|
||||
"FlashInfer XQA speculative decode is not wired in vLLM yet."
|
||||
)
|
||||
@@ -2472,6 +2508,8 @@ class FlashInferImpl(AttentionImpl):
|
||||
kv_layout=get_flashinfer_layout_string(self.kv_cache_layout),
|
||||
backend=attn_metadata.decode.kernel.value,
|
||||
q_len_per_req=q_len_per_req,
|
||||
max_q_len=max_q_len,
|
||||
cum_seq_lens_q=q_cu_seq_lens,
|
||||
kv_cache_sf=(
|
||||
nvfp4_kv_block_scales if self.is_kvcache_nvfp4 else None
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user