diff --git a/tests/kernels/attention/test_flashinfer_trtllm_attention.py b/tests/kernels/attention/test_flashinfer_trtllm_attention.py index 20c34e589f..55d11f99ba 100644 --- a/tests/kernels/attention/test_flashinfer_trtllm_attention.py +++ b/tests/kernels/attention/test_flashinfer_trtllm_attention.py @@ -358,6 +358,99 @@ def test_flashinfer_trtllm_decode_with_baseline( ) +@pytest.mark.parametrize("quant_dtypes", [(None, None, None), (None, FP8_DTYPE, None)]) +@pytest.mark.parametrize("q_lens_cfg", [[1, 3, 2, 4], [2, 2, 1, 5, 3]]) +@torch.inference_mode +def test_flashinfer_trtllm_decode_varlen( + quant_dtypes: tuple[torch.dtype | None, torch.dtype | None, torch.dtype | None], + q_lens_cfg: list[int], +) -> None: + """Ragged decode via cum_seq_lens_q must match the varlen baseline.""" + dtype = torch.bfloat16 + num_qo_heads, num_kv_heads = 40, 8 + head_size = 128 + block_size = 16 + max_kv_len = 4096 + torch.set_default_device("cuda") + set_random_seed(42) + + _, kv_quant_dtype, _ = quant_dtypes + kv_quant_dtype = kv_quant_dtype or dtype + sm_scale = float(1.0 / (head_size**0.5)) + batch_size = len(q_lens_cfg) + + q_lens = torch.tensor(q_lens_cfg, dtype=torch.int32) + q_indptr = torch.cat( + [ + torch.tensor([0], dtype=torch.int32), + torch.cumsum(q_lens, dim=0, dtype=torch.int32), + ] + ) + query = torch.randn(int(q_lens.sum()), num_qo_heads, head_size, dtype=dtype) + + kv_lens = torch.randint(1, max_kv_len, (batch_size,), dtype=torch.int32) + kv_lens[-1] = max_kv_len + seq_lens = kv_lens + q_lens + max_seq_len = int(seq_lens.max()) + + kv_cache_shape = (NUM_BLOCKS, 2, num_kv_heads, block_size, head_size) + kv_cache = torch.randn(kv_cache_shape, dtype=dtype) + kv_cache, kv_cache_sf, kv_scale, ref_kv_cache, _ = make_quantized_kv_cache( + kv_cache, kv_quant_dtype, block_size, head_size + ) + + max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size + block_tables = torch.randint( + 0, NUM_BLOCKS, (batch_size, max_num_blocks_per_seq), dtype=torch.int32 + ) + kv_indptr, kv_indices, kv_last_page_lens = build_paged_kv_metadata( + seq_lens, block_tables, block_size + ) + workspace_buffer = torch.zeros(128 * 1024 * 1024, dtype=torch.int8) + + wrapper = flashinfer.BatchPrefillWithPagedKVCacheWrapper( + float_workspace_buffer=workspace_buffer, kv_layout="HND", backend="fa2" + ) + wrapper.plan( + qo_indptr=q_indptr, + paged_kv_indptr=kv_indptr, + paged_kv_indices=kv_indices, + paged_kv_last_page_len=kv_last_page_lens, + num_qo_heads=num_qo_heads, + num_kv_heads=num_kv_heads, + head_dim_qk=head_size, + page_size=block_size, + causal=True, + sm_scale=sm_scale, + window_left=-1, + q_data_type=dtype, + kv_data_type=dtype, + ) + output = torch.empty(query.shape, dtype=dtype) + wrapper.run(query, ref_kv_cache, out=output) + + output_trtllm = torch.empty(query.shape, dtype=dtype) + flashinfer.decode.trtllm_batch_decode_with_kv_cache( + query=query, + kv_cache=kv_cache, + workspace_buffer=workspace_buffer, + block_tables=block_tables, + seq_lens=seq_lens, + max_seq_len=max_seq_len, + bmm1_scale=kv_scale * sm_scale, + bmm2_scale=kv_scale, + window_left=-1, + out=output_trtllm, + q_len_per_req=None, + max_q_len=int(q_lens.max()), + cum_seq_lens_q=q_indptr, + kv_cache_sf=kv_cache_sf, + ) + + rtol, atol = (4e-2, 6e-2) if kv_quant_dtype == FP8_DTYPE else (1e-2, 1e-2) + torch.testing.assert_close(output, output_trtllm, atol=atol, rtol=rtol) + + @pytest.mark.parametrize("dtype", DTYPE) @pytest.mark.parametrize("quant_dtypes", QUANT_DTYPES) @pytest.mark.parametrize("batch_size", BATCH_SIZE) diff --git a/vllm/v1/attention/backends/flashinfer.py b/vllm/v1/attention/backends/flashinfer.py index c9aea6cb83..a479b2ca48 100755 --- a/vllm/v1/attention/backends/flashinfer.py +++ b/vllm/v1/attention/backends/flashinfer.py @@ -450,9 +450,19 @@ class FlashInferBackend(AttentionBackend): @classmethod def supports_device_cpu_query_lens_mismatch(cls) -> bool: - # The wrappers are planned from qo_indptr_cpu, so the CPU query offsets - # have to be the ones the kernel runs on. - return False + # Only trtllm-gen decode (SM100 family) reads query offsets from the + # device tensor; XQA and wrapper decode plan from qo_indptr_cpu, and + # under DCP spec queries are routed to the prefill path. + if not current_platform.is_device_capability_family(100): + return False + vllm_config = get_current_vllm_config_or_none() + if vllm_config is None or vllm_config.model_config is None: + return True + pc = vllm_config.parallel_config + mc = vllm_config.model_config + return pc.decode_context_parallel_size == 1 and can_use_trtllm_attention( + mc.get_num_attention_heads(pc), mc.get_num_kv_heads(pc), is_prefill=False + ) @classmethod def supports_sliding_window(cls) -> bool: @@ -834,6 +844,16 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): current_platform.is_device_capability_family(120) and self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.XQA ) + # Adaptive verification trims drafts on device, so decode query lengths + # must come from the device qo_indptr; only trtllm-gen supports that + # (the selector already rejects the other configurations). + self.use_trtllm_gen_varlen_decode = ( + speculative_config is not None + and speculative_config.enable_adaptive_verification + and self.flashinfer_trtllm_api_decode_kernel + == FlashInferDecodeKernel.TRTLLM_GEN + and not self.use_dcp + ) supports_spec_as_decode = ( self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.TRTLLM_GEN @@ -1619,7 +1639,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): if num_decodes > 0: if decode_with_flashinfer_trtllm_api: assert self.flashinfer_trtllm_api_decode_kernel is not None - if not self.use_dedicated_xqa: + if not self.use_dedicated_xqa and not self.use_trtllm_gen_varlen_decode: assert num_decode_tokens % num_decodes == 0, ( "XQA/trtllm-gen decode requires uniform query lengths " f"per request. Got {num_decode_tokens=} and {num_decodes=}." @@ -1648,6 +1668,16 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]): num_decodes, bool(causal), ) + elif self.use_trtllm_gen_varlen_decode: + # CPU lens are only an upper bound, device qo_indptr is + # the truth. + decode_q_lens = ( + qo_indptr_cpu[1 : num_decodes + 1] - qo_indptr_cpu[:num_decodes] + ) + max_q_upper = int(decode_q_lens.max()) + if max_q_upper > 1: + q_len_per_req = max_q_upper + q_cu_seq_lens = qo_indptr[: num_decodes + 1] attn_metadata.decode = FlashInferTrtllmAPIDecode( kernel=self.flashinfer_trtllm_api_decode_kernel, block_tables=block_table_tensor[:num_decodes], @@ -2382,7 +2412,7 @@ class FlashInferImpl(AttentionImpl): bmm1_scale = self.get_xqa_bmm1_scale( layer, attn_metadata.q_data_type_decode ) - q_len_per_req = attn_metadata.decode.q_len_per_req + q_len_per_req: int | None = attn_metadata.decode.q_len_per_req flashinfer_xqa_batch_decode_with_kv_cache( query=decode_query, @@ -2421,14 +2451,20 @@ class FlashInferImpl(AttentionImpl): if needs_fp8_out: out = self._nvfp4_fp8_out[:num_decode_tokens] - if num_decode_tokens % attn_metadata.num_decodes != 0: + q_cu_seq_lens = attn_metadata.decode.q_cu_seq_lens + max_q_len = None + q_len_per_req = None + if q_cu_seq_lens is not None: + # q_len_per_req must stay None for the kernel's varlen path. + max_q_len = attn_metadata.decode.q_len_per_req + elif num_decode_tokens % attn_metadata.num_decodes != 0: # This gets triggered when the dummy_run forces # attention to be initialized with q_len = 0 q_len_per_req = 1 else: q_len_per_req = num_decode_tokens // attn_metadata.num_decodes - if decode_with_xqa and q_len_per_req > 1: + if decode_with_xqa and q_len_per_req is not None and q_len_per_req > 1: raise NotImplementedError( "FlashInfer XQA speculative decode is not wired in vLLM yet." ) @@ -2472,6 +2508,8 @@ class FlashInferImpl(AttentionImpl): kv_layout=get_flashinfer_layout_string(self.kv_cache_layout), backend=attn_metadata.decode.kernel.value, q_len_per_req=q_len_per_req, + max_q_len=max_q_len, + cum_seq_lens_q=q_cu_seq_lens, kv_cache_sf=( nvfp4_kv_block_scales if self.is_kvcache_nvfp4 else None ),