From a309d4fe60bef7657b88805a1fcc9b014c414314 Mon Sep 17 00:00:00 2001 From: Woosuk Kwon Date: Mon, 29 Jun 2026 13:24:29 -0700 Subject: [PATCH] Support DCP with FlashInfer MLA (#43729) Signed-off-by: Woosuk Kwon --- docs/design/attention_backends.md | 2 +- vllm/v1/attention/backends/mla/flashinfer_mla.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/design/attention_backends.md b/docs/design/attention_backends.md index 9278ab6761..f3067dfc85 100644 --- a/docs/design/attention_backends.md +++ b/docs/design/attention_backends.md @@ -220,7 +220,7 @@ MLA decode backends are selected using the standard | Backend | Dtypes | KV Dtypes | Block Sizes | Head Sizes | Sink | Non-Causal | Sparse | MM Prefix | DCP | Attention Types | Compute Cap. | | ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ | | `CUTLASS_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 128 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x | -| `FLASHINFER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 10.x | +| `FLASHINFER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x | | `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 10.x | | `FLASHINFER_MLA_SPARSE_SM120` | bf16 | `auto`, `fp8`, `fp8_e4m3`, `fp8_ds_mla` | 64, 256 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 12.x | | `FLASHMLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x-10.x | diff --git a/vllm/v1/attention/backends/mla/flashinfer_mla.py b/vllm/v1/attention/backends/mla/flashinfer_mla.py index 25ab3d7f65..07e2e44140 100644 --- a/vllm/v1/attention/backends/mla/flashinfer_mla.py +++ b/vllm/v1/attention/backends/mla/flashinfer_mla.py @@ -113,6 +113,8 @@ g_fi_workspace = torch.zeros( class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): + can_return_lse_for_decode: bool = True + def __init__( self, num_heads: int, @@ -196,7 +198,8 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): if is_quantized_kv_cache(self.kv_cache_dtype): self.bmm2_scale *= layer._k_scale_float - o = trtllm_batch_decode_with_kv_cache_mla( + return_lse = self.need_to_return_lse_for_decode + kernel_out = trtllm_batch_decode_with_kv_cache_mla( query=q, kv_cache=kv_c_and_k_pe_cache.unsqueeze(1), workspace_buffer=self._workspace_buffer, @@ -208,11 +211,14 @@ class FlashInferMLAImpl(MLACommonImpl[MLACommonMetadata]): max_seq_len=attn_metadata.max_seq_len, bmm1_scale=self.bmm1_scale, bmm2_scale=self.bmm2_scale, + return_lse=return_lse, ) + if return_lse: + o, lse = kernel_out + else: + o, lse = kernel_out, None # Flatten the output for consistent shape o = o.view(-1, o.shape[-2], o.shape[-1]) - # TODO: Return LSE pending support from Flashinfer API: - # https://github.com/flashinfer-ai/flashinfer/pull/1566 - return o, None + return o, lse