[Attention] Extend XQA decode support on SM90 (#50439)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Andrii Skliar
2026-09-11 06:19:55 +00:00
committed by GitHub
co-authored by Andrii Skliar OpenAI Codex
parent fbf51c7026
commit 4b839c3378
3 changed files with 239 additions and 105 deletions
+147 -13
View File
@@ -889,7 +889,7 @@ def test_flashinfer_xqa_query_lens_preserve_cudagraph_padding():
device = torch.device("cpu")
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
builder.use_dedicated_xqa = True
builder.use_xqa = True
qo_indptr = torch.tensor([0, 3, 9, 15, 15], dtype=torch.int32, device=device)
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
@@ -905,6 +905,95 @@ def test_flashinfer_xqa_query_lens_preserve_cudagraph_padding():
assert q_cu_seq_lens.tolist() == [0, 3, 9, 15, 15]
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypatch):
"""Non-ragged XQA keeps one query/output row per padded request."""
import unittest.mock
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
impl = object.__new__(flashinfer_backend.FlashInferImpl)
impl.scale = 1.0
impl.bmm1_scale = None
impl.bmm2_scale = None
impl.kv_cache_dtype = "auto"
impl.is_kvcache_nvfp4 = False
impl.head_size = 16
impl.dcp_world_size = 1
impl.o_sf_scale = None
impl.window_left = -1
impl.sinks = None
impl.cache_config = unittest.mock.Mock()
impl.cache_config.get_resolved_kv_cache_layout.return_value = KVCacheLayout.LBHNC
layer = unittest.mock.Mock(
_q_scale=torch.tensor(1.0),
_q_scale_float=1.0,
_k_scale_float=1.0,
_v_scale_float=1.0,
)
decode = flashinfer_backend.FlashInferTrtllmAPIDecode(
kernel=flashinfer_backend.FlashInferDecodeKernel.XQA,
block_tables=torch.zeros((4, 1), dtype=torch.int32),
seq_lens=torch.tensor([8, 8, 0, 0], dtype=torch.int32),
max_seq_len=8,
q_len_per_req=1,
)
attn_metadata = flashinfer_backend.FlashInferMetadata(
num_actual_tokens=2,
slot_mapping=torch.empty(0, dtype=torch.int64),
q_data_type_prefill=torch.bfloat16,
q_data_type_decode=torch.bfloat16,
num_decodes=4,
num_decode_tokens=2,
num_prefills=0,
num_prefill_tokens=0,
causal=True,
prefill=None,
decode=decode,
use_cascade=False,
cascade_wrapper=None,
)
seen_shapes = {}
def mock_xqa(**kwargs):
seen_shapes["query"] = kwargs["query"].shape
seen_shapes["out"] = kwargs["out"].shape
seen_shapes["block_tables"] = kwargs["block_tables"].shape
monkeypatch.setattr(
flashinfer_backend,
"_get_trtllm_workspace_buffer",
lambda: torch.empty(1, dtype=torch.uint8),
)
monkeypatch.setattr(
flashinfer_backend, "flashinfer_xqa_batch_decode_with_kv_cache", mock_xqa
)
query = torch.zeros((4, 1, 16), dtype=torch.bfloat16)
output = torch.empty_like(query)
result = impl.forward(
layer,
query,
torch.empty_like(query),
torch.empty_like(query),
torch.zeros((1, 1, 1, 32), dtype=torch.bfloat16),
attn_metadata,
output,
)
assert result is output
assert seen_shapes == {
"query": torch.Size([4, 1, 16]),
"out": torch.Size([4, 1, 16]),
"block_tables": torch.Size([4, 1]),
}
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
@@ -913,7 +1002,7 @@ def test_flashinfer_xqa_query_lens_require_exact_uniform_product():
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
builder.use_dedicated_xqa = True
builder.use_xqa = True
qo_indptr = torch.tensor([0, 3, 3, 3], dtype=torch.int32)
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
@@ -929,6 +1018,42 @@ def test_flashinfer_xqa_query_lens_require_exact_uniform_product():
assert q_cu_seq_lens.tolist() == [0, 3, 3, 3]
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
@pytest.mark.parametrize(
("qo_indptr_values", "num_decode_tokens", "expected_q_len"),
[
([0, 3, 3], 3, 3),
([0, 2, 4, 4, 4], 4, 2),
],
)
def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
qo_indptr_values: list[int],
num_decode_tokens: int,
expected_q_len: int,
):
"""Padded speculative decode keeps its actual packed query width."""
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
builder.use_xqa = False
qo_indptr = torch.tensor(qo_indptr_values, dtype=torch.int32)
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
qo_indptr,
qo_indptr,
num_decodes=len(qo_indptr_values) - 1,
num_decode_tokens=num_decode_tokens,
)
assert q_len == expected_q_len
assert q_lens is None
assert q_cu_seq_lens is not None
assert q_cu_seq_lens.tolist() == qo_indptr_values
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
@@ -961,10 +1086,12 @@ def test_flashinfer_attention_sinks_refreshed_after_reload(dtype):
reason="FlashInfer is not available.",
)
def test_flashinfer_native_prefill_with_sinks(default_vllm_config):
if not (
current_platform.is_cuda() and current_platform.is_device_capability_family(120)
):
pytest.skip("Native FlashInfer prefill with sinks requires SM12x.")
supported = current_platform.is_cuda() and (
current_platform.is_device_capability(90)
or current_platform.is_device_capability_family(120)
)
if not supported:
pytest.skip("Native FlashInfer prefill with sinks requires SM90 or SM12x.")
from vllm.v1.attention.backends.flashinfer import FlashInferBackend
@@ -995,7 +1122,7 @@ def test_flashinfer_native_prefill_with_sinks(default_vllm_config):
reason="FlashInfer is not available.",
)
def test_flashinfer_xqa_decode_correctness(default_vllm_config):
"""FlashInfer should route supported decode through XQA and match SDPA."""
"""FlashInfer should route SM90/SM12x decode through XQA and match SDPA."""
supported = current_platform.is_cuda() and (
current_platform.is_device_capability(90)
or current_platform.is_device_capability_family(120)
@@ -1066,16 +1193,23 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
)
attn_metadata = builder.build(0, common_attn_metadata)
expected_cg_support = (
AttentionCGSupport.UNIFORM_BATCH
if current_platform.is_device_capability_family(120)
else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
assert (
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
vllm_config, kv_cache_spec
)
== expected_cg_support
== AttentionCGSupport.UNIFORM_BATCH
)
wide_head_kv_cache_spec = FullAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=kv_cache_spec.num_kv_heads,
head_size=512,
dtype=vllm_config.model_config.dtype,
)
assert (
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
vllm_config, wide_head_kv_cache_spec
)
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
assert isinstance(
attn_metadata.decode,
@@ -13,6 +13,7 @@ import torch
from tests.v1.attention.utils import create_vllm_config
from vllm.config import SpeculativeConfig, set_current_vllm_config
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
from vllm.v1.attention.backends.flashinfer import (
FlashInferDecodeKernel,
@@ -73,3 +74,7 @@ def test_flashinfer_gqa_dcp_spec_decode_clamps_reorder_threshold(monkeypatch):
builder.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.TRTLLM_GEN
)
assert builder.reorder_batch_threshold == 1
assert (
FlashInferMetadataBuilder.get_cudagraph_support(vllm_config, kv_cache_spec)
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
+87 -92
View File
@@ -519,7 +519,7 @@ class FlashInferBackend(AttentionBackend):
@classmethod
def supports_sink(cls) -> bool:
"""FlashInfer supports sinks on SM12x XQA and SM100 trtllm-gen."""
"""Whether FlashInfer can serve attention sinks on this platform."""
from vllm.utils.flashinfer import (
force_use_trtllm_attention,
)
@@ -527,7 +527,9 @@ class FlashInferBackend(AttentionBackend):
if force_use_trtllm_attention() is False:
return False
if current_platform.is_device_capability_family(120):
if current_platform.is_device_capability(
90
) or current_platform.is_device_capability_family(120):
return supports_trtllm_attention(is_prefill=False)
if not current_platform.is_device_capability_family(100):
@@ -570,6 +572,10 @@ class FlashInferDecodeKernel(Enum):
TRTLLM_GEN = "trtllm-gen"
def _is_xqa_head_dim_supported(head_dim: int) -> bool:
return 16 <= head_dim <= 256 and head_dim % 16 == 0
@dataclass
class TRTLLMPrefill:
"""Metadata for the TRTLLM prefill pathway."""
@@ -827,7 +833,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# the wider groups fall back to native FlashInfer decode.
if (
self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.XQA
and not (16 <= self.head_dim <= 256 and self.head_dim % 16 == 0)
and not _is_xqa_head_dim_supported(self.head_dim)
):
logger.warning_once(
"FlashInfer XQA decode does not support head_dim=%d; "
@@ -847,9 +853,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
)
self.use_trtllm_decode_attention = False
self.flashinfer_trtllm_api_decode_kernel = None
self.use_dedicated_xqa = (
current_platform.is_device_capability_family(120)
and self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.XQA
self.use_xqa = (
self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.XQA
)
# Adaptive verification trims drafts on device, so decode query lengths
# must come from the device qo_indptr; only trtllm-gen supports that
@@ -861,14 +866,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
== FlashInferDecodeKernel.TRTLLM_GEN
and not self.use_dcp
)
supports_spec_as_decode = (
self.flashinfer_trtllm_api_decode_kernel
== FlashInferDecodeKernel.TRTLLM_GEN
or self.use_dedicated_xqa
)
self._init_reorder_batch_threshold(
1,
supports_spec_as_decode=supports_spec_as_decode,
supports_spec_as_decode=(
self.flashinfer_trtllm_api_decode_kernel is not None
),
# trtllm-gen decode receives no cp_rank/global-seq-len information,
# so its end-aligned causal mask is wrong for q_len > 1 over the
# DCP-interleaved local KV shard (spec token i misses up to
@@ -886,16 +888,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
per_layer_parameters = get_per_layer_parameters(
vllm_config, layer_names, FlashInferImpl
)
if current_platform.is_device_capability(90) and any(
params.window_left != -1 for params in per_layer_parameters.values()
):
# FlashInfer SM90 sliding-window prefill is not reliable with FP8-Q:
# https://github.com/flashinfer-ai/flashinfer/issues/3578
raise NotImplementedError(
"FlashInfer backend on SM90 currently crashes with "
"sliding-window attention layers. Use the default attention "
"backend."
)
self.global_hyperparameters = infer_global_hyperparameters(per_layer_parameters)
self.sm_scale = self.global_hyperparameters.sm_scale
self.window_left = self.global_hyperparameters.window_left
@@ -950,7 +942,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# if cache_config requests a quantized dtype globally.
cache_dtype = self.cache_dtype
# XQA decode requires BF16/FP16-Q even with FP8 KV cache.
# On SM90/SM12x, XQA decode requires BF16/FP16-Q even with FP8 KV cache.
# FI native prefill on SM90 still uses FP8-Q in that case; SM12x prefill
# is fa2-only and keeps the model dtype (handled below).
if (
(
current_platform.is_device_capability(90)
@@ -964,7 +958,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# Otherwise, match Q dtype to the KV cache dtype.
if cache_dtype.startswith("fp8"):
# FP8-Q requires an fp8 tensor-core attention path.
# FP8-Q requires an fp8 tensor-core attention path. SM90 uses
# native FA3 for prefill; XQA decode on SM90/SM12x is handled
# above with model-dtype Q, while SM100 uses trtllm-gen.
# Architectures with only fa2 (e.g. SM89, SM120) cannot
# consume FP8 queries, so keep the model dtype for Q there.
if current_platform.is_device_capability(
@@ -983,24 +979,19 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
vllm_config: VllmConfig,
kv_cache_spec: KVCacheSpec,
) -> AttentionCGSupport:
"""Get the cudagraph support level for FlashInfer attention.
SM90 XQA supports only single-token decode. SM12x uses the dedicated
XQA API, which supports speculative and non-causal decode.
"""
if current_platform.is_device_capability(90):
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
is_sm12x = current_platform.is_device_capability_family(120)
# XQA does not return LSE and therefore does not support DCP.
if is_sm12x and vllm_config.parallel_config.decode_context_parallel_size > 1:
"""Get the cudagraph support level for FlashInfer attention."""
# XQA lacks LSE for DCP; DCP also cannot graph variable-length trtllm-gen.
if vllm_config.parallel_config.decode_context_parallel_size > 1:
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
kv_specs = iter_layer_specs(kv_cache_spec)
num_qo_heads = vllm_config.model_config.get_num_attention_heads(
vllm_config.parallel_config
)
has_trtllm_support: bool = len(kv_specs) > 0
is_xqa_arch = current_platform.is_device_capability(
90
) or current_platform.is_device_capability_family(120)
has_uniform_batch_support: bool = len(kv_specs) > 0
for spec in kv_specs:
if not isinstance(spec, AttentionSpec):
# FlashInfer only applies to attention, so we don't consider other types
@@ -1010,13 +1001,12 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
num_qo_heads=num_qo_heads,
num_kv_heads=spec.num_kv_heads,
is_prefill=False,
):
has_trtllm_support = False
) or (is_xqa_arch and not _is_xqa_head_dim_supported(spec.head_size)):
has_uniform_batch_support = False
break
if has_trtllm_support and (
is_sm12x or not vllm_config.attention_config.use_non_causal
):
use_non_causal = vllm_config.attention_config.use_non_causal
if has_uniform_batch_support and (not use_non_causal or is_xqa_arch):
return AttentionCGSupport.UNIFORM_BATCH
else:
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
@@ -1066,10 +1056,27 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
num_decode_tokens: int,
) -> tuple[int, torch.Tensor | None, list[int] | None]:
"""Return the query width, ragged offsets, and effective query lengths."""
assert self.use_dedicated_xqa
if num_decodes == 0 or num_decode_tokens == 0:
return 1, None, None
if not self.use_xqa:
decode_q_lens = (
qo_indptr_cpu[1 : num_decodes + 1] - qo_indptr_cpu[:num_decodes]
)
q_len_per_req = num_decode_tokens // num_decodes
if num_decode_tokens % num_decodes == 0 and bool(
(decode_q_lens == q_len_per_req).all().item()
):
return q_len_per_req, None, None
# CUDA-graph padding can leave zero-length requests after a
# uniform speculative decode batch. Keep the packed query and
# describe it through the trtllm-gen varlen API.
max_q_len = int(decode_q_lens.max().item())
if max_q_len > 1:
return max_q_len, qo_indptr[: num_decodes + 1], None
return 1, None, None
decode_q_lens = qo_indptr_cpu[1 : num_decodes + 1] - qo_indptr_cpu[:num_decodes]
nonzero = decode_q_lens[decode_q_lens > 0]
q_len_per_req = int(nonzero.max().item()) if nonzero.numel() > 0 else 1
@@ -1128,7 +1135,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
"NVFP4 KV cache."
)
if self._noncausal_prefill_wrapper is None:
if self.has_sinks and current_platform.is_device_capability_family(120):
if self.has_sinks and (
current_platform.is_device_capability(90)
or current_platform.is_device_capability_family(120)
):
self._noncausal_prefill_wrapper = (
BatchAttentionWithAttentionSinkWrapper(
self._get_workspace_buffer(),
@@ -1159,7 +1169,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
dcp_a2a=self.dcp_a2a,
)
else:
if self.has_sinks and current_platform.is_device_capability_family(120):
if self.has_sinks and (
current_platform.is_device_capability(90)
or current_platform.is_device_capability_family(120)
):
assert not self.is_kvcache_nvfp4
self._prefill_wrapper = BatchAttentionWithAttentionSinkWrapper(
self._get_workspace_buffer(),
@@ -1296,13 +1309,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
num_reqs = common_attn_metadata.num_reqs
num_actual_tokens = common_attn_metadata.num_actual_tokens
causal = common_attn_metadata.causal
route_decode = causal or self.use_dedicated_xqa
route_decode = causal or self.use_xqa
if route_decode:
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
split_decodes_and_prefills(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold,
require_uniform=not self.use_dedicated_xqa,
require_uniform=not self.use_xqa,
)
)
else:
@@ -1342,7 +1355,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
has_spec=uses_spec_reorder,
)
decode_with_flashinfer_trtllm_api = self.use_trtllm_decode_attention and (
causal or self.use_dedicated_xqa
causal or self.use_xqa
)
if not causal and self.use_dcp:
@@ -1360,13 +1373,12 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
)
if not all_uses_trtllm:
if self.has_sinks and (
not self.use_dedicated_xqa or self.use_dcp or use_cascade
):
sinks_would_be_dropped = self.use_dcp or use_cascade
if self.has_sinks and sinks_would_be_dropped:
raise NotImplementedError(
"FlashInfer backend currently does not support attention "
"sinks, please use trtllm on blackwell or flash attention "
"on earlier GPUs."
"FlashInfer BatchDCPPrefillWrapper and "
"MultiLevelCascadeAttentionWrapper do not support attention sinks. "
"Please use TRTLLM on Blackwell or FlashAttention on earlier GPUs."
)
if not self.global_hyperparameters.has_same_window_lefts:
@@ -1663,29 +1675,19 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
if num_decodes > 0:
if decode_with_flashinfer_trtllm_api:
assert self.flashinfer_trtllm_api_decode_kernel is not None
if not self.use_dedicated_xqa and not self.use_trtllm_gen_varlen_decode:
assert num_decode_tokens % num_decodes == 0, (
"XQA/trtllm-gen decode requires uniform query lengths "
f"per request. Got {num_decode_tokens=} and {num_decodes=}."
)
seq_lens_decode = seq_lens[:num_decodes]
if self.use_dcp:
assert common_attn_metadata.dcp_local_seq_lens is not None
seq_lens_decode = common_attn_metadata.dcp_local_seq_lens[
:num_decodes
]
q_len_per_req = 1
q_cu_seq_lens = None
decode_mask = None
if self.use_dedicated_xqa:
q_len_per_req, q_cu_seq_lens, ragged_q_lens = (
self._compute_decode_query_lens(
qo_indptr,
qo_indptr_cpu,
num_decodes,
num_decode_tokens,
)
q_len_per_req, q_cu_seq_lens, ragged_q_lens = (
self._compute_decode_query_lens(
qo_indptr, qo_indptr_cpu, num_decodes, num_decode_tokens
)
)
decode_mask = None
if self.use_xqa:
decode_mask = self._get_decode_mask(
q_len_per_req,
ragged_q_lens,
@@ -2066,7 +2068,10 @@ class FlashInferImpl(AttentionImpl):
if fp8_view_dtype is not None:
kv_cache = kv_cache.view(fp8_view_dtype)
# Inputs and outputs may be padded for CUDA graphs
# Inputs and outputs may be padded for CUDA graphs. Keep the original
# query for single-token XQA, whose non-ragged API requires one row for
# every (including padded) decode request.
query_padded = query
query = query[:num_actual_tokens]
output_padded = output
output = output[:num_actual_tokens]
@@ -2131,10 +2136,7 @@ class FlashInferImpl(AttentionImpl):
kv_cache_tuple = kv_cache_permute.split(hs, dim=-1)
use_dcp = self.dcp_world_size > 1
decode_with_dedicated_xqa = (
decode_with_xqa and current_platform.is_device_capability_family(120)
)
if decode_with_dedicated_xqa:
if decode_with_xqa:
assert not use_dcp
assert not self.is_kvcache_nvfp4
assert self.o_sf_scale is None
@@ -2351,8 +2353,13 @@ class FlashInferImpl(AttentionImpl):
].copy_(out[:num_prefill_tokens])
if num_decode_tokens > 0:
decode_query = query[:num_decode_tokens]
assert decode_query.shape[0] == num_decode_tokens
decode_query_tokens = num_decode_tokens
if decode_with_xqa:
assert isinstance(attn_metadata.decode, FlashInferTrtllmAPIDecode)
if attn_metadata.decode.q_len_per_req == 1:
decode_query_tokens = attn_metadata.num_decodes
decode_query = query_padded[:decode_query_tokens]
assert decode_query.shape[0] == decode_query_tokens
# Convert query to the expected dtype for decode if needed.
decode_query = self.maybe_quant_query(
@@ -2467,7 +2474,7 @@ class FlashInferImpl(AttentionImpl):
)
decode_query = canonicalize_singleton_dim_strides(decode_query)
if decode_with_dedicated_xqa:
if decode_with_xqa:
bmm1_scale = self.get_xqa_bmm1_scale(
layer, attn_metadata.q_data_type_decode
)
@@ -2483,7 +2490,7 @@ class FlashInferImpl(AttentionImpl):
bmm1_scale=bmm1_scale,
bmm2_scale=self.bmm2_scale,
window_left=self.window_left,
out=output[:num_decode_tokens],
out=output_padded[:decode_query_tokens],
sinks=self.sinks,
kv_layout=get_flashinfer_layout_string(self.kv_cache_layout),
q_len_per_req=q_len_per_req,
@@ -2492,6 +2499,7 @@ class FlashInferImpl(AttentionImpl):
)
return output_padded
assert decode_with_trtllm_gen
if output.dtype == FP4_DTYPE:
assert self.o_sf_scale is not None
out = FP4Tensor(
@@ -2523,19 +2531,6 @@ class FlashInferImpl(AttentionImpl):
else:
q_len_per_req = num_decode_tokens // attn_metadata.num_decodes
if decode_with_xqa and q_len_per_req is not None and q_len_per_req > 1:
raise NotImplementedError(
"FlashInfer XQA speculative decode is not wired in vLLM yet."
)
# XQA decode can use model-dtype Q with FP8 KV, so only include
# q_scale when the decode query is actually FP8.
bmm1_scale = (
self.get_xqa_bmm1_scale(layer, attn_metadata.q_data_type_decode)
if decode_with_xqa
else self.bmm1_scale
)
lse = None
if use_dcp:
out = torch.empty(
@@ -2558,7 +2553,7 @@ class FlashInferImpl(AttentionImpl):
block_tables=block_tables_decode,
seq_lens=seq_lens_decode,
max_seq_len=attn_metadata.decode.max_seq_len,
bmm1_scale=bmm1_scale,
bmm1_scale=self.bmm1_scale,
bmm2_scale=self.bmm2_scale,
window_left=self.window_left,
sinks=self.sinks,