[Attention] Extend XQA decode support on SM90 (#50439)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Andrii Skliar
OpenAI Codex
parent
fbf51c7026
commit
4b839c3378
@@ -889,7 +889,7 @@ def test_flashinfer_xqa_query_lens_preserve_cudagraph_padding():
|
||||
|
||||
device = torch.device("cpu")
|
||||
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
|
||||
builder.use_dedicated_xqa = True
|
||||
builder.use_xqa = True
|
||||
qo_indptr = torch.tensor([0, 3, 9, 15, 15], dtype=torch.int32, device=device)
|
||||
|
||||
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
|
||||
@@ -905,6 +905,95 @@ def test_flashinfer_xqa_query_lens_preserve_cudagraph_padding():
|
||||
assert q_cu_seq_lens.tolist() == [0, 3, 9, 15, 15]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypatch):
|
||||
"""Non-ragged XQA keeps one query/output row per padded request."""
|
||||
import unittest.mock
|
||||
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
|
||||
impl = object.__new__(flashinfer_backend.FlashInferImpl)
|
||||
impl.scale = 1.0
|
||||
impl.bmm1_scale = None
|
||||
impl.bmm2_scale = None
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.is_kvcache_nvfp4 = False
|
||||
impl.head_size = 16
|
||||
impl.dcp_world_size = 1
|
||||
impl.o_sf_scale = None
|
||||
impl.window_left = -1
|
||||
impl.sinks = None
|
||||
impl.cache_config = unittest.mock.Mock()
|
||||
impl.cache_config.get_resolved_kv_cache_layout.return_value = KVCacheLayout.LBHNC
|
||||
|
||||
layer = unittest.mock.Mock(
|
||||
_q_scale=torch.tensor(1.0),
|
||||
_q_scale_float=1.0,
|
||||
_k_scale_float=1.0,
|
||||
_v_scale_float=1.0,
|
||||
)
|
||||
decode = flashinfer_backend.FlashInferTrtllmAPIDecode(
|
||||
kernel=flashinfer_backend.FlashInferDecodeKernel.XQA,
|
||||
block_tables=torch.zeros((4, 1), dtype=torch.int32),
|
||||
seq_lens=torch.tensor([8, 8, 0, 0], dtype=torch.int32),
|
||||
max_seq_len=8,
|
||||
q_len_per_req=1,
|
||||
)
|
||||
attn_metadata = flashinfer_backend.FlashInferMetadata(
|
||||
num_actual_tokens=2,
|
||||
slot_mapping=torch.empty(0, dtype=torch.int64),
|
||||
q_data_type_prefill=torch.bfloat16,
|
||||
q_data_type_decode=torch.bfloat16,
|
||||
num_decodes=4,
|
||||
num_decode_tokens=2,
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
causal=True,
|
||||
prefill=None,
|
||||
decode=decode,
|
||||
use_cascade=False,
|
||||
cascade_wrapper=None,
|
||||
)
|
||||
|
||||
seen_shapes = {}
|
||||
|
||||
def mock_xqa(**kwargs):
|
||||
seen_shapes["query"] = kwargs["query"].shape
|
||||
seen_shapes["out"] = kwargs["out"].shape
|
||||
seen_shapes["block_tables"] = kwargs["block_tables"].shape
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer_backend,
|
||||
"_get_trtllm_workspace_buffer",
|
||||
lambda: torch.empty(1, dtype=torch.uint8),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
flashinfer_backend, "flashinfer_xqa_batch_decode_with_kv_cache", mock_xqa
|
||||
)
|
||||
|
||||
query = torch.zeros((4, 1, 16), dtype=torch.bfloat16)
|
||||
output = torch.empty_like(query)
|
||||
result = impl.forward(
|
||||
layer,
|
||||
query,
|
||||
torch.empty_like(query),
|
||||
torch.empty_like(query),
|
||||
torch.zeros((1, 1, 1, 32), dtype=torch.bfloat16),
|
||||
attn_metadata,
|
||||
output,
|
||||
)
|
||||
|
||||
assert result is output
|
||||
assert seen_shapes == {
|
||||
"query": torch.Size([4, 1, 16]),
|
||||
"out": torch.Size([4, 1, 16]),
|
||||
"block_tables": torch.Size([4, 1]),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
@@ -913,7 +1002,7 @@ def test_flashinfer_xqa_query_lens_require_exact_uniform_product():
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
|
||||
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
|
||||
builder.use_dedicated_xqa = True
|
||||
builder.use_xqa = True
|
||||
qo_indptr = torch.tensor([0, 3, 3, 3], dtype=torch.int32)
|
||||
|
||||
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
|
||||
@@ -929,6 +1018,42 @@ def test_flashinfer_xqa_query_lens_require_exact_uniform_product():
|
||||
assert q_cu_seq_lens.tolist() == [0, 3, 3, 3]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("qo_indptr_values", "num_decode_tokens", "expected_q_len"),
|
||||
[
|
||||
([0, 3, 3], 3, 3),
|
||||
([0, 2, 4, 4, 4], 4, 2),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
|
||||
qo_indptr_values: list[int],
|
||||
num_decode_tokens: int,
|
||||
expected_q_len: int,
|
||||
):
|
||||
"""Padded speculative decode keeps its actual packed query width."""
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
|
||||
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
|
||||
builder.use_xqa = False
|
||||
qo_indptr = torch.tensor(qo_indptr_values, dtype=torch.int32)
|
||||
|
||||
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
|
||||
qo_indptr,
|
||||
qo_indptr,
|
||||
num_decodes=len(qo_indptr_values) - 1,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
)
|
||||
|
||||
assert q_len == expected_q_len
|
||||
assert q_lens is None
|
||||
assert q_cu_seq_lens is not None
|
||||
assert q_cu_seq_lens.tolist() == qo_indptr_values
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
@@ -961,10 +1086,12 @@ def test_flashinfer_attention_sinks_refreshed_after_reload(dtype):
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_native_prefill_with_sinks(default_vllm_config):
|
||||
if not (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability_family(120)
|
||||
):
|
||||
pytest.skip("Native FlashInfer prefill with sinks requires SM12x.")
|
||||
supported = current_platform.is_cuda() and (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
)
|
||||
if not supported:
|
||||
pytest.skip("Native FlashInfer prefill with sinks requires SM90 or SM12x.")
|
||||
|
||||
from vllm.v1.attention.backends.flashinfer import FlashInferBackend
|
||||
|
||||
@@ -995,7 +1122,7 @@ def test_flashinfer_native_prefill_with_sinks(default_vllm_config):
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_xqa_decode_correctness(default_vllm_config):
|
||||
"""FlashInfer should route supported decode through XQA and match SDPA."""
|
||||
"""FlashInfer should route SM90/SM12x decode through XQA and match SDPA."""
|
||||
supported = current_platform.is_cuda() and (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
@@ -1066,16 +1193,23 @@ def test_flashinfer_xqa_decode_correctness(default_vllm_config):
|
||||
)
|
||||
attn_metadata = builder.build(0, common_attn_metadata)
|
||||
|
||||
expected_cg_support = (
|
||||
AttentionCGSupport.UNIFORM_BATCH
|
||||
if current_platform.is_device_capability_family(120)
|
||||
else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
assert (
|
||||
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
|
||||
vllm_config, kv_cache_spec
|
||||
)
|
||||
== expected_cg_support
|
||||
== AttentionCGSupport.UNIFORM_BATCH
|
||||
)
|
||||
wide_head_kv_cache_spec = FullAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=kv_cache_spec.num_kv_heads,
|
||||
head_size=512,
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
)
|
||||
assert (
|
||||
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
|
||||
vllm_config, wide_head_kv_cache_spec
|
||||
)
|
||||
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
assert isinstance(
|
||||
attn_metadata.decode,
|
||||
|
||||
@@ -13,6 +13,7 @@ import torch
|
||||
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
from vllm.config import SpeculativeConfig, set_current_vllm_config
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
from vllm.v1.attention.backends.flashinfer import (
|
||||
FlashInferDecodeKernel,
|
||||
@@ -73,3 +74,7 @@ def test_flashinfer_gqa_dcp_spec_decode_clamps_reorder_threshold(monkeypatch):
|
||||
builder.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.TRTLLM_GEN
|
||||
)
|
||||
assert builder.reorder_batch_threshold == 1
|
||||
assert (
|
||||
FlashInferMetadataBuilder.get_cudagraph_support(vllm_config, kv_cache_spec)
|
||||
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
|
||||
@@ -519,7 +519,7 @@ class FlashInferBackend(AttentionBackend):
|
||||
|
||||
@classmethod
|
||||
def supports_sink(cls) -> bool:
|
||||
"""FlashInfer supports sinks on SM12x XQA and SM100 trtllm-gen."""
|
||||
"""Whether FlashInfer can serve attention sinks on this platform."""
|
||||
from vllm.utils.flashinfer import (
|
||||
force_use_trtllm_attention,
|
||||
)
|
||||
@@ -527,7 +527,9 @@ class FlashInferBackend(AttentionBackend):
|
||||
if force_use_trtllm_attention() is False:
|
||||
return False
|
||||
|
||||
if current_platform.is_device_capability_family(120):
|
||||
if current_platform.is_device_capability(
|
||||
90
|
||||
) or current_platform.is_device_capability_family(120):
|
||||
return supports_trtllm_attention(is_prefill=False)
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
@@ -570,6 +572,10 @@ class FlashInferDecodeKernel(Enum):
|
||||
TRTLLM_GEN = "trtllm-gen"
|
||||
|
||||
|
||||
def _is_xqa_head_dim_supported(head_dim: int) -> bool:
|
||||
return 16 <= head_dim <= 256 and head_dim % 16 == 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class TRTLLMPrefill:
|
||||
"""Metadata for the TRTLLM prefill pathway."""
|
||||
@@ -827,7 +833,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
# the wider groups fall back to native FlashInfer decode.
|
||||
if (
|
||||
self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.XQA
|
||||
and not (16 <= self.head_dim <= 256 and self.head_dim % 16 == 0)
|
||||
and not _is_xqa_head_dim_supported(self.head_dim)
|
||||
):
|
||||
logger.warning_once(
|
||||
"FlashInfer XQA decode does not support head_dim=%d; "
|
||||
@@ -847,9 +853,8 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
)
|
||||
self.use_trtllm_decode_attention = False
|
||||
self.flashinfer_trtllm_api_decode_kernel = None
|
||||
self.use_dedicated_xqa = (
|
||||
current_platform.is_device_capability_family(120)
|
||||
and self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.XQA
|
||||
self.use_xqa = (
|
||||
self.flashinfer_trtllm_api_decode_kernel == FlashInferDecodeKernel.XQA
|
||||
)
|
||||
# Adaptive verification trims drafts on device, so decode query lengths
|
||||
# must come from the device qo_indptr; only trtllm-gen supports that
|
||||
@@ -861,14 +866,11 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
== FlashInferDecodeKernel.TRTLLM_GEN
|
||||
and not self.use_dcp
|
||||
)
|
||||
supports_spec_as_decode = (
|
||||
self.flashinfer_trtllm_api_decode_kernel
|
||||
== FlashInferDecodeKernel.TRTLLM_GEN
|
||||
or self.use_dedicated_xqa
|
||||
)
|
||||
self._init_reorder_batch_threshold(
|
||||
1,
|
||||
supports_spec_as_decode=supports_spec_as_decode,
|
||||
supports_spec_as_decode=(
|
||||
self.flashinfer_trtllm_api_decode_kernel is not None
|
||||
),
|
||||
# trtllm-gen decode receives no cp_rank/global-seq-len information,
|
||||
# so its end-aligned causal mask is wrong for q_len > 1 over the
|
||||
# DCP-interleaved local KV shard (spec token i misses up to
|
||||
@@ -886,16 +888,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
per_layer_parameters = get_per_layer_parameters(
|
||||
vllm_config, layer_names, FlashInferImpl
|
||||
)
|
||||
if current_platform.is_device_capability(90) and any(
|
||||
params.window_left != -1 for params in per_layer_parameters.values()
|
||||
):
|
||||
# FlashInfer SM90 sliding-window prefill is not reliable with FP8-Q:
|
||||
# https://github.com/flashinfer-ai/flashinfer/issues/3578
|
||||
raise NotImplementedError(
|
||||
"FlashInfer backend on SM90 currently crashes with "
|
||||
"sliding-window attention layers. Use the default attention "
|
||||
"backend."
|
||||
)
|
||||
self.global_hyperparameters = infer_global_hyperparameters(per_layer_parameters)
|
||||
self.sm_scale = self.global_hyperparameters.sm_scale
|
||||
self.window_left = self.global_hyperparameters.window_left
|
||||
@@ -950,7 +942,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
# if cache_config requests a quantized dtype globally.
|
||||
cache_dtype = self.cache_dtype
|
||||
|
||||
# XQA decode requires BF16/FP16-Q even with FP8 KV cache.
|
||||
# On SM90/SM12x, XQA decode requires BF16/FP16-Q even with FP8 KV cache.
|
||||
# FI native prefill on SM90 still uses FP8-Q in that case; SM12x prefill
|
||||
# is fa2-only and keeps the model dtype (handled below).
|
||||
if (
|
||||
(
|
||||
current_platform.is_device_capability(90)
|
||||
@@ -964,7 +958,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
|
||||
# Otherwise, match Q dtype to the KV cache dtype.
|
||||
if cache_dtype.startswith("fp8"):
|
||||
# FP8-Q requires an fp8 tensor-core attention path.
|
||||
# FP8-Q requires an fp8 tensor-core attention path. SM90 uses
|
||||
# native FA3 for prefill; XQA decode on SM90/SM12x is handled
|
||||
# above with model-dtype Q, while SM100 uses trtllm-gen.
|
||||
# Architectures with only fa2 (e.g. SM89, SM120) cannot
|
||||
# consume FP8 queries, so keep the model dtype for Q there.
|
||||
if current_platform.is_device_capability(
|
||||
@@ -983,24 +979,19 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: KVCacheSpec,
|
||||
) -> AttentionCGSupport:
|
||||
"""Get the cudagraph support level for FlashInfer attention.
|
||||
|
||||
SM90 XQA supports only single-token decode. SM12x uses the dedicated
|
||||
XQA API, which supports speculative and non-causal decode.
|
||||
"""
|
||||
if current_platform.is_device_capability(90):
|
||||
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
|
||||
is_sm12x = current_platform.is_device_capability_family(120)
|
||||
# XQA does not return LSE and therefore does not support DCP.
|
||||
if is_sm12x and vllm_config.parallel_config.decode_context_parallel_size > 1:
|
||||
"""Get the cudagraph support level for FlashInfer attention."""
|
||||
# XQA lacks LSE for DCP; DCP also cannot graph variable-length trtllm-gen.
|
||||
if vllm_config.parallel_config.decode_context_parallel_size > 1:
|
||||
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
|
||||
kv_specs = iter_layer_specs(kv_cache_spec)
|
||||
num_qo_heads = vllm_config.model_config.get_num_attention_heads(
|
||||
vllm_config.parallel_config
|
||||
)
|
||||
has_trtllm_support: bool = len(kv_specs) > 0
|
||||
is_xqa_arch = current_platform.is_device_capability(
|
||||
90
|
||||
) or current_platform.is_device_capability_family(120)
|
||||
has_uniform_batch_support: bool = len(kv_specs) > 0
|
||||
for spec in kv_specs:
|
||||
if not isinstance(spec, AttentionSpec):
|
||||
# FlashInfer only applies to attention, so we don't consider other types
|
||||
@@ -1010,13 +1001,12 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
num_qo_heads=num_qo_heads,
|
||||
num_kv_heads=spec.num_kv_heads,
|
||||
is_prefill=False,
|
||||
):
|
||||
has_trtllm_support = False
|
||||
) or (is_xqa_arch and not _is_xqa_head_dim_supported(spec.head_size)):
|
||||
has_uniform_batch_support = False
|
||||
break
|
||||
|
||||
if has_trtllm_support and (
|
||||
is_sm12x or not vllm_config.attention_config.use_non_causal
|
||||
):
|
||||
use_non_causal = vllm_config.attention_config.use_non_causal
|
||||
if has_uniform_batch_support and (not use_non_causal or is_xqa_arch):
|
||||
return AttentionCGSupport.UNIFORM_BATCH
|
||||
else:
|
||||
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
@@ -1066,10 +1056,27 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
num_decode_tokens: int,
|
||||
) -> tuple[int, torch.Tensor | None, list[int] | None]:
|
||||
"""Return the query width, ragged offsets, and effective query lengths."""
|
||||
assert self.use_dedicated_xqa
|
||||
if num_decodes == 0 or num_decode_tokens == 0:
|
||||
return 1, None, None
|
||||
|
||||
if not self.use_xqa:
|
||||
decode_q_lens = (
|
||||
qo_indptr_cpu[1 : num_decodes + 1] - qo_indptr_cpu[:num_decodes]
|
||||
)
|
||||
q_len_per_req = num_decode_tokens // num_decodes
|
||||
if num_decode_tokens % num_decodes == 0 and bool(
|
||||
(decode_q_lens == q_len_per_req).all().item()
|
||||
):
|
||||
return q_len_per_req, None, None
|
||||
|
||||
# CUDA-graph padding can leave zero-length requests after a
|
||||
# uniform speculative decode batch. Keep the packed query and
|
||||
# describe it through the trtllm-gen varlen API.
|
||||
max_q_len = int(decode_q_lens.max().item())
|
||||
if max_q_len > 1:
|
||||
return max_q_len, qo_indptr[: num_decodes + 1], None
|
||||
return 1, None, None
|
||||
|
||||
decode_q_lens = qo_indptr_cpu[1 : num_decodes + 1] - qo_indptr_cpu[:num_decodes]
|
||||
nonzero = decode_q_lens[decode_q_lens > 0]
|
||||
q_len_per_req = int(nonzero.max().item()) if nonzero.numel() > 0 else 1
|
||||
@@ -1128,7 +1135,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
"NVFP4 KV cache."
|
||||
)
|
||||
if self._noncausal_prefill_wrapper is None:
|
||||
if self.has_sinks and current_platform.is_device_capability_family(120):
|
||||
if self.has_sinks and (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
):
|
||||
self._noncausal_prefill_wrapper = (
|
||||
BatchAttentionWithAttentionSinkWrapper(
|
||||
self._get_workspace_buffer(),
|
||||
@@ -1159,7 +1169,10 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
dcp_a2a=self.dcp_a2a,
|
||||
)
|
||||
else:
|
||||
if self.has_sinks and current_platform.is_device_capability_family(120):
|
||||
if self.has_sinks and (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
):
|
||||
assert not self.is_kvcache_nvfp4
|
||||
self._prefill_wrapper = BatchAttentionWithAttentionSinkWrapper(
|
||||
self._get_workspace_buffer(),
|
||||
@@ -1296,13 +1309,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
num_actual_tokens = common_attn_metadata.num_actual_tokens
|
||||
causal = common_attn_metadata.causal
|
||||
route_decode = causal or self.use_dedicated_xqa
|
||||
route_decode = causal or self.use_xqa
|
||||
if route_decode:
|
||||
num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = (
|
||||
split_decodes_and_prefills(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
require_uniform=not self.use_dedicated_xqa,
|
||||
require_uniform=not self.use_xqa,
|
||||
)
|
||||
)
|
||||
else:
|
||||
@@ -1342,7 +1355,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
has_spec=uses_spec_reorder,
|
||||
)
|
||||
decode_with_flashinfer_trtllm_api = self.use_trtllm_decode_attention and (
|
||||
causal or self.use_dedicated_xqa
|
||||
causal or self.use_xqa
|
||||
)
|
||||
|
||||
if not causal and self.use_dcp:
|
||||
@@ -1360,13 +1373,12 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
)
|
||||
|
||||
if not all_uses_trtllm:
|
||||
if self.has_sinks and (
|
||||
not self.use_dedicated_xqa or self.use_dcp or use_cascade
|
||||
):
|
||||
sinks_would_be_dropped = self.use_dcp or use_cascade
|
||||
if self.has_sinks and sinks_would_be_dropped:
|
||||
raise NotImplementedError(
|
||||
"FlashInfer backend currently does not support attention "
|
||||
"sinks, please use trtllm on blackwell or flash attention "
|
||||
"on earlier GPUs."
|
||||
"FlashInfer BatchDCPPrefillWrapper and "
|
||||
"MultiLevelCascadeAttentionWrapper do not support attention sinks. "
|
||||
"Please use TRTLLM on Blackwell or FlashAttention on earlier GPUs."
|
||||
)
|
||||
|
||||
if not self.global_hyperparameters.has_same_window_lefts:
|
||||
@@ -1663,29 +1675,19 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
if num_decodes > 0:
|
||||
if decode_with_flashinfer_trtllm_api:
|
||||
assert self.flashinfer_trtllm_api_decode_kernel is not None
|
||||
if not self.use_dedicated_xqa and not self.use_trtllm_gen_varlen_decode:
|
||||
assert num_decode_tokens % num_decodes == 0, (
|
||||
"XQA/trtllm-gen decode requires uniform query lengths "
|
||||
f"per request. Got {num_decode_tokens=} and {num_decodes=}."
|
||||
)
|
||||
seq_lens_decode = seq_lens[:num_decodes]
|
||||
if self.use_dcp:
|
||||
assert common_attn_metadata.dcp_local_seq_lens is not None
|
||||
seq_lens_decode = common_attn_metadata.dcp_local_seq_lens[
|
||||
:num_decodes
|
||||
]
|
||||
q_len_per_req = 1
|
||||
q_cu_seq_lens = None
|
||||
decode_mask = None
|
||||
if self.use_dedicated_xqa:
|
||||
q_len_per_req, q_cu_seq_lens, ragged_q_lens = (
|
||||
self._compute_decode_query_lens(
|
||||
qo_indptr,
|
||||
qo_indptr_cpu,
|
||||
num_decodes,
|
||||
num_decode_tokens,
|
||||
)
|
||||
q_len_per_req, q_cu_seq_lens, ragged_q_lens = (
|
||||
self._compute_decode_query_lens(
|
||||
qo_indptr, qo_indptr_cpu, num_decodes, num_decode_tokens
|
||||
)
|
||||
)
|
||||
decode_mask = None
|
||||
if self.use_xqa:
|
||||
decode_mask = self._get_decode_mask(
|
||||
q_len_per_req,
|
||||
ragged_q_lens,
|
||||
@@ -2066,7 +2068,10 @@ class FlashInferImpl(AttentionImpl):
|
||||
if fp8_view_dtype is not None:
|
||||
kv_cache = kv_cache.view(fp8_view_dtype)
|
||||
|
||||
# Inputs and outputs may be padded for CUDA graphs
|
||||
# Inputs and outputs may be padded for CUDA graphs. Keep the original
|
||||
# query for single-token XQA, whose non-ragged API requires one row for
|
||||
# every (including padded) decode request.
|
||||
query_padded = query
|
||||
query = query[:num_actual_tokens]
|
||||
output_padded = output
|
||||
output = output[:num_actual_tokens]
|
||||
@@ -2131,10 +2136,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
kv_cache_tuple = kv_cache_permute.split(hs, dim=-1)
|
||||
|
||||
use_dcp = self.dcp_world_size > 1
|
||||
decode_with_dedicated_xqa = (
|
||||
decode_with_xqa and current_platform.is_device_capability_family(120)
|
||||
)
|
||||
if decode_with_dedicated_xqa:
|
||||
if decode_with_xqa:
|
||||
assert not use_dcp
|
||||
assert not self.is_kvcache_nvfp4
|
||||
assert self.o_sf_scale is None
|
||||
@@ -2351,8 +2353,13 @@ class FlashInferImpl(AttentionImpl):
|
||||
].copy_(out[:num_prefill_tokens])
|
||||
|
||||
if num_decode_tokens > 0:
|
||||
decode_query = query[:num_decode_tokens]
|
||||
assert decode_query.shape[0] == num_decode_tokens
|
||||
decode_query_tokens = num_decode_tokens
|
||||
if decode_with_xqa:
|
||||
assert isinstance(attn_metadata.decode, FlashInferTrtllmAPIDecode)
|
||||
if attn_metadata.decode.q_len_per_req == 1:
|
||||
decode_query_tokens = attn_metadata.num_decodes
|
||||
decode_query = query_padded[:decode_query_tokens]
|
||||
assert decode_query.shape[0] == decode_query_tokens
|
||||
|
||||
# Convert query to the expected dtype for decode if needed.
|
||||
decode_query = self.maybe_quant_query(
|
||||
@@ -2467,7 +2474,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
)
|
||||
decode_query = canonicalize_singleton_dim_strides(decode_query)
|
||||
|
||||
if decode_with_dedicated_xqa:
|
||||
if decode_with_xqa:
|
||||
bmm1_scale = self.get_xqa_bmm1_scale(
|
||||
layer, attn_metadata.q_data_type_decode
|
||||
)
|
||||
@@ -2483,7 +2490,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
bmm1_scale=bmm1_scale,
|
||||
bmm2_scale=self.bmm2_scale,
|
||||
window_left=self.window_left,
|
||||
out=output[:num_decode_tokens],
|
||||
out=output_padded[:decode_query_tokens],
|
||||
sinks=self.sinks,
|
||||
kv_layout=get_flashinfer_layout_string(self.kv_cache_layout),
|
||||
q_len_per_req=q_len_per_req,
|
||||
@@ -2492,6 +2499,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
)
|
||||
return output_padded
|
||||
|
||||
assert decode_with_trtllm_gen
|
||||
if output.dtype == FP4_DTYPE:
|
||||
assert self.o_sf_scale is not None
|
||||
out = FP4Tensor(
|
||||
@@ -2523,19 +2531,6 @@ class FlashInferImpl(AttentionImpl):
|
||||
else:
|
||||
q_len_per_req = num_decode_tokens // attn_metadata.num_decodes
|
||||
|
||||
if decode_with_xqa and q_len_per_req is not None and q_len_per_req > 1:
|
||||
raise NotImplementedError(
|
||||
"FlashInfer XQA speculative decode is not wired in vLLM yet."
|
||||
)
|
||||
|
||||
# XQA decode can use model-dtype Q with FP8 KV, so only include
|
||||
# q_scale when the decode query is actually FP8.
|
||||
bmm1_scale = (
|
||||
self.get_xqa_bmm1_scale(layer, attn_metadata.q_data_type_decode)
|
||||
if decode_with_xqa
|
||||
else self.bmm1_scale
|
||||
)
|
||||
|
||||
lse = None
|
||||
if use_dcp:
|
||||
out = torch.empty(
|
||||
@@ -2558,7 +2553,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
block_tables=block_tables_decode,
|
||||
seq_lens=seq_lens_decode,
|
||||
max_seq_len=attn_metadata.decode.max_seq_len,
|
||||
bmm1_scale=bmm1_scale,
|
||||
bmm1_scale=self.bmm1_scale,
|
||||
bmm2_scale=self.bmm2_scale,
|
||||
window_left=self.window_left,
|
||||
sinks=self.sinks,
|
||||
|
||||
Reference in New Issue
Block a user