Xqa decode kernels (#43232)

Signed-off-by: Dan Blanaru <48605845+DanBlanaru@users.noreply.github.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Dan Blanaru
2026-07-02 12:32:05 -07:00
committed by GitHub
co-authored by Matthew Bonanni
parent d715b3aa1e
commit d29125c085
7 changed files with 538 additions and 143 deletions
+3 -2
View File
@@ -161,7 +161,8 @@ Priority is **1 = highest** (tried first).
| ------- | ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | --------- | --- | --------------- | ------------ |
| `CPU_ATTN` | | fp16, bf16, fp32 | `auto`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 112, 128, 160, 192, 224, 256, 512 | ❌ | ✅ | ❌ | ❌ | All | N/A |
| `FLASHINFER` | Native† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ✅ | ❌ | ✅ | Decoder | 8.x-9.x |
| `FLASHINFER` | TRTLLM† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `nvfp4` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ✅ | ✅ | ❌ | ✅ | Decoder | 10.x |
| `FLASHINFER` | XQA† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ❌ | ❌ | ✅ | Decoder | 9.0 |
| `FLASHINFER` | trtllm-gen† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `nvfp4` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ✅ | ✅ | ❌ | ✅ | Decoder | 10.x |
| `FLASH_ATTN` | FA2* | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ✅ | ❌ | ✅ | All | ≥8.0 |
| `FLASH_ATTN` | FA3* | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | 9.x |
| `FLASH_ATTN` | FA4* | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | ≥10.0 |
@@ -175,7 +176,7 @@ Priority is **1 = highest** (tried first).
| `TRITON_ATTN_DIFFKV` | | fp16, bf16 | `auto`, `bfloat16` | Any | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
| `TURBOQUANT` | | fp16, bf16 | `turboquant_k8v4`, `turboquant_4bit_nc`, `turboquant_k3v4_nc`, `turboquant_3bit_nc` | 16, 32, 64, 128 | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
> **†** FlashInfer uses TRTLLM attention on Blackwell (SM100), which supports sinks. Disable via `--attention-config.use_trtllm_attention=0`.
> **†** FlashInfer Native is the regular FlashInfer path. XQA is the SM90 decode path exposed through FlashInfer's TRTLLM decode API. trtllm-gen is used on SM100 and supports sinks. Disable XQA/trtllm-gen via `--attention-config.use_trtllm_attention=0`.
>
> **\*** Specify the FlashAttention version via `--attention-config.flash_attn_version=2`, `3`, or `4`. Default is FA4 on SM100+ (Blackwell), FA3 on SM90 (Hopper), FA2 otherwise.
@@ -78,14 +78,29 @@ def test_supports_sm100_with_artifactory(_art, _cap):
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
@patch(
"vllm.utils.flashinfer.current_platform.is_device_capability", return_value=False
)
@patch(
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
return_value=False,
)
def test_supports_non_sm100_platform(_cap):
def test_supports_unsupported_platform(_family, _cap):
assert supports_trtllm_attention() is False
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
@patch("vllm.utils.flashinfer.current_platform.is_device_capability", return_value=True)
@patch(
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
return_value=False,
)
@patch("vllm.utils.flashinfer.has_nvidia_artifactory", return_value=True)
def test_supports_sm90_decode_only(_art, _family, _cap):
assert supports_trtllm_attention(is_prefill=False) is True
assert supports_trtllm_attention(is_prefill=True) is False
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
@patch(
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
+120 -4
View File
@@ -16,7 +16,7 @@ from tests.v1.attention.utils import (
try_backend_includes_kv_cache_update,
try_get_attention_backend,
)
from vllm.config import ModelConfig
from vllm.config import ModelConfig, set_current_vllm_config
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import (
@@ -24,7 +24,11 @@ from vllm.utils.torch_utils import (
is_torch_equal_or_newer,
set_random_seed,
)
from vllm.v1.attention.backend import AttentionType, CommonAttentionMetadata
from vllm.v1.attention.backend import (
AttentionCGSupport,
AttentionType,
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.backends.utils import (
set_kv_cache_layout,
@@ -626,6 +630,118 @@ def test_causal_backend_correctness(
)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
def test_flashinfer_xqa_bmm1_scale_matches_decode_q_dtype():
"""XQA decode should only apply q_scale when decode Q is FP8."""
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
class MockLayer:
_q_scale_float = 2.0
_k_scale_float = 3.0
impl = object.__new__(flashinfer_backend.FlashInferImpl)
impl.scale = 0.5
impl.kv_cache_dtype = "fp8"
assert impl.get_xqa_bmm1_scale(MockLayer, torch.bfloat16) == 1.5
assert impl.get_xqa_bmm1_scale(MockLayer, torch.float8_e4m3fn) == 3.0
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
def test_flashinfer_sm90_xqa_decode_correctness(default_vllm_config):
"""FlashInfer should route Hopper decode through XQA and match SDPA."""
if not current_platform.is_cuda() or not current_platform.is_device_capability(90):
pytest.skip("FlashInfer XQA decode requires SM90.")
import unittest.mock
from vllm.utils.flashinfer import can_use_trtllm_attention
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
from vllm.v1.attention.backends.utils import PerLayerParameters
def mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls):
return {
"placeholder": PerLayerParameters(
window_left=-1,
logits_soft_cap=0.0,
sm_scale=1.0,
)
}
def causal_mask_mod(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
):
return (q_idx + context_len) >= kv_idx
batch_spec = BATCH_SPECS["small_decode"]
vllm_config = create_vllm_config(
model_name="meta-llama/Meta-Llama-3-8B",
max_model_len=max(batch_spec.seq_lens),
block_size=16,
)
device = torch.device(f"{DEVICE_TYPE}:0")
kv_cache_spec = FullAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=vllm_config.model_config.get_num_kv_heads(
vllm_config.parallel_config
),
head_size=vllm_config.model_config.get_head_size(),
dtype=vllm_config.model_config.dtype,
)
with set_current_vllm_config(vllm_config):
if not can_use_trtllm_attention(
vllm_config.model_config.get_num_attention_heads(
vllm_config.parallel_config
),
kv_cache_spec.num_kv_heads,
is_prefill=False,
):
pytest.skip("FlashInfer XQA decode is not available in this setup.")
with unittest.mock.patch(
"vllm.v1.attention.backends.flashinfer.get_per_layer_parameters",
mock_get_per_layer_parameters,
):
builder = flashinfer_backend.FlashInferMetadataBuilder(
kv_cache_spec, ["placeholder"], vllm_config, device
)
common_attn_metadata = create_common_attn_metadata(
batch_spec, vllm_config.cache_config.block_size, device
)
attn_metadata = builder.build(0, common_attn_metadata)
assert (
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
vllm_config, kv_cache_spec
)
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
assert isinstance(
attn_metadata.decode,
flashinfer_backend.FlashInferTrtllmAPIDecode,
)
assert attn_metadata.decode.kernel == flashinfer_backend.FlashInferDecodeKernel.XQA
_test_backend_correctness(
batch_spec,
"meta-llama/Meta-Llama-3-8B",
[AttentionBackendEnum.FLASHINFER],
causal_mask_mod,
)
if current_platform.is_rocm():
# FLASH_ATTN is not supported on ROCm
SLIDING_WINDOW_BACKENDS_TO_TEST = [
@@ -656,7 +772,7 @@ else:
@pytest.mark.parametrize("model", ["microsoft/Phi-tiny-MoE-instruct"])
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
def test_sliding_window_backend_correctness(
batch_spec_name: str, model: str, tensor_parallel_size: int
default_vllm_config, batch_spec_name: str, model: str, tensor_parallel_size: int
):
"""Test backend's correctness with sliding window attention."""
@@ -718,7 +834,7 @@ def test_sliding_window_backend_correctness(
@pytest.mark.parametrize("model", ["google/embeddinggemma-300m"])
@pytest.mark.parametrize("tensor_parallel_size", [1, 2])
def test_sliding_window_encoder_backend_correctness(
batch_spec_name: str, model: str, tensor_parallel_size: int
default_vllm_config, batch_spec_name: str, model: str, tensor_parallel_size: int
):
"""Test backend's correctness with sliding window attention."""
@@ -32,9 +32,10 @@ if not current_platform.is_device_capability_family(100):
)
from vllm.v1.attention.backends.flashinfer import ( # noqa: E402
FlashInferDecodeKernel,
FlashInferImpl,
FlashInferMetadataBuilder,
TRTLLMDecode,
FlashInferTrtllmAPIDecode,
TRTLLMPrefill,
)
@@ -435,9 +436,11 @@ def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL)
f"Expected TRTLLMPrefill, got {type(attn_metadata.prefill)}"
)
if has_decodes:
assert isinstance(attn_metadata.decode, TRTLLMDecode), (
f"Expected TRTLLMDecode, got {type(attn_metadata.decode)}"
assert isinstance(attn_metadata.decode, FlashInferTrtllmAPIDecode), (
"Expected FlashInferTrtllmAPIDecode, got "
f"{type(attn_metadata.decode)}"
)
assert attn_metadata.decode.kernel == FlashInferDecodeKernel.TRTLLM_GEN
impl = FlashInferImpl(
num_heads=num_q_heads,
@@ -261,6 +261,25 @@ def _find_cc_in_function(tree: ast.AST, func_name: str) -> str | None:
return None
def _find_exact_cc_in_function(tree: ast.AST, func_name: str) -> str | None:
"""Find a compute capability from is_device_capability() calls in a function."""
for node in ast.walk(tree):
if not isinstance(node, ast.FunctionDef) or node.name != func_name:
continue
for n in ast.walk(node):
if (
isinstance(n, ast.Call)
and isinstance(n.func, ast.Attribute)
and n.func.attr == "is_device_capability"
and n.args
and isinstance(n.args[0], ast.Constant)
and isinstance(n.args[0].value, int)
):
capability = n.args[0].value
return f"{capability // 10}.{capability % 10}"
return None
# ---------------------------------------------------------------------------
# Registry and file resolution
# ---------------------------------------------------------------------------
@@ -1048,10 +1067,11 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]:
def parse_flashinfer_trtllm_features() -> dict[str, dict[str, Any]]:
"""Parse flashinfer.py to detect TRTLLM-specific features.
"""Parse flashinfer.py to detect FlashInfer TRTLLM API variants.
FLASHINFER uses TRTLLM attention on SM100 (Blackwell), which has different
capabilities (e.g., sink support) than native FlashInfer on earlier GPUs.
FLASHINFER uses XQA on SM90 and trtllm-gen on SM100 through FlashInfer's
TRTLLM decode API. These variants have different capabilities than native
FlashInfer.
"""
if not FLASHINFER_UTILS_FILE.exists():
return {}
@@ -1061,9 +1081,10 @@ def parse_flashinfer_trtllm_features() -> dict[str, dict[str, Any]]:
except Exception:
return {}
trtllm_compute_cap = _find_cc_in_function(tree, "supports_trtllm_attention")
xqa_compute_cap = _find_exact_cc_in_function(tree, "supports_trtllm_attention")
trtllm_gen_compute_cap = _find_cc_in_function(tree, "supports_trtllm_attention")
if not trtllm_compute_cap:
if not xqa_compute_cap and not trtllm_gen_compute_cap:
return {}
# KV cache dtypes that only work with a dedicated kernel (e.g. nvfp4
@@ -1073,12 +1094,17 @@ def parse_flashinfer_trtllm_features() -> dict[str, dict[str, Any]]:
return {
"native": {
# Native FlashInfer: everything except SM100
# Native FlashInfer path.
"supports_sink": False,
},
"trtllm": {
# TRTLLM pathway on Blackwell
"compute_capability": trtllm_compute_cap,
"xqa": {
# XQA decode path on Hopper.
"compute_capability": xqa_compute_cap,
"supports_sink": False,
},
"trtllm_gen": {
# trtllm-gen pathway on Blackwell.
"compute_capability": trtllm_gen_compute_cap,
"supports_sink": True,
},
"exclude_kv_dtypes": kernel_only_kv_dtypes,
@@ -1086,7 +1112,7 @@ def parse_flashinfer_trtllm_features() -> dict[str, dict[str, Any]]:
# ---------------------------------------------------------------------------
# Backend variant expansion (FA2/FA3/FA4, FlashInfer native/TRTLLM)
# Backend variant expansion (FA2/FA3/FA4, FlashInfer native/XQA/trtllm-gen)
# ---------------------------------------------------------------------------
@@ -1146,7 +1172,7 @@ def _expand_flashinfer_variants(
all_backends: list[dict[str, Any]],
fi_features: dict[str, dict[str, Any]],
) -> list[dict[str, Any]]:
"""Expand FLASHINFER into native and TRTLLM variants."""
"""Expand FLASHINFER into native, XQA, and trtllm-gen variants."""
expanded = []
for backend in all_backends:
if backend["name"] != "FLASHINFER":
@@ -1157,9 +1183,8 @@ def _expand_flashinfer_variants(
orig_cap = backend["compute_capability"]
parts = orig_cap.replace(".x", "").split("-")
min_cc = parts[0] if parts else "7"
trtllm_cc = fi_features["trtllm"]["compute_capability"]
# Create native entry (pre-Blackwell GPUs)
# Create native entry.
native = backend.copy()
native["version"] = "Native†"
native["_sort_key"] = "FLASHINFER"
@@ -1176,16 +1201,36 @@ def _expand_flashinfer_variants(
if d not in exclude
)
# Create TRTLLM entry
trtllm = backend.copy()
trtllm["version"] = "TRTLLM†"
trtllm["_sort_key"] = "FLASHINFER"
trtllm["_sort_order"] = 1
trtllm["compute_capability"] = trtllm_cc
trtllm["supports_sink"] = fi_features["trtllm"]["supports_sink"]
# Create XQA entry.
xqa = backend.copy()
xqa["version"] = "XQA†"
xqa["_sort_key"] = "FLASHINFER"
xqa["_sort_order"] = 1
xqa["compute_capability"] = fi_features["xqa"]["compute_capability"]
xqa["supports_sink"] = fi_features["xqa"]["supports_sink"]
xqa["supports_non_causal"] = False
if exclude:
xqa["kv_cache_dtypes"] = ", ".join(
d
for d in (d.strip() for d in xqa["kv_cache_dtypes"].split(","))
if d not in exclude
)
# Create trtllm-gen entry.
trtllm_gen = backend.copy()
trtllm_gen["version"] = "trtllm-gen†"
trtllm_gen["_sort_key"] = "FLASHINFER"
trtllm_gen["_sort_order"] = 2
trtllm_gen["compute_capability"] = fi_features["trtllm_gen"][
"compute_capability"
]
trtllm_gen["supports_sink"] = fi_features["trtllm_gen"]["supports_sink"]
expanded.append(native)
expanded.append(trtllm)
if fi_features["xqa"]["compute_capability"]:
expanded.append(xqa)
if fi_features["trtllm_gen"]["compute_capability"]:
expanded.append(trtllm_gen)
return expanded
@@ -1788,8 +1833,10 @@ def generate_docs() -> str:
footnotes = []
if fi_features:
footnotes.append(
"> **†** FlashInfer uses TRTLLM attention on Blackwell (SM100), which "
"supports sinks. Disable via `--attention-config.use_trtllm_attention=0`."
"> **†** FlashInfer Native is the regular FlashInfer path. XQA is the "
"SM90 decode path exposed through FlashInfer's TRTLLM decode API. "
"trtllm-gen is used on SM100 and supports sinks. Disable XQA/trtllm-gen "
"via `--attention-config.use_trtllm_attention=0`."
)
if fa_features:
footnotes.append(
+37 -18
View File
@@ -371,19 +371,27 @@ def has_nvidia_artifactory() -> bool:
@functools.cache
def supports_trtllm_attention() -> bool:
"""
TRTLLM attention is supported if the platform is SM100,
NVIDIA artifactory is accessible, and batch-invariant mode is not enabled.
def supports_trtllm_attention(is_prefill: bool = False) -> bool:
"""Return whether TRTLLM attention is available on the current platform
for the given attention phase.
SM90 (Hopper) supports the XQA decode kernel but not TRTLLM prefill.
SM100+ supports TRTLLM for both phases. All others are unsupported.
"""
# Batch-invariant mode disables TRTLLM attention
if envs.VLLM_BATCH_INVARIANT:
return False
# Requires SM100 and NVIDIA artifactory to be accessible to download cubins
return (
current_platform.is_device_capability_family(100) and has_nvidia_artifactory()
)
# Requires NVIDIA artifactory to be accessible to download cubins
if not has_nvidia_artifactory():
return False
# SM90 has XQA decode; prefill is not supported.
if current_platform.is_device_capability(90):
return not is_prefill
# SM100/SM103 has both prefill and decode TRTLLM kernels.
return current_platform.is_device_capability_family(100)
def force_use_trtllm_attention() -> bool | None:
@@ -400,12 +408,15 @@ def force_use_trtllm_attention() -> bool | None:
return vllm_config.attention_config.use_trtllm_attention
def can_use_trtllm_attention(num_qo_heads: int, num_kv_heads: int) -> bool:
def can_use_trtllm_attention(
num_qo_heads: int, num_kv_heads: int, is_prefill: bool = False
) -> bool:
"""Check if the current configuration supports TRTLLM attention."""
if force_use_trtllm_attention() is False:
return False
has_trtllm = supports_trtllm_attention()
return has_trtllm and (num_qo_heads % num_kv_heads == 0)
return supports_trtllm_attention(is_prefill=is_prefill) and (
num_qo_heads % num_kv_heads == 0
)
def use_trtllm_attention(
@@ -437,11 +448,12 @@ def use_trtllm_attention(
return False
# The platform is not supported
if not supports_trtllm_attention():
if not supports_trtllm_attention(is_prefill=is_prefill):
if force_use_trtllm:
logger.warning_once(
"TRTLLM attention is not supported on this platform, "
"but --attention-config.use_trtllm_attention is set to 1"
"TRTLLM attention is not supported on this platform for %s, "
"but --attention-config.use_trtllm_attention is set to 1",
"prefill" if is_prefill else "decode",
)
return False
@@ -476,13 +488,20 @@ def use_trtllm_attention(
if is_prefill:
# Prefill auto-detection
use_trtllm = kv_cache_dtype == "auto"
if use_trtllm:
logger.warning_once("Using TRTLLM prefill attention (auto-detected).")
elif current_platform.is_device_capability(90) and kv_cache_dtype.startswith(
"fp8"
):
# SM90 + FP8 KV cache: prefer the XQA decode kernel. XQA does not
# support NVFP4 KV (that is an SM100 trtllm-gen path only).
use_trtllm = True
else:
# Decode auto-detection
use_trtllm = num_tokens <= 256 and kv_cache_dtype == "auto"
if use_trtllm:
logger.warning_once("Using TRTLLM decode attention (auto-detected).")
if use_trtllm:
logger.warning_once(
"Using TRTLLM %s attention (auto-detected).",
"prefill" if is_prefill else "decode",
)
return use_trtllm
# CLI argument is set to 1 - respect it
+286 -92
View File
@@ -3,6 +3,7 @@
"""Attention layer with FlashInfer."""
from dataclasses import dataclass
from enum import Enum
from functools import partial
from typing import ClassVar
@@ -19,6 +20,7 @@ from flashinfer.prefill import trtllm_batch_context_with_kv_cache
from flashinfer.utils import FP4Tensor
from typing_extensions import override
from vllm import _custom_ops as custom_ops
from vllm import envs
from vllm.config import (
CUDAGraphMode,
@@ -38,6 +40,8 @@ from vllm.platforms.interface import DeviceCapability
from vllm.triton_utils import tl, triton
from vllm.utils.flashinfer import (
can_use_trtllm_attention,
force_use_trtllm_attention,
supports_trtllm_attention,
use_trtllm_attention,
)
from vllm.utils.math_utils import cdiv
@@ -84,16 +88,16 @@ FP4_DTYPE = torch.uint8
logger = init_logger(__name__)
trtllm_gen_workspace_buffer = None
trtllm_workspace_buffer = None
def _get_trtllm_gen_workspace_buffer():
global trtllm_gen_workspace_buffer
if trtllm_gen_workspace_buffer is None:
trtllm_gen_workspace_buffer = torch.zeros(
def _get_trtllm_workspace_buffer():
global trtllm_workspace_buffer
if trtllm_workspace_buffer is None:
trtllm_workspace_buffer = torch.zeros(
envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE, dtype=torch.uint8, device="cuda"
)
return trtllm_gen_workspace_buffer
return trtllm_workspace_buffer
@triton.jit
@@ -350,6 +354,7 @@ class FlashInferBackend(AttentionBackend):
use_large_pages = (
num_kv_heads > 0
and num_qo_heads // num_kv_heads > 1
and current_platform.is_device_capability_family(100)
and can_use_trtllm_attention(num_qo_heads, num_kv_heads)
)
if not use_large_pages:
@@ -418,6 +423,16 @@ class FlashInferBackend(AttentionBackend):
else:
raise ValueError(f"Unrecognized dtype: {kv_cache_dtype}")
@classmethod
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
if kv_cache_dtype == "nvfp4":
return (
current_platform.is_device_capability_family(100)
and supports_trtllm_attention(is_prefill=True)
and supports_trtllm_attention(is_prefill=False)
)
return super().supports_kv_cache_dtype(kv_cache_dtype)
@classmethod
def get_supported_head_sizes(cls) -> list[int]:
# https://github.com/flashinfer-ai/flashinfer/blob/3d55c71a62052c590c130897d3a3db49b14fcc34/include/flashinfer/utils.cuh#L157
@@ -436,10 +451,9 @@ class FlashInferBackend(AttentionBackend):
@classmethod
def supports_sink(cls) -> bool:
"""FlashInfer supports sinks when TRTLLM attention is available (SM100)."""
"""FlashInfer supports sinks only on the SM100 trtllm-gen path."""
from vllm.utils.flashinfer import (
force_use_trtllm_attention,
supports_trtllm_attention,
)
# Respect explicit disable flag (e.g.,
@@ -447,8 +461,13 @@ class FlashInferBackend(AttentionBackend):
if force_use_trtllm_attention() is False:
return False
if not current_platform.is_device_capability_family(100):
return False
# Check if TRTLLM is supported on this platform
return supports_trtllm_attention()
return supports_trtllm_attention(
is_prefill=False
) and supports_trtllm_attention(is_prefill=True)
@classmethod
def get_required_kv_cache_layout(cls) -> KVCacheLayoutType | None:
@@ -474,6 +493,13 @@ class FIDecode:
wrapper: BatchDecodeWithPagedKVCacheWrapper
class FlashInferDecodeKernel(Enum):
"""Decode kernels selected inside the FlashInfer backend."""
XQA = "xqa"
TRTLLM_GEN = "trtllm-gen"
@dataclass
class TRTLLMPrefill:
"""Metadata for the TRTLLM prefill pathway."""
@@ -503,8 +529,15 @@ class TRTLLMPrefill:
@dataclass
class TRTLLMDecode:
"""Metadata for the TRTLLM decode pathway."""
class FlashInferTrtllmAPIDecode:
"""Metadata for decode paths using FlashInfer's TRTLLM decode API.
FlashInfer exposes both XQA (SM90) and trtllm-gen (SM100) through
``trtllm_batch_decode_with_kv_cache``. Keep them as distinct vLLM
decode kernels because their dtype/layout/output constraints differ.
"""
kernel: FlashInferDecodeKernel
block_tables: torch.Tensor
"""
@@ -530,7 +563,10 @@ class FlashInferMetadata:
slot_mapping: torch.Tensor
"""Tensor for writing K/V to the cache. Shape: [num_actual_tokens]"""
q_data_type: torch.dtype
# The data types of the query for prefill and decode.
# On SM90, these two data types may be different.
q_data_type_prefill: torch.dtype
q_data_type_decode: torch.dtype
num_decodes: int
num_decode_tokens: int
@@ -544,7 +580,7 @@ class FlashInferMetadata:
Will be `None` if `num_prefill_tokens == 0`.
"""
decode: FIDecode | TRTLLMDecode | None
decode: FIDecode | FlashInferTrtllmAPIDecode | None
"""
Holds the metadata for the decode portion of the batch.
Will be `None` if `num_decode_tokens == 0`.
@@ -651,12 +687,14 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# storage dtype may not be the same as the op dtype (uint8 vs fp8_e4m3)
self.is_kvcache_nvfp4 = self.cache_dtype == "nvfp4"
if self.is_kvcache_nvfp4:
# trtllm-gen FP4 FMHA kernels only exist for sm100f (sm_100/sm_103).
# Fail fast at init rather than crashing on the first request.
if not current_platform.is_device_capability_family(100):
if (
force_use_trtllm_attention() is False
or not supports_trtllm_attention(is_prefill=True)
or not supports_trtllm_attention(is_prefill=False)
):
raise ValueError(
"--kv-cache-dtype nvfp4 requires sm100f, "
"please try a different dtype or remove"
"--kv-cache-dtype nvfp4 requires the SM100 trtllm-gen "
"FlashInfer path."
)
# For NVFP4, kv_cache_dtype stays as the string "nvfp4"
# which is passed to FlashInferImpl
@@ -671,54 +709,87 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
assert self.kv_cache_spec.dtype == self.model_config.dtype
self.kv_cache_dtype = self.kv_cache_spec.dtype
# Use model dtype as q dtype when TRTLLM attn is not supported, or
# --attention-config.disable_flashinfer_q_quantization is set to 1. Otherwise,
# try to use fp8 q if kv cache is fp8, and will fall back to model dtype
# if TRTLLM attention kernel is not used when building attn metadata
can_use_trtllm = can_use_trtllm_attention(self.num_qo_heads, self.num_kv_heads)
# Compute per-phase Q dtype. On SM90 (XQA decode), the prefill and
# decode phases require different Q dtypes when the KV cache is FP8
# (FP8-Q for the FI native prefill, BF16/FP16-Q for XQA decode),
# so both values must be tracked independently.
self.q_data_type_prefill = self.get_q_data_type(
vllm_config, self.kv_cache_spec, is_prefill=True
)
self.q_data_type_decode = self.get_q_data_type(
vllm_config, self.kv_cache_spec, is_prefill=False
)
# Prefer TRTLLM/XQA for decoding whenever supported. The decode kernel
# must be selected statically for FULL cudagraph capture.
can_use_xqa_or_trtllm_gen_decode = can_use_trtllm_attention(
self.num_qo_heads, self.num_kv_heads, is_prefill=False
)
# Page sizes >= 128 require the trtllm-gen GQA/MQA path (guaranteed by
# get_supported_kernel_block_sizes).
assert self.page_size <= 64 or (
can_use_trtllm and self.num_qo_heads // self.num_kv_heads > 1
current_platform.is_device_capability_family(100)
and can_use_xqa_or_trtllm_gen_decode
and self.num_qo_heads // self.num_kv_heads > 1
), f"Unexpected FlashInfer page size {self.page_size} without trtllm-gen GQA"
if (
can_use_trtllm
and not vllm_config.attention_config.disable_flashinfer_q_quantization
):
if self.is_kvcache_nvfp4:
# NVFP4 KV cache uses FP8 quantized queries
self.q_data_type = FlashInferBackend.get_dtype_for_flashinfer(
"fp8_e4m3"
)
else:
self.q_data_type = self.kv_cache_dtype
else:
self.q_data_type = self.model_config.dtype
# Prefer TRTLLM attention for decoding in all cases.
# This allows us to use AttentionCGSupport.UNIFORM_BATCH mode.
self.use_trtllm_decode_attention = can_use_trtllm
self._init_reorder_batch_threshold(1, supports_spec_as_decode=can_use_trtllm)
self.use_trtllm_decode_attention = can_use_xqa_or_trtllm_gen_decode
self.flashinfer_trtllm_api_decode_kernel: FlashInferDecodeKernel | None = (
self._get_flashinfer_trtllm_api_decode_kernel()
if can_use_xqa_or_trtllm_gen_decode
else None
)
supports_spec_as_decode = (
self.flashinfer_trtllm_api_decode_kernel
== FlashInferDecodeKernel.TRTLLM_GEN
)
self._init_reorder_batch_threshold(
1, supports_spec_as_decode=supports_spec_as_decode
)
self._cascade_wrapper = None # Wrapper for cascade attention
# Global hyperparameters shared by all attention layers
# TODO: discard this for trtllm-gen backend
self.global_hyperparameters = infer_global_hyperparameters(
get_per_layer_parameters(vllm_config, layer_names, FlashInferImpl)
per_layer_parameters = get_per_layer_parameters(
vllm_config, layer_names, FlashInferImpl
)
if current_platform.is_device_capability(90) and any(
params.window_left != -1 for params in per_layer_parameters.values()
):
# FlashInfer SM90 sliding-window prefill is not reliable with FP8-Q:
# https://github.com/flashinfer-ai/flashinfer/issues/3578
raise NotImplementedError(
"FlashInfer backend on SM90 currently crashes with "
"sliding-window attention layers. Use the default attention "
"backend."
)
self.global_hyperparameters = infer_global_hyperparameters(per_layer_parameters)
self.sm_scale = self.global_hyperparameters.sm_scale
self.window_left = self.global_hyperparameters.window_left
self.logits_soft_cap = self.global_hyperparameters.logits_soft_cap
self.has_sinks = self.global_hyperparameters.has_sinks
if self.has_sinks and not can_use_trtllm:
if self.has_sinks and not FlashInferBackend.supports_sink():
raise NotImplementedError(
"FlashInfer backend currently does not support attention "
"sinks, please use trtllm on blackwell or flash attention on "
"earlier GPUs."
)
capability = current_platform.get_device_capability()
arch = f"sm{capability.major}{capability.minor}" if capability else "unknown"
decode_backend = (
self.flashinfer_trtllm_api_decode_kernel.value
if self.flashinfer_trtllm_api_decode_kernel is not None
else "flashinfer-native"
)
logger.info_once(
"FlashInfer resolved query dtypes: prefill=%s, decode=%s, "
"decode_backend=%s, kv_cache_dtype=%s, arch=%s",
self.q_data_type_prefill,
self.q_data_type_decode,
decode_backend,
self.kv_cache_dtype,
arch,
)
# Preparing persistent buffers
# Since we do not have explicit synchronization in ModelRunnerV2, we do not pin
# reused CPU buffers to avoid a race condition between step N async copies to
@@ -731,6 +802,37 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
self.paged_kv_indices = self._make_buffer(max_num_pages)
self.paged_kv_last_page_len = self._make_buffer(max_num_reqs)
# Keep SM90 prefill/decode Q dtype selection in one place.
@classmethod
def get_q_data_type(
cls,
vllm_config: VllmConfig,
kv_cache_spec: AttentionSpec,
is_prefill: bool,
) -> torch.dtype:
# The user sets --attention-config.disable_flashinfer_q_quantization
# to 1 explicitly, use model dtype for query.
if vllm_config.attention_config.disable_flashinfer_q_quantization:
return vllm_config.model_config.dtype
# On SM90, XQA decode requires BF16/FP16-Q even with FP8 KV cache.
# FI native prefill on SM90 still uses FP8-Q in that case.
cache_dtype = vllm_config.cache_config.cache_dtype
if (
current_platform.is_device_capability(90)
and not is_prefill
and force_use_trtllm_attention() is not False
and cache_dtype.startswith("fp8")
):
return vllm_config.model_config.dtype
# Otherwise, match Q dtype to the KV cache dtype.
if cache_dtype.startswith("fp8"):
return FlashInferBackend.get_dtype_for_flashinfer(cache_dtype)
if cache_dtype == "nvfp4":
return FlashInferBackend.get_dtype_for_flashinfer("fp8_e4m3")
return kv_cache_spec.dtype
def _make_buffer(
self, *size: int | torch.SymInt, dtype: torch.dtype = torch.int32
) -> CpuGpuBuffer:
@@ -751,12 +853,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
) -> AttentionCGSupport:
"""Get the cudagraph support level for FlashInfer attention.
This depends on whether we can use TRTLLM attention for decodes, since we can
only do UNIFORM_SINGLE_TOKEN_DECODE if it is unavailable.
To check this, we must call can_use_trtllm_attention with the number of KV
heads from the kv_cache_spec. We check all available KV cache specs and
only return UNIFORM_BATCH if all of them support TRTLLM attention.
The SM90 XQA integration only enables single-token decode today. Keep
specdec CUDA graphs limited to trtllm-gen until vLLM wires the XQA
specdec mask.
"""
if current_platform.is_device_capability(90):
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
# For UniformTypeKVCacheSpecs, check all contained specs
kv_specs = (
kv_cache_spec.kv_cache_specs.values()
@@ -775,6 +878,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
if not can_use_trtllm_attention(
num_qo_heads=num_qo_heads,
num_kv_heads=spec.num_kv_heads,
is_prefill=False,
):
has_trtllm_support = False
break
@@ -797,6 +901,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
def set_workspace_buffer(self, workspace_buffer: torch.Tensor):
self._workspace_buffer = workspace_buffer
@staticmethod
def _get_flashinfer_trtllm_api_decode_kernel() -> FlashInferDecodeKernel:
if current_platform.is_device_capability(90):
return FlashInferDecodeKernel.XQA
assert current_platform.is_device_capability_family(100)
return FlashInferDecodeKernel.TRTLLM_GEN
def _get_prefill_wrapper(
self,
causal: bool = True,
@@ -976,7 +1087,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# Step 1: Decide which dispatch modes to use:
# - Cascade attention (distinct mode)
# - Prefill (FI native or TRTLLM)
# - Decode (FI native or TRTLLM)
# - Decode (FI native, XQA, or trtllm-gen)
use_cascade = common_prefix_len > 0
uses_spec_reorder = self.reorder_batch_threshold > 1
# Page sizes >= 128 must use trtllm-gen; force it for prefill too.
@@ -990,13 +1101,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
max_seq_len,
self.dcp_world_size,
self.cache_dtype,
self.q_data_type,
self.q_data_type_prefill,
is_prefill=True,
force_use_trtllm=prefill_force_trtllm,
has_sinks=self.has_sinks,
has_spec=uses_spec_reorder,
)
decode_use_trtllm = (
decode_with_flashinfer_trtllm_api = (
causal and self.use_trtllm_decode_attention and self.dcp_world_size <= 1
)
@@ -1011,7 +1122,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
)
all_uses_trtllm = causal and (
(num_prefills == 0 or prefill_use_trtllm)
and (num_decodes == 0 or decode_use_trtllm)
and (num_decodes == 0 or decode_with_flashinfer_trtllm_api)
)
if not all_uses_trtllm:
@@ -1035,17 +1146,14 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
"`sm_scale`."
)
# The q quantization is not supported for non-trtllm attention,
# fall back to model dtype.
self.q_data_type = self.model_config.dtype
# Step 2: Initialize the output metadata
# Leave prefill/decode/cascade_wrapper empty, to be populated
# case by case depending on the batch contents and backend selection.
attn_metadata = FlashInferMetadata(
num_actual_tokens=num_actual_tokens,
slot_mapping=common_attn_metadata.slot_mapping,
q_data_type=self.q_data_type,
q_data_type_prefill=self.q_data_type_prefill,
q_data_type_decode=self.q_data_type_decode,
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
num_prefills=num_prefills,
@@ -1101,7 +1209,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
# Compute paged_kv_indices if necessary
# paged_kv_indices is only needed for FlashInfer native paths;
# TRTLLM paths use block_tables directly on GPU.
# XQA/trtllm-gen paths use block_tables directly on GPU.
needs_paged_kv_indices = use_cascade or not all_uses_trtllm
if needs_paged_kv_indices:
assert num_blocks_np is not None
@@ -1143,6 +1251,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[:num_reqs]
attn_metadata.cascade_wrapper = self._get_cascade_wrapper()
# Cascade attention must use the same q dtype for prefill and decode
# because it does not support FP8 kv-cache or FP8 query yet.
assert self.q_data_type_prefill == self.q_data_type_decode
attn_metadata.cascade_wrapper.plan(
qo_indptr_arr=[shared_qo_indptr_cpu, qo_indptr_cpu],
paged_kv_indptr_arr=[shared_kv_page_indptr_cpu, paged_kv_indptr_cpu],
@@ -1159,7 +1270,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
sm_scale=self.sm_scale,
window_left=self.window_left,
logits_soft_cap=self.logits_soft_cap,
q_data_type=self.q_data_type,
q_data_type=self.q_data_type_prefill,
kv_data_type=self.kv_cache_dtype,
)
return attn_metadata
@@ -1233,7 +1344,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
sm_scale=self.sm_scale,
window_left=self.window_left,
logits_soft_cap=self.logits_soft_cap,
q_data_type=self.q_data_type,
q_data_type=self.q_data_type_prefill,
kv_cache_dtype=self.kv_cache_dtype,
prefill_fixed_split_size=self.prefill_fixed_split_size,
disable_split_kv=self.disable_split_kv,
@@ -1262,7 +1373,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
sm_scale=self.sm_scale,
window_left=self.window_left,
logits_soft_cap=self.logits_soft_cap,
q_data_type=self.q_data_type,
q_data_type=self.q_data_type_prefill,
kv_data_type=self.kv_cache_dtype,
o_data_type=o_dtype,
fixed_split_size=self.prefill_fixed_split_size,
@@ -1272,12 +1383,14 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
## DECODE PATHWAY
if num_decodes > 0:
if decode_use_trtllm:
if decode_with_flashinfer_trtllm_api:
assert num_decode_tokens % num_decodes == 0, (
"TRTLLM decode requires uniform query lengths per request. "
"XQA/trtllm-gen decode requires uniform query lengths per request. "
f"Got {num_decode_tokens=} and {num_decodes=}."
)
attn_metadata.decode = TRTLLMDecode(
assert self.flashinfer_trtllm_api_decode_kernel is not None
attn_metadata.decode = FlashInferTrtllmAPIDecode(
kernel=self.flashinfer_trtllm_api_decode_kernel,
block_tables=block_table_tensor[:num_decodes],
seq_lens=seq_lens[:num_decodes],
max_seq_len=max_seq_len,
@@ -1320,7 +1433,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
sm_scale=self.sm_scale,
window_left=self.window_left,
logits_soft_cap=self.logits_soft_cap,
q_data_type=self.q_data_type,
q_data_type=self.q_data_type_decode,
kv_data_type=self.kv_cache_dtype,
o_data_type=o_dtype,
fixed_split_size=self.decode_fixed_split_size,
@@ -1396,10 +1509,17 @@ class FlashInferImpl(AttentionImpl):
)
self.sinks = sinks
self.support_trtllm_attn = can_use_trtllm_attention(num_heads, num_kv_heads)
self.supports_xqa_or_trtllm_gen_decode = can_use_trtllm_attention(
num_heads, num_kv_heads, is_prefill=False
)
vllm_config = get_current_vllm_config_or_none()
# Query pre-quantization needs a single dtype for the whole query tensor.
# SM90 XQA needs BF16/FP16-Q for decode and FP8 for prefill,
# so only enable this for SM100 trtllm-gen where both use FP8-Q.
self.supports_quant_query_input = (
self.support_trtllm_attn
self.supports_xqa_or_trtllm_gen_decode
and is_quantized_kv_cache(self.kv_cache_dtype)
and current_platform.is_device_capability_family(100)
and vllm_config is not None
and not vllm_config.attention_config.disable_flashinfer_q_quantization
)
@@ -1429,9 +1549,12 @@ class FlashInferImpl(AttentionImpl):
self.dcp_combine = partial(cp_lse_ag_out_rs, is_lse_base_on_e=False)
def fused_output_quant_supported(self, quant_key: QuantKey):
# XQA does not support FP8/NVFP4 output, so require trtllm-gen
# (SM100+) here. Without that we cannot fuse the output quant.
return (
self.support_trtllm_attn
self.supports_xqa_or_trtllm_gen_decode
and is_quantized_kv_cache(self.kv_cache_dtype)
and current_platform.is_device_capability_family(100)
and quant_key in (kFp8StaticTensorSym, kNvfp4Dynamic)
)
@@ -1440,6 +1563,37 @@ class FlashInferImpl(AttentionImpl):
if self.sinks is not None and self.sinks.dtype != torch.float32:
self.sinks = self.sinks.to(torch.float32)
def get_xqa_bmm1_scale(self, layer: torch.nn.Module, q_data_type: torch.dtype):
bmm1_scale = self.scale
if is_quantized_kv_cache(self.kv_cache_dtype):
if q_data_type in (torch.float8_e4m3fn, torch.float8_e5m2):
bmm1_scale *= layer._q_scale_float
bmm1_scale *= layer._k_scale_float
return bmm1_scale
# SM90 may need FP8-Q for native prefill and BF16/FP16-Q for XQA decode,
# so quantize only the slice whose target dtype differs.
def maybe_quant_query(
self,
query: torch.Tensor,
q_data_type: torch.dtype,
scale: torch.Tensor,
) -> torch.Tensor:
if query.dtype != q_data_type:
assert query.dtype in [torch.float16, torch.bfloat16]
assert q_data_type in [torch.float8_e4m3fn, torch.float8_e5m2]
assert query.is_contiguous()
assert query.dim() == 3
num_tokens = query.shape[0]
num_heads = query.shape[1]
head_size = query.shape[2]
query_quantized, _ = custom_ops.scaled_fp8_quant(
query.view(num_tokens, num_heads * head_size), scale=scale
)
return query_quantized.view(num_tokens, num_heads, head_size)
return query
def forward(
self,
layer: torch.nn.Module,
@@ -1469,12 +1623,6 @@ class FlashInferImpl(AttentionImpl):
# Profiling run.
return output.fill_(0)
# Ensure query dtype matches the expected dtype from attention metadata
assert attn_metadata.q_data_type == query.dtype, (
f"Query dtype mismatch: expected {attn_metadata.q_data_type}, "
f"got {query.dtype}"
)
if self.bmm1_scale is None:
self.bmm1_scale = self.scale
if is_quantized_kv_cache(self.kv_cache_dtype):
@@ -1486,7 +1634,14 @@ class FlashInferImpl(AttentionImpl):
self.bmm2_scale *= layer._v_scale_float
prefill_use_trtllm = isinstance(attn_metadata.prefill, TRTLLMPrefill)
decode_use_trtllm = isinstance(attn_metadata.decode, TRTLLMDecode)
decode_kernel = (
attn_metadata.decode.kernel
if isinstance(attn_metadata.decode, FlashInferTrtllmAPIDecode)
else None
)
decode_with_xqa = decode_kernel == FlashInferDecodeKernel.XQA
decode_with_trtllm_gen = decode_kernel == FlashInferDecodeKernel.TRTLLM_GEN
decode_with_flashinfer_trtllm_api = decode_with_xqa or decode_with_trtllm_gen
# The attn+quant fusion happens when output_scale is provided.
if output_scale is None:
@@ -1494,12 +1649,15 @@ class FlashInferImpl(AttentionImpl):
"output_block_scale is not supported when fusion has not happened"
)
else:
assert attn_metadata.q_data_type == FP8_DTYPE, (
"Query must be FP8 when attn+quant fusion happened."
assert attn_metadata.q_data_type_prefill == FP8_DTYPE, (
"Query must be FP8 when attn+quant fusion happened for prefill."
)
assert attn_metadata.q_data_type_decode == FP8_DTYPE, (
"Query must be FP8 when attn+quant fusion happened for decode."
)
assert (attn_metadata.num_prefills == 0 or prefill_use_trtllm) and (
attn_metadata.num_decodes == 0 or decode_use_trtllm
), "Must use TRT-LLM attn"
attn_metadata.num_decodes == 0 or decode_with_trtllm_gen
), "Output quant fusion requires TRTLLM prefill/trtllm-gen decode"
if output.dtype == FP8_DTYPE:
assert output_block_scale is None, (
@@ -1597,6 +1755,13 @@ class FlashInferImpl(AttentionImpl):
prefill_query = query[num_decode_tokens:]
assert prefill_query.shape[0] == num_prefill_tokens
# Convert query to the expected dtype for prefill if needed.
prefill_query = self.maybe_quant_query(
prefill_query,
attn_metadata.q_data_type_prefill,
layer._q_scale,
)
if not prefill_use_trtllm:
assert isinstance(attn_metadata.prefill, FIPrefill)
prefill_wrapper = attn_metadata.prefill.wrapper
@@ -1655,6 +1820,7 @@ class FlashInferImpl(AttentionImpl):
prefill_wrapper.run(
prefill_query,
kv_cache_permute,
q_scale=layer._q_scale_float,
k_scale=layer._k_scale_float,
v_scale=layer._v_scale_float,
out=out_prefill,
@@ -1673,7 +1839,7 @@ class FlashInferImpl(AttentionImpl):
# degenerate strides on size=1 dims for TMA alignment.
prefill_query = prefill_query.contiguous()
prefill_query = canonicalize_singleton_dim_strides(prefill_query)
workspace_buffer = _get_trtllm_gen_workspace_buffer()
workspace_buffer = _get_trtllm_workspace_buffer()
block_tables_prefill = attn_metadata.prefill.block_tables
seq_lens_prefill = attn_metadata.prefill.seq_lens
@@ -1705,7 +1871,7 @@ class FlashInferImpl(AttentionImpl):
prefill_kv_block_scales = None
if self.is_kvcache_nvfp4:
# NVFP4 trtllm-gen kernel requires FP8 query.
assert attn_metadata.q_data_type == FP8_DTYPE, (
assert attn_metadata.q_data_type_prefill == FP8_DTYPE, (
"NVFP4 KV cache requires FP8 quantized queries for "
"trtllm-gen prefill. Set "
"disable_flashinfer_q_quantization=False."
@@ -1714,7 +1880,7 @@ class FlashInferImpl(AttentionImpl):
mock_block_table = block_tables_prefill
prefill_kv_block_scales = nvfp4_kv_block_scales
elif (
attn_metadata.q_data_type != FP8_DTYPE
attn_metadata.q_data_type_prefill != FP8_DTYPE
and self.kv_cache_dtype.startswith("fp8")
):
# TRTLLM prefill attention does not support BF16 Q
@@ -1738,7 +1904,7 @@ class FlashInferImpl(AttentionImpl):
block_tables_prefill,
layer._k_scale,
layer._v_scale,
attn_metadata.q_data_type,
attn_metadata.q_data_type_prefill,
)
else:
mock_kv_cache = kv_cache_permute
@@ -1773,7 +1939,14 @@ class FlashInferImpl(AttentionImpl):
decode_query = query[:num_decode_tokens]
assert decode_query.shape[0] == num_decode_tokens
if not decode_use_trtllm:
# Convert query to the expected dtype for decode if needed.
decode_query = self.maybe_quant_query(
decode_query,
attn_metadata.q_data_type_decode,
layer._q_scale,
)
if not decode_with_flashinfer_trtllm_api:
assert isinstance(attn_metadata.decode, FIDecode)
decode_wrapper = attn_metadata.decode.wrapper
assert decode_wrapper is not None
@@ -1806,6 +1979,7 @@ class FlashInferImpl(AttentionImpl):
decode_wrapper.run(
decode_query,
kv_cache_permute,
q_scale=layer._q_scale_float,
k_scale=layer._k_scale_float,
v_scale=layer._v_scale_float,
out=output_tmp,
@@ -1822,6 +1996,7 @@ class FlashInferImpl(AttentionImpl):
decode_wrapper.run(
decode_query,
kv_cache_permute,
q_scale=layer._q_scale_float,
k_scale=layer._k_scale_float,
v_scale=layer._v_scale_float,
out=out_decode,
@@ -1831,19 +2006,23 @@ class FlashInferImpl(AttentionImpl):
if needs_fp8_out:
output[:num_decode_tokens].copy_(out_decode.to(output.dtype))
else:
assert isinstance(attn_metadata.decode, TRTLLMDecode)
assert isinstance(attn_metadata.decode, FlashInferTrtllmAPIDecode)
# decode_query may be non-contiguous or have degenerate strides
# on size=1 dims. contiguous() ensures memory layout; then
# canonicalize_singleton_dim_strides fixes any remaining
# degenerate strides on size=1 dims for TMA alignment.
decode_query = decode_query.contiguous()
decode_query = canonicalize_singleton_dim_strides(decode_query)
workspace_buffer = _get_trtllm_gen_workspace_buffer()
workspace_buffer = _get_trtllm_workspace_buffer()
block_tables_decode = attn_metadata.decode.block_tables
seq_lens_decode = attn_metadata.decode.seq_lens
# This path needs to be enabled with VLLM_KV_CACHE_LAYOUT = HND
assert get_kv_cache_layout() == "HND"
# trtllm-gen needs HND layout on SM100. XQA is selected
# separately on SM90 and does not use this SM100 layout gate.
if decode_with_trtllm_gen:
assert get_kv_cache_layout() == "HND"
else:
assert decode_with_xqa
assert is_strictly_contiguous(decode_query)
assert is_strictly_contiguous(workspace_buffer)
assert is_strictly_contiguous(block_tables_decode)
@@ -1882,6 +2061,19 @@ class FlashInferImpl(AttentionImpl):
else:
q_len_per_req = num_decode_tokens // attn_metadata.num_decodes
if decode_with_xqa and q_len_per_req > 1:
raise NotImplementedError(
"FlashInfer XQA speculative decode is not wired in vLLM yet."
)
# XQA decode can use model-dtype Q with FP8 KV, so only include
# q_scale when the decode query is actually FP8.
bmm1_scale = (
self.get_xqa_bmm1_scale(layer, attn_metadata.q_data_type_decode)
if decode_with_xqa
else self.bmm1_scale
)
trtllm_batch_decode_with_kv_cache(
query=decode_query,
kv_cache=(
@@ -1891,12 +2083,14 @@ class FlashInferImpl(AttentionImpl):
block_tables=block_tables_decode,
seq_lens=seq_lens_decode,
max_seq_len=attn_metadata.decode.max_seq_len,
bmm1_scale=self.bmm1_scale,
bmm1_scale=bmm1_scale,
bmm2_scale=self.bmm2_scale,
window_left=self.window_left,
sinks=self.sinks,
o_sf_scale=self.o_sf_scale,
out=out,
kv_layout=get_kv_cache_layout(),
backend=attn_metadata.decode.kernel.value,
q_len_per_req=q_len_per_req,
kv_cache_sf=(
nvfp4_kv_block_scales if self.is_kvcache_nvfp4 else None