Xqa decode kernels (#43232)
Signed-off-by: Dan Blanaru <48605845+DanBlanaru@users.noreply.github.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
parent
d715b3aa1e
commit
d29125c085
@@ -161,7 +161,8 @@ Priority is **1 = highest** (tried first).
|
||||
| ------- | ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | --------- | --- | --------------- | ------------ |
|
||||
| `CPU_ATTN` | | fp16, bf16, fp32 | `auto`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 112, 128, 160, 192, 224, 256, 512 | ❌ | ✅ | ❌ | ❌ | All | N/A |
|
||||
| `FLASHINFER` | Native† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ✅ | ❌ | ✅ | Decoder | 8.x-9.x |
|
||||
| `FLASHINFER` | TRTLLM† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `nvfp4` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ✅ | ✅ | ❌ | ✅ | Decoder | 10.x |
|
||||
| `FLASHINFER` | XQA† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ❌ | ❌ | ✅ | Decoder | 9.0 |
|
||||
| `FLASHINFER` | trtllm-gen† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `nvfp4` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ✅ | ✅ | ❌ | ✅ | Decoder | 10.x |
|
||||
| `FLASH_ATTN` | FA2* | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ✅ | ❌ | ✅ | All | ≥8.0 |
|
||||
| `FLASH_ATTN` | FA3* | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | 9.x |
|
||||
| `FLASH_ATTN` | FA4* | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | ≥10.0 |
|
||||
@@ -175,7 +176,7 @@ Priority is **1 = highest** (tried first).
|
||||
| `TRITON_ATTN_DIFFKV` | | fp16, bf16 | `auto`, `bfloat16` | Any | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
|
||||
| `TURBOQUANT` | | fp16, bf16 | `turboquant_k8v4`, `turboquant_4bit_nc`, `turboquant_k3v4_nc`, `turboquant_3bit_nc` | 16, 32, 64, 128 | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |
|
||||
|
||||
> **†** FlashInfer uses TRTLLM attention on Blackwell (SM100), which supports sinks. Disable via `--attention-config.use_trtllm_attention=0`.
|
||||
> **†** FlashInfer Native is the regular FlashInfer path. XQA is the SM90 decode path exposed through FlashInfer's TRTLLM decode API. trtllm-gen is used on SM100 and supports sinks. Disable XQA/trtllm-gen via `--attention-config.use_trtllm_attention=0`.
|
||||
>
|
||||
> **\*** Specify the FlashAttention version via `--attention-config.flash_attn_version=2`, `3`, or `4`. Default is FA4 on SM100+ (Blackwell), FA3 on SM90 (Hopper), FA2 otherwise.
|
||||
|
||||
|
||||
@@ -78,14 +78,29 @@ def test_supports_sm100_with_artifactory(_art, _cap):
|
||||
|
||||
|
||||
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability", return_value=False
|
||||
)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
|
||||
return_value=False,
|
||||
)
|
||||
def test_supports_non_sm100_platform(_cap):
|
||||
def test_supports_unsupported_platform(_family, _cap):
|
||||
assert supports_trtllm_attention() is False
|
||||
|
||||
|
||||
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
|
||||
@patch("vllm.utils.flashinfer.current_platform.is_device_capability", return_value=True)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
|
||||
return_value=False,
|
||||
)
|
||||
@patch("vllm.utils.flashinfer.has_nvidia_artifactory", return_value=True)
|
||||
def test_supports_sm90_decode_only(_art, _family, _cap):
|
||||
assert supports_trtllm_attention(is_prefill=False) is True
|
||||
assert supports_trtllm_attention(is_prefill=True) is False
|
||||
|
||||
|
||||
@patch("vllm.envs.VLLM_BATCH_INVARIANT", False)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.current_platform.is_device_capability_family",
|
||||
|
||||
@@ -16,7 +16,7 @@ from tests.v1.attention.utils import (
|
||||
try_backend_includes_kv_cache_update,
|
||||
try_get_attention_backend,
|
||||
)
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config import ModelConfig, set_current_vllm_config
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import (
|
||||
@@ -24,7 +24,11 @@ from vllm.utils.torch_utils import (
|
||||
is_torch_equal_or_newer,
|
||||
set_random_seed,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionType, CommonAttentionMetadata
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionCGSupport,
|
||||
AttentionType,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
set_kv_cache_layout,
|
||||
@@ -626,6 +630,118 @@ def test_causal_backend_correctness(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_xqa_bmm1_scale_matches_decode_q_dtype():
|
||||
"""XQA decode should only apply q_scale when decode Q is FP8."""
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
|
||||
class MockLayer:
|
||||
_q_scale_float = 2.0
|
||||
_k_scale_float = 3.0
|
||||
|
||||
impl = object.__new__(flashinfer_backend.FlashInferImpl)
|
||||
impl.scale = 0.5
|
||||
impl.kv_cache_dtype = "fp8"
|
||||
|
||||
assert impl.get_xqa_bmm1_scale(MockLayer, torch.bfloat16) == 1.5
|
||||
assert impl.get_xqa_bmm1_scale(MockLayer, torch.float8_e4m3fn) == 3.0
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
||||
reason="FlashInfer is not available.",
|
||||
)
|
||||
def test_flashinfer_sm90_xqa_decode_correctness(default_vllm_config):
|
||||
"""FlashInfer should route Hopper decode through XQA and match SDPA."""
|
||||
if not current_platform.is_cuda() or not current_platform.is_device_capability(90):
|
||||
pytest.skip("FlashInfer XQA decode requires SM90.")
|
||||
|
||||
import unittest.mock
|
||||
|
||||
from vllm.utils.flashinfer import can_use_trtllm_attention
|
||||
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
||||
from vllm.v1.attention.backends.utils import PerLayerParameters
|
||||
|
||||
def mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls):
|
||||
return {
|
||||
"placeholder": PerLayerParameters(
|
||||
window_left=-1,
|
||||
logits_soft_cap=0.0,
|
||||
sm_scale=1.0,
|
||||
)
|
||||
}
|
||||
|
||||
def causal_mask_mod(
|
||||
b: torch.Tensor,
|
||||
h: torch.Tensor,
|
||||
q_idx: torch.Tensor,
|
||||
kv_idx: torch.Tensor,
|
||||
*,
|
||||
context_len: int,
|
||||
):
|
||||
return (q_idx + context_len) >= kv_idx
|
||||
|
||||
batch_spec = BATCH_SPECS["small_decode"]
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="meta-llama/Meta-Llama-3-8B",
|
||||
max_model_len=max(batch_spec.seq_lens),
|
||||
block_size=16,
|
||||
)
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
kv_cache_spec = FullAttentionSpec(
|
||||
block_size=vllm_config.cache_config.block_size,
|
||||
num_kv_heads=vllm_config.model_config.get_num_kv_heads(
|
||||
vllm_config.parallel_config
|
||||
),
|
||||
head_size=vllm_config.model_config.get_head_size(),
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
if not can_use_trtllm_attention(
|
||||
vllm_config.model_config.get_num_attention_heads(
|
||||
vllm_config.parallel_config
|
||||
),
|
||||
kv_cache_spec.num_kv_heads,
|
||||
is_prefill=False,
|
||||
):
|
||||
pytest.skip("FlashInfer XQA decode is not available in this setup.")
|
||||
|
||||
with unittest.mock.patch(
|
||||
"vllm.v1.attention.backends.flashinfer.get_per_layer_parameters",
|
||||
mock_get_per_layer_parameters,
|
||||
):
|
||||
builder = flashinfer_backend.FlashInferMetadataBuilder(
|
||||
kv_cache_spec, ["placeholder"], vllm_config, device
|
||||
)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, vllm_config.cache_config.block_size, device
|
||||
)
|
||||
attn_metadata = builder.build(0, common_attn_metadata)
|
||||
|
||||
assert (
|
||||
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
|
||||
vllm_config, kv_cache_spec
|
||||
)
|
||||
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
)
|
||||
assert isinstance(
|
||||
attn_metadata.decode,
|
||||
flashinfer_backend.FlashInferTrtllmAPIDecode,
|
||||
)
|
||||
assert attn_metadata.decode.kernel == flashinfer_backend.FlashInferDecodeKernel.XQA
|
||||
|
||||
_test_backend_correctness(
|
||||
batch_spec,
|
||||
"meta-llama/Meta-Llama-3-8B",
|
||||
[AttentionBackendEnum.FLASHINFER],
|
||||
causal_mask_mod,
|
||||
)
|
||||
|
||||
|
||||
if current_platform.is_rocm():
|
||||
# FLASH_ATTN is not supported on ROCm
|
||||
SLIDING_WINDOW_BACKENDS_TO_TEST = [
|
||||
@@ -656,7 +772,7 @@ else:
|
||||
@pytest.mark.parametrize("model", ["microsoft/Phi-tiny-MoE-instruct"])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
|
||||
def test_sliding_window_backend_correctness(
|
||||
batch_spec_name: str, model: str, tensor_parallel_size: int
|
||||
default_vllm_config, batch_spec_name: str, model: str, tensor_parallel_size: int
|
||||
):
|
||||
"""Test backend's correctness with sliding window attention."""
|
||||
|
||||
@@ -718,7 +834,7 @@ def test_sliding_window_backend_correctness(
|
||||
@pytest.mark.parametrize("model", ["google/embeddinggemma-300m"])
|
||||
@pytest.mark.parametrize("tensor_parallel_size", [1, 2])
|
||||
def test_sliding_window_encoder_backend_correctness(
|
||||
batch_spec_name: str, model: str, tensor_parallel_size: int
|
||||
default_vllm_config, batch_spec_name: str, model: str, tensor_parallel_size: int
|
||||
):
|
||||
"""Test backend's correctness with sliding window attention."""
|
||||
|
||||
|
||||
@@ -32,9 +32,10 @@ if not current_platform.is_device_capability_family(100):
|
||||
)
|
||||
|
||||
from vllm.v1.attention.backends.flashinfer import ( # noqa: E402
|
||||
FlashInferDecodeKernel,
|
||||
FlashInferImpl,
|
||||
FlashInferMetadataBuilder,
|
||||
TRTLLMDecode,
|
||||
FlashInferTrtllmAPIDecode,
|
||||
TRTLLMPrefill,
|
||||
)
|
||||
|
||||
@@ -435,9 +436,11 @@ def _run_trtllm_integration(batch_spec, kv_cache_dtype="auto", model_name=MODEL)
|
||||
f"Expected TRTLLMPrefill, got {type(attn_metadata.prefill)}"
|
||||
)
|
||||
if has_decodes:
|
||||
assert isinstance(attn_metadata.decode, TRTLLMDecode), (
|
||||
f"Expected TRTLLMDecode, got {type(attn_metadata.decode)}"
|
||||
assert isinstance(attn_metadata.decode, FlashInferTrtllmAPIDecode), (
|
||||
"Expected FlashInferTrtllmAPIDecode, got "
|
||||
f"{type(attn_metadata.decode)}"
|
||||
)
|
||||
assert attn_metadata.decode.kernel == FlashInferDecodeKernel.TRTLLM_GEN
|
||||
|
||||
impl = FlashInferImpl(
|
||||
num_heads=num_q_heads,
|
||||
|
||||
@@ -261,6 +261,25 @@ def _find_cc_in_function(tree: ast.AST, func_name: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _find_exact_cc_in_function(tree: ast.AST, func_name: str) -> str | None:
|
||||
"""Find a compute capability from is_device_capability() calls in a function."""
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.FunctionDef) or node.name != func_name:
|
||||
continue
|
||||
for n in ast.walk(node):
|
||||
if (
|
||||
isinstance(n, ast.Call)
|
||||
and isinstance(n.func, ast.Attribute)
|
||||
and n.func.attr == "is_device_capability"
|
||||
and n.args
|
||||
and isinstance(n.args[0], ast.Constant)
|
||||
and isinstance(n.args[0].value, int)
|
||||
):
|
||||
capability = n.args[0].value
|
||||
return f"{capability // 10}.{capability % 10}"
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Registry and file resolution
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1048,10 +1067,11 @@ def parse_flash_attn_features() -> dict[str, dict[str, Any]]:
|
||||
|
||||
|
||||
def parse_flashinfer_trtllm_features() -> dict[str, dict[str, Any]]:
|
||||
"""Parse flashinfer.py to detect TRTLLM-specific features.
|
||||
"""Parse flashinfer.py to detect FlashInfer TRTLLM API variants.
|
||||
|
||||
FLASHINFER uses TRTLLM attention on SM100 (Blackwell), which has different
|
||||
capabilities (e.g., sink support) than native FlashInfer on earlier GPUs.
|
||||
FLASHINFER uses XQA on SM90 and trtllm-gen on SM100 through FlashInfer's
|
||||
TRTLLM decode API. These variants have different capabilities than native
|
||||
FlashInfer.
|
||||
"""
|
||||
if not FLASHINFER_UTILS_FILE.exists():
|
||||
return {}
|
||||
@@ -1061,9 +1081,10 @@ def parse_flashinfer_trtllm_features() -> dict[str, dict[str, Any]]:
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
trtllm_compute_cap = _find_cc_in_function(tree, "supports_trtllm_attention")
|
||||
xqa_compute_cap = _find_exact_cc_in_function(tree, "supports_trtllm_attention")
|
||||
trtllm_gen_compute_cap = _find_cc_in_function(tree, "supports_trtllm_attention")
|
||||
|
||||
if not trtllm_compute_cap:
|
||||
if not xqa_compute_cap and not trtllm_gen_compute_cap:
|
||||
return {}
|
||||
|
||||
# KV cache dtypes that only work with a dedicated kernel (e.g. nvfp4
|
||||
@@ -1073,12 +1094,17 @@ def parse_flashinfer_trtllm_features() -> dict[str, dict[str, Any]]:
|
||||
|
||||
return {
|
||||
"native": {
|
||||
# Native FlashInfer: everything except SM100
|
||||
# Native FlashInfer path.
|
||||
"supports_sink": False,
|
||||
},
|
||||
"trtllm": {
|
||||
# TRTLLM pathway on Blackwell
|
||||
"compute_capability": trtllm_compute_cap,
|
||||
"xqa": {
|
||||
# XQA decode path on Hopper.
|
||||
"compute_capability": xqa_compute_cap,
|
||||
"supports_sink": False,
|
||||
},
|
||||
"trtllm_gen": {
|
||||
# trtllm-gen pathway on Blackwell.
|
||||
"compute_capability": trtllm_gen_compute_cap,
|
||||
"supports_sink": True,
|
||||
},
|
||||
"exclude_kv_dtypes": kernel_only_kv_dtypes,
|
||||
@@ -1086,7 +1112,7 @@ def parse_flashinfer_trtllm_features() -> dict[str, dict[str, Any]]:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Backend variant expansion (FA2/FA3/FA4, FlashInfer native/TRTLLM)
|
||||
# Backend variant expansion (FA2/FA3/FA4, FlashInfer native/XQA/trtllm-gen)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -1146,7 +1172,7 @@ def _expand_flashinfer_variants(
|
||||
all_backends: list[dict[str, Any]],
|
||||
fi_features: dict[str, dict[str, Any]],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Expand FLASHINFER into native and TRTLLM variants."""
|
||||
"""Expand FLASHINFER into native, XQA, and trtllm-gen variants."""
|
||||
expanded = []
|
||||
for backend in all_backends:
|
||||
if backend["name"] != "FLASHINFER":
|
||||
@@ -1157,9 +1183,8 @@ def _expand_flashinfer_variants(
|
||||
orig_cap = backend["compute_capability"]
|
||||
parts = orig_cap.replace(".x", "").split("-")
|
||||
min_cc = parts[0] if parts else "7"
|
||||
trtllm_cc = fi_features["trtllm"]["compute_capability"]
|
||||
|
||||
# Create native entry (pre-Blackwell GPUs)
|
||||
# Create native entry.
|
||||
native = backend.copy()
|
||||
native["version"] = "Native†"
|
||||
native["_sort_key"] = "FLASHINFER"
|
||||
@@ -1176,16 +1201,36 @@ def _expand_flashinfer_variants(
|
||||
if d not in exclude
|
||||
)
|
||||
|
||||
# Create TRTLLM entry
|
||||
trtllm = backend.copy()
|
||||
trtllm["version"] = "TRTLLM†"
|
||||
trtllm["_sort_key"] = "FLASHINFER"
|
||||
trtllm["_sort_order"] = 1
|
||||
trtllm["compute_capability"] = trtllm_cc
|
||||
trtllm["supports_sink"] = fi_features["trtllm"]["supports_sink"]
|
||||
# Create XQA entry.
|
||||
xqa = backend.copy()
|
||||
xqa["version"] = "XQA†"
|
||||
xqa["_sort_key"] = "FLASHINFER"
|
||||
xqa["_sort_order"] = 1
|
||||
xqa["compute_capability"] = fi_features["xqa"]["compute_capability"]
|
||||
xqa["supports_sink"] = fi_features["xqa"]["supports_sink"]
|
||||
xqa["supports_non_causal"] = False
|
||||
if exclude:
|
||||
xqa["kv_cache_dtypes"] = ", ".join(
|
||||
d
|
||||
for d in (d.strip() for d in xqa["kv_cache_dtypes"].split(","))
|
||||
if d not in exclude
|
||||
)
|
||||
|
||||
# Create trtllm-gen entry.
|
||||
trtllm_gen = backend.copy()
|
||||
trtllm_gen["version"] = "trtllm-gen†"
|
||||
trtllm_gen["_sort_key"] = "FLASHINFER"
|
||||
trtllm_gen["_sort_order"] = 2
|
||||
trtllm_gen["compute_capability"] = fi_features["trtllm_gen"][
|
||||
"compute_capability"
|
||||
]
|
||||
trtllm_gen["supports_sink"] = fi_features["trtllm_gen"]["supports_sink"]
|
||||
|
||||
expanded.append(native)
|
||||
expanded.append(trtllm)
|
||||
if fi_features["xqa"]["compute_capability"]:
|
||||
expanded.append(xqa)
|
||||
if fi_features["trtllm_gen"]["compute_capability"]:
|
||||
expanded.append(trtllm_gen)
|
||||
return expanded
|
||||
|
||||
|
||||
@@ -1788,8 +1833,10 @@ def generate_docs() -> str:
|
||||
footnotes = []
|
||||
if fi_features:
|
||||
footnotes.append(
|
||||
"> **†** FlashInfer uses TRTLLM attention on Blackwell (SM100), which "
|
||||
"supports sinks. Disable via `--attention-config.use_trtllm_attention=0`."
|
||||
"> **†** FlashInfer Native is the regular FlashInfer path. XQA is the "
|
||||
"SM90 decode path exposed through FlashInfer's TRTLLM decode API. "
|
||||
"trtllm-gen is used on SM100 and supports sinks. Disable XQA/trtllm-gen "
|
||||
"via `--attention-config.use_trtllm_attention=0`."
|
||||
)
|
||||
if fa_features:
|
||||
footnotes.append(
|
||||
|
||||
+37
-18
@@ -371,19 +371,27 @@ def has_nvidia_artifactory() -> bool:
|
||||
|
||||
|
||||
@functools.cache
|
||||
def supports_trtllm_attention() -> bool:
|
||||
"""
|
||||
TRTLLM attention is supported if the platform is SM100,
|
||||
NVIDIA artifactory is accessible, and batch-invariant mode is not enabled.
|
||||
def supports_trtllm_attention(is_prefill: bool = False) -> bool:
|
||||
"""Return whether TRTLLM attention is available on the current platform
|
||||
for the given attention phase.
|
||||
|
||||
SM90 (Hopper) supports the XQA decode kernel but not TRTLLM prefill.
|
||||
SM100+ supports TRTLLM for both phases. All others are unsupported.
|
||||
"""
|
||||
# Batch-invariant mode disables TRTLLM attention
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
return False
|
||||
|
||||
# Requires SM100 and NVIDIA artifactory to be accessible to download cubins
|
||||
return (
|
||||
current_platform.is_device_capability_family(100) and has_nvidia_artifactory()
|
||||
)
|
||||
# Requires NVIDIA artifactory to be accessible to download cubins
|
||||
if not has_nvidia_artifactory():
|
||||
return False
|
||||
|
||||
# SM90 has XQA decode; prefill is not supported.
|
||||
if current_platform.is_device_capability(90):
|
||||
return not is_prefill
|
||||
|
||||
# SM100/SM103 has both prefill and decode TRTLLM kernels.
|
||||
return current_platform.is_device_capability_family(100)
|
||||
|
||||
|
||||
def force_use_trtllm_attention() -> bool | None:
|
||||
@@ -400,12 +408,15 @@ def force_use_trtllm_attention() -> bool | None:
|
||||
return vllm_config.attention_config.use_trtllm_attention
|
||||
|
||||
|
||||
def can_use_trtllm_attention(num_qo_heads: int, num_kv_heads: int) -> bool:
|
||||
def can_use_trtllm_attention(
|
||||
num_qo_heads: int, num_kv_heads: int, is_prefill: bool = False
|
||||
) -> bool:
|
||||
"""Check if the current configuration supports TRTLLM attention."""
|
||||
if force_use_trtllm_attention() is False:
|
||||
return False
|
||||
has_trtllm = supports_trtllm_attention()
|
||||
return has_trtllm and (num_qo_heads % num_kv_heads == 0)
|
||||
return supports_trtllm_attention(is_prefill=is_prefill) and (
|
||||
num_qo_heads % num_kv_heads == 0
|
||||
)
|
||||
|
||||
|
||||
def use_trtllm_attention(
|
||||
@@ -437,11 +448,12 @@ def use_trtllm_attention(
|
||||
return False
|
||||
|
||||
# The platform is not supported
|
||||
if not supports_trtllm_attention():
|
||||
if not supports_trtllm_attention(is_prefill=is_prefill):
|
||||
if force_use_trtllm:
|
||||
logger.warning_once(
|
||||
"TRTLLM attention is not supported on this platform, "
|
||||
"but --attention-config.use_trtllm_attention is set to 1"
|
||||
"TRTLLM attention is not supported on this platform for %s, "
|
||||
"but --attention-config.use_trtllm_attention is set to 1",
|
||||
"prefill" if is_prefill else "decode",
|
||||
)
|
||||
return False
|
||||
|
||||
@@ -476,13 +488,20 @@ def use_trtllm_attention(
|
||||
if is_prefill:
|
||||
# Prefill auto-detection
|
||||
use_trtllm = kv_cache_dtype == "auto"
|
||||
if use_trtllm:
|
||||
logger.warning_once("Using TRTLLM prefill attention (auto-detected).")
|
||||
elif current_platform.is_device_capability(90) and kv_cache_dtype.startswith(
|
||||
"fp8"
|
||||
):
|
||||
# SM90 + FP8 KV cache: prefer the XQA decode kernel. XQA does not
|
||||
# support NVFP4 KV (that is an SM100 trtllm-gen path only).
|
||||
use_trtllm = True
|
||||
else:
|
||||
# Decode auto-detection
|
||||
use_trtllm = num_tokens <= 256 and kv_cache_dtype == "auto"
|
||||
if use_trtllm:
|
||||
logger.warning_once("Using TRTLLM decode attention (auto-detected).")
|
||||
if use_trtllm:
|
||||
logger.warning_once(
|
||||
"Using TRTLLM %s attention (auto-detected).",
|
||||
"prefill" if is_prefill else "decode",
|
||||
)
|
||||
return use_trtllm
|
||||
|
||||
# CLI argument is set to 1 - respect it
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"""Attention layer with FlashInfer."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from functools import partial
|
||||
from typing import ClassVar
|
||||
|
||||
@@ -19,6 +20,7 @@ from flashinfer.prefill import trtllm_batch_context_with_kv_cache
|
||||
from flashinfer.utils import FP4Tensor
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm import _custom_ops as custom_ops
|
||||
from vllm import envs
|
||||
from vllm.config import (
|
||||
CUDAGraphMode,
|
||||
@@ -38,6 +40,8 @@ from vllm.platforms.interface import DeviceCapability
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.flashinfer import (
|
||||
can_use_trtllm_attention,
|
||||
force_use_trtllm_attention,
|
||||
supports_trtllm_attention,
|
||||
use_trtllm_attention,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -84,16 +88,16 @@ FP4_DTYPE = torch.uint8
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
trtllm_gen_workspace_buffer = None
|
||||
trtllm_workspace_buffer = None
|
||||
|
||||
|
||||
def _get_trtllm_gen_workspace_buffer():
|
||||
global trtllm_gen_workspace_buffer
|
||||
if trtllm_gen_workspace_buffer is None:
|
||||
trtllm_gen_workspace_buffer = torch.zeros(
|
||||
def _get_trtllm_workspace_buffer():
|
||||
global trtllm_workspace_buffer
|
||||
if trtllm_workspace_buffer is None:
|
||||
trtllm_workspace_buffer = torch.zeros(
|
||||
envs.VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE, dtype=torch.uint8, device="cuda"
|
||||
)
|
||||
return trtllm_gen_workspace_buffer
|
||||
return trtllm_workspace_buffer
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -350,6 +354,7 @@ class FlashInferBackend(AttentionBackend):
|
||||
use_large_pages = (
|
||||
num_kv_heads > 0
|
||||
and num_qo_heads // num_kv_heads > 1
|
||||
and current_platform.is_device_capability_family(100)
|
||||
and can_use_trtllm_attention(num_qo_heads, num_kv_heads)
|
||||
)
|
||||
if not use_large_pages:
|
||||
@@ -418,6 +423,16 @@ class FlashInferBackend(AttentionBackend):
|
||||
else:
|
||||
raise ValueError(f"Unrecognized dtype: {kv_cache_dtype}")
|
||||
|
||||
@classmethod
|
||||
def supports_kv_cache_dtype(cls, kv_cache_dtype: CacheDType | None) -> bool:
|
||||
if kv_cache_dtype == "nvfp4":
|
||||
return (
|
||||
current_platform.is_device_capability_family(100)
|
||||
and supports_trtllm_attention(is_prefill=True)
|
||||
and supports_trtllm_attention(is_prefill=False)
|
||||
)
|
||||
return super().supports_kv_cache_dtype(kv_cache_dtype)
|
||||
|
||||
@classmethod
|
||||
def get_supported_head_sizes(cls) -> list[int]:
|
||||
# https://github.com/flashinfer-ai/flashinfer/blob/3d55c71a62052c590c130897d3a3db49b14fcc34/include/flashinfer/utils.cuh#L157
|
||||
@@ -436,10 +451,9 @@ class FlashInferBackend(AttentionBackend):
|
||||
|
||||
@classmethod
|
||||
def supports_sink(cls) -> bool:
|
||||
"""FlashInfer supports sinks when TRTLLM attention is available (SM100)."""
|
||||
"""FlashInfer supports sinks only on the SM100 trtllm-gen path."""
|
||||
from vllm.utils.flashinfer import (
|
||||
force_use_trtllm_attention,
|
||||
supports_trtllm_attention,
|
||||
)
|
||||
|
||||
# Respect explicit disable flag (e.g.,
|
||||
@@ -447,8 +461,13 @@ class FlashInferBackend(AttentionBackend):
|
||||
if force_use_trtllm_attention() is False:
|
||||
return False
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
return False
|
||||
|
||||
# Check if TRTLLM is supported on this platform
|
||||
return supports_trtllm_attention()
|
||||
return supports_trtllm_attention(
|
||||
is_prefill=False
|
||||
) and supports_trtllm_attention(is_prefill=True)
|
||||
|
||||
@classmethod
|
||||
def get_required_kv_cache_layout(cls) -> KVCacheLayoutType | None:
|
||||
@@ -474,6 +493,13 @@ class FIDecode:
|
||||
wrapper: BatchDecodeWithPagedKVCacheWrapper
|
||||
|
||||
|
||||
class FlashInferDecodeKernel(Enum):
|
||||
"""Decode kernels selected inside the FlashInfer backend."""
|
||||
|
||||
XQA = "xqa"
|
||||
TRTLLM_GEN = "trtllm-gen"
|
||||
|
||||
|
||||
@dataclass
|
||||
class TRTLLMPrefill:
|
||||
"""Metadata for the TRTLLM prefill pathway."""
|
||||
@@ -503,8 +529,15 @@ class TRTLLMPrefill:
|
||||
|
||||
|
||||
@dataclass
|
||||
class TRTLLMDecode:
|
||||
"""Metadata for the TRTLLM decode pathway."""
|
||||
class FlashInferTrtllmAPIDecode:
|
||||
"""Metadata for decode paths using FlashInfer's TRTLLM decode API.
|
||||
|
||||
FlashInfer exposes both XQA (SM90) and trtllm-gen (SM100) through
|
||||
``trtllm_batch_decode_with_kv_cache``. Keep them as distinct vLLM
|
||||
decode kernels because their dtype/layout/output constraints differ.
|
||||
"""
|
||||
|
||||
kernel: FlashInferDecodeKernel
|
||||
|
||||
block_tables: torch.Tensor
|
||||
"""
|
||||
@@ -530,7 +563,10 @@ class FlashInferMetadata:
|
||||
slot_mapping: torch.Tensor
|
||||
"""Tensor for writing K/V to the cache. Shape: [num_actual_tokens]"""
|
||||
|
||||
q_data_type: torch.dtype
|
||||
# The data types of the query for prefill and decode.
|
||||
# On SM90, these two data types may be different.
|
||||
q_data_type_prefill: torch.dtype
|
||||
q_data_type_decode: torch.dtype
|
||||
|
||||
num_decodes: int
|
||||
num_decode_tokens: int
|
||||
@@ -544,7 +580,7 @@ class FlashInferMetadata:
|
||||
Will be `None` if `num_prefill_tokens == 0`.
|
||||
"""
|
||||
|
||||
decode: FIDecode | TRTLLMDecode | None
|
||||
decode: FIDecode | FlashInferTrtllmAPIDecode | None
|
||||
"""
|
||||
Holds the metadata for the decode portion of the batch.
|
||||
Will be `None` if `num_decode_tokens == 0`.
|
||||
@@ -651,12 +687,14 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
# storage dtype may not be the same as the op dtype (uint8 vs fp8_e4m3)
|
||||
self.is_kvcache_nvfp4 = self.cache_dtype == "nvfp4"
|
||||
if self.is_kvcache_nvfp4:
|
||||
# trtllm-gen FP4 FMHA kernels only exist for sm100f (sm_100/sm_103).
|
||||
# Fail fast at init rather than crashing on the first request.
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
if (
|
||||
force_use_trtllm_attention() is False
|
||||
or not supports_trtllm_attention(is_prefill=True)
|
||||
or not supports_trtllm_attention(is_prefill=False)
|
||||
):
|
||||
raise ValueError(
|
||||
"--kv-cache-dtype nvfp4 requires sm100f, "
|
||||
"please try a different dtype or remove"
|
||||
"--kv-cache-dtype nvfp4 requires the SM100 trtllm-gen "
|
||||
"FlashInfer path."
|
||||
)
|
||||
# For NVFP4, kv_cache_dtype stays as the string "nvfp4"
|
||||
# which is passed to FlashInferImpl
|
||||
@@ -671,54 +709,87 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
assert self.kv_cache_spec.dtype == self.model_config.dtype
|
||||
self.kv_cache_dtype = self.kv_cache_spec.dtype
|
||||
|
||||
# Use model dtype as q dtype when TRTLLM attn is not supported, or
|
||||
# --attention-config.disable_flashinfer_q_quantization is set to 1. Otherwise,
|
||||
# try to use fp8 q if kv cache is fp8, and will fall back to model dtype
|
||||
# if TRTLLM attention kernel is not used when building attn metadata
|
||||
can_use_trtllm = can_use_trtllm_attention(self.num_qo_heads, self.num_kv_heads)
|
||||
# Compute per-phase Q dtype. On SM90 (XQA decode), the prefill and
|
||||
# decode phases require different Q dtypes when the KV cache is FP8
|
||||
# (FP8-Q for the FI native prefill, BF16/FP16-Q for XQA decode),
|
||||
# so both values must be tracked independently.
|
||||
self.q_data_type_prefill = self.get_q_data_type(
|
||||
vllm_config, self.kv_cache_spec, is_prefill=True
|
||||
)
|
||||
self.q_data_type_decode = self.get_q_data_type(
|
||||
vllm_config, self.kv_cache_spec, is_prefill=False
|
||||
)
|
||||
|
||||
# Prefer TRTLLM/XQA for decoding whenever supported. The decode kernel
|
||||
# must be selected statically for FULL cudagraph capture.
|
||||
can_use_xqa_or_trtllm_gen_decode = can_use_trtllm_attention(
|
||||
self.num_qo_heads, self.num_kv_heads, is_prefill=False
|
||||
)
|
||||
# Page sizes >= 128 require the trtllm-gen GQA/MQA path (guaranteed by
|
||||
# get_supported_kernel_block_sizes).
|
||||
assert self.page_size <= 64 or (
|
||||
can_use_trtllm and self.num_qo_heads // self.num_kv_heads > 1
|
||||
current_platform.is_device_capability_family(100)
|
||||
and can_use_xqa_or_trtllm_gen_decode
|
||||
and self.num_qo_heads // self.num_kv_heads > 1
|
||||
), f"Unexpected FlashInfer page size {self.page_size} without trtllm-gen GQA"
|
||||
|
||||
if (
|
||||
can_use_trtllm
|
||||
and not vllm_config.attention_config.disable_flashinfer_q_quantization
|
||||
):
|
||||
if self.is_kvcache_nvfp4:
|
||||
# NVFP4 KV cache uses FP8 quantized queries
|
||||
self.q_data_type = FlashInferBackend.get_dtype_for_flashinfer(
|
||||
"fp8_e4m3"
|
||||
)
|
||||
else:
|
||||
self.q_data_type = self.kv_cache_dtype
|
||||
else:
|
||||
self.q_data_type = self.model_config.dtype
|
||||
|
||||
# Prefer TRTLLM attention for decoding in all cases.
|
||||
# This allows us to use AttentionCGSupport.UNIFORM_BATCH mode.
|
||||
self.use_trtllm_decode_attention = can_use_trtllm
|
||||
self._init_reorder_batch_threshold(1, supports_spec_as_decode=can_use_trtllm)
|
||||
self.use_trtllm_decode_attention = can_use_xqa_or_trtllm_gen_decode
|
||||
self.flashinfer_trtllm_api_decode_kernel: FlashInferDecodeKernel | None = (
|
||||
self._get_flashinfer_trtllm_api_decode_kernel()
|
||||
if can_use_xqa_or_trtllm_gen_decode
|
||||
else None
|
||||
)
|
||||
supports_spec_as_decode = (
|
||||
self.flashinfer_trtllm_api_decode_kernel
|
||||
== FlashInferDecodeKernel.TRTLLM_GEN
|
||||
)
|
||||
self._init_reorder_batch_threshold(
|
||||
1, supports_spec_as_decode=supports_spec_as_decode
|
||||
)
|
||||
|
||||
self._cascade_wrapper = None # Wrapper for cascade attention
|
||||
|
||||
# Global hyperparameters shared by all attention layers
|
||||
# TODO: discard this for trtllm-gen backend
|
||||
self.global_hyperparameters = infer_global_hyperparameters(
|
||||
get_per_layer_parameters(vllm_config, layer_names, FlashInferImpl)
|
||||
per_layer_parameters = get_per_layer_parameters(
|
||||
vllm_config, layer_names, FlashInferImpl
|
||||
)
|
||||
if current_platform.is_device_capability(90) and any(
|
||||
params.window_left != -1 for params in per_layer_parameters.values()
|
||||
):
|
||||
# FlashInfer SM90 sliding-window prefill is not reliable with FP8-Q:
|
||||
# https://github.com/flashinfer-ai/flashinfer/issues/3578
|
||||
raise NotImplementedError(
|
||||
"FlashInfer backend on SM90 currently crashes with "
|
||||
"sliding-window attention layers. Use the default attention "
|
||||
"backend."
|
||||
)
|
||||
self.global_hyperparameters = infer_global_hyperparameters(per_layer_parameters)
|
||||
self.sm_scale = self.global_hyperparameters.sm_scale
|
||||
self.window_left = self.global_hyperparameters.window_left
|
||||
self.logits_soft_cap = self.global_hyperparameters.logits_soft_cap
|
||||
self.has_sinks = self.global_hyperparameters.has_sinks
|
||||
if self.has_sinks and not can_use_trtllm:
|
||||
if self.has_sinks and not FlashInferBackend.supports_sink():
|
||||
raise NotImplementedError(
|
||||
"FlashInfer backend currently does not support attention "
|
||||
"sinks, please use trtllm on blackwell or flash attention on "
|
||||
"earlier GPUs."
|
||||
)
|
||||
capability = current_platform.get_device_capability()
|
||||
arch = f"sm{capability.major}{capability.minor}" if capability else "unknown"
|
||||
decode_backend = (
|
||||
self.flashinfer_trtllm_api_decode_kernel.value
|
||||
if self.flashinfer_trtllm_api_decode_kernel is not None
|
||||
else "flashinfer-native"
|
||||
)
|
||||
logger.info_once(
|
||||
"FlashInfer resolved query dtypes: prefill=%s, decode=%s, "
|
||||
"decode_backend=%s, kv_cache_dtype=%s, arch=%s",
|
||||
self.q_data_type_prefill,
|
||||
self.q_data_type_decode,
|
||||
decode_backend,
|
||||
self.kv_cache_dtype,
|
||||
arch,
|
||||
)
|
||||
# Preparing persistent buffers
|
||||
# Since we do not have explicit synchronization in ModelRunnerV2, we do not pin
|
||||
# reused CPU buffers to avoid a race condition between step N async copies to
|
||||
@@ -731,6 +802,37 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
self.paged_kv_indices = self._make_buffer(max_num_pages)
|
||||
self.paged_kv_last_page_len = self._make_buffer(max_num_reqs)
|
||||
|
||||
# Keep SM90 prefill/decode Q dtype selection in one place.
|
||||
@classmethod
|
||||
def get_q_data_type(
|
||||
cls,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
is_prefill: bool,
|
||||
) -> torch.dtype:
|
||||
# The user sets --attention-config.disable_flashinfer_q_quantization
|
||||
# to 1 explicitly, use model dtype for query.
|
||||
if vllm_config.attention_config.disable_flashinfer_q_quantization:
|
||||
return vllm_config.model_config.dtype
|
||||
|
||||
# On SM90, XQA decode requires BF16/FP16-Q even with FP8 KV cache.
|
||||
# FI native prefill on SM90 still uses FP8-Q in that case.
|
||||
cache_dtype = vllm_config.cache_config.cache_dtype
|
||||
if (
|
||||
current_platform.is_device_capability(90)
|
||||
and not is_prefill
|
||||
and force_use_trtllm_attention() is not False
|
||||
and cache_dtype.startswith("fp8")
|
||||
):
|
||||
return vllm_config.model_config.dtype
|
||||
|
||||
# Otherwise, match Q dtype to the KV cache dtype.
|
||||
if cache_dtype.startswith("fp8"):
|
||||
return FlashInferBackend.get_dtype_for_flashinfer(cache_dtype)
|
||||
if cache_dtype == "nvfp4":
|
||||
return FlashInferBackend.get_dtype_for_flashinfer("fp8_e4m3")
|
||||
return kv_cache_spec.dtype
|
||||
|
||||
def _make_buffer(
|
||||
self, *size: int | torch.SymInt, dtype: torch.dtype = torch.int32
|
||||
) -> CpuGpuBuffer:
|
||||
@@ -751,12 +853,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
) -> AttentionCGSupport:
|
||||
"""Get the cudagraph support level for FlashInfer attention.
|
||||
|
||||
This depends on whether we can use TRTLLM attention for decodes, since we can
|
||||
only do UNIFORM_SINGLE_TOKEN_DECODE if it is unavailable.
|
||||
To check this, we must call can_use_trtllm_attention with the number of KV
|
||||
heads from the kv_cache_spec. We check all available KV cache specs and
|
||||
only return UNIFORM_BATCH if all of them support TRTLLM attention.
|
||||
The SM90 XQA integration only enables single-token decode today. Keep
|
||||
specdec CUDA graphs limited to trtllm-gen until vLLM wires the XQA
|
||||
specdec mask.
|
||||
"""
|
||||
if current_platform.is_device_capability(90):
|
||||
return AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
||||
|
||||
# For UniformTypeKVCacheSpecs, check all contained specs
|
||||
kv_specs = (
|
||||
kv_cache_spec.kv_cache_specs.values()
|
||||
@@ -775,6 +878,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
if not can_use_trtllm_attention(
|
||||
num_qo_heads=num_qo_heads,
|
||||
num_kv_heads=spec.num_kv_heads,
|
||||
is_prefill=False,
|
||||
):
|
||||
has_trtllm_support = False
|
||||
break
|
||||
@@ -797,6 +901,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
def set_workspace_buffer(self, workspace_buffer: torch.Tensor):
|
||||
self._workspace_buffer = workspace_buffer
|
||||
|
||||
@staticmethod
|
||||
def _get_flashinfer_trtllm_api_decode_kernel() -> FlashInferDecodeKernel:
|
||||
if current_platform.is_device_capability(90):
|
||||
return FlashInferDecodeKernel.XQA
|
||||
assert current_platform.is_device_capability_family(100)
|
||||
return FlashInferDecodeKernel.TRTLLM_GEN
|
||||
|
||||
def _get_prefill_wrapper(
|
||||
self,
|
||||
causal: bool = True,
|
||||
@@ -976,7 +1087,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
# Step 1: Decide which dispatch modes to use:
|
||||
# - Cascade attention (distinct mode)
|
||||
# - Prefill (FI native or TRTLLM)
|
||||
# - Decode (FI native or TRTLLM)
|
||||
# - Decode (FI native, XQA, or trtllm-gen)
|
||||
use_cascade = common_prefix_len > 0
|
||||
uses_spec_reorder = self.reorder_batch_threshold > 1
|
||||
# Page sizes >= 128 must use trtllm-gen; force it for prefill too.
|
||||
@@ -990,13 +1101,13 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
max_seq_len,
|
||||
self.dcp_world_size,
|
||||
self.cache_dtype,
|
||||
self.q_data_type,
|
||||
self.q_data_type_prefill,
|
||||
is_prefill=True,
|
||||
force_use_trtllm=prefill_force_trtllm,
|
||||
has_sinks=self.has_sinks,
|
||||
has_spec=uses_spec_reorder,
|
||||
)
|
||||
decode_use_trtllm = (
|
||||
decode_with_flashinfer_trtllm_api = (
|
||||
causal and self.use_trtllm_decode_attention and self.dcp_world_size <= 1
|
||||
)
|
||||
|
||||
@@ -1011,7 +1122,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
)
|
||||
all_uses_trtllm = causal and (
|
||||
(num_prefills == 0 or prefill_use_trtllm)
|
||||
and (num_decodes == 0 or decode_use_trtllm)
|
||||
and (num_decodes == 0 or decode_with_flashinfer_trtllm_api)
|
||||
)
|
||||
|
||||
if not all_uses_trtllm:
|
||||
@@ -1035,17 +1146,14 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
"`sm_scale`."
|
||||
)
|
||||
|
||||
# The q quantization is not supported for non-trtllm attention,
|
||||
# fall back to model dtype.
|
||||
self.q_data_type = self.model_config.dtype
|
||||
|
||||
# Step 2: Initialize the output metadata
|
||||
# Leave prefill/decode/cascade_wrapper empty, to be populated
|
||||
# case by case depending on the batch contents and backend selection.
|
||||
attn_metadata = FlashInferMetadata(
|
||||
num_actual_tokens=num_actual_tokens,
|
||||
slot_mapping=common_attn_metadata.slot_mapping,
|
||||
q_data_type=self.q_data_type,
|
||||
q_data_type_prefill=self.q_data_type_prefill,
|
||||
q_data_type_decode=self.q_data_type_decode,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_prefills=num_prefills,
|
||||
@@ -1101,7 +1209,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
|
||||
# Compute paged_kv_indices if necessary
|
||||
# paged_kv_indices is only needed for FlashInfer native paths;
|
||||
# TRTLLM paths use block_tables directly on GPU.
|
||||
# XQA/trtllm-gen paths use block_tables directly on GPU.
|
||||
needs_paged_kv_indices = use_cascade or not all_uses_trtllm
|
||||
if needs_paged_kv_indices:
|
||||
assert num_blocks_np is not None
|
||||
@@ -1143,6 +1251,9 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
paged_kv_last_page_len_cpu = self.paged_kv_last_page_len.cpu[:num_reqs]
|
||||
|
||||
attn_metadata.cascade_wrapper = self._get_cascade_wrapper()
|
||||
# Cascade attention must use the same q dtype for prefill and decode
|
||||
# because it does not support FP8 kv-cache or FP8 query yet.
|
||||
assert self.q_data_type_prefill == self.q_data_type_decode
|
||||
attn_metadata.cascade_wrapper.plan(
|
||||
qo_indptr_arr=[shared_qo_indptr_cpu, qo_indptr_cpu],
|
||||
paged_kv_indptr_arr=[shared_kv_page_indptr_cpu, paged_kv_indptr_cpu],
|
||||
@@ -1159,7 +1270,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
sm_scale=self.sm_scale,
|
||||
window_left=self.window_left,
|
||||
logits_soft_cap=self.logits_soft_cap,
|
||||
q_data_type=self.q_data_type,
|
||||
q_data_type=self.q_data_type_prefill,
|
||||
kv_data_type=self.kv_cache_dtype,
|
||||
)
|
||||
return attn_metadata
|
||||
@@ -1233,7 +1344,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
sm_scale=self.sm_scale,
|
||||
window_left=self.window_left,
|
||||
logits_soft_cap=self.logits_soft_cap,
|
||||
q_data_type=self.q_data_type,
|
||||
q_data_type=self.q_data_type_prefill,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
prefill_fixed_split_size=self.prefill_fixed_split_size,
|
||||
disable_split_kv=self.disable_split_kv,
|
||||
@@ -1262,7 +1373,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
sm_scale=self.sm_scale,
|
||||
window_left=self.window_left,
|
||||
logits_soft_cap=self.logits_soft_cap,
|
||||
q_data_type=self.q_data_type,
|
||||
q_data_type=self.q_data_type_prefill,
|
||||
kv_data_type=self.kv_cache_dtype,
|
||||
o_data_type=o_dtype,
|
||||
fixed_split_size=self.prefill_fixed_split_size,
|
||||
@@ -1272,12 +1383,14 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
|
||||
## DECODE PATHWAY
|
||||
if num_decodes > 0:
|
||||
if decode_use_trtllm:
|
||||
if decode_with_flashinfer_trtllm_api:
|
||||
assert num_decode_tokens % num_decodes == 0, (
|
||||
"TRTLLM decode requires uniform query lengths per request. "
|
||||
"XQA/trtllm-gen decode requires uniform query lengths per request. "
|
||||
f"Got {num_decode_tokens=} and {num_decodes=}."
|
||||
)
|
||||
attn_metadata.decode = TRTLLMDecode(
|
||||
assert self.flashinfer_trtllm_api_decode_kernel is not None
|
||||
attn_metadata.decode = FlashInferTrtllmAPIDecode(
|
||||
kernel=self.flashinfer_trtllm_api_decode_kernel,
|
||||
block_tables=block_table_tensor[:num_decodes],
|
||||
seq_lens=seq_lens[:num_decodes],
|
||||
max_seq_len=max_seq_len,
|
||||
@@ -1320,7 +1433,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
|
||||
sm_scale=self.sm_scale,
|
||||
window_left=self.window_left,
|
||||
logits_soft_cap=self.logits_soft_cap,
|
||||
q_data_type=self.q_data_type,
|
||||
q_data_type=self.q_data_type_decode,
|
||||
kv_data_type=self.kv_cache_dtype,
|
||||
o_data_type=o_dtype,
|
||||
fixed_split_size=self.decode_fixed_split_size,
|
||||
@@ -1396,10 +1509,17 @@ class FlashInferImpl(AttentionImpl):
|
||||
)
|
||||
self.sinks = sinks
|
||||
|
||||
self.support_trtllm_attn = can_use_trtllm_attention(num_heads, num_kv_heads)
|
||||
self.supports_xqa_or_trtllm_gen_decode = can_use_trtllm_attention(
|
||||
num_heads, num_kv_heads, is_prefill=False
|
||||
)
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
# Query pre-quantization needs a single dtype for the whole query tensor.
|
||||
# SM90 XQA needs BF16/FP16-Q for decode and FP8 for prefill,
|
||||
# so only enable this for SM100 trtllm-gen where both use FP8-Q.
|
||||
self.supports_quant_query_input = (
|
||||
self.support_trtllm_attn
|
||||
self.supports_xqa_or_trtllm_gen_decode
|
||||
and is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
and current_platform.is_device_capability_family(100)
|
||||
and vllm_config is not None
|
||||
and not vllm_config.attention_config.disable_flashinfer_q_quantization
|
||||
)
|
||||
@@ -1429,9 +1549,12 @@ class FlashInferImpl(AttentionImpl):
|
||||
self.dcp_combine = partial(cp_lse_ag_out_rs, is_lse_base_on_e=False)
|
||||
|
||||
def fused_output_quant_supported(self, quant_key: QuantKey):
|
||||
# XQA does not support FP8/NVFP4 output, so require trtllm-gen
|
||||
# (SM100+) here. Without that we cannot fuse the output quant.
|
||||
return (
|
||||
self.support_trtllm_attn
|
||||
self.supports_xqa_or_trtllm_gen_decode
|
||||
and is_quantized_kv_cache(self.kv_cache_dtype)
|
||||
and current_platform.is_device_capability_family(100)
|
||||
and quant_key in (kFp8StaticTensorSym, kNvfp4Dynamic)
|
||||
)
|
||||
|
||||
@@ -1440,6 +1563,37 @@ class FlashInferImpl(AttentionImpl):
|
||||
if self.sinks is not None and self.sinks.dtype != torch.float32:
|
||||
self.sinks = self.sinks.to(torch.float32)
|
||||
|
||||
def get_xqa_bmm1_scale(self, layer: torch.nn.Module, q_data_type: torch.dtype):
|
||||
bmm1_scale = self.scale
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
if q_data_type in (torch.float8_e4m3fn, torch.float8_e5m2):
|
||||
bmm1_scale *= layer._q_scale_float
|
||||
bmm1_scale *= layer._k_scale_float
|
||||
return bmm1_scale
|
||||
|
||||
# SM90 may need FP8-Q for native prefill and BF16/FP16-Q for XQA decode,
|
||||
# so quantize only the slice whose target dtype differs.
|
||||
def maybe_quant_query(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
q_data_type: torch.dtype,
|
||||
scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if query.dtype != q_data_type:
|
||||
assert query.dtype in [torch.float16, torch.bfloat16]
|
||||
assert q_data_type in [torch.float8_e4m3fn, torch.float8_e5m2]
|
||||
assert query.is_contiguous()
|
||||
assert query.dim() == 3
|
||||
num_tokens = query.shape[0]
|
||||
num_heads = query.shape[1]
|
||||
head_size = query.shape[2]
|
||||
query_quantized, _ = custom_ops.scaled_fp8_quant(
|
||||
query.view(num_tokens, num_heads * head_size), scale=scale
|
||||
)
|
||||
return query_quantized.view(num_tokens, num_heads, head_size)
|
||||
|
||||
return query
|
||||
|
||||
def forward(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -1469,12 +1623,6 @@ class FlashInferImpl(AttentionImpl):
|
||||
# Profiling run.
|
||||
return output.fill_(0)
|
||||
|
||||
# Ensure query dtype matches the expected dtype from attention metadata
|
||||
assert attn_metadata.q_data_type == query.dtype, (
|
||||
f"Query dtype mismatch: expected {attn_metadata.q_data_type}, "
|
||||
f"got {query.dtype}"
|
||||
)
|
||||
|
||||
if self.bmm1_scale is None:
|
||||
self.bmm1_scale = self.scale
|
||||
if is_quantized_kv_cache(self.kv_cache_dtype):
|
||||
@@ -1486,7 +1634,14 @@ class FlashInferImpl(AttentionImpl):
|
||||
self.bmm2_scale *= layer._v_scale_float
|
||||
|
||||
prefill_use_trtllm = isinstance(attn_metadata.prefill, TRTLLMPrefill)
|
||||
decode_use_trtllm = isinstance(attn_metadata.decode, TRTLLMDecode)
|
||||
decode_kernel = (
|
||||
attn_metadata.decode.kernel
|
||||
if isinstance(attn_metadata.decode, FlashInferTrtllmAPIDecode)
|
||||
else None
|
||||
)
|
||||
decode_with_xqa = decode_kernel == FlashInferDecodeKernel.XQA
|
||||
decode_with_trtllm_gen = decode_kernel == FlashInferDecodeKernel.TRTLLM_GEN
|
||||
decode_with_flashinfer_trtllm_api = decode_with_xqa or decode_with_trtllm_gen
|
||||
|
||||
# The attn+quant fusion happens when output_scale is provided.
|
||||
if output_scale is None:
|
||||
@@ -1494,12 +1649,15 @@ class FlashInferImpl(AttentionImpl):
|
||||
"output_block_scale is not supported when fusion has not happened"
|
||||
)
|
||||
else:
|
||||
assert attn_metadata.q_data_type == FP8_DTYPE, (
|
||||
"Query must be FP8 when attn+quant fusion happened."
|
||||
assert attn_metadata.q_data_type_prefill == FP8_DTYPE, (
|
||||
"Query must be FP8 when attn+quant fusion happened for prefill."
|
||||
)
|
||||
assert attn_metadata.q_data_type_decode == FP8_DTYPE, (
|
||||
"Query must be FP8 when attn+quant fusion happened for decode."
|
||||
)
|
||||
assert (attn_metadata.num_prefills == 0 or prefill_use_trtllm) and (
|
||||
attn_metadata.num_decodes == 0 or decode_use_trtllm
|
||||
), "Must use TRT-LLM attn"
|
||||
attn_metadata.num_decodes == 0 or decode_with_trtllm_gen
|
||||
), "Output quant fusion requires TRTLLM prefill/trtllm-gen decode"
|
||||
|
||||
if output.dtype == FP8_DTYPE:
|
||||
assert output_block_scale is None, (
|
||||
@@ -1597,6 +1755,13 @@ class FlashInferImpl(AttentionImpl):
|
||||
prefill_query = query[num_decode_tokens:]
|
||||
assert prefill_query.shape[0] == num_prefill_tokens
|
||||
|
||||
# Convert query to the expected dtype for prefill if needed.
|
||||
prefill_query = self.maybe_quant_query(
|
||||
prefill_query,
|
||||
attn_metadata.q_data_type_prefill,
|
||||
layer._q_scale,
|
||||
)
|
||||
|
||||
if not prefill_use_trtllm:
|
||||
assert isinstance(attn_metadata.prefill, FIPrefill)
|
||||
prefill_wrapper = attn_metadata.prefill.wrapper
|
||||
@@ -1655,6 +1820,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
prefill_wrapper.run(
|
||||
prefill_query,
|
||||
kv_cache_permute,
|
||||
q_scale=layer._q_scale_float,
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
out=out_prefill,
|
||||
@@ -1673,7 +1839,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
# degenerate strides on size=1 dims for TMA alignment.
|
||||
prefill_query = prefill_query.contiguous()
|
||||
prefill_query = canonicalize_singleton_dim_strides(prefill_query)
|
||||
workspace_buffer = _get_trtllm_gen_workspace_buffer()
|
||||
workspace_buffer = _get_trtllm_workspace_buffer()
|
||||
block_tables_prefill = attn_metadata.prefill.block_tables
|
||||
seq_lens_prefill = attn_metadata.prefill.seq_lens
|
||||
|
||||
@@ -1705,7 +1871,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
prefill_kv_block_scales = None
|
||||
if self.is_kvcache_nvfp4:
|
||||
# NVFP4 trtllm-gen kernel requires FP8 query.
|
||||
assert attn_metadata.q_data_type == FP8_DTYPE, (
|
||||
assert attn_metadata.q_data_type_prefill == FP8_DTYPE, (
|
||||
"NVFP4 KV cache requires FP8 quantized queries for "
|
||||
"trtllm-gen prefill. Set "
|
||||
"disable_flashinfer_q_quantization=False."
|
||||
@@ -1714,7 +1880,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
mock_block_table = block_tables_prefill
|
||||
prefill_kv_block_scales = nvfp4_kv_block_scales
|
||||
elif (
|
||||
attn_metadata.q_data_type != FP8_DTYPE
|
||||
attn_metadata.q_data_type_prefill != FP8_DTYPE
|
||||
and self.kv_cache_dtype.startswith("fp8")
|
||||
):
|
||||
# TRTLLM prefill attention does not support BF16 Q
|
||||
@@ -1738,7 +1904,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
block_tables_prefill,
|
||||
layer._k_scale,
|
||||
layer._v_scale,
|
||||
attn_metadata.q_data_type,
|
||||
attn_metadata.q_data_type_prefill,
|
||||
)
|
||||
else:
|
||||
mock_kv_cache = kv_cache_permute
|
||||
@@ -1773,7 +1939,14 @@ class FlashInferImpl(AttentionImpl):
|
||||
decode_query = query[:num_decode_tokens]
|
||||
assert decode_query.shape[0] == num_decode_tokens
|
||||
|
||||
if not decode_use_trtllm:
|
||||
# Convert query to the expected dtype for decode if needed.
|
||||
decode_query = self.maybe_quant_query(
|
||||
decode_query,
|
||||
attn_metadata.q_data_type_decode,
|
||||
layer._q_scale,
|
||||
)
|
||||
|
||||
if not decode_with_flashinfer_trtllm_api:
|
||||
assert isinstance(attn_metadata.decode, FIDecode)
|
||||
decode_wrapper = attn_metadata.decode.wrapper
|
||||
assert decode_wrapper is not None
|
||||
@@ -1806,6 +1979,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
decode_wrapper.run(
|
||||
decode_query,
|
||||
kv_cache_permute,
|
||||
q_scale=layer._q_scale_float,
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
out=output_tmp,
|
||||
@@ -1822,6 +1996,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
decode_wrapper.run(
|
||||
decode_query,
|
||||
kv_cache_permute,
|
||||
q_scale=layer._q_scale_float,
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
out=out_decode,
|
||||
@@ -1831,19 +2006,23 @@ class FlashInferImpl(AttentionImpl):
|
||||
if needs_fp8_out:
|
||||
output[:num_decode_tokens].copy_(out_decode.to(output.dtype))
|
||||
else:
|
||||
assert isinstance(attn_metadata.decode, TRTLLMDecode)
|
||||
assert isinstance(attn_metadata.decode, FlashInferTrtllmAPIDecode)
|
||||
# decode_query may be non-contiguous or have degenerate strides
|
||||
# on size=1 dims. contiguous() ensures memory layout; then
|
||||
# canonicalize_singleton_dim_strides fixes any remaining
|
||||
# degenerate strides on size=1 dims for TMA alignment.
|
||||
decode_query = decode_query.contiguous()
|
||||
decode_query = canonicalize_singleton_dim_strides(decode_query)
|
||||
workspace_buffer = _get_trtllm_gen_workspace_buffer()
|
||||
workspace_buffer = _get_trtllm_workspace_buffer()
|
||||
block_tables_decode = attn_metadata.decode.block_tables
|
||||
seq_lens_decode = attn_metadata.decode.seq_lens
|
||||
|
||||
# This path needs to be enabled with VLLM_KV_CACHE_LAYOUT = HND
|
||||
assert get_kv_cache_layout() == "HND"
|
||||
# trtllm-gen needs HND layout on SM100. XQA is selected
|
||||
# separately on SM90 and does not use this SM100 layout gate.
|
||||
if decode_with_trtllm_gen:
|
||||
assert get_kv_cache_layout() == "HND"
|
||||
else:
|
||||
assert decode_with_xqa
|
||||
assert is_strictly_contiguous(decode_query)
|
||||
assert is_strictly_contiguous(workspace_buffer)
|
||||
assert is_strictly_contiguous(block_tables_decode)
|
||||
@@ -1882,6 +2061,19 @@ class FlashInferImpl(AttentionImpl):
|
||||
else:
|
||||
q_len_per_req = num_decode_tokens // attn_metadata.num_decodes
|
||||
|
||||
if decode_with_xqa and q_len_per_req > 1:
|
||||
raise NotImplementedError(
|
||||
"FlashInfer XQA speculative decode is not wired in vLLM yet."
|
||||
)
|
||||
|
||||
# XQA decode can use model-dtype Q with FP8 KV, so only include
|
||||
# q_scale when the decode query is actually FP8.
|
||||
bmm1_scale = (
|
||||
self.get_xqa_bmm1_scale(layer, attn_metadata.q_data_type_decode)
|
||||
if decode_with_xqa
|
||||
else self.bmm1_scale
|
||||
)
|
||||
|
||||
trtllm_batch_decode_with_kv_cache(
|
||||
query=decode_query,
|
||||
kv_cache=(
|
||||
@@ -1891,12 +2083,14 @@ class FlashInferImpl(AttentionImpl):
|
||||
block_tables=block_tables_decode,
|
||||
seq_lens=seq_lens_decode,
|
||||
max_seq_len=attn_metadata.decode.max_seq_len,
|
||||
bmm1_scale=self.bmm1_scale,
|
||||
bmm1_scale=bmm1_scale,
|
||||
bmm2_scale=self.bmm2_scale,
|
||||
window_left=self.window_left,
|
||||
sinks=self.sinks,
|
||||
o_sf_scale=self.o_sf_scale,
|
||||
out=out,
|
||||
kv_layout=get_kv_cache_layout(),
|
||||
backend=attn_metadata.decode.kernel.value,
|
||||
q_len_per_req=q_len_per_req,
|
||||
kv_cache_sf=(
|
||||
nvfp4_kv_block_scales if self.is_kvcache_nvfp4 else None
|
||||
|
||||
Reference in New Issue
Block a user