Files
vLLM/tests/v1/attention/test_attention_backends.py

2044 lines
70 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for v1 attention backends without GPUModelRunner dependency."""
from dataclasses import replace
from functools import partial
from types import SimpleNamespace
import pytest
import torch
from torch.nn.attention.flex_attention import create_block_mask, flex_attention
from transformers import LlamaConfig
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_standard_kv_cache_spec,
create_vllm_config,
try_backend_includes_kv_cache_update,
try_get_attention_backend,
)
from vllm.config import (
DiffusionConfig,
ModelConfig,
SpeculativeConfig,
set_current_vllm_config,
)
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import (
STR_DTYPE_TO_TORCH_DTYPE,
is_quantized_kv_cache,
is_torch_equal_or_newer,
nvfp4_kv_cache_full_dim,
set_random_seed,
)
from vllm.v1.attention.backend import (
AttentionCGSupport,
AttentionMetadataBuilder,
AttentionType,
CommonAttentionMetadata,
max_decode_query_len,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout, KVQuantMode
BACKENDS_TO_TEST = [
AttentionBackendEnum.FLASH_ATTN,
AttentionBackendEnum.FLASHINFER,
AttentionBackendEnum.FLEX_ATTENTION,
AttentionBackendEnum.TRITON_ATTN,
"FLEX_ATTENTION_SLOW",
]
def _actual_backend(backend: AttentionBackendEnum | str) -> AttentionBackendEnum:
"""Resolve pseudo-backends (FLEX_ATTENTION_SLOW) to their real enum."""
if isinstance(backend, str):
if backend != "FLEX_ATTENTION_SLOW":
raise ValueError(f"Unknown pseudo-backend: {backend}")
return AttentionBackendEnum.FLEX_ATTENTION
return backend
DEVICE_TYPE = current_platform.device_type
# Use the platform's preferred FP8 type so the stored cache matches what the
# backends reinterpret at runtime. On ROCm gfx94x this is e4m3fnuz, not e4m3fn;
# storing e4m3fn bytes there would be re-read as fnuz and produce NaNs.
FP8_KV_CACHE_DTYPES = {
"fp8": current_platform.fp8_dtype(),
"fp8_e4m3": current_platform.fp8_dtype(),
}
# Remove flashinfer from the list if it's not available
try:
import flashinfer # noqa: F401
except ImportError:
BACKENDS_TO_TEST.remove(AttentionBackendEnum.FLASHINFER)
def _convert_dtype_to_torch(dtype):
"""Convert ModelDType to torch.dtype."""
if isinstance(dtype, str):
if dtype == "auto":
return torch.float16 # Default dtype for testing
elif dtype in STR_DTYPE_TO_TORCH_DTYPE:
return STR_DTYPE_TO_TORCH_DTYPE[dtype]
else:
raise ValueError(f"Unknown dtype: {dtype}")
elif isinstance(dtype, torch.dtype):
return dtype
else:
raise ValueError(f"Unknown dtype: {dtype}")
# Define common batch configurations
BATCH_SPECS = {
"small_decode": BatchSpec(seq_lens=[32, 40], query_lens=[1, 1]),
"small_prefill": BatchSpec(seq_lens=[32, 40], query_lens=[8, 8]),
"mixed_small": BatchSpec(seq_lens=[32, 40, 48, 56], query_lens=[1, 1, 5, 5]),
"medium_decode": BatchSpec(
seq_lens=[128, 256, 512, 1024, 128, 256, 512, 1024],
query_lens=[1, 1, 1, 1, 1, 1, 1, 1],
),
"medium_prefill": BatchSpec(
seq_lens=[256, 512, 1024, 2048], query_lens=[16, 16, 16, 16]
),
"mixed_medium": BatchSpec(
seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[1, 1, 1, 7, 7, 7]
),
"large_decode": BatchSpec(seq_lens=[2048] * 32, query_lens=[1] * 32),
"large_prefill": BatchSpec(seq_lens=[4096] * 8, query_lens=[32] * 8),
"mixed_large": BatchSpec(
seq_lens=[1024, 2048, 4096, 1024, 2048, 4096], query_lens=[1, 1, 1, 32, 32, 32]
),
"single_decode": BatchSpec(seq_lens=[1024], query_lens=[1]),
"single_prefill": BatchSpec(seq_lens=[1024], query_lens=[64]),
# encoder-only
"small_encoder_prefill": BatchSpec(
seq_lens=[32, 64, 128, 256], query_lens=[32, 64, 128, 256]
),
"medium_encoder_prefill": BatchSpec(
seq_lens=[256, 512, 1024, 2048], query_lens=[256, 512, 1024, 2048]
),
}
def create_and_prepopulate_kv_cache(
k_contexts: list[torch.Tensor],
v_contexts: list[torch.Tensor],
block_size: int,
num_kv_heads: int,
head_size: int,
dtype: torch.dtype,
device: torch.device,
num_blocks: int,
common_attn_metadata: CommonAttentionMetadata,
layout: KVCacheLayout,
randomize_blocks: bool = True,
kv_cache_dtype: str = "auto",
) -> torch.Tensor:
"""Create and prepopulate a KV cache with context data.
Args:
k_contexts: List of key context tensors for each sequence
v_contexts: List of value context tensors for each sequence
block_size: Size of each block
num_kv_heads: Number of KV heads
head_size: Size of each head
dtype: Data type for the cache
device: Device to create the cache on
num_blocks: Total number of blocks in the cache
common_attn_metadata: Provides seq lens, block table and slot mapping
layout: Physical layout to allocate in; the cache is returned as the
logical ``[B, H, N, C]`` view (as ``create_kv_cache_views`` does)
randomize_blocks: Whether to randomly permute blocks
or use sequential order
kv_cache_dtype: Cache dtype string; fp8 caches use fp8 storage
Returns:
A 4D tensor in logical ``(num_blocks, num_kv_heads, block_size,
2 * head_size)`` order with strides determined by ``layout``.
"""
batch_size = len(k_contexts)
seq_lens = common_attn_metadata.seq_lens.cpu()
query_lens = (
common_attn_metadata.query_start_loc_cpu[1:]
- common_attn_metadata.query_start_loc_cpu[:-1]
)
context_lens = seq_lens - query_lens
block_table = common_attn_metadata.block_table_tensor
slot_mapping = common_attn_metadata.slot_mapping
# For an fp8 kv cache, store the cache in the fp8 dtype so that assigning
# the higher-precision context tensors quantizes them, mirroring runtime.
fp8_kv_cache = is_quantized_kv_cache(kv_cache_dtype)
storage_dtype = FP8_KV_CACHE_DTYPES[kv_cache_dtype] if fp8_kv_cache else dtype
# Logical 5D shape is always [L, B, H, N, C]. Cross-layer layouts need
# at least two layers to reproduce the inter-layer gaps in a layer view.
logical_4d = (num_blocks, num_kv_heads, block_size, 2 * head_size)
num_layers = 1 if layout.is_layer_compact else 2
logical_5d = (num_layers, *logical_4d)
physical_5d = tuple(logical_5d[i] for i in layout.stride_order)
inv_order = [layout.stride_order.index(i) for i in range(5)]
kv_cache_physical = torch.zeros(physical_5d, dtype=storage_dtype, device=device)
# Permute to logical [L, B, H, N, C], then select a layer. This mirrors
# create_kv_cache_views and retains cross-layer strides in the 4D view.
kv_cache = kv_cache_physical.permute(*inv_order)[0]
# Write context tokens into the cache via the logical view:
# kv_cache[block, :, token_in_block, :] routes correctly regardless
# of physical layout.
# Start from block_id=1 since block_id=0 is considered the null block
start_block_idx = 1
for i in range(batch_size):
k_context, v_context = k_contexts[i], v_contexts[i]
t = torch.arange(k_context.shape[0], device=device)
blk = start_block_idx + t // block_size
off = t % block_size
# Advanced indexing on (blk, off) yields [T, H, hs] destinations.
# index_put is dtype-strict; cast like the scalar path would.
kv_cache[blk, :, off, :head_size] = k_context.to(kv_cache.dtype)
kv_cache[blk, :, off, head_size:] = v_context.to(kv_cache.dtype)
# Stay block aligned and allocate enough blocks for the new tokens
start_block_idx += cdiv(int(seq_lens[i]), block_size)
blocks_end = start_block_idx
# Permute the context blocks (excluding block 0 which is null)
if randomize_blocks:
# Random permutation starting from block 1
perm = torch.randperm(blocks_end - 1) + 1
else:
# Sequential order starting from block 1
perm = torch.arange(1, blocks_end)
inv_perm = torch.zeros(blocks_end, dtype=torch.long, device=device)
# Add 1 to account for starting from block 1
inv_perm[1:] = torch.argsort(perm) + 1
kv_cache[1:blocks_end, ...] = kv_cache[perm, ...]
# Construct the right block table
# Start from block_id=1 since block_id=0 is considered the null block
start_block_idx = 1
for i in range(batch_size):
num_blocks_for_seq = cdiv(int(seq_lens[i]), block_size)
start = start_block_idx
end = start + num_blocks_for_seq
block_table[i, :num_blocks_for_seq] = inv_perm[start:end]
start_block_idx += num_blocks_for_seq
# Create a realistic slot mapping that corresponds to the block table
for i in range(batch_size):
token_offsets = torch.arange(int(query_lens[i])) + int(context_lens[i])
block_indices = token_offsets // block_size
token_inter_block_offsets = token_offsets % block_size
start = common_attn_metadata.query_start_loc_cpu[i]
end = common_attn_metadata.query_start_loc_cpu[i + 1]
slot_mapping[start:end] = block_table[
i, block_indices
] * block_size + token_inter_block_offsets.to(device)
if fp8_kv_cache:
kv_cache = kv_cache.view(torch.uint8)
return kv_cache
class MockAttentionLayer:
"""A mock attention layer for testing."""
def __init__(
self,
device: torch.device,
k_scale: float = 1.0,
v_scale: float = 1.0,
):
self._q_scale = torch.tensor(1.0, device=device)
self._k_scale = torch.tensor(k_scale, device=device)
self._v_scale = torch.tensor(v_scale, device=device)
# Add float versions for flashinfer
self._q_scale_float = 1.0
self._k_scale_float = k_scale
self._v_scale_float = v_scale
def _clone_kv_cache_in_layout(
kv_cache: torch.Tensor, layout: KVCacheLayout
) -> torch.Tensor:
"""Copy a logical [B, H, N, C] cache into a fresh allocation in `layout`."""
logical_5d = (1, *kv_cache.shape)
physical = torch.zeros(
tuple(logical_5d[i] for i in layout.stride_order),
dtype=kv_cache.dtype,
device=kv_cache.device,
)
inv_order = [layout.stride_order.index(i) for i in range(5)]
view = physical.permute(*inv_order)[0]
view.copy_(kv_cache)
return view
def run_attention_backend(
backend: AttentionBackendEnum | str,
kv_cache_spec: FullAttentionSpec,
layer_names: list[str],
vllm_config,
device: torch.device,
common_attn_metadata: CommonAttentionMetadata,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
kv_cache: torch.Tensor,
attn_type: AttentionType = AttentionType.DECODER,
sliding_window: int | None = None,
kv_cache_dtype: str = "auto",
sinks: torch.Tensor | None = None,
use_cuda_graph: bool = False,
layer_k_scale: float = 1.0,
layer_v_scale: float = 1.0,
) -> torch.Tensor:
"""Run attention computation using the specified backend's AttentionImpl."""
use_direct_block_mask = is_torch_equal_or_newer("2.9.0.dev0")
if backend == "FLEX_ATTENTION_SLOW":
use_direct_block_mask = False
backend = _actual_backend(backend)
builder_cls, impl_cls = try_get_attention_backend(backend)
# Mock flashinfer's get_per_layer_parameters if needed
if backend == AttentionBackendEnum.FLASHINFER:
import unittest.mock
from vllm.v1.attention.backends.utils import PerLayerParameters
def mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls):
# Return mock parameters for a single layer
head_size = vllm_config.model_config.get_head_size()
return {
layer_name: PerLayerParameters(
window_left=-1, # No sliding window
logits_soft_cap=0.0, # No soft cap
sm_scale=1.0 / (head_size**0.5), # Standard scale
has_sinks=sinks is not None,
)
for layer_name in layer_names
}
with unittest.mock.patch(
"vllm.v1.attention.backends.flashinfer.get_per_layer_parameters",
mock_get_per_layer_parameters,
):
builder = builder_cls(kv_cache_spec, layer_names, vllm_config, device)
attn_metadata = builder.build(
common_prefix_len=0,
common_attn_metadata=common_attn_metadata,
)
else:
# Build metadata
builder = builder_cls(kv_cache_spec, layer_names, vllm_config, device)
if backend == AttentionBackendEnum.FLEX_ATTENTION:
builder.direct_build = use_direct_block_mask
attn_metadata = builder.build(
common_prefix_len=0,
common_attn_metadata=common_attn_metadata,
)
# Instantiate implementation
num_heads = vllm_config.model_config.get_num_attention_heads(
vllm_config.parallel_config
)
num_kv_heads = vllm_config.model_config.get_num_kv_heads(
vllm_config.parallel_config
)
head_size = vllm_config.model_config.get_head_size()
scale = 1.0 / (head_size**0.5)
# Impls capture the current vllm config at construction, as in model loading.
with set_current_vllm_config(vllm_config):
impl = impl_cls(
num_heads=num_heads,
head_size=head_size,
scale=scale,
num_kv_heads=num_kv_heads,
alibi_slopes=None,
sliding_window=sliding_window,
attn_type=attn_type,
kv_cache_dtype=kv_cache_dtype,
**({"sinks": sinks} if sinks is not None else {}),
)
# Create mock layer and output buffer
mock_layer = MockAttentionLayer(device, layer_k_scale, layer_v_scale)
output = torch.empty_like(query)
if is_quantized_kv_cache(kv_cache_dtype) and impl.supports_quant_query_input:
query = query.to(current_platform.fp8_dtype())
# Run forward pass
# NOTE: The query, key, and value are already shaped correctly
# in the calling test function.
if not try_backend_includes_kv_cache_update(backend):
impl.do_kv_cache_update(
mock_layer, key, value, kv_cache, attn_metadata.slot_mapping
)
if use_cuda_graph:
impl.forward(
mock_layer, query, key, value, kv_cache, attn_metadata, output=output
)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
impl.forward(
mock_layer,
query,
key,
value,
kv_cache,
attn_metadata,
output=output,
)
graph.replay()
else:
backend_output = impl.forward(
mock_layer, query, key, value, kv_cache, attn_metadata, output=output
)
if backend_output is not None:
output = backend_output
return output
def _test_backend_correctness(
batch_spec: BatchSpec,
model: str,
backend_to_test: list[AttentionBackendEnum | str],
mask_mod,
*,
causal: bool = True,
attn_type: AttentionType = AttentionType.DECODER,
block_size: int = 16,
atol: float = 1e-2,
rtol: float = 1e-2,
tensor_parallel_size: int = 1,
kv_cache_dtype: str = "auto",
use_sinks: bool = False,
layout: KVCacheLayout | None = None,
use_cuda_graph: bool = False,
num_speculative_tokens: int = 0,
model_dtype: torch.dtype | None = None,
max_num_seqs: int | None = None,
max_num_batched_tokens: int | None = None,
layer_k_scale: float = 1.0,
layer_v_scale: float = 1.0,
):
"""Test that all backends produce similar outputs to a reference implementation
using FlexAttention or an explicit attention-sink reference.
This test works by:
1. Generating a batch of sequences with specified context and query lengths.
2. Computing a ground-truth attention output using torch.sdpa on
contiguous Q, K, and V tensors.
3. Simulating vLLM's paged KV cache: It takes the context portion of the
K/V tensors and manually places them into a paged buffer according to
the test's (randomly generated) block table.
4. Running each vLLM attention backend with the new queries and the
simulated paged KV cache.
5. Comparing the vLLM backend's output to the ground-truth SDPA output.
Note: When tensor_parallel_size > 1, we simulate the head partitioning
by overriding the model config to use fewer heads, without requiring
multiple GPUs. This tests that backends work correctly with different
head counts.
"""
set_random_seed(42)
hf_config_override = None
if tensor_parallel_size > 1:
from vllm.config import ModelConfig
temp_config = ModelConfig(model=model, max_model_len=1)
original_num_heads = temp_config.hf_text_config.num_attention_heads
original_num_kv_heads = getattr(
temp_config.hf_text_config, "num_key_value_heads", None
)
hf_config_override = {
"num_attention_heads": original_num_heads // tensor_parallel_size,
}
if original_num_kv_heads is not None:
hf_config_override["num_key_value_heads"] = max(
1, original_num_kv_heads // tensor_parallel_size
)
vllm_config = create_vllm_config(
model_name=model,
tensor_parallel_size=1, # Always use TP=1 to avoid multi-GPU requirements
max_model_len=max(batch_spec.seq_lens),
dtype=model_dtype or "auto",
block_size=block_size,
num_gpu_blocks=8192,
hf_config_override=hf_config_override,
)
if max_num_seqs is not None:
vllm_config.scheduler_config.max_num_seqs = max_num_seqs
if max_num_batched_tokens is not None:
vllm_config.scheduler_config.max_num_batched_tokens = max_num_batched_tokens
if num_speculative_tokens > 0:
vllm_config.speculative_config = SimpleNamespace(
num_speculative_tokens=num_speculative_tokens
)
vllm_config.cache_config.cache_dtype = kv_cache_dtype
device = torch.device(f"{DEVICE_TYPE}:0")
kv_cache_spec = create_standard_kv_cache_spec(vllm_config, attn_type)
# 1. Setup
batch_size = batch_spec.batch_size
seq_lens = batch_spec.seq_lens
query_lens = batch_spec.query_lens
num_q_heads = vllm_config.model_config.get_num_attention_heads(
vllm_config.parallel_config
)
num_kv_heads = vllm_config.model_config.get_num_kv_heads(
vllm_config.parallel_config
)
sinks = (
torch.linspace(-1.0, 1.0, num_q_heads, dtype=torch.float32, device=device)
if use_sinks
else None
)
head_size = vllm_config.model_config.get_head_size()
sliding_window = vllm_config.model_config.get_sliding_window()
dtype = _convert_dtype_to_torch(vllm_config.model_config.dtype)
block_size = vllm_config.cache_config.block_size
scale = 1.0 / (head_size**0.5)
fp8_kv_cache = is_quantized_kv_cache(kv_cache_dtype)
if fp8_kv_cache:
query_fp8_dtype = current_platform.fp8_dtype()
kv_fp8_dtype = FP8_KV_CACHE_DTYPES[kv_cache_dtype]
atol = max(atol, 6e-2)
rtol = max(rtol, 1e-1)
# 2. Generate data and compute SDPA reference output
all_q_vllm, all_k_vllm, all_v_vllm = [], [], []
all_sdpa_outputs = []
k_contexts, v_contexts = [], []
for i in range(batch_size):
s_len = seq_lens[i]
q_len = query_lens[i]
context_len = s_len - q_len
# Generate Q, K, V for the whole sequence to be used in SDPA
q = torch.randn(q_len, num_q_heads, head_size, dtype=dtype, device=device)
k_full = torch.randn(s_len, num_kv_heads, head_size, dtype=dtype, device=device)
v_full = torch.randn(s_len, num_kv_heads, head_size, dtype=dtype, device=device)
if fp8_kv_cache:
q_ref = q.to(query_fp8_dtype).to(dtype)
k_ref = k_full.to(kv_fp8_dtype).to(dtype)
v_ref = v_full.to(kv_fp8_dtype).to(dtype)
else:
q_ref, k_ref, v_ref = q, k_full, v_full
# SDPA expects (N, H, L, D), so unsqueeze batch and permute
q_sdpa_in = q_ref.unsqueeze(0).transpose(1, 2)
k_sdpa_in = k_ref.unsqueeze(0).transpose(1, 2)
v_sdpa_in = v_ref.unsqueeze(0).transpose(1, 2)
if num_q_heads != num_kv_heads:
assert num_q_heads % num_kv_heads == 0, (
f"num_q_heads ({num_q_heads}) must be divisible by "
f"num_kv_heads ({num_kv_heads})"
)
repeats = num_q_heads // num_kv_heads
k_sdpa_in = k_sdpa_in.repeat_interleave(repeats, dim=1)
v_sdpa_in = v_sdpa_in.repeat_interleave(repeats, dim=1)
# Create causal mask: query token i attends to positions 0 to
# (context_len + i)
kv_len = s_len
final_mask_mod = partial(mask_mod, context_len=context_len)
if sinks is None:
block_mask = create_block_mask(
final_mask_mod,
B=None,
H=None,
Q_LEN=q_len,
KV_LEN=kv_len,
device=device,
)
sdpa_out_i = flex_attention(
q_sdpa_in,
k_sdpa_in,
v_sdpa_in,
block_mask=block_mask,
scale=scale,
enable_gqa=True,
)
else:
scores = (
torch.matmul(q_sdpa_in.float(), k_sdpa_in.float().transpose(-2, -1))
* scale
)
q_idx = torch.arange(q_len, device=device).unsqueeze(1)
kv_idx = torch.arange(kv_len, device=device).unsqueeze(0)
zero = torch.zeros((), dtype=torch.int32, device=device)
valid = final_mask_mod(zero, zero, q_idx, kv_idx)
scores.masked_fill_(~valid.unsqueeze(0).unsqueeze(0), -torch.inf)
sink_logits = sinks.view(1, num_q_heads, 1, 1).expand(1, -1, q_len, -1)
weights = torch.softmax(torch.cat((scores, sink_logits), dim=-1), dim=-1)[
..., :kv_len
]
sdpa_out_i = torch.matmul(weights, v_sdpa_in.float()).to(dtype)
all_sdpa_outputs.append(sdpa_out_i.transpose(1, 2).squeeze(0))
# Inputs for vLLM backends are just the new tokens
all_q_vllm.append(q)
all_k_vllm.append(k_full[context_len:])
all_v_vllm.append(v_full[context_len:])
# Contextual K/V data used to populate the paged cache
k_contexts.append(k_full[:context_len])
v_contexts.append(v_full[:context_len])
query_vllm = torch.cat(all_q_vllm, dim=0)
key_vllm = torch.cat(all_k_vllm, dim=0)
value_vllm = torch.cat(all_v_vllm, dim=0)
sdpa_output = torch.cat(all_sdpa_outputs, dim=0)
common_attn_metadata = create_common_attn_metadata(
batch_spec, vllm_config.cache_config.block_size, device
)
common_attn_metadata.causal = causal
# 3. Simulate Paged KV Cache and a realistic slot_mapping
# Mirror selector-time resolution locally (caller-requested layout, then
# any backend-required layout, then the default chain).
declared = [
supported
for backend in backend_to_test
if (
supported := _actual_backend(backend)
.get_class()
.supported_kv_cache_layouts()
)
is not None
]
if layout is None:
# Mirror the resolver: the shared cache uses the declared sets' preferred
# layout; backends that don't support it get a per-backend copy below.
layout = min(declared, key=len)[0] if declared else KVCacheLayout.LBNHC
kv_cache = create_and_prepopulate_kv_cache(
k_contexts=k_contexts,
v_contexts=v_contexts,
block_size=block_size,
num_kv_heads=num_kv_heads,
head_size=head_size,
dtype=dtype,
device=device,
num_blocks=vllm_config.cache_config.num_gpu_blocks or 1000,
common_attn_metadata=common_attn_metadata,
layout=layout,
randomize_blocks=True,
kv_cache_dtype=kv_cache_dtype,
)
# 4. Run vLLM backends and compare
# Note: flex_attention has known Triton kernel compatibility issues
# with test infrastructures
for backend_name in backend_to_test:
backend_cls = _actual_backend(backend_name).get_class()
if is_quantized_kv_cache(kv_cache_dtype) and (
not backend_cls.supports_kv_cache_dtype(kv_cache_dtype)
):
continue
kv_cache_for_backend = kv_cache
backend_layout = layout
backend_supported = backend_cls.supported_kv_cache_layouts()
if backend_supported is not None and backend_layout not in backend_supported:
# Production resolution never pairs this backend with the shared layout
# (e.g. flex is LBNHC-only next to FlashInfer's LBHNC); give it a copy
# of the cache in a layout it supports.
backend_layout = backend_supported[0]
kv_cache_for_backend = _clone_kv_cache_in_layout(kv_cache, backend_layout)
if backend_name == AttentionBackendEnum.B12X:
cache_dtype = (
FP8_KV_CACHE_DTYPES[kv_cache_dtype]
if is_quantized_kv_cache(kv_cache_dtype)
else kv_cache.dtype
)
typed_cache = kv_cache.view(cache_dtype)
key_cache = typed_cache[..., :head_size].permute(0, 2, 1, 3)
value_cache = typed_cache[..., head_size:].permute(0, 2, 1, 3)
packed_cache = torch.stack((key_cache, value_cache), dim=1).flatten(-2)
if is_quantized_kv_cache(kv_cache_dtype):
packed_cache = packed_cache.view(torch.uint8)
kv_cache_for_backend = _clone_kv_cache_in_layout(
packed_cache, backend_layout
)
# FlashInfer reads the layout at plan time; set it to match
# the physical order of the test cache.
vllm_config.cache_config.kv_cache_layout = backend_layout.name
backend_output = run_attention_backend(
backend_name,
kv_cache_spec,
["placeholder"],
vllm_config,
device,
common_attn_metadata,
query_vllm,
key_vllm,
value_vllm,
kv_cache_for_backend,
sliding_window=sliding_window,
attn_type=attn_type,
kv_cache_dtype=kv_cache_dtype,
sinks=sinks,
use_cuda_graph=use_cuda_graph,
layer_k_scale=layer_k_scale,
layer_v_scale=layer_v_scale,
)
# Check shape and dtype consistency
assert backend_output.shape == sdpa_output.shape, (
f"[{backend_name}] shape {backend_output.shape} != "
f"SDPA shape {sdpa_output.shape}"
)
assert backend_output.dtype == sdpa_output.dtype, (
f"[{backend_name}] dtype {backend_output.dtype} != "
f"SDPA dtype {sdpa_output.dtype}"
)
assert torch.isfinite(backend_output).all(), (
f"[{backend_name}] produced non-finite values"
)
# Check numerical similarity
def error_msg(msg: str, backend_name: AttentionBackendEnum | str):
return f"[{backend_name}] output differs from SDPA baseline. {msg}"
torch.testing.assert_close(
backend_output,
sdpa_output,
rtol=rtol,
atol=atol,
msg=partial(error_msg, backend_name=backend_name),
)
@pytest.mark.parametrize("layout", ["BLHNC", "BHLNC"])
@pytest.mark.parametrize("batch_spec_name", ["small_decode", "small_prefill"])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
def test_flashinfer_cross_layer_layout(
default_vllm_config,
layout: str,
batch_spec_name: str,
kv_cache_dtype: str,
):
if AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST:
pytest.skip("FlashInfer is not installed")
def causal_mask_mod(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
):
return (q_idx + context_len) >= kv_idx
_test_backend_correctness(
batch_spec=BATCH_SPECS[batch_spec_name],
model="Qwen/Qwen3-0.6B",
backend_to_test=[AttentionBackendEnum.FLASHINFER],
mask_mod=causal_mask_mod,
kv_cache_dtype=kv_cache_dtype,
layout=KVCacheLayout[layout],
)
@pytest.mark.parametrize(
"batch_spec_name",
[
"small_decode",
"small_prefill",
"mixed_small",
"medium_decode",
"medium_prefill",
"mixed_medium",
"large_decode",
"large_prefill",
"single_decode",
"single_prefill",
],
)
@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"])
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
def test_causal_backend_correctness(
default_vllm_config,
batch_spec_name: str,
model: str,
tensor_parallel_size: int,
kv_cache_dtype: str,
):
"""Test backend's correctness with causal attention."""
def causal_mask_mod(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
):
return (q_idx + context_len) >= kv_idx
batch_spec = BATCH_SPECS[batch_spec_name]
LARGE_BLOCK_BACKENDS = (
[AttentionBackendEnum.FLEX_ATTENTION]
if is_torch_equal_or_newer("2.9.0.dev0")
else []
)
if current_platform.is_rocm():
SMALL_BLOCK_BACKENDS = [
x
for x in BACKENDS_TO_TEST
if (
x not in LARGE_BLOCK_BACKENDS
and x is not AttentionBackendEnum.FLASH_ATTN
)
]
else:
SMALL_BLOCK_BACKENDS = [
x for x in BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS
]
_test_backend_correctness(
batch_spec,
model,
SMALL_BLOCK_BACKENDS,
causal_mask_mod,
tensor_parallel_size=tensor_parallel_size,
kv_cache_dtype=kv_cache_dtype,
)
# Fast FlexAttention needs to run with block_size=128
if LARGE_BLOCK_BACKENDS:
_test_backend_correctness(
batch_spec,
model,
LARGE_BLOCK_BACKENDS,
causal_mask_mod,
block_size=128,
tensor_parallel_size=tensor_parallel_size,
kv_cache_dtype=kv_cache_dtype,
)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
def test_flashinfer_xqa_bmm1_scale_matches_decode_q_dtype():
"""XQA decode should only apply q_scale when decode Q is FP8."""
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
class MockLayer:
_q_scale_float = 2.0
_k_scale_float = 3.0
impl = object.__new__(flashinfer_backend.FlashInferImpl)
impl.scale = 0.5
impl.kv_cache_dtype = "fp8"
assert impl.get_xqa_bmm1_scale(MockLayer, torch.bfloat16) == 1.5
assert impl.get_xqa_bmm1_scale(MockLayer, torch.float8_e4m3fn) == 3.0
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
def test_flashinfer_xqa_draft_masks():
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
device = torch.device("cpu")
causal = flashinfer_backend._make_xqa_draft_block_mask(3, True, device)
full = flashinfer_backend._make_xqa_draft_block_mask(3, False, device)
ragged = flashinfer_backend._make_xqa_ragged_draft_block_mask(
[2, 3], 3, True, device
)
assert torch.equal(
causal.view(torch.int16),
torch.tensor([[1, 0], [3, 0], [7, 0]], dtype=torch.int16),
)
assert torch.equal(
full.view(torch.int16), torch.tensor([[7, 0]] * 3, dtype=torch.int16)
)
assert torch.equal(
ragged.view(torch.int16),
torch.tensor(
[[1, 0], [3, 0], [1, 0], [3, 0], [7, 0]],
dtype=torch.int16,
),
)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
def test_flashinfer_xqa_query_lens_preserve_cudagraph_padding():
"""CUDA-graph padding stays as zero-length requests in ragged offsets."""
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
device = torch.device("cpu")
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
builder.use_xqa = True
qo_indptr = torch.tensor([0, 3, 9, 15, 15], dtype=torch.int32, device=device)
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
qo_indptr,
qo_indptr,
num_decodes=4,
num_decode_tokens=21,
)
assert q_len == 6
assert q_lens == [3, 6, 6, 0]
assert q_cu_seq_lens is not None
assert q_cu_seq_lens.tolist() == [0, 3, 9, 15, 15]
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypatch):
"""Non-ragged XQA keeps one query/output row per padded request."""
import unittest.mock
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
impl = object.__new__(flashinfer_backend.FlashInferImpl)
impl.scale = 1.0
impl.bmm1_scale = None
impl.bmm2_scale = None
impl.kv_cache_dtype = "auto"
impl.is_kvcache_nvfp4 = False
impl.head_size = 16
impl.dcp_world_size = 1
impl.o_sf_scale = None
impl.window_left = -1
impl.sinks = None
layer = unittest.mock.Mock(
_q_scale=torch.tensor(1.0),
_q_scale_float=1.0,
_k_scale_float=1.0,
_v_scale_float=1.0,
)
decode = flashinfer_backend.FlashInferTrtllmAPIDecode(
kernel=flashinfer_backend.FlashInferDecodeKernel.XQA,
block_tables=torch.zeros((4, 1), dtype=torch.int32),
seq_lens=torch.tensor([8, 8, 0, 0], dtype=torch.int32),
max_seq_len=8,
q_len_per_req=1,
)
attn_metadata = flashinfer_backend.FlashInferMetadata(
kv_cache_layout=KVCacheLayout.LBHNC,
num_actual_tokens=2,
slot_mapping=torch.empty(0, dtype=torch.int64),
q_data_type_prefill=torch.bfloat16,
q_data_type_decode=torch.bfloat16,
num_decodes=4,
num_decode_tokens=2,
num_prefills=0,
num_prefill_tokens=0,
causal=True,
prefill=None,
decode=decode,
)
seen_shapes = {}
def mock_xqa(**kwargs):
seen_shapes["query"] = kwargs["query"].shape
seen_shapes["out"] = kwargs["out"].shape
seen_shapes["block_tables"] = kwargs["block_tables"].shape
monkeypatch.setattr(
flashinfer_backend,
"_get_trtllm_workspace_buffer",
lambda: torch.empty(1, dtype=torch.uint8),
)
monkeypatch.setattr(
flashinfer_backend, "flashinfer_xqa_batch_decode_with_kv_cache", mock_xqa
)
query = torch.zeros((4, 1, 16), dtype=torch.bfloat16)
output = torch.empty_like(query)
result = impl.forward(
layer,
query,
torch.empty_like(query),
torch.empty_like(query),
torch.zeros((1, 1, 1, 32), dtype=torch.bfloat16),
attn_metadata,
output,
)
assert result is output
assert seen_shapes == {
"query": torch.Size([4, 1, 16]),
"out": torch.Size([4, 1, 16]),
"block_tables": torch.Size([4, 1]),
}
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
def test_flashinfer_xqa_query_lens_require_exact_uniform_product():
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
builder.use_xqa = True
qo_indptr = torch.tensor([0, 3, 3, 3], dtype=torch.int32)
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
qo_indptr,
qo_indptr,
num_decodes=3,
num_decode_tokens=6,
)
assert q_len == 3
assert q_lens == [3, 0, 0]
assert q_cu_seq_lens is not None
assert q_cu_seq_lens.tolist() == [0, 3, 3, 3]
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
@pytest.mark.parametrize(
("qo_indptr_values", "num_decode_tokens", "expected_q_len"),
[
([0, 3, 3], 3, 3),
([0, 2, 4, 4, 4], 4, 2),
],
)
def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
qo_indptr_values: list[int],
num_decode_tokens: int,
expected_q_len: int,
):
"""Padded speculative decode keeps its actual packed query width."""
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
builder.use_xqa = False
qo_indptr = torch.tensor(qo_indptr_values, dtype=torch.int32)
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
qo_indptr,
qo_indptr,
num_decodes=len(qo_indptr_values) - 1,
num_decode_tokens=num_decode_tokens,
)
assert q_len == expected_q_len
assert q_lens is None
assert q_cu_seq_lens is not None
assert q_cu_seq_lens.tolist() == qo_indptr_values
@pytest.mark.skipif(
not current_platform.is_cuda()
or AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer with CUDA is required.",
)
@pytest.mark.parametrize(
"runner_type,spec_kind,has_upper_bound,expected_copies",
[
("generate", None, True, 0),
("pooling", None, True, 0),
("generate", None, False, 1),
("generate", "spec_decode", True, 1),
("generate", "diffusion", True, 1),
],
)
def test_flashinfer_avoids_seq_lens_copy_without_spec_tokens(
monkeypatch, runner_type, spec_kind, has_upper_bound, expected_copies
):
"""Planning reuses the host seq_lens bound only when it is exact.
Without speculative token accounting (no spec-decode drafts, no dLLM
canvas tokens) scheduled tokens always land in KV, so the CPU upper bound
is exact regardless of runner type. Any speculative accounting keeps the
bound optimistic (rejected tokens are in-flight) and forces the copy.
"""
import unittest.mock
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
from vllm.v1.attention.backends.utils import PerLayerParameters
monkeypatch.setattr(
flashinfer_backend,
"get_per_layer_parameters",
lambda *args: {
"layer.0": PerLayerParameters(
window_left=-1, logits_soft_cap=0.0, sm_scale=0.125
)
},
)
config = create_vllm_config(
model_name="Qwen/Qwen3-0.6B", max_model_len=64, max_num_seqs=3
)
config.model_config.runner_type = runner_type
config.attention_config.use_trtllm_attention = False
if spec_kind == "spec_decode":
config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=3
)
elif spec_kind == "diffusion":
config.diffusion_config = DiffusionConfig(canvas_length=8)
device = torch.device(f"{DEVICE_TYPE}:0")
# Include a one-token final chunk and both partial and full KV pages.
common_attn_metadata = create_common_attn_metadata(
BatchSpec(seq_lens=[17, 20, 32], query_lens=[1, 4, 16]), 16, device
)
if not has_upper_bound:
common_attn_metadata.seq_lens_cpu_upper_bound = None
elif spec_kind is not None:
# Simulate rejected speculative tokens: CPU says 18, GPU says 17.
assert common_attn_metadata.seq_lens_cpu_upper_bound is not None
common_attn_metadata.seq_lens_cpu_upper_bound[0] += 1
with set_current_vllm_config(config):
builder = flashinfer_backend.FlashInferMetadataBuilder(
create_standard_kv_cache_spec(config), ["layer.0"], config, device
)
with unittest.mock.patch.object(
common_attn_metadata.seq_lens,
"cpu",
wraps=common_attn_metadata.seq_lens.cpu,
) as copy_to_cpu:
metadata = builder.build(0, common_attn_metadata)
assert copy_to_cpu.call_count == expected_copies
assert metadata.num_decodes == 1
assert metadata.num_prefills == 2
assert builder.paged_kv_indptr.cpu[:4].tolist() == [0, 2, 4, 6]
assert builder.paged_kv_last_page_len.cpu[:3].tolist() == [1, 4, 16]
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
@pytest.mark.parametrize(
"adaptive,decode_kernel,dcp_size,expected_bound",
[
(None, "TRTLLM_GEN", 1, None),
(False, "TRTLLM_GEN", 1, None),
(True, "TRTLLM_GEN", 1, 8),
(True, "XQA", 1, None),
(True, "TRTLLM_GEN", 2, None),
],
)
def test_flashinfer_varlen_cudagraph_capability(
monkeypatch, adaptive, decode_kernel, dcp_size, expected_bound
):
"""Only the trtllm-gen path that adaptive verification enables replays
varlen decode graphs, up to the decode width; the uniform level is fixed."""
from vllm.model_executor.layers.attention.chunked_local_attention import (
create_chunked_local_attention_backend,
)
from vllm.v1.attention.backends import flashinfer as fi
config = SimpleNamespace(
attention_config=SimpleNamespace(use_non_causal=False),
speculative_config=SimpleNamespace(
enable_adaptive_verification=adaptive,
num_speculative_tokens=7,
parallel_drafting=True,
)
if adaptive is not None
else None,
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size),
model_config=SimpleNamespace(get_num_attention_heads=lambda _: 32),
use_v2_model_runner=True,
)
monkeypatch.setattr(fi, "can_use_trtllm_attention", lambda *_, **__: True)
monkeypatch.setattr(
fi.FlashInferMetadataBuilder,
"_get_flashinfer_trtllm_api_decode_kernel",
staticmethod(lambda: fi.FlashInferDecodeKernel[decode_kernel]),
)
spec = FullAttentionSpec(
block_size=16,
num_kv_heads=4,
head_size=64,
dtype=torch.bfloat16,
)
builder_cls = fi.FlashInferMetadataBuilder
assert builder_cls.get_cudagraph_support(config, spec) == (
AttentionCGSupport.UNIFORM_BATCH
if dcp_size == 1
else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
bound = builder_cls.get_varlen_cudagraph_max_query_len(config, spec)
assert bound == expected_bound
if bound is not None:
# A wrapper forcing NEVER inherits the override but not the bound.
chunked_local = create_chunked_local_attention_backend(fi.FlashInferBackend, 16)
wrapped_cls = chunked_local.get_builder_cls()
assert wrapped_cls.get_varlen_cudagraph_max_query_len(config, spec) is None
@pytest.mark.parametrize(
"use_v2_model_runner,parallel_drafting,expected",
[
(True, None, 1),
(True, False, 8),
(True, True, 8),
(False, False, 8),
(False, True, 15),
],
)
def test_spec_as_decode_width_follows_model_runner(
use_v2_model_runner, parallel_drafting, expected
):
"""Only model runner V1's parallel drafter widens decode requests."""
class Builder(AttentionMetadataBuilder):
def __init__(self, vllm_config):
self.vllm_config = vllm_config
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
raise NotImplementedError
config = SimpleNamespace(
speculative_config=None
if parallel_drafting is None
else SimpleNamespace(
num_speculative_tokens=7, parallel_drafting=parallel_drafting
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
use_v2_model_runner=use_v2_model_runner,
)
assert max_decode_query_len(config) == expected
builder = Builder(config)
builder._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
assert builder.reorder_batch_threshold == expected
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
or not current_platform.is_device_capability_family(100),
reason="TRTLLM-GEN varlen decode graphs require FlashInfer on SM100-family GPUs",
)
@pytest.mark.parametrize(
"head_size,sliding_window,num_tokens,num_reqs,num_heads,num_kv_heads",
[
pytest.param(64, None, 8, 8, 32, 4, id="single-token-capture"),
pytest.param(
256,
1024,
40,
32,
16,
8,
id="gemma4-sliding-gqa",
),
pytest.param(
512,
None,
40,
32,
16,
1,
id="gemma4-global-mqa",
),
],
)
def test_flashinfer_varlen_decode_graph_replays_changed_layout(
tmp_path,
monkeypatch,
head_size,
sliding_window,
num_tokens,
num_reqs,
num_heads,
num_kv_heads,
):
"""Replay changes device offsets, active slots, context, and KV writes.
The graph is captured with evenly spread dummy queries and an explicit
bound of eight. CPU offsets then remain stale while device offsets change.
Compare real output rows with independent FP32 causal/windowed attention,
and verify cache writes including preservation of the null padding block.
"""
from vllm.v1.attention.backends import flashinfer as fi
from vllm.v1.attention.backends.utils import PerLayerParameters
set_random_seed(0)
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
block_size = 16
dtype = torch.bfloat16
LlamaConfig(
architectures=["LlamaForCausalLM"],
hidden_size=num_heads * head_size,
head_dim=head_size,
num_attention_heads=num_heads,
num_key_value_heads=num_kv_heads,
num_hidden_layers=1,
max_position_embeddings=4096,
).save_pretrained(tmp_path)
num_blocks = 1 + num_reqs * cdiv(2112, block_size)
config = create_vllm_config(
model_name=str(tmp_path),
max_model_len=4096,
dtype=dtype,
num_gpu_blocks=num_blocks,
block_size=block_size,
max_num_seqs=32,
max_num_batched_tokens=256,
)
config.cache_config.kv_cache_layout = "BLHNC"
config.speculative_config = SimpleNamespace(
num_speculative_tokens=7,
enable_adaptive_verification=True,
parallel_drafting=False,
)
spec = create_standard_kv_cache_spec(config)
scale = head_size**-0.5
monkeypatch.setattr(
fi,
"get_per_layer_parameters",
lambda *_: {
"layer": PerLayerParameters(
window_left=-1 if sliding_window is None else sliding_window - 1,
logits_soft_cap=0.0,
sm_scale=scale,
has_sinks=False,
),
},
)
device = torch.device("cuda")
layout = KVCacheLayout.BLHNC
query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype, device=device)
key = torch.empty(num_tokens, num_kv_heads, head_size, dtype=dtype, device=device)
value = torch.empty_like(key)
output = torch.empty_like(query)
dummy_lens = [
num_tokens // num_reqs + (i < num_tokens % num_reqs) for i in range(num_reqs)
]
common = create_common_attn_metadata(
BatchSpec(seq_lens=[2112] * num_reqs, query_lens=dummy_lens),
block_size,
device,
)
common.max_query_len = 8
cache = None
def prepare(query_lens):
nonlocal cache
query_lens = query_lens + [0] * (num_reqs - len(query_lens))
query.normal_()
key.normal_()
value.normal_()
# Alternating short prefills, nonempty decode contexts, and contexts
# beyond the sliding window; inactive requests have zero sequence length.
contexts = [
([0, 17, 2079, 39][i % 4] if q else 0) for i, q in enumerate(query_lens)
]
ks = [
torch.randn(c, num_kv_heads, head_size, dtype=dtype, device=device)
for c in contexts
]
vs = [torch.randn_like(k) for k in ks]
actual = create_common_attn_metadata(
BatchSpec(
seq_lens=[c + q for c, q in zip(contexts, query_lens)],
query_lens=query_lens,
),
block_size,
device,
)
fresh_cache = create_and_prepopulate_kv_cache(
ks,
vs,
block_size,
num_kv_heads,
head_size,
dtype,
device,
num_blocks,
actual,
layout,
randomize_blocks=True,
)
if cache is None:
cache = fresh_cache
else:
cache.copy_(fresh_cache)
common.query_start_loc.copy_(actual.query_start_loc)
common.seq_lens.copy_(actual.seq_lens)
common.block_table_tensor.zero_()
common.block_table_tensor[:, : actual.block_table_tensor.shape[1]].copy_(
actual.block_table_tensor
)
common.slot_mapping.fill_(-1)
common.slot_mapping[: sum(query_lens)].copy_(actual.slot_mapping)
return contexts, ks, vs
prepare(dummy_lens)
with set_current_vllm_config(config):
builder = fi.FlashInferMetadataBuilder(spec, ["layer"], config, device)
metadata = builder.build(0, common)
assert metadata.num_prefills == 0
assert metadata.decode.q_len_per_req == 8
impl = fi.FlashInferImpl(
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
scale=scale,
alibi_slopes=None,
sliding_window=sliding_window,
kv_cache_dtype="auto",
attn_type=AttentionType.DECODER,
)
layer = MockAttentionLayer(device)
def forward(attn_metadata=metadata):
impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping)
impl.forward(layer, query, key, value, cache, attn_metadata, output=output)
for _ in range(3):
forward()
torch.accelerator.synchronize()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
forward()
layouts = [[8], [1] * num_reqs]
if num_tokens >= 16:
layouts.append([8, 4, 3, 1])
if num_tokens >= 40:
layouts.append([8] * 5)
def check(query_lens, run):
contexts, ks, vs = prepare(query_lens)
assert cache is not None
output.fill_(float("nan"))
run()
start = 0
for i, q_len in enumerate(query_lens):
if q_len == 0:
continue
end = start + q_len
full_k = (
torch.cat((ks[i], key[start:end]))
.float()
.repeat_interleave(num_heads // num_kv_heads, dim=1)
)
full_v = (
torch.cat((vs[i], value[start:end]))
.float()
.repeat_interleave(num_heads // num_kv_heads, dim=1)
)
scores = (
torch.einsum("qhd,khd->hqk", query[start:end].float(), full_k) * scale
)
q_pos = torch.arange(q_len, device=device) + contexts[i]
k_pos = torch.arange(full_k.shape[0], device=device)
allowed = k_pos[None, :] <= q_pos[:, None]
if sliding_window is not None:
allowed &= k_pos[None, :] > q_pos[:, None] - sliding_window
probs = scores.masked_fill(~allowed[None], float("-inf")).softmax(-1)
reference = torch.einsum("hqk,khd->qhd", probs, full_v)
torch.testing.assert_close(
output[start:end].float(),
reference,
atol=2e-2,
rtol=2e-2,
)
slots = common.slot_mapping[start:end]
written = cache[slots // block_size, :, slots % block_size, :]
torch.testing.assert_close(written[..., :head_size], key[start:end])
torch.testing.assert_close(written[..., head_size:], value[start:end])
start = end
assert torch.count_nonzero(cache[0]) == 0
for query_lens in layouts:
check(query_lens, graph.replay)
if num_tokens == 40:
# A longer prefill forces a separate launch in eager/piecewise mode.
# Its length must not become the decode kernel's query bound, and
# compacted CPU placeholders still need not match the GPU allocation.
common.num_reqs = 5
common.max_query_len = 24
common.query_start_loc_cpu[:6] = torch.tensor([0, 4, 8, 12, 16, 40])
common.query_start_loc_cpu[6:] = 40
def mixed_forward():
with set_current_vllm_config(config):
metadata = builder.build(
0,
replace(
common,
query_start_loc=common.query_start_loc[:6],
query_start_loc_cpu=common.query_start_loc_cpu[:6],
seq_lens=common.seq_lens[:5],
seq_lens_cpu_upper_bound=common.seq_lens_cpu_upper_bound[:5],
block_table_tensor=common.block_table_tensor[:5],
),
)
assert metadata.num_decodes == 4
assert metadata.num_prefills == 1
assert metadata.decode.q_len_per_req == 8
forward(metadata)
check([8, 4, 3, 1, 24], mixed_forward)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
def test_flashinfer_attention_sinks_refreshed_after_reload(dtype):
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
source_sinks = torch.tensor([1.0, 2.0], dtype=dtype)
impl = object.__new__(flashinfer_backend.FlashInferImpl)
impl._sinks_source = source_sinks
impl.sinks = source_sinks
impl.process_weights_after_loading(dtype)
assert impl.sinks is not None
sinks_ptr = impl.sinks.data_ptr()
assert impl.sinks.dtype == torch.float32
torch.testing.assert_close(impl.sinks, source_sinks.float())
source_sinks.copy_(torch.tensor([3.0, 4.0], dtype=dtype))
impl.process_weights_after_loading(dtype)
assert impl.sinks.data_ptr() == sinks_ptr
torch.testing.assert_close(impl.sinks, source_sinks.float())
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
def test_flashinfer_native_prefill_with_sinks(default_vllm_config):
supported = current_platform.is_cuda() and (
current_platform.is_device_capability(90)
or current_platform.is_device_capability_family(120)
)
if not supported:
pytest.skip("Native FlashInfer prefill with sinks requires SM90 or SM12x.")
from vllm.v1.attention.backends.flashinfer import FlashInferBackend
if not FlashInferBackend.supports_sink():
pytest.skip("FlashInfer attention sinks are not available in this setup.")
def causal_mask_mod(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
):
return (q_idx + context_len) >= kv_idx
_test_backend_correctness(
BATCH_SPECS["small_prefill"],
"Qwen/Qwen3-0.6B",
[AttentionBackendEnum.FLASHINFER],
causal_mask_mod,
use_sinks=True,
)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
reason="FlashInfer is not available.",
)
def test_flashinfer_xqa_decode_correctness(default_vllm_config):
"""FlashInfer should route SM90/SM12x decode through XQA and match SDPA."""
supported = current_platform.is_cuda() and (
current_platform.is_device_capability(90)
or current_platform.is_device_capability_family(120)
)
if not supported:
pytest.skip("FlashInfer XQA decode requires SM90 or SM12x.")
import unittest.mock
from vllm.utils.flashinfer import can_use_trtllm_attention
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
from vllm.v1.attention.backends.utils import PerLayerParameters
def mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls):
return {
"placeholder": PerLayerParameters(
window_left=-1,
logits_soft_cap=0.0,
sm_scale=1.0,
)
}
def causal_mask_mod(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
):
return (q_idx + context_len) >= kv_idx
batch_spec = BATCH_SPECS["small_decode"]
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3-0.6B",
max_model_len=max(batch_spec.seq_lens),
block_size=16,
)
device = torch.device(f"{DEVICE_TYPE}:0")
kv_cache_spec = FullAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=vllm_config.model_config.get_num_kv_heads(
vllm_config.parallel_config
),
head_size=vllm_config.model_config.get_head_size(),
dtype=vllm_config.model_config.dtype,
)
with set_current_vllm_config(vllm_config):
if not can_use_trtllm_attention(
vllm_config.model_config.get_num_attention_heads(
vllm_config.parallel_config
),
kv_cache_spec.num_kv_heads,
is_prefill=False,
):
pytest.skip("FlashInfer XQA decode is not available in this setup.")
with unittest.mock.patch(
"vllm.v1.attention.backends.flashinfer.get_per_layer_parameters",
mock_get_per_layer_parameters,
):
builder = flashinfer_backend.FlashInferMetadataBuilder(
kv_cache_spec, ["placeholder"], vllm_config, device
)
common_attn_metadata = create_common_attn_metadata(
batch_spec, vllm_config.cache_config.block_size, device
)
attn_metadata = builder.build(0, common_attn_metadata)
assert (
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
vllm_config, kv_cache_spec
)
== AttentionCGSupport.UNIFORM_BATCH
)
wide_head_kv_cache_spec = FullAttentionSpec(
block_size=vllm_config.cache_config.block_size,
num_kv_heads=kv_cache_spec.num_kv_heads,
head_size=512,
dtype=vllm_config.model_config.dtype,
)
assert (
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
vllm_config, wide_head_kv_cache_spec
)
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
)
assert isinstance(
attn_metadata.decode,
flashinfer_backend.FlashInferTrtllmAPIDecode,
)
assert attn_metadata.decode.kernel == flashinfer_backend.FlashInferDecodeKernel.XQA
_test_backend_correctness(
batch_spec,
"Qwen/Qwen3-0.6B",
[AttentionBackendEnum.FLASHINFER],
causal_mask_mod,
)
@pytest.mark.skipif(
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
or not current_platform.is_device_capability_family(120),
reason="NVFP4 XQA decode requires SM12x.",
)
@pytest.mark.parametrize("use_cuda_graph", [False, True])
@pytest.mark.parametrize("q_len", [1, 4])
def test_flashinfer_xqa_nvfp4_decode_correctness(
default_vllm_config, q_len, use_cuda_graph
):
"""Single-token and spec-decode queries over an NVFP4 cache decode with
XQA, eager and replayed, and match the fa2 prefill that wrote the cache."""
import unittest.mock
from vllm.v1.attention.backends import flashinfer as fi
seq_lens, bs, d = [40, 300, 1027], 16, 256
config = create_vllm_config(
model_name="Qwen/Qwen3-0.6B",
max_model_len=max(seq_lens),
block_size=bs,
)
# GQA 6 at head_dim 256, as Qwen3.8-27B at TP=2.
config.model_config.model_arch_config = replace(
config.model_config.model_arch_config,
total_num_attention_heads=24,
total_num_kv_heads=4,
head_size=d,
)
config.cache_config.cache_dtype = "nvfp4"
if q_len > 1:
config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=q_len - 1
)
spec = FullAttentionSpec(
block_size=bs,
num_kv_heads=4,
head_size=d,
dtype=torch.uint8,
kv_quant_mode=KVQuantMode.NVFP4,
)
cg_support = fi.FlashInferMetadataBuilder.get_cudagraph_support(config, spec)
assert cg_support == AttentionCGSupport.UNIFORM_BATCH
device = torch.device(f"{DEVICE_TYPE}:0")
# Request i owns consecutive pages from starts[i]; page 0 stays null.
pages = [0] + [cdiv(s, bs) for s in seq_lens[:-1]]
starts = 1 + torch.tensor(pages).cumsum(0)
max_pages = cdiv(max(seq_lens), bs)
block_table = (starts[:, None] + torch.arange(max_pages)).int().to(device)
full_dim = nvfp4_kv_cache_full_dim(d)
kv_cache = torch.zeros(
int(starts[-1]) + max_pages, bs, 8, full_dim, dtype=torch.uint8, device=device
).transpose(1, 2)
q, k, v = (
torch.randn(sum(seq_lens), h, d).bfloat16().to(device) for h in (24, 4, 4)
)
def run(query_lens, use_cuda_graph=False):
req = torch.repeat_interleave(torch.arange(3), torch.tensor(query_lens))
pos = torch.cat([torch.arange(s - n, s) for s, n in zip(seq_lens, query_lens)])
rows = pos + torch.tensor([0, *seq_lens]).cumsum(0)[req]
metadata = replace(
create_common_attn_metadata(BatchSpec(seq_lens, query_lens), bs, device),
block_table_tensor=block_table,
slot_mapping=(starts[req] * bs + pos).to(device),
)
with set_current_vllm_config(config):
return rows, run_attention_backend(
AttentionBackendEnum.FLASHINFER,
spec,
["placeholder"],
config,
device,
metadata,
q[rows],
k[rows],
v[rows],
kv_cache,
kv_cache_dtype="nvfp4",
use_cuda_graph=use_cuda_graph,
layer_k_scale=0.02,
layer_v_scale=0.03,
)
# The full prefill writes every token through the NVFP4 writer; its fa2
# output for the last q_len tokens of each request is the reference.
_, reference = run(seq_lens)
xqa_fn = fi.flashinfer_xqa_batch_decode_with_kv_cache
with unittest.mock.patch.object(
fi, "flashinfer_xqa_batch_decode_with_kv_cache", wraps=xqa_fn
) as xqa:
rows, output = run([q_len] * len(seq_lens), use_cuda_graph)
assert xqa.called
torch.testing.assert_close(output, reference[rows], atol=2e-2, rtol=2e-2)
if current_platform.is_rocm():
# FLASH_ATTN is not supported on ROCm
SLIDING_WINDOW_BACKENDS_TO_TEST = [
AttentionBackendEnum.FLEX_ATTENTION,
AttentionBackendEnum.TRITON_ATTN,
"FLEX_ATTENTION_SLOW",
]
else:
SLIDING_WINDOW_BACKENDS_TO_TEST = [
AttentionBackendEnum.FLASH_ATTN,
AttentionBackendEnum.FLEX_ATTENTION,
AttentionBackendEnum.TRITON_ATTN,
"FLEX_ATTENTION_SLOW",
]
# Encoder-only FlexAttention always uses the slow builder, so the pseudo-backend
# would run an identical implementation twice.
SLIDING_WINDOW_ENCODER_BACKENDS_TO_TEST = [
backend
for backend in SLIDING_WINDOW_BACKENDS_TO_TEST
if backend != "FLEX_ATTENTION_SLOW"
]
@pytest.mark.parametrize(
"batch_spec_name",
[
"small_decode",
"small_prefill",
"mixed_medium",
"large_decode",
"large_prefill",
"mixed_large",
],
)
@pytest.mark.parametrize("model", ["microsoft/Phi-tiny-MoE-instruct"])
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
def test_sliding_window_backend_correctness(
default_vllm_config,
batch_spec_name: str,
model: str,
tensor_parallel_size: int,
):
"""Test backend's correctness with sliding window attention."""
def sliding_window_mask_mod(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
sliding_window: int,
):
causal_mask = q_idx + context_len >= kv_idx
window_mask = q_idx + context_len - kv_idx < sliding_window
return causal_mask & window_mask
batch_spec = BATCH_SPECS[batch_spec_name]
model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens))
sliding_window = model_config.get_sliding_window()
sliding_window_mask_mod_fn = partial(
sliding_window_mask_mod, sliding_window=sliding_window
)
LARGE_BLOCK_BACKENDS = (
[AttentionBackendEnum.FLEX_ATTENTION]
if is_torch_equal_or_newer("2.9.0.dev0")
else []
)
SMALL_BLOCK_BACKENDS = [
x for x in SLIDING_WINDOW_BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS
]
_test_backend_correctness(
batch_spec,
model,
SMALL_BLOCK_BACKENDS,
sliding_window_mask_mod_fn,
tensor_parallel_size=tensor_parallel_size,
)
# Fast FlexAttention needs to run with block_size=128
if LARGE_BLOCK_BACKENDS:
_test_backend_correctness(
batch_spec,
model,
LARGE_BLOCK_BACKENDS,
sliding_window_mask_mod_fn,
block_size=128,
tensor_parallel_size=tensor_parallel_size,
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test")
def test_rocm_aiter_fa_unquantized_cache_ignores_kv_scales(
default_vllm_config,
):
"""Stale scale tensors must not affect an unquantized KV cache."""
from vllm._aiter_ops import is_aiter_found_and_supported
if not is_aiter_found_and_supported():
pytest.skip("AITER is required")
def sliding_window_mask_mod(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
sliding_window: int,
):
causal_mask = q_idx + context_len >= kv_idx
window_mask = q_idx + context_len - kv_idx < sliding_window
return causal_mask & window_mask
model = "microsoft/Phi-tiny-MoE-instruct"
batch_spec = BATCH_SPECS["small_decode"]
model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens))
sliding_window = model_config.get_sliding_window()
assert sliding_window is not None
_test_backend_correctness(
batch_spec,
model,
[AttentionBackendEnum.ROCM_AITER_FA],
partial(sliding_window_mask_mod, sliding_window=sliding_window),
layer_k_scale=0.25,
layer_v_scale=0.5,
)
@pytest.mark.parametrize(
"batch_spec_name",
[
"small_encoder_prefill",
"medium_encoder_prefill",
],
)
@pytest.mark.parametrize("model", ["google/embeddinggemma-300m"])
@pytest.mark.parametrize("tensor_parallel_size", [1, 2])
def test_sliding_window_encoder_backend_correctness(
default_vllm_config,
batch_spec_name: str,
model: str,
tensor_parallel_size: int,
):
"""Test backend's correctness with sliding window attention."""
def bidi_sliding_window_mask_mod(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
sliding_window: int,
):
return torch.abs(q_idx + context_len - kv_idx) < sliding_window
batch_spec = BATCH_SPECS[batch_spec_name]
model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens))
sliding_window = model_config.get_sliding_window()
sliding_window_mask_mod_fn = partial(
bidi_sliding_window_mask_mod, sliding_window=sliding_window
)
_test_backend_correctness(
batch_spec,
model,
SLIDING_WINDOW_ENCODER_BACKENDS_TO_TEST,
sliding_window_mask_mod_fn,
causal=False,
attn_type=AttentionType.ENCODER_ONLY,
tensor_parallel_size=tensor_parallel_size,
)
NON_CAUSAL_BACKENDS_TO_TEST = [
AttentionBackendEnum.FLASH_ATTN,
AttentionBackendEnum.FLEX_ATTENTION,
"FLEX_ATTENTION_SLOW",
]
if current_platform.is_rocm():
NON_CAUSAL_BACKENDS_TO_TEST = [
x
for x in NON_CAUSAL_BACKENDS_TO_TEST
if x is not AttentionBackendEnum.FLASH_ATTN
]
@pytest.mark.parametrize(
"batch_spec_name",
[
"small_decode",
"small_prefill",
"mixed_small",
],
)
@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"])
def test_non_causal_backend_correctness(
default_vllm_config,
batch_spec_name: str,
model: str,
):
"""Test backend's correctness with non-causal (bidirectional) decoder
attention, as used by DFlash speculative decoding."""
def bidirectional_mask_mod(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
):
return q_idx >= 0 # Always True
batch_spec = BATCH_SPECS[batch_spec_name]
LARGE_BLOCK_BACKENDS = (
[AttentionBackendEnum.FLEX_ATTENTION]
if is_torch_equal_or_newer("2.9.0.dev0")
else []
)
SMALL_BLOCK_BACKENDS = [
x for x in NON_CAUSAL_BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS
]
_test_backend_correctness(
batch_spec,
model,
SMALL_BLOCK_BACKENDS,
bidirectional_mask_mod,
causal=False,
)
if LARGE_BLOCK_BACKENDS:
_test_backend_correctness(
batch_spec,
model,
LARGE_BLOCK_BACKENDS,
bidirectional_mask_mod,
causal=False,
block_size=128,
)