Signed-off-by: lesj0610 <lesj0610@gmail.com> Co-authored-by: Wei Zhao <51183510+wzhao18@users.noreply.github.com>
2044 lines
70 KiB
Python
2044 lines
70 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Tests for v1 attention backends without GPUModelRunner dependency."""
|
|
|
|
from dataclasses import replace
|
|
from functools import partial
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
import torch
|
|
from torch.nn.attention.flex_attention import create_block_mask, flex_attention
|
|
from transformers import LlamaConfig
|
|
|
|
from tests.v1.attention.utils import (
|
|
BatchSpec,
|
|
create_common_attn_metadata,
|
|
create_standard_kv_cache_spec,
|
|
create_vllm_config,
|
|
try_backend_includes_kv_cache_update,
|
|
try_get_attention_backend,
|
|
)
|
|
from vllm.config import (
|
|
DiffusionConfig,
|
|
ModelConfig,
|
|
SpeculativeConfig,
|
|
set_current_vllm_config,
|
|
)
|
|
from vllm.platforms import current_platform
|
|
from vllm.utils.math_utils import cdiv
|
|
from vllm.utils.torch_utils import (
|
|
STR_DTYPE_TO_TORCH_DTYPE,
|
|
is_quantized_kv_cache,
|
|
is_torch_equal_or_newer,
|
|
nvfp4_kv_cache_full_dim,
|
|
set_random_seed,
|
|
)
|
|
from vllm.v1.attention.backend import (
|
|
AttentionCGSupport,
|
|
AttentionMetadataBuilder,
|
|
AttentionType,
|
|
CommonAttentionMetadata,
|
|
max_decode_query_len,
|
|
)
|
|
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
|
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout, KVQuantMode
|
|
|
|
BACKENDS_TO_TEST = [
|
|
AttentionBackendEnum.FLASH_ATTN,
|
|
AttentionBackendEnum.FLASHINFER,
|
|
AttentionBackendEnum.FLEX_ATTENTION,
|
|
AttentionBackendEnum.TRITON_ATTN,
|
|
"FLEX_ATTENTION_SLOW",
|
|
]
|
|
|
|
|
|
def _actual_backend(backend: AttentionBackendEnum | str) -> AttentionBackendEnum:
|
|
"""Resolve pseudo-backends (FLEX_ATTENTION_SLOW) to their real enum."""
|
|
if isinstance(backend, str):
|
|
if backend != "FLEX_ATTENTION_SLOW":
|
|
raise ValueError(f"Unknown pseudo-backend: {backend}")
|
|
return AttentionBackendEnum.FLEX_ATTENTION
|
|
return backend
|
|
|
|
|
|
DEVICE_TYPE = current_platform.device_type
|
|
|
|
# Use the platform's preferred FP8 type so the stored cache matches what the
|
|
# backends reinterpret at runtime. On ROCm gfx94x this is e4m3fnuz, not e4m3fn;
|
|
# storing e4m3fn bytes there would be re-read as fnuz and produce NaNs.
|
|
FP8_KV_CACHE_DTYPES = {
|
|
"fp8": current_platform.fp8_dtype(),
|
|
"fp8_e4m3": current_platform.fp8_dtype(),
|
|
}
|
|
|
|
# Remove flashinfer from the list if it's not available
|
|
try:
|
|
import flashinfer # noqa: F401
|
|
except ImportError:
|
|
BACKENDS_TO_TEST.remove(AttentionBackendEnum.FLASHINFER)
|
|
|
|
|
|
def _convert_dtype_to_torch(dtype):
|
|
"""Convert ModelDType to torch.dtype."""
|
|
if isinstance(dtype, str):
|
|
if dtype == "auto":
|
|
return torch.float16 # Default dtype for testing
|
|
elif dtype in STR_DTYPE_TO_TORCH_DTYPE:
|
|
return STR_DTYPE_TO_TORCH_DTYPE[dtype]
|
|
else:
|
|
raise ValueError(f"Unknown dtype: {dtype}")
|
|
elif isinstance(dtype, torch.dtype):
|
|
return dtype
|
|
else:
|
|
raise ValueError(f"Unknown dtype: {dtype}")
|
|
|
|
|
|
# Define common batch configurations
|
|
BATCH_SPECS = {
|
|
"small_decode": BatchSpec(seq_lens=[32, 40], query_lens=[1, 1]),
|
|
"small_prefill": BatchSpec(seq_lens=[32, 40], query_lens=[8, 8]),
|
|
"mixed_small": BatchSpec(seq_lens=[32, 40, 48, 56], query_lens=[1, 1, 5, 5]),
|
|
"medium_decode": BatchSpec(
|
|
seq_lens=[128, 256, 512, 1024, 128, 256, 512, 1024],
|
|
query_lens=[1, 1, 1, 1, 1, 1, 1, 1],
|
|
),
|
|
"medium_prefill": BatchSpec(
|
|
seq_lens=[256, 512, 1024, 2048], query_lens=[16, 16, 16, 16]
|
|
),
|
|
"mixed_medium": BatchSpec(
|
|
seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[1, 1, 1, 7, 7, 7]
|
|
),
|
|
"large_decode": BatchSpec(seq_lens=[2048] * 32, query_lens=[1] * 32),
|
|
"large_prefill": BatchSpec(seq_lens=[4096] * 8, query_lens=[32] * 8),
|
|
"mixed_large": BatchSpec(
|
|
seq_lens=[1024, 2048, 4096, 1024, 2048, 4096], query_lens=[1, 1, 1, 32, 32, 32]
|
|
),
|
|
"single_decode": BatchSpec(seq_lens=[1024], query_lens=[1]),
|
|
"single_prefill": BatchSpec(seq_lens=[1024], query_lens=[64]),
|
|
# encoder-only
|
|
"small_encoder_prefill": BatchSpec(
|
|
seq_lens=[32, 64, 128, 256], query_lens=[32, 64, 128, 256]
|
|
),
|
|
"medium_encoder_prefill": BatchSpec(
|
|
seq_lens=[256, 512, 1024, 2048], query_lens=[256, 512, 1024, 2048]
|
|
),
|
|
}
|
|
|
|
|
|
def create_and_prepopulate_kv_cache(
|
|
k_contexts: list[torch.Tensor],
|
|
v_contexts: list[torch.Tensor],
|
|
block_size: int,
|
|
num_kv_heads: int,
|
|
head_size: int,
|
|
dtype: torch.dtype,
|
|
device: torch.device,
|
|
num_blocks: int,
|
|
common_attn_metadata: CommonAttentionMetadata,
|
|
layout: KVCacheLayout,
|
|
randomize_blocks: bool = True,
|
|
kv_cache_dtype: str = "auto",
|
|
) -> torch.Tensor:
|
|
"""Create and prepopulate a KV cache with context data.
|
|
|
|
Args:
|
|
k_contexts: List of key context tensors for each sequence
|
|
v_contexts: List of value context tensors for each sequence
|
|
block_size: Size of each block
|
|
num_kv_heads: Number of KV heads
|
|
head_size: Size of each head
|
|
dtype: Data type for the cache
|
|
device: Device to create the cache on
|
|
num_blocks: Total number of blocks in the cache
|
|
common_attn_metadata: Provides seq lens, block table and slot mapping
|
|
layout: Physical layout to allocate in; the cache is returned as the
|
|
logical ``[B, H, N, C]`` view (as ``create_kv_cache_views`` does)
|
|
randomize_blocks: Whether to randomly permute blocks
|
|
or use sequential order
|
|
kv_cache_dtype: Cache dtype string; fp8 caches use fp8 storage
|
|
|
|
Returns:
|
|
A 4D tensor in logical ``(num_blocks, num_kv_heads, block_size,
|
|
2 * head_size)`` order with strides determined by ``layout``.
|
|
|
|
"""
|
|
batch_size = len(k_contexts)
|
|
seq_lens = common_attn_metadata.seq_lens.cpu()
|
|
query_lens = (
|
|
common_attn_metadata.query_start_loc_cpu[1:]
|
|
- common_attn_metadata.query_start_loc_cpu[:-1]
|
|
)
|
|
context_lens = seq_lens - query_lens
|
|
block_table = common_attn_metadata.block_table_tensor
|
|
slot_mapping = common_attn_metadata.slot_mapping
|
|
|
|
# For an fp8 kv cache, store the cache in the fp8 dtype so that assigning
|
|
# the higher-precision context tensors quantizes them, mirroring runtime.
|
|
fp8_kv_cache = is_quantized_kv_cache(kv_cache_dtype)
|
|
storage_dtype = FP8_KV_CACHE_DTYPES[kv_cache_dtype] if fp8_kv_cache else dtype
|
|
|
|
# Logical 5D shape is always [L, B, H, N, C]. Cross-layer layouts need
|
|
# at least two layers to reproduce the inter-layer gaps in a layer view.
|
|
logical_4d = (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
|
num_layers = 1 if layout.is_layer_compact else 2
|
|
logical_5d = (num_layers, *logical_4d)
|
|
physical_5d = tuple(logical_5d[i] for i in layout.stride_order)
|
|
inv_order = [layout.stride_order.index(i) for i in range(5)]
|
|
|
|
kv_cache_physical = torch.zeros(physical_5d, dtype=storage_dtype, device=device)
|
|
# Permute to logical [L, B, H, N, C], then select a layer. This mirrors
|
|
# create_kv_cache_views and retains cross-layer strides in the 4D view.
|
|
kv_cache = kv_cache_physical.permute(*inv_order)[0]
|
|
|
|
# Write context tokens into the cache via the logical view:
|
|
# kv_cache[block, :, token_in_block, :] routes correctly regardless
|
|
# of physical layout.
|
|
# Start from block_id=1 since block_id=0 is considered the null block
|
|
start_block_idx = 1
|
|
for i in range(batch_size):
|
|
k_context, v_context = k_contexts[i], v_contexts[i]
|
|
t = torch.arange(k_context.shape[0], device=device)
|
|
blk = start_block_idx + t // block_size
|
|
off = t % block_size
|
|
# Advanced indexing on (blk, off) yields [T, H, hs] destinations.
|
|
# index_put is dtype-strict; cast like the scalar path would.
|
|
kv_cache[blk, :, off, :head_size] = k_context.to(kv_cache.dtype)
|
|
kv_cache[blk, :, off, head_size:] = v_context.to(kv_cache.dtype)
|
|
# Stay block aligned and allocate enough blocks for the new tokens
|
|
start_block_idx += cdiv(int(seq_lens[i]), block_size)
|
|
|
|
blocks_end = start_block_idx
|
|
|
|
# Permute the context blocks (excluding block 0 which is null)
|
|
if randomize_blocks:
|
|
# Random permutation starting from block 1
|
|
perm = torch.randperm(blocks_end - 1) + 1
|
|
else:
|
|
# Sequential order starting from block 1
|
|
perm = torch.arange(1, blocks_end)
|
|
|
|
inv_perm = torch.zeros(blocks_end, dtype=torch.long, device=device)
|
|
# Add 1 to account for starting from block 1
|
|
inv_perm[1:] = torch.argsort(perm) + 1
|
|
kv_cache[1:blocks_end, ...] = kv_cache[perm, ...]
|
|
|
|
# Construct the right block table
|
|
# Start from block_id=1 since block_id=0 is considered the null block
|
|
start_block_idx = 1
|
|
for i in range(batch_size):
|
|
num_blocks_for_seq = cdiv(int(seq_lens[i]), block_size)
|
|
start = start_block_idx
|
|
end = start + num_blocks_for_seq
|
|
block_table[i, :num_blocks_for_seq] = inv_perm[start:end]
|
|
start_block_idx += num_blocks_for_seq
|
|
|
|
# Create a realistic slot mapping that corresponds to the block table
|
|
for i in range(batch_size):
|
|
token_offsets = torch.arange(int(query_lens[i])) + int(context_lens[i])
|
|
block_indices = token_offsets // block_size
|
|
token_inter_block_offsets = token_offsets % block_size
|
|
start = common_attn_metadata.query_start_loc_cpu[i]
|
|
end = common_attn_metadata.query_start_loc_cpu[i + 1]
|
|
slot_mapping[start:end] = block_table[
|
|
i, block_indices
|
|
] * block_size + token_inter_block_offsets.to(device)
|
|
|
|
if fp8_kv_cache:
|
|
kv_cache = kv_cache.view(torch.uint8)
|
|
|
|
return kv_cache
|
|
|
|
|
|
class MockAttentionLayer:
|
|
"""A mock attention layer for testing."""
|
|
|
|
def __init__(
|
|
self,
|
|
device: torch.device,
|
|
k_scale: float = 1.0,
|
|
v_scale: float = 1.0,
|
|
):
|
|
self._q_scale = torch.tensor(1.0, device=device)
|
|
self._k_scale = torch.tensor(k_scale, device=device)
|
|
self._v_scale = torch.tensor(v_scale, device=device)
|
|
# Add float versions for flashinfer
|
|
self._q_scale_float = 1.0
|
|
self._k_scale_float = k_scale
|
|
self._v_scale_float = v_scale
|
|
|
|
|
|
def _clone_kv_cache_in_layout(
|
|
kv_cache: torch.Tensor, layout: KVCacheLayout
|
|
) -> torch.Tensor:
|
|
"""Copy a logical [B, H, N, C] cache into a fresh allocation in `layout`."""
|
|
logical_5d = (1, *kv_cache.shape)
|
|
physical = torch.zeros(
|
|
tuple(logical_5d[i] for i in layout.stride_order),
|
|
dtype=kv_cache.dtype,
|
|
device=kv_cache.device,
|
|
)
|
|
inv_order = [layout.stride_order.index(i) for i in range(5)]
|
|
view = physical.permute(*inv_order)[0]
|
|
view.copy_(kv_cache)
|
|
return view
|
|
|
|
|
|
def run_attention_backend(
|
|
backend: AttentionBackendEnum | str,
|
|
kv_cache_spec: FullAttentionSpec,
|
|
layer_names: list[str],
|
|
vllm_config,
|
|
device: torch.device,
|
|
common_attn_metadata: CommonAttentionMetadata,
|
|
query: torch.Tensor,
|
|
key: torch.Tensor,
|
|
value: torch.Tensor,
|
|
kv_cache: torch.Tensor,
|
|
attn_type: AttentionType = AttentionType.DECODER,
|
|
sliding_window: int | None = None,
|
|
kv_cache_dtype: str = "auto",
|
|
sinks: torch.Tensor | None = None,
|
|
use_cuda_graph: bool = False,
|
|
layer_k_scale: float = 1.0,
|
|
layer_v_scale: float = 1.0,
|
|
) -> torch.Tensor:
|
|
"""Run attention computation using the specified backend's AttentionImpl."""
|
|
use_direct_block_mask = is_torch_equal_or_newer("2.9.0.dev0")
|
|
if backend == "FLEX_ATTENTION_SLOW":
|
|
use_direct_block_mask = False
|
|
backend = _actual_backend(backend)
|
|
|
|
builder_cls, impl_cls = try_get_attention_backend(backend)
|
|
|
|
# Mock flashinfer's get_per_layer_parameters if needed
|
|
if backend == AttentionBackendEnum.FLASHINFER:
|
|
import unittest.mock
|
|
|
|
from vllm.v1.attention.backends.utils import PerLayerParameters
|
|
|
|
def mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls):
|
|
# Return mock parameters for a single layer
|
|
head_size = vllm_config.model_config.get_head_size()
|
|
return {
|
|
layer_name: PerLayerParameters(
|
|
window_left=-1, # No sliding window
|
|
logits_soft_cap=0.0, # No soft cap
|
|
sm_scale=1.0 / (head_size**0.5), # Standard scale
|
|
has_sinks=sinks is not None,
|
|
)
|
|
for layer_name in layer_names
|
|
}
|
|
|
|
with unittest.mock.patch(
|
|
"vllm.v1.attention.backends.flashinfer.get_per_layer_parameters",
|
|
mock_get_per_layer_parameters,
|
|
):
|
|
builder = builder_cls(kv_cache_spec, layer_names, vllm_config, device)
|
|
attn_metadata = builder.build(
|
|
common_prefix_len=0,
|
|
common_attn_metadata=common_attn_metadata,
|
|
)
|
|
else:
|
|
# Build metadata
|
|
builder = builder_cls(kv_cache_spec, layer_names, vllm_config, device)
|
|
if backend == AttentionBackendEnum.FLEX_ATTENTION:
|
|
builder.direct_build = use_direct_block_mask
|
|
attn_metadata = builder.build(
|
|
common_prefix_len=0,
|
|
common_attn_metadata=common_attn_metadata,
|
|
)
|
|
|
|
# Instantiate implementation
|
|
num_heads = vllm_config.model_config.get_num_attention_heads(
|
|
vllm_config.parallel_config
|
|
)
|
|
num_kv_heads = vllm_config.model_config.get_num_kv_heads(
|
|
vllm_config.parallel_config
|
|
)
|
|
head_size = vllm_config.model_config.get_head_size()
|
|
scale = 1.0 / (head_size**0.5)
|
|
# Impls capture the current vllm config at construction, as in model loading.
|
|
with set_current_vllm_config(vllm_config):
|
|
impl = impl_cls(
|
|
num_heads=num_heads,
|
|
head_size=head_size,
|
|
scale=scale,
|
|
num_kv_heads=num_kv_heads,
|
|
alibi_slopes=None,
|
|
sliding_window=sliding_window,
|
|
attn_type=attn_type,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
**({"sinks": sinks} if sinks is not None else {}),
|
|
)
|
|
|
|
# Create mock layer and output buffer
|
|
mock_layer = MockAttentionLayer(device, layer_k_scale, layer_v_scale)
|
|
output = torch.empty_like(query)
|
|
|
|
if is_quantized_kv_cache(kv_cache_dtype) and impl.supports_quant_query_input:
|
|
query = query.to(current_platform.fp8_dtype())
|
|
|
|
# Run forward pass
|
|
# NOTE: The query, key, and value are already shaped correctly
|
|
# in the calling test function.
|
|
if not try_backend_includes_kv_cache_update(backend):
|
|
impl.do_kv_cache_update(
|
|
mock_layer, key, value, kv_cache, attn_metadata.slot_mapping
|
|
)
|
|
if use_cuda_graph:
|
|
impl.forward(
|
|
mock_layer, query, key, value, kv_cache, attn_metadata, output=output
|
|
)
|
|
graph = torch.cuda.CUDAGraph()
|
|
with torch.cuda.graph(graph):
|
|
impl.forward(
|
|
mock_layer,
|
|
query,
|
|
key,
|
|
value,
|
|
kv_cache,
|
|
attn_metadata,
|
|
output=output,
|
|
)
|
|
graph.replay()
|
|
else:
|
|
backend_output = impl.forward(
|
|
mock_layer, query, key, value, kv_cache, attn_metadata, output=output
|
|
)
|
|
if backend_output is not None:
|
|
output = backend_output
|
|
|
|
return output
|
|
|
|
|
|
def _test_backend_correctness(
|
|
batch_spec: BatchSpec,
|
|
model: str,
|
|
backend_to_test: list[AttentionBackendEnum | str],
|
|
mask_mod,
|
|
*,
|
|
causal: bool = True,
|
|
attn_type: AttentionType = AttentionType.DECODER,
|
|
block_size: int = 16,
|
|
atol: float = 1e-2,
|
|
rtol: float = 1e-2,
|
|
tensor_parallel_size: int = 1,
|
|
kv_cache_dtype: str = "auto",
|
|
use_sinks: bool = False,
|
|
layout: KVCacheLayout | None = None,
|
|
use_cuda_graph: bool = False,
|
|
num_speculative_tokens: int = 0,
|
|
model_dtype: torch.dtype | None = None,
|
|
max_num_seqs: int | None = None,
|
|
max_num_batched_tokens: int | None = None,
|
|
layer_k_scale: float = 1.0,
|
|
layer_v_scale: float = 1.0,
|
|
):
|
|
"""Test that all backends produce similar outputs to a reference implementation
|
|
using FlexAttention or an explicit attention-sink reference.
|
|
|
|
This test works by:
|
|
1. Generating a batch of sequences with specified context and query lengths.
|
|
2. Computing a ground-truth attention output using torch.sdpa on
|
|
contiguous Q, K, and V tensors.
|
|
3. Simulating vLLM's paged KV cache: It takes the context portion of the
|
|
K/V tensors and manually places them into a paged buffer according to
|
|
the test's (randomly generated) block table.
|
|
4. Running each vLLM attention backend with the new queries and the
|
|
simulated paged KV cache.
|
|
5. Comparing the vLLM backend's output to the ground-truth SDPA output.
|
|
|
|
Note: When tensor_parallel_size > 1, we simulate the head partitioning
|
|
by overriding the model config to use fewer heads, without requiring
|
|
multiple GPUs. This tests that backends work correctly with different
|
|
head counts.
|
|
"""
|
|
set_random_seed(42)
|
|
|
|
hf_config_override = None
|
|
if tensor_parallel_size > 1:
|
|
from vllm.config import ModelConfig
|
|
|
|
temp_config = ModelConfig(model=model, max_model_len=1)
|
|
original_num_heads = temp_config.hf_text_config.num_attention_heads
|
|
original_num_kv_heads = getattr(
|
|
temp_config.hf_text_config, "num_key_value_heads", None
|
|
)
|
|
hf_config_override = {
|
|
"num_attention_heads": original_num_heads // tensor_parallel_size,
|
|
}
|
|
if original_num_kv_heads is not None:
|
|
hf_config_override["num_key_value_heads"] = max(
|
|
1, original_num_kv_heads // tensor_parallel_size
|
|
)
|
|
|
|
vllm_config = create_vllm_config(
|
|
model_name=model,
|
|
tensor_parallel_size=1, # Always use TP=1 to avoid multi-GPU requirements
|
|
max_model_len=max(batch_spec.seq_lens),
|
|
dtype=model_dtype or "auto",
|
|
block_size=block_size,
|
|
num_gpu_blocks=8192,
|
|
hf_config_override=hf_config_override,
|
|
)
|
|
if max_num_seqs is not None:
|
|
vllm_config.scheduler_config.max_num_seqs = max_num_seqs
|
|
if max_num_batched_tokens is not None:
|
|
vllm_config.scheduler_config.max_num_batched_tokens = max_num_batched_tokens
|
|
if num_speculative_tokens > 0:
|
|
vllm_config.speculative_config = SimpleNamespace(
|
|
num_speculative_tokens=num_speculative_tokens
|
|
)
|
|
vllm_config.cache_config.cache_dtype = kv_cache_dtype
|
|
device = torch.device(f"{DEVICE_TYPE}:0")
|
|
|
|
kv_cache_spec = create_standard_kv_cache_spec(vllm_config, attn_type)
|
|
|
|
# 1. Setup
|
|
batch_size = batch_spec.batch_size
|
|
seq_lens = batch_spec.seq_lens
|
|
query_lens = batch_spec.query_lens
|
|
num_q_heads = vllm_config.model_config.get_num_attention_heads(
|
|
vllm_config.parallel_config
|
|
)
|
|
num_kv_heads = vllm_config.model_config.get_num_kv_heads(
|
|
vllm_config.parallel_config
|
|
)
|
|
sinks = (
|
|
torch.linspace(-1.0, 1.0, num_q_heads, dtype=torch.float32, device=device)
|
|
if use_sinks
|
|
else None
|
|
)
|
|
head_size = vllm_config.model_config.get_head_size()
|
|
sliding_window = vllm_config.model_config.get_sliding_window()
|
|
dtype = _convert_dtype_to_torch(vllm_config.model_config.dtype)
|
|
block_size = vllm_config.cache_config.block_size
|
|
scale = 1.0 / (head_size**0.5)
|
|
|
|
fp8_kv_cache = is_quantized_kv_cache(kv_cache_dtype)
|
|
if fp8_kv_cache:
|
|
query_fp8_dtype = current_platform.fp8_dtype()
|
|
kv_fp8_dtype = FP8_KV_CACHE_DTYPES[kv_cache_dtype]
|
|
atol = max(atol, 6e-2)
|
|
rtol = max(rtol, 1e-1)
|
|
|
|
# 2. Generate data and compute SDPA reference output
|
|
all_q_vllm, all_k_vllm, all_v_vllm = [], [], []
|
|
all_sdpa_outputs = []
|
|
k_contexts, v_contexts = [], []
|
|
|
|
for i in range(batch_size):
|
|
s_len = seq_lens[i]
|
|
q_len = query_lens[i]
|
|
context_len = s_len - q_len
|
|
|
|
# Generate Q, K, V for the whole sequence to be used in SDPA
|
|
q = torch.randn(q_len, num_q_heads, head_size, dtype=dtype, device=device)
|
|
k_full = torch.randn(s_len, num_kv_heads, head_size, dtype=dtype, device=device)
|
|
v_full = torch.randn(s_len, num_kv_heads, head_size, dtype=dtype, device=device)
|
|
|
|
if fp8_kv_cache:
|
|
q_ref = q.to(query_fp8_dtype).to(dtype)
|
|
k_ref = k_full.to(kv_fp8_dtype).to(dtype)
|
|
v_ref = v_full.to(kv_fp8_dtype).to(dtype)
|
|
else:
|
|
q_ref, k_ref, v_ref = q, k_full, v_full
|
|
|
|
# SDPA expects (N, H, L, D), so unsqueeze batch and permute
|
|
q_sdpa_in = q_ref.unsqueeze(0).transpose(1, 2)
|
|
k_sdpa_in = k_ref.unsqueeze(0).transpose(1, 2)
|
|
v_sdpa_in = v_ref.unsqueeze(0).transpose(1, 2)
|
|
|
|
if num_q_heads != num_kv_heads:
|
|
assert num_q_heads % num_kv_heads == 0, (
|
|
f"num_q_heads ({num_q_heads}) must be divisible by "
|
|
f"num_kv_heads ({num_kv_heads})"
|
|
)
|
|
repeats = num_q_heads // num_kv_heads
|
|
k_sdpa_in = k_sdpa_in.repeat_interleave(repeats, dim=1)
|
|
v_sdpa_in = v_sdpa_in.repeat_interleave(repeats, dim=1)
|
|
|
|
# Create causal mask: query token i attends to positions 0 to
|
|
# (context_len + i)
|
|
kv_len = s_len
|
|
|
|
final_mask_mod = partial(mask_mod, context_len=context_len)
|
|
if sinks is None:
|
|
block_mask = create_block_mask(
|
|
final_mask_mod,
|
|
B=None,
|
|
H=None,
|
|
Q_LEN=q_len,
|
|
KV_LEN=kv_len,
|
|
device=device,
|
|
)
|
|
sdpa_out_i = flex_attention(
|
|
q_sdpa_in,
|
|
k_sdpa_in,
|
|
v_sdpa_in,
|
|
block_mask=block_mask,
|
|
scale=scale,
|
|
enable_gqa=True,
|
|
)
|
|
else:
|
|
scores = (
|
|
torch.matmul(q_sdpa_in.float(), k_sdpa_in.float().transpose(-2, -1))
|
|
* scale
|
|
)
|
|
q_idx = torch.arange(q_len, device=device).unsqueeze(1)
|
|
kv_idx = torch.arange(kv_len, device=device).unsqueeze(0)
|
|
zero = torch.zeros((), dtype=torch.int32, device=device)
|
|
valid = final_mask_mod(zero, zero, q_idx, kv_idx)
|
|
scores.masked_fill_(~valid.unsqueeze(0).unsqueeze(0), -torch.inf)
|
|
sink_logits = sinks.view(1, num_q_heads, 1, 1).expand(1, -1, q_len, -1)
|
|
weights = torch.softmax(torch.cat((scores, sink_logits), dim=-1), dim=-1)[
|
|
..., :kv_len
|
|
]
|
|
sdpa_out_i = torch.matmul(weights, v_sdpa_in.float()).to(dtype)
|
|
|
|
all_sdpa_outputs.append(sdpa_out_i.transpose(1, 2).squeeze(0))
|
|
|
|
# Inputs for vLLM backends are just the new tokens
|
|
all_q_vllm.append(q)
|
|
all_k_vllm.append(k_full[context_len:])
|
|
all_v_vllm.append(v_full[context_len:])
|
|
|
|
# Contextual K/V data used to populate the paged cache
|
|
k_contexts.append(k_full[:context_len])
|
|
v_contexts.append(v_full[:context_len])
|
|
|
|
query_vllm = torch.cat(all_q_vllm, dim=0)
|
|
key_vllm = torch.cat(all_k_vllm, dim=0)
|
|
value_vllm = torch.cat(all_v_vllm, dim=0)
|
|
sdpa_output = torch.cat(all_sdpa_outputs, dim=0)
|
|
|
|
common_attn_metadata = create_common_attn_metadata(
|
|
batch_spec, vllm_config.cache_config.block_size, device
|
|
)
|
|
common_attn_metadata.causal = causal
|
|
|
|
# 3. Simulate Paged KV Cache and a realistic slot_mapping
|
|
# Mirror selector-time resolution locally (caller-requested layout, then
|
|
# any backend-required layout, then the default chain).
|
|
declared = [
|
|
supported
|
|
for backend in backend_to_test
|
|
if (
|
|
supported := _actual_backend(backend)
|
|
.get_class()
|
|
.supported_kv_cache_layouts()
|
|
)
|
|
is not None
|
|
]
|
|
if layout is None:
|
|
# Mirror the resolver: the shared cache uses the declared sets' preferred
|
|
# layout; backends that don't support it get a per-backend copy below.
|
|
layout = min(declared, key=len)[0] if declared else KVCacheLayout.LBNHC
|
|
kv_cache = create_and_prepopulate_kv_cache(
|
|
k_contexts=k_contexts,
|
|
v_contexts=v_contexts,
|
|
block_size=block_size,
|
|
num_kv_heads=num_kv_heads,
|
|
head_size=head_size,
|
|
dtype=dtype,
|
|
device=device,
|
|
num_blocks=vllm_config.cache_config.num_gpu_blocks or 1000,
|
|
common_attn_metadata=common_attn_metadata,
|
|
layout=layout,
|
|
randomize_blocks=True,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
)
|
|
|
|
# 4. Run vLLM backends and compare
|
|
# Note: flex_attention has known Triton kernel compatibility issues
|
|
# with test infrastructures
|
|
for backend_name in backend_to_test:
|
|
backend_cls = _actual_backend(backend_name).get_class()
|
|
|
|
if is_quantized_kv_cache(kv_cache_dtype) and (
|
|
not backend_cls.supports_kv_cache_dtype(kv_cache_dtype)
|
|
):
|
|
continue
|
|
|
|
kv_cache_for_backend = kv_cache
|
|
backend_layout = layout
|
|
|
|
backend_supported = backend_cls.supported_kv_cache_layouts()
|
|
if backend_supported is not None and backend_layout not in backend_supported:
|
|
# Production resolution never pairs this backend with the shared layout
|
|
# (e.g. flex is LBNHC-only next to FlashInfer's LBHNC); give it a copy
|
|
# of the cache in a layout it supports.
|
|
backend_layout = backend_supported[0]
|
|
kv_cache_for_backend = _clone_kv_cache_in_layout(kv_cache, backend_layout)
|
|
|
|
if backend_name == AttentionBackendEnum.B12X:
|
|
cache_dtype = (
|
|
FP8_KV_CACHE_DTYPES[kv_cache_dtype]
|
|
if is_quantized_kv_cache(kv_cache_dtype)
|
|
else kv_cache.dtype
|
|
)
|
|
typed_cache = kv_cache.view(cache_dtype)
|
|
key_cache = typed_cache[..., :head_size].permute(0, 2, 1, 3)
|
|
value_cache = typed_cache[..., head_size:].permute(0, 2, 1, 3)
|
|
packed_cache = torch.stack((key_cache, value_cache), dim=1).flatten(-2)
|
|
if is_quantized_kv_cache(kv_cache_dtype):
|
|
packed_cache = packed_cache.view(torch.uint8)
|
|
kv_cache_for_backend = _clone_kv_cache_in_layout(
|
|
packed_cache, backend_layout
|
|
)
|
|
|
|
# FlashInfer reads the layout at plan time; set it to match
|
|
# the physical order of the test cache.
|
|
vllm_config.cache_config.kv_cache_layout = backend_layout.name
|
|
|
|
backend_output = run_attention_backend(
|
|
backend_name,
|
|
kv_cache_spec,
|
|
["placeholder"],
|
|
vllm_config,
|
|
device,
|
|
common_attn_metadata,
|
|
query_vllm,
|
|
key_vllm,
|
|
value_vllm,
|
|
kv_cache_for_backend,
|
|
sliding_window=sliding_window,
|
|
attn_type=attn_type,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
sinks=sinks,
|
|
use_cuda_graph=use_cuda_graph,
|
|
layer_k_scale=layer_k_scale,
|
|
layer_v_scale=layer_v_scale,
|
|
)
|
|
|
|
# Check shape and dtype consistency
|
|
assert backend_output.shape == sdpa_output.shape, (
|
|
f"[{backend_name}] shape {backend_output.shape} != "
|
|
f"SDPA shape {sdpa_output.shape}"
|
|
)
|
|
assert backend_output.dtype == sdpa_output.dtype, (
|
|
f"[{backend_name}] dtype {backend_output.dtype} != "
|
|
f"SDPA dtype {sdpa_output.dtype}"
|
|
)
|
|
|
|
assert torch.isfinite(backend_output).all(), (
|
|
f"[{backend_name}] produced non-finite values"
|
|
)
|
|
|
|
# Check numerical similarity
|
|
def error_msg(msg: str, backend_name: AttentionBackendEnum | str):
|
|
return f"[{backend_name}] output differs from SDPA baseline. {msg}"
|
|
|
|
torch.testing.assert_close(
|
|
backend_output,
|
|
sdpa_output,
|
|
rtol=rtol,
|
|
atol=atol,
|
|
msg=partial(error_msg, backend_name=backend_name),
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("layout", ["BLHNC", "BHLNC"])
|
|
@pytest.mark.parametrize("batch_spec_name", ["small_decode", "small_prefill"])
|
|
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
|
def test_flashinfer_cross_layer_layout(
|
|
default_vllm_config,
|
|
layout: str,
|
|
batch_spec_name: str,
|
|
kv_cache_dtype: str,
|
|
):
|
|
if AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST:
|
|
pytest.skip("FlashInfer is not installed")
|
|
|
|
def causal_mask_mod(
|
|
b: torch.Tensor,
|
|
h: torch.Tensor,
|
|
q_idx: torch.Tensor,
|
|
kv_idx: torch.Tensor,
|
|
*,
|
|
context_len: int,
|
|
):
|
|
return (q_idx + context_len) >= kv_idx
|
|
|
|
_test_backend_correctness(
|
|
batch_spec=BATCH_SPECS[batch_spec_name],
|
|
model="Qwen/Qwen3-0.6B",
|
|
backend_to_test=[AttentionBackendEnum.FLASHINFER],
|
|
mask_mod=causal_mask_mod,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
layout=KVCacheLayout[layout],
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"batch_spec_name",
|
|
[
|
|
"small_decode",
|
|
"small_prefill",
|
|
"mixed_small",
|
|
"medium_decode",
|
|
"medium_prefill",
|
|
"mixed_medium",
|
|
"large_decode",
|
|
"large_prefill",
|
|
"single_decode",
|
|
"single_prefill",
|
|
],
|
|
)
|
|
@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"])
|
|
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
|
|
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"])
|
|
def test_causal_backend_correctness(
|
|
default_vllm_config,
|
|
batch_spec_name: str,
|
|
model: str,
|
|
tensor_parallel_size: int,
|
|
kv_cache_dtype: str,
|
|
):
|
|
"""Test backend's correctness with causal attention."""
|
|
|
|
def causal_mask_mod(
|
|
b: torch.Tensor,
|
|
h: torch.Tensor,
|
|
q_idx: torch.Tensor,
|
|
kv_idx: torch.Tensor,
|
|
*,
|
|
context_len: int,
|
|
):
|
|
return (q_idx + context_len) >= kv_idx
|
|
|
|
batch_spec = BATCH_SPECS[batch_spec_name]
|
|
LARGE_BLOCK_BACKENDS = (
|
|
[AttentionBackendEnum.FLEX_ATTENTION]
|
|
if is_torch_equal_or_newer("2.9.0.dev0")
|
|
else []
|
|
)
|
|
|
|
if current_platform.is_rocm():
|
|
SMALL_BLOCK_BACKENDS = [
|
|
x
|
|
for x in BACKENDS_TO_TEST
|
|
if (
|
|
x not in LARGE_BLOCK_BACKENDS
|
|
and x is not AttentionBackendEnum.FLASH_ATTN
|
|
)
|
|
]
|
|
else:
|
|
SMALL_BLOCK_BACKENDS = [
|
|
x for x in BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS
|
|
]
|
|
|
|
_test_backend_correctness(
|
|
batch_spec,
|
|
model,
|
|
SMALL_BLOCK_BACKENDS,
|
|
causal_mask_mod,
|
|
tensor_parallel_size=tensor_parallel_size,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
)
|
|
|
|
# Fast FlexAttention needs to run with block_size=128
|
|
if LARGE_BLOCK_BACKENDS:
|
|
_test_backend_correctness(
|
|
batch_spec,
|
|
model,
|
|
LARGE_BLOCK_BACKENDS,
|
|
causal_mask_mod,
|
|
block_size=128,
|
|
tensor_parallel_size=tensor_parallel_size,
|
|
kv_cache_dtype=kv_cache_dtype,
|
|
)
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
|
reason="FlashInfer is not available.",
|
|
)
|
|
def test_flashinfer_xqa_bmm1_scale_matches_decode_q_dtype():
|
|
"""XQA decode should only apply q_scale when decode Q is FP8."""
|
|
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
|
|
|
class MockLayer:
|
|
_q_scale_float = 2.0
|
|
_k_scale_float = 3.0
|
|
|
|
impl = object.__new__(flashinfer_backend.FlashInferImpl)
|
|
impl.scale = 0.5
|
|
impl.kv_cache_dtype = "fp8"
|
|
|
|
assert impl.get_xqa_bmm1_scale(MockLayer, torch.bfloat16) == 1.5
|
|
assert impl.get_xqa_bmm1_scale(MockLayer, torch.float8_e4m3fn) == 3.0
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
|
reason="FlashInfer is not available.",
|
|
)
|
|
def test_flashinfer_xqa_draft_masks():
|
|
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
|
|
|
device = torch.device("cpu")
|
|
causal = flashinfer_backend._make_xqa_draft_block_mask(3, True, device)
|
|
full = flashinfer_backend._make_xqa_draft_block_mask(3, False, device)
|
|
ragged = flashinfer_backend._make_xqa_ragged_draft_block_mask(
|
|
[2, 3], 3, True, device
|
|
)
|
|
|
|
assert torch.equal(
|
|
causal.view(torch.int16),
|
|
torch.tensor([[1, 0], [3, 0], [7, 0]], dtype=torch.int16),
|
|
)
|
|
assert torch.equal(
|
|
full.view(torch.int16), torch.tensor([[7, 0]] * 3, dtype=torch.int16)
|
|
)
|
|
assert torch.equal(
|
|
ragged.view(torch.int16),
|
|
torch.tensor(
|
|
[[1, 0], [3, 0], [1, 0], [3, 0], [7, 0]],
|
|
dtype=torch.int16,
|
|
),
|
|
)
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
|
reason="FlashInfer is not available.",
|
|
)
|
|
def test_flashinfer_xqa_query_lens_preserve_cudagraph_padding():
|
|
"""CUDA-graph padding stays as zero-length requests in ragged offsets."""
|
|
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
|
|
|
device = torch.device("cpu")
|
|
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
|
|
builder.use_xqa = True
|
|
qo_indptr = torch.tensor([0, 3, 9, 15, 15], dtype=torch.int32, device=device)
|
|
|
|
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
|
|
qo_indptr,
|
|
qo_indptr,
|
|
num_decodes=4,
|
|
num_decode_tokens=21,
|
|
)
|
|
|
|
assert q_len == 6
|
|
assert q_lens == [3, 6, 6, 0]
|
|
assert q_cu_seq_lens is not None
|
|
assert q_cu_seq_lens.tolist() == [0, 3, 9, 15, 15]
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
|
reason="FlashInfer is not available.",
|
|
)
|
|
def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypatch):
|
|
"""Non-ragged XQA keeps one query/output row per padded request."""
|
|
import unittest.mock
|
|
|
|
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
|
|
|
impl = object.__new__(flashinfer_backend.FlashInferImpl)
|
|
impl.scale = 1.0
|
|
impl.bmm1_scale = None
|
|
impl.bmm2_scale = None
|
|
impl.kv_cache_dtype = "auto"
|
|
impl.is_kvcache_nvfp4 = False
|
|
impl.head_size = 16
|
|
impl.dcp_world_size = 1
|
|
impl.o_sf_scale = None
|
|
impl.window_left = -1
|
|
impl.sinks = None
|
|
|
|
layer = unittest.mock.Mock(
|
|
_q_scale=torch.tensor(1.0),
|
|
_q_scale_float=1.0,
|
|
_k_scale_float=1.0,
|
|
_v_scale_float=1.0,
|
|
)
|
|
decode = flashinfer_backend.FlashInferTrtllmAPIDecode(
|
|
kernel=flashinfer_backend.FlashInferDecodeKernel.XQA,
|
|
block_tables=torch.zeros((4, 1), dtype=torch.int32),
|
|
seq_lens=torch.tensor([8, 8, 0, 0], dtype=torch.int32),
|
|
max_seq_len=8,
|
|
q_len_per_req=1,
|
|
)
|
|
attn_metadata = flashinfer_backend.FlashInferMetadata(
|
|
kv_cache_layout=KVCacheLayout.LBHNC,
|
|
num_actual_tokens=2,
|
|
slot_mapping=torch.empty(0, dtype=torch.int64),
|
|
q_data_type_prefill=torch.bfloat16,
|
|
q_data_type_decode=torch.bfloat16,
|
|
num_decodes=4,
|
|
num_decode_tokens=2,
|
|
num_prefills=0,
|
|
num_prefill_tokens=0,
|
|
causal=True,
|
|
prefill=None,
|
|
decode=decode,
|
|
)
|
|
|
|
seen_shapes = {}
|
|
|
|
def mock_xqa(**kwargs):
|
|
seen_shapes["query"] = kwargs["query"].shape
|
|
seen_shapes["out"] = kwargs["out"].shape
|
|
seen_shapes["block_tables"] = kwargs["block_tables"].shape
|
|
|
|
monkeypatch.setattr(
|
|
flashinfer_backend,
|
|
"_get_trtllm_workspace_buffer",
|
|
lambda: torch.empty(1, dtype=torch.uint8),
|
|
)
|
|
monkeypatch.setattr(
|
|
flashinfer_backend, "flashinfer_xqa_batch_decode_with_kv_cache", mock_xqa
|
|
)
|
|
|
|
query = torch.zeros((4, 1, 16), dtype=torch.bfloat16)
|
|
output = torch.empty_like(query)
|
|
result = impl.forward(
|
|
layer,
|
|
query,
|
|
torch.empty_like(query),
|
|
torch.empty_like(query),
|
|
torch.zeros((1, 1, 1, 32), dtype=torch.bfloat16),
|
|
attn_metadata,
|
|
output,
|
|
)
|
|
|
|
assert result is output
|
|
assert seen_shapes == {
|
|
"query": torch.Size([4, 1, 16]),
|
|
"out": torch.Size([4, 1, 16]),
|
|
"block_tables": torch.Size([4, 1]),
|
|
}
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
|
reason="FlashInfer is not available.",
|
|
)
|
|
def test_flashinfer_xqa_query_lens_require_exact_uniform_product():
|
|
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
|
|
|
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
|
|
builder.use_xqa = True
|
|
qo_indptr = torch.tensor([0, 3, 3, 3], dtype=torch.int32)
|
|
|
|
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
|
|
qo_indptr,
|
|
qo_indptr,
|
|
num_decodes=3,
|
|
num_decode_tokens=6,
|
|
)
|
|
|
|
assert q_len == 3
|
|
assert q_lens == [3, 0, 0]
|
|
assert q_cu_seq_lens is not None
|
|
assert q_cu_seq_lens.tolist() == [0, 3, 3, 3]
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
|
reason="FlashInfer is not available.",
|
|
)
|
|
@pytest.mark.parametrize(
|
|
("qo_indptr_values", "num_decode_tokens", "expected_q_len"),
|
|
[
|
|
([0, 3, 3], 3, 3),
|
|
([0, 2, 4, 4, 4], 4, 2),
|
|
],
|
|
)
|
|
def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets(
|
|
qo_indptr_values: list[int],
|
|
num_decode_tokens: int,
|
|
expected_q_len: int,
|
|
):
|
|
"""Padded speculative decode keeps its actual packed query width."""
|
|
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
|
|
|
builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder)
|
|
builder.use_xqa = False
|
|
qo_indptr = torch.tensor(qo_indptr_values, dtype=torch.int32)
|
|
|
|
q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens(
|
|
qo_indptr,
|
|
qo_indptr,
|
|
num_decodes=len(qo_indptr_values) - 1,
|
|
num_decode_tokens=num_decode_tokens,
|
|
)
|
|
|
|
assert q_len == expected_q_len
|
|
assert q_lens is None
|
|
assert q_cu_seq_lens is not None
|
|
assert q_cu_seq_lens.tolist() == qo_indptr_values
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
not current_platform.is_cuda()
|
|
or AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
|
reason="FlashInfer with CUDA is required.",
|
|
)
|
|
@pytest.mark.parametrize(
|
|
"runner_type,spec_kind,has_upper_bound,expected_copies",
|
|
[
|
|
("generate", None, True, 0),
|
|
("pooling", None, True, 0),
|
|
("generate", None, False, 1),
|
|
("generate", "spec_decode", True, 1),
|
|
("generate", "diffusion", True, 1),
|
|
],
|
|
)
|
|
def test_flashinfer_avoids_seq_lens_copy_without_spec_tokens(
|
|
monkeypatch, runner_type, spec_kind, has_upper_bound, expected_copies
|
|
):
|
|
"""Planning reuses the host seq_lens bound only when it is exact.
|
|
|
|
Without speculative token accounting (no spec-decode drafts, no dLLM
|
|
canvas tokens) scheduled tokens always land in KV, so the CPU upper bound
|
|
is exact regardless of runner type. Any speculative accounting keeps the
|
|
bound optimistic (rejected tokens are in-flight) and forces the copy.
|
|
"""
|
|
import unittest.mock
|
|
|
|
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
|
from vllm.v1.attention.backends.utils import PerLayerParameters
|
|
|
|
monkeypatch.setattr(
|
|
flashinfer_backend,
|
|
"get_per_layer_parameters",
|
|
lambda *args: {
|
|
"layer.0": PerLayerParameters(
|
|
window_left=-1, logits_soft_cap=0.0, sm_scale=0.125
|
|
)
|
|
},
|
|
)
|
|
|
|
config = create_vllm_config(
|
|
model_name="Qwen/Qwen3-0.6B", max_model_len=64, max_num_seqs=3
|
|
)
|
|
config.model_config.runner_type = runner_type
|
|
config.attention_config.use_trtllm_attention = False
|
|
if spec_kind == "spec_decode":
|
|
config.speculative_config = SpeculativeConfig(
|
|
method="ngram", num_speculative_tokens=3
|
|
)
|
|
elif spec_kind == "diffusion":
|
|
config.diffusion_config = DiffusionConfig(canvas_length=8)
|
|
device = torch.device(f"{DEVICE_TYPE}:0")
|
|
# Include a one-token final chunk and both partial and full KV pages.
|
|
common_attn_metadata = create_common_attn_metadata(
|
|
BatchSpec(seq_lens=[17, 20, 32], query_lens=[1, 4, 16]), 16, device
|
|
)
|
|
if not has_upper_bound:
|
|
common_attn_metadata.seq_lens_cpu_upper_bound = None
|
|
elif spec_kind is not None:
|
|
# Simulate rejected speculative tokens: CPU says 18, GPU says 17.
|
|
assert common_attn_metadata.seq_lens_cpu_upper_bound is not None
|
|
common_attn_metadata.seq_lens_cpu_upper_bound[0] += 1
|
|
|
|
with set_current_vllm_config(config):
|
|
builder = flashinfer_backend.FlashInferMetadataBuilder(
|
|
create_standard_kv_cache_spec(config), ["layer.0"], config, device
|
|
)
|
|
with unittest.mock.patch.object(
|
|
common_attn_metadata.seq_lens,
|
|
"cpu",
|
|
wraps=common_attn_metadata.seq_lens.cpu,
|
|
) as copy_to_cpu:
|
|
metadata = builder.build(0, common_attn_metadata)
|
|
|
|
assert copy_to_cpu.call_count == expected_copies
|
|
assert metadata.num_decodes == 1
|
|
assert metadata.num_prefills == 2
|
|
assert builder.paged_kv_indptr.cpu[:4].tolist() == [0, 2, 4, 6]
|
|
assert builder.paged_kv_last_page_len.cpu[:3].tolist() == [1, 4, 16]
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
|
reason="FlashInfer is not available.",
|
|
)
|
|
@pytest.mark.parametrize(
|
|
"adaptive,decode_kernel,dcp_size,expected_bound",
|
|
[
|
|
(None, "TRTLLM_GEN", 1, None),
|
|
(False, "TRTLLM_GEN", 1, None),
|
|
(True, "TRTLLM_GEN", 1, 8),
|
|
(True, "XQA", 1, None),
|
|
(True, "TRTLLM_GEN", 2, None),
|
|
],
|
|
)
|
|
def test_flashinfer_varlen_cudagraph_capability(
|
|
monkeypatch, adaptive, decode_kernel, dcp_size, expected_bound
|
|
):
|
|
"""Only the trtllm-gen path that adaptive verification enables replays
|
|
varlen decode graphs, up to the decode width; the uniform level is fixed."""
|
|
from vllm.model_executor.layers.attention.chunked_local_attention import (
|
|
create_chunked_local_attention_backend,
|
|
)
|
|
from vllm.v1.attention.backends import flashinfer as fi
|
|
|
|
config = SimpleNamespace(
|
|
attention_config=SimpleNamespace(use_non_causal=False),
|
|
speculative_config=SimpleNamespace(
|
|
enable_adaptive_verification=adaptive,
|
|
num_speculative_tokens=7,
|
|
parallel_drafting=True,
|
|
)
|
|
if adaptive is not None
|
|
else None,
|
|
parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size),
|
|
model_config=SimpleNamespace(get_num_attention_heads=lambda _: 32),
|
|
use_v2_model_runner=True,
|
|
)
|
|
monkeypatch.setattr(fi, "can_use_trtllm_attention", lambda *_, **__: True)
|
|
monkeypatch.setattr(
|
|
fi.FlashInferMetadataBuilder,
|
|
"_get_flashinfer_trtllm_api_decode_kernel",
|
|
staticmethod(lambda: fi.FlashInferDecodeKernel[decode_kernel]),
|
|
)
|
|
spec = FullAttentionSpec(
|
|
block_size=16,
|
|
num_kv_heads=4,
|
|
head_size=64,
|
|
dtype=torch.bfloat16,
|
|
)
|
|
builder_cls = fi.FlashInferMetadataBuilder
|
|
assert builder_cls.get_cudagraph_support(config, spec) == (
|
|
AttentionCGSupport.UNIFORM_BATCH
|
|
if dcp_size == 1
|
|
else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
|
)
|
|
bound = builder_cls.get_varlen_cudagraph_max_query_len(config, spec)
|
|
assert bound == expected_bound
|
|
if bound is not None:
|
|
# A wrapper forcing NEVER inherits the override but not the bound.
|
|
chunked_local = create_chunked_local_attention_backend(fi.FlashInferBackend, 16)
|
|
wrapped_cls = chunked_local.get_builder_cls()
|
|
assert wrapped_cls.get_varlen_cudagraph_max_query_len(config, spec) is None
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"use_v2_model_runner,parallel_drafting,expected",
|
|
[
|
|
(True, None, 1),
|
|
(True, False, 8),
|
|
(True, True, 8),
|
|
(False, False, 8),
|
|
(False, True, 15),
|
|
],
|
|
)
|
|
def test_spec_as_decode_width_follows_model_runner(
|
|
use_v2_model_runner, parallel_drafting, expected
|
|
):
|
|
"""Only model runner V1's parallel drafter widens decode requests."""
|
|
|
|
class Builder(AttentionMetadataBuilder):
|
|
def __init__(self, vllm_config):
|
|
self.vllm_config = vllm_config
|
|
|
|
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
|
|
raise NotImplementedError
|
|
|
|
config = SimpleNamespace(
|
|
speculative_config=None
|
|
if parallel_drafting is None
|
|
else SimpleNamespace(
|
|
num_speculative_tokens=7, parallel_drafting=parallel_drafting
|
|
),
|
|
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
|
use_v2_model_runner=use_v2_model_runner,
|
|
)
|
|
assert max_decode_query_len(config) == expected
|
|
builder = Builder(config)
|
|
builder._init_reorder_batch_threshold(1, supports_spec_as_decode=True)
|
|
assert builder.reorder_batch_threshold == expected
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
|
|
or not current_platform.is_device_capability_family(100),
|
|
reason="TRTLLM-GEN varlen decode graphs require FlashInfer on SM100-family GPUs",
|
|
)
|
|
@pytest.mark.parametrize(
|
|
"head_size,sliding_window,num_tokens,num_reqs,num_heads,num_kv_heads",
|
|
[
|
|
pytest.param(64, None, 8, 8, 32, 4, id="single-token-capture"),
|
|
pytest.param(
|
|
256,
|
|
1024,
|
|
40,
|
|
32,
|
|
16,
|
|
8,
|
|
id="gemma4-sliding-gqa",
|
|
),
|
|
pytest.param(
|
|
512,
|
|
None,
|
|
40,
|
|
32,
|
|
16,
|
|
1,
|
|
id="gemma4-global-mqa",
|
|
),
|
|
],
|
|
)
|
|
def test_flashinfer_varlen_decode_graph_replays_changed_layout(
|
|
tmp_path,
|
|
monkeypatch,
|
|
head_size,
|
|
sliding_window,
|
|
num_tokens,
|
|
num_reqs,
|
|
num_heads,
|
|
num_kv_heads,
|
|
):
|
|
"""Replay changes device offsets, active slots, context, and KV writes.
|
|
|
|
The graph is captured with evenly spread dummy queries and an explicit
|
|
bound of eight. CPU offsets then remain stale while device offsets change.
|
|
Compare real output rows with independent FP32 causal/windowed attention,
|
|
and verify cache writes including preservation of the null padding block.
|
|
"""
|
|
from vllm.v1.attention.backends import flashinfer as fi
|
|
from vllm.v1.attention.backends.utils import PerLayerParameters
|
|
|
|
set_random_seed(0)
|
|
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
|
block_size = 16
|
|
dtype = torch.bfloat16
|
|
LlamaConfig(
|
|
architectures=["LlamaForCausalLM"],
|
|
hidden_size=num_heads * head_size,
|
|
head_dim=head_size,
|
|
num_attention_heads=num_heads,
|
|
num_key_value_heads=num_kv_heads,
|
|
num_hidden_layers=1,
|
|
max_position_embeddings=4096,
|
|
).save_pretrained(tmp_path)
|
|
num_blocks = 1 + num_reqs * cdiv(2112, block_size)
|
|
config = create_vllm_config(
|
|
model_name=str(tmp_path),
|
|
max_model_len=4096,
|
|
dtype=dtype,
|
|
num_gpu_blocks=num_blocks,
|
|
block_size=block_size,
|
|
max_num_seqs=32,
|
|
max_num_batched_tokens=256,
|
|
)
|
|
config.cache_config.kv_cache_layout = "BLHNC"
|
|
config.speculative_config = SimpleNamespace(
|
|
num_speculative_tokens=7,
|
|
enable_adaptive_verification=True,
|
|
parallel_drafting=False,
|
|
)
|
|
spec = create_standard_kv_cache_spec(config)
|
|
scale = head_size**-0.5
|
|
monkeypatch.setattr(
|
|
fi,
|
|
"get_per_layer_parameters",
|
|
lambda *_: {
|
|
"layer": PerLayerParameters(
|
|
window_left=-1 if sliding_window is None else sliding_window - 1,
|
|
logits_soft_cap=0.0,
|
|
sm_scale=scale,
|
|
has_sinks=False,
|
|
),
|
|
},
|
|
)
|
|
device = torch.device("cuda")
|
|
layout = KVCacheLayout.BLHNC
|
|
query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype, device=device)
|
|
key = torch.empty(num_tokens, num_kv_heads, head_size, dtype=dtype, device=device)
|
|
value = torch.empty_like(key)
|
|
output = torch.empty_like(query)
|
|
dummy_lens = [
|
|
num_tokens // num_reqs + (i < num_tokens % num_reqs) for i in range(num_reqs)
|
|
]
|
|
common = create_common_attn_metadata(
|
|
BatchSpec(seq_lens=[2112] * num_reqs, query_lens=dummy_lens),
|
|
block_size,
|
|
device,
|
|
)
|
|
common.max_query_len = 8
|
|
cache = None
|
|
|
|
def prepare(query_lens):
|
|
nonlocal cache
|
|
query_lens = query_lens + [0] * (num_reqs - len(query_lens))
|
|
query.normal_()
|
|
key.normal_()
|
|
value.normal_()
|
|
# Alternating short prefills, nonempty decode contexts, and contexts
|
|
# beyond the sliding window; inactive requests have zero sequence length.
|
|
contexts = [
|
|
([0, 17, 2079, 39][i % 4] if q else 0) for i, q in enumerate(query_lens)
|
|
]
|
|
ks = [
|
|
torch.randn(c, num_kv_heads, head_size, dtype=dtype, device=device)
|
|
for c in contexts
|
|
]
|
|
vs = [torch.randn_like(k) for k in ks]
|
|
actual = create_common_attn_metadata(
|
|
BatchSpec(
|
|
seq_lens=[c + q for c, q in zip(contexts, query_lens)],
|
|
query_lens=query_lens,
|
|
),
|
|
block_size,
|
|
device,
|
|
)
|
|
fresh_cache = create_and_prepopulate_kv_cache(
|
|
ks,
|
|
vs,
|
|
block_size,
|
|
num_kv_heads,
|
|
head_size,
|
|
dtype,
|
|
device,
|
|
num_blocks,
|
|
actual,
|
|
layout,
|
|
randomize_blocks=True,
|
|
)
|
|
if cache is None:
|
|
cache = fresh_cache
|
|
else:
|
|
cache.copy_(fresh_cache)
|
|
common.query_start_loc.copy_(actual.query_start_loc)
|
|
common.seq_lens.copy_(actual.seq_lens)
|
|
common.block_table_tensor.zero_()
|
|
common.block_table_tensor[:, : actual.block_table_tensor.shape[1]].copy_(
|
|
actual.block_table_tensor
|
|
)
|
|
common.slot_mapping.fill_(-1)
|
|
common.slot_mapping[: sum(query_lens)].copy_(actual.slot_mapping)
|
|
return contexts, ks, vs
|
|
|
|
prepare(dummy_lens)
|
|
with set_current_vllm_config(config):
|
|
builder = fi.FlashInferMetadataBuilder(spec, ["layer"], config, device)
|
|
metadata = builder.build(0, common)
|
|
assert metadata.num_prefills == 0
|
|
assert metadata.decode.q_len_per_req == 8
|
|
impl = fi.FlashInferImpl(
|
|
num_heads=num_heads,
|
|
num_kv_heads=num_kv_heads,
|
|
head_size=head_size,
|
|
scale=scale,
|
|
alibi_slopes=None,
|
|
sliding_window=sliding_window,
|
|
kv_cache_dtype="auto",
|
|
attn_type=AttentionType.DECODER,
|
|
)
|
|
layer = MockAttentionLayer(device)
|
|
|
|
def forward(attn_metadata=metadata):
|
|
impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping)
|
|
impl.forward(layer, query, key, value, cache, attn_metadata, output=output)
|
|
|
|
for _ in range(3):
|
|
forward()
|
|
torch.accelerator.synchronize()
|
|
graph = torch.cuda.CUDAGraph()
|
|
with torch.cuda.graph(graph):
|
|
forward()
|
|
|
|
layouts = [[8], [1] * num_reqs]
|
|
if num_tokens >= 16:
|
|
layouts.append([8, 4, 3, 1])
|
|
if num_tokens >= 40:
|
|
layouts.append([8] * 5)
|
|
|
|
def check(query_lens, run):
|
|
contexts, ks, vs = prepare(query_lens)
|
|
assert cache is not None
|
|
output.fill_(float("nan"))
|
|
run()
|
|
start = 0
|
|
for i, q_len in enumerate(query_lens):
|
|
if q_len == 0:
|
|
continue
|
|
end = start + q_len
|
|
full_k = (
|
|
torch.cat((ks[i], key[start:end]))
|
|
.float()
|
|
.repeat_interleave(num_heads // num_kv_heads, dim=1)
|
|
)
|
|
full_v = (
|
|
torch.cat((vs[i], value[start:end]))
|
|
.float()
|
|
.repeat_interleave(num_heads // num_kv_heads, dim=1)
|
|
)
|
|
scores = (
|
|
torch.einsum("qhd,khd->hqk", query[start:end].float(), full_k) * scale
|
|
)
|
|
q_pos = torch.arange(q_len, device=device) + contexts[i]
|
|
k_pos = torch.arange(full_k.shape[0], device=device)
|
|
allowed = k_pos[None, :] <= q_pos[:, None]
|
|
if sliding_window is not None:
|
|
allowed &= k_pos[None, :] > q_pos[:, None] - sliding_window
|
|
probs = scores.masked_fill(~allowed[None], float("-inf")).softmax(-1)
|
|
reference = torch.einsum("hqk,khd->qhd", probs, full_v)
|
|
torch.testing.assert_close(
|
|
output[start:end].float(),
|
|
reference,
|
|
atol=2e-2,
|
|
rtol=2e-2,
|
|
)
|
|
slots = common.slot_mapping[start:end]
|
|
written = cache[slots // block_size, :, slots % block_size, :]
|
|
torch.testing.assert_close(written[..., :head_size], key[start:end])
|
|
torch.testing.assert_close(written[..., head_size:], value[start:end])
|
|
start = end
|
|
assert torch.count_nonzero(cache[0]) == 0
|
|
|
|
for query_lens in layouts:
|
|
check(query_lens, graph.replay)
|
|
|
|
if num_tokens == 40:
|
|
# A longer prefill forces a separate launch in eager/piecewise mode.
|
|
# Its length must not become the decode kernel's query bound, and
|
|
# compacted CPU placeholders still need not match the GPU allocation.
|
|
common.num_reqs = 5
|
|
common.max_query_len = 24
|
|
common.query_start_loc_cpu[:6] = torch.tensor([0, 4, 8, 12, 16, 40])
|
|
common.query_start_loc_cpu[6:] = 40
|
|
|
|
def mixed_forward():
|
|
with set_current_vllm_config(config):
|
|
metadata = builder.build(
|
|
0,
|
|
replace(
|
|
common,
|
|
query_start_loc=common.query_start_loc[:6],
|
|
query_start_loc_cpu=common.query_start_loc_cpu[:6],
|
|
seq_lens=common.seq_lens[:5],
|
|
seq_lens_cpu_upper_bound=common.seq_lens_cpu_upper_bound[:5],
|
|
block_table_tensor=common.block_table_tensor[:5],
|
|
),
|
|
)
|
|
assert metadata.num_decodes == 4
|
|
assert metadata.num_prefills == 1
|
|
assert metadata.decode.q_len_per_req == 8
|
|
forward(metadata)
|
|
|
|
check([8, 4, 3, 1, 24], mixed_forward)
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
|
reason="FlashInfer is not available.",
|
|
)
|
|
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
|
def test_flashinfer_attention_sinks_refreshed_after_reload(dtype):
|
|
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
|
|
|
source_sinks = torch.tensor([1.0, 2.0], dtype=dtype)
|
|
impl = object.__new__(flashinfer_backend.FlashInferImpl)
|
|
impl._sinks_source = source_sinks
|
|
impl.sinks = source_sinks
|
|
|
|
impl.process_weights_after_loading(dtype)
|
|
|
|
assert impl.sinks is not None
|
|
sinks_ptr = impl.sinks.data_ptr()
|
|
assert impl.sinks.dtype == torch.float32
|
|
torch.testing.assert_close(impl.sinks, source_sinks.float())
|
|
|
|
source_sinks.copy_(torch.tensor([3.0, 4.0], dtype=dtype))
|
|
impl.process_weights_after_loading(dtype)
|
|
|
|
assert impl.sinks.data_ptr() == sinks_ptr
|
|
torch.testing.assert_close(impl.sinks, source_sinks.float())
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
|
reason="FlashInfer is not available.",
|
|
)
|
|
def test_flashinfer_native_prefill_with_sinks(default_vllm_config):
|
|
supported = current_platform.is_cuda() and (
|
|
current_platform.is_device_capability(90)
|
|
or current_platform.is_device_capability_family(120)
|
|
)
|
|
if not supported:
|
|
pytest.skip("Native FlashInfer prefill with sinks requires SM90 or SM12x.")
|
|
|
|
from vllm.v1.attention.backends.flashinfer import FlashInferBackend
|
|
|
|
if not FlashInferBackend.supports_sink():
|
|
pytest.skip("FlashInfer attention sinks are not available in this setup.")
|
|
|
|
def causal_mask_mod(
|
|
b: torch.Tensor,
|
|
h: torch.Tensor,
|
|
q_idx: torch.Tensor,
|
|
kv_idx: torch.Tensor,
|
|
*,
|
|
context_len: int,
|
|
):
|
|
return (q_idx + context_len) >= kv_idx
|
|
|
|
_test_backend_correctness(
|
|
BATCH_SPECS["small_prefill"],
|
|
"Qwen/Qwen3-0.6B",
|
|
[AttentionBackendEnum.FLASHINFER],
|
|
causal_mask_mod,
|
|
use_sinks=True,
|
|
)
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST,
|
|
reason="FlashInfer is not available.",
|
|
)
|
|
def test_flashinfer_xqa_decode_correctness(default_vllm_config):
|
|
"""FlashInfer should route SM90/SM12x decode through XQA and match SDPA."""
|
|
supported = current_platform.is_cuda() and (
|
|
current_platform.is_device_capability(90)
|
|
or current_platform.is_device_capability_family(120)
|
|
)
|
|
if not supported:
|
|
pytest.skip("FlashInfer XQA decode requires SM90 or SM12x.")
|
|
|
|
import unittest.mock
|
|
|
|
from vllm.utils.flashinfer import can_use_trtllm_attention
|
|
from vllm.v1.attention.backends import flashinfer as flashinfer_backend
|
|
from vllm.v1.attention.backends.utils import PerLayerParameters
|
|
|
|
def mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls):
|
|
return {
|
|
"placeholder": PerLayerParameters(
|
|
window_left=-1,
|
|
logits_soft_cap=0.0,
|
|
sm_scale=1.0,
|
|
)
|
|
}
|
|
|
|
def causal_mask_mod(
|
|
b: torch.Tensor,
|
|
h: torch.Tensor,
|
|
q_idx: torch.Tensor,
|
|
kv_idx: torch.Tensor,
|
|
*,
|
|
context_len: int,
|
|
):
|
|
return (q_idx + context_len) >= kv_idx
|
|
|
|
batch_spec = BATCH_SPECS["small_decode"]
|
|
vllm_config = create_vllm_config(
|
|
model_name="Qwen/Qwen3-0.6B",
|
|
max_model_len=max(batch_spec.seq_lens),
|
|
block_size=16,
|
|
)
|
|
device = torch.device(f"{DEVICE_TYPE}:0")
|
|
kv_cache_spec = FullAttentionSpec(
|
|
block_size=vllm_config.cache_config.block_size,
|
|
num_kv_heads=vllm_config.model_config.get_num_kv_heads(
|
|
vllm_config.parallel_config
|
|
),
|
|
head_size=vllm_config.model_config.get_head_size(),
|
|
dtype=vllm_config.model_config.dtype,
|
|
)
|
|
|
|
with set_current_vllm_config(vllm_config):
|
|
if not can_use_trtllm_attention(
|
|
vllm_config.model_config.get_num_attention_heads(
|
|
vllm_config.parallel_config
|
|
),
|
|
kv_cache_spec.num_kv_heads,
|
|
is_prefill=False,
|
|
):
|
|
pytest.skip("FlashInfer XQA decode is not available in this setup.")
|
|
|
|
with unittest.mock.patch(
|
|
"vllm.v1.attention.backends.flashinfer.get_per_layer_parameters",
|
|
mock_get_per_layer_parameters,
|
|
):
|
|
builder = flashinfer_backend.FlashInferMetadataBuilder(
|
|
kv_cache_spec, ["placeholder"], vllm_config, device
|
|
)
|
|
common_attn_metadata = create_common_attn_metadata(
|
|
batch_spec, vllm_config.cache_config.block_size, device
|
|
)
|
|
attn_metadata = builder.build(0, common_attn_metadata)
|
|
|
|
assert (
|
|
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
|
|
vllm_config, kv_cache_spec
|
|
)
|
|
== AttentionCGSupport.UNIFORM_BATCH
|
|
)
|
|
wide_head_kv_cache_spec = FullAttentionSpec(
|
|
block_size=vllm_config.cache_config.block_size,
|
|
num_kv_heads=kv_cache_spec.num_kv_heads,
|
|
head_size=512,
|
|
dtype=vllm_config.model_config.dtype,
|
|
)
|
|
assert (
|
|
flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support(
|
|
vllm_config, wide_head_kv_cache_spec
|
|
)
|
|
== AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE
|
|
)
|
|
assert isinstance(
|
|
attn_metadata.decode,
|
|
flashinfer_backend.FlashInferTrtllmAPIDecode,
|
|
)
|
|
assert attn_metadata.decode.kernel == flashinfer_backend.FlashInferDecodeKernel.XQA
|
|
|
|
_test_backend_correctness(
|
|
batch_spec,
|
|
"Qwen/Qwen3-0.6B",
|
|
[AttentionBackendEnum.FLASHINFER],
|
|
causal_mask_mod,
|
|
)
|
|
|
|
|
|
@pytest.mark.skipif(
|
|
AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST
|
|
or not current_platform.is_device_capability_family(120),
|
|
reason="NVFP4 XQA decode requires SM12x.",
|
|
)
|
|
@pytest.mark.parametrize("use_cuda_graph", [False, True])
|
|
@pytest.mark.parametrize("q_len", [1, 4])
|
|
def test_flashinfer_xqa_nvfp4_decode_correctness(
|
|
default_vllm_config, q_len, use_cuda_graph
|
|
):
|
|
"""Single-token and spec-decode queries over an NVFP4 cache decode with
|
|
XQA, eager and replayed, and match the fa2 prefill that wrote the cache."""
|
|
import unittest.mock
|
|
|
|
from vllm.v1.attention.backends import flashinfer as fi
|
|
|
|
seq_lens, bs, d = [40, 300, 1027], 16, 256
|
|
config = create_vllm_config(
|
|
model_name="Qwen/Qwen3-0.6B",
|
|
max_model_len=max(seq_lens),
|
|
block_size=bs,
|
|
)
|
|
# GQA 6 at head_dim 256, as Qwen3.8-27B at TP=2.
|
|
config.model_config.model_arch_config = replace(
|
|
config.model_config.model_arch_config,
|
|
total_num_attention_heads=24,
|
|
total_num_kv_heads=4,
|
|
head_size=d,
|
|
)
|
|
config.cache_config.cache_dtype = "nvfp4"
|
|
if q_len > 1:
|
|
config.speculative_config = SpeculativeConfig(
|
|
method="ngram", num_speculative_tokens=q_len - 1
|
|
)
|
|
spec = FullAttentionSpec(
|
|
block_size=bs,
|
|
num_kv_heads=4,
|
|
head_size=d,
|
|
dtype=torch.uint8,
|
|
kv_quant_mode=KVQuantMode.NVFP4,
|
|
)
|
|
cg_support = fi.FlashInferMetadataBuilder.get_cudagraph_support(config, spec)
|
|
assert cg_support == AttentionCGSupport.UNIFORM_BATCH
|
|
|
|
device = torch.device(f"{DEVICE_TYPE}:0")
|
|
# Request i owns consecutive pages from starts[i]; page 0 stays null.
|
|
pages = [0] + [cdiv(s, bs) for s in seq_lens[:-1]]
|
|
starts = 1 + torch.tensor(pages).cumsum(0)
|
|
max_pages = cdiv(max(seq_lens), bs)
|
|
block_table = (starts[:, None] + torch.arange(max_pages)).int().to(device)
|
|
full_dim = nvfp4_kv_cache_full_dim(d)
|
|
kv_cache = torch.zeros(
|
|
int(starts[-1]) + max_pages, bs, 8, full_dim, dtype=torch.uint8, device=device
|
|
).transpose(1, 2)
|
|
q, k, v = (
|
|
torch.randn(sum(seq_lens), h, d).bfloat16().to(device) for h in (24, 4, 4)
|
|
)
|
|
|
|
def run(query_lens, use_cuda_graph=False):
|
|
req = torch.repeat_interleave(torch.arange(3), torch.tensor(query_lens))
|
|
pos = torch.cat([torch.arange(s - n, s) for s, n in zip(seq_lens, query_lens)])
|
|
rows = pos + torch.tensor([0, *seq_lens]).cumsum(0)[req]
|
|
metadata = replace(
|
|
create_common_attn_metadata(BatchSpec(seq_lens, query_lens), bs, device),
|
|
block_table_tensor=block_table,
|
|
slot_mapping=(starts[req] * bs + pos).to(device),
|
|
)
|
|
with set_current_vllm_config(config):
|
|
return rows, run_attention_backend(
|
|
AttentionBackendEnum.FLASHINFER,
|
|
spec,
|
|
["placeholder"],
|
|
config,
|
|
device,
|
|
metadata,
|
|
q[rows],
|
|
k[rows],
|
|
v[rows],
|
|
kv_cache,
|
|
kv_cache_dtype="nvfp4",
|
|
use_cuda_graph=use_cuda_graph,
|
|
layer_k_scale=0.02,
|
|
layer_v_scale=0.03,
|
|
)
|
|
|
|
# The full prefill writes every token through the NVFP4 writer; its fa2
|
|
# output for the last q_len tokens of each request is the reference.
|
|
_, reference = run(seq_lens)
|
|
xqa_fn = fi.flashinfer_xqa_batch_decode_with_kv_cache
|
|
with unittest.mock.patch.object(
|
|
fi, "flashinfer_xqa_batch_decode_with_kv_cache", wraps=xqa_fn
|
|
) as xqa:
|
|
rows, output = run([q_len] * len(seq_lens), use_cuda_graph)
|
|
assert xqa.called
|
|
torch.testing.assert_close(output, reference[rows], atol=2e-2, rtol=2e-2)
|
|
|
|
|
|
if current_platform.is_rocm():
|
|
# FLASH_ATTN is not supported on ROCm
|
|
SLIDING_WINDOW_BACKENDS_TO_TEST = [
|
|
AttentionBackendEnum.FLEX_ATTENTION,
|
|
AttentionBackendEnum.TRITON_ATTN,
|
|
"FLEX_ATTENTION_SLOW",
|
|
]
|
|
else:
|
|
SLIDING_WINDOW_BACKENDS_TO_TEST = [
|
|
AttentionBackendEnum.FLASH_ATTN,
|
|
AttentionBackendEnum.FLEX_ATTENTION,
|
|
AttentionBackendEnum.TRITON_ATTN,
|
|
"FLEX_ATTENTION_SLOW",
|
|
]
|
|
|
|
# Encoder-only FlexAttention always uses the slow builder, so the pseudo-backend
|
|
# would run an identical implementation twice.
|
|
SLIDING_WINDOW_ENCODER_BACKENDS_TO_TEST = [
|
|
backend
|
|
for backend in SLIDING_WINDOW_BACKENDS_TO_TEST
|
|
if backend != "FLEX_ATTENTION_SLOW"
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"batch_spec_name",
|
|
[
|
|
"small_decode",
|
|
"small_prefill",
|
|
"mixed_medium",
|
|
"large_decode",
|
|
"large_prefill",
|
|
"mixed_large",
|
|
],
|
|
)
|
|
@pytest.mark.parametrize("model", ["microsoft/Phi-tiny-MoE-instruct"])
|
|
@pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4])
|
|
def test_sliding_window_backend_correctness(
|
|
default_vllm_config,
|
|
batch_spec_name: str,
|
|
model: str,
|
|
tensor_parallel_size: int,
|
|
):
|
|
"""Test backend's correctness with sliding window attention."""
|
|
|
|
def sliding_window_mask_mod(
|
|
b: torch.Tensor,
|
|
h: torch.Tensor,
|
|
q_idx: torch.Tensor,
|
|
kv_idx: torch.Tensor,
|
|
*,
|
|
context_len: int,
|
|
sliding_window: int,
|
|
):
|
|
causal_mask = q_idx + context_len >= kv_idx
|
|
window_mask = q_idx + context_len - kv_idx < sliding_window
|
|
return causal_mask & window_mask
|
|
|
|
batch_spec = BATCH_SPECS[batch_spec_name]
|
|
model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens))
|
|
sliding_window = model_config.get_sliding_window()
|
|
sliding_window_mask_mod_fn = partial(
|
|
sliding_window_mask_mod, sliding_window=sliding_window
|
|
)
|
|
|
|
LARGE_BLOCK_BACKENDS = (
|
|
[AttentionBackendEnum.FLEX_ATTENTION]
|
|
if is_torch_equal_or_newer("2.9.0.dev0")
|
|
else []
|
|
)
|
|
SMALL_BLOCK_BACKENDS = [
|
|
x for x in SLIDING_WINDOW_BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS
|
|
]
|
|
_test_backend_correctness(
|
|
batch_spec,
|
|
model,
|
|
SMALL_BLOCK_BACKENDS,
|
|
sliding_window_mask_mod_fn,
|
|
tensor_parallel_size=tensor_parallel_size,
|
|
)
|
|
|
|
# Fast FlexAttention needs to run with block_size=128
|
|
if LARGE_BLOCK_BACKENDS:
|
|
_test_backend_correctness(
|
|
batch_spec,
|
|
model,
|
|
LARGE_BLOCK_BACKENDS,
|
|
sliding_window_mask_mod_fn,
|
|
block_size=128,
|
|
tensor_parallel_size=tensor_parallel_size,
|
|
)
|
|
|
|
|
|
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test")
|
|
def test_rocm_aiter_fa_unquantized_cache_ignores_kv_scales(
|
|
default_vllm_config,
|
|
):
|
|
"""Stale scale tensors must not affect an unquantized KV cache."""
|
|
from vllm._aiter_ops import is_aiter_found_and_supported
|
|
|
|
if not is_aiter_found_and_supported():
|
|
pytest.skip("AITER is required")
|
|
|
|
def sliding_window_mask_mod(
|
|
b: torch.Tensor,
|
|
h: torch.Tensor,
|
|
q_idx: torch.Tensor,
|
|
kv_idx: torch.Tensor,
|
|
*,
|
|
context_len: int,
|
|
sliding_window: int,
|
|
):
|
|
causal_mask = q_idx + context_len >= kv_idx
|
|
window_mask = q_idx + context_len - kv_idx < sliding_window
|
|
return causal_mask & window_mask
|
|
|
|
model = "microsoft/Phi-tiny-MoE-instruct"
|
|
batch_spec = BATCH_SPECS["small_decode"]
|
|
model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens))
|
|
sliding_window = model_config.get_sliding_window()
|
|
assert sliding_window is not None
|
|
|
|
_test_backend_correctness(
|
|
batch_spec,
|
|
model,
|
|
[AttentionBackendEnum.ROCM_AITER_FA],
|
|
partial(sliding_window_mask_mod, sliding_window=sliding_window),
|
|
layer_k_scale=0.25,
|
|
layer_v_scale=0.5,
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"batch_spec_name",
|
|
[
|
|
"small_encoder_prefill",
|
|
"medium_encoder_prefill",
|
|
],
|
|
)
|
|
@pytest.mark.parametrize("model", ["google/embeddinggemma-300m"])
|
|
@pytest.mark.parametrize("tensor_parallel_size", [1, 2])
|
|
def test_sliding_window_encoder_backend_correctness(
|
|
default_vllm_config,
|
|
batch_spec_name: str,
|
|
model: str,
|
|
tensor_parallel_size: int,
|
|
):
|
|
"""Test backend's correctness with sliding window attention."""
|
|
|
|
def bidi_sliding_window_mask_mod(
|
|
b: torch.Tensor,
|
|
h: torch.Tensor,
|
|
q_idx: torch.Tensor,
|
|
kv_idx: torch.Tensor,
|
|
*,
|
|
context_len: int,
|
|
sliding_window: int,
|
|
):
|
|
return torch.abs(q_idx + context_len - kv_idx) < sliding_window
|
|
|
|
batch_spec = BATCH_SPECS[batch_spec_name]
|
|
model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens))
|
|
sliding_window = model_config.get_sliding_window()
|
|
sliding_window_mask_mod_fn = partial(
|
|
bidi_sliding_window_mask_mod, sliding_window=sliding_window
|
|
)
|
|
|
|
_test_backend_correctness(
|
|
batch_spec,
|
|
model,
|
|
SLIDING_WINDOW_ENCODER_BACKENDS_TO_TEST,
|
|
sliding_window_mask_mod_fn,
|
|
causal=False,
|
|
attn_type=AttentionType.ENCODER_ONLY,
|
|
tensor_parallel_size=tensor_parallel_size,
|
|
)
|
|
|
|
|
|
NON_CAUSAL_BACKENDS_TO_TEST = [
|
|
AttentionBackendEnum.FLASH_ATTN,
|
|
AttentionBackendEnum.FLEX_ATTENTION,
|
|
"FLEX_ATTENTION_SLOW",
|
|
]
|
|
|
|
if current_platform.is_rocm():
|
|
NON_CAUSAL_BACKENDS_TO_TEST = [
|
|
x
|
|
for x in NON_CAUSAL_BACKENDS_TO_TEST
|
|
if x is not AttentionBackendEnum.FLASH_ATTN
|
|
]
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"batch_spec_name",
|
|
[
|
|
"small_decode",
|
|
"small_prefill",
|
|
"mixed_small",
|
|
],
|
|
)
|
|
@pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"])
|
|
def test_non_causal_backend_correctness(
|
|
default_vllm_config,
|
|
batch_spec_name: str,
|
|
model: str,
|
|
):
|
|
"""Test backend's correctness with non-causal (bidirectional) decoder
|
|
attention, as used by DFlash speculative decoding."""
|
|
|
|
def bidirectional_mask_mod(
|
|
b: torch.Tensor,
|
|
h: torch.Tensor,
|
|
q_idx: torch.Tensor,
|
|
kv_idx: torch.Tensor,
|
|
*,
|
|
context_len: int,
|
|
):
|
|
return q_idx >= 0 # Always True
|
|
|
|
batch_spec = BATCH_SPECS[batch_spec_name]
|
|
LARGE_BLOCK_BACKENDS = (
|
|
[AttentionBackendEnum.FLEX_ATTENTION]
|
|
if is_torch_equal_or_newer("2.9.0.dev0")
|
|
else []
|
|
)
|
|
|
|
SMALL_BLOCK_BACKENDS = [
|
|
x for x in NON_CAUSAL_BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS
|
|
]
|
|
|
|
_test_backend_correctness(
|
|
batch_spec,
|
|
model,
|
|
SMALL_BLOCK_BACKENDS,
|
|
bidirectional_mask_mod,
|
|
causal=False,
|
|
)
|
|
|
|
if LARGE_BLOCK_BACKENDS:
|
|
_test_backend_correctness(
|
|
batch_spec,
|
|
model,
|
|
LARGE_BLOCK_BACKENDS,
|
|
bidirectional_mask_mod,
|
|
causal=False,
|
|
block_size=128,
|
|
)
|