# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Tests for v1 attention backends without GPUModelRunner dependency.""" from dataclasses import replace from functools import partial from types import SimpleNamespace import pytest import torch from torch.nn.attention.flex_attention import create_block_mask, flex_attention from transformers import LlamaConfig from tests.v1.attention.utils import ( BatchSpec, create_common_attn_metadata, create_standard_kv_cache_spec, create_vllm_config, try_backend_includes_kv_cache_update, try_get_attention_backend, ) from vllm.config import ( DiffusionConfig, ModelConfig, SpeculativeConfig, set_current_vllm_config, ) from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv from vllm.utils.torch_utils import ( STR_DTYPE_TO_TORCH_DTYPE, is_quantized_kv_cache, is_torch_equal_or_newer, nvfp4_kv_cache_full_dim, set_random_seed, ) from vllm.v1.attention.backend import ( AttentionCGSupport, AttentionMetadataBuilder, AttentionType, CommonAttentionMetadata, max_decode_query_len, ) from vllm.v1.attention.backends.registry import AttentionBackendEnum from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout, KVQuantMode BACKENDS_TO_TEST = [ AttentionBackendEnum.FLASH_ATTN, AttentionBackendEnum.FLASHINFER, AttentionBackendEnum.FLEX_ATTENTION, AttentionBackendEnum.TRITON_ATTN, "FLEX_ATTENTION_SLOW", ] def _actual_backend(backend: AttentionBackendEnum | str) -> AttentionBackendEnum: """Resolve pseudo-backends (FLEX_ATTENTION_SLOW) to their real enum.""" if isinstance(backend, str): if backend != "FLEX_ATTENTION_SLOW": raise ValueError(f"Unknown pseudo-backend: {backend}") return AttentionBackendEnum.FLEX_ATTENTION return backend DEVICE_TYPE = current_platform.device_type # Use the platform's preferred FP8 type so the stored cache matches what the # backends reinterpret at runtime. On ROCm gfx94x this is e4m3fnuz, not e4m3fn; # storing e4m3fn bytes there would be re-read as fnuz and produce NaNs. FP8_KV_CACHE_DTYPES = { "fp8": current_platform.fp8_dtype(), "fp8_e4m3": current_platform.fp8_dtype(), } # Remove flashinfer from the list if it's not available try: import flashinfer # noqa: F401 except ImportError: BACKENDS_TO_TEST.remove(AttentionBackendEnum.FLASHINFER) def _convert_dtype_to_torch(dtype): """Convert ModelDType to torch.dtype.""" if isinstance(dtype, str): if dtype == "auto": return torch.float16 # Default dtype for testing elif dtype in STR_DTYPE_TO_TORCH_DTYPE: return STR_DTYPE_TO_TORCH_DTYPE[dtype] else: raise ValueError(f"Unknown dtype: {dtype}") elif isinstance(dtype, torch.dtype): return dtype else: raise ValueError(f"Unknown dtype: {dtype}") # Define common batch configurations BATCH_SPECS = { "small_decode": BatchSpec(seq_lens=[32, 40], query_lens=[1, 1]), "small_prefill": BatchSpec(seq_lens=[32, 40], query_lens=[8, 8]), "mixed_small": BatchSpec(seq_lens=[32, 40, 48, 56], query_lens=[1, 1, 5, 5]), "medium_decode": BatchSpec( seq_lens=[128, 256, 512, 1024, 128, 256, 512, 1024], query_lens=[1, 1, 1, 1, 1, 1, 1, 1], ), "medium_prefill": BatchSpec( seq_lens=[256, 512, 1024, 2048], query_lens=[16, 16, 16, 16] ), "mixed_medium": BatchSpec( seq_lens=[512, 1024, 2048, 512, 1024, 2048], query_lens=[1, 1, 1, 7, 7, 7] ), "large_decode": BatchSpec(seq_lens=[2048] * 32, query_lens=[1] * 32), "large_prefill": BatchSpec(seq_lens=[4096] * 8, query_lens=[32] * 8), "mixed_large": BatchSpec( seq_lens=[1024, 2048, 4096, 1024, 2048, 4096], query_lens=[1, 1, 1, 32, 32, 32] ), "single_decode": BatchSpec(seq_lens=[1024], query_lens=[1]), "single_prefill": BatchSpec(seq_lens=[1024], query_lens=[64]), # encoder-only "small_encoder_prefill": BatchSpec( seq_lens=[32, 64, 128, 256], query_lens=[32, 64, 128, 256] ), "medium_encoder_prefill": BatchSpec( seq_lens=[256, 512, 1024, 2048], query_lens=[256, 512, 1024, 2048] ), } def create_and_prepopulate_kv_cache( k_contexts: list[torch.Tensor], v_contexts: list[torch.Tensor], block_size: int, num_kv_heads: int, head_size: int, dtype: torch.dtype, device: torch.device, num_blocks: int, common_attn_metadata: CommonAttentionMetadata, layout: KVCacheLayout, randomize_blocks: bool = True, kv_cache_dtype: str = "auto", ) -> torch.Tensor: """Create and prepopulate a KV cache with context data. Args: k_contexts: List of key context tensors for each sequence v_contexts: List of value context tensors for each sequence block_size: Size of each block num_kv_heads: Number of KV heads head_size: Size of each head dtype: Data type for the cache device: Device to create the cache on num_blocks: Total number of blocks in the cache common_attn_metadata: Provides seq lens, block table and slot mapping layout: Physical layout to allocate in; the cache is returned as the logical ``[B, H, N, C]`` view (as ``create_kv_cache_views`` does) randomize_blocks: Whether to randomly permute blocks or use sequential order kv_cache_dtype: Cache dtype string; fp8 caches use fp8 storage Returns: A 4D tensor in logical ``(num_blocks, num_kv_heads, block_size, 2 * head_size)`` order with strides determined by ``layout``. """ batch_size = len(k_contexts) seq_lens = common_attn_metadata.seq_lens.cpu() query_lens = ( common_attn_metadata.query_start_loc_cpu[1:] - common_attn_metadata.query_start_loc_cpu[:-1] ) context_lens = seq_lens - query_lens block_table = common_attn_metadata.block_table_tensor slot_mapping = common_attn_metadata.slot_mapping # For an fp8 kv cache, store the cache in the fp8 dtype so that assigning # the higher-precision context tensors quantizes them, mirroring runtime. fp8_kv_cache = is_quantized_kv_cache(kv_cache_dtype) storage_dtype = FP8_KV_CACHE_DTYPES[kv_cache_dtype] if fp8_kv_cache else dtype # Logical 5D shape is always [L, B, H, N, C]. Cross-layer layouts need # at least two layers to reproduce the inter-layer gaps in a layer view. logical_4d = (num_blocks, num_kv_heads, block_size, 2 * head_size) num_layers = 1 if layout.is_layer_compact else 2 logical_5d = (num_layers, *logical_4d) physical_5d = tuple(logical_5d[i] for i in layout.stride_order) inv_order = [layout.stride_order.index(i) for i in range(5)] kv_cache_physical = torch.zeros(physical_5d, dtype=storage_dtype, device=device) # Permute to logical [L, B, H, N, C], then select a layer. This mirrors # create_kv_cache_views and retains cross-layer strides in the 4D view. kv_cache = kv_cache_physical.permute(*inv_order)[0] # Write context tokens into the cache via the logical view: # kv_cache[block, :, token_in_block, :] routes correctly regardless # of physical layout. # Start from block_id=1 since block_id=0 is considered the null block start_block_idx = 1 for i in range(batch_size): k_context, v_context = k_contexts[i], v_contexts[i] t = torch.arange(k_context.shape[0], device=device) blk = start_block_idx + t // block_size off = t % block_size # Advanced indexing on (blk, off) yields [T, H, hs] destinations. # index_put is dtype-strict; cast like the scalar path would. kv_cache[blk, :, off, :head_size] = k_context.to(kv_cache.dtype) kv_cache[blk, :, off, head_size:] = v_context.to(kv_cache.dtype) # Stay block aligned and allocate enough blocks for the new tokens start_block_idx += cdiv(int(seq_lens[i]), block_size) blocks_end = start_block_idx # Permute the context blocks (excluding block 0 which is null) if randomize_blocks: # Random permutation starting from block 1 perm = torch.randperm(blocks_end - 1) + 1 else: # Sequential order starting from block 1 perm = torch.arange(1, blocks_end) inv_perm = torch.zeros(blocks_end, dtype=torch.long, device=device) # Add 1 to account for starting from block 1 inv_perm[1:] = torch.argsort(perm) + 1 kv_cache[1:blocks_end, ...] = kv_cache[perm, ...] # Construct the right block table # Start from block_id=1 since block_id=0 is considered the null block start_block_idx = 1 for i in range(batch_size): num_blocks_for_seq = cdiv(int(seq_lens[i]), block_size) start = start_block_idx end = start + num_blocks_for_seq block_table[i, :num_blocks_for_seq] = inv_perm[start:end] start_block_idx += num_blocks_for_seq # Create a realistic slot mapping that corresponds to the block table for i in range(batch_size): token_offsets = torch.arange(int(query_lens[i])) + int(context_lens[i]) block_indices = token_offsets // block_size token_inter_block_offsets = token_offsets % block_size start = common_attn_metadata.query_start_loc_cpu[i] end = common_attn_metadata.query_start_loc_cpu[i + 1] slot_mapping[start:end] = block_table[ i, block_indices ] * block_size + token_inter_block_offsets.to(device) if fp8_kv_cache: kv_cache = kv_cache.view(torch.uint8) return kv_cache class MockAttentionLayer: """A mock attention layer for testing.""" def __init__( self, device: torch.device, k_scale: float = 1.0, v_scale: float = 1.0, ): self._q_scale = torch.tensor(1.0, device=device) self._k_scale = torch.tensor(k_scale, device=device) self._v_scale = torch.tensor(v_scale, device=device) # Add float versions for flashinfer self._q_scale_float = 1.0 self._k_scale_float = k_scale self._v_scale_float = v_scale def _clone_kv_cache_in_layout( kv_cache: torch.Tensor, layout: KVCacheLayout ) -> torch.Tensor: """Copy a logical [B, H, N, C] cache into a fresh allocation in `layout`.""" logical_5d = (1, *kv_cache.shape) physical = torch.zeros( tuple(logical_5d[i] for i in layout.stride_order), dtype=kv_cache.dtype, device=kv_cache.device, ) inv_order = [layout.stride_order.index(i) for i in range(5)] view = physical.permute(*inv_order)[0] view.copy_(kv_cache) return view def run_attention_backend( backend: AttentionBackendEnum | str, kv_cache_spec: FullAttentionSpec, layer_names: list[str], vllm_config, device: torch.device, common_attn_metadata: CommonAttentionMetadata, query: torch.Tensor, key: torch.Tensor, value: torch.Tensor, kv_cache: torch.Tensor, attn_type: AttentionType = AttentionType.DECODER, sliding_window: int | None = None, kv_cache_dtype: str = "auto", sinks: torch.Tensor | None = None, use_cuda_graph: bool = False, layer_k_scale: float = 1.0, layer_v_scale: float = 1.0, ) -> torch.Tensor: """Run attention computation using the specified backend's AttentionImpl.""" use_direct_block_mask = is_torch_equal_or_newer("2.9.0.dev0") if backend == "FLEX_ATTENTION_SLOW": use_direct_block_mask = False backend = _actual_backend(backend) builder_cls, impl_cls = try_get_attention_backend(backend) # Mock flashinfer's get_per_layer_parameters if needed if backend == AttentionBackendEnum.FLASHINFER: import unittest.mock from vllm.v1.attention.backends.utils import PerLayerParameters def mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls): # Return mock parameters for a single layer head_size = vllm_config.model_config.get_head_size() return { layer_name: PerLayerParameters( window_left=-1, # No sliding window logits_soft_cap=0.0, # No soft cap sm_scale=1.0 / (head_size**0.5), # Standard scale has_sinks=sinks is not None, ) for layer_name in layer_names } with unittest.mock.patch( "vllm.v1.attention.backends.flashinfer.get_per_layer_parameters", mock_get_per_layer_parameters, ): builder = builder_cls(kv_cache_spec, layer_names, vllm_config, device) attn_metadata = builder.build( common_prefix_len=0, common_attn_metadata=common_attn_metadata, ) else: # Build metadata builder = builder_cls(kv_cache_spec, layer_names, vllm_config, device) if backend == AttentionBackendEnum.FLEX_ATTENTION: builder.direct_build = use_direct_block_mask attn_metadata = builder.build( common_prefix_len=0, common_attn_metadata=common_attn_metadata, ) # Instantiate implementation num_heads = vllm_config.model_config.get_num_attention_heads( vllm_config.parallel_config ) num_kv_heads = vllm_config.model_config.get_num_kv_heads( vllm_config.parallel_config ) head_size = vllm_config.model_config.get_head_size() scale = 1.0 / (head_size**0.5) # Impls capture the current vllm config at construction, as in model loading. with set_current_vllm_config(vllm_config): impl = impl_cls( num_heads=num_heads, head_size=head_size, scale=scale, num_kv_heads=num_kv_heads, alibi_slopes=None, sliding_window=sliding_window, attn_type=attn_type, kv_cache_dtype=kv_cache_dtype, **({"sinks": sinks} if sinks is not None else {}), ) # Create mock layer and output buffer mock_layer = MockAttentionLayer(device, layer_k_scale, layer_v_scale) output = torch.empty_like(query) if is_quantized_kv_cache(kv_cache_dtype) and impl.supports_quant_query_input: query = query.to(current_platform.fp8_dtype()) # Run forward pass # NOTE: The query, key, and value are already shaped correctly # in the calling test function. if not try_backend_includes_kv_cache_update(backend): impl.do_kv_cache_update( mock_layer, key, value, kv_cache, attn_metadata.slot_mapping ) if use_cuda_graph: impl.forward( mock_layer, query, key, value, kv_cache, attn_metadata, output=output ) graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): impl.forward( mock_layer, query, key, value, kv_cache, attn_metadata, output=output, ) graph.replay() else: backend_output = impl.forward( mock_layer, query, key, value, kv_cache, attn_metadata, output=output ) if backend_output is not None: output = backend_output return output def _test_backend_correctness( batch_spec: BatchSpec, model: str, backend_to_test: list[AttentionBackendEnum | str], mask_mod, *, causal: bool = True, attn_type: AttentionType = AttentionType.DECODER, block_size: int = 16, atol: float = 1e-2, rtol: float = 1e-2, tensor_parallel_size: int = 1, kv_cache_dtype: str = "auto", use_sinks: bool = False, layout: KVCacheLayout | None = None, use_cuda_graph: bool = False, num_speculative_tokens: int = 0, model_dtype: torch.dtype | None = None, max_num_seqs: int | None = None, max_num_batched_tokens: int | None = None, layer_k_scale: float = 1.0, layer_v_scale: float = 1.0, ): """Test that all backends produce similar outputs to a reference implementation using FlexAttention or an explicit attention-sink reference. This test works by: 1. Generating a batch of sequences with specified context and query lengths. 2. Computing a ground-truth attention output using torch.sdpa on contiguous Q, K, and V tensors. 3. Simulating vLLM's paged KV cache: It takes the context portion of the K/V tensors and manually places them into a paged buffer according to the test's (randomly generated) block table. 4. Running each vLLM attention backend with the new queries and the simulated paged KV cache. 5. Comparing the vLLM backend's output to the ground-truth SDPA output. Note: When tensor_parallel_size > 1, we simulate the head partitioning by overriding the model config to use fewer heads, without requiring multiple GPUs. This tests that backends work correctly with different head counts. """ set_random_seed(42) hf_config_override = None if tensor_parallel_size > 1: from vllm.config import ModelConfig temp_config = ModelConfig(model=model, max_model_len=1) original_num_heads = temp_config.hf_text_config.num_attention_heads original_num_kv_heads = getattr( temp_config.hf_text_config, "num_key_value_heads", None ) hf_config_override = { "num_attention_heads": original_num_heads // tensor_parallel_size, } if original_num_kv_heads is not None: hf_config_override["num_key_value_heads"] = max( 1, original_num_kv_heads // tensor_parallel_size ) vllm_config = create_vllm_config( model_name=model, tensor_parallel_size=1, # Always use TP=1 to avoid multi-GPU requirements max_model_len=max(batch_spec.seq_lens), dtype=model_dtype or "auto", block_size=block_size, num_gpu_blocks=8192, hf_config_override=hf_config_override, ) if max_num_seqs is not None: vllm_config.scheduler_config.max_num_seqs = max_num_seqs if max_num_batched_tokens is not None: vllm_config.scheduler_config.max_num_batched_tokens = max_num_batched_tokens if num_speculative_tokens > 0: vllm_config.speculative_config = SimpleNamespace( num_speculative_tokens=num_speculative_tokens ) vllm_config.cache_config.cache_dtype = kv_cache_dtype device = torch.device(f"{DEVICE_TYPE}:0") kv_cache_spec = create_standard_kv_cache_spec(vllm_config, attn_type) # 1. Setup batch_size = batch_spec.batch_size seq_lens = batch_spec.seq_lens query_lens = batch_spec.query_lens num_q_heads = vllm_config.model_config.get_num_attention_heads( vllm_config.parallel_config ) num_kv_heads = vllm_config.model_config.get_num_kv_heads( vllm_config.parallel_config ) sinks = ( torch.linspace(-1.0, 1.0, num_q_heads, dtype=torch.float32, device=device) if use_sinks else None ) head_size = vllm_config.model_config.get_head_size() sliding_window = vllm_config.model_config.get_sliding_window() dtype = _convert_dtype_to_torch(vllm_config.model_config.dtype) block_size = vllm_config.cache_config.block_size scale = 1.0 / (head_size**0.5) fp8_kv_cache = is_quantized_kv_cache(kv_cache_dtype) if fp8_kv_cache: query_fp8_dtype = current_platform.fp8_dtype() kv_fp8_dtype = FP8_KV_CACHE_DTYPES[kv_cache_dtype] atol = max(atol, 6e-2) rtol = max(rtol, 1e-1) # 2. Generate data and compute SDPA reference output all_q_vllm, all_k_vllm, all_v_vllm = [], [], [] all_sdpa_outputs = [] k_contexts, v_contexts = [], [] for i in range(batch_size): s_len = seq_lens[i] q_len = query_lens[i] context_len = s_len - q_len # Generate Q, K, V for the whole sequence to be used in SDPA q = torch.randn(q_len, num_q_heads, head_size, dtype=dtype, device=device) k_full = torch.randn(s_len, num_kv_heads, head_size, dtype=dtype, device=device) v_full = torch.randn(s_len, num_kv_heads, head_size, dtype=dtype, device=device) if fp8_kv_cache: q_ref = q.to(query_fp8_dtype).to(dtype) k_ref = k_full.to(kv_fp8_dtype).to(dtype) v_ref = v_full.to(kv_fp8_dtype).to(dtype) else: q_ref, k_ref, v_ref = q, k_full, v_full # SDPA expects (N, H, L, D), so unsqueeze batch and permute q_sdpa_in = q_ref.unsqueeze(0).transpose(1, 2) k_sdpa_in = k_ref.unsqueeze(0).transpose(1, 2) v_sdpa_in = v_ref.unsqueeze(0).transpose(1, 2) if num_q_heads != num_kv_heads: assert num_q_heads % num_kv_heads == 0, ( f"num_q_heads ({num_q_heads}) must be divisible by " f"num_kv_heads ({num_kv_heads})" ) repeats = num_q_heads // num_kv_heads k_sdpa_in = k_sdpa_in.repeat_interleave(repeats, dim=1) v_sdpa_in = v_sdpa_in.repeat_interleave(repeats, dim=1) # Create causal mask: query token i attends to positions 0 to # (context_len + i) kv_len = s_len final_mask_mod = partial(mask_mod, context_len=context_len) if sinks is None: block_mask = create_block_mask( final_mask_mod, B=None, H=None, Q_LEN=q_len, KV_LEN=kv_len, device=device, ) sdpa_out_i = flex_attention( q_sdpa_in, k_sdpa_in, v_sdpa_in, block_mask=block_mask, scale=scale, enable_gqa=True, ) else: scores = ( torch.matmul(q_sdpa_in.float(), k_sdpa_in.float().transpose(-2, -1)) * scale ) q_idx = torch.arange(q_len, device=device).unsqueeze(1) kv_idx = torch.arange(kv_len, device=device).unsqueeze(0) zero = torch.zeros((), dtype=torch.int32, device=device) valid = final_mask_mod(zero, zero, q_idx, kv_idx) scores.masked_fill_(~valid.unsqueeze(0).unsqueeze(0), -torch.inf) sink_logits = sinks.view(1, num_q_heads, 1, 1).expand(1, -1, q_len, -1) weights = torch.softmax(torch.cat((scores, sink_logits), dim=-1), dim=-1)[ ..., :kv_len ] sdpa_out_i = torch.matmul(weights, v_sdpa_in.float()).to(dtype) all_sdpa_outputs.append(sdpa_out_i.transpose(1, 2).squeeze(0)) # Inputs for vLLM backends are just the new tokens all_q_vllm.append(q) all_k_vllm.append(k_full[context_len:]) all_v_vllm.append(v_full[context_len:]) # Contextual K/V data used to populate the paged cache k_contexts.append(k_full[:context_len]) v_contexts.append(v_full[:context_len]) query_vllm = torch.cat(all_q_vllm, dim=0) key_vllm = torch.cat(all_k_vllm, dim=0) value_vllm = torch.cat(all_v_vllm, dim=0) sdpa_output = torch.cat(all_sdpa_outputs, dim=0) common_attn_metadata = create_common_attn_metadata( batch_spec, vllm_config.cache_config.block_size, device ) common_attn_metadata.causal = causal # 3. Simulate Paged KV Cache and a realistic slot_mapping # Mirror selector-time resolution locally (caller-requested layout, then # any backend-required layout, then the default chain). declared = [ supported for backend in backend_to_test if ( supported := _actual_backend(backend) .get_class() .supported_kv_cache_layouts() ) is not None ] if layout is None: # Mirror the resolver: the shared cache uses the declared sets' preferred # layout; backends that don't support it get a per-backend copy below. layout = min(declared, key=len)[0] if declared else KVCacheLayout.LBNHC kv_cache = create_and_prepopulate_kv_cache( k_contexts=k_contexts, v_contexts=v_contexts, block_size=block_size, num_kv_heads=num_kv_heads, head_size=head_size, dtype=dtype, device=device, num_blocks=vllm_config.cache_config.num_gpu_blocks or 1000, common_attn_metadata=common_attn_metadata, layout=layout, randomize_blocks=True, kv_cache_dtype=kv_cache_dtype, ) # 4. Run vLLM backends and compare # Note: flex_attention has known Triton kernel compatibility issues # with test infrastructures for backend_name in backend_to_test: backend_cls = _actual_backend(backend_name).get_class() if is_quantized_kv_cache(kv_cache_dtype) and ( not backend_cls.supports_kv_cache_dtype(kv_cache_dtype) ): continue kv_cache_for_backend = kv_cache backend_layout = layout backend_supported = backend_cls.supported_kv_cache_layouts() if backend_supported is not None and backend_layout not in backend_supported: # Production resolution never pairs this backend with the shared layout # (e.g. flex is LBNHC-only next to FlashInfer's LBHNC); give it a copy # of the cache in a layout it supports. backend_layout = backend_supported[0] kv_cache_for_backend = _clone_kv_cache_in_layout(kv_cache, backend_layout) if backend_name == AttentionBackendEnum.B12X: cache_dtype = ( FP8_KV_CACHE_DTYPES[kv_cache_dtype] if is_quantized_kv_cache(kv_cache_dtype) else kv_cache.dtype ) typed_cache = kv_cache.view(cache_dtype) key_cache = typed_cache[..., :head_size].permute(0, 2, 1, 3) value_cache = typed_cache[..., head_size:].permute(0, 2, 1, 3) packed_cache = torch.stack((key_cache, value_cache), dim=1).flatten(-2) if is_quantized_kv_cache(kv_cache_dtype): packed_cache = packed_cache.view(torch.uint8) kv_cache_for_backend = _clone_kv_cache_in_layout( packed_cache, backend_layout ) # FlashInfer reads the layout at plan time; set it to match # the physical order of the test cache. vllm_config.cache_config.kv_cache_layout = backend_layout.name backend_output = run_attention_backend( backend_name, kv_cache_spec, ["placeholder"], vllm_config, device, common_attn_metadata, query_vllm, key_vllm, value_vllm, kv_cache_for_backend, sliding_window=sliding_window, attn_type=attn_type, kv_cache_dtype=kv_cache_dtype, sinks=sinks, use_cuda_graph=use_cuda_graph, layer_k_scale=layer_k_scale, layer_v_scale=layer_v_scale, ) # Check shape and dtype consistency assert backend_output.shape == sdpa_output.shape, ( f"[{backend_name}] shape {backend_output.shape} != " f"SDPA shape {sdpa_output.shape}" ) assert backend_output.dtype == sdpa_output.dtype, ( f"[{backend_name}] dtype {backend_output.dtype} != " f"SDPA dtype {sdpa_output.dtype}" ) assert torch.isfinite(backend_output).all(), ( f"[{backend_name}] produced non-finite values" ) # Check numerical similarity def error_msg(msg: str, backend_name: AttentionBackendEnum | str): return f"[{backend_name}] output differs from SDPA baseline. {msg}" torch.testing.assert_close( backend_output, sdpa_output, rtol=rtol, atol=atol, msg=partial(error_msg, backend_name=backend_name), ) @pytest.mark.parametrize("layout", ["BLHNC", "BHLNC"]) @pytest.mark.parametrize("batch_spec_name", ["small_decode", "small_prefill"]) @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) def test_flashinfer_cross_layer_layout( default_vllm_config, layout: str, batch_spec_name: str, kv_cache_dtype: str, ): if AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST: pytest.skip("FlashInfer is not installed") def causal_mask_mod( b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor, *, context_len: int, ): return (q_idx + context_len) >= kv_idx _test_backend_correctness( batch_spec=BATCH_SPECS[batch_spec_name], model="Qwen/Qwen3-0.6B", backend_to_test=[AttentionBackendEnum.FLASHINFER], mask_mod=causal_mask_mod, kv_cache_dtype=kv_cache_dtype, layout=KVCacheLayout[layout], ) @pytest.mark.parametrize( "batch_spec_name", [ "small_decode", "small_prefill", "mixed_small", "medium_decode", "medium_prefill", "mixed_medium", "large_decode", "large_prefill", "single_decode", "single_prefill", ], ) @pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"]) @pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4]) @pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8", "fp8_e4m3"]) def test_causal_backend_correctness( default_vllm_config, batch_spec_name: str, model: str, tensor_parallel_size: int, kv_cache_dtype: str, ): """Test backend's correctness with causal attention.""" def causal_mask_mod( b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor, *, context_len: int, ): return (q_idx + context_len) >= kv_idx batch_spec = BATCH_SPECS[batch_spec_name] LARGE_BLOCK_BACKENDS = ( [AttentionBackendEnum.FLEX_ATTENTION] if is_torch_equal_or_newer("2.9.0.dev0") else [] ) if current_platform.is_rocm(): SMALL_BLOCK_BACKENDS = [ x for x in BACKENDS_TO_TEST if ( x not in LARGE_BLOCK_BACKENDS and x is not AttentionBackendEnum.FLASH_ATTN ) ] else: SMALL_BLOCK_BACKENDS = [ x for x in BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS ] _test_backend_correctness( batch_spec, model, SMALL_BLOCK_BACKENDS, causal_mask_mod, tensor_parallel_size=tensor_parallel_size, kv_cache_dtype=kv_cache_dtype, ) # Fast FlexAttention needs to run with block_size=128 if LARGE_BLOCK_BACKENDS: _test_backend_correctness( batch_spec, model, LARGE_BLOCK_BACKENDS, causal_mask_mod, block_size=128, tensor_parallel_size=tensor_parallel_size, kv_cache_dtype=kv_cache_dtype, ) @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", ) def test_flashinfer_xqa_bmm1_scale_matches_decode_q_dtype(): """XQA decode should only apply q_scale when decode Q is FP8.""" from vllm.v1.attention.backends import flashinfer as flashinfer_backend class MockLayer: _q_scale_float = 2.0 _k_scale_float = 3.0 impl = object.__new__(flashinfer_backend.FlashInferImpl) impl.scale = 0.5 impl.kv_cache_dtype = "fp8" assert impl.get_xqa_bmm1_scale(MockLayer, torch.bfloat16) == 1.5 assert impl.get_xqa_bmm1_scale(MockLayer, torch.float8_e4m3fn) == 3.0 @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", ) def test_flashinfer_xqa_draft_masks(): from vllm.v1.attention.backends import flashinfer as flashinfer_backend device = torch.device("cpu") causal = flashinfer_backend._make_xqa_draft_block_mask(3, True, device) full = flashinfer_backend._make_xqa_draft_block_mask(3, False, device) ragged = flashinfer_backend._make_xqa_ragged_draft_block_mask( [2, 3], 3, True, device ) assert torch.equal( causal.view(torch.int16), torch.tensor([[1, 0], [3, 0], [7, 0]], dtype=torch.int16), ) assert torch.equal( full.view(torch.int16), torch.tensor([[7, 0]] * 3, dtype=torch.int16) ) assert torch.equal( ragged.view(torch.int16), torch.tensor( [[1, 0], [3, 0], [1, 0], [3, 0], [7, 0]], dtype=torch.int16, ), ) @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", ) def test_flashinfer_xqa_query_lens_preserve_cudagraph_padding(): """CUDA-graph padding stays as zero-length requests in ragged offsets.""" from vllm.v1.attention.backends import flashinfer as flashinfer_backend device = torch.device("cpu") builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder) builder.use_xqa = True qo_indptr = torch.tensor([0, 3, 9, 15, 15], dtype=torch.int32, device=device) q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens( qo_indptr, qo_indptr, num_decodes=4, num_decode_tokens=21, ) assert q_len == 6 assert q_lens == [3, 6, 6, 0] assert q_cu_seq_lens is not None assert q_cu_seq_lens.tolist() == [0, 3, 9, 15, 15] @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", ) def test_flashinfer_xqa_single_token_decode_preserves_cudagraph_padding(monkeypatch): """Non-ragged XQA keeps one query/output row per padded request.""" import unittest.mock from vllm.v1.attention.backends import flashinfer as flashinfer_backend impl = object.__new__(flashinfer_backend.FlashInferImpl) impl.scale = 1.0 impl.bmm1_scale = None impl.bmm2_scale = None impl.kv_cache_dtype = "auto" impl.is_kvcache_nvfp4 = False impl.head_size = 16 impl.dcp_world_size = 1 impl.o_sf_scale = None impl.window_left = -1 impl.sinks = None layer = unittest.mock.Mock( _q_scale=torch.tensor(1.0), _q_scale_float=1.0, _k_scale_float=1.0, _v_scale_float=1.0, ) decode = flashinfer_backend.FlashInferTrtllmAPIDecode( kernel=flashinfer_backend.FlashInferDecodeKernel.XQA, block_tables=torch.zeros((4, 1), dtype=torch.int32), seq_lens=torch.tensor([8, 8, 0, 0], dtype=torch.int32), max_seq_len=8, q_len_per_req=1, ) attn_metadata = flashinfer_backend.FlashInferMetadata( kv_cache_layout=KVCacheLayout.LBHNC, num_actual_tokens=2, slot_mapping=torch.empty(0, dtype=torch.int64), q_data_type_prefill=torch.bfloat16, q_data_type_decode=torch.bfloat16, num_decodes=4, num_decode_tokens=2, num_prefills=0, num_prefill_tokens=0, causal=True, prefill=None, decode=decode, ) seen_shapes = {} def mock_xqa(**kwargs): seen_shapes["query"] = kwargs["query"].shape seen_shapes["out"] = kwargs["out"].shape seen_shapes["block_tables"] = kwargs["block_tables"].shape monkeypatch.setattr( flashinfer_backend, "_get_trtllm_workspace_buffer", lambda: torch.empty(1, dtype=torch.uint8), ) monkeypatch.setattr( flashinfer_backend, "flashinfer_xqa_batch_decode_with_kv_cache", mock_xqa ) query = torch.zeros((4, 1, 16), dtype=torch.bfloat16) output = torch.empty_like(query) result = impl.forward( layer, query, torch.empty_like(query), torch.empty_like(query), torch.zeros((1, 1, 1, 32), dtype=torch.bfloat16), attn_metadata, output, ) assert result is output assert seen_shapes == { "query": torch.Size([4, 1, 16]), "out": torch.Size([4, 1, 16]), "block_tables": torch.Size([4, 1]), } @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", ) def test_flashinfer_xqa_query_lens_require_exact_uniform_product(): from vllm.v1.attention.backends import flashinfer as flashinfer_backend builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder) builder.use_xqa = True qo_indptr = torch.tensor([0, 3, 3, 3], dtype=torch.int32) q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens( qo_indptr, qo_indptr, num_decodes=3, num_decode_tokens=6, ) assert q_len == 3 assert q_lens == [3, 0, 0] assert q_cu_seq_lens is not None assert q_cu_seq_lens.tolist() == [0, 3, 3, 3] @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", ) @pytest.mark.parametrize( ("qo_indptr_values", "num_decode_tokens", "expected_q_len"), [ ([0, 3, 3], 3, 3), ([0, 2, 4, 4, 4], 4, 2), ], ) def test_flashinfer_trtllm_gen_padded_decode_uses_varlen_offsets( qo_indptr_values: list[int], num_decode_tokens: int, expected_q_len: int, ): """Padded speculative decode keeps its actual packed query width.""" from vllm.v1.attention.backends import flashinfer as flashinfer_backend builder = object.__new__(flashinfer_backend.FlashInferMetadataBuilder) builder.use_xqa = False qo_indptr = torch.tensor(qo_indptr_values, dtype=torch.int32) q_len, q_cu_seq_lens, q_lens = builder._compute_decode_query_lens( qo_indptr, qo_indptr, num_decodes=len(qo_indptr_values) - 1, num_decode_tokens=num_decode_tokens, ) assert q_len == expected_q_len assert q_lens is None assert q_cu_seq_lens is not None assert q_cu_seq_lens.tolist() == qo_indptr_values @pytest.mark.skipif( not current_platform.is_cuda() or AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer with CUDA is required.", ) @pytest.mark.parametrize( "runner_type,spec_kind,has_upper_bound,expected_copies", [ ("generate", None, True, 0), ("pooling", None, True, 0), ("generate", None, False, 1), ("generate", "spec_decode", True, 1), ("generate", "diffusion", True, 1), ], ) def test_flashinfer_avoids_seq_lens_copy_without_spec_tokens( monkeypatch, runner_type, spec_kind, has_upper_bound, expected_copies ): """Planning reuses the host seq_lens bound only when it is exact. Without speculative token accounting (no spec-decode drafts, no dLLM canvas tokens) scheduled tokens always land in KV, so the CPU upper bound is exact regardless of runner type. Any speculative accounting keeps the bound optimistic (rejected tokens are in-flight) and forces the copy. """ import unittest.mock from vllm.v1.attention.backends import flashinfer as flashinfer_backend from vllm.v1.attention.backends.utils import PerLayerParameters monkeypatch.setattr( flashinfer_backend, "get_per_layer_parameters", lambda *args: { "layer.0": PerLayerParameters( window_left=-1, logits_soft_cap=0.0, sm_scale=0.125 ) }, ) config = create_vllm_config( model_name="Qwen/Qwen3-0.6B", max_model_len=64, max_num_seqs=3 ) config.model_config.runner_type = runner_type config.attention_config.use_trtllm_attention = False if spec_kind == "spec_decode": config.speculative_config = SpeculativeConfig( method="ngram", num_speculative_tokens=3 ) elif spec_kind == "diffusion": config.diffusion_config = DiffusionConfig(canvas_length=8) device = torch.device(f"{DEVICE_TYPE}:0") # Include a one-token final chunk and both partial and full KV pages. common_attn_metadata = create_common_attn_metadata( BatchSpec(seq_lens=[17, 20, 32], query_lens=[1, 4, 16]), 16, device ) if not has_upper_bound: common_attn_metadata.seq_lens_cpu_upper_bound = None elif spec_kind is not None: # Simulate rejected speculative tokens: CPU says 18, GPU says 17. assert common_attn_metadata.seq_lens_cpu_upper_bound is not None common_attn_metadata.seq_lens_cpu_upper_bound[0] += 1 with set_current_vllm_config(config): builder = flashinfer_backend.FlashInferMetadataBuilder( create_standard_kv_cache_spec(config), ["layer.0"], config, device ) with unittest.mock.patch.object( common_attn_metadata.seq_lens, "cpu", wraps=common_attn_metadata.seq_lens.cpu, ) as copy_to_cpu: metadata = builder.build(0, common_attn_metadata) assert copy_to_cpu.call_count == expected_copies assert metadata.num_decodes == 1 assert metadata.num_prefills == 2 assert builder.paged_kv_indptr.cpu[:4].tolist() == [0, 2, 4, 6] assert builder.paged_kv_last_page_len.cpu[:3].tolist() == [1, 4, 16] @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", ) @pytest.mark.parametrize( "adaptive,decode_kernel,dcp_size,expected_bound", [ (None, "TRTLLM_GEN", 1, None), (False, "TRTLLM_GEN", 1, None), (True, "TRTLLM_GEN", 1, 8), (True, "XQA", 1, None), (True, "TRTLLM_GEN", 2, None), ], ) def test_flashinfer_varlen_cudagraph_capability( monkeypatch, adaptive, decode_kernel, dcp_size, expected_bound ): """Only the trtllm-gen path that adaptive verification enables replays varlen decode graphs, up to the decode width; the uniform level is fixed.""" from vllm.model_executor.layers.attention.chunked_local_attention import ( create_chunked_local_attention_backend, ) from vllm.v1.attention.backends import flashinfer as fi config = SimpleNamespace( attention_config=SimpleNamespace(use_non_causal=False), speculative_config=SimpleNamespace( enable_adaptive_verification=adaptive, num_speculative_tokens=7, parallel_drafting=True, ) if adaptive is not None else None, parallel_config=SimpleNamespace(decode_context_parallel_size=dcp_size), model_config=SimpleNamespace(get_num_attention_heads=lambda _: 32), use_v2_model_runner=True, ) monkeypatch.setattr(fi, "can_use_trtllm_attention", lambda *_, **__: True) monkeypatch.setattr( fi.FlashInferMetadataBuilder, "_get_flashinfer_trtllm_api_decode_kernel", staticmethod(lambda: fi.FlashInferDecodeKernel[decode_kernel]), ) spec = FullAttentionSpec( block_size=16, num_kv_heads=4, head_size=64, dtype=torch.bfloat16, ) builder_cls = fi.FlashInferMetadataBuilder assert builder_cls.get_cudagraph_support(config, spec) == ( AttentionCGSupport.UNIFORM_BATCH if dcp_size == 1 else AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE ) bound = builder_cls.get_varlen_cudagraph_max_query_len(config, spec) assert bound == expected_bound if bound is not None: # A wrapper forcing NEVER inherits the override but not the bound. chunked_local = create_chunked_local_attention_backend(fi.FlashInferBackend, 16) wrapped_cls = chunked_local.get_builder_cls() assert wrapped_cls.get_varlen_cudagraph_max_query_len(config, spec) is None @pytest.mark.parametrize( "use_v2_model_runner,parallel_drafting,expected", [ (True, None, 1), (True, False, 8), (True, True, 8), (False, False, 8), (False, True, 15), ], ) def test_spec_as_decode_width_follows_model_runner( use_v2_model_runner, parallel_drafting, expected ): """Only model runner V1's parallel drafter widens decode requests.""" class Builder(AttentionMetadataBuilder): def __init__(self, vllm_config): self.vllm_config = vllm_config def build(self, common_prefix_len, common_attn_metadata, fast_build=False): raise NotImplementedError config = SimpleNamespace( speculative_config=None if parallel_drafting is None else SimpleNamespace( num_speculative_tokens=7, parallel_drafting=parallel_drafting ), parallel_config=SimpleNamespace(decode_context_parallel_size=1), use_v2_model_runner=use_v2_model_runner, ) assert max_decode_query_len(config) == expected builder = Builder(config) builder._init_reorder_batch_threshold(1, supports_spec_as_decode=True) assert builder.reorder_batch_threshold == expected @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST or not current_platform.is_device_capability_family(100), reason="TRTLLM-GEN varlen decode graphs require FlashInfer on SM100-family GPUs", ) @pytest.mark.parametrize( "head_size,sliding_window,num_tokens,num_reqs,num_heads,num_kv_heads", [ pytest.param(64, None, 8, 8, 32, 4, id="single-token-capture"), pytest.param( 256, 1024, 40, 32, 16, 8, id="gemma4-sliding-gqa", ), pytest.param( 512, None, 40, 32, 16, 1, id="gemma4-global-mqa", ), ], ) def test_flashinfer_varlen_decode_graph_replays_changed_layout( tmp_path, monkeypatch, head_size, sliding_window, num_tokens, num_reqs, num_heads, num_kv_heads, ): """Replay changes device offsets, active slots, context, and KV writes. The graph is captured with evenly spread dummy queries and an explicit bound of eight. CPU offsets then remain stale while device offsets change. Compare real output rows with independent FP32 causal/windowed attention, and verify cache writes including preservation of the null padding block. """ from vllm.v1.attention.backends import flashinfer as fi from vllm.v1.attention.backends.utils import PerLayerParameters set_random_seed(0) monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") block_size = 16 dtype = torch.bfloat16 LlamaConfig( architectures=["LlamaForCausalLM"], hidden_size=num_heads * head_size, head_dim=head_size, num_attention_heads=num_heads, num_key_value_heads=num_kv_heads, num_hidden_layers=1, max_position_embeddings=4096, ).save_pretrained(tmp_path) num_blocks = 1 + num_reqs * cdiv(2112, block_size) config = create_vllm_config( model_name=str(tmp_path), max_model_len=4096, dtype=dtype, num_gpu_blocks=num_blocks, block_size=block_size, max_num_seqs=32, max_num_batched_tokens=256, ) config.cache_config.kv_cache_layout = "BLHNC" config.speculative_config = SimpleNamespace( num_speculative_tokens=7, enable_adaptive_verification=True, parallel_drafting=False, ) spec = create_standard_kv_cache_spec(config) scale = head_size**-0.5 monkeypatch.setattr( fi, "get_per_layer_parameters", lambda *_: { "layer": PerLayerParameters( window_left=-1 if sliding_window is None else sliding_window - 1, logits_soft_cap=0.0, sm_scale=scale, has_sinks=False, ), }, ) device = torch.device("cuda") layout = KVCacheLayout.BLHNC query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype, device=device) key = torch.empty(num_tokens, num_kv_heads, head_size, dtype=dtype, device=device) value = torch.empty_like(key) output = torch.empty_like(query) dummy_lens = [ num_tokens // num_reqs + (i < num_tokens % num_reqs) for i in range(num_reqs) ] common = create_common_attn_metadata( BatchSpec(seq_lens=[2112] * num_reqs, query_lens=dummy_lens), block_size, device, ) common.max_query_len = 8 cache = None def prepare(query_lens): nonlocal cache query_lens = query_lens + [0] * (num_reqs - len(query_lens)) query.normal_() key.normal_() value.normal_() # Alternating short prefills, nonempty decode contexts, and contexts # beyond the sliding window; inactive requests have zero sequence length. contexts = [ ([0, 17, 2079, 39][i % 4] if q else 0) for i, q in enumerate(query_lens) ] ks = [ torch.randn(c, num_kv_heads, head_size, dtype=dtype, device=device) for c in contexts ] vs = [torch.randn_like(k) for k in ks] actual = create_common_attn_metadata( BatchSpec( seq_lens=[c + q for c, q in zip(contexts, query_lens)], query_lens=query_lens, ), block_size, device, ) fresh_cache = create_and_prepopulate_kv_cache( ks, vs, block_size, num_kv_heads, head_size, dtype, device, num_blocks, actual, layout, randomize_blocks=True, ) if cache is None: cache = fresh_cache else: cache.copy_(fresh_cache) common.query_start_loc.copy_(actual.query_start_loc) common.seq_lens.copy_(actual.seq_lens) common.block_table_tensor.zero_() common.block_table_tensor[:, : actual.block_table_tensor.shape[1]].copy_( actual.block_table_tensor ) common.slot_mapping.fill_(-1) common.slot_mapping[: sum(query_lens)].copy_(actual.slot_mapping) return contexts, ks, vs prepare(dummy_lens) with set_current_vllm_config(config): builder = fi.FlashInferMetadataBuilder(spec, ["layer"], config, device) metadata = builder.build(0, common) assert metadata.num_prefills == 0 assert metadata.decode.q_len_per_req == 8 impl = fi.FlashInferImpl( num_heads=num_heads, num_kv_heads=num_kv_heads, head_size=head_size, scale=scale, alibi_slopes=None, sliding_window=sliding_window, kv_cache_dtype="auto", attn_type=AttentionType.DECODER, ) layer = MockAttentionLayer(device) def forward(attn_metadata=metadata): impl.do_kv_cache_update(layer, key, value, cache, common.slot_mapping) impl.forward(layer, query, key, value, cache, attn_metadata, output=output) for _ in range(3): forward() torch.accelerator.synchronize() graph = torch.cuda.CUDAGraph() with torch.cuda.graph(graph): forward() layouts = [[8], [1] * num_reqs] if num_tokens >= 16: layouts.append([8, 4, 3, 1]) if num_tokens >= 40: layouts.append([8] * 5) def check(query_lens, run): contexts, ks, vs = prepare(query_lens) assert cache is not None output.fill_(float("nan")) run() start = 0 for i, q_len in enumerate(query_lens): if q_len == 0: continue end = start + q_len full_k = ( torch.cat((ks[i], key[start:end])) .float() .repeat_interleave(num_heads // num_kv_heads, dim=1) ) full_v = ( torch.cat((vs[i], value[start:end])) .float() .repeat_interleave(num_heads // num_kv_heads, dim=1) ) scores = ( torch.einsum("qhd,khd->hqk", query[start:end].float(), full_k) * scale ) q_pos = torch.arange(q_len, device=device) + contexts[i] k_pos = torch.arange(full_k.shape[0], device=device) allowed = k_pos[None, :] <= q_pos[:, None] if sliding_window is not None: allowed &= k_pos[None, :] > q_pos[:, None] - sliding_window probs = scores.masked_fill(~allowed[None], float("-inf")).softmax(-1) reference = torch.einsum("hqk,khd->qhd", probs, full_v) torch.testing.assert_close( output[start:end].float(), reference, atol=2e-2, rtol=2e-2, ) slots = common.slot_mapping[start:end] written = cache[slots // block_size, :, slots % block_size, :] torch.testing.assert_close(written[..., :head_size], key[start:end]) torch.testing.assert_close(written[..., head_size:], value[start:end]) start = end assert torch.count_nonzero(cache[0]) == 0 for query_lens in layouts: check(query_lens, graph.replay) if num_tokens == 40: # A longer prefill forces a separate launch in eager/piecewise mode. # Its length must not become the decode kernel's query bound, and # compacted CPU placeholders still need not match the GPU allocation. common.num_reqs = 5 common.max_query_len = 24 common.query_start_loc_cpu[:6] = torch.tensor([0, 4, 8, 12, 16, 40]) common.query_start_loc_cpu[6:] = 40 def mixed_forward(): with set_current_vllm_config(config): metadata = builder.build( 0, replace( common, query_start_loc=common.query_start_loc[:6], query_start_loc_cpu=common.query_start_loc_cpu[:6], seq_lens=common.seq_lens[:5], seq_lens_cpu_upper_bound=common.seq_lens_cpu_upper_bound[:5], block_table_tensor=common.block_table_tensor[:5], ), ) assert metadata.num_decodes == 4 assert metadata.num_prefills == 1 assert metadata.decode.q_len_per_req == 8 forward(metadata) check([8, 4, 3, 1, 24], mixed_forward) @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", ) @pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16]) def test_flashinfer_attention_sinks_refreshed_after_reload(dtype): from vllm.v1.attention.backends import flashinfer as flashinfer_backend source_sinks = torch.tensor([1.0, 2.0], dtype=dtype) impl = object.__new__(flashinfer_backend.FlashInferImpl) impl._sinks_source = source_sinks impl.sinks = source_sinks impl.process_weights_after_loading(dtype) assert impl.sinks is not None sinks_ptr = impl.sinks.data_ptr() assert impl.sinks.dtype == torch.float32 torch.testing.assert_close(impl.sinks, source_sinks.float()) source_sinks.copy_(torch.tensor([3.0, 4.0], dtype=dtype)) impl.process_weights_after_loading(dtype) assert impl.sinks.data_ptr() == sinks_ptr torch.testing.assert_close(impl.sinks, source_sinks.float()) @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", ) def test_flashinfer_native_prefill_with_sinks(default_vllm_config): supported = current_platform.is_cuda() and ( current_platform.is_device_capability(90) or current_platform.is_device_capability_family(120) ) if not supported: pytest.skip("Native FlashInfer prefill with sinks requires SM90 or SM12x.") from vllm.v1.attention.backends.flashinfer import FlashInferBackend if not FlashInferBackend.supports_sink(): pytest.skip("FlashInfer attention sinks are not available in this setup.") def causal_mask_mod( b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor, *, context_len: int, ): return (q_idx + context_len) >= kv_idx _test_backend_correctness( BATCH_SPECS["small_prefill"], "Qwen/Qwen3-0.6B", [AttentionBackendEnum.FLASHINFER], causal_mask_mod, use_sinks=True, ) @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST, reason="FlashInfer is not available.", ) def test_flashinfer_xqa_decode_correctness(default_vllm_config): """FlashInfer should route SM90/SM12x decode through XQA and match SDPA.""" supported = current_platform.is_cuda() and ( current_platform.is_device_capability(90) or current_platform.is_device_capability_family(120) ) if not supported: pytest.skip("FlashInfer XQA decode requires SM90 or SM12x.") import unittest.mock from vllm.utils.flashinfer import can_use_trtllm_attention from vllm.v1.attention.backends import flashinfer as flashinfer_backend from vllm.v1.attention.backends.utils import PerLayerParameters def mock_get_per_layer_parameters(vllm_config, layer_names, impl_cls): return { "placeholder": PerLayerParameters( window_left=-1, logits_soft_cap=0.0, sm_scale=1.0, ) } def causal_mask_mod( b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor, *, context_len: int, ): return (q_idx + context_len) >= kv_idx batch_spec = BATCH_SPECS["small_decode"] vllm_config = create_vllm_config( model_name="Qwen/Qwen3-0.6B", max_model_len=max(batch_spec.seq_lens), block_size=16, ) device = torch.device(f"{DEVICE_TYPE}:0") kv_cache_spec = FullAttentionSpec( block_size=vllm_config.cache_config.block_size, num_kv_heads=vllm_config.model_config.get_num_kv_heads( vllm_config.parallel_config ), head_size=vllm_config.model_config.get_head_size(), dtype=vllm_config.model_config.dtype, ) with set_current_vllm_config(vllm_config): if not can_use_trtllm_attention( vllm_config.model_config.get_num_attention_heads( vllm_config.parallel_config ), kv_cache_spec.num_kv_heads, is_prefill=False, ): pytest.skip("FlashInfer XQA decode is not available in this setup.") with unittest.mock.patch( "vllm.v1.attention.backends.flashinfer.get_per_layer_parameters", mock_get_per_layer_parameters, ): builder = flashinfer_backend.FlashInferMetadataBuilder( kv_cache_spec, ["placeholder"], vllm_config, device ) common_attn_metadata = create_common_attn_metadata( batch_spec, vllm_config.cache_config.block_size, device ) attn_metadata = builder.build(0, common_attn_metadata) assert ( flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support( vllm_config, kv_cache_spec ) == AttentionCGSupport.UNIFORM_BATCH ) wide_head_kv_cache_spec = FullAttentionSpec( block_size=vllm_config.cache_config.block_size, num_kv_heads=kv_cache_spec.num_kv_heads, head_size=512, dtype=vllm_config.model_config.dtype, ) assert ( flashinfer_backend.FlashInferMetadataBuilder.get_cudagraph_support( vllm_config, wide_head_kv_cache_spec ) == AttentionCGSupport.UNIFORM_SINGLE_TOKEN_DECODE ) assert isinstance( attn_metadata.decode, flashinfer_backend.FlashInferTrtllmAPIDecode, ) assert attn_metadata.decode.kernel == flashinfer_backend.FlashInferDecodeKernel.XQA _test_backend_correctness( batch_spec, "Qwen/Qwen3-0.6B", [AttentionBackendEnum.FLASHINFER], causal_mask_mod, ) @pytest.mark.skipif( AttentionBackendEnum.FLASHINFER not in BACKENDS_TO_TEST or not current_platform.is_device_capability_family(120), reason="NVFP4 XQA decode requires SM12x.", ) @pytest.mark.parametrize("use_cuda_graph", [False, True]) @pytest.mark.parametrize("q_len", [1, 4]) def test_flashinfer_xqa_nvfp4_decode_correctness( default_vllm_config, q_len, use_cuda_graph ): """Single-token and spec-decode queries over an NVFP4 cache decode with XQA, eager and replayed, and match the fa2 prefill that wrote the cache.""" import unittest.mock from vllm.v1.attention.backends import flashinfer as fi seq_lens, bs, d = [40, 300, 1027], 16, 256 config = create_vllm_config( model_name="Qwen/Qwen3-0.6B", max_model_len=max(seq_lens), block_size=bs, ) # GQA 6 at head_dim 256, as Qwen3.8-27B at TP=2. config.model_config.model_arch_config = replace( config.model_config.model_arch_config, total_num_attention_heads=24, total_num_kv_heads=4, head_size=d, ) config.cache_config.cache_dtype = "nvfp4" if q_len > 1: config.speculative_config = SpeculativeConfig( method="ngram", num_speculative_tokens=q_len - 1 ) spec = FullAttentionSpec( block_size=bs, num_kv_heads=4, head_size=d, dtype=torch.uint8, kv_quant_mode=KVQuantMode.NVFP4, ) cg_support = fi.FlashInferMetadataBuilder.get_cudagraph_support(config, spec) assert cg_support == AttentionCGSupport.UNIFORM_BATCH device = torch.device(f"{DEVICE_TYPE}:0") # Request i owns consecutive pages from starts[i]; page 0 stays null. pages = [0] + [cdiv(s, bs) for s in seq_lens[:-1]] starts = 1 + torch.tensor(pages).cumsum(0) max_pages = cdiv(max(seq_lens), bs) block_table = (starts[:, None] + torch.arange(max_pages)).int().to(device) full_dim = nvfp4_kv_cache_full_dim(d) kv_cache = torch.zeros( int(starts[-1]) + max_pages, bs, 8, full_dim, dtype=torch.uint8, device=device ).transpose(1, 2) q, k, v = ( torch.randn(sum(seq_lens), h, d).bfloat16().to(device) for h in (24, 4, 4) ) def run(query_lens, use_cuda_graph=False): req = torch.repeat_interleave(torch.arange(3), torch.tensor(query_lens)) pos = torch.cat([torch.arange(s - n, s) for s, n in zip(seq_lens, query_lens)]) rows = pos + torch.tensor([0, *seq_lens]).cumsum(0)[req] metadata = replace( create_common_attn_metadata(BatchSpec(seq_lens, query_lens), bs, device), block_table_tensor=block_table, slot_mapping=(starts[req] * bs + pos).to(device), ) with set_current_vllm_config(config): return rows, run_attention_backend( AttentionBackendEnum.FLASHINFER, spec, ["placeholder"], config, device, metadata, q[rows], k[rows], v[rows], kv_cache, kv_cache_dtype="nvfp4", use_cuda_graph=use_cuda_graph, layer_k_scale=0.02, layer_v_scale=0.03, ) # The full prefill writes every token through the NVFP4 writer; its fa2 # output for the last q_len tokens of each request is the reference. _, reference = run(seq_lens) xqa_fn = fi.flashinfer_xqa_batch_decode_with_kv_cache with unittest.mock.patch.object( fi, "flashinfer_xqa_batch_decode_with_kv_cache", wraps=xqa_fn ) as xqa: rows, output = run([q_len] * len(seq_lens), use_cuda_graph) assert xqa.called torch.testing.assert_close(output, reference[rows], atol=2e-2, rtol=2e-2) if current_platform.is_rocm(): # FLASH_ATTN is not supported on ROCm SLIDING_WINDOW_BACKENDS_TO_TEST = [ AttentionBackendEnum.FLEX_ATTENTION, AttentionBackendEnum.TRITON_ATTN, "FLEX_ATTENTION_SLOW", ] else: SLIDING_WINDOW_BACKENDS_TO_TEST = [ AttentionBackendEnum.FLASH_ATTN, AttentionBackendEnum.FLEX_ATTENTION, AttentionBackendEnum.TRITON_ATTN, "FLEX_ATTENTION_SLOW", ] # Encoder-only FlexAttention always uses the slow builder, so the pseudo-backend # would run an identical implementation twice. SLIDING_WINDOW_ENCODER_BACKENDS_TO_TEST = [ backend for backend in SLIDING_WINDOW_BACKENDS_TO_TEST if backend != "FLEX_ATTENTION_SLOW" ] @pytest.mark.parametrize( "batch_spec_name", [ "small_decode", "small_prefill", "mixed_medium", "large_decode", "large_prefill", "mixed_large", ], ) @pytest.mark.parametrize("model", ["microsoft/Phi-tiny-MoE-instruct"]) @pytest.mark.parametrize("tensor_parallel_size", [1, 2, 4]) def test_sliding_window_backend_correctness( default_vllm_config, batch_spec_name: str, model: str, tensor_parallel_size: int, ): """Test backend's correctness with sliding window attention.""" def sliding_window_mask_mod( b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor, *, context_len: int, sliding_window: int, ): causal_mask = q_idx + context_len >= kv_idx window_mask = q_idx + context_len - kv_idx < sliding_window return causal_mask & window_mask batch_spec = BATCH_SPECS[batch_spec_name] model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens)) sliding_window = model_config.get_sliding_window() sliding_window_mask_mod_fn = partial( sliding_window_mask_mod, sliding_window=sliding_window ) LARGE_BLOCK_BACKENDS = ( [AttentionBackendEnum.FLEX_ATTENTION] if is_torch_equal_or_newer("2.9.0.dev0") else [] ) SMALL_BLOCK_BACKENDS = [ x for x in SLIDING_WINDOW_BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS ] _test_backend_correctness( batch_spec, model, SMALL_BLOCK_BACKENDS, sliding_window_mask_mod_fn, tensor_parallel_size=tensor_parallel_size, ) # Fast FlexAttention needs to run with block_size=128 if LARGE_BLOCK_BACKENDS: _test_backend_correctness( batch_spec, model, LARGE_BLOCK_BACKENDS, sliding_window_mask_mod_fn, block_size=128, tensor_parallel_size=tensor_parallel_size, ) @pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm-specific test") def test_rocm_aiter_fa_unquantized_cache_ignores_kv_scales( default_vllm_config, ): """Stale scale tensors must not affect an unquantized KV cache.""" from vllm._aiter_ops import is_aiter_found_and_supported if not is_aiter_found_and_supported(): pytest.skip("AITER is required") def sliding_window_mask_mod( b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor, *, context_len: int, sliding_window: int, ): causal_mask = q_idx + context_len >= kv_idx window_mask = q_idx + context_len - kv_idx < sliding_window return causal_mask & window_mask model = "microsoft/Phi-tiny-MoE-instruct" batch_spec = BATCH_SPECS["small_decode"] model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens)) sliding_window = model_config.get_sliding_window() assert sliding_window is not None _test_backend_correctness( batch_spec, model, [AttentionBackendEnum.ROCM_AITER_FA], partial(sliding_window_mask_mod, sliding_window=sliding_window), layer_k_scale=0.25, layer_v_scale=0.5, ) @pytest.mark.parametrize( "batch_spec_name", [ "small_encoder_prefill", "medium_encoder_prefill", ], ) @pytest.mark.parametrize("model", ["google/embeddinggemma-300m"]) @pytest.mark.parametrize("tensor_parallel_size", [1, 2]) def test_sliding_window_encoder_backend_correctness( default_vllm_config, batch_spec_name: str, model: str, tensor_parallel_size: int, ): """Test backend's correctness with sliding window attention.""" def bidi_sliding_window_mask_mod( b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor, *, context_len: int, sliding_window: int, ): return torch.abs(q_idx + context_len - kv_idx) < sliding_window batch_spec = BATCH_SPECS[batch_spec_name] model_config = ModelConfig(model=model, max_model_len=max(batch_spec.seq_lens)) sliding_window = model_config.get_sliding_window() sliding_window_mask_mod_fn = partial( bidi_sliding_window_mask_mod, sliding_window=sliding_window ) _test_backend_correctness( batch_spec, model, SLIDING_WINDOW_ENCODER_BACKENDS_TO_TEST, sliding_window_mask_mod_fn, causal=False, attn_type=AttentionType.ENCODER_ONLY, tensor_parallel_size=tensor_parallel_size, ) NON_CAUSAL_BACKENDS_TO_TEST = [ AttentionBackendEnum.FLASH_ATTN, AttentionBackendEnum.FLEX_ATTENTION, "FLEX_ATTENTION_SLOW", ] if current_platform.is_rocm(): NON_CAUSAL_BACKENDS_TO_TEST = [ x for x in NON_CAUSAL_BACKENDS_TO_TEST if x is not AttentionBackendEnum.FLASH_ATTN ] @pytest.mark.parametrize( "batch_spec_name", [ "small_decode", "small_prefill", "mixed_small", ], ) @pytest.mark.parametrize("model", ["Qwen/Qwen3-0.6B"]) def test_non_causal_backend_correctness( default_vllm_config, batch_spec_name: str, model: str, ): """Test backend's correctness with non-causal (bidirectional) decoder attention, as used by DFlash speculative decoding.""" def bidirectional_mask_mod( b: torch.Tensor, h: torch.Tensor, q_idx: torch.Tensor, kv_idx: torch.Tensor, *, context_len: int, ): return q_idx >= 0 # Always True batch_spec = BATCH_SPECS[batch_spec_name] LARGE_BLOCK_BACKENDS = ( [AttentionBackendEnum.FLEX_ATTENTION] if is_torch_equal_or_newer("2.9.0.dev0") else [] ) SMALL_BLOCK_BACKENDS = [ x for x in NON_CAUSAL_BACKENDS_TO_TEST if x not in LARGE_BLOCK_BACKENDS ] _test_backend_correctness( batch_spec, model, SMALL_BLOCK_BACKENDS, bidirectional_mask_mod, causal=False, ) if LARGE_BLOCK_BACKENDS: _test_backend_correctness( batch_spec, model, LARGE_BLOCK_BACKENDS, bidirectional_mask_mod, causal=False, block_size=128, )