From 0d4ad47981b6133c8f835d78f7a19ca4eda113d5 Mon Sep 17 00:00:00 2001 From: Luke Alonso Date: Tue, 1 Sep 2026 07:42:02 -0700 Subject: [PATCH] [Kernel] Add B12X causal paged attention backend (#52017) Signed-off-by: Luke Alonso Co-authored-by: Lucas Wilkinson --- .buildkite/test_areas/kernels.yaml | 2 +- docs/design/attention_backends.md | 10 + setup.py | 2 +- tests/v1/attention/test_attention_backends.py | 56 +- tests/v1/attention/test_b12x.py | 360 ++++++ vllm/utils/b12x.py | 5 + vllm/v1/attention/backends/b12x.py | 1096 +++++++++++++++++ vllm/v1/attention/backends/registry.py | 1 + 8 files changed, 1527 insertions(+), 5 deletions(-) create mode 100644 tests/v1/attention/test_b12x.py create mode 100644 vllm/v1/attention/backends/b12x.py diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index fc13887799..37fe5489e6 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -389,7 +389,7 @@ steps: - tests/quantization/reference_mxfp4.py - tests/quantization/test_auto_round.py commands: - - uv pip install --system b12x==1.2.6 + - uv pip install --system b12x==1.3.0 - pytest -v -s model_executor/kernels/test_b12x_linear.py model_executor/test_b12x_warmup.py kernels/quantization/test_block_fp8.py diff --git a/docs/design/attention_backends.md b/docs/design/attention_backends.md index f0f363223f..3fdc23ff84 100644 --- a/docs/design/attention_backends.md +++ b/docs/design/attention_backends.md @@ -136,6 +136,16 @@ Priority is **1 = highest** (tried first). > `--block-size` that is not a multiple of 128 instead makes FlashAttention ineligible > for such models, which is an error if the backend was requested explicitly. +### b12x + +The optional [b12x](https://pypi.org/project/b12x/) backend supports causal +decoder attention on NVIDIA SM120 and SM121 GPUs. Install and select it with: + +```bash +uv pip install "vllm[b12x]" +vllm serve --attention-backend b12x +``` + ## MiniMax M3 Sparse Attention Backends Block-sparse GQA backend used by MiniMax M3 sparse ("lightning indexer") diff --git a/setup.py b/setup.py index 4b3488b5fb..6de21c793d 100644 --- a/setup.py +++ b/setup.py @@ -1522,7 +1522,7 @@ setup( # only; also needs system GStreamer + libv4l (see docs). "deepstream": ["nvidia-deepstream-videodecode-cu13>=9.0.2"], "flashinfer": [], # Kept for backwards compatibility - "b12x": ["b12x==1.2.6"], + "b12x": ["b12x==1.3.0"], # Optional deps for Helion kernel development # NOTE: When updating helion version, also update CI files: # - .buildkite/test_areas/kernels.yaml diff --git a/tests/v1/attention/test_attention_backends.py b/tests/v1/attention/test_attention_backends.py index cc16242753..6609705c22 100644 --- a/tests/v1/attention/test_attention_backends.py +++ b/tests/v1/attention/test_attention_backends.py @@ -3,6 +3,7 @@ """Tests for v1 attention backends without GPUModelRunner dependency.""" from functools import partial +from types import SimpleNamespace import pytest import torch @@ -282,6 +283,7 @@ def run_attention_backend( sliding_window: int | None = None, kv_cache_dtype: str = "auto", sinks: torch.Tensor | None = None, + use_cuda_graph: bool = False, ) -> torch.Tensor: """Run attention computation using the specified backend's AttentionImpl.""" @@ -367,9 +369,26 @@ def run_attention_backend( impl.do_kv_cache_update( mock_layer, key, value, kv_cache, attn_metadata.slot_mapping ) - output = impl.forward( - mock_layer, query, key, value, kv_cache, attn_metadata, output=output - ) + if use_cuda_graph: + impl.forward( + mock_layer, query, key, value, kv_cache, attn_metadata, output=output + ) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + impl.forward( + mock_layer, + query, + key, + value, + kv_cache, + attn_metadata, + output=output, + ) + graph.replay() + else: + output = impl.forward( + mock_layer, query, key, value, kv_cache, attn_metadata, output=output + ) return output @@ -389,6 +408,11 @@ def _test_backend_correctness( kv_cache_dtype: str = "auto", use_sinks: bool = False, layout: KVCacheLayout | None = None, + use_cuda_graph: bool = False, + num_speculative_tokens: int = 0, + model_dtype: torch.dtype | None = None, + max_num_seqs: int | None = None, + max_num_batched_tokens: int | None = None, ): """ Test that all backends produce similar outputs to a reference implementation @@ -433,10 +457,19 @@ def _test_backend_correctness( model_name=model, tensor_parallel_size=1, # Always use TP=1 to avoid multi-GPU requirements max_model_len=max(batch_spec.seq_lens), + dtype=model_dtype or "auto", block_size=block_size, num_gpu_blocks=8192, hf_config_override=hf_config_override, ) + if max_num_seqs is not None: + vllm_config.scheduler_config.max_num_seqs = max_num_seqs + if max_num_batched_tokens is not None: + vllm_config.scheduler_config.max_num_batched_tokens = max_num_batched_tokens + if num_speculative_tokens > 0: + vllm_config.speculative_config = SimpleNamespace( + num_speculative_tokens=num_speculative_tokens + ) vllm_config.cache_config.cache_dtype = kv_cache_dtype device = torch.device(f"{DEVICE_TYPE}:0") @@ -619,6 +652,22 @@ def _test_backend_correctness( backend_layout = backend_supported[0] kv_cache_for_backend = _clone_kv_cache_in_layout(kv_cache, backend_layout) + if backend_name == AttentionBackendEnum.B12X: + cache_dtype = ( + FP8_KV_CACHE_DTYPES[kv_cache_dtype] + if is_quantized_kv_cache(kv_cache_dtype) + else kv_cache.dtype + ) + typed_cache = kv_cache.view(cache_dtype) + key_cache = typed_cache[..., :head_size].permute(0, 2, 1, 3) + value_cache = typed_cache[..., head_size:].permute(0, 2, 1, 3) + packed_cache = torch.stack((key_cache, value_cache), dim=1).flatten(-2) + if is_quantized_kv_cache(kv_cache_dtype): + packed_cache = packed_cache.view(torch.uint8) + kv_cache_for_backend = _clone_kv_cache_in_layout( + packed_cache, backend_layout + ) + # FlashInfer reads the layout at plan time; set it to match # the physical order of the test cache. vllm_config.cache_config.kv_cache_layout = backend_layout.name @@ -638,6 +687,7 @@ def _test_backend_correctness( attn_type=attn_type, kv_cache_dtype=kv_cache_dtype, sinks=sinks, + use_cuda_graph=use_cuda_graph, ) # Check shape and dtype consistency diff --git a/tests/v1/attention/test_b12x.py b/tests/v1/attention/test_b12x.py new file mode 100644 index 0000000000..d4fb03764f --- /dev/null +++ b/tests/v1/attention/test_b12x.py @@ -0,0 +1,360 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from functools import partial +from types import SimpleNamespace + +import pytest +import torch + +from tests.v1.attention.test_attention_backends import ( + BATCH_SPECS, + _test_backend_correctness, +) +from tests.v1.attention.utils import BatchSpec +from vllm.config import ModelConfig +from vllm.platforms import current_platform +from vllm.platforms.interface import DeviceCapability +from vllm.utils.b12x import get_b12x_paged_attention +from vllm.v1.attention.backends import b12x +from vllm.v1.attention.backends.b12x import ( + B12xPagedAttentionBackend, + B12xPagedAttentionImpl, + _kv_page_size, + _max_page_table_width, +) +from vllm.v1.attention.backends.registry import AttentionBackendEnum +from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout + + +def _require_b12x_paged_attention() -> None: + capability = current_platform.get_device_capability() + if ( + not current_platform.is_cuda() + or capability is None + or not B12xPagedAttentionBackend.supports_compute_capability(capability) + ): + pytest.skip("b12x paged attention requires SM120 or SM121.") + + paged_attention = get_b12x_paged_attention() + if paged_attention is None or not paged_attention.is_supported(): + pytest.skip("b12x paged attention is not available.") + + +def _causal_mask( + b: torch.Tensor, + h: torch.Tensor, + q_idx: torch.Tensor, + kv_idx: torch.Tensor, + *, + context_len: int, +): + return q_idx + context_len >= kv_idx + + +def _causal_sliding_window_mask( + b: torch.Tensor, + h: torch.Tensor, + q_idx: torch.Tensor, + kv_idx: torch.Tensor, + *, + context_len: int, + sliding_window: int, +): + causal_mask = q_idx + context_len >= kv_idx + window_mask = q_idx + context_len - kv_idx < sliding_window + return causal_mask & window_mask + + +@pytest.mark.parametrize( + ("dtype", "kv_cache_dtype", "paged_attention", "expected_reason"), + [ + pytest.param( + torch.float16, + "fp8_e4m3", + None, + "b12x currently requires bfloat16 queries", + id="query-dtype", + ), + pytest.param( + torch.bfloat16, + "float16", + None, + "b12x does not support float16 KV cache", + id="kv-cache-dtype", + ), + pytest.param( + torch.bfloat16, + "auto", + None, + "Install the b12x backend with `pip install vllm[b12x]`", + id="package-not-installed", + ), + pytest.param( + torch.bfloat16, + "auto", + SimpleNamespace(is_supported=lambda: False), + "b12x paged attention is not supported on the current device", + id="device-api", + ), + pytest.param( + torch.bfloat16, + "auto", + SimpleNamespace(is_supported=lambda: True), + None, + id="supported", + ), + ], +) +def test_b12x_attention_config_support( + monkeypatch: pytest.MonkeyPatch, + dtype: torch.dtype, + kv_cache_dtype: str, + paged_attention, + expected_reason: str | None, +) -> None: + monkeypatch.setattr( + b12x, + "get_b12x_paged_attention", + lambda: paged_attention, + ) + + reason = B12xPagedAttentionBackend.supports_combination( + head_size=128, + dtype=dtype, + kv_cache_dtype=kv_cache_dtype, + block_size=128, + use_mla=False, + has_sink=False, + use_sparse=False, + use_mm_prefix=False, + device_capability=DeviceCapability(12, 0), + ) + + assert reason == expected_reason + + +def test_b12x_attention_uses_two_plane_nhd_cache() -> None: + spec = B12xPagedAttentionBackend.customize_spec( + FullAttentionSpec( + block_size=128, + num_kv_heads=4, + head_size=128, + dtype=torch.bfloat16, + ) + ) + + assert spec.num_head_slots == 2 + assert spec.state_content_bytes == 4 * 128 * 2 + assert B12xPagedAttentionBackend.supported_kv_cache_layouts() == ( + KVCacheLayout.LBHNC, + KVCacheLayout.BLHNC, + ) + assert B12xPagedAttentionBackend.supports_block_size(128) + assert not B12xPagedAttentionBackend.supports_block_size(32) + + +def test_b12x_attention_hybrid_cache_capacity_includes_expansion() -> None: + assert _max_page_table_width(4096, 128, 4096, False) == 32 + assert _max_page_table_width(4096, 128, 4096, True) == 64 + + +def test_b12x_attention_runtime_page_size_comes_from_cache() -> None: + key_cache = torch.empty((3, 64, 4, 128), device="meta") + value_cache = torch.empty_like(key_cache) + + assert _kv_page_size(key_cache, value_cache) == 64 + with pytest.raises(ValueError, match="matching K/V page sizes"): + _kv_page_size(key_cache, torch.empty((3, 128, 4, 128), device="meta")) + + +def test_b12x_attention_lazily_prepares_decode_bucket(monkeypatch) -> None: + impl = object.__new__(B12xPagedAttentionImpl) + plan = SimpleNamespace(layout=SimpleNamespace(nbytes=96)) + created: list[tuple[int, int]] = [] + + def create_plan(page_size: int, batch_size: int) -> SimpleNamespace: + created.append((page_size, batch_size)) + return plan + + impl._decode_plans = {} + impl._create_decode_plan = create_plan + impl._scratch_nbytes = 128 + impl._extend_plans = {} + impl._verify_q_per_req = 0 + metadata = SimpleNamespace(max_query_len=1) + monkeypatch.setattr(b12x, "_capture_alloc_forbidden", lambda: False) + + assert impl._select_plan(metadata, 7, 7, 7, 64) is plan + assert impl._select_plan(metadata, 7, 7, 7, 64) is plan + assert created == [(64, 7)] + + +def test_b12x_attention_fp8_descales_follow_request_batch() -> None: + impl = object.__new__(B12xPagedAttentionImpl) + impl.kv_cache_dtype = "fp8_e4m3" + layer = SimpleNamespace( + _k_scale=torch.tensor(2.0), + _v_scale=torch.tensor([3.0, 4.0, 5.0]), + ) + + k_descale, v_descale = impl._prepare_fp8_descales( + layer, num_reqs=2, device=torch.device("cpu") + ) + + torch.testing.assert_close(k_descale, torch.tensor([2.0, 2.0])) + torch.testing.assert_close(v_descale, torch.tensor([3.0, 4.0])) + assert k_descale.stride() == (0,) + assert v_descale.stride() == (1,) + + +def test_b12x_attention_sinks_refresh_in_place_after_reload() -> None: + impl = object.__new__(B12xPagedAttentionImpl) + source = torch.tensor([1.0, 2.0], dtype=torch.bfloat16) + impl._sinks_source = source + impl.sinks = None + + impl.process_weights_after_loading(torch.bfloat16) + assert impl.sinks is not None + sinks_ptr = impl.sinks.data_ptr() + source.copy_(torch.tensor([3.0, 4.0], dtype=torch.bfloat16)) + impl.process_weights_after_loading(torch.bfloat16) + + assert impl.sinks.data_ptr() == sinks_ptr + torch.testing.assert_close(impl.sinks, source.float()) + + +@pytest.mark.parametrize( + "batch_spec_name", + ["small_decode", "small_prefill", "mixed_small", "medium_decode"], +) +@pytest.mark.parametrize( + ("kv_cache_dtype", "model_dtype"), + [ + ("auto", None), + ("bfloat16", torch.bfloat16), + ("fp8_e4m3", torch.bfloat16), + ], +) +@pytest.mark.parametrize("block_size", [64, 128]) +def test_b12x_causal_backend_correctness( + default_vllm_config, + workspace_init, + batch_spec_name: str, + kv_cache_dtype: str, + model_dtype: torch.dtype | None, + block_size: int, +) -> None: + """b12x causal paged attention matches the shared SDPA reference.""" + _require_b12x_paged_attention() + batch_spec = BATCH_SPECS[batch_spec_name] + + _test_backend_correctness( + batch_spec, + "Qwen/Qwen3-0.6B", + [AttentionBackendEnum.B12X], + _causal_mask, + block_size=block_size, + kv_cache_dtype=kv_cache_dtype, + model_dtype=model_dtype, + max_num_seqs=batch_spec.batch_size, + max_num_batched_tokens=max(sum(batch_spec.query_lens), 64), + ) + + +@pytest.mark.parametrize( + "batch_spec", + [ + pytest.param(BatchSpec(seq_lens=[2080, 2200], query_lens=[1, 1]), id="decode"), + pytest.param(BatchSpec(seq_lens=[2080, 2200], query_lens=[8, 8]), id="prefill"), + ], +) +def test_b12x_causal_sliding_window( + default_vllm_config, + workspace_init, + batch_spec: BatchSpec, +) -> None: + """b12x causal sliding-window attention matches the shared reference.""" + _require_b12x_paged_attention() + + model = "microsoft/Phi-tiny-MoE-instruct" + sliding_window = ModelConfig( + model=model, max_model_len=max(batch_spec.seq_lens) + ).get_sliding_window() + assert sliding_window is not None + mask = partial(_causal_sliding_window_mask, sliding_window=sliding_window) + + _test_backend_correctness( + batch_spec, + model, + [AttentionBackendEnum.B12X], + mask, + block_size=64, + atol=3e-2, + rtol=3e-2, + max_num_seqs=batch_spec.batch_size, + max_num_batched_tokens=max(sum(batch_spec.query_lens), 64), + ) + + +def test_b12x_attention_sinks( + default_vllm_config, + workspace_init, +) -> None: + """b12x attention sinks match the explicit sink reference.""" + _require_b12x_paged_attention() + batch_spec = BATCH_SPECS["small_prefill"] + + _test_backend_correctness( + batch_spec, + "Qwen/Qwen3-0.6B", + [AttentionBackendEnum.B12X], + _causal_mask, + block_size=64, + atol=3e-2, + rtol=3e-2, + use_sinks=True, + max_num_seqs=batch_spec.batch_size, + max_num_batched_tokens=max(sum(batch_spec.query_lens), 64), + ) + + +def test_b12x_decode_cuda_graph_replay( + default_vllm_config, + workspace_init, +) -> None: + """b12x decode output remains correct after CUDA graph replay.""" + _require_b12x_paged_attention() + batch_spec = BATCH_SPECS["small_decode"] + + _test_backend_correctness( + batch_spec, + "Qwen/Qwen3-0.6B", + [AttentionBackendEnum.B12X], + _causal_mask, + block_size=64, + use_cuda_graph=True, + max_num_seqs=batch_spec.batch_size, + max_num_batched_tokens=max(sum(batch_spec.query_lens), 64), + ) + + +def test_b12x_speculative_verification_uses_cuda_graph_plan( + default_vllm_config, + workspace_init, +) -> None: + """Exercise the verifier plan rather than the general extend plan.""" + _require_b12x_paged_attention() + batch_spec = BatchSpec(seq_lens=[32, 40], query_lens=[4, 4]) + + _test_backend_correctness( + batch_spec, + "Qwen/Qwen3-0.6B", + [AttentionBackendEnum.B12X], + _causal_mask, + block_size=128, + num_speculative_tokens=3, + use_cuda_graph=True, + max_num_seqs=batch_spec.batch_size, + max_num_batched_tokens=max(sum(batch_spec.query_lens), 64), + ) diff --git a/vllm/utils/b12x.py b/vllm/utils/b12x.py index dd4fa2940b..1ed0febc85 100644 --- a/vllm/utils/b12x.py +++ b/vllm/utils/b12x.py @@ -34,6 +34,7 @@ def _import_submodule(module_name: str) -> ModuleType | None: _B12X_SUBMODULES = { module_name: _import_submodule(module_name) for module_name in ( + "b12x.attention.paged", "b12x.gemm.blockscaled", # TODO: Remove once B12X exposes the scale-swizzle API publicly. "b12x._lib.intrinsics", @@ -73,6 +74,10 @@ def get_b12x_fused_moe() -> ModuleType | None: return _get_submodule("b12x.moe.fused_moe") +def get_b12x_paged_attention() -> ModuleType | None: + return _get_submodule("b12x.attention.paged") + + def b12x_warmup_token_counts( *, max_tokens: int, diff --git a/vllm/v1/attention/backends/b12x.py b/vllm/v1/attention/backends/b12x.py new file mode 100644 index 0000000000..d2b28eed0a --- /dev/null +++ b/vllm/v1/attention/backends/b12x.py @@ -0,0 +1,1096 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""b12x paged causal attention backend for SM12x.""" + +from __future__ import annotations + +import copy +import math +from dataclasses import dataclass, replace +from functools import partial +from typing import Any, ClassVar + +import torch + +from vllm.config import VllmConfig, get_current_vllm_config +from vllm.config.cache import CacheDType +from vllm.logger import init_logger +from vllm.model_executor.warmup.cutedsl_warmup import ( + CuTeDSLCompileUnit, + register_cutedsl_warmup_provider, +) +from vllm.platforms import current_platform +from vllm.platforms.interface import DeviceCapability +from vllm.utils.b12x import get_b12x_paged_attention +from vllm.utils.math_utils import cdiv +from vllm.utils.torch_utils import ( + canonicalize_singleton_dim_strides, + get_dtype_size, + is_quantized_kv_cache, +) +from vllm.v1.attention.backend import ( + AttentionBackend, + AttentionCGSupport, + AttentionImpl, + AttentionLayer, + AttentionMetadata, + AttentionMetadataBuilder, + AttentionType, + CommonAttentionMetadata, + MultipleOf, +) +from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout, KVCacheSpec +from vllm.v1.worker.workspace import current_workspace_manager + +logger = init_logger(__name__) + +_B12X_SUPPORTED_PAGE_SIZES = (64, 128) +_B12X_PREFERRED_PAGE_SIZE = 128 +_MIN_PAGED_TILE_Q = 16 +_B12X_FP8_KV_CACHE_DTYPES: tuple[CacheDType, ...] = ("fp8", "fp8_e4m3") +_B12X_SUPPORTED_KV_CACHE_DTYPES: tuple[CacheDType, ...] = ( + "auto", + "bfloat16", + *_B12X_FP8_KV_CACHE_DTYPES, +) + + +def _max_page_table_width( + max_model_len: int, + block_size: int, + max_num_batched_tokens: int, + is_hybrid: bool, +) -> int: + width = max(cdiv(max(max_model_len, 1), block_size), 1) + if is_hybrid: + # Hybrid cache setup can enlarge the storage block after attention + # layers are initialized. Its expansion into kernel-sized blocks adds + # at most one storage block of trailing page-table capacity. + width += cdiv(max_num_batched_tokens, block_size) + return width + + +def _kv_page_size(key_cache: torch.Tensor, value_cache: torch.Tensor) -> int: + """Return the static kernel page geometry negotiated by vLLM. + + The KV manager can split the configured storage block into a smaller + kernel page when another backend shares its cache group. Cache shapes are + fixed before graph capture, so this is not a live-length policy decision. + """ + if key_cache.ndim < 2 or value_cache.ndim < 2: + raise ValueError( + "b12x expects paged K/V caches with a page dimension, got " + f"{tuple(key_cache.shape)} and {tuple(value_cache.shape)}." + ) + key_page_size = int(key_cache.shape[1]) + value_page_size = int(value_cache.shape[1]) + if key_page_size != value_page_size: + raise ValueError( + "b12x requires matching K/V page sizes, got " + f"{key_page_size} and {value_page_size}." + ) + if key_page_size not in _B12X_SUPPORTED_PAGE_SIZES: + raise ValueError( + "b12x requires runtime page size in " + f"{_B12X_SUPPORTED_PAGE_SIZES}, got {key_page_size}." + ) + return key_page_size + + +def _capture_alloc_forbidden() -> bool: + if not torch.cuda.is_available(): + return False + try: + return bool(torch.cuda.is_current_stream_capturing()) + except RuntimeError: + return False + + +def _ensure_i32_contiguous(tensor: torch.Tensor, name: str) -> torch.Tensor: + if tensor.dtype != torch.int32: + if _capture_alloc_forbidden(): + raise RuntimeError( + f"b12x would convert {name} to int32 during CUDA graph " + "capture. Prepare int32 metadata before capture." + ) + tensor = tensor.to(torch.int32) + if not tensor.is_contiguous(): + if _capture_alloc_forbidden(): + raise RuntimeError( + f"b12x would make {name} contiguous during CUDA graph " + "capture. Prepare contiguous metadata before capture." + ) + tensor = tensor.contiguous() + return tensor + + +def _dtype_from_cache_config( + kv_cache_dtype: str, + vllm_config: VllmConfig, +) -> torch.dtype: + if kv_cache_dtype == "bfloat16": + return torch.bfloat16 + if kv_cache_dtype in _B12X_FP8_KV_CACHE_DTYPES: + return current_platform.fp8_dtype() + if kv_cache_dtype != "auto": + raise NotImplementedError( + "b12x currently supports only auto, bfloat16, " + "fp8, and fp8_e4m3 " + f"KV cache dtypes; got {kv_cache_dtype!r}." + ) + return vllm_config.model_config.dtype + + +def _is_b12x_fp8_kv_cache(kv_cache_dtype: str) -> bool: + return kv_cache_dtype in _B12X_FP8_KV_CACHE_DTYPES + + +class B12xPagedAttentionBackend(AttentionBackend): + """b12x paged attention backend for regular/GQA decoder layers.""" + + @classmethod + def customize_spec(cls, spec: AttentionSpec) -> AttentionSpec: + if spec.state_content_bytes is not None: + return spec + assert spec.head_size == spec.head_size_v, ( + "Separate K/V planes require symmetric K/V head sizes." + ) + return replace( + spec, + num_head_slots=2, + state_content_bytes=spec.num_kv_heads + * spec.head_size + * get_dtype_size(spec.dtype), + ) + + supported_dtypes: ClassVar[list[torch.dtype]] = [torch.bfloat16] + supported_kv_cache_dtypes: ClassVar[list[CacheDType]] = list( + _B12X_SUPPORTED_KV_CACHE_DTYPES + ) + + forward_includes_kv_cache_update: bool = False + + @staticmethod + def get_name() -> str: + return "B12X" + + @classmethod + def get_impl_cls(cls) -> type[B12xPagedAttentionImpl]: + return B12xPagedAttentionImpl + + @staticmethod + def get_builder_cls() -> type[B12xPagedMetadataBuilder]: + return B12xPagedMetadataBuilder + + @staticmethod + def get_supported_kernel_block_sizes() -> list[int | MultipleOf]: + return list(_B12X_SUPPORTED_PAGE_SIZES) + + @classmethod + def supports_block_size(cls, block_size: int | None) -> bool: + return block_size is None or int(block_size) in _B12X_SUPPORTED_PAGE_SIZES + + @classmethod + def get_preferred_block_size(cls, default_block_size: int) -> int: + if int(default_block_size) in _B12X_SUPPORTED_PAGE_SIZES: + return int(default_block_size) + return _B12X_PREFERRED_PAGE_SIZE + + @classmethod + def get_supported_head_sizes(cls) -> list[int]: + return [64, 128, 192, 256] + + @classmethod + def supports_sink(cls) -> bool: + return True + + @classmethod + def supports_sliding_window(cls) -> bool: + return True + + @classmethod + def supports_compute_capability(cls, capability: DeviceCapability) -> bool: + # Consumer Blackwell SM120 / SM121. The b12x paged kernels also gate + # internally, but keep vLLM selection fail-fast and explicit. + return (capability.major, capability.minor) in ((12, 0), (12, 1)) + + @classmethod + def supports_combination( + cls, + head_size: int, + dtype: torch.dtype, + kv_cache_dtype: CacheDType | None, + block_size: int | None, + use_mla: bool, + has_sink: bool, + use_sparse: bool, + use_mm_prefix: bool, + device_capability: DeviceCapability, + ) -> str | None: + if dtype != torch.bfloat16: + return "b12x currently requires bfloat16 queries" + if kv_cache_dtype == "float16": + return "b12x does not support float16 KV cache" + if ( + kv_cache_dtype is not None + and is_quantized_kv_cache(kv_cache_dtype) + and not _is_b12x_fp8_kv_cache(kv_cache_dtype) + ): + return "b12x currently supports only fp8/fp8_e4m3 quantized KV cache dtypes" + paged_attention = get_b12x_paged_attention() + if paged_attention is None: + return "Install the b12x backend with `pip install vllm[b12x]`" + if not paged_attention.is_supported(): + return "b12x paged attention is not supported on the current device" + return None + + @classmethod + def supported_kv_cache_layouts(cls) -> tuple[KVCacheLayout, ...]: + return (KVCacheLayout.LBHNC, KVCacheLayout.BLHNC) + + +@dataclass +class B12xPagedMetadata(AttentionMetadata): + num_actual_tokens: int + max_query_len: int + query_start_loc: torch.Tensor + max_seq_len: int + seq_lens: torch.Tensor + block_table: torch.Tensor + slot_mapping: torch.Tensor + causal: bool = True + + +class B12xPagedMetadataBuilder(AttentionMetadataBuilder[B12xPagedMetadata]): + """Metadata builder for b12x. + + Decode and uniform speculative-verifier batches use preplanned graph + buckets. Extend/prefill remains eager and does not affect uniform decode + graph eligibility. + """ + + _cudagraph_support: ClassVar[AttentionCGSupport] = AttentionCGSupport.UNIFORM_BATCH + supports_update_block_table: bool = True + + @classmethod + def get_cudagraph_support( + cls, + vllm_config: VllmConfig, + kv_cache_spec: KVCacheSpec, + ) -> AttentionCGSupport: + del vllm_config, kv_cache_spec + return cls._cudagraph_support + + def __init__( + self, + kv_cache_spec: AttentionSpec, + layer_names: list[str], + vllm_config: VllmConfig, + device: torch.device, + ) -> None: + super().__init__(kv_cache_spec, layer_names, vllm_config, device) + + def build( + self, + common_prefix_len: int, + common_attn_metadata: CommonAttentionMetadata, + fast_build: bool = False, + ) -> B12xPagedMetadata: + del common_prefix_len, fast_build + cm = common_attn_metadata + return B12xPagedMetadata( + num_actual_tokens=cm.num_actual_tokens, + max_query_len=cm.max_query_len, + query_start_loc=cm.query_start_loc, + max_seq_len=cm.max_seq_len, + seq_lens=cm.seq_lens, + block_table=cm.block_table_tensor, + slot_mapping=cm.slot_mapping, + causal=cm.causal, + ) + + def update_block_table( + self, + metadata: B12xPagedMetadata, + blk_table: torch.Tensor, + slot_mapping: torch.Tensor, + ) -> B12xPagedMetadata: + new_metadata = copy.copy(metadata) + new_metadata.block_table = blk_table + new_metadata.slot_mapping = slot_mapping + return new_metadata + + +class B12xPagedAttentionImpl(AttentionImpl[B12xPagedMetadata]): + """b12x paged GQA attention implementation.""" + + can_return_lse_for_decode: bool = False + supports_dcp: bool = False + + def __init__( + self, + num_heads: int, + head_size: int, + scale: float, + num_kv_heads: int, + alibi_slopes: list[float] | None, + sliding_window: int | None, + kv_cache_dtype: str, + logits_soft_cap: float | None = None, + attn_type: AttentionType = AttentionType.DECODER, + kv_sharing_target_layer_name: str | None = None, + sinks: torch.Tensor | None = None, + ) -> None: + if alibi_slopes is not None: + raise NotImplementedError("b12x does not support ALiBi.") + if logits_soft_cap not in (None, 0): + raise NotImplementedError("b12x does not support logits soft cap.") + if attn_type != AttentionType.DECODER: + raise NotImplementedError( + "b12x currently supports decoder self-attention only." + ) + if is_quantized_kv_cache(kv_cache_dtype) and not _is_b12x_fp8_kv_cache( + kv_cache_dtype + ): + raise NotImplementedError( + "b12x currently supports only fp8/fp8_e4m3 quantized KV cache dtypes." + ) + if num_heads % num_kv_heads != 0: + raise ValueError("b12x requires q heads divisible by kv heads.") + + expected_scale = head_size**-0.5 + if not math.isclose(float(scale), expected_scale, rel_tol=1e-5, abs_tol=1e-7): + raise NotImplementedError( + "b12x currently requires canonical softmax scale " + f"head_dim**-0.5={expected_scale}, got {scale}." + ) + if self.total_cp_world_size > 1: + raise NotImplementedError( + "b12x does not yet support decode/prefill context parallelism." + ) + + self.num_heads = int(num_heads) + self.head_size = int(head_size) + self.output_head_size = self.head_size + self.scale = float(scale) + self.num_kv_heads = int(num_kv_heads) + self.num_queries_per_kv = self.num_heads // self.num_kv_heads + self.kv_cache_dtype = kv_cache_dtype + self.attn_type = attn_type + self.kv_sharing_target_layer_name = kv_sharing_target_layer_name + self.window_left = -1 if sliding_window is None else int(sliding_window) - 1 + + self._sinks_source = sinks + if sinks is not None and ( + sinks.ndim != 1 or int(sinks.shape[0]) != self.num_heads + ): + raise ValueError( + "b12x sinks must have shape " + f"[{self.num_heads}], got {tuple(sinks.shape)}." + ) + self.sinks = sinks if sinks is None or sinks.dtype == torch.float32 else None + + vllm_config = get_current_vllm_config() + scheduler_config = vllm_config.scheduler_config + model_config = vllm_config.model_config + cache_config = vllm_config.cache_config + spec_config = vllm_config.speculative_config + default_block_size = int(cache_config.block_size) + if default_block_size not in _B12X_SUPPORTED_PAGE_SIZES: + raise ValueError( + "b12x requires --block-size in " + f"{_B12X_SUPPORTED_PAGE_SIZES}, got " + f"{cache_config.block_size}." + ) + + self.device = torch.device("cuda", torch.accelerator.current_device_index()) + self.dtype = model_config.dtype + self.kv_torch_dtype = _dtype_from_cache_config(kv_cache_dtype, vllm_config) + if self.dtype != torch.bfloat16: + raise NotImplementedError("b12x currently requires bfloat16 queries.") + max_batched = int(scheduler_config.max_num_batched_tokens) + max_num_seqs = int(scheduler_config.max_num_seqs) + max_model_len = int(model_config.max_model_len) + self._max_num_seqs = max_num_seqs + max_page_table_widths = { + page_size: _max_page_table_width( + max_model_len, + page_size, + max_batched, + model_config.is_hybrid, + ) + for page_size in _B12X_SUPPORTED_PAGE_SIZES + } + + # Extend dispatch may depend on the static Q tensor capacity, but never + # on live per-request lengths. Keep a small set of capacity buckets so + # short/tail prefills do not replay the maximum 8K CTA grid. + self._extend_q_capacities = tuple( + sorted( + { + min(max_batched, q_capacity) + for q_capacity in (128, 512, 1024, 2048, 4096, max_batched) + if q_capacity > 0 + } + ) + ) + + paged_attention = get_b12x_paged_attention() + assert paged_attention is not None and paged_attention.is_supported() + + def _extend_work_items( + page_size: int, + q_capacity: int, + batch_size: int, + ) -> int: + capacity = paged_attention.extend_graph_capacity( + device=self.device, + q_dtype=self.dtype, + kv_dtype=self.kv_torch_dtype, + num_q_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + head_dim_qk=self.head_size, + head_dim_vo=self.output_head_size, + page_size=page_size, + batch=batch_size, + total_q_capacity=q_capacity, + max_cache_page_count=max_page_table_widths[page_size], + window_left=self.window_left, + ) + return capacity.max_work_items + + self._paged_attention = paged_attention + + def _make_plan( + page_size: int, + mode: str, + max_total_q: int, + max_batch: int, + max_work_items: int, + max_partial_rows: int, + use_cuda_graph: bool, + num_cache_pages: int, + copy_runtime_metadata: bool, + ) -> Any: + return paged_attention.plan( + paged_attention.Caps( + device=self.device, + mode=mode, + dtype=self.dtype, + kv_dtype=self.kv_torch_dtype, + num_q_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + head_dim_qk=self.head_size, + head_dim_vo=self.output_head_size, + page_size=page_size, + max_total_q=max_total_q, + max_batch=max_batch, + max_page_table_width=max_page_table_widths[page_size], + max_work_items=max_work_items, + max_partial_rows=max_partial_rows, + # Shape-only planning tensor; runtime cache shape is + # validated by head/page geometry, not page count. + num_cache_pages=num_cache_pages, + use_cuda_graph=use_cuda_graph, + copy_runtime_metadata=copy_runtime_metadata, + ) + ) + + capture_sizes = vllm_config.compilation_config.cudagraph_capture_sizes or [] + decode_plan_sizes = { + int(size) for size in capture_sizes if 0 < int(size) <= max_num_seqs + } + decode_plan_sizes.add(max_num_seqs) + + def _create_decode_plan(page_size: int, batch_size: int) -> Any: + max_page_table_width = max_page_table_widths[page_size] + capacity = paged_attention.decode_graph_capacity( + device=self.device, + q_dtype=self.dtype, + kv_dtype=self.kv_torch_dtype, + num_q_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + head_dim_qk=self.head_size, + head_dim_vo=self.output_head_size, + page_size=page_size, + batch=batch_size, + max_cache_page_count=max_page_table_width, + window_left=self.window_left, + ) + plan = _make_plan( + page_size, + "decode", + batch_size, + batch_size, + capacity.max_work_items, + capacity.max_partial_rows, + True, + max_page_table_width, + True, + ) + plan.prepare_decode_graph_replay_state( + batch=batch_size, + total_q_capacity=batch_size, + max_page_table_width=max_page_table_width, + max_cache_page_count=max_page_table_width, + window_left=self.window_left, + ) + return plan + + self._create_decode_plan = _create_decode_plan + self._verify_q_per_req = 0 + if spec_config is not None: + self._verify_q_per_req = 1 + int( + getattr(spec_config, "num_speculative_tokens", None) or 0 + ) + if self._verify_q_per_req <= 1: + self._verify_q_per_req = 0 + + def _create_verify_plan(page_size: int, batch_size: int) -> Any: + if self._verify_q_per_req <= 1: + raise RuntimeError("b12x verifier plan requested without speculation") + max_page_table_width = max_page_table_widths[page_size] + total_q = batch_size * self._verify_q_per_req + capacity = paged_attention.verify_graph_capacity( + device=self.device, + q_dtype=self.dtype, + kv_dtype=self.kv_torch_dtype, + num_q_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + head_dim_qk=self.head_size, + head_dim_vo=self.output_head_size, + page_size=page_size, + batch=batch_size, + query_len=self._verify_q_per_req, + max_cache_page_count=max_page_table_width, + window_left=self.window_left, + ) + plan = _make_plan( + page_size, + "verify", + total_q, + batch_size, + capacity.max_work_items, + capacity.max_partial_rows, + True, + max_page_table_width, + True, + ) + page_ids = torch.arange( + max_page_table_width, + dtype=torch.int32, + device=self.device, + ) + max_page_table = page_ids.unsqueeze(0).expand(batch_size, -1).contiguous() + max_cache_seqlens = torch.full( + (batch_size,), + capacity.representative_cache_seqlen, + dtype=torch.int32, + device=self.device, + ) + max_cu_seqlens_q = torch.arange( + 0, + total_q + 1, + self._verify_q_per_req, + dtype=torch.int32, + device=self.device, + ) + plan.prepare_graph_replay_state( + page_table=max_page_table, + cache_seqlens=max_cache_seqlens, + cu_seqlens_q=max_cu_seqlens_q, + active_total_q=total_q, + window_left=self.window_left, + ) + return plan + + self._create_verify_plan = _create_verify_plan + + def _create_extend_plan( + page_size: int, + batch_size: int, + q_capacity: int, + ) -> Any: + """Prepare a fixed-capacity extend plan without reading live lengths.""" + max_page_table_width = max_page_table_widths[page_size] + plan = _make_plan( + page_size, + "extend", + q_capacity, + batch_size, + _extend_work_items(page_size, q_capacity, batch_size), + 0, + True, + max_page_table_width, + False, + ) + page_ids = torch.arange( + max_page_table_width, + dtype=torch.int32, + device=self.device, + ) + max_page_table = page_ids.unsqueeze(0).expand(batch_size, -1).contiguous() + max_cache_seqlens = torch.full( + (batch_size,), + min(max_model_len, max_page_table_width * page_size), + dtype=torch.int32, + device=self.device, + ) + # Put one row in every request except the last, which owns the + # remainder. This represents the full total-Q capacity while the + # replay kernel remains responsible for packing arbitrary live + # per-request lengths from device cu_seqlens_q. + max_cu_seqlens_q = torch.arange( + 0, + batch_size + 1, + dtype=torch.int32, + device=self.device, + ) + max_cu_seqlens_q[-1] = q_capacity + plan.prepare_graph_replay_state( + page_table=max_page_table, + cache_seqlens=max_cache_seqlens, + cu_seqlens_q=max_cu_seqlens_q, + active_total_q=q_capacity, + window_left=self.window_left, + ) + return plan + + self._create_extend_plan = _create_extend_plan + decode_scratch_envelopes = { + page_size: paged_attention.decode_graph_scratch_envelope( + device=self.device, + q_dtype=self.dtype, + kv_dtype=self.kv_torch_dtype, + num_q_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + head_dim_qk=self.head_size, + head_dim_vo=self.output_head_size, + page_size=page_size, + max_batch=max_num_seqs, + max_page_table_width=max_page_table_widths[page_size], + max_cache_page_count=max_page_table_widths[page_size], + window_left=self.window_left, + copy_runtime_metadata=True, + ) + for page_size in _B12X_SUPPORTED_PAGE_SIZES + } + self._decode_plans: dict[tuple[int, int], Any] = {} + self._verify_plans: dict[tuple[int, int], Any] = {} + self._extend_plans: dict[tuple[int, int, int], Any] = {} + for page_size in _B12X_SUPPORTED_PAGE_SIZES: + for batch_size in sorted(decode_plan_sizes): + self._decode_plans[page_size, batch_size] = self._create_decode_plan( + page_size, batch_size + ) + if self._verify_q_per_req > 1: + for batch_size in range(1, max_num_seqs + 1): + self._verify_plans[page_size, batch_size] = ( + self._create_verify_plan(page_size, batch_size) + ) + for batch_size in range(1, max_num_seqs + 1): + for q_capacity in self._extend_q_capacities: + # Equal capacity is necessarily one query token per + # request, which is handled by the decode plan. + if batch_size >= q_capacity: + continue + self._extend_plans[page_size, batch_size, q_capacity] = ( + self._create_extend_plan( + page_size, + batch_size, + q_capacity, + ) + ) + self._scratch_nbytes = max( + *(int(envelope.nbytes) for envelope in decode_scratch_envelopes.values()), + *(int(plan.layout.nbytes) for plan in self._verify_plans.values()), + *(int(plan.layout.nbytes) for plan in self._extend_plans.values()), + ) + + current_workspace_manager().get_simultaneous( + ((self._scratch_nbytes,), torch.uint8), + ) + + self.supports_quant_query_input = False + register_cutedsl_warmup_provider(self) + + logger.info_once( + "Using b12x with q_heads=%d kv_heads=%d head_dim_qk=%d " + "head_dim_vo=%d window_left=%d planned_page_sizes=%s " + "verify_q_per_req=%d extend_q_capacities=%s scratch=%d bytes.", + self.num_heads, + self.num_kv_heads, + self.head_size, + self.output_head_size, + self.window_left, + _B12X_SUPPORTED_PAGE_SIZES, + self._verify_q_per_req, + self._extend_q_capacities, + self._scratch_nbytes, + ) + + def _compile_paged_extend_entry(self, page_size: int) -> None: + """Compile fixed-capacity paged-prefill entries without a live plan.""" + warmup_plans: list[tuple[int, int, Any, bool]] = [] + for batch_size in range(1, self._max_num_seqs + 1): + candidates = sorted( + (q_capacity, plan) + for (plan_page_size, plan_batch, q_capacity), plan in ( + self._extend_plans.items() + ) + if plan_page_size == page_size and plan_batch == batch_size + ) + for index, (q_capacity, plan) in enumerate(candidates): + warmup_plans.append((batch_size, q_capacity, plan, index == 0)) + if not warmup_plans: + return + + max_q_rows = max( + min(q_capacity, max(64, batch_size + 1)) + for batch_size, q_capacity, _, _ in warmup_plans + ) + q = torch.zeros( + (max_q_rows, self.num_heads, self.head_size), + dtype=self.dtype, + device=self.device, + ) + output = torch.zeros( + (max_q_rows, self.num_heads, self.output_head_size), + dtype=self.dtype, + device=self.device, + ) + kv_cache = torch.zeros( + (1, 2, page_size, self.num_kv_heads * self.head_size), + dtype=self.kv_torch_dtype, + device=self.device, + ) + key_cache, value_cache = self._kv_cache_views(kv_cache) + (scratch_storage,) = current_workspace_manager().get_simultaneous( + ((self._scratch_nbytes,), torch.uint8), + ) + for batch_size, q_capacity, plan, execute in warmup_plans: + q_rows = min(q_capacity, max(64, batch_size + 1)) + page_table = torch.zeros( + (batch_size, plan.caps.max_page_table_width), + dtype=torch.int32, + device=self.device, + ) + cache_seqlens = torch.full( + (batch_size,), page_size, dtype=torch.int32, device=self.device + ) + cu_seqlens_q = torch.arange( + 0, + batch_size + 1, + dtype=torch.int32, + device=self.device, + ) + cu_seqlens_q[-1] = q_rows + k_descale = None + v_descale = None + if _is_b12x_fp8_kv_cache(self.kv_cache_dtype): + k_descale = torch.ones( + (), dtype=torch.float32, device=self.device + ).expand(batch_size) + v_descale = torch.ones( + (), dtype=torch.float32, device=self.device + ).expand(batch_size) + binding = plan.bind( + scratch=scratch_storage, + q=q[:q_rows], + k_cache=key_cache, + v_cache=value_cache, + output=output[:q_rows], + page_table=page_table, + cache_seqlens=cache_seqlens, + cu_seqlens_q=cu_seqlens_q, + window_left=self.window_left, + attention_sink_bias=self.sinks, + k_descale=k_descale, + v_descale=v_descale, + ) + self._paged_attention.compile(binding=binding) + if execute: + # Compile-only warmup does not launch the device-side compact + # scheduler. One execution per batch covers the capture-static + # metadata variant shared by its Q-capacity plans. + self._paged_attention.run(binding=binding) + + def get_cutedsl_warmup_compile_units(self) -> tuple[CuTeDSLCompileUnit, ...]: + common_key = ( + "b12x_paged_extend", + str(self.device), + str(self.dtype), + str(self.kv_torch_dtype), + self.num_heads, + self.num_kv_heads, + self.head_size, + self.output_head_size, + self.window_left, + self.sinks is not None, + ) + return tuple( + CuTeDSLCompileUnit( + name="b12x_paged_extend", + key=(*common_key, page_size), + compile=partial(self._compile_paged_extend_entry, page_size), + ) + for page_size in _B12X_SUPPORTED_PAGE_SIZES + ) + + def process_weights_after_loading(self, act_dtype: torch.dtype) -> None: + del act_dtype + source_sinks = self._sinks_source + if source_sinks is None: + return + if source_sinks.dtype == torch.float32: + self.sinks = source_sinks + elif self.sinks is None or self.sinks.dtype != torch.float32: + self.sinks = source_sinks.to(torch.float32) + else: + self.sinks.copy_(source_sinks) + + def _prepare_fp8_descales( + self, + layer: AttentionLayer, + num_reqs: int, + device: torch.device, + ) -> tuple[torch.Tensor | None, torch.Tensor | None]: + if not _is_b12x_fp8_kv_cache(self.kv_cache_dtype): + return None, None + if num_reqs <= 0: + raise ValueError("b12x fp8 KV descale request count must be positive.") + + def _prepare(scale: torch.Tensor, name: str) -> torch.Tensor: + if scale.device != device: + raise RuntimeError(f"b12x {name} must be on the query device.") + if scale.dtype != torch.float32: + raise RuntimeError(f"b12x {name} must be float32.") + if scale.ndim == 0: + return scale.expand(num_reqs) + if scale.ndim == 1: + if int(scale.shape[0]) == 1: + return scale.expand(num_reqs) + if int(scale.shape[0]) >= num_reqs: + return scale[:num_reqs] + raise ValueError( + f"b12x {name} must be scalar or rank-1 with at least " + f"{num_reqs} values; got shape {tuple(scale.shape)}." + ) + + return _prepare(layer._k_scale, "k_scale"), _prepare(layer._v_scale, "v_scale") + + def _kv_cache_views( + self, + kv_cache: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor]: + key_cache, value_cache = kv_cache.unbind(1) + key_cache = key_cache.unflatten(-1, (self.num_kv_heads, self.head_size)) + value_cache = value_cache.unflatten(-1, (self.num_kv_heads, self.head_size)) + key_cache = canonicalize_singleton_dim_strides(key_cache) + value_cache = canonicalize_singleton_dim_strides(value_cache) + if _is_b12x_fp8_kv_cache(self.kv_cache_dtype): + fp8_dtype = current_platform.fp8_dtype() + if key_cache.dtype == torch.uint8: + key_cache = key_cache.view(fp8_dtype) + if value_cache.dtype == torch.uint8: + value_cache = value_cache.view(fp8_dtype) + if ( + key_cache.dtype != self.kv_torch_dtype + or value_cache.dtype != self.kv_torch_dtype + ): + raise TypeError( + f"b12x plan expects KV dtype {self.kv_torch_dtype}, got " + f"{key_cache.dtype}/{value_cache.dtype}." + ) + return key_cache, value_cache + + def _select_plan( + self, + attn_metadata: B12xPagedMetadata, + total_q: int, + q_capacity: int, + num_reqs: int, + page_size: int, + ) -> Any: + if attn_metadata.max_query_len <= 1 and int(total_q) == int(num_reqs): + batch_size = int(total_q) + plan_key = (page_size, batch_size) + plan = self._decode_plans.get(plan_key) + if plan is None: + if _capture_alloc_forbidden(): + raise RuntimeError( + "b12x decode plan was not prepared before CUDA graph " + f"capture for page size {page_size}, batch size " + f"{batch_size}." + ) + plan = self._create_decode_plan(page_size, batch_size) + if int(plan.layout.nbytes) > self._scratch_nbytes: + raise RuntimeError( + "b12x lazily created decode plan exceeds reserved " + f"scratch: {int(plan.layout.nbytes)} > " + f"{self._scratch_nbytes} bytes." + ) + self._decode_plans[plan_key] = plan + return plan + elif ( + self._verify_q_per_req > 1 + and attn_metadata.max_query_len == self._verify_q_per_req + and int(total_q) == int(num_reqs) * self._verify_q_per_req + ): + plan_key = (page_size, int(num_reqs)) + plan = self._verify_plans.get(plan_key) + if plan is None: + if _capture_alloc_forbidden(): + raise RuntimeError( + "b12x verifier plan was not prepared before CUDA " + f"graph capture for page size {page_size}, batch size " + f"{num_reqs}." + ) + plan = self._create_verify_plan(page_size, int(num_reqs)) + if int(plan.layout.nbytes) > self._scratch_nbytes: + raise RuntimeError( + "b12x lazily created verifier plan exceeds reserved " + f"scratch: {int(plan.layout.nbytes)} > " + f"{self._scratch_nbytes} bytes." + ) + self._verify_plans[plan_key] = plan + return plan + extend_q_capacity = next( + ( + capacity + for capacity in self._extend_q_capacities + if q_capacity <= capacity + ), + None, + ) + if extend_q_capacity is None: + raise ValueError( + f"b12x extend Q capacity {q_capacity} exceeds prepared " + f"maximum {self._extend_q_capacities[-1]}." + ) + return self._extend_plans[ + page_size, + int(num_reqs), + extend_q_capacity, + ] + + def forward( + self, + layer: AttentionLayer, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + attn_metadata: B12xPagedMetadata, + output: torch.Tensor, + output_scale: torch.Tensor | None = None, + output_block_scale: torch.Tensor | None = None, + ) -> torch.Tensor: + del key, value + if output_scale is not None or output_block_scale is not None: + raise NotImplementedError( + "b12x does not support fused output quantization." + ) + if attn_metadata is None: + return output.fill_(0) + if output.shape[-1] != self.output_head_size: + raise ValueError( + f"b12x expected output head dim {self.output_head_size}, got " + f"{output.shape[-1]}." + ) + if kv_cache.numel() == 0: + return output.fill_(0) + + # In FULL cudagraph mode vLLM may pad attention metadata to the graph + # bucket while still passing per-layer Q/output tensors with only the + # real rows. Use tensor capacity as the launch contract and avoid + # selecting decode graph replay for padded virtual requests. + q_capacity = min( + int(query.shape[0]), + int(output.shape[0]), + ) + num_actual_tokens = min( + int(attn_metadata.num_actual_tokens), + q_capacity, + ) + if num_actual_tokens <= 0: + return output + q = query[:num_actual_tokens] + out = output[:num_actual_tokens] + if q.dtype != self.dtype or out.dtype != self.dtype: + raise TypeError( + f"b12x plan expects dtype {self.dtype}, got " + f"q={q.dtype}, output={out.dtype}." + ) + + key_cache, value_cache = self._kv_cache_views(kv_cache) + page_size = _kv_page_size(key_cache, value_cache) + if not attn_metadata.causal: + raise NotImplementedError("b12x supports causal attention only.") + + page_table = _ensure_i32_contiguous(attn_metadata.block_table, "block_table") + cache_seqlens = _ensure_i32_contiguous(attn_metadata.seq_lens, "seq_lens") + cu_seqlens_q = _ensure_i32_contiguous( + attn_metadata.query_start_loc, + "query_start_loc", + ) + num_reqs = int(cache_seqlens.shape[0]) + if attn_metadata.max_query_len <= 1 and num_actual_tokens < num_reqs: + num_reqs = num_actual_tokens + page_table = page_table[:num_reqs] + cache_seqlens = cache_seqlens[:num_reqs] + cu_seqlens_q = cu_seqlens_q[: num_reqs + 1] + k_descale, v_descale = self._prepare_fp8_descales( + layer, + num_reqs, + q.device, + ) + plan = self._select_plan( + attn_metadata, + num_actual_tokens, + q_capacity, + num_reqs, + page_size, + ) + (scratch_storage,) = current_workspace_manager().get_simultaneous( + ((self._scratch_nbytes,), torch.uint8), + ) + binding = plan.bind( + scratch=scratch_storage, + q=q, + k_cache=key_cache, + v_cache=value_cache, + output=out, + page_table=page_table, + cache_seqlens=cache_seqlens, + cu_seqlens_q=cu_seqlens_q, + window_left=self.window_left, + active_total_q=(None if plan.caps.mode == "extend" else num_actual_tokens), + attention_sink_bias=self.sinks, + k_descale=k_descale, + v_descale=v_descale, + ) + self._paged_attention.run(binding=binding) + return output + + def do_kv_cache_update( + self, + layer: AttentionLayer, + key: torch.Tensor, + value: torch.Tensor, + kv_cache: torch.Tensor, + slot_mapping: torch.Tensor, + ) -> None: + if kv_cache.numel() == 0: + return + key_cache, value_cache = self._kv_cache_views(kv_cache) + torch.ops._C_cache_ops.reshape_and_cache_flash( + key, + value, + key_cache, + value_cache, + slot_mapping, + self.kv_cache_dtype, + layer._k_scale, + layer._v_scale, + ) diff --git a/vllm/v1/attention/backends/registry.py b/vllm/v1/attention/backends/registry.py index 6fc9dd1e1f..fa8a445353 100644 --- a/vllm/v1/attention/backends/registry.py +++ b/vllm/v1/attention/backends/registry.py @@ -94,6 +94,7 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta): ROCM_FLASHMLA_SPARSE_DSV4 = ( "vllm.models.deepseek_v4.amd.rocm.DeepseekV4ROCMAiterMLASparseBackend" ) + B12X = "vllm.v1.attention.backends.b12x.B12xPagedAttentionBackend" FLASH_ATTN_MLA = "vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend" FLASH_ATTN_MLA_SPARSE = ( "vllm.v1.attention.backends.mla.flashattn_mla_sparse.FlashAttnMLASparseBackend"