[Kernel] Add B12X causal paged attention backend (#52017)

Signed-off-by: Luke Alonso <lalonso@gmail.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Luke Alonso
2026-09-01 10:42:02 -04:00
committed by GitHub
co-authored by Lucas Wilkinson
parent adebc41b7e
commit 0d4ad47981
8 changed files with 1527 additions and 5 deletions
+1 -1
View File
@@ -389,7 +389,7 @@ steps:
- tests/quantization/reference_mxfp4.py
- tests/quantization/test_auto_round.py
commands:
- uv pip install --system b12x==1.2.6
- uv pip install --system b12x==1.3.0
- pytest -v -s model_executor/kernels/test_b12x_linear.py
model_executor/test_b12x_warmup.py
kernels/quantization/test_block_fp8.py
+10
View File
@@ -136,6 +136,16 @@ Priority is **1 = highest** (tried first).
> `--block-size` that is not a multiple of 128 instead makes FlashAttention ineligible
> for such models, which is an error if the backend was requested explicitly.
### b12x
The optional [b12x](https://pypi.org/project/b12x/) backend supports causal
decoder attention on NVIDIA SM120 and SM121 GPUs. Install and select it with:
```bash
uv pip install "vllm[b12x]"
vllm serve <model> --attention-backend b12x
```
## MiniMax M3 Sparse Attention Backends
Block-sparse GQA backend used by MiniMax M3 sparse ("lightning indexer")
+1 -1
View File
@@ -1522,7 +1522,7 @@ setup(
# only; also needs system GStreamer + libv4l (see docs).
"deepstream": ["nvidia-deepstream-videodecode-cu13>=9.0.2"],
"flashinfer": [], # Kept for backwards compatibility
"b12x": ["b12x==1.2.6"],
"b12x": ["b12x==1.3.0"],
# Optional deps for Helion kernel development
# NOTE: When updating helion version, also update CI files:
# - .buildkite/test_areas/kernels.yaml
+53 -3
View File
@@ -3,6 +3,7 @@
"""Tests for v1 attention backends without GPUModelRunner dependency."""
from functools import partial
from types import SimpleNamespace
import pytest
import torch
@@ -282,6 +283,7 @@ def run_attention_backend(
sliding_window: int | None = None,
kv_cache_dtype: str = "auto",
sinks: torch.Tensor | None = None,
use_cuda_graph: bool = False,
) -> torch.Tensor:
"""Run attention computation using the specified backend's AttentionImpl."""
@@ -367,9 +369,26 @@ def run_attention_backend(
impl.do_kv_cache_update(
mock_layer, key, value, kv_cache, attn_metadata.slot_mapping
)
output = impl.forward(
mock_layer, query, key, value, kv_cache, attn_metadata, output=output
)
if use_cuda_graph:
impl.forward(
mock_layer, query, key, value, kv_cache, attn_metadata, output=output
)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
impl.forward(
mock_layer,
query,
key,
value,
kv_cache,
attn_metadata,
output=output,
)
graph.replay()
else:
output = impl.forward(
mock_layer, query, key, value, kv_cache, attn_metadata, output=output
)
return output
@@ -389,6 +408,11 @@ def _test_backend_correctness(
kv_cache_dtype: str = "auto",
use_sinks: bool = False,
layout: KVCacheLayout | None = None,
use_cuda_graph: bool = False,
num_speculative_tokens: int = 0,
model_dtype: torch.dtype | None = None,
max_num_seqs: int | None = None,
max_num_batched_tokens: int | None = None,
):
"""
Test that all backends produce similar outputs to a reference implementation
@@ -433,10 +457,19 @@ def _test_backend_correctness(
model_name=model,
tensor_parallel_size=1, # Always use TP=1 to avoid multi-GPU requirements
max_model_len=max(batch_spec.seq_lens),
dtype=model_dtype or "auto",
block_size=block_size,
num_gpu_blocks=8192,
hf_config_override=hf_config_override,
)
if max_num_seqs is not None:
vllm_config.scheduler_config.max_num_seqs = max_num_seqs
if max_num_batched_tokens is not None:
vllm_config.scheduler_config.max_num_batched_tokens = max_num_batched_tokens
if num_speculative_tokens > 0:
vllm_config.speculative_config = SimpleNamespace(
num_speculative_tokens=num_speculative_tokens
)
vllm_config.cache_config.cache_dtype = kv_cache_dtype
device = torch.device(f"{DEVICE_TYPE}:0")
@@ -619,6 +652,22 @@ def _test_backend_correctness(
backend_layout = backend_supported[0]
kv_cache_for_backend = _clone_kv_cache_in_layout(kv_cache, backend_layout)
if backend_name == AttentionBackendEnum.B12X:
cache_dtype = (
FP8_KV_CACHE_DTYPES[kv_cache_dtype]
if is_quantized_kv_cache(kv_cache_dtype)
else kv_cache.dtype
)
typed_cache = kv_cache.view(cache_dtype)
key_cache = typed_cache[..., :head_size].permute(0, 2, 1, 3)
value_cache = typed_cache[..., head_size:].permute(0, 2, 1, 3)
packed_cache = torch.stack((key_cache, value_cache), dim=1).flatten(-2)
if is_quantized_kv_cache(kv_cache_dtype):
packed_cache = packed_cache.view(torch.uint8)
kv_cache_for_backend = _clone_kv_cache_in_layout(
packed_cache, backend_layout
)
# FlashInfer reads the layout at plan time; set it to match
# the physical order of the test cache.
vllm_config.cache_config.kv_cache_layout = backend_layout.name
@@ -638,6 +687,7 @@ def _test_backend_correctness(
attn_type=attn_type,
kv_cache_dtype=kv_cache_dtype,
sinks=sinks,
use_cuda_graph=use_cuda_graph,
)
# Check shape and dtype consistency
+360
View File
@@ -0,0 +1,360 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
from types import SimpleNamespace
import pytest
import torch
from tests.v1.attention.test_attention_backends import (
BATCH_SPECS,
_test_backend_correctness,
)
from tests.v1.attention.utils import BatchSpec
from vllm.config import ModelConfig
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
from vllm.utils.b12x import get_b12x_paged_attention
from vllm.v1.attention.backends import b12x
from vllm.v1.attention.backends.b12x import (
B12xPagedAttentionBackend,
B12xPagedAttentionImpl,
_kv_page_size,
_max_page_table_width,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheLayout
def _require_b12x_paged_attention() -> None:
capability = current_platform.get_device_capability()
if (
not current_platform.is_cuda()
or capability is None
or not B12xPagedAttentionBackend.supports_compute_capability(capability)
):
pytest.skip("b12x paged attention requires SM120 or SM121.")
paged_attention = get_b12x_paged_attention()
if paged_attention is None or not paged_attention.is_supported():
pytest.skip("b12x paged attention is not available.")
def _causal_mask(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
):
return q_idx + context_len >= kv_idx
def _causal_sliding_window_mask(
b: torch.Tensor,
h: torch.Tensor,
q_idx: torch.Tensor,
kv_idx: torch.Tensor,
*,
context_len: int,
sliding_window: int,
):
causal_mask = q_idx + context_len >= kv_idx
window_mask = q_idx + context_len - kv_idx < sliding_window
return causal_mask & window_mask
@pytest.mark.parametrize(
("dtype", "kv_cache_dtype", "paged_attention", "expected_reason"),
[
pytest.param(
torch.float16,
"fp8_e4m3",
None,
"b12x currently requires bfloat16 queries",
id="query-dtype",
),
pytest.param(
torch.bfloat16,
"float16",
None,
"b12x does not support float16 KV cache",
id="kv-cache-dtype",
),
pytest.param(
torch.bfloat16,
"auto",
None,
"Install the b12x backend with `pip install vllm[b12x]`",
id="package-not-installed",
),
pytest.param(
torch.bfloat16,
"auto",
SimpleNamespace(is_supported=lambda: False),
"b12x paged attention is not supported on the current device",
id="device-api",
),
pytest.param(
torch.bfloat16,
"auto",
SimpleNamespace(is_supported=lambda: True),
None,
id="supported",
),
],
)
def test_b12x_attention_config_support(
monkeypatch: pytest.MonkeyPatch,
dtype: torch.dtype,
kv_cache_dtype: str,
paged_attention,
expected_reason: str | None,
) -> None:
monkeypatch.setattr(
b12x,
"get_b12x_paged_attention",
lambda: paged_attention,
)
reason = B12xPagedAttentionBackend.supports_combination(
head_size=128,
dtype=dtype,
kv_cache_dtype=kv_cache_dtype,
block_size=128,
use_mla=False,
has_sink=False,
use_sparse=False,
use_mm_prefix=False,
device_capability=DeviceCapability(12, 0),
)
assert reason == expected_reason
def test_b12x_attention_uses_two_plane_nhd_cache() -> None:
spec = B12xPagedAttentionBackend.customize_spec(
FullAttentionSpec(
block_size=128,
num_kv_heads=4,
head_size=128,
dtype=torch.bfloat16,
)
)
assert spec.num_head_slots == 2
assert spec.state_content_bytes == 4 * 128 * 2
assert B12xPagedAttentionBackend.supported_kv_cache_layouts() == (
KVCacheLayout.LBHNC,
KVCacheLayout.BLHNC,
)
assert B12xPagedAttentionBackend.supports_block_size(128)
assert not B12xPagedAttentionBackend.supports_block_size(32)
def test_b12x_attention_hybrid_cache_capacity_includes_expansion() -> None:
assert _max_page_table_width(4096, 128, 4096, False) == 32
assert _max_page_table_width(4096, 128, 4096, True) == 64
def test_b12x_attention_runtime_page_size_comes_from_cache() -> None:
key_cache = torch.empty((3, 64, 4, 128), device="meta")
value_cache = torch.empty_like(key_cache)
assert _kv_page_size(key_cache, value_cache) == 64
with pytest.raises(ValueError, match="matching K/V page sizes"):
_kv_page_size(key_cache, torch.empty((3, 128, 4, 128), device="meta"))
def test_b12x_attention_lazily_prepares_decode_bucket(monkeypatch) -> None:
impl = object.__new__(B12xPagedAttentionImpl)
plan = SimpleNamespace(layout=SimpleNamespace(nbytes=96))
created: list[tuple[int, int]] = []
def create_plan(page_size: int, batch_size: int) -> SimpleNamespace:
created.append((page_size, batch_size))
return plan
impl._decode_plans = {}
impl._create_decode_plan = create_plan
impl._scratch_nbytes = 128
impl._extend_plans = {}
impl._verify_q_per_req = 0
metadata = SimpleNamespace(max_query_len=1)
monkeypatch.setattr(b12x, "_capture_alloc_forbidden", lambda: False)
assert impl._select_plan(metadata, 7, 7, 7, 64) is plan
assert impl._select_plan(metadata, 7, 7, 7, 64) is plan
assert created == [(64, 7)]
def test_b12x_attention_fp8_descales_follow_request_batch() -> None:
impl = object.__new__(B12xPagedAttentionImpl)
impl.kv_cache_dtype = "fp8_e4m3"
layer = SimpleNamespace(
_k_scale=torch.tensor(2.0),
_v_scale=torch.tensor([3.0, 4.0, 5.0]),
)
k_descale, v_descale = impl._prepare_fp8_descales(
layer, num_reqs=2, device=torch.device("cpu")
)
torch.testing.assert_close(k_descale, torch.tensor([2.0, 2.0]))
torch.testing.assert_close(v_descale, torch.tensor([3.0, 4.0]))
assert k_descale.stride() == (0,)
assert v_descale.stride() == (1,)
def test_b12x_attention_sinks_refresh_in_place_after_reload() -> None:
impl = object.__new__(B12xPagedAttentionImpl)
source = torch.tensor([1.0, 2.0], dtype=torch.bfloat16)
impl._sinks_source = source
impl.sinks = None
impl.process_weights_after_loading(torch.bfloat16)
assert impl.sinks is not None
sinks_ptr = impl.sinks.data_ptr()
source.copy_(torch.tensor([3.0, 4.0], dtype=torch.bfloat16))
impl.process_weights_after_loading(torch.bfloat16)
assert impl.sinks.data_ptr() == sinks_ptr
torch.testing.assert_close(impl.sinks, source.float())
@pytest.mark.parametrize(
"batch_spec_name",
["small_decode", "small_prefill", "mixed_small", "medium_decode"],
)
@pytest.mark.parametrize(
("kv_cache_dtype", "model_dtype"),
[
("auto", None),
("bfloat16", torch.bfloat16),
("fp8_e4m3", torch.bfloat16),
],
)
@pytest.mark.parametrize("block_size", [64, 128])
def test_b12x_causal_backend_correctness(
default_vllm_config,
workspace_init,
batch_spec_name: str,
kv_cache_dtype: str,
model_dtype: torch.dtype | None,
block_size: int,
) -> None:
"""b12x causal paged attention matches the shared SDPA reference."""
_require_b12x_paged_attention()
batch_spec = BATCH_SPECS[batch_spec_name]
_test_backend_correctness(
batch_spec,
"Qwen/Qwen3-0.6B",
[AttentionBackendEnum.B12X],
_causal_mask,
block_size=block_size,
kv_cache_dtype=kv_cache_dtype,
model_dtype=model_dtype,
max_num_seqs=batch_spec.batch_size,
max_num_batched_tokens=max(sum(batch_spec.query_lens), 64),
)
@pytest.mark.parametrize(
"batch_spec",
[
pytest.param(BatchSpec(seq_lens=[2080, 2200], query_lens=[1, 1]), id="decode"),
pytest.param(BatchSpec(seq_lens=[2080, 2200], query_lens=[8, 8]), id="prefill"),
],
)
def test_b12x_causal_sliding_window(
default_vllm_config,
workspace_init,
batch_spec: BatchSpec,
) -> None:
"""b12x causal sliding-window attention matches the shared reference."""
_require_b12x_paged_attention()
model = "microsoft/Phi-tiny-MoE-instruct"
sliding_window = ModelConfig(
model=model, max_model_len=max(batch_spec.seq_lens)
).get_sliding_window()
assert sliding_window is not None
mask = partial(_causal_sliding_window_mask, sliding_window=sliding_window)
_test_backend_correctness(
batch_spec,
model,
[AttentionBackendEnum.B12X],
mask,
block_size=64,
atol=3e-2,
rtol=3e-2,
max_num_seqs=batch_spec.batch_size,
max_num_batched_tokens=max(sum(batch_spec.query_lens), 64),
)
def test_b12x_attention_sinks(
default_vllm_config,
workspace_init,
) -> None:
"""b12x attention sinks match the explicit sink reference."""
_require_b12x_paged_attention()
batch_spec = BATCH_SPECS["small_prefill"]
_test_backend_correctness(
batch_spec,
"Qwen/Qwen3-0.6B",
[AttentionBackendEnum.B12X],
_causal_mask,
block_size=64,
atol=3e-2,
rtol=3e-2,
use_sinks=True,
max_num_seqs=batch_spec.batch_size,
max_num_batched_tokens=max(sum(batch_spec.query_lens), 64),
)
def test_b12x_decode_cuda_graph_replay(
default_vllm_config,
workspace_init,
) -> None:
"""b12x decode output remains correct after CUDA graph replay."""
_require_b12x_paged_attention()
batch_spec = BATCH_SPECS["small_decode"]
_test_backend_correctness(
batch_spec,
"Qwen/Qwen3-0.6B",
[AttentionBackendEnum.B12X],
_causal_mask,
block_size=64,
use_cuda_graph=True,
max_num_seqs=batch_spec.batch_size,
max_num_batched_tokens=max(sum(batch_spec.query_lens), 64),
)
def test_b12x_speculative_verification_uses_cuda_graph_plan(
default_vllm_config,
workspace_init,
) -> None:
"""Exercise the verifier plan rather than the general extend plan."""
_require_b12x_paged_attention()
batch_spec = BatchSpec(seq_lens=[32, 40], query_lens=[4, 4])
_test_backend_correctness(
batch_spec,
"Qwen/Qwen3-0.6B",
[AttentionBackendEnum.B12X],
_causal_mask,
block_size=128,
num_speculative_tokens=3,
use_cuda_graph=True,
max_num_seqs=batch_spec.batch_size,
max_num_batched_tokens=max(sum(batch_spec.query_lens), 64),
)
+5
View File
@@ -34,6 +34,7 @@ def _import_submodule(module_name: str) -> ModuleType | None:
_B12X_SUBMODULES = {
module_name: _import_submodule(module_name)
for module_name in (
"b12x.attention.paged",
"b12x.gemm.blockscaled",
# TODO: Remove once B12X exposes the scale-swizzle API publicly.
"b12x._lib.intrinsics",
@@ -73,6 +74,10 @@ def get_b12x_fused_moe() -> ModuleType | None:
return _get_submodule("b12x.moe.fused_moe")
def get_b12x_paged_attention() -> ModuleType | None:
return _get_submodule("b12x.attention.paged")
def b12x_warmup_token_counts(
*,
max_tokens: int,
File diff suppressed because it is too large Load Diff
+1
View File
@@ -94,6 +94,7 @@ class AttentionBackendEnum(Enum, metaclass=_AttentionBackendEnumMeta):
ROCM_FLASHMLA_SPARSE_DSV4 = (
"vllm.models.deepseek_v4.amd.rocm.DeepseekV4ROCMAiterMLASparseBackend"
)
B12X = "vllm.v1.attention.backends.b12x.B12xPagedAttentionBackend"
FLASH_ATTN_MLA = "vllm.v1.attention.backends.mla.flashattn_mla.FlashAttnMLABackend"
FLASH_ATTN_MLA_SPARSE = (
"vllm.v1.attention.backends.mla.flashattn_mla_sparse.FlashAttnMLASparseBackend"