[Model] add GLM-5.3-Flash support (#53906)

This commit is contained in:
Jiangyun Zhu
2026-09-03 09:40:35 -07:00
committed by GitHub
parent da8ec28268
commit 98ed0856f3
94 changed files with 17927 additions and 266 deletions
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for error handling in the CUDA attention backend probe.
"""Tests for CUDA attention backend selection and probe error handling.
Environment-shaped probe failures (missing packages, unreadable caches,
broken driver installs) must mark the backend unavailable rather than
@@ -89,6 +89,48 @@ def test_get_valid_backends_keeps_probing_after_failure():
assert valid
def test_sm90_nope_mla_prefers_flashinfer_without_changing_rope_order():
backend_cls = MagicMock()
backend_cls.validate_configuration.return_value = []
sparse_backends = {
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
AttentionBackendEnum.FLASHMLA_SPARSE,
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
}
def sparse_order(head_size: int) -> list[AttentionBackendEnum]:
config = SELECTOR_CONFIG._replace(
head_size=head_size,
use_mla=True,
use_sparse=True,
)
with patch(
"vllm.platforms.cuda._get_attn_backend_class",
return_value=backend_cls,
):
valid, _ = CudaPlatform.get_valid_backends(
device_capability=SM90,
attn_selector_config=config,
num_heads=32,
)
return [
candidate.backend
for candidate in valid
if candidate.backend in sparse_backends
]
assert sparse_order(512) == [
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
AttentionBackendEnum.FLASHMLA_SPARSE,
]
assert sparse_order(576) == [
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
AttentionBackendEnum.FLASHMLA_SPARSE,
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
]
def test_selected_backend_probe_failure_raises_value_error_with_cause():
exc = OSError("libcuda.so.1: cannot open shared object file")
with (
@@ -0,0 +1,280 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU tests for the FlashInfer SM90 sparse MLA backend wiring (no GPU).
The FlashInfer wrapper and top-k conversion are replaced by CPU recorders;
the tests pin the contract between the impl and the kernel API: page_size=1
varlen rows, reserved-buffer refresh, plan parameters (dims, NoPE/rope scale,
causality), ckv/kpe cache splitting, and the backend's model-shape gates.
"""
from types import SimpleNamespace
import pytest
import torch
# isort: off
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm90 as sm90_mod
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm90 import (
FlashInferMLASparseSM90Backend,
FlashInferMLASparseSM90Builder,
FlashInferMLASparseSM90Impl,
)
# isort: on
BLOCK_SIZE = 64
HEAD = 512
TOPK = 128 # triton convert requires width % 128 == 0
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
out = torch.full_like(token_indices, -1)
counts = torch.zeros(token_indices.shape[0], dtype=torch.int32)
for t in range(token_indices.shape[0]):
vals = []
for j in range(token_indices.shape[1]):
pos = int(token_indices[t, j])
if pos == -1:
continue
blk = int(block_table[int(req_id[t]), pos // BLOCK_SIZE])
if blk < 0:
continue
vals.append(blk * BLOCK_SIZE + pos % BLOCK_SIZE)
out[t, : len(vals)] = torch.tensor(vals, dtype=out.dtype)
counts[t] = len(vals)
return out, counts
class FakeWrapper:
def __init__(self):
self.plan_args = None
self.run_args = None
def plan(self, *args, **kwargs):
self.plan_args = (args, kwargs)
def run(self, *args, **kwargs):
q_nope, q_pe, ckv, kpe = args
self.run_args = (q_nope, q_pe, ckv, kpe, kwargs)
return torch.zeros(
q_nope.shape[0], q_nope.shape[1], ckv.shape[-1], dtype=torch.bfloat16
)
class FakeState:
def __init__(self, width, max_tokens=64):
self.kv_indices = torch.zeros(max_tokens * width, dtype=torch.int32)
self.kv_len_arr = torch.zeros(max_tokens, dtype=torch.int32)
self.wrapper = FakeWrapper()
self.plan_calls = []
def plan(self, num_tokens, kv_lens):
self.plan_calls.append((num_tokens, kv_lens))
def make_impl(qk_rope, kv_dtype="fp8_e4m3", num_heads=2, topk_width=TOPK):
impl = object.__new__(FlashInferMLASparseSM90Impl)
impl.num_heads = num_heads
impl.head_size = HEAD + qk_rope
impl.scale = (HEAD + qk_rope) ** -0.5
impl.kv_lora_rank = HEAD
impl.qk_rope_head_dim = qk_rope
impl.kv_cache_dtype = kv_dtype
impl.use_fp8_kv_cache = kv_dtype in ("fp8", "fp8_e4m3")
rows = 4
impl.topk_indices_buffer = torch.full((rows, topk_width), -1, dtype=torch.int32)
return impl, rows
def make_batch(rows, topk_rows, own_blocks):
req_id = torch.tensor([0] * rows, dtype=torch.int32)
block_table = torch.zeros(1, 16, dtype=torch.int32)
block_table[:, 0] = own_blocks[0]
topk = torch.full((rows, TOPK), -1, dtype=torch.int32)
for t, row in enumerate(topk_rows):
topk[t, : len(row)] = torch.tensor(row, dtype=torch.int32)
return SimpleNamespace(
req_id_per_token=req_id, block_table=block_table, block_size=BLOCK_SIZE
)
@pytest.mark.parametrize("qk_rope,kv_dtype", [(0, "fp8_e4m3"), (64, "auto")])
def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
impl, rows = make_impl(qk_rope, kv_dtype)
state = FakeState(TOPK)
monkeypatch.setattr(
sm90_mod, "triton_convert_req_index_to_global_index", ref_convert
)
# req with context 10 < topk: 8 valid + -1 padding.
topk_rows = [
[7, 3, 1, 9, 0, 2, 5, 8] + [-1] * (TOPK - 8),
[4, 0, 2, 3, 1] + [-1] * (TOPK - 5),
[6, 5, 4] + [-1] * (TOPK - 3),
[2] + [-1] * (TOPK - 1),
]
meta = make_batch(rows, topk_rows, [3])
meta.state = state
q_nope = torch.randn(rows, impl.num_heads, HEAD)
q_rope = torch.randn(rows, impl.num_heads, qk_rope)
cache = torch.zeros(
8 * BLOCK_SIZE,
impl.head_size,
dtype=torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16,
)
out, lse = impl.forward_mqa(
(q_nope, q_rope), cache, meta, SimpleNamespace(_k_scale_float=0.5)
)
assert lse is None and out.shape == (rows, impl.num_heads, HEAD)
# Reserved buffers carry this step's slots; lengths are NOT refreshed
# here (the builder plans them host-side before capture/replay).
ref_slots, ref_counts = ref_convert(
meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer
)
width = TOPK
got_slots = state.kv_indices[: rows * width].view(rows, width)
for t in range(rows):
k = int(ref_counts[t])
assert got_slots[t, :k].tolist() == ref_slots[t, :k].tolist()
assert state.plan_calls == []
assert state.wrapper.run_args is not None
q_pe, ckv, kpe, kwargs = state.wrapper.run_args[1:]
assert q_pe.shape == (rows, impl.num_heads, qk_rope)
assert ckv.shape == (8 * BLOCK_SIZE, 1, HEAD)
assert kpe.shape[-1] == qk_rope
if impl.use_fp8_kv_cache:
assert kwargs["ckv_scale"] == 0.5 and kwargs["kpe_scale"] == 1.0
else:
assert kwargs == {}
def test_builder_attaches_its_state(monkeypatch):
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
builder._async_scheduling = False
builder.state = FakeState(TOPK)
metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata)
metadata.state = None
monkeypatch.setattr(
sm90_mod.FlashInferMLASparseMetadataBuilder,
"build",
lambda *_args, **_kwargs: metadata,
)
cam = SimpleNamespace(
num_reqs=1,
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
seq_lens=torch.tensor([1], dtype=torch.int32),
seq_lens_cpu_upper_bound=torch.tensor([1], dtype=torch.int32),
positions=None,
)
result = builder.build(0, cam)
assert result.state is builder.state
assert builder.state.plan_calls[0][0] == 1
assert builder.state.plan_calls[0][1].tolist() == [1]
def test_plan_uses_state_params(monkeypatch):
"""The NoPE/rope dims and scale live on the builder state, not the layer.
plan() takes exact per-row KV lengths; the schedule is rebuilt on every
call (contexts grow between steps) and the indptrs are always full-size
with zero-query padding rows past num_tokens.
"""
impl, rows = make_impl(64, "auto")
wrapper = FakeWrapper()
state = sm90_mod._SM90State.__new__(sm90_mod._SM90State)
state.device = torch.device("cpu")
state.wrapper = wrapper
state.num_heads = 4
state.kv_dtype = torch.bfloat16
state.kv_lora_rank = HEAD
state.qk_rope_head_dim = 64
state.sm_scale = 576**-0.5
state.max_tokens = 4
state.topk_width = TOPK
state.kv_indices = torch.zeros(4 * TOPK)
state._arange_cpu = torch.arange(5, dtype=torch.int32)
state._qo_cpu = torch.empty(5, dtype=torch.int32)
state._kv_cpu = torch.empty(5, dtype=torch.int32)
state._lens_cpu = torch.full((4,), TOPK, dtype=torch.int32)
state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32))
assert wrapper.plan_args is not None
args, kwargs = wrapper.plan_args
(qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args
assert qo.tolist() == [0, 1, 2, 3, 3] # clamp: rows past 3 have no queries
assert kv.tolist() == [i * TOPK for i in (0, 1, 2, 3, 3)]
assert kv_len.tolist() == [2, 5, 7, TOPK] # padded row keeps full width
assert (heads, ckv, kpe, page, causal) == (4, HEAD, 64, 1, False)
assert scale == 576**-0.5
assert kwargs["q_data_type"] == torch.bfloat16
assert kwargs["kv_data_type"] == torch.bfloat16
def test_kv_lens_host_formula():
"""Per-row host lengths: context == position + 1; capped at
index_topk + trailing-pool remainder past the sparse threshold."""
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
builder._async_scheduling = False
cam = SimpleNamespace(
num_reqs=3,
query_start_loc_cpu=torch.tensor([0, 5, 7, 10], dtype=torch.int32),
seq_lens=torch.tensor([100, 9, 3000], dtype=torch.int32),
seq_lens_cpu_upper_bound=torch.tensor([100, 9, 3000], dtype=torch.int32),
positions=None,
)
num_rows, lens = builder._kv_lens_host(cam)
assert num_rows == 10
# req0: positions 95..99 -> ctx 96..100 (all <= 2048: full context)
# req1: positions 7,8 -> ctx 8,9
# req2: positions 2997..2999 -> ctx 2998..3000 (> 2048: topk + ctx%4)
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2050, 2051, 2048]
def test_kv_lens_host_empty():
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
cam = SimpleNamespace(
num_reqs=0,
query_start_loc_cpu=torch.tensor([0], dtype=torch.int32),
seq_lens=torch.zeros(0, dtype=torch.int32),
)
num_rows, lens = builder._kv_lens_host(cam)
assert num_rows == 0 and lens.numel() == 0
def test_supports_combination_gates(monkeypatch, default_vllm_config):
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True)
call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination(
head_size=576,
dtype=torch.bfloat16,
kv_cache_dtype="fp8_e4m3",
block_size=64,
use_mla=True,
has_sink=False,
use_sparse=True,
use_mm_prefix=False,
device_capability=SimpleNamespace(major=9),
**kw,
)
assert call() is None # no model config: only the feature gate applies
import vllm.config as cfg
monkeypatch.setattr(
cfg,
"get_current_vllm_config",
lambda: SimpleNamespace(model_config=None),
)
assert call() is None
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: False)
assert "requires FlashInfer" in (call() or "")
@@ -0,0 +1,369 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU tests for the kpool tail slot mapping (no GPU required).
The kpool tail cache is a 1-block-per-request circular ring addressed by
``pos % kpool`` (``KpoolTailSpec`` / ``KpoolTailManager``: exactly one block
allocated per request, never grown, so only column 0 of its block table is
ever written; the rest stays zero-initialized).
The generic per-group slot kernel cannot express that layout: it maps
``pos -> bt[req][pos // bs] * bs + pos % bs`` (``_compute_slot_mappings_kernel``
in vllm/v1/worker/gpu/block_table.py), so every token at ``pos >= kpool``
reads a zero column and collapses onto physical tail block 0. All concurrent
requests then share one ``kpool``-slot ring and corrupt each other's pool
compression.
These tests pin that defect's arithmetic, verify the circular replacement
(``compute_kpool_tail_slot_mapping``), and mirror the tail kernels' index
math to show cross-request pollution before the fix and isolation after.
"""
from types import SimpleNamespace
import pytest
import torch
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.indexer import (
KpoolTailBackend,
KpoolTailMetadataBuilder,
compute_kpool_tail_slot_mapping,
)
from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides
from vllm.v1.kv_cache_layout import KVCacheLayout
KPOOL = 4
def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
(layout,) = KpoolTailBackend.supported_kv_cache_layouts()
spec = KpoolTailSpec(
block_size=KPOOL,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=KPOOL,
)
strides = compute_layout_strides(spec, num_blocks=8, num_layers=3, layout=layout)
_, _, head_stride, state_stride, content_stride = strides
assert layout is KVCacheLayout.LBHNC
assert head_stride == KPOOL * 128 * torch.bfloat16.itemsize
assert state_stride == 128 * torch.bfloat16.itemsize
assert content_stride == 1
def make_tail_block_table(own_blocks, width=64):
"""Tail-group block table as BlockTables produces it: column 0 holds the
request's single KpoolTailManager block, the remaining columns are never
written and stay zero."""
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
return bt
def legacy_generic_tail_slots(block_table, query_start_loc, positions):
"""Reference of the generic ``_compute_slot_mappings_kernel`` arithmetic
(block_table.py:305-313) applied to the tail group's table."""
slots = []
for req in range(block_table.shape[0]):
for i in range(query_start_loc[req], query_start_loc[req + 1]):
pos = int(positions[i])
block_number = int(block_table[req, pos // KPOOL])
slots.append(block_number * KPOOL + pos % KPOOL)
return torch.tensor(slots, dtype=torch.int64)
def circular_tail_slots(
slot_mapping, block_table, query_start_loc, positions, num_actual, num_reqs
):
return compute_kpool_tail_slot_mapping(
slot_mapping,
block_table,
query_start_loc,
positions,
num_actual,
num_reqs,
KPOOL,
)
def make_batch(per_req_positions, padded_len=None):
positions = torch.cat(
[torch.tensor(p, dtype=torch.int64) for p in per_req_positions]
)
num_actual = positions.numel()
num_reqs = len(per_req_positions)
lens = [len(p) for p in per_req_positions]
qsl = torch.zeros(num_reqs + 1, dtype=torch.int64)
torch.cumsum(torch.tensor(lens, dtype=torch.int64), 0, out=qsl[1:])
if padded_len is None:
padded_len = num_actual
slot_mapping = torch.full((padded_len,), -1, dtype=torch.int64)
return positions, qsl, slot_mapping, num_actual, num_reqs
def test_legacy_generic_mapping_collapses_onto_block_zero():
"""The bug: with the manager's 1-column block table, the generic kernel
maps every pos >= kpool onto tail block 0, and distinct requests collide."""
own_blocks = [5, 9]
per_req = [list(range(10)), list(range(12))] # prompts of len 10 and 12
positions, qsl, _, num_actual, num_reqs = make_batch(per_req)
bt = make_tail_block_table(own_blocks)
legacy = legacy_generic_tail_slots(bt, qsl, positions)
# Every token at pos >= kpool resolves to block 0, not the request's own.
off = 0
for req, prompt in enumerate(per_req):
req_slots = legacy[off : off + len(prompt)]
for pos in range(len(prompt)):
slot = int(req_slots[pos])
if pos >= KPOOL:
assert slot // KPOOL == 0, (
f"expected collapse onto block 0 at pos {pos}"
)
assert slot // KPOOL != own_blocks[req]
off += len(prompt)
# The two requests share ring slots -> cross-request pollution.
a_slots = set(legacy[: len(per_req[0])].tolist())
b_slots = set(legacy[len(per_req[0]) :].tolist())
assert a_slots & b_slots, "legacy mapping must collide across requests"
def test_circular_mapping_isolates_requests():
"""The fix: every token lands in its own request's block at pos % kpool,
and no slot is ever shared by two different requests (slots do recur
within a request every kpool positions -- that is the circular design)."""
own_blocks = [5, 9]
per_req = [list(range(10)), list(range(12))]
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
bt = make_tail_block_table(own_blocks)
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
off = 0
per_req_slots = []
for req, prompt in enumerate(per_req):
req_slots = set()
for pos in range(len(prompt)):
slot = int(out[off + pos])
assert slot // KPOOL == own_blocks[req], (
f"req {req} pos {pos} left its tail block"
)
assert slot % KPOOL == pos % KPOOL
req_slots.add(slot)
per_req_slots.append(req_slots)
off += len(prompt)
assert not per_req_slots[0] & per_req_slots[1]
@pytest.mark.parametrize("prompt_len", [1, 2, 3, 4])
def test_circular_mapping_matches_generic_for_short_requests(prompt_len):
"""For pos < kpool the generic kernel already picks the own block, so the
two mappings agree while every position fits the request's first block
(single-request behavior is unchanged)."""
own_blocks = [7]
per_req = [list(range(prompt_len))]
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
bt = make_tail_block_table(own_blocks)
legacy = legacy_generic_tail_slots(bt, qsl, positions)
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
assert torch.equal(out, legacy)
def test_circular_mapping_preserves_padding_and_empty_batch():
own_blocks = [5, 9]
per_req = [list(range(10)), list(range(12))]
padded_len = sum(len(p) for p in per_req) + 8
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(
per_req, padded_len=padded_len
)
bt = make_tail_block_table(own_blocks)
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
assert out.shape == slot_mapping.shape
assert torch.equal(out[num_actual:], torch.full_like(out[num_actual:], -1))
empty = circular_tail_slots(slot_mapping, bt, qsl, positions[:0], 0, num_reqs)
assert torch.equal(empty, slot_mapping)
def make_common_metadata(per_req_positions, own_blocks, with_positions=True):
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(
per_req_positions, padded_len=sum(len(p) for p in per_req_positions) + 4
)
bt = make_tail_block_table(own_blocks)
seq_lens = torch.tensor(
[max(p) + 1 if p else 1 for p in per_req_positions], dtype=torch.int64
)
return CommonAttentionMetadata(
query_start_loc=qsl,
query_start_loc_cpu=qsl.clone(),
seq_lens=seq_lens,
num_reqs=num_reqs,
num_actual_tokens=num_actual,
max_query_len=max((len(p) for p in per_req_positions), default=1),
max_seq_len=int(seq_lens.max()) if num_reqs else 1,
block_table_tensor=bt,
slot_mapping=slot_mapping,
positions=positions if with_positions else None,
)
def make_tail_builder(block_size=KPOOL, max_num_batched_tokens=128):
builder = object.__new__(KpoolTailMetadataBuilder)
builder.kv_cache_spec = SimpleNamespace(block_size=block_size)
builder.slot_mapping_buffer = torch.empty(max_num_batched_tokens, dtype=torch.int64)
return builder
def test_builder_build_uses_circular_mapping():
per_req = [list(range(10)), list(range(12))]
own_blocks = [5, 9]
cam = make_common_metadata(per_req, own_blocks)
meta = KpoolTailMetadataBuilder.build(make_tail_builder(), 0, cam)
out = meta.slot_mapping
off = 0
for req, prompt in enumerate(per_req):
for pos in range(len(prompt)):
slot = int(out[off + pos])
assert slot // KPOOL == own_blocks[req]
assert slot % KPOOL == pos % KPOOL
off += len(prompt)
# Padding tail of the buffer keeps the -1 sentinel.
assert torch.equal(
out[cam.num_actual_tokens :], torch.full_like(out[cam.num_actual_tokens :], -1)
)
def test_builder_build_falls_back_without_positions():
"""Capture / dummy builds without positions keep the generic mapping."""
per_req = [list(range(10))]
cam = make_common_metadata(per_req, [5], with_positions=False)
meta = KpoolTailMetadataBuilder.build(make_tail_builder(), 0, cam)
assert meta.slot_mapping is cam.slot_mapping
def test_builder_reuses_slot_mapping_storage():
builder = make_tail_builder()
first = make_common_metadata([list(range(10))], [5])
first_meta = KpoolTailMetadataBuilder.build(builder, 0, first)
data_ptr = first_meta.slot_mapping.data_ptr()
second = make_common_metadata([list(range(12))], [9])
second_meta = KpoolTailMetadataBuilder.build(builder, 0, second)
assert second_meta.slot_mapping.data_ptr() == data_ptr
assert second_meta.slot_mapping[:12].tolist() == [
9 * KPOOL + pos % KPOOL for pos in range(12)
]
# ---------------------------------------------------------------------------
# Index-level mirror of the tail kernels: seed / stash / pool completion
# (addressing replicated from kpool_compress.py's Triton kernels).
# ---------------------------------------------------------------------------
class TailRingMirror:
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
completing at pos reads ring slots (pool_start + s) % kpool and uses the
current token's own K/score for the last member."""
def __init__(self, num_blocks, kpool=KPOOL):
self.kpool = kpool
self.k = torch.full((num_blocks, kpool, 3), float("nan"))
self.s = torch.full((num_blocks, kpool, 3), float("nan"))
def stash(self, tail_slot, pos, k, s):
blk, off = tail_slot // self.kpool, pos % self.kpool
self.k[blk, off] = k
self.s[blk, off] = s
seed = stash # the seed kernel writes with the same addressing
def complete(self, tail_slot, pos, k, s):
blk = tail_slot // self.kpool
start = pos - (self.kpool - 1)
kk = torch.stack(
[self.k[blk, (start + i) % self.kpool] for i in range(self.kpool)]
)
ss = torch.stack(
[self.s[blk, (start + i) % self.kpool] for i in range(self.kpool)]
)
kk[-1], ss[-1] = k, s # is_current for the completing token
w = torch.softmax(ss, dim=0)
return (kk * w).sum(0)
def token_kv(req, pos):
k = torch.tensor([pos + 100.0 * req, pos + 0.5, 2.0 * pos + 0.25])
s = torch.tensor([0.1 * (pos + 1) + req, 0.2 * pos, 0.05 * pos])
return k, s
def tail_slot_for(mapping, req, pos, own_block):
if mapping == "legacy":
bt_val = own_block if pos < KPOOL else 0
return bt_val * KPOOL + pos % KPOOL
return own_block * KPOOL + pos % KPOOL
def run_scenario(mapping, interleave):
"""Requests A (block 5, prompt len 9) and B (block 9, prompt len 11)
decode concurrently; returns A's boundary pool [8, 9, 10, 11]."""
ring = TailRingMirror(num_blocks=16)
blocks = {"A": 5, "B": 9}
prompts = {"A": 9, "B": 11}
def slot(req, pos):
return tail_slot_for(mapping, 0 if req == "A" else 1, pos, blocks[req])
# Prefill: seed each request's trailing incomplete pool.
for req, L in prompts.items():
for pos in range(L - (L % KPOOL or KPOOL), L):
if pos < 0:
continue
ring.stash(slot(req, pos), pos, *token_kv(0 if req == "A" else 1, pos))
# Decode: A emits pos 9, 10, 11; B emits 11, 12, 13. `interleave`
# processes B before A within a step, which is exactly what concurrent
# Triton programs do when they share tail block 0.
order = ["B", "A"] if interleave else ["A", "B"]
decode = {"A": [9, 10, 11], "B": [11, 12, 13]}
result = None
for step in range(3):
for req in order:
pos = decode[req][step]
k, s = token_kv(0 if req == "A" else 1, pos)
if pos % KPOOL == KPOOL - 1:
pool = ring.complete(slot(req, pos), pos, k, s)
if req == "A":
result = pool
ring.stash(slot(req, pos), pos, k, s)
return result
def test_interleaved_decode_pollution_legacy_vs_circular():
"""Ground truth: request A decoding alone (its ring used exclusively)."""
torch.manual_seed(0)
ground_truth = run_scenario("circular", interleave=False)
legacy = run_scenario("legacy", interleave=True)
circular = run_scenario("circular", interleave=True)
# The old mapping lets request B's tokens into A's ring: A's boundary
# pool is compressed from 2 of B's tokens -> wrong.
assert not torch.allclose(legacy, ground_truth), (
f"legacy mapping unexpectedly clean: {legacy} vs {ground_truth}"
)
# The circular mapping keeps the rings isolated under interleaving.
torch.testing.assert_close(circular, ground_truth)
@@ -0,0 +1,127 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
_use_rocm_sparse_triton,
fit_kpool_indices_to_aiter,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_sparse_kv_row_offset,
_validate_dsv4_sparse_dims,
_validate_sparse_dims,
)
@triton.jit
def _store_sparse_kv_row_offset_kernel(slot_ptr, output_ptr, stride: tl.constexpr):
slot = tl.load(slot_ptr)
tl.store(output_ptr, _sparse_kv_row_offset(slot, stride))
def test_fit_kpool_indices_preserves_tail_and_best_history():
token_indices = torch.tensor(
[
[10, 9, 8, 7, 6, 5, 100, 101],
[10, 9, 8, -1, -1, -1, 100, -1],
[-1, -1, -1, -1, -1, -1, -1, -1],
],
dtype=torch.int32,
)
fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=6)
assert fitted.tolist() == [
[10, 9, 8, 7, 100, 101],
[10, 9, 8, 100, -1, -1],
[-1, -1, -1, -1, -1, -1],
]
def test_fit_kpool_indices_exact_width_is_noop():
token_indices = torch.tensor([[3, 2, 1, -1]], dtype=torch.int32)
fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=4)
assert fitted.data_ptr() == token_indices.data_ptr()
def test_fit_kpool_indices_rejects_narrow_input():
with pytest.raises(ValueError, match="at least topk_tokens"):
fit_kpool_indices_to_aiter(
torch.zeros((1, 3), dtype=torch.int32), topk_tokens=4
)
@pytest.mark.parametrize(
(
"kv_cache_dtype",
"head_size",
"num_prefills",
"num_decodes",
"num_decode_tokens",
"max_query_len",
"expected",
),
[
("auto", 512, 1, 0, 0, 32, True),
("auto", 512, 1, 2, 2, 32, True),
("auto", 512, 0, 2, 2, 1, True),
("fp8", 512, 1, 0, 0, 32, False),
("auto", 576, 1, 0, 0, 32, False),
("auto", 512, 0, 2, 4, 2, False),
],
)
def test_rocm_sparse_triton_route(
kv_cache_dtype,
head_size,
num_prefills,
num_decodes,
num_decode_tokens,
max_query_len,
expected,
):
assert (
_use_rocm_sparse_triton(
kv_cache_dtype=kv_cache_dtype,
head_size=head_size,
kv_lora_rank=512,
num_prefills=num_prefills,
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
max_query_len=max_query_len,
)
is expected
)
def test_rocm_sparse_attention_accepts_glm_nope_dimensions():
_validate_sparse_dims(512, 512, 0, "test")
def test_rocm_sparse_attention_rejects_inconsistent_dimensions():
with pytest.raises(AssertionError, match="expected head_dim"):
_validate_sparse_dims(511, 512, 0, "test")
def test_dsv4_sparse_attention_keeps_layout_constraint():
_validate_dsv4_sparse_dims(512, 448, 64, "test")
with pytest.raises(AssertionError, match="expects 448 NoPE dims"):
_validate_dsv4_sparse_dims(512, 512, 0, "test")
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
def test_sparse_prefill_kv_row_offset_does_not_overflow_int32():
# GLM's 640-token pages cross the signed-int32 address boundary at block
# 6554 for a 512-element KV row. The production kernel must promote the
# slot before multiplying by the row stride.
slot = torch.tensor([6554 * 640], dtype=torch.int32, device="cuda")
output = torch.empty(1, dtype=torch.int64, device="cuda")
_store_sparse_kv_row_offset_kernel[(1,)](slot, output, stride=512)
assert output.item() == 6554 * 640 * 512
@@ -0,0 +1,184 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU tests for the decode top-k row->token mapping (no GPU required).
On non-uniform decode batches (``requires_padding`` -- mixed plain-decode and
spec-verify requests, or variable MTP verify lens; taken on Hopper where the
varlen/flatten logits path is unavailable), the pool-topk rows follow the
PADDED ``[batch_size, next_n]`` grid: row ``(b, t)`` is flat decode token
``offset_b + t``. The former inline ``dec_seq = positions[:n] + 1``
(``n = batch_size * next_n``) indexes the flat per-token layout with padded
coordinates, so rows after the first non-uniform request read another
request's positions and rows past the decode region read prefill tokens --
``expand_pools_and_append_tail`` then anchors the tail at a foreign length.
These tests pin the defect with the exact production arithmetic and verify
the layout-aware replacement (``_decode_topk_seq_lens``), including the tail
expansion consequences via the pure-torch expand/append pair that the fused
kernel is documented to replicate.
"""
import torch
# Bootstrap the glm5next package before entering the indexer module: its
# kpool_compress import runs glm5next/__init__, which pulls model ->
# attention -> back into sparse_attn_indexer_kpool (attention.py imports the
# class at module scope). Production always enters via attention.py first.
# isort: off
import vllm.models.glm5next # noqa: F401
from vllm.models.glm5next.nvidia.ops.kpool_compress import ( # noqa: E402
append_tail_to_topk,
expand_pools_to_tokens,
)
# isort: on
import vllm.model_executor.layers.sparse_attn_indexer_kpool as indexer_mod
from vllm.model_executor.layers.sparse_attn_indexer_kpool import (
_decode_topk_seq_lens,
_fill_short_decode_causal_indices,
)
from vllm.platforms import current_platform
KPOOL = 4
TOPK_TOKENS = 16
SELECT_K = TOPK_TOKENS // KPOOL
def test_kpool_ops_dispatch_matches_platform():
expected_backend = ".amd." if current_platform.is_rocm() else ".nvidia."
assert expected_backend in indexer_mod.kpool_ops.__name__
def test_short_decode_fills_exact_causal_rows():
topk = torch.full((3, 8), 99, dtype=torch.int32)
positions = torch.tensor([0, 3, 7], dtype=torch.int64)
assert _fill_short_decode_causal_indices(topk, positions, 3, 8, 8)
assert topk.tolist() == [
[0, -1, -1, -1, -1, -1, -1, -1],
[0, 1, 2, 3, -1, -1, -1, -1],
[0, 1, 2, 3, 4, 5, 6, 7],
]
def test_short_decode_leaves_buffer_unchanged_for_sparse_context():
topk = torch.full((2, 8), 99, dtype=torch.int32)
before = topk.clone()
assert not _fill_short_decode_causal_indices(topk, torch.tensor([7, 8]), 2, 9, 8)
assert torch.equal(topk, before)
def make_non_uniform_batch():
"""3 requests: plain decode (1 token), MTP verify (4), adaptive verify (3).
Flat decode positions (production layout: decode tokens first, then any
prefill tokens of the same batch):
req0: [30] (context len 30)
req1: [100..103] (verify at context len 100)
req2: [7, 8, 9] (verify at context len 7)
Followed by 4 prefill tokens at positions 555..558 so the flat tensor is
at least ``n = 3 * 4`` long, as it is in a real mixed batch.
"""
decode_lens = torch.tensor([1, 4, 3], dtype=torch.int64)
per_req_positions = [[30], [100, 101, 102, 103], [7, 8, 9]]
flat_decode = [p for req in per_req_positions for p in req]
positions = torch.tensor(flat_decode + [555, 556, 557, 558], dtype=torch.int64)
return decode_lens, per_req_positions, positions
def expected_row_seq_lens(per_req_positions, batch_size, next_n):
"""Ground truth: row (b, t) -> pos + 1 for real rows, 0 for pad rows."""
out = torch.zeros(batch_size * next_n, dtype=torch.int32)
for b, req in enumerate(per_req_positions):
for t, pos in enumerate(req):
out[b * next_n + t] = pos + 1
return out
def test_legacy_flat_layout_misaligns_and_bleeds():
"""The bug: ``positions[:n] + 1`` with padded-row coordinates reads other
requests' positions and prefill positions."""
decode_lens, per_req, positions = make_non_uniform_batch()
batch_size = decode_lens.shape[0]
next_n = int(decode_lens.max())
n = batch_size * next_n
assert n == 12
legacy = positions[:n].to(torch.int32) + 1
expected = expected_row_seq_lens(per_req, batch_size, next_n)
# Row (1, 0): req1's first verify token (pos 100, seq 101) reads flat
# index 4 -> req1's LAST verify token (pos 103). Its true tail token is
# dropped and the tail anchors 3 tokens late.
assert int(legacy[4]) == 104 and int(expected[4]) == 101
# Rows (2, *): req2 starts at flat offset 5, but padded coordinates point
# at flat indices 8..11 -- the batch's PREFILL positions.
for t in range(3):
assert int(legacy[2 * next_n + t]) >= 556, (
"legacy row (2, t) should read prefill positions"
)
assert not torch.equal(legacy, expected)
def test_helper_uniform_layout_matches_flat_slice():
"""Uniform batches keep the flat shortcut (zero behavior/perf change)."""
per_req = [[200, 201, 202, 203], [50, 51, 52, 53]]
positions = torch.tensor([p for r in per_req for p in r], dtype=torch.int64)
decode_lens = torch.tensor([4, 4], dtype=torch.int64)
out = _decode_topk_seq_lens(positions, decode_lens, 8, 2, 4, requires_padding=False)
assert torch.equal(out, positions[:8].to(torch.int32) + 1)
def test_helper_padded_layout_per_row():
"""Non-uniform batches map every padded row to its own token's position;
pad rows get 0 (empty tail)."""
decode_lens, per_req, positions = make_non_uniform_batch()
out = _decode_topk_seq_lens(positions, decode_lens, 8, 3, 4, requires_padding=True)
expected = expected_row_seq_lens(per_req, 3, 4)
assert torch.equal(out, expected)
# Pad rows (0, 1..3) and (2, 3) collapse to 0 -> no tail appended.
assert out[1] == 0 and out[2] == 0 and out[3] == 0 and out[11] == 0
def expand_tail_region(dec_seq):
"""Run the production pure-torch expand + append pair (the fused kernel
replicates it exactly on the identity path) and return the tail columns
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1)."""
rows = dec_seq.shape[0]
pool_ids = torch.arange(SELECT_K, dtype=torch.int64).expand(rows, SELECT_K)
valid = torch.ones_like(pool_ids, dtype=torch.bool)
expanded = expand_pools_to_tokens(pool_ids, valid, TOPK_TOKENS, KPOOL)
seq_lens = dec_seq.to(torch.int32)
pool_lens = (seq_lens // KPOOL).to(torch.int32)
out = append_tail_to_topk(expanded, seq_lens, pool_lens, KPOOL)
return out[:, TOPK_TOKENS:]
def test_tail_expansion_legacy_vs_fixed():
"""End-to-end tail consequence: the fixed mapping appends exactly the
request's trailing incomplete pool; the legacy flat slice drops real tail
tokens and emits indices far past the request's own sequence."""
decode_lens, per_req, positions = make_non_uniform_batch()
n = 12
fixed = _decode_topk_seq_lens(
positions, decode_lens, 8, 3, 4, requires_padding=True
)
legacy = positions[:n].to(torch.int32) + 1
fixed_tail = expand_tail_region(fixed)
legacy_tail = expand_tail_region(legacy)
# Fixed: row (1, 0) (seq 101) keeps its single tail token 100; row (2, 2)
# (seq 10) keeps its full trailing pool [8, 9].
assert fixed_tail[4, 0].item() == 100
assert fixed_tail[10, :2].tolist() == [8, 9]
# Legacy: row (1, 0) loses its tail token (anchored at 104, count 0)...
assert legacy_tail[4, 0].item() == -1
# ...and row (2, 2) reads prefill position 557 -> tail indices 556/557,
# way past req2's 10-token sequence -> out-of-bounds block-table reads.
assert legacy_tail[10, :2].tolist() == [556, 557]
assert not torch.equal(legacy_tail, fixed_tail)
+61 -5
View File
@@ -40,11 +40,13 @@ if not current_platform.is_cuda():
allow_module_level=True,
)
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as flashinfer_sparse_mod
from vllm.model_executor.layers.attention.mla_attention import (
_canonicalize_sparse_mla_kv_cache_dtype,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseImpl,
FlashInferMLASparseTRTLLMBackend,
)
from vllm.v1.attention.backends.mla.flashmla_sparse import (
@@ -82,6 +84,60 @@ SPARSE_BACKEND_BATCH_SPECS["large_q_pure_prefill"] = BatchSpec(
DEVICE_TYPE = current_platform.device_type
def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
"""Weight absorption must not change the model's attention temperature."""
model_scale = 256**-0.5
kv_lora_rank = 512
topk = torch.zeros((1, 1), dtype=torch.int32)
metadata = SimpleNamespace(
req_id_per_token=torch.zeros(1, dtype=torch.int32),
block_table=torch.zeros((1, 1), dtype=torch.int32),
block_size=1,
)
recorded_scale = None
impl = object.__new__(FlashInferMLASparseImpl)
impl.scale = model_scale
impl.qk_nope_head_dim = 256
impl.kv_lora_rank = kv_lora_rank
impl.qk_rope_head_dim = 0
impl.kv_cache_dtype = "auto"
impl.topk_indices_buffer = topk
impl.dcp_world_size = 1
impl._workspace_buffer = torch.empty(1)
impl.bmm1_scale = None
impl.bmm2_scale = None
impl.is_nope_mla = True
impl.need_to_return_lse_for_decode = False
monkeypatch.setattr(
flashinfer_sparse_mod,
"triton_convert_req_index_to_global_index",
lambda *args, **kwargs: (topk, torch.ones(1, dtype=torch.int32)),
)
import flashinfer.decode
def fake_flashinfer(**kwargs):
nonlocal recorded_scale
recorded_scale = kwargs["bmm1_scale"]
return torch.zeros((1, 1, 1, kv_lora_rank))
monkeypatch.setattr(
flashinfer.decode,
"trtllm_batch_decode_with_kv_cache_mla",
fake_flashinfer,
)
impl.forward_mqa(
torch.zeros(1, 1, kv_lora_rank),
torch.zeros(1, 1, kv_lora_rank),
metadata,
SimpleNamespace(),
)
assert recorded_scale == model_scale
assert recorded_scale != kv_lora_rank**-0.5
def _float_to_e8m0_truncate(f: float) -> float:
"""Simulate SM100's float -> e8m0 -> bf16 scale conversion.
e8m0 format only stores the exponent (power of 2).
@@ -1451,14 +1507,14 @@ def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
return_valid_counts=False,
)
assert isinstance(result_only, torch.Tensor)
torch.testing.assert_close(result_only, result, rtol=0, atol=0)
for row, num_valid in enumerate(expected_valid):
compact_valid = result[row, :num_valid].sort().values
original_valid = result_only[row][result_only[row] >= 0].sort().values
torch.testing.assert_close(compact_valid, original_valid, rtol=0, atol=0)
assert torch.all(result[row, num_valid:] == -1)
def test_flashmla_cache_dtype_aliases_use_ds_layout():
from vllm.model_executor.layers.attention.mla_attention import (
_canonicalize_sparse_mla_kv_cache_dtype,
)
# kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer
# stores kv_cache_dtype, so they cannot bypass the gate.
for alias in ("fp8", "fp8_e4m3"):