[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for error handling in the CUDA attention backend probe.
|
||||
"""Tests for CUDA attention backend selection and probe error handling.
|
||||
|
||||
Environment-shaped probe failures (missing packages, unreadable caches,
|
||||
broken driver installs) must mark the backend unavailable rather than
|
||||
@@ -89,6 +89,48 @@ def test_get_valid_backends_keeps_probing_after_failure():
|
||||
assert valid
|
||||
|
||||
|
||||
def test_sm90_nope_mla_prefers_flashinfer_without_changing_rope_order():
|
||||
backend_cls = MagicMock()
|
||||
backend_cls.validate_configuration.return_value = []
|
||||
sparse_backends = {
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
|
||||
}
|
||||
|
||||
def sparse_order(head_size: int) -> list[AttentionBackendEnum]:
|
||||
config = SELECTOR_CONFIG._replace(
|
||||
head_size=head_size,
|
||||
use_mla=True,
|
||||
use_sparse=True,
|
||||
)
|
||||
with patch(
|
||||
"vllm.platforms.cuda._get_attn_backend_class",
|
||||
return_value=backend_cls,
|
||||
):
|
||||
valid, _ = CudaPlatform.get_valid_backends(
|
||||
device_capability=SM90,
|
||||
attn_selector_config=config,
|
||||
num_heads=32,
|
||||
)
|
||||
return [
|
||||
candidate.backend
|
||||
for candidate in valid
|
||||
if candidate.backend in sparse_backends
|
||||
]
|
||||
|
||||
assert sparse_order(512) == [
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
]
|
||||
assert sparse_order(576) == [
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
|
||||
]
|
||||
|
||||
|
||||
def test_selected_backend_probe_failure_raises_value_error_with_cause():
|
||||
exc = OSError("libcuda.so.1: cannot open shared object file")
|
||||
with (
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for the FlashInfer SM90 sparse MLA backend wiring (no GPU).
|
||||
|
||||
The FlashInfer wrapper and top-k conversion are replaced by CPU recorders;
|
||||
the tests pin the contract between the impl and the kernel API: page_size=1
|
||||
varlen rows, reserved-buffer refresh, plan parameters (dims, NoPE/rope scale,
|
||||
causality), ckv/kpe cache splitting, and the backend's model-shape gates.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# isort: off
|
||||
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm90 as sm90_mod
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm90 import (
|
||||
FlashInferMLASparseSM90Backend,
|
||||
FlashInferMLASparseSM90Builder,
|
||||
FlashInferMLASparseSM90Impl,
|
||||
)
|
||||
# isort: on
|
||||
|
||||
BLOCK_SIZE = 64
|
||||
HEAD = 512
|
||||
TOPK = 128 # triton convert requires width % 128 == 0
|
||||
|
||||
|
||||
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
|
||||
out = torch.full_like(token_indices, -1)
|
||||
counts = torch.zeros(token_indices.shape[0], dtype=torch.int32)
|
||||
for t in range(token_indices.shape[0]):
|
||||
vals = []
|
||||
for j in range(token_indices.shape[1]):
|
||||
pos = int(token_indices[t, j])
|
||||
if pos == -1:
|
||||
continue
|
||||
blk = int(block_table[int(req_id[t]), pos // BLOCK_SIZE])
|
||||
if blk < 0:
|
||||
continue
|
||||
vals.append(blk * BLOCK_SIZE + pos % BLOCK_SIZE)
|
||||
out[t, : len(vals)] = torch.tensor(vals, dtype=out.dtype)
|
||||
counts[t] = len(vals)
|
||||
return out, counts
|
||||
|
||||
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.plan_args = None
|
||||
self.run_args = None
|
||||
|
||||
def plan(self, *args, **kwargs):
|
||||
self.plan_args = (args, kwargs)
|
||||
|
||||
def run(self, *args, **kwargs):
|
||||
q_nope, q_pe, ckv, kpe = args
|
||||
self.run_args = (q_nope, q_pe, ckv, kpe, kwargs)
|
||||
return torch.zeros(
|
||||
q_nope.shape[0], q_nope.shape[1], ckv.shape[-1], dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
|
||||
class FakeState:
|
||||
def __init__(self, width, max_tokens=64):
|
||||
self.kv_indices = torch.zeros(max_tokens * width, dtype=torch.int32)
|
||||
self.kv_len_arr = torch.zeros(max_tokens, dtype=torch.int32)
|
||||
self.wrapper = FakeWrapper()
|
||||
self.plan_calls = []
|
||||
|
||||
def plan(self, num_tokens, kv_lens):
|
||||
self.plan_calls.append((num_tokens, kv_lens))
|
||||
|
||||
|
||||
def make_impl(qk_rope, kv_dtype="fp8_e4m3", num_heads=2, topk_width=TOPK):
|
||||
impl = object.__new__(FlashInferMLASparseSM90Impl)
|
||||
impl.num_heads = num_heads
|
||||
impl.head_size = HEAD + qk_rope
|
||||
impl.scale = (HEAD + qk_rope) ** -0.5
|
||||
impl.kv_lora_rank = HEAD
|
||||
impl.qk_rope_head_dim = qk_rope
|
||||
impl.kv_cache_dtype = kv_dtype
|
||||
impl.use_fp8_kv_cache = kv_dtype in ("fp8", "fp8_e4m3")
|
||||
rows = 4
|
||||
impl.topk_indices_buffer = torch.full((rows, topk_width), -1, dtype=torch.int32)
|
||||
return impl, rows
|
||||
|
||||
|
||||
def make_batch(rows, topk_rows, own_blocks):
|
||||
req_id = torch.tensor([0] * rows, dtype=torch.int32)
|
||||
block_table = torch.zeros(1, 16, dtype=torch.int32)
|
||||
block_table[:, 0] = own_blocks[0]
|
||||
topk = torch.full((rows, TOPK), -1, dtype=torch.int32)
|
||||
for t, row in enumerate(topk_rows):
|
||||
topk[t, : len(row)] = torch.tensor(row, dtype=torch.int32)
|
||||
return SimpleNamespace(
|
||||
req_id_per_token=req_id, block_table=block_table, block_size=BLOCK_SIZE
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("qk_rope,kv_dtype", [(0, "fp8_e4m3"), (64, "auto")])
|
||||
def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
impl, rows = make_impl(qk_rope, kv_dtype)
|
||||
state = FakeState(TOPK)
|
||||
monkeypatch.setattr(
|
||||
sm90_mod, "triton_convert_req_index_to_global_index", ref_convert
|
||||
)
|
||||
|
||||
# req with context 10 < topk: 8 valid + -1 padding.
|
||||
topk_rows = [
|
||||
[7, 3, 1, 9, 0, 2, 5, 8] + [-1] * (TOPK - 8),
|
||||
[4, 0, 2, 3, 1] + [-1] * (TOPK - 5),
|
||||
[6, 5, 4] + [-1] * (TOPK - 3),
|
||||
[2] + [-1] * (TOPK - 1),
|
||||
]
|
||||
meta = make_batch(rows, topk_rows, [3])
|
||||
meta.state = state
|
||||
q_nope = torch.randn(rows, impl.num_heads, HEAD)
|
||||
q_rope = torch.randn(rows, impl.num_heads, qk_rope)
|
||||
cache = torch.zeros(
|
||||
8 * BLOCK_SIZE,
|
||||
impl.head_size,
|
||||
dtype=torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16,
|
||||
)
|
||||
|
||||
out, lse = impl.forward_mqa(
|
||||
(q_nope, q_rope), cache, meta, SimpleNamespace(_k_scale_float=0.5)
|
||||
)
|
||||
assert lse is None and out.shape == (rows, impl.num_heads, HEAD)
|
||||
|
||||
# Reserved buffers carry this step's slots; lengths are NOT refreshed
|
||||
# here (the builder plans them host-side before capture/replay).
|
||||
ref_slots, ref_counts = ref_convert(
|
||||
meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer
|
||||
)
|
||||
width = TOPK
|
||||
got_slots = state.kv_indices[: rows * width].view(rows, width)
|
||||
for t in range(rows):
|
||||
k = int(ref_counts[t])
|
||||
assert got_slots[t, :k].tolist() == ref_slots[t, :k].tolist()
|
||||
assert state.plan_calls == []
|
||||
|
||||
assert state.wrapper.run_args is not None
|
||||
q_pe, ckv, kpe, kwargs = state.wrapper.run_args[1:]
|
||||
assert q_pe.shape == (rows, impl.num_heads, qk_rope)
|
||||
assert ckv.shape == (8 * BLOCK_SIZE, 1, HEAD)
|
||||
assert kpe.shape[-1] == qk_rope
|
||||
if impl.use_fp8_kv_cache:
|
||||
assert kwargs["ckv_scale"] == 0.5 and kwargs["kpe_scale"] == 1.0
|
||||
else:
|
||||
assert kwargs == {}
|
||||
|
||||
|
||||
def test_builder_attaches_its_state(monkeypatch):
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
builder._async_scheduling = False
|
||||
builder.state = FakeState(TOPK)
|
||||
metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata)
|
||||
metadata.state = None
|
||||
monkeypatch.setattr(
|
||||
sm90_mod.FlashInferMLASparseMetadataBuilder,
|
||||
"build",
|
||||
lambda *_args, **_kwargs: metadata,
|
||||
)
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=1,
|
||||
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([1], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([1], dtype=torch.int32),
|
||||
positions=None,
|
||||
)
|
||||
|
||||
result = builder.build(0, cam)
|
||||
|
||||
assert result.state is builder.state
|
||||
assert builder.state.plan_calls[0][0] == 1
|
||||
assert builder.state.plan_calls[0][1].tolist() == [1]
|
||||
|
||||
|
||||
def test_plan_uses_state_params(monkeypatch):
|
||||
"""The NoPE/rope dims and scale live on the builder state, not the layer.
|
||||
|
||||
plan() takes exact per-row KV lengths; the schedule is rebuilt on every
|
||||
call (contexts grow between steps) and the indptrs are always full-size
|
||||
with zero-query padding rows past num_tokens.
|
||||
"""
|
||||
impl, rows = make_impl(64, "auto")
|
||||
wrapper = FakeWrapper()
|
||||
state = sm90_mod._SM90State.__new__(sm90_mod._SM90State)
|
||||
state.device = torch.device("cpu")
|
||||
state.wrapper = wrapper
|
||||
state.num_heads = 4
|
||||
state.kv_dtype = torch.bfloat16
|
||||
state.kv_lora_rank = HEAD
|
||||
state.qk_rope_head_dim = 64
|
||||
state.sm_scale = 576**-0.5
|
||||
state.max_tokens = 4
|
||||
state.topk_width = TOPK
|
||||
state.kv_indices = torch.zeros(4 * TOPK)
|
||||
state._arange_cpu = torch.arange(5, dtype=torch.int32)
|
||||
state._qo_cpu = torch.empty(5, dtype=torch.int32)
|
||||
state._kv_cpu = torch.empty(5, dtype=torch.int32)
|
||||
state._lens_cpu = torch.full((4,), TOPK, dtype=torch.int32)
|
||||
|
||||
state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32))
|
||||
assert wrapper.plan_args is not None
|
||||
args, kwargs = wrapper.plan_args
|
||||
(qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args
|
||||
assert qo.tolist() == [0, 1, 2, 3, 3] # clamp: rows past 3 have no queries
|
||||
assert kv.tolist() == [i * TOPK for i in (0, 1, 2, 3, 3)]
|
||||
assert kv_len.tolist() == [2, 5, 7, TOPK] # padded row keeps full width
|
||||
assert (heads, ckv, kpe, page, causal) == (4, HEAD, 64, 1, False)
|
||||
assert scale == 576**-0.5
|
||||
assert kwargs["q_data_type"] == torch.bfloat16
|
||||
assert kwargs["kv_data_type"] == torch.bfloat16
|
||||
|
||||
|
||||
def test_kv_lens_host_formula():
|
||||
"""Per-row host lengths: context == position + 1; capped at
|
||||
index_topk + trailing-pool remainder past the sparse threshold."""
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
builder._async_scheduling = False
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=3,
|
||||
query_start_loc_cpu=torch.tensor([0, 5, 7, 10], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([100, 9, 3000], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([100, 9, 3000], dtype=torch.int32),
|
||||
positions=None,
|
||||
)
|
||||
num_rows, lens = builder._kv_lens_host(cam)
|
||||
assert num_rows == 10
|
||||
# req0: positions 95..99 -> ctx 96..100 (all <= 2048: full context)
|
||||
# req1: positions 7,8 -> ctx 8,9
|
||||
# req2: positions 2997..2999 -> ctx 2998..3000 (> 2048: topk + ctx%4)
|
||||
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2050, 2051, 2048]
|
||||
|
||||
|
||||
def test_kv_lens_host_empty():
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=0,
|
||||
query_start_loc_cpu=torch.tensor([0], dtype=torch.int32),
|
||||
seq_lens=torch.zeros(0, dtype=torch.int32),
|
||||
)
|
||||
num_rows, lens = builder._kv_lens_host(cam)
|
||||
assert num_rows == 0 and lens.numel() == 0
|
||||
|
||||
|
||||
def test_supports_combination_gates(monkeypatch, default_vllm_config):
|
||||
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True)
|
||||
call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8_e4m3",
|
||||
block_size=64,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=True,
|
||||
use_mm_prefix=False,
|
||||
device_capability=SimpleNamespace(major=9),
|
||||
**kw,
|
||||
)
|
||||
assert call() is None # no model config: only the feature gate applies
|
||||
|
||||
import vllm.config as cfg
|
||||
|
||||
monkeypatch.setattr(
|
||||
cfg,
|
||||
"get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=None),
|
||||
)
|
||||
assert call() is None
|
||||
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: False)
|
||||
assert "requires FlashInfer" in (call() or "")
|
||||
@@ -0,0 +1,369 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for the kpool tail slot mapping (no GPU required).
|
||||
|
||||
The kpool tail cache is a 1-block-per-request circular ring addressed by
|
||||
``pos % kpool`` (``KpoolTailSpec`` / ``KpoolTailManager``: exactly one block
|
||||
allocated per request, never grown, so only column 0 of its block table is
|
||||
ever written; the rest stays zero-initialized).
|
||||
|
||||
The generic per-group slot kernel cannot express that layout: it maps
|
||||
``pos -> bt[req][pos // bs] * bs + pos % bs`` (``_compute_slot_mappings_kernel``
|
||||
in vllm/v1/worker/gpu/block_table.py), so every token at ``pos >= kpool``
|
||||
reads a zero column and collapses onto physical tail block 0. All concurrent
|
||||
requests then share one ``kpool``-slot ring and corrupt each other's pool
|
||||
compression.
|
||||
|
||||
These tests pin that defect's arithmetic, verify the circular replacement
|
||||
(``compute_kpool_tail_slot_mapping``), and mirror the tail kernels' index
|
||||
math to show cross-request pollution before the fix and isolation after.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
KpoolTailBackend,
|
||||
KpoolTailMetadataBuilder,
|
||||
compute_kpool_tail_slot_mapping,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
KPOOL = 4
|
||||
|
||||
|
||||
def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
|
||||
(layout,) = KpoolTailBackend.supported_kv_cache_layouts()
|
||||
spec = KpoolTailSpec(
|
||||
block_size=KPOOL,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=KPOOL,
|
||||
)
|
||||
strides = compute_layout_strides(spec, num_blocks=8, num_layers=3, layout=layout)
|
||||
_, _, head_stride, state_stride, content_stride = strides
|
||||
|
||||
assert layout is KVCacheLayout.LBHNC
|
||||
assert head_stride == KPOOL * 128 * torch.bfloat16.itemsize
|
||||
assert state_stride == 128 * torch.bfloat16.itemsize
|
||||
assert content_stride == 1
|
||||
|
||||
|
||||
def make_tail_block_table(own_blocks, width=64):
|
||||
"""Tail-group block table as BlockTables produces it: column 0 holds the
|
||||
request's single KpoolTailManager block, the remaining columns are never
|
||||
written and stay zero."""
|
||||
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
|
||||
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
|
||||
return bt
|
||||
|
||||
|
||||
def legacy_generic_tail_slots(block_table, query_start_loc, positions):
|
||||
"""Reference of the generic ``_compute_slot_mappings_kernel`` arithmetic
|
||||
(block_table.py:305-313) applied to the tail group's table."""
|
||||
slots = []
|
||||
for req in range(block_table.shape[0]):
|
||||
for i in range(query_start_loc[req], query_start_loc[req + 1]):
|
||||
pos = int(positions[i])
|
||||
block_number = int(block_table[req, pos // KPOOL])
|
||||
slots.append(block_number * KPOOL + pos % KPOOL)
|
||||
return torch.tensor(slots, dtype=torch.int64)
|
||||
|
||||
|
||||
def circular_tail_slots(
|
||||
slot_mapping, block_table, query_start_loc, positions, num_actual, num_reqs
|
||||
):
|
||||
return compute_kpool_tail_slot_mapping(
|
||||
slot_mapping,
|
||||
block_table,
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_actual,
|
||||
num_reqs,
|
||||
KPOOL,
|
||||
)
|
||||
|
||||
|
||||
def make_batch(per_req_positions, padded_len=None):
|
||||
positions = torch.cat(
|
||||
[torch.tensor(p, dtype=torch.int64) for p in per_req_positions]
|
||||
)
|
||||
num_actual = positions.numel()
|
||||
num_reqs = len(per_req_positions)
|
||||
lens = [len(p) for p in per_req_positions]
|
||||
qsl = torch.zeros(num_reqs + 1, dtype=torch.int64)
|
||||
torch.cumsum(torch.tensor(lens, dtype=torch.int64), 0, out=qsl[1:])
|
||||
if padded_len is None:
|
||||
padded_len = num_actual
|
||||
slot_mapping = torch.full((padded_len,), -1, dtype=torch.int64)
|
||||
return positions, qsl, slot_mapping, num_actual, num_reqs
|
||||
|
||||
|
||||
def test_legacy_generic_mapping_collapses_onto_block_zero():
|
||||
"""The bug: with the manager's 1-column block table, the generic kernel
|
||||
maps every pos >= kpool onto tail block 0, and distinct requests collide."""
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))] # prompts of len 10 and 12
|
||||
positions, qsl, _, num_actual, num_reqs = make_batch(per_req)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
legacy = legacy_generic_tail_slots(bt, qsl, positions)
|
||||
|
||||
# Every token at pos >= kpool resolves to block 0, not the request's own.
|
||||
off = 0
|
||||
for req, prompt in enumerate(per_req):
|
||||
req_slots = legacy[off : off + len(prompt)]
|
||||
for pos in range(len(prompt)):
|
||||
slot = int(req_slots[pos])
|
||||
if pos >= KPOOL:
|
||||
assert slot // KPOOL == 0, (
|
||||
f"expected collapse onto block 0 at pos {pos}"
|
||||
)
|
||||
assert slot // KPOOL != own_blocks[req]
|
||||
off += len(prompt)
|
||||
|
||||
# The two requests share ring slots -> cross-request pollution.
|
||||
a_slots = set(legacy[: len(per_req[0])].tolist())
|
||||
b_slots = set(legacy[len(per_req[0]) :].tolist())
|
||||
assert a_slots & b_slots, "legacy mapping must collide across requests"
|
||||
|
||||
|
||||
def test_circular_mapping_isolates_requests():
|
||||
"""The fix: every token lands in its own request's block at pos % kpool,
|
||||
and no slot is ever shared by two different requests (slots do recur
|
||||
within a request every kpool positions -- that is the circular design)."""
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))]
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
|
||||
|
||||
off = 0
|
||||
per_req_slots = []
|
||||
for req, prompt in enumerate(per_req):
|
||||
req_slots = set()
|
||||
for pos in range(len(prompt)):
|
||||
slot = int(out[off + pos])
|
||||
assert slot // KPOOL == own_blocks[req], (
|
||||
f"req {req} pos {pos} left its tail block"
|
||||
)
|
||||
assert slot % KPOOL == pos % KPOOL
|
||||
req_slots.add(slot)
|
||||
per_req_slots.append(req_slots)
|
||||
off += len(prompt)
|
||||
assert not per_req_slots[0] & per_req_slots[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt_len", [1, 2, 3, 4])
|
||||
def test_circular_mapping_matches_generic_for_short_requests(prompt_len):
|
||||
"""For pos < kpool the generic kernel already picks the own block, so the
|
||||
two mappings agree while every position fits the request's first block
|
||||
(single-request behavior is unchanged)."""
|
||||
own_blocks = [7]
|
||||
per_req = [list(range(prompt_len))]
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
legacy = legacy_generic_tail_slots(bt, qsl, positions)
|
||||
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
|
||||
assert torch.equal(out, legacy)
|
||||
|
||||
|
||||
def test_circular_mapping_preserves_padding_and_empty_batch():
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))]
|
||||
padded_len = sum(len(p) for p in per_req) + 8
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(
|
||||
per_req, padded_len=padded_len
|
||||
)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
|
||||
assert out.shape == slot_mapping.shape
|
||||
assert torch.equal(out[num_actual:], torch.full_like(out[num_actual:], -1))
|
||||
|
||||
empty = circular_tail_slots(slot_mapping, bt, qsl, positions[:0], 0, num_reqs)
|
||||
assert torch.equal(empty, slot_mapping)
|
||||
|
||||
|
||||
def make_common_metadata(per_req_positions, own_blocks, with_positions=True):
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(
|
||||
per_req_positions, padded_len=sum(len(p) for p in per_req_positions) + 4
|
||||
)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
seq_lens = torch.tensor(
|
||||
[max(p) + 1 if p else 1 for p in per_req_positions], dtype=torch.int64
|
||||
)
|
||||
return CommonAttentionMetadata(
|
||||
query_start_loc=qsl,
|
||||
query_start_loc_cpu=qsl.clone(),
|
||||
seq_lens=seq_lens,
|
||||
num_reqs=num_reqs,
|
||||
num_actual_tokens=num_actual,
|
||||
max_query_len=max((len(p) for p in per_req_positions), default=1),
|
||||
max_seq_len=int(seq_lens.max()) if num_reqs else 1,
|
||||
block_table_tensor=bt,
|
||||
slot_mapping=slot_mapping,
|
||||
positions=positions if with_positions else None,
|
||||
)
|
||||
|
||||
|
||||
def make_tail_builder(block_size=KPOOL, max_num_batched_tokens=128):
|
||||
builder = object.__new__(KpoolTailMetadataBuilder)
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=block_size)
|
||||
builder.slot_mapping_buffer = torch.empty(max_num_batched_tokens, dtype=torch.int64)
|
||||
return builder
|
||||
|
||||
|
||||
def test_builder_build_uses_circular_mapping():
|
||||
per_req = [list(range(10)), list(range(12))]
|
||||
own_blocks = [5, 9]
|
||||
cam = make_common_metadata(per_req, own_blocks)
|
||||
meta = KpoolTailMetadataBuilder.build(make_tail_builder(), 0, cam)
|
||||
|
||||
out = meta.slot_mapping
|
||||
off = 0
|
||||
for req, prompt in enumerate(per_req):
|
||||
for pos in range(len(prompt)):
|
||||
slot = int(out[off + pos])
|
||||
assert slot // KPOOL == own_blocks[req]
|
||||
assert slot % KPOOL == pos % KPOOL
|
||||
off += len(prompt)
|
||||
# Padding tail of the buffer keeps the -1 sentinel.
|
||||
assert torch.equal(
|
||||
out[cam.num_actual_tokens :], torch.full_like(out[cam.num_actual_tokens :], -1)
|
||||
)
|
||||
|
||||
|
||||
def test_builder_build_falls_back_without_positions():
|
||||
"""Capture / dummy builds without positions keep the generic mapping."""
|
||||
per_req = [list(range(10))]
|
||||
cam = make_common_metadata(per_req, [5], with_positions=False)
|
||||
meta = KpoolTailMetadataBuilder.build(make_tail_builder(), 0, cam)
|
||||
assert meta.slot_mapping is cam.slot_mapping
|
||||
|
||||
|
||||
def test_builder_reuses_slot_mapping_storage():
|
||||
builder = make_tail_builder()
|
||||
first = make_common_metadata([list(range(10))], [5])
|
||||
first_meta = KpoolTailMetadataBuilder.build(builder, 0, first)
|
||||
data_ptr = first_meta.slot_mapping.data_ptr()
|
||||
|
||||
second = make_common_metadata([list(range(12))], [9])
|
||||
second_meta = KpoolTailMetadataBuilder.build(builder, 0, second)
|
||||
|
||||
assert second_meta.slot_mapping.data_ptr() == data_ptr
|
||||
assert second_meta.slot_mapping[:12].tolist() == [
|
||||
9 * KPOOL + pos % KPOOL for pos in range(12)
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Index-level mirror of the tail kernels: seed / stash / pool completion
|
||||
# (addressing replicated from kpool_compress.py's Triton kernels).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TailRingMirror:
|
||||
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
|
||||
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
|
||||
completing at pos reads ring slots (pool_start + s) % kpool and uses the
|
||||
current token's own K/score for the last member."""
|
||||
|
||||
def __init__(self, num_blocks, kpool=KPOOL):
|
||||
self.kpool = kpool
|
||||
self.k = torch.full((num_blocks, kpool, 3), float("nan"))
|
||||
self.s = torch.full((num_blocks, kpool, 3), float("nan"))
|
||||
|
||||
def stash(self, tail_slot, pos, k, s):
|
||||
blk, off = tail_slot // self.kpool, pos % self.kpool
|
||||
self.k[blk, off] = k
|
||||
self.s[blk, off] = s
|
||||
|
||||
seed = stash # the seed kernel writes with the same addressing
|
||||
|
||||
def complete(self, tail_slot, pos, k, s):
|
||||
blk = tail_slot // self.kpool
|
||||
start = pos - (self.kpool - 1)
|
||||
kk = torch.stack(
|
||||
[self.k[blk, (start + i) % self.kpool] for i in range(self.kpool)]
|
||||
)
|
||||
ss = torch.stack(
|
||||
[self.s[blk, (start + i) % self.kpool] for i in range(self.kpool)]
|
||||
)
|
||||
kk[-1], ss[-1] = k, s # is_current for the completing token
|
||||
w = torch.softmax(ss, dim=0)
|
||||
return (kk * w).sum(0)
|
||||
|
||||
|
||||
def token_kv(req, pos):
|
||||
k = torch.tensor([pos + 100.0 * req, pos + 0.5, 2.0 * pos + 0.25])
|
||||
s = torch.tensor([0.1 * (pos + 1) + req, 0.2 * pos, 0.05 * pos])
|
||||
return k, s
|
||||
|
||||
|
||||
def tail_slot_for(mapping, req, pos, own_block):
|
||||
if mapping == "legacy":
|
||||
bt_val = own_block if pos < KPOOL else 0
|
||||
return bt_val * KPOOL + pos % KPOOL
|
||||
return own_block * KPOOL + pos % KPOOL
|
||||
|
||||
|
||||
def run_scenario(mapping, interleave):
|
||||
"""Requests A (block 5, prompt len 9) and B (block 9, prompt len 11)
|
||||
decode concurrently; returns A's boundary pool [8, 9, 10, 11]."""
|
||||
ring = TailRingMirror(num_blocks=16)
|
||||
blocks = {"A": 5, "B": 9}
|
||||
prompts = {"A": 9, "B": 11}
|
||||
|
||||
def slot(req, pos):
|
||||
return tail_slot_for(mapping, 0 if req == "A" else 1, pos, blocks[req])
|
||||
|
||||
# Prefill: seed each request's trailing incomplete pool.
|
||||
for req, L in prompts.items():
|
||||
for pos in range(L - (L % KPOOL or KPOOL), L):
|
||||
if pos < 0:
|
||||
continue
|
||||
ring.stash(slot(req, pos), pos, *token_kv(0 if req == "A" else 1, pos))
|
||||
|
||||
# Decode: A emits pos 9, 10, 11; B emits 11, 12, 13. `interleave`
|
||||
# processes B before A within a step, which is exactly what concurrent
|
||||
# Triton programs do when they share tail block 0.
|
||||
order = ["B", "A"] if interleave else ["A", "B"]
|
||||
decode = {"A": [9, 10, 11], "B": [11, 12, 13]}
|
||||
result = None
|
||||
for step in range(3):
|
||||
for req in order:
|
||||
pos = decode[req][step]
|
||||
k, s = token_kv(0 if req == "A" else 1, pos)
|
||||
if pos % KPOOL == KPOOL - 1:
|
||||
pool = ring.complete(slot(req, pos), pos, k, s)
|
||||
if req == "A":
|
||||
result = pool
|
||||
ring.stash(slot(req, pos), pos, k, s)
|
||||
return result
|
||||
|
||||
|
||||
def test_interleaved_decode_pollution_legacy_vs_circular():
|
||||
"""Ground truth: request A decoding alone (its ring used exclusively)."""
|
||||
torch.manual_seed(0)
|
||||
ground_truth = run_scenario("circular", interleave=False)
|
||||
|
||||
legacy = run_scenario("legacy", interleave=True)
|
||||
circular = run_scenario("circular", interleave=True)
|
||||
|
||||
# The old mapping lets request B's tokens into A's ring: A's boundary
|
||||
# pool is compressed from 2 of B's tokens -> wrong.
|
||||
assert not torch.allclose(legacy, ground_truth), (
|
||||
f"legacy mapping unexpectedly clean: {legacy} vs {ground_truth}"
|
||||
)
|
||||
|
||||
# The circular mapping keeps the rings isolated under interleaving.
|
||||
torch.testing.assert_close(circular, ground_truth)
|
||||
@@ -0,0 +1,127 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
_use_rocm_sparse_triton,
|
||||
fit_kpool_indices_to_aiter,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_sparse_kv_row_offset,
|
||||
_validate_dsv4_sparse_dims,
|
||||
_validate_sparse_dims,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_sparse_kv_row_offset_kernel(slot_ptr, output_ptr, stride: tl.constexpr):
|
||||
slot = tl.load(slot_ptr)
|
||||
tl.store(output_ptr, _sparse_kv_row_offset(slot, stride))
|
||||
|
||||
|
||||
def test_fit_kpool_indices_preserves_tail_and_best_history():
|
||||
token_indices = torch.tensor(
|
||||
[
|
||||
[10, 9, 8, 7, 6, 5, 100, 101],
|
||||
[10, 9, 8, -1, -1, -1, 100, -1],
|
||||
[-1, -1, -1, -1, -1, -1, -1, -1],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=6)
|
||||
|
||||
assert fitted.tolist() == [
|
||||
[10, 9, 8, 7, 100, 101],
|
||||
[10, 9, 8, 100, -1, -1],
|
||||
[-1, -1, -1, -1, -1, -1],
|
||||
]
|
||||
|
||||
|
||||
def test_fit_kpool_indices_exact_width_is_noop():
|
||||
token_indices = torch.tensor([[3, 2, 1, -1]], dtype=torch.int32)
|
||||
|
||||
fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=4)
|
||||
|
||||
assert fitted.data_ptr() == token_indices.data_ptr()
|
||||
|
||||
|
||||
def test_fit_kpool_indices_rejects_narrow_input():
|
||||
with pytest.raises(ValueError, match="at least topk_tokens"):
|
||||
fit_kpool_indices_to_aiter(
|
||||
torch.zeros((1, 3), dtype=torch.int32), topk_tokens=4
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"kv_cache_dtype",
|
||||
"head_size",
|
||||
"num_prefills",
|
||||
"num_decodes",
|
||||
"num_decode_tokens",
|
||||
"max_query_len",
|
||||
"expected",
|
||||
),
|
||||
[
|
||||
("auto", 512, 1, 0, 0, 32, True),
|
||||
("auto", 512, 1, 2, 2, 32, True),
|
||||
("auto", 512, 0, 2, 2, 1, True),
|
||||
("fp8", 512, 1, 0, 0, 32, False),
|
||||
("auto", 576, 1, 0, 0, 32, False),
|
||||
("auto", 512, 0, 2, 4, 2, False),
|
||||
],
|
||||
)
|
||||
def test_rocm_sparse_triton_route(
|
||||
kv_cache_dtype,
|
||||
head_size,
|
||||
num_prefills,
|
||||
num_decodes,
|
||||
num_decode_tokens,
|
||||
max_query_len,
|
||||
expected,
|
||||
):
|
||||
assert (
|
||||
_use_rocm_sparse_triton(
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
head_size=head_size,
|
||||
kv_lora_rank=512,
|
||||
num_prefills=num_prefills,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_sparse_attention_accepts_glm_nope_dimensions():
|
||||
_validate_sparse_dims(512, 512, 0, "test")
|
||||
|
||||
|
||||
def test_rocm_sparse_attention_rejects_inconsistent_dimensions():
|
||||
with pytest.raises(AssertionError, match="expected head_dim"):
|
||||
_validate_sparse_dims(511, 512, 0, "test")
|
||||
|
||||
|
||||
def test_dsv4_sparse_attention_keeps_layout_constraint():
|
||||
_validate_dsv4_sparse_dims(512, 448, 64, "test")
|
||||
with pytest.raises(AssertionError, match="expects 448 NoPE dims"):
|
||||
_validate_dsv4_sparse_dims(512, 512, 0, "test")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
|
||||
def test_sparse_prefill_kv_row_offset_does_not_overflow_int32():
|
||||
# GLM's 640-token pages cross the signed-int32 address boundary at block
|
||||
# 6554 for a 512-element KV row. The production kernel must promote the
|
||||
# slot before multiplying by the row stride.
|
||||
slot = torch.tensor([6554 * 640], dtype=torch.int32, device="cuda")
|
||||
output = torch.empty(1, dtype=torch.int64, device="cuda")
|
||||
|
||||
_store_sparse_kv_row_offset_kernel[(1,)](slot, output, stride=512)
|
||||
|
||||
assert output.item() == 6554 * 640 * 512
|
||||
@@ -0,0 +1,184 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for the decode top-k row->token mapping (no GPU required).
|
||||
|
||||
On non-uniform decode batches (``requires_padding`` -- mixed plain-decode and
|
||||
spec-verify requests, or variable MTP verify lens; taken on Hopper where the
|
||||
varlen/flatten logits path is unavailable), the pool-topk rows follow the
|
||||
PADDED ``[batch_size, next_n]`` grid: row ``(b, t)`` is flat decode token
|
||||
``offset_b + t``. The former inline ``dec_seq = positions[:n] + 1``
|
||||
(``n = batch_size * next_n``) indexes the flat per-token layout with padded
|
||||
coordinates, so rows after the first non-uniform request read another
|
||||
request's positions and rows past the decode region read prefill tokens --
|
||||
``expand_pools_and_append_tail`` then anchors the tail at a foreign length.
|
||||
|
||||
These tests pin the defect with the exact production arithmetic and verify
|
||||
the layout-aware replacement (``_decode_topk_seq_lens``), including the tail
|
||||
expansion consequences via the pure-torch expand/append pair that the fused
|
||||
kernel is documented to replicate.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
# Bootstrap the glm5next package before entering the indexer module: its
|
||||
# kpool_compress import runs glm5next/__init__, which pulls model ->
|
||||
# attention -> back into sparse_attn_indexer_kpool (attention.py imports the
|
||||
# class at module scope). Production always enters via attention.py first.
|
||||
# isort: off
|
||||
import vllm.models.glm5next # noqa: F401
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import ( # noqa: E402
|
||||
append_tail_to_topk,
|
||||
expand_pools_to_tokens,
|
||||
)
|
||||
# isort: on
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer_kpool as indexer_mod
|
||||
from vllm.model_executor.layers.sparse_attn_indexer_kpool import (
|
||||
_decode_topk_seq_lens,
|
||||
_fill_short_decode_causal_indices,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
KPOOL = 4
|
||||
TOPK_TOKENS = 16
|
||||
SELECT_K = TOPK_TOKENS // KPOOL
|
||||
|
||||
|
||||
def test_kpool_ops_dispatch_matches_platform():
|
||||
expected_backend = ".amd." if current_platform.is_rocm() else ".nvidia."
|
||||
assert expected_backend in indexer_mod.kpool_ops.__name__
|
||||
|
||||
|
||||
def test_short_decode_fills_exact_causal_rows():
|
||||
topk = torch.full((3, 8), 99, dtype=torch.int32)
|
||||
positions = torch.tensor([0, 3, 7], dtype=torch.int64)
|
||||
|
||||
assert _fill_short_decode_causal_indices(topk, positions, 3, 8, 8)
|
||||
assert topk.tolist() == [
|
||||
[0, -1, -1, -1, -1, -1, -1, -1],
|
||||
[0, 1, 2, 3, -1, -1, -1, -1],
|
||||
[0, 1, 2, 3, 4, 5, 6, 7],
|
||||
]
|
||||
|
||||
|
||||
def test_short_decode_leaves_buffer_unchanged_for_sparse_context():
|
||||
topk = torch.full((2, 8), 99, dtype=torch.int32)
|
||||
before = topk.clone()
|
||||
|
||||
assert not _fill_short_decode_causal_indices(topk, torch.tensor([7, 8]), 2, 9, 8)
|
||||
assert torch.equal(topk, before)
|
||||
|
||||
|
||||
def make_non_uniform_batch():
|
||||
"""3 requests: plain decode (1 token), MTP verify (4), adaptive verify (3).
|
||||
|
||||
Flat decode positions (production layout: decode tokens first, then any
|
||||
prefill tokens of the same batch):
|
||||
req0: [30] (context len 30)
|
||||
req1: [100..103] (verify at context len 100)
|
||||
req2: [7, 8, 9] (verify at context len 7)
|
||||
Followed by 4 prefill tokens at positions 555..558 so the flat tensor is
|
||||
at least ``n = 3 * 4`` long, as it is in a real mixed batch.
|
||||
"""
|
||||
decode_lens = torch.tensor([1, 4, 3], dtype=torch.int64)
|
||||
per_req_positions = [[30], [100, 101, 102, 103], [7, 8, 9]]
|
||||
flat_decode = [p for req in per_req_positions for p in req]
|
||||
positions = torch.tensor(flat_decode + [555, 556, 557, 558], dtype=torch.int64)
|
||||
return decode_lens, per_req_positions, positions
|
||||
|
||||
|
||||
def expected_row_seq_lens(per_req_positions, batch_size, next_n):
|
||||
"""Ground truth: row (b, t) -> pos + 1 for real rows, 0 for pad rows."""
|
||||
out = torch.zeros(batch_size * next_n, dtype=torch.int32)
|
||||
for b, req in enumerate(per_req_positions):
|
||||
for t, pos in enumerate(req):
|
||||
out[b * next_n + t] = pos + 1
|
||||
return out
|
||||
|
||||
|
||||
def test_legacy_flat_layout_misaligns_and_bleeds():
|
||||
"""The bug: ``positions[:n] + 1`` with padded-row coordinates reads other
|
||||
requests' positions and prefill positions."""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
batch_size = decode_lens.shape[0]
|
||||
next_n = int(decode_lens.max())
|
||||
n = batch_size * next_n
|
||||
assert n == 12
|
||||
|
||||
legacy = positions[:n].to(torch.int32) + 1
|
||||
expected = expected_row_seq_lens(per_req, batch_size, next_n)
|
||||
|
||||
# Row (1, 0): req1's first verify token (pos 100, seq 101) reads flat
|
||||
# index 4 -> req1's LAST verify token (pos 103). Its true tail token is
|
||||
# dropped and the tail anchors 3 tokens late.
|
||||
assert int(legacy[4]) == 104 and int(expected[4]) == 101
|
||||
|
||||
# Rows (2, *): req2 starts at flat offset 5, but padded coordinates point
|
||||
# at flat indices 8..11 -- the batch's PREFILL positions.
|
||||
for t in range(3):
|
||||
assert int(legacy[2 * next_n + t]) >= 556, (
|
||||
"legacy row (2, t) should read prefill positions"
|
||||
)
|
||||
assert not torch.equal(legacy, expected)
|
||||
|
||||
|
||||
def test_helper_uniform_layout_matches_flat_slice():
|
||||
"""Uniform batches keep the flat shortcut (zero behavior/perf change)."""
|
||||
per_req = [[200, 201, 202, 203], [50, 51, 52, 53]]
|
||||
positions = torch.tensor([p for r in per_req for p in r], dtype=torch.int64)
|
||||
decode_lens = torch.tensor([4, 4], dtype=torch.int64)
|
||||
out = _decode_topk_seq_lens(positions, decode_lens, 8, 2, 4, requires_padding=False)
|
||||
assert torch.equal(out, positions[:8].to(torch.int32) + 1)
|
||||
|
||||
|
||||
def test_helper_padded_layout_per_row():
|
||||
"""Non-uniform batches map every padded row to its own token's position;
|
||||
pad rows get 0 (empty tail)."""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
out = _decode_topk_seq_lens(positions, decode_lens, 8, 3, 4, requires_padding=True)
|
||||
expected = expected_row_seq_lens(per_req, 3, 4)
|
||||
assert torch.equal(out, expected)
|
||||
# Pad rows (0, 1..3) and (2, 3) collapse to 0 -> no tail appended.
|
||||
assert out[1] == 0 and out[2] == 0 and out[3] == 0 and out[11] == 0
|
||||
|
||||
|
||||
def expand_tail_region(dec_seq):
|
||||
"""Run the production pure-torch expand + append pair (the fused kernel
|
||||
replicates it exactly on the identity path) and return the tail columns
|
||||
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1)."""
|
||||
rows = dec_seq.shape[0]
|
||||
pool_ids = torch.arange(SELECT_K, dtype=torch.int64).expand(rows, SELECT_K)
|
||||
valid = torch.ones_like(pool_ids, dtype=torch.bool)
|
||||
expanded = expand_pools_to_tokens(pool_ids, valid, TOPK_TOKENS, KPOOL)
|
||||
seq_lens = dec_seq.to(torch.int32)
|
||||
pool_lens = (seq_lens // KPOOL).to(torch.int32)
|
||||
out = append_tail_to_topk(expanded, seq_lens, pool_lens, KPOOL)
|
||||
return out[:, TOPK_TOKENS:]
|
||||
|
||||
|
||||
def test_tail_expansion_legacy_vs_fixed():
|
||||
"""End-to-end tail consequence: the fixed mapping appends exactly the
|
||||
request's trailing incomplete pool; the legacy flat slice drops real tail
|
||||
tokens and emits indices far past the request's own sequence."""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
n = 12
|
||||
fixed = _decode_topk_seq_lens(
|
||||
positions, decode_lens, 8, 3, 4, requires_padding=True
|
||||
)
|
||||
legacy = positions[:n].to(torch.int32) + 1
|
||||
|
||||
fixed_tail = expand_tail_region(fixed)
|
||||
legacy_tail = expand_tail_region(legacy)
|
||||
|
||||
# Fixed: row (1, 0) (seq 101) keeps its single tail token 100; row (2, 2)
|
||||
# (seq 10) keeps its full trailing pool [8, 9].
|
||||
assert fixed_tail[4, 0].item() == 100
|
||||
assert fixed_tail[10, :2].tolist() == [8, 9]
|
||||
|
||||
# Legacy: row (1, 0) loses its tail token (anchored at 104, count 0)...
|
||||
assert legacy_tail[4, 0].item() == -1
|
||||
# ...and row (2, 2) reads prefill position 557 -> tail indices 556/557,
|
||||
# way past req2's 10-token sequence -> out-of-bounds block-table reads.
|
||||
assert legacy_tail[10, :2].tolist() == [556, 557]
|
||||
|
||||
assert not torch.equal(legacy_tail, fixed_tail)
|
||||
@@ -40,11 +40,13 @@ if not current_platform.is_cuda():
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as flashinfer_sparse_mod
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseImpl,
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
@@ -82,6 +84,60 @@ SPARSE_BACKEND_BATCH_SPECS["large_q_pure_prefill"] = BatchSpec(
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
|
||||
"""Weight absorption must not change the model's attention temperature."""
|
||||
model_scale = 256**-0.5
|
||||
kv_lora_rank = 512
|
||||
topk = torch.zeros((1, 1), dtype=torch.int32)
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.zeros(1, dtype=torch.int32),
|
||||
block_table=torch.zeros((1, 1), dtype=torch.int32),
|
||||
block_size=1,
|
||||
)
|
||||
recorded_scale = None
|
||||
|
||||
impl = object.__new__(FlashInferMLASparseImpl)
|
||||
impl.scale = model_scale
|
||||
impl.qk_nope_head_dim = 256
|
||||
impl.kv_lora_rank = kv_lora_rank
|
||||
impl.qk_rope_head_dim = 0
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.topk_indices_buffer = topk
|
||||
impl.dcp_world_size = 1
|
||||
impl._workspace_buffer = torch.empty(1)
|
||||
impl.bmm1_scale = None
|
||||
impl.bmm2_scale = None
|
||||
impl.is_nope_mla = True
|
||||
impl.need_to_return_lse_for_decode = False
|
||||
monkeypatch.setattr(
|
||||
flashinfer_sparse_mod,
|
||||
"triton_convert_req_index_to_global_index",
|
||||
lambda *args, **kwargs: (topk, torch.ones(1, dtype=torch.int32)),
|
||||
)
|
||||
|
||||
import flashinfer.decode
|
||||
|
||||
def fake_flashinfer(**kwargs):
|
||||
nonlocal recorded_scale
|
||||
recorded_scale = kwargs["bmm1_scale"]
|
||||
return torch.zeros((1, 1, 1, kv_lora_rank))
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer.decode,
|
||||
"trtllm_batch_decode_with_kv_cache_mla",
|
||||
fake_flashinfer,
|
||||
)
|
||||
impl.forward_mqa(
|
||||
torch.zeros(1, 1, kv_lora_rank),
|
||||
torch.zeros(1, 1, kv_lora_rank),
|
||||
metadata,
|
||||
SimpleNamespace(),
|
||||
)
|
||||
|
||||
assert recorded_scale == model_scale
|
||||
assert recorded_scale != kv_lora_rank**-0.5
|
||||
|
||||
|
||||
def _float_to_e8m0_truncate(f: float) -> float:
|
||||
"""Simulate SM100's float -> e8m0 -> bf16 scale conversion.
|
||||
e8m0 format only stores the exponent (power of 2).
|
||||
@@ -1451,14 +1507,14 @@ def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
|
||||
return_valid_counts=False,
|
||||
)
|
||||
assert isinstance(result_only, torch.Tensor)
|
||||
torch.testing.assert_close(result_only, result, rtol=0, atol=0)
|
||||
for row, num_valid in enumerate(expected_valid):
|
||||
compact_valid = result[row, :num_valid].sort().values
|
||||
original_valid = result_only[row][result_only[row] >= 0].sort().values
|
||||
torch.testing.assert_close(compact_valid, original_valid, rtol=0, atol=0)
|
||||
assert torch.all(result[row, num_valid:] == -1)
|
||||
|
||||
|
||||
def test_flashmla_cache_dtype_aliases_use_ds_layout():
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype,
|
||||
)
|
||||
|
||||
# kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer
|
||||
# stores kv_cache_dtype, so they cannot bypass the gate.
|
||||
for alias in ("fp8", "fp8_e4m3"):
|
||||
|
||||
Reference in New Issue
Block a user