[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for error handling in the CUDA attention backend probe.
|
||||
"""Tests for CUDA attention backend selection and probe error handling.
|
||||
|
||||
Environment-shaped probe failures (missing packages, unreadable caches,
|
||||
broken driver installs) must mark the backend unavailable rather than
|
||||
@@ -89,6 +89,48 @@ def test_get_valid_backends_keeps_probing_after_failure():
|
||||
assert valid
|
||||
|
||||
|
||||
def test_sm90_nope_mla_prefers_flashinfer_without_changing_rope_order():
|
||||
backend_cls = MagicMock()
|
||||
backend_cls.validate_configuration.return_value = []
|
||||
sparse_backends = {
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
|
||||
}
|
||||
|
||||
def sparse_order(head_size: int) -> list[AttentionBackendEnum]:
|
||||
config = SELECTOR_CONFIG._replace(
|
||||
head_size=head_size,
|
||||
use_mla=True,
|
||||
use_sparse=True,
|
||||
)
|
||||
with patch(
|
||||
"vllm.platforms.cuda._get_attn_backend_class",
|
||||
return_value=backend_cls,
|
||||
):
|
||||
valid, _ = CudaPlatform.get_valid_backends(
|
||||
device_capability=SM90,
|
||||
attn_selector_config=config,
|
||||
num_heads=32,
|
||||
)
|
||||
return [
|
||||
candidate.backend
|
||||
for candidate in valid
|
||||
if candidate.backend in sparse_backends
|
||||
]
|
||||
|
||||
assert sparse_order(512) == [
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
]
|
||||
assert sparse_order(576) == [
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
|
||||
]
|
||||
|
||||
|
||||
def test_selected_backend_probe_failure_raises_value_error_with_cause():
|
||||
exc = OSError("libcuda.so.1: cannot open shared object file")
|
||||
with (
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for the FlashInfer SM90 sparse MLA backend wiring (no GPU).
|
||||
|
||||
The FlashInfer wrapper and top-k conversion are replaced by CPU recorders;
|
||||
the tests pin the contract between the impl and the kernel API: page_size=1
|
||||
varlen rows, reserved-buffer refresh, plan parameters (dims, NoPE/rope scale,
|
||||
causality), ckv/kpe cache splitting, and the backend's model-shape gates.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# isort: off
|
||||
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm90 as sm90_mod
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm90 import (
|
||||
FlashInferMLASparseSM90Backend,
|
||||
FlashInferMLASparseSM90Builder,
|
||||
FlashInferMLASparseSM90Impl,
|
||||
)
|
||||
# isort: on
|
||||
|
||||
BLOCK_SIZE = 64
|
||||
HEAD = 512
|
||||
TOPK = 128 # triton convert requires width % 128 == 0
|
||||
|
||||
|
||||
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
|
||||
out = torch.full_like(token_indices, -1)
|
||||
counts = torch.zeros(token_indices.shape[0], dtype=torch.int32)
|
||||
for t in range(token_indices.shape[0]):
|
||||
vals = []
|
||||
for j in range(token_indices.shape[1]):
|
||||
pos = int(token_indices[t, j])
|
||||
if pos == -1:
|
||||
continue
|
||||
blk = int(block_table[int(req_id[t]), pos // BLOCK_SIZE])
|
||||
if blk < 0:
|
||||
continue
|
||||
vals.append(blk * BLOCK_SIZE + pos % BLOCK_SIZE)
|
||||
out[t, : len(vals)] = torch.tensor(vals, dtype=out.dtype)
|
||||
counts[t] = len(vals)
|
||||
return out, counts
|
||||
|
||||
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.plan_args = None
|
||||
self.run_args = None
|
||||
|
||||
def plan(self, *args, **kwargs):
|
||||
self.plan_args = (args, kwargs)
|
||||
|
||||
def run(self, *args, **kwargs):
|
||||
q_nope, q_pe, ckv, kpe = args
|
||||
self.run_args = (q_nope, q_pe, ckv, kpe, kwargs)
|
||||
return torch.zeros(
|
||||
q_nope.shape[0], q_nope.shape[1], ckv.shape[-1], dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
|
||||
class FakeState:
|
||||
def __init__(self, width, max_tokens=64):
|
||||
self.kv_indices = torch.zeros(max_tokens * width, dtype=torch.int32)
|
||||
self.kv_len_arr = torch.zeros(max_tokens, dtype=torch.int32)
|
||||
self.wrapper = FakeWrapper()
|
||||
self.plan_calls = []
|
||||
|
||||
def plan(self, num_tokens, kv_lens):
|
||||
self.plan_calls.append((num_tokens, kv_lens))
|
||||
|
||||
|
||||
def make_impl(qk_rope, kv_dtype="fp8_e4m3", num_heads=2, topk_width=TOPK):
|
||||
impl = object.__new__(FlashInferMLASparseSM90Impl)
|
||||
impl.num_heads = num_heads
|
||||
impl.head_size = HEAD + qk_rope
|
||||
impl.scale = (HEAD + qk_rope) ** -0.5
|
||||
impl.kv_lora_rank = HEAD
|
||||
impl.qk_rope_head_dim = qk_rope
|
||||
impl.kv_cache_dtype = kv_dtype
|
||||
impl.use_fp8_kv_cache = kv_dtype in ("fp8", "fp8_e4m3")
|
||||
rows = 4
|
||||
impl.topk_indices_buffer = torch.full((rows, topk_width), -1, dtype=torch.int32)
|
||||
return impl, rows
|
||||
|
||||
|
||||
def make_batch(rows, topk_rows, own_blocks):
|
||||
req_id = torch.tensor([0] * rows, dtype=torch.int32)
|
||||
block_table = torch.zeros(1, 16, dtype=torch.int32)
|
||||
block_table[:, 0] = own_blocks[0]
|
||||
topk = torch.full((rows, TOPK), -1, dtype=torch.int32)
|
||||
for t, row in enumerate(topk_rows):
|
||||
topk[t, : len(row)] = torch.tensor(row, dtype=torch.int32)
|
||||
return SimpleNamespace(
|
||||
req_id_per_token=req_id, block_table=block_table, block_size=BLOCK_SIZE
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("qk_rope,kv_dtype", [(0, "fp8_e4m3"), (64, "auto")])
|
||||
def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
impl, rows = make_impl(qk_rope, kv_dtype)
|
||||
state = FakeState(TOPK)
|
||||
monkeypatch.setattr(
|
||||
sm90_mod, "triton_convert_req_index_to_global_index", ref_convert
|
||||
)
|
||||
|
||||
# req with context 10 < topk: 8 valid + -1 padding.
|
||||
topk_rows = [
|
||||
[7, 3, 1, 9, 0, 2, 5, 8] + [-1] * (TOPK - 8),
|
||||
[4, 0, 2, 3, 1] + [-1] * (TOPK - 5),
|
||||
[6, 5, 4] + [-1] * (TOPK - 3),
|
||||
[2] + [-1] * (TOPK - 1),
|
||||
]
|
||||
meta = make_batch(rows, topk_rows, [3])
|
||||
meta.state = state
|
||||
q_nope = torch.randn(rows, impl.num_heads, HEAD)
|
||||
q_rope = torch.randn(rows, impl.num_heads, qk_rope)
|
||||
cache = torch.zeros(
|
||||
8 * BLOCK_SIZE,
|
||||
impl.head_size,
|
||||
dtype=torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16,
|
||||
)
|
||||
|
||||
out, lse = impl.forward_mqa(
|
||||
(q_nope, q_rope), cache, meta, SimpleNamespace(_k_scale_float=0.5)
|
||||
)
|
||||
assert lse is None and out.shape == (rows, impl.num_heads, HEAD)
|
||||
|
||||
# Reserved buffers carry this step's slots; lengths are NOT refreshed
|
||||
# here (the builder plans them host-side before capture/replay).
|
||||
ref_slots, ref_counts = ref_convert(
|
||||
meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer
|
||||
)
|
||||
width = TOPK
|
||||
got_slots = state.kv_indices[: rows * width].view(rows, width)
|
||||
for t in range(rows):
|
||||
k = int(ref_counts[t])
|
||||
assert got_slots[t, :k].tolist() == ref_slots[t, :k].tolist()
|
||||
assert state.plan_calls == []
|
||||
|
||||
assert state.wrapper.run_args is not None
|
||||
q_pe, ckv, kpe, kwargs = state.wrapper.run_args[1:]
|
||||
assert q_pe.shape == (rows, impl.num_heads, qk_rope)
|
||||
assert ckv.shape == (8 * BLOCK_SIZE, 1, HEAD)
|
||||
assert kpe.shape[-1] == qk_rope
|
||||
if impl.use_fp8_kv_cache:
|
||||
assert kwargs["ckv_scale"] == 0.5 and kwargs["kpe_scale"] == 1.0
|
||||
else:
|
||||
assert kwargs == {}
|
||||
|
||||
|
||||
def test_builder_attaches_its_state(monkeypatch):
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
builder._async_scheduling = False
|
||||
builder.state = FakeState(TOPK)
|
||||
metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata)
|
||||
metadata.state = None
|
||||
monkeypatch.setattr(
|
||||
sm90_mod.FlashInferMLASparseMetadataBuilder,
|
||||
"build",
|
||||
lambda *_args, **_kwargs: metadata,
|
||||
)
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=1,
|
||||
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([1], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([1], dtype=torch.int32),
|
||||
positions=None,
|
||||
)
|
||||
|
||||
result = builder.build(0, cam)
|
||||
|
||||
assert result.state is builder.state
|
||||
assert builder.state.plan_calls[0][0] == 1
|
||||
assert builder.state.plan_calls[0][1].tolist() == [1]
|
||||
|
||||
|
||||
def test_plan_uses_state_params(monkeypatch):
|
||||
"""The NoPE/rope dims and scale live on the builder state, not the layer.
|
||||
|
||||
plan() takes exact per-row KV lengths; the schedule is rebuilt on every
|
||||
call (contexts grow between steps) and the indptrs are always full-size
|
||||
with zero-query padding rows past num_tokens.
|
||||
"""
|
||||
impl, rows = make_impl(64, "auto")
|
||||
wrapper = FakeWrapper()
|
||||
state = sm90_mod._SM90State.__new__(sm90_mod._SM90State)
|
||||
state.device = torch.device("cpu")
|
||||
state.wrapper = wrapper
|
||||
state.num_heads = 4
|
||||
state.kv_dtype = torch.bfloat16
|
||||
state.kv_lora_rank = HEAD
|
||||
state.qk_rope_head_dim = 64
|
||||
state.sm_scale = 576**-0.5
|
||||
state.max_tokens = 4
|
||||
state.topk_width = TOPK
|
||||
state.kv_indices = torch.zeros(4 * TOPK)
|
||||
state._arange_cpu = torch.arange(5, dtype=torch.int32)
|
||||
state._qo_cpu = torch.empty(5, dtype=torch.int32)
|
||||
state._kv_cpu = torch.empty(5, dtype=torch.int32)
|
||||
state._lens_cpu = torch.full((4,), TOPK, dtype=torch.int32)
|
||||
|
||||
state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32))
|
||||
assert wrapper.plan_args is not None
|
||||
args, kwargs = wrapper.plan_args
|
||||
(qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args
|
||||
assert qo.tolist() == [0, 1, 2, 3, 3] # clamp: rows past 3 have no queries
|
||||
assert kv.tolist() == [i * TOPK for i in (0, 1, 2, 3, 3)]
|
||||
assert kv_len.tolist() == [2, 5, 7, TOPK] # padded row keeps full width
|
||||
assert (heads, ckv, kpe, page, causal) == (4, HEAD, 64, 1, False)
|
||||
assert scale == 576**-0.5
|
||||
assert kwargs["q_data_type"] == torch.bfloat16
|
||||
assert kwargs["kv_data_type"] == torch.bfloat16
|
||||
|
||||
|
||||
def test_kv_lens_host_formula():
|
||||
"""Per-row host lengths: context == position + 1; capped at
|
||||
index_topk + trailing-pool remainder past the sparse threshold."""
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
builder._async_scheduling = False
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=3,
|
||||
query_start_loc_cpu=torch.tensor([0, 5, 7, 10], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([100, 9, 3000], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([100, 9, 3000], dtype=torch.int32),
|
||||
positions=None,
|
||||
)
|
||||
num_rows, lens = builder._kv_lens_host(cam)
|
||||
assert num_rows == 10
|
||||
# req0: positions 95..99 -> ctx 96..100 (all <= 2048: full context)
|
||||
# req1: positions 7,8 -> ctx 8,9
|
||||
# req2: positions 2997..2999 -> ctx 2998..3000 (> 2048: topk + ctx%4)
|
||||
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2050, 2051, 2048]
|
||||
|
||||
|
||||
def test_kv_lens_host_empty():
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=0,
|
||||
query_start_loc_cpu=torch.tensor([0], dtype=torch.int32),
|
||||
seq_lens=torch.zeros(0, dtype=torch.int32),
|
||||
)
|
||||
num_rows, lens = builder._kv_lens_host(cam)
|
||||
assert num_rows == 0 and lens.numel() == 0
|
||||
|
||||
|
||||
def test_supports_combination_gates(monkeypatch, default_vllm_config):
|
||||
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True)
|
||||
call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8_e4m3",
|
||||
block_size=64,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=True,
|
||||
use_mm_prefix=False,
|
||||
device_capability=SimpleNamespace(major=9),
|
||||
**kw,
|
||||
)
|
||||
assert call() is None # no model config: only the feature gate applies
|
||||
|
||||
import vllm.config as cfg
|
||||
|
||||
monkeypatch.setattr(
|
||||
cfg,
|
||||
"get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=None),
|
||||
)
|
||||
assert call() is None
|
||||
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: False)
|
||||
assert "requires FlashInfer" in (call() or "")
|
||||
@@ -0,0 +1,369 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for the kpool tail slot mapping (no GPU required).
|
||||
|
||||
The kpool tail cache is a 1-block-per-request circular ring addressed by
|
||||
``pos % kpool`` (``KpoolTailSpec`` / ``KpoolTailManager``: exactly one block
|
||||
allocated per request, never grown, so only column 0 of its block table is
|
||||
ever written; the rest stays zero-initialized).
|
||||
|
||||
The generic per-group slot kernel cannot express that layout: it maps
|
||||
``pos -> bt[req][pos // bs] * bs + pos % bs`` (``_compute_slot_mappings_kernel``
|
||||
in vllm/v1/worker/gpu/block_table.py), so every token at ``pos >= kpool``
|
||||
reads a zero column and collapses onto physical tail block 0. All concurrent
|
||||
requests then share one ``kpool``-slot ring and corrupt each other's pool
|
||||
compression.
|
||||
|
||||
These tests pin that defect's arithmetic, verify the circular replacement
|
||||
(``compute_kpool_tail_slot_mapping``), and mirror the tail kernels' index
|
||||
math to show cross-request pollution before the fix and isolation after.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
KpoolTailBackend,
|
||||
KpoolTailMetadataBuilder,
|
||||
compute_kpool_tail_slot_mapping,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
KPOOL = 4
|
||||
|
||||
|
||||
def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
|
||||
(layout,) = KpoolTailBackend.supported_kv_cache_layouts()
|
||||
spec = KpoolTailSpec(
|
||||
block_size=KPOOL,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=KPOOL,
|
||||
)
|
||||
strides = compute_layout_strides(spec, num_blocks=8, num_layers=3, layout=layout)
|
||||
_, _, head_stride, state_stride, content_stride = strides
|
||||
|
||||
assert layout is KVCacheLayout.LBHNC
|
||||
assert head_stride == KPOOL * 128 * torch.bfloat16.itemsize
|
||||
assert state_stride == 128 * torch.bfloat16.itemsize
|
||||
assert content_stride == 1
|
||||
|
||||
|
||||
def make_tail_block_table(own_blocks, width=64):
|
||||
"""Tail-group block table as BlockTables produces it: column 0 holds the
|
||||
request's single KpoolTailManager block, the remaining columns are never
|
||||
written and stay zero."""
|
||||
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
|
||||
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
|
||||
return bt
|
||||
|
||||
|
||||
def legacy_generic_tail_slots(block_table, query_start_loc, positions):
|
||||
"""Reference of the generic ``_compute_slot_mappings_kernel`` arithmetic
|
||||
(block_table.py:305-313) applied to the tail group's table."""
|
||||
slots = []
|
||||
for req in range(block_table.shape[0]):
|
||||
for i in range(query_start_loc[req], query_start_loc[req + 1]):
|
||||
pos = int(positions[i])
|
||||
block_number = int(block_table[req, pos // KPOOL])
|
||||
slots.append(block_number * KPOOL + pos % KPOOL)
|
||||
return torch.tensor(slots, dtype=torch.int64)
|
||||
|
||||
|
||||
def circular_tail_slots(
|
||||
slot_mapping, block_table, query_start_loc, positions, num_actual, num_reqs
|
||||
):
|
||||
return compute_kpool_tail_slot_mapping(
|
||||
slot_mapping,
|
||||
block_table,
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_actual,
|
||||
num_reqs,
|
||||
KPOOL,
|
||||
)
|
||||
|
||||
|
||||
def make_batch(per_req_positions, padded_len=None):
|
||||
positions = torch.cat(
|
||||
[torch.tensor(p, dtype=torch.int64) for p in per_req_positions]
|
||||
)
|
||||
num_actual = positions.numel()
|
||||
num_reqs = len(per_req_positions)
|
||||
lens = [len(p) for p in per_req_positions]
|
||||
qsl = torch.zeros(num_reqs + 1, dtype=torch.int64)
|
||||
torch.cumsum(torch.tensor(lens, dtype=torch.int64), 0, out=qsl[1:])
|
||||
if padded_len is None:
|
||||
padded_len = num_actual
|
||||
slot_mapping = torch.full((padded_len,), -1, dtype=torch.int64)
|
||||
return positions, qsl, slot_mapping, num_actual, num_reqs
|
||||
|
||||
|
||||
def test_legacy_generic_mapping_collapses_onto_block_zero():
|
||||
"""The bug: with the manager's 1-column block table, the generic kernel
|
||||
maps every pos >= kpool onto tail block 0, and distinct requests collide."""
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))] # prompts of len 10 and 12
|
||||
positions, qsl, _, num_actual, num_reqs = make_batch(per_req)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
legacy = legacy_generic_tail_slots(bt, qsl, positions)
|
||||
|
||||
# Every token at pos >= kpool resolves to block 0, not the request's own.
|
||||
off = 0
|
||||
for req, prompt in enumerate(per_req):
|
||||
req_slots = legacy[off : off + len(prompt)]
|
||||
for pos in range(len(prompt)):
|
||||
slot = int(req_slots[pos])
|
||||
if pos >= KPOOL:
|
||||
assert slot // KPOOL == 0, (
|
||||
f"expected collapse onto block 0 at pos {pos}"
|
||||
)
|
||||
assert slot // KPOOL != own_blocks[req]
|
||||
off += len(prompt)
|
||||
|
||||
# The two requests share ring slots -> cross-request pollution.
|
||||
a_slots = set(legacy[: len(per_req[0])].tolist())
|
||||
b_slots = set(legacy[len(per_req[0]) :].tolist())
|
||||
assert a_slots & b_slots, "legacy mapping must collide across requests"
|
||||
|
||||
|
||||
def test_circular_mapping_isolates_requests():
|
||||
"""The fix: every token lands in its own request's block at pos % kpool,
|
||||
and no slot is ever shared by two different requests (slots do recur
|
||||
within a request every kpool positions -- that is the circular design)."""
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))]
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
|
||||
|
||||
off = 0
|
||||
per_req_slots = []
|
||||
for req, prompt in enumerate(per_req):
|
||||
req_slots = set()
|
||||
for pos in range(len(prompt)):
|
||||
slot = int(out[off + pos])
|
||||
assert slot // KPOOL == own_blocks[req], (
|
||||
f"req {req} pos {pos} left its tail block"
|
||||
)
|
||||
assert slot % KPOOL == pos % KPOOL
|
||||
req_slots.add(slot)
|
||||
per_req_slots.append(req_slots)
|
||||
off += len(prompt)
|
||||
assert not per_req_slots[0] & per_req_slots[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt_len", [1, 2, 3, 4])
|
||||
def test_circular_mapping_matches_generic_for_short_requests(prompt_len):
|
||||
"""For pos < kpool the generic kernel already picks the own block, so the
|
||||
two mappings agree while every position fits the request's first block
|
||||
(single-request behavior is unchanged)."""
|
||||
own_blocks = [7]
|
||||
per_req = [list(range(prompt_len))]
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
legacy = legacy_generic_tail_slots(bt, qsl, positions)
|
||||
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
|
||||
assert torch.equal(out, legacy)
|
||||
|
||||
|
||||
def test_circular_mapping_preserves_padding_and_empty_batch():
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))]
|
||||
padded_len = sum(len(p) for p in per_req) + 8
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(
|
||||
per_req, padded_len=padded_len
|
||||
)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
|
||||
assert out.shape == slot_mapping.shape
|
||||
assert torch.equal(out[num_actual:], torch.full_like(out[num_actual:], -1))
|
||||
|
||||
empty = circular_tail_slots(slot_mapping, bt, qsl, positions[:0], 0, num_reqs)
|
||||
assert torch.equal(empty, slot_mapping)
|
||||
|
||||
|
||||
def make_common_metadata(per_req_positions, own_blocks, with_positions=True):
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(
|
||||
per_req_positions, padded_len=sum(len(p) for p in per_req_positions) + 4
|
||||
)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
seq_lens = torch.tensor(
|
||||
[max(p) + 1 if p else 1 for p in per_req_positions], dtype=torch.int64
|
||||
)
|
||||
return CommonAttentionMetadata(
|
||||
query_start_loc=qsl,
|
||||
query_start_loc_cpu=qsl.clone(),
|
||||
seq_lens=seq_lens,
|
||||
num_reqs=num_reqs,
|
||||
num_actual_tokens=num_actual,
|
||||
max_query_len=max((len(p) for p in per_req_positions), default=1),
|
||||
max_seq_len=int(seq_lens.max()) if num_reqs else 1,
|
||||
block_table_tensor=bt,
|
||||
slot_mapping=slot_mapping,
|
||||
positions=positions if with_positions else None,
|
||||
)
|
||||
|
||||
|
||||
def make_tail_builder(block_size=KPOOL, max_num_batched_tokens=128):
|
||||
builder = object.__new__(KpoolTailMetadataBuilder)
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=block_size)
|
||||
builder.slot_mapping_buffer = torch.empty(max_num_batched_tokens, dtype=torch.int64)
|
||||
return builder
|
||||
|
||||
|
||||
def test_builder_build_uses_circular_mapping():
|
||||
per_req = [list(range(10)), list(range(12))]
|
||||
own_blocks = [5, 9]
|
||||
cam = make_common_metadata(per_req, own_blocks)
|
||||
meta = KpoolTailMetadataBuilder.build(make_tail_builder(), 0, cam)
|
||||
|
||||
out = meta.slot_mapping
|
||||
off = 0
|
||||
for req, prompt in enumerate(per_req):
|
||||
for pos in range(len(prompt)):
|
||||
slot = int(out[off + pos])
|
||||
assert slot // KPOOL == own_blocks[req]
|
||||
assert slot % KPOOL == pos % KPOOL
|
||||
off += len(prompt)
|
||||
# Padding tail of the buffer keeps the -1 sentinel.
|
||||
assert torch.equal(
|
||||
out[cam.num_actual_tokens :], torch.full_like(out[cam.num_actual_tokens :], -1)
|
||||
)
|
||||
|
||||
|
||||
def test_builder_build_falls_back_without_positions():
|
||||
"""Capture / dummy builds without positions keep the generic mapping."""
|
||||
per_req = [list(range(10))]
|
||||
cam = make_common_metadata(per_req, [5], with_positions=False)
|
||||
meta = KpoolTailMetadataBuilder.build(make_tail_builder(), 0, cam)
|
||||
assert meta.slot_mapping is cam.slot_mapping
|
||||
|
||||
|
||||
def test_builder_reuses_slot_mapping_storage():
|
||||
builder = make_tail_builder()
|
||||
first = make_common_metadata([list(range(10))], [5])
|
||||
first_meta = KpoolTailMetadataBuilder.build(builder, 0, first)
|
||||
data_ptr = first_meta.slot_mapping.data_ptr()
|
||||
|
||||
second = make_common_metadata([list(range(12))], [9])
|
||||
second_meta = KpoolTailMetadataBuilder.build(builder, 0, second)
|
||||
|
||||
assert second_meta.slot_mapping.data_ptr() == data_ptr
|
||||
assert second_meta.slot_mapping[:12].tolist() == [
|
||||
9 * KPOOL + pos % KPOOL for pos in range(12)
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Index-level mirror of the tail kernels: seed / stash / pool completion
|
||||
# (addressing replicated from kpool_compress.py's Triton kernels).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TailRingMirror:
|
||||
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
|
||||
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
|
||||
completing at pos reads ring slots (pool_start + s) % kpool and uses the
|
||||
current token's own K/score for the last member."""
|
||||
|
||||
def __init__(self, num_blocks, kpool=KPOOL):
|
||||
self.kpool = kpool
|
||||
self.k = torch.full((num_blocks, kpool, 3), float("nan"))
|
||||
self.s = torch.full((num_blocks, kpool, 3), float("nan"))
|
||||
|
||||
def stash(self, tail_slot, pos, k, s):
|
||||
blk, off = tail_slot // self.kpool, pos % self.kpool
|
||||
self.k[blk, off] = k
|
||||
self.s[blk, off] = s
|
||||
|
||||
seed = stash # the seed kernel writes with the same addressing
|
||||
|
||||
def complete(self, tail_slot, pos, k, s):
|
||||
blk = tail_slot // self.kpool
|
||||
start = pos - (self.kpool - 1)
|
||||
kk = torch.stack(
|
||||
[self.k[blk, (start + i) % self.kpool] for i in range(self.kpool)]
|
||||
)
|
||||
ss = torch.stack(
|
||||
[self.s[blk, (start + i) % self.kpool] for i in range(self.kpool)]
|
||||
)
|
||||
kk[-1], ss[-1] = k, s # is_current for the completing token
|
||||
w = torch.softmax(ss, dim=0)
|
||||
return (kk * w).sum(0)
|
||||
|
||||
|
||||
def token_kv(req, pos):
|
||||
k = torch.tensor([pos + 100.0 * req, pos + 0.5, 2.0 * pos + 0.25])
|
||||
s = torch.tensor([0.1 * (pos + 1) + req, 0.2 * pos, 0.05 * pos])
|
||||
return k, s
|
||||
|
||||
|
||||
def tail_slot_for(mapping, req, pos, own_block):
|
||||
if mapping == "legacy":
|
||||
bt_val = own_block if pos < KPOOL else 0
|
||||
return bt_val * KPOOL + pos % KPOOL
|
||||
return own_block * KPOOL + pos % KPOOL
|
||||
|
||||
|
||||
def run_scenario(mapping, interleave):
|
||||
"""Requests A (block 5, prompt len 9) and B (block 9, prompt len 11)
|
||||
decode concurrently; returns A's boundary pool [8, 9, 10, 11]."""
|
||||
ring = TailRingMirror(num_blocks=16)
|
||||
blocks = {"A": 5, "B": 9}
|
||||
prompts = {"A": 9, "B": 11}
|
||||
|
||||
def slot(req, pos):
|
||||
return tail_slot_for(mapping, 0 if req == "A" else 1, pos, blocks[req])
|
||||
|
||||
# Prefill: seed each request's trailing incomplete pool.
|
||||
for req, L in prompts.items():
|
||||
for pos in range(L - (L % KPOOL or KPOOL), L):
|
||||
if pos < 0:
|
||||
continue
|
||||
ring.stash(slot(req, pos), pos, *token_kv(0 if req == "A" else 1, pos))
|
||||
|
||||
# Decode: A emits pos 9, 10, 11; B emits 11, 12, 13. `interleave`
|
||||
# processes B before A within a step, which is exactly what concurrent
|
||||
# Triton programs do when they share tail block 0.
|
||||
order = ["B", "A"] if interleave else ["A", "B"]
|
||||
decode = {"A": [9, 10, 11], "B": [11, 12, 13]}
|
||||
result = None
|
||||
for step in range(3):
|
||||
for req in order:
|
||||
pos = decode[req][step]
|
||||
k, s = token_kv(0 if req == "A" else 1, pos)
|
||||
if pos % KPOOL == KPOOL - 1:
|
||||
pool = ring.complete(slot(req, pos), pos, k, s)
|
||||
if req == "A":
|
||||
result = pool
|
||||
ring.stash(slot(req, pos), pos, k, s)
|
||||
return result
|
||||
|
||||
|
||||
def test_interleaved_decode_pollution_legacy_vs_circular():
|
||||
"""Ground truth: request A decoding alone (its ring used exclusively)."""
|
||||
torch.manual_seed(0)
|
||||
ground_truth = run_scenario("circular", interleave=False)
|
||||
|
||||
legacy = run_scenario("legacy", interleave=True)
|
||||
circular = run_scenario("circular", interleave=True)
|
||||
|
||||
# The old mapping lets request B's tokens into A's ring: A's boundary
|
||||
# pool is compressed from 2 of B's tokens -> wrong.
|
||||
assert not torch.allclose(legacy, ground_truth), (
|
||||
f"legacy mapping unexpectedly clean: {legacy} vs {ground_truth}"
|
||||
)
|
||||
|
||||
# The circular mapping keeps the rings isolated under interleaving.
|
||||
torch.testing.assert_close(circular, ground_truth)
|
||||
@@ -0,0 +1,127 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
_use_rocm_sparse_triton,
|
||||
fit_kpool_indices_to_aiter,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_sparse_kv_row_offset,
|
||||
_validate_dsv4_sparse_dims,
|
||||
_validate_sparse_dims,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_sparse_kv_row_offset_kernel(slot_ptr, output_ptr, stride: tl.constexpr):
|
||||
slot = tl.load(slot_ptr)
|
||||
tl.store(output_ptr, _sparse_kv_row_offset(slot, stride))
|
||||
|
||||
|
||||
def test_fit_kpool_indices_preserves_tail_and_best_history():
|
||||
token_indices = torch.tensor(
|
||||
[
|
||||
[10, 9, 8, 7, 6, 5, 100, 101],
|
||||
[10, 9, 8, -1, -1, -1, 100, -1],
|
||||
[-1, -1, -1, -1, -1, -1, -1, -1],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=6)
|
||||
|
||||
assert fitted.tolist() == [
|
||||
[10, 9, 8, 7, 100, 101],
|
||||
[10, 9, 8, 100, -1, -1],
|
||||
[-1, -1, -1, -1, -1, -1],
|
||||
]
|
||||
|
||||
|
||||
def test_fit_kpool_indices_exact_width_is_noop():
|
||||
token_indices = torch.tensor([[3, 2, 1, -1]], dtype=torch.int32)
|
||||
|
||||
fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=4)
|
||||
|
||||
assert fitted.data_ptr() == token_indices.data_ptr()
|
||||
|
||||
|
||||
def test_fit_kpool_indices_rejects_narrow_input():
|
||||
with pytest.raises(ValueError, match="at least topk_tokens"):
|
||||
fit_kpool_indices_to_aiter(
|
||||
torch.zeros((1, 3), dtype=torch.int32), topk_tokens=4
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"kv_cache_dtype",
|
||||
"head_size",
|
||||
"num_prefills",
|
||||
"num_decodes",
|
||||
"num_decode_tokens",
|
||||
"max_query_len",
|
||||
"expected",
|
||||
),
|
||||
[
|
||||
("auto", 512, 1, 0, 0, 32, True),
|
||||
("auto", 512, 1, 2, 2, 32, True),
|
||||
("auto", 512, 0, 2, 2, 1, True),
|
||||
("fp8", 512, 1, 0, 0, 32, False),
|
||||
("auto", 576, 1, 0, 0, 32, False),
|
||||
("auto", 512, 0, 2, 4, 2, False),
|
||||
],
|
||||
)
|
||||
def test_rocm_sparse_triton_route(
|
||||
kv_cache_dtype,
|
||||
head_size,
|
||||
num_prefills,
|
||||
num_decodes,
|
||||
num_decode_tokens,
|
||||
max_query_len,
|
||||
expected,
|
||||
):
|
||||
assert (
|
||||
_use_rocm_sparse_triton(
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
head_size=head_size,
|
||||
kv_lora_rank=512,
|
||||
num_prefills=num_prefills,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_sparse_attention_accepts_glm_nope_dimensions():
|
||||
_validate_sparse_dims(512, 512, 0, "test")
|
||||
|
||||
|
||||
def test_rocm_sparse_attention_rejects_inconsistent_dimensions():
|
||||
with pytest.raises(AssertionError, match="expected head_dim"):
|
||||
_validate_sparse_dims(511, 512, 0, "test")
|
||||
|
||||
|
||||
def test_dsv4_sparse_attention_keeps_layout_constraint():
|
||||
_validate_dsv4_sparse_dims(512, 448, 64, "test")
|
||||
with pytest.raises(AssertionError, match="expects 448 NoPE dims"):
|
||||
_validate_dsv4_sparse_dims(512, 512, 0, "test")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
|
||||
def test_sparse_prefill_kv_row_offset_does_not_overflow_int32():
|
||||
# GLM's 640-token pages cross the signed-int32 address boundary at block
|
||||
# 6554 for a 512-element KV row. The production kernel must promote the
|
||||
# slot before multiplying by the row stride.
|
||||
slot = torch.tensor([6554 * 640], dtype=torch.int32, device="cuda")
|
||||
output = torch.empty(1, dtype=torch.int64, device="cuda")
|
||||
|
||||
_store_sparse_kv_row_offset_kernel[(1,)](slot, output, stride=512)
|
||||
|
||||
assert output.item() == 6554 * 640 * 512
|
||||
@@ -0,0 +1,184 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for the decode top-k row->token mapping (no GPU required).
|
||||
|
||||
On non-uniform decode batches (``requires_padding`` -- mixed plain-decode and
|
||||
spec-verify requests, or variable MTP verify lens; taken on Hopper where the
|
||||
varlen/flatten logits path is unavailable), the pool-topk rows follow the
|
||||
PADDED ``[batch_size, next_n]`` grid: row ``(b, t)`` is flat decode token
|
||||
``offset_b + t``. The former inline ``dec_seq = positions[:n] + 1``
|
||||
(``n = batch_size * next_n``) indexes the flat per-token layout with padded
|
||||
coordinates, so rows after the first non-uniform request read another
|
||||
request's positions and rows past the decode region read prefill tokens --
|
||||
``expand_pools_and_append_tail`` then anchors the tail at a foreign length.
|
||||
|
||||
These tests pin the defect with the exact production arithmetic and verify
|
||||
the layout-aware replacement (``_decode_topk_seq_lens``), including the tail
|
||||
expansion consequences via the pure-torch expand/append pair that the fused
|
||||
kernel is documented to replicate.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
# Bootstrap the glm5next package before entering the indexer module: its
|
||||
# kpool_compress import runs glm5next/__init__, which pulls model ->
|
||||
# attention -> back into sparse_attn_indexer_kpool (attention.py imports the
|
||||
# class at module scope). Production always enters via attention.py first.
|
||||
# isort: off
|
||||
import vllm.models.glm5next # noqa: F401
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import ( # noqa: E402
|
||||
append_tail_to_topk,
|
||||
expand_pools_to_tokens,
|
||||
)
|
||||
# isort: on
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer_kpool as indexer_mod
|
||||
from vllm.model_executor.layers.sparse_attn_indexer_kpool import (
|
||||
_decode_topk_seq_lens,
|
||||
_fill_short_decode_causal_indices,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
KPOOL = 4
|
||||
TOPK_TOKENS = 16
|
||||
SELECT_K = TOPK_TOKENS // KPOOL
|
||||
|
||||
|
||||
def test_kpool_ops_dispatch_matches_platform():
|
||||
expected_backend = ".amd." if current_platform.is_rocm() else ".nvidia."
|
||||
assert expected_backend in indexer_mod.kpool_ops.__name__
|
||||
|
||||
|
||||
def test_short_decode_fills_exact_causal_rows():
|
||||
topk = torch.full((3, 8), 99, dtype=torch.int32)
|
||||
positions = torch.tensor([0, 3, 7], dtype=torch.int64)
|
||||
|
||||
assert _fill_short_decode_causal_indices(topk, positions, 3, 8, 8)
|
||||
assert topk.tolist() == [
|
||||
[0, -1, -1, -1, -1, -1, -1, -1],
|
||||
[0, 1, 2, 3, -1, -1, -1, -1],
|
||||
[0, 1, 2, 3, 4, 5, 6, 7],
|
||||
]
|
||||
|
||||
|
||||
def test_short_decode_leaves_buffer_unchanged_for_sparse_context():
|
||||
topk = torch.full((2, 8), 99, dtype=torch.int32)
|
||||
before = topk.clone()
|
||||
|
||||
assert not _fill_short_decode_causal_indices(topk, torch.tensor([7, 8]), 2, 9, 8)
|
||||
assert torch.equal(topk, before)
|
||||
|
||||
|
||||
def make_non_uniform_batch():
|
||||
"""3 requests: plain decode (1 token), MTP verify (4), adaptive verify (3).
|
||||
|
||||
Flat decode positions (production layout: decode tokens first, then any
|
||||
prefill tokens of the same batch):
|
||||
req0: [30] (context len 30)
|
||||
req1: [100..103] (verify at context len 100)
|
||||
req2: [7, 8, 9] (verify at context len 7)
|
||||
Followed by 4 prefill tokens at positions 555..558 so the flat tensor is
|
||||
at least ``n = 3 * 4`` long, as it is in a real mixed batch.
|
||||
"""
|
||||
decode_lens = torch.tensor([1, 4, 3], dtype=torch.int64)
|
||||
per_req_positions = [[30], [100, 101, 102, 103], [7, 8, 9]]
|
||||
flat_decode = [p for req in per_req_positions for p in req]
|
||||
positions = torch.tensor(flat_decode + [555, 556, 557, 558], dtype=torch.int64)
|
||||
return decode_lens, per_req_positions, positions
|
||||
|
||||
|
||||
def expected_row_seq_lens(per_req_positions, batch_size, next_n):
|
||||
"""Ground truth: row (b, t) -> pos + 1 for real rows, 0 for pad rows."""
|
||||
out = torch.zeros(batch_size * next_n, dtype=torch.int32)
|
||||
for b, req in enumerate(per_req_positions):
|
||||
for t, pos in enumerate(req):
|
||||
out[b * next_n + t] = pos + 1
|
||||
return out
|
||||
|
||||
|
||||
def test_legacy_flat_layout_misaligns_and_bleeds():
|
||||
"""The bug: ``positions[:n] + 1`` with padded-row coordinates reads other
|
||||
requests' positions and prefill positions."""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
batch_size = decode_lens.shape[0]
|
||||
next_n = int(decode_lens.max())
|
||||
n = batch_size * next_n
|
||||
assert n == 12
|
||||
|
||||
legacy = positions[:n].to(torch.int32) + 1
|
||||
expected = expected_row_seq_lens(per_req, batch_size, next_n)
|
||||
|
||||
# Row (1, 0): req1's first verify token (pos 100, seq 101) reads flat
|
||||
# index 4 -> req1's LAST verify token (pos 103). Its true tail token is
|
||||
# dropped and the tail anchors 3 tokens late.
|
||||
assert int(legacy[4]) == 104 and int(expected[4]) == 101
|
||||
|
||||
# Rows (2, *): req2 starts at flat offset 5, but padded coordinates point
|
||||
# at flat indices 8..11 -- the batch's PREFILL positions.
|
||||
for t in range(3):
|
||||
assert int(legacy[2 * next_n + t]) >= 556, (
|
||||
"legacy row (2, t) should read prefill positions"
|
||||
)
|
||||
assert not torch.equal(legacy, expected)
|
||||
|
||||
|
||||
def test_helper_uniform_layout_matches_flat_slice():
|
||||
"""Uniform batches keep the flat shortcut (zero behavior/perf change)."""
|
||||
per_req = [[200, 201, 202, 203], [50, 51, 52, 53]]
|
||||
positions = torch.tensor([p for r in per_req for p in r], dtype=torch.int64)
|
||||
decode_lens = torch.tensor([4, 4], dtype=torch.int64)
|
||||
out = _decode_topk_seq_lens(positions, decode_lens, 8, 2, 4, requires_padding=False)
|
||||
assert torch.equal(out, positions[:8].to(torch.int32) + 1)
|
||||
|
||||
|
||||
def test_helper_padded_layout_per_row():
|
||||
"""Non-uniform batches map every padded row to its own token's position;
|
||||
pad rows get 0 (empty tail)."""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
out = _decode_topk_seq_lens(positions, decode_lens, 8, 3, 4, requires_padding=True)
|
||||
expected = expected_row_seq_lens(per_req, 3, 4)
|
||||
assert torch.equal(out, expected)
|
||||
# Pad rows (0, 1..3) and (2, 3) collapse to 0 -> no tail appended.
|
||||
assert out[1] == 0 and out[2] == 0 and out[3] == 0 and out[11] == 0
|
||||
|
||||
|
||||
def expand_tail_region(dec_seq):
|
||||
"""Run the production pure-torch expand + append pair (the fused kernel
|
||||
replicates it exactly on the identity path) and return the tail columns
|
||||
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1)."""
|
||||
rows = dec_seq.shape[0]
|
||||
pool_ids = torch.arange(SELECT_K, dtype=torch.int64).expand(rows, SELECT_K)
|
||||
valid = torch.ones_like(pool_ids, dtype=torch.bool)
|
||||
expanded = expand_pools_to_tokens(pool_ids, valid, TOPK_TOKENS, KPOOL)
|
||||
seq_lens = dec_seq.to(torch.int32)
|
||||
pool_lens = (seq_lens // KPOOL).to(torch.int32)
|
||||
out = append_tail_to_topk(expanded, seq_lens, pool_lens, KPOOL)
|
||||
return out[:, TOPK_TOKENS:]
|
||||
|
||||
|
||||
def test_tail_expansion_legacy_vs_fixed():
|
||||
"""End-to-end tail consequence: the fixed mapping appends exactly the
|
||||
request's trailing incomplete pool; the legacy flat slice drops real tail
|
||||
tokens and emits indices far past the request's own sequence."""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
n = 12
|
||||
fixed = _decode_topk_seq_lens(
|
||||
positions, decode_lens, 8, 3, 4, requires_padding=True
|
||||
)
|
||||
legacy = positions[:n].to(torch.int32) + 1
|
||||
|
||||
fixed_tail = expand_tail_region(fixed)
|
||||
legacy_tail = expand_tail_region(legacy)
|
||||
|
||||
# Fixed: row (1, 0) (seq 101) keeps its single tail token 100; row (2, 2)
|
||||
# (seq 10) keeps its full trailing pool [8, 9].
|
||||
assert fixed_tail[4, 0].item() == 100
|
||||
assert fixed_tail[10, :2].tolist() == [8, 9]
|
||||
|
||||
# Legacy: row (1, 0) loses its tail token (anchored at 104, count 0)...
|
||||
assert legacy_tail[4, 0].item() == -1
|
||||
# ...and row (2, 2) reads prefill position 557 -> tail indices 556/557,
|
||||
# way past req2's 10-token sequence -> out-of-bounds block-table reads.
|
||||
assert legacy_tail[10, :2].tolist() == [556, 557]
|
||||
|
||||
assert not torch.equal(legacy_tail, fixed_tail)
|
||||
@@ -40,11 +40,13 @@ if not current_platform.is_cuda():
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as flashinfer_sparse_mod
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseImpl,
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
@@ -82,6 +84,60 @@ SPARSE_BACKEND_BATCH_SPECS["large_q_pure_prefill"] = BatchSpec(
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
|
||||
"""Weight absorption must not change the model's attention temperature."""
|
||||
model_scale = 256**-0.5
|
||||
kv_lora_rank = 512
|
||||
topk = torch.zeros((1, 1), dtype=torch.int32)
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.zeros(1, dtype=torch.int32),
|
||||
block_table=torch.zeros((1, 1), dtype=torch.int32),
|
||||
block_size=1,
|
||||
)
|
||||
recorded_scale = None
|
||||
|
||||
impl = object.__new__(FlashInferMLASparseImpl)
|
||||
impl.scale = model_scale
|
||||
impl.qk_nope_head_dim = 256
|
||||
impl.kv_lora_rank = kv_lora_rank
|
||||
impl.qk_rope_head_dim = 0
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.topk_indices_buffer = topk
|
||||
impl.dcp_world_size = 1
|
||||
impl._workspace_buffer = torch.empty(1)
|
||||
impl.bmm1_scale = None
|
||||
impl.bmm2_scale = None
|
||||
impl.is_nope_mla = True
|
||||
impl.need_to_return_lse_for_decode = False
|
||||
monkeypatch.setattr(
|
||||
flashinfer_sparse_mod,
|
||||
"triton_convert_req_index_to_global_index",
|
||||
lambda *args, **kwargs: (topk, torch.ones(1, dtype=torch.int32)),
|
||||
)
|
||||
|
||||
import flashinfer.decode
|
||||
|
||||
def fake_flashinfer(**kwargs):
|
||||
nonlocal recorded_scale
|
||||
recorded_scale = kwargs["bmm1_scale"]
|
||||
return torch.zeros((1, 1, 1, kv_lora_rank))
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer.decode,
|
||||
"trtllm_batch_decode_with_kv_cache_mla",
|
||||
fake_flashinfer,
|
||||
)
|
||||
impl.forward_mqa(
|
||||
torch.zeros(1, 1, kv_lora_rank),
|
||||
torch.zeros(1, 1, kv_lora_rank),
|
||||
metadata,
|
||||
SimpleNamespace(),
|
||||
)
|
||||
|
||||
assert recorded_scale == model_scale
|
||||
assert recorded_scale != kv_lora_rank**-0.5
|
||||
|
||||
|
||||
def _float_to_e8m0_truncate(f: float) -> float:
|
||||
"""Simulate SM100's float -> e8m0 -> bf16 scale conversion.
|
||||
e8m0 format only stores the exponent (power of 2).
|
||||
@@ -1451,14 +1507,14 @@ def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
|
||||
return_valid_counts=False,
|
||||
)
|
||||
assert isinstance(result_only, torch.Tensor)
|
||||
torch.testing.assert_close(result_only, result, rtol=0, atol=0)
|
||||
for row, num_valid in enumerate(expected_valid):
|
||||
compact_valid = result[row, :num_valid].sort().values
|
||||
original_valid = result_only[row][result_only[row] >= 0].sort().values
|
||||
torch.testing.assert_close(compact_valid, original_valid, rtol=0, atol=0)
|
||||
assert torch.all(result[row, num_valid:] == -1)
|
||||
|
||||
|
||||
def test_flashmla_cache_dtype_aliases_use_ds_layout():
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype,
|
||||
)
|
||||
|
||||
# kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer
|
||||
# stores kv_cache_dtype, so they cannot bypass the gate.
|
||||
for alias in ("fp8", "fp8_e4m3"):
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
@@ -1801,6 +1802,127 @@ def test_hybrid_sliding_window_group_disables_partial_hash_hits():
|
||||
assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed
|
||||
|
||||
|
||||
def test_opted_out_scratch_group_keeps_partial_hash_hits():
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
KpoolTailSpec(
|
||||
block_size=mamba_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=mamba_block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
assert manager.coordinator.enable_partial_hash_hits
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
manager.free(req0)
|
||||
manager.new_step_starts()
|
||||
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||
_, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
|
||||
|
||||
def test_kpool_tail_supports_128_token_partial_hash_hits():
|
||||
hash_block_size = 128
|
||||
cache_block_size = 9 * hash_block_size
|
||||
kpool_block_size = 64
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=cache_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=cache_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
KpoolTailSpec(
|
||||
block_size=kpool_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=kpool_block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
assert manager.coordinator.scheduler_block_size == cache_block_size
|
||||
assert manager.coordinator.hash_block_size == hash_block_size
|
||||
assert manager.coordinator.enable_partial_hash_hits
|
||||
|
||||
shared_prefix = [10] * (12 * hash_block_size)
|
||||
req0 = make_request("0", shared_prefix, hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert (
|
||||
manager.allocate_slots(req0, len(shared_prefix), num_computed, computed_blocks)
|
||||
is not None
|
||||
)
|
||||
manager.free(req0)
|
||||
manager.new_step_starts()
|
||||
|
||||
req1 = make_request("1", shared_prefix + [12] * 64, hash_block_size, sha256)
|
||||
_, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == len(shared_prefix)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dcp_world_size", [1, 2, 4])
|
||||
def test_hybrid_partial_hash_hit_uses_cow_under_dcp(dcp_world_size: int):
|
||||
hash_block_size = 2
|
||||
|
||||
@@ -6,13 +6,19 @@ import importlib
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
from vllm.config import CacheConfig, ModelConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -47,6 +53,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
@@ -2191,34 +2198,567 @@ def test_generate_scheduler_kv_cache_config():
|
||||
)
|
||||
|
||||
|
||||
def test_mixed_precision_kv_cache_with_uniform_type_specs():
|
||||
fp8_spec = new_kv_cache_spec(dtype=torch.float8_e4m3fn)
|
||||
bf16_spec = new_kv_cache_spec(dtype=torch.bfloat16)
|
||||
worker_config = KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["fp8_layer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=16, kv_cache_specs={"fp8_layer": fp8_spec}
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["bf16_layer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=16, kv_cache_specs={"bf16_layer": bf16_spec}
|
||||
),
|
||||
),
|
||||
],
|
||||
def _glm5_like_kv_cache_spec(
|
||||
mamba_spec_factory=new_mamba_spec,
|
||||
) -> tuple[dict[str, KVCacheSpec], list[str]]:
|
||||
"""(mamba, mamba, mamba, MLA + indexer) * 11 plus a trailing mamba."""
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
mamba_layers = []
|
||||
for i in range(45):
|
||||
if i % 4 == 3:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=132,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=16,
|
||||
)
|
||||
else:
|
||||
name = f"layers.{i}.linear_attn"
|
||||
kv_cache_spec[name] = mamba_spec_factory()
|
||||
mamba_layers.append(name)
|
||||
return kv_cache_spec, mamba_layers
|
||||
|
||||
|
||||
def _glm5_like_kv_cache_spec_with_tail(
|
||||
mamba_spec_factory=new_mamba_spec,
|
||||
) -> dict[str, KVCacheSpec]:
|
||||
"""Production kpool=4 proportions: (mamba*3, MLA + indexer + tail) * 11.
|
||||
|
||||
The tail's logical page (2 * kpool * 2*indexer_head_dim * bf16) must fit
|
||||
inside the indexer page it parasitizes (block_size//kpool * 132 B), which
|
||||
is why the fixture drops the tokens_per_state=16 shape used above.
|
||||
"""
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec(mamba_spec_factory)
|
||||
for i in range(3, 45, 4):
|
||||
kpool = 4
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = replace(
|
||||
cast(MLAAttentionSpec, kv_cache_spec[f"layers.{i}.indexer"]),
|
||||
tokens_per_state=kpool,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.tail"] = KpoolTailSpec(
|
||||
block_size=kpool,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=kpool,
|
||||
)
|
||||
return kv_cache_spec
|
||||
|
||||
|
||||
def _tensor_by_layer(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheTensor]:
|
||||
return {
|
||||
layer_name: tensor
|
||||
for tensor in kv_cache_config.kv_cache_tensors
|
||||
for layer_name in tensor.layers
|
||||
}
|
||||
|
||||
|
||||
def _layer_offset(tensor: KVCacheTensor, layer_name: str) -> int:
|
||||
return tensor.offset + tensor.layers.index(layer_name) * tensor.layer_stride
|
||||
|
||||
|
||||
def test_get_kv_cache_config_balanced_mamba_hybrid():
|
||||
"""Hybrid slot sharing: mamba layers co-own the MLA slot tensors."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
uniform_groups = [
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
]
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
|
||||
# Round-robin into G = ceil(34 / 11) = 4 groups so every mamba layer
|
||||
# gets an MLA slot.
|
||||
assert len(groups) == 5
|
||||
assert len(uniform_groups) == 1
|
||||
assert len(uniform_groups[0].layer_names) == 22
|
||||
assert uniform_groups[0].kv_cache_spec.get_max_layers_per_page_size() == 11
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
|
||||
for k, name in enumerate(mamba_layers):
|
||||
assert name in mamba_groups[k % 4].layer_names
|
||||
# Mamba pages are padded up to the MLA page.
|
||||
for group in mamba_groups:
|
||||
assert group.kv_cache_spec.page_size_padded == mla_page
|
||||
assert group.kv_cache_spec.page_size_bytes == mla_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 11 * mla_page + 11 * idx_page
|
||||
|
||||
# Every block id is charged the full per-block sum.
|
||||
attn_blocks = uniform_groups[0].kv_cache_spec.max_memory_usage_pages(vllm_config)
|
||||
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
|
||||
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group
|
||||
assert (
|
||||
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
|
||||
== blocks_per_request * bytes_per_block
|
||||
)
|
||||
scheduler_config = generate_scheduler_kv_cache_config([worker_config])
|
||||
|
||||
assert worker_config.needs_kv_cache_zeroing
|
||||
assert scheduler_config.needs_kv_cache_zeroing
|
||||
available_memory = bytes_per_block * 100 + 1
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, available_memory
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
|
||||
# Every logical layer has a view into one backing allocation. Mamba views
|
||||
# alias their corresponding MLA slot by using the same byte offset.
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 56
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
mla_layer_names = [f"layers.{i}.attn" for i in range(3, 45, 4)]
|
||||
for i, mla_name in enumerate(mla_layer_names):
|
||||
mla_tensor = tensors[mla_name]
|
||||
assert mla_tensor.block_stride == mla_page
|
||||
for group in mamba_groups:
|
||||
if i < len(group.layer_names):
|
||||
assert tensors[group.layer_names[i]].offset == mla_tensor.offset
|
||||
for i in range(11):
|
||||
assert tensors[f"layers.{4 * i + 3}.indexer"].block_stride == idx_page
|
||||
|
||||
total_allocated = next(iter({t.size for t in kv_cache_config.kv_cache_tensors}))
|
||||
assert total_allocated == bytes_per_block * 100
|
||||
assert 0 <= available_memory - total_allocated < bytes_per_block
|
||||
|
||||
assert get_max_concurrency_for_kv_cache_config(
|
||||
vllm_config, kv_cache_config
|
||||
) == pytest.approx(100 / blocks_per_request)
|
||||
|
||||
|
||||
def new_mla_spec(cache_dtype_str=None, block_size=16):
|
||||
def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
|
||||
"""The kpool tail parasitizes the indexer tensors instead of getting its
|
||||
own: sibling idx/tail tensors paired by layer order, zero standalone tail
|
||||
bytes, one shared block per request, and no prefix-caching leakage from
|
||||
the kpool-sized scratch group."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec = _glm5_like_kv_cache_spec_with_tail()
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
tail_logical_page = kv_cache_spec["layers.3.tail"].page_size_bytes
|
||||
assert tail_logical_page < idx_page
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
tail_group = next(
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
and all(
|
||||
isinstance(spec, KpoolTailSpec)
|
||||
for spec in group.kv_cache_spec.kv_cache_specs.values()
|
||||
)
|
||||
)
|
||||
# The tail never prefix-caches; its page is padded up to the indexer page
|
||||
# so the runner's strided view rides the indexer storage.
|
||||
assert not tail_group.kv_cache_spec.prefix_cacheable
|
||||
tail_inner = cast(
|
||||
KpoolTailSpec, tail_group.kv_cache_spec.kv_cache_specs["layers.3.tail"]
|
||||
)
|
||||
assert tail_inner.page_size_padded == idx_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 11 * mla_page + 11 * idx_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
# Tail layers get logical views but no additional storage: each view aliases
|
||||
# its sibling indexer at the same offset in the shared allocation.
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 67
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
for i in range(11):
|
||||
idx_name = f"layers.{4 * i + 3}.indexer"
|
||||
tail_name = f"layers.{4 * i + 3}.tail"
|
||||
assert tensors[idx_name].offset == tensors[tail_name].offset
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
# The layout detector surfaces the sibling names in layer order for the
|
||||
# accounting and connector paths.
|
||||
layout = kv_cache_utils._glm5_next_tensor_layout(kv_cache_config.kv_cache_groups)
|
||||
assert layout is not None
|
||||
assert layout[3] == [f"layers.{4 * i + 3}.indexer" for i in range(11)]
|
||||
assert layout[6] == [f"layers.{4 * i + 3}.tail" for i in range(11)]
|
||||
|
||||
# Accounting charges exactly one extra shared block per request for the
|
||||
# tail, not a per-sequence allocation.
|
||||
attn_group = next(
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
and group is not tail_group
|
||||
)
|
||||
attn_blocks = attn_group.kv_cache_spec.max_memory_usage_pages(vllm_config)
|
||||
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
|
||||
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group + 1
|
||||
assert (
|
||||
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
|
||||
== blocks_per_request * bytes_per_block
|
||||
)
|
||||
|
||||
|
||||
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
|
||||
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
|
||||
prefix-cache hash granularity: participating groups alone decide it."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
def align_mamba():
|
||||
return new_mamba_spec(mamba_cache_mode="align")
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(
|
||||
vllm_config, _glm5_like_kv_cache_spec_with_tail(align_mamba)
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
hash_vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=16,
|
||||
enable_prefix_caching=True,
|
||||
prefix_match_unit=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
kv_transfer_config=object(),
|
||||
)
|
||||
# gcd(attn 1024, mamba 16) with the tail's 4 excluded; scheduler size is
|
||||
# the lcm including the tail (1024 % 4 == 0, so it coincides).
|
||||
assert kv_cache_utils.resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, hash_vllm_config
|
||||
) == (1024, 16)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible():
|
||||
"""Reject GLM-5.3-Flash layouts whose Mamba page exceeds the MLA page."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
# (512, 1024) fp32 state = 2 MiB per page > 1.125 MiB MLA page.
|
||||
def big_mamba_spec():
|
||||
return new_mamba_spec(shapes=((512, 1024),), dtypes=(torch.float32,))
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec(big_mamba_spec)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
assert big_mamba_spec().page_size_bytes > mla_page
|
||||
|
||||
with pytest.raises(ValueError, match="does not fit the MLA page"):
|
||||
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
|
||||
"""Use the generic layout error when no kpool indexer is present."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBHNC"
|
||||
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
for i in range(27):
|
||||
if i % 4 == 3 or i == 26:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
else:
|
||||
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec(
|
||||
shapes=((512, 1024),), dtypes=(torch.float32,)
|
||||
)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
assert kv_cache_spec["layers.0.linear_attn"].page_size_bytes > mla_page
|
||||
|
||||
with pytest.raises(NotImplementedError, match="page size"):
|
||||
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
|
||||
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
|
||||
disable slot sharing; the layout re-pads them to the MLA page."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
def prepadded_mamba_spec():
|
||||
return new_mamba_spec(page_size_padded=294_912)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec(prepadded_mamba_spec)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
assert prepadded_mamba_spec().page_size_bytes < mla_page
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
assert len(groups) == 5
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
|
||||
for group in mamba_groups:
|
||||
assert group.kv_cache_spec.page_size_bytes == mla_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 56
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
|
||||
"""Round-robin mamba grouping keeps practical PP splits balanced: every
|
||||
stage's largest projected mamba group slice fits its projected MLA
|
||||
layers, so per-stage slot tensors all have an MLA owner."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
|
||||
global_groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
|
||||
# PP=2 split at transformer layer 22/23.
|
||||
# Stage 0 of a PP=2 split at transformer layer 22/23: 5 MLA(+indexer)
|
||||
# layers, projected mamba groups [5, 5, 4, 4].
|
||||
worker_spec = {n: s for n, s in kv_cache_spec.items() if int(n.split(".")[1]) <= 22}
|
||||
groups = kv_cache_utils._project_kv_cache_groups_to_worker(
|
||||
global_groups, worker_spec
|
||||
)
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [5, 5, 4, 4]
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 5 * mla_page + 5 * idx_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
# Every projected Mamba slot aliases an MLA owner.
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.0.linear_attn"].offset
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.1.linear_attn"].offset
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.2.linear_attn"].offset
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.4.linear_attn"].offset
|
||||
# The last slot only hosts the two groups with a 5th projected layer.
|
||||
assert tensors["layers.19.attn"].offset == tensors["layers.21.linear_attn"].offset
|
||||
assert tensors["layers.19.attn"].offset == tensors["layers.22.linear_attn"].offset
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatch):
|
||||
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
|
||||
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
|
||||
grouping bumps to 5 groups so every stage's projection keeps sharing
|
||||
on instead of silently falling back on stage 0."""
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
|
||||
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(max_model_len=8192),
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=4),
|
||||
)
|
||||
|
||||
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mamba_groups = [g for g in groups if isinstance(g.kv_cache_spec, MambaSpec)]
|
||||
assert [len(g.layer_names) for g in mamba_groups] == [7, 7, 7, 7, 6]
|
||||
for k, name in enumerate(mamba_layers):
|
||||
assert name in mamba_groups[k % 5].layer_names
|
||||
|
||||
for rank in range(4):
|
||||
start, end = get_pp_indices(45, rank, 4)
|
||||
worker_spec = {
|
||||
n: s
|
||||
for n, s in kv_cache_spec.items()
|
||||
if start <= int(n.split(".")[1]) < end
|
||||
}
|
||||
projected = kv_cache_utils._project_kv_cache_groups_to_worker(
|
||||
groups, worker_spec
|
||||
)
|
||||
assert kv_cache_utils._glm5_next_tensor_layout(projected) is not None
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
|
||||
"""Reject PP stages whose Mamba layers have no MLA slot to share."""
|
||||
from vllm.config import ParallelConfig
|
||||
|
||||
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
|
||||
monkeypatch.setenv("VLLM_PP_LAYER_PARTITION", "3,42")
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(max_model_len=8192),
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=2),
|
||||
)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
|
||||
with pytest.raises(ValueError, match="VLLM_PP_LAYER_PARTITION"):
|
||||
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
|
||||
"""Hybrid slot sharing must take precedence over the experimental
|
||||
enable_cross_layers_blocks packed layout: generic packing would give the
|
||||
MLA slots a strided view, breaking the contiguous virtual split."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="NixlConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"enable_cross_layers_blocks": "true"},
|
||||
)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config, kv_transfer_config=kv_transfer_config
|
||||
)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 11 * mla_page + 11 * idx_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 56
|
||||
assert all(
|
||||
tensor.block_stride in (mla_page, idx_page)
|
||||
for tensor in kv_cache_config.kv_cache_tensors
|
||||
)
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_no_indexer():
|
||||
"""Kimi-Linear-like: MLA without indexer layers, idx_stride == 0."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
|
||||
# 20 mamba + 7 MLA layers -> G = ceil(20 / 7) = 3 groups of [7, 7, 6].
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
for i in range(27):
|
||||
if i % 4 == 3 or i == 26:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
else:
|
||||
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
assert len(groups) == 4
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [7, 7, 6]
|
||||
for group in mamba_groups:
|
||||
assert group.kv_cache_spec.page_size_bytes == mla_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 7 * mla_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
# Each group has one tensor descriptor and all groups alias the same slots.
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 4
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
mla_names = [f"layers.{i}.attn" for i in (*range(3, 27, 4), 26)]
|
||||
for index, mla_name in enumerate(mla_names):
|
||||
mla_offset = _layer_offset(tensors[mla_name], mla_name)
|
||||
for group in mamba_groups:
|
||||
if index < len(group.layer_names):
|
||||
mamba_name = group.layer_names[index]
|
||||
assert _layer_offset(tensors[mamba_name], mamba_name) == mla_offset
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
|
||||
def test_get_kv_cache_capacity_after_scheduler_unwrap():
|
||||
"""max_concurrency must survive the scheduler-config unwrap.
|
||||
|
||||
Regression for the balanced-mamba hybrid layout:
|
||||
``generate_scheduler_kv_cache_config`` flattens the MLA
|
||||
``UniformTypeKVCacheSpecs`` group into a single ``MLAAttentionSpec``, so the
|
||||
scheduler config holds an MLA spec (page size A) next to ``MambaSpec``
|
||||
groups (page size B). ``EngineCore._initialize_kv_caches`` calls
|
||||
``get_kv_cache_capacity`` on that unwrapped config; the uniform-page-size
|
||||
path used to assert-fail (``assert len(page_sizes) == 1``) on this topology.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
for i in range(45):
|
||||
if i % 4 == 3:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=132,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=16,
|
||||
)
|
||||
else:
|
||||
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
|
||||
scheduler_config = generate_scheduler_kv_cache_config([kv_cache_config])
|
||||
# Confirm we are exercising the regression path: the MLA group is no longer
|
||||
# UniformTypeKVCacheSpecs, so sibling specs have differing page sizes.
|
||||
assert not any(
|
||||
isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
for g in scheduler_config.kv_cache_groups
|
||||
)
|
||||
|
||||
unwrapped_groups = scheduler_config.kv_cache_groups
|
||||
expected_max_mem = kv_cache_utils._max_memory_usage_bytes_from_groups(
|
||||
vllm_config, unwrapped_groups
|
||||
)
|
||||
expected_pool = kv_cache_utils._pool_bytes_per_block(unwrapped_groups)
|
||||
expected_blocks_per_request = (
|
||||
expected_max_mem + expected_pool - 1
|
||||
) // expected_pool
|
||||
|
||||
_, max_concurrency = get_kv_cache_capacity(vllm_config, scheduler_config)
|
||||
assert max_concurrency > 0
|
||||
assert max_concurrency == pytest.approx(
|
||||
scheduler_config.num_blocks / expected_blocks_per_request
|
||||
)
|
||||
|
||||
|
||||
def new_mla_spec(cache_dtype_str=None, block_size: int = 16):
|
||||
# head_size = kv_lora_rank(512) + qk_rope_head_dim(64) = 576
|
||||
return MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
@@ -3305,9 +3845,10 @@ def _hybrid_specs_with_draft(draft: bool, draft_shares_target_spec: bool = False
|
||||
"""A K3-shaped hybrid: MLA full attention + Mamba, optionally plus a
|
||||
DSpark-style draft MLA layer marked non_causal_multi_token_decode.
|
||||
|
||||
The target's fp8 KV dtype is what keeps the draft in its own bucket, as it
|
||||
does on Kimi-K3 (target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
|
||||
draft_shares_target_spec to collapse them into one group instead.
|
||||
The target's fp8 KV dtype is one difference from the draft on Kimi-K3
|
||||
(target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
|
||||
draft_shares_target_spec to match that dtype while retaining the draft
|
||||
marker as a group boundary.
|
||||
"""
|
||||
target_dtype = None if draft_shares_target_spec else "fp8_e4m3"
|
||||
specs = {
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
@@ -605,3 +606,195 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
|
||||
token_dbs[1].key_for(hs[2]): [10_000 + mamba_cow_block * 512],
|
||||
}
|
||||
assert store.puts == expected
|
||||
|
||||
|
||||
def test_worker_lookup_hits_sub_block_partial_tail():
|
||||
"""worker.lookup must query sub-block keys when partial hash hits are on.
|
||||
|
||||
Regression test: ``lookup`` hard-coded ``fine_grained = False``, so the
|
||||
sub-block keys persisted by ``_sub_block_tail_puts`` were never probed and
|
||||
partial prefix hits silently returned 0 while the store side kept writing
|
||||
them. Here the mamba block (16) exceeds the hash unit (4), so
|
||||
``enable_partial_hash_hits`` is on and the lookup must find the stored
|
||||
boundary at 12.
|
||||
"""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
cfg = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=4 * full.page_size_bytes,
|
||||
layers=["L0"],
|
||||
layer_stride=4 * full.page_size_bytes,
|
||||
block_stride=full.page_size_bytes,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=4 * mamba.page_size_bytes,
|
||||
layers=["L1"],
|
||||
layer_stride=4 * mamba.page_size_bytes,
|
||||
block_stride=mamba.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], mamba),
|
||||
],
|
||||
)
|
||||
vllm_config = _minimal_vllm_config(cache_block_size=16)
|
||||
# Hash unit 4 < mamba block 16 -> partial hash hits are enabled.
|
||||
vllm_config.cache_config.prefix_match_unit = 4
|
||||
store = _DictStore()
|
||||
|
||||
worker = _build_worker_with_dict_store(vllm_config, cfg, store)
|
||||
worker.tp_size = 1
|
||||
worker.pp_size = 1
|
||||
worker.num_kv_head = 8
|
||||
assert worker.coord.enable_partial_hash_hits
|
||||
|
||||
for g_idx, db in enumerate(worker.token_dbs):
|
||||
db.set_kv_caches_base_addr([g_idx * 10_000])
|
||||
db.set_block_len([512])
|
||||
|
||||
send_thread = KVCacheStoreSendingThread(
|
||||
store=store,
|
||||
token_databases=worker.token_dbs,
|
||||
block_size=worker.block_size,
|
||||
coord=worker.coord,
|
||||
tp_rank=0,
|
||||
group_put_steps=[1, 1],
|
||||
kv_role="kv_both",
|
||||
ready_event=threading.Event(),
|
||||
replicate_config=MagicMock(),
|
||||
)
|
||||
|
||||
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//4-1]).
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(5)]
|
||||
req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1], [2]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=20,
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
)
|
||||
send_thread._maybe_offload_boundary_states(req)
|
||||
|
||||
worker.store = store
|
||||
|
||||
# A 13-token prompt sharing the prefix must hit the stored boundary at 12.
|
||||
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 12
|
||||
|
||||
|
||||
def test_worker_setup_tolerates_finer_scratch_group():
|
||||
"""Setup and lookup must tolerate a non-prefix-cacheable scratch group.
|
||||
|
||||
Regression: GLM-5.3-Flash carries a kpool-tail scratch group whose block
|
||||
size (``index_kpool`` tokens) is finer than the hash unit, so the
|
||||
coordinator's divisibility assert and the scratch group's
|
||||
``ChunkedTokenDatabase`` both rejected worker setup for any hash unit the
|
||||
scratch block does not divide. Scratch groups never participate in
|
||||
store/load/lookup, so setup must skip them and lookup must still hit
|
||||
stored sub-block boundaries on the participating groups.
|
||||
"""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
# Scratch block 4 is not divisible by the hash unit 8 below.
|
||||
scratch = KpoolTailSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
)
|
||||
cfg = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=4 * full.page_size_bytes,
|
||||
layers=["L0"],
|
||||
layer_stride=4 * full.page_size_bytes,
|
||||
block_stride=full.page_size_bytes,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=4 * mamba.page_size_bytes,
|
||||
layers=["L1"],
|
||||
layer_stride=4 * mamba.page_size_bytes,
|
||||
block_stride=mamba.page_size_bytes,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=4 * scratch.page_size_bytes,
|
||||
layers=["L2"],
|
||||
layer_stride=4 * scratch.page_size_bytes,
|
||||
block_stride=scratch.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], mamba),
|
||||
KVCacheGroupSpec(["L2"], scratch),
|
||||
],
|
||||
)
|
||||
vllm_config = _minimal_vllm_config(cache_block_size=16)
|
||||
# Hash unit 8 divides the participating groups (16) but not the scratch
|
||||
# group (4); mamba block 16 > 8 keeps partial hash hits on.
|
||||
vllm_config.cache_config.prefix_match_unit = 8
|
||||
store = _DictStore()
|
||||
|
||||
worker = _build_worker_with_dict_store(vllm_config, cfg, store)
|
||||
worker.tp_size = 1
|
||||
worker.pp_size = 1
|
||||
worker.num_kv_head = 8
|
||||
assert worker.coord.enable_partial_hash_hits
|
||||
# The scratch DB is keyed at its own block size and never probed.
|
||||
assert worker.token_dbs[2].hash_block_size == 4
|
||||
|
||||
for g_idx, db in enumerate(worker.token_dbs):
|
||||
db.set_kv_caches_base_addr([g_idx * 10_000])
|
||||
db.set_block_len([512])
|
||||
|
||||
send_thread = KVCacheStoreSendingThread(
|
||||
store=store,
|
||||
token_databases=worker.token_dbs,
|
||||
block_size=worker.block_size,
|
||||
coord=worker.coord,
|
||||
tp_rank=0,
|
||||
group_put_steps=[1, 1, 1],
|
||||
kv_role="kv_both",
|
||||
ready_event=threading.Event(),
|
||||
replicate_config=MagicMock(),
|
||||
group_participates=[True, True, False],
|
||||
)
|
||||
|
||||
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//8-1]).
|
||||
hs = [BlockHash(bytes([i + 1]) * 8) for i in range(3)]
|
||||
req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1], [2], [3]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=20,
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
)
|
||||
send_thread._maybe_offload_boundary_states(req)
|
||||
|
||||
worker.store = store
|
||||
|
||||
# A 13-token prompt sharing the prefix must hit the first hash unit.
|
||||
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 8
|
||||
# The scratch group's namespace never enters the store.
|
||||
scratch_prefix = worker.token_dbs[2].key_for(hs[0]).rsplit("@", 1)[0]
|
||||
assert not any(key.startswith(scratch_prefix) for key in store._data)
|
||||
|
||||
@@ -211,6 +211,167 @@ def _make_mla_hybrid_worker(local_block_size, kernel_block_size, num_logical_blo
|
||||
return worker
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("logical_block_size", [1152, 640])
|
||||
@pytest.mark.parametrize("tail_first", [False, True])
|
||||
def test_register_compressed_indexer_uses_virtual_transfer_pages(
|
||||
logical_block_size, tail_first
|
||||
):
|
||||
"""Compressed indexer rows must split into contiguous NIXL transfer pages."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
|
||||
base_worker as bw,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
create_kv_cache_views,
|
||||
)
|
||||
|
||||
num_logical_blocks = 3
|
||||
transfer_block_size = 64
|
||||
kernel_block_size = 128
|
||||
tokens_per_state = 4
|
||||
state_content_bytes = 132
|
||||
|
||||
indexer_spec = MLAAttentionSpec(
|
||||
block_size=logical_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.uint8,
|
||||
state_content_bytes=state_content_bytes,
|
||||
tokens_per_state=tokens_per_state,
|
||||
)
|
||||
indexer_page_size = indexer_spec.page_size_bytes
|
||||
tail_spec = KpoolTailSpec(
|
||||
block_size=tokens_per_state,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
page_size_padded=indexer_page_size,
|
||||
sliding_window=tokens_per_state,
|
||||
)
|
||||
|
||||
allocation_size = num_logical_blocks * indexer_page_size
|
||||
indexer_tensor = KVCacheTensor(
|
||||
size=allocation_size,
|
||||
layers=["indexer"],
|
||||
layer_stride=allocation_size,
|
||||
block_stride=indexer_page_size,
|
||||
)
|
||||
tail_tensor = KVCacheTensor(
|
||||
size=allocation_size,
|
||||
layers=["tail"],
|
||||
layer_stride=allocation_size,
|
||||
block_stride=indexer_page_size,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_logical_blocks,
|
||||
kv_cache_tensors=[indexer_tensor, tail_tensor],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["indexer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=logical_block_size,
|
||||
kv_cache_specs={"indexer": indexer_spec},
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=tokens_per_state,
|
||||
kv_cache_specs={"tail": tail_spec},
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
raw = torch.zeros(allocation_size, dtype=torch.int8)
|
||||
(indexer_cache,) = create_kv_cache_views(
|
||||
raw,
|
||||
indexer_spec,
|
||||
num_logical_blocks,
|
||||
KVCacheLayout.LBHNC,
|
||||
indexer_tensor,
|
||||
kernel_block_size=kernel_block_size,
|
||||
)
|
||||
(tail_cache,) = create_kv_cache_views(
|
||||
raw,
|
||||
tail_spec,
|
||||
num_logical_blocks,
|
||||
KVCacheLayout.LBHNC,
|
||||
tail_tensor,
|
||||
)
|
||||
assert indexer_cache.data_ptr() == tail_cache.data_ptr() == raw.data_ptr()
|
||||
|
||||
vllm_config = create_vllm_config(block_size=logical_block_size)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBHNC"
|
||||
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
|
||||
fake_backend = MagicMock()
|
||||
fake_backend.get_supported_kernel_block_sizes.return_value = [transfer_block_size]
|
||||
fake_backend.get_name.return_value = "DEEPSEEK_V32_INDEXER"
|
||||
fake_backend.full_cls_name.return_value = "fake.DEEPSEEK_V32_INDEXER"
|
||||
fake_platform = MagicMock()
|
||||
fake_platform.device_type = "cuda"
|
||||
fake_platform.get_nixl_memory_type.return_value = "VRAM"
|
||||
|
||||
caches = [("indexer", indexer_cache), ("tail", tail_cache)]
|
||||
if tail_first:
|
||||
caches.reverse()
|
||||
|
||||
with (
|
||||
patch.object(bw, "NixlWrapper", _RecordingNixl),
|
||||
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
|
||||
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
|
||||
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
|
||||
patch.object(bw, "current_platform", fake_platform),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config)
|
||||
worker.use_mla = True
|
||||
worker.register_kv_caches(dict(caches))
|
||||
|
||||
transfer_page_size = transfer_block_size // tokens_per_state * state_content_bytes
|
||||
num_transfer_blocks = num_logical_blocks * (
|
||||
logical_block_size // transfer_block_size
|
||||
)
|
||||
expected_descs = np.asarray(
|
||||
[
|
||||
[
|
||||
raw.data_ptr() + block_idx * transfer_page_size,
|
||||
transfer_page_size,
|
||||
0,
|
||||
]
|
||||
for block_idx in range(num_transfer_blocks)
|
||||
],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
|
||||
assert worker.block_size == transfer_block_size
|
||||
assert worker.num_regions == 1
|
||||
assert worker.block_len_per_layer == [transfer_page_size]
|
||||
assert worker.block_stride_per_layer == [transfer_page_size]
|
||||
assert worker._region_is_mla == [True]
|
||||
assert worker.kv_caches_base_addr[worker.engine_id][0] == [raw.data_ptr()]
|
||||
assert worker._registered_descs[0] == [(raw.data_ptr(), raw.nbytes, 0, "")]
|
||||
np.testing.assert_array_equal(worker.src_blocks_data, expected_descs)
|
||||
assert expected_descs[-1, 0] + expected_descs[-1, 1] == (
|
||||
raw.data_ptr() + raw.nbytes
|
||||
)
|
||||
|
||||
|
||||
def _make_remote_meta(
|
||||
worker,
|
||||
remote_block_size,
|
||||
|
||||
@@ -143,15 +143,22 @@ def test_reshape_padded_kv_cache_strides_by_padded_page():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kernel_block_sizes", "expected_num_blocks", "expected_num_states"),
|
||||
(
|
||||
"kernel_block_sizes",
|
||||
"storage_block_size",
|
||||
"expected_num_blocks",
|
||||
"expected_num_states",
|
||||
),
|
||||
[
|
||||
(None, 4, 64),
|
||||
([256], 4, 64),
|
||||
([64], 16, 16),
|
||||
(None, None, 4, 64),
|
||||
([256], None, 4, 64),
|
||||
([64], None, 16, 16),
|
||||
([64], 256, 4, 64),
|
||||
],
|
||||
)
|
||||
def test_allocate_compressed_mla_cache(
|
||||
kernel_block_sizes: list[int] | None,
|
||||
storage_block_size: int | None,
|
||||
expected_num_blocks: int,
|
||||
expected_num_states: int,
|
||||
):
|
||||
@@ -161,6 +168,7 @@ def test_allocate_compressed_mla_cache(
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=4,
|
||||
storage_block_size=storage_block_size,
|
||||
)
|
||||
num_pages = 4
|
||||
config = KVCacheConfig(
|
||||
|
||||
@@ -76,6 +76,7 @@ def test_packed_dsv4_zeroer_zeroes_only_each_layers_page():
|
||||
static_forward_context={
|
||||
f"layer.{i}": SimpleNamespace(kv_cache=views[i]) for i in range(NUM_LAYERS)
|
||||
},
|
||||
num_blocks=NUM_BLOCKS,
|
||||
)
|
||||
seg_addrs, seg_block_strides, seg_page_sizes, _, _, n_segs = zeroer._meta
|
||||
|
||||
@@ -153,6 +154,7 @@ def test_overlaid_zeroer_dedups_segments_with_max_span():
|
||||
static_forward_context={
|
||||
name: SimpleNamespace(kv_cache=views[name]) for name in views
|
||||
},
|
||||
num_blocks=config.num_blocks,
|
||||
)
|
||||
seg_addrs, seg_block_strides, seg_page_sizes, _, _, n_segs = zeroer._meta
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ def test_attention_blocks_are_zeroed(spec):
|
||||
static_forward_context={
|
||||
layer_name: SimpleNamespace(kv_cache=storage),
|
||||
},
|
||||
num_blocks=4,
|
||||
)
|
||||
|
||||
zeroer.zero_block_ids([1])
|
||||
@@ -64,6 +65,56 @@ def test_attention_blocks_are_zeroed(spec):
|
||||
assert torch.equal(storage, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_layers_in_one_group_may_use_different_kernel_pages_per_block():
|
||||
"""Derive kernel pages per block from each layer's allocation."""
|
||||
device = torch.device("cuda")
|
||||
num_blocks = 4
|
||||
spec = SlidingWindowSpec(
|
||||
block_size=2,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.int32,
|
||||
sliding_window=2,
|
||||
)
|
||||
# Two kernel pages per logical block.
|
||||
wide = torch.ones((num_blocks * 2, 3), dtype=torch.int32, device=device)
|
||||
# One kernel page per logical block, carved out of a larger allocation so
|
||||
# an over-strided write lands in the guard region instead of faulting or
|
||||
# silently hitting another tensor.
|
||||
narrow_backing = torch.ones((num_blocks * 3, 5), dtype=torch.int32, device=device)
|
||||
narrow = narrow_backing[:num_blocks]
|
||||
|
||||
zeroer = KVBlockZeroer(
|
||||
device,
|
||||
attn_groups_iter=[
|
||||
AttentionGroup(
|
||||
None,
|
||||
["wide", "narrow"],
|
||||
spec,
|
||||
0,
|
||||
)
|
||||
],
|
||||
kernel_block_sizes=[1],
|
||||
static_forward_context={
|
||||
"wide": SimpleNamespace(kv_cache=wide),
|
||||
"narrow": SimpleNamespace(kv_cache=narrow),
|
||||
},
|
||||
num_blocks=num_blocks,
|
||||
)
|
||||
|
||||
zeroer.zero_block_ids([num_blocks - 1])
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected_wide = torch.ones_like(wide)
|
||||
expected_wide[2 * (num_blocks - 1) :] = 0
|
||||
assert torch.equal(wide, expected_wide)
|
||||
|
||||
expected_narrow = torch.ones_like(narrow_backing)
|
||||
expected_narrow[num_blocks - 1] = 0
|
||||
assert torch.equal(narrow_backing, expected_narrow)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_block_ids_are_not_overwritten_while_copy_is_in_flight():
|
||||
device = torch.device("cuda")
|
||||
@@ -207,6 +258,7 @@ def test_large_dsv4_launch_geometry(monkeypatch):
|
||||
name: SimpleNamespace(kv_cache=storage)
|
||||
for name, storage in storages.items()
|
||||
},
|
||||
num_blocks=1,
|
||||
)
|
||||
|
||||
assert zeroer._meta is not None
|
||||
@@ -332,6 +384,7 @@ def test_zeroes_exactly_one_block_per_layer(layout: KVCacheLayout):
|
||||
attn_groups_iter=iter(groups),
|
||||
kernel_block_sizes=[spec.block_size],
|
||||
static_forward_context=ctx,
|
||||
num_blocks=num_blocks,
|
||||
)
|
||||
zeroer.zero_block_ids([2])
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -7,15 +7,57 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.recoverssm_metadata import (
|
||||
RecoverSSMMetadata,
|
||||
RecoverSSMPostprocessMetadata,
|
||||
)
|
||||
from vllm.v1.worker.gpu.model_states import mamba_hybrid
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
from vllm.v1.worker.gpu.model_states.recoverssm import RecoverSSMState
|
||||
|
||||
|
||||
def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
state = object.__new__(MambaHybridModelState)
|
||||
state.vllm_config = SimpleNamespace(num_speculative_tokens=0)
|
||||
state.max_model_len = 8192
|
||||
state._align_mode = False
|
||||
state.recoverssm = None
|
||||
|
||||
positions = torch.tensor([1536], dtype=torch.int64)
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=1,
|
||||
num_tokens=1,
|
||||
num_reqs_after_padding=1,
|
||||
num_tokens_after_padding=1,
|
||||
query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
|
||||
num_scheduled_tokens=torch.tensor([1], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([1537], dtype=torch.int32),
|
||||
is_prefilling_np=torch.tensor([False]).numpy(),
|
||||
dcp_local_seq_lens=None,
|
||||
positions=positions,
|
||||
prompt_lens=torch.tensor([1024], dtype=torch.int32),
|
||||
)
|
||||
expected_metadata = {"layer": object()}
|
||||
build_attn_metadata = Mock(return_value=expected_metadata)
|
||||
monkeypatch.setattr(mamba_hybrid, "build_attn_metadata", build_attn_metadata)
|
||||
|
||||
metadata = state.prepare_attn(
|
||||
input_batch=input_batch,
|
||||
cudagraph_mode=CUDAGraphMode.NONE,
|
||||
block_tables=(),
|
||||
slot_mappings=torch.empty(0, dtype=torch.int64),
|
||||
attn_groups=[],
|
||||
kv_cache_config=Mock(),
|
||||
)
|
||||
|
||||
assert metadata is expected_metadata
|
||||
assert build_attn_metadata.call_args.kwargs["positions"] is positions
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
@pytest.mark.parametrize(("num_sampled", "expected_value"), [(0, 1), (3, 3)])
|
||||
def test_postprocess_state_scalar_with_int32_mapping(
|
||||
|
||||
Reference in New Issue
Block a user