[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+102 -10
View File
@@ -15,7 +15,11 @@ import pytest
import torch
import vllm.v1.hisparse.binding as attn_utils_module
from tests.v1.attention.utils import dense_kv_cache_views
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
dense_kv_cache_views,
)
from vllm.config.compilation import CompilationConfig, CUDAGraphMode
from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
@@ -42,6 +46,8 @@ from vllm.v1.worker.utils import (
AttentionGroup,
allocate_kv_cache,
copy_kv_cache_blocks_inplace,
customize_attention_spec,
map_kv_caches_to_kernel_blocks,
)
@@ -79,7 +85,7 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
backend = SimpleNamespace(
get_name=lambda name=name: name,
customize_spec=AttentionBackend.customize_spec,
get_supported_kernel_block_sizes=lambda sizes=sizes: sizes,
get_supported_kernel_block_sizes=lambda _=None, sizes=sizes: sizes,
)
layers[name] = SimpleNamespace(
get_kv_cache_spec=lambda _, spec=specs[name]: spec,
@@ -99,7 +105,8 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
resolved = attn_utils.get_kv_cache_spec(config)
assert resolved["main"].block_size == resolved["indexer"].block_size == expected
assert resolved["dense"] is specs["dense"]
dense_backend = layers["dense"].get_attn_backend()
assert resolved["dense"] == customize_attention_spec(dense_backend, specs["dense"])
assert all(spec.block_size == block_size for spec in specs.values())
@@ -495,20 +502,17 @@ def test_reshape_padded_kv_cache_strides_by_padded_page():
@pytest.mark.parametrize(
(
"kernel_block_sizes",
"storage_block_size",
"expected_num_blocks",
"expected_num_states",
),
[
(None, None, 4, 64),
([256], None, 4, 64),
([64], None, 16, 16),
([64], 256, 4, 64),
(None, 4, 64),
([256], 4, 64),
([64], 16, 16),
],
)
def test_allocate_compressed_mla_cache(
kernel_block_sizes: list[int] | None,
storage_block_size: int | None,
expected_num_blocks: int,
expected_num_states: int,
):
@@ -518,7 +522,6 @@ def test_allocate_compressed_mla_cache(
head_size=128,
dtype=torch.bfloat16,
tokens_per_state=4,
storage_block_size=storage_block_size,
)
num_pages = 4
config = KVCacheConfig(
@@ -744,3 +747,92 @@ def test_allocate_hisparse_kv_caches_host_pool_and_view_less_specs():
backing.untyped_storage().data_ptr()
== caches["indexer"].untyped_storage().data_ptr()
)
class _TableBuilder:
requires_block_table_width = False
supports_update_block_table = False
def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
self.kv_cache_spec = kv_cache_spec
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
return common_attn_metadata.block_table_tensor
_VLLM_CONFIG = SimpleNamespace(
model_config=SimpleNamespace(max_model_len=8192),
scheduler_config=SimpleNamespace(max_num_seqs=4),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
)
def _packed_group(tokens_per_state, row_bytes, supported_sizes):
spec = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=128,
dtype=torch.uint8,
state_content_bytes=row_bytes,
tokens_per_state=tokens_per_state,
)
backend = SimpleNamespace(
get_builder_cls=lambda: _TableBuilder,
get_supported_kernel_block_sizes=lambda _=None: supported_sizes,
get_name=lambda: "FAKE",
customize_spec=lambda spec: spec,
)
return AttentionGroup(backend, ["layer.0"], spec, 0)
def test_packed_compressed_group_gets_kernel_block_table():
"""A packed indexer cache whose kernel needs 64-state pages gets a table and
a bound view in kernel blocks that address the same packed-block rows."""
row, stride_rows = 132, 5 * 64
group = _packed_group(4, row, [128, 256])
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=1024)
assert group.get_metadata_builder().kv_cache_spec.block_size == 256
backing = torch.arange(3 * stride_rows).repeat_interleave(row)
manager = backing.view(3, 1, stride_rows, row)[:, :, :256]
pages = map_kv_caches_to_kernel_blocks({"layer.0": manager}, [group])["layer.0"]
metadata = create_common_attn_metadata(BatchSpec([2048], [1]), 1024, "cpu")
metadata.block_table_tensor = torch.tensor([[0, 2]], dtype=torch.int32)
table = group.build_metadata(metadata)
assert table.tolist() == [[0, 1, 2, 3, 10, 11, 12, 13]]
assert torch.equal(pages[table[0, 4:], 0].flatten(0, 1), manager[2, 0])
def test_split_group_maps_manager_block_table():
"""Groups split in a dense layout keep manager block tables; builders get
b * blocks_per_kv_block + j against the view allocated in kernel blocks."""
group = _packed_group(4, 132, [128, 256])
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=256)
kernel_view = torch.zeros(12, 1, 64, 132)
bound = map_kv_caches_to_kernel_blocks({"layer.0": kernel_view}, [group])
assert bound["layer.0"] is kernel_view
metadata = create_common_attn_metadata(BatchSpec([2048], [1]), 1024, "cpu")
metadata.block_table_tensor = torch.tensor([[0, 2]], dtype=torch.int32)
table = group.build_metadata(metadata)
assert table.tolist() == [[0, 1, 2, 3, 8, 9, 10, 11]]
@pytest.mark.parametrize(
("block_size", "page_states"), [(1024, 64), (384, 32), (256, 0)]
)
def test_split_blocks_align_to_kernel_blocks(block_size, page_states):
"""Backends taking only fixed kernel blocks align packed blocks to the kernel
block the final block size is split into, and not at all when unsplit."""
group = _packed_group(4, 132, [128, 256])
spec = customize_attention_spec(group.backend, group.kv_cache_spec)
spec = spec.copy_with_new_block_size(block_size)
assert spec.get_block_stride_alignment() == max(page_states * 132, 1)
def test_packed_token_cache_cannot_be_split():
group = _packed_group(1, 1152, [64])
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=1024)
packed = torch.zeros(3, 1, 2048, 1152, dtype=torch.uint8)[:, :, :1024]
with pytest.raises(ValueError, match="cannot be split"):
map_kv_caches_to_kernel_blocks({"layer.0": packed}, [group])
+11 -20
View File
@@ -22,7 +22,6 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
max_num_batched_tokens=64,
max_num_blocks_per_group=[8, 8, 8],
device=device,
kernel_block_sizes=[16, 16, 8],
)
def fail_if_apply_write_called():
@@ -48,33 +47,31 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
block_tables.block_tables[0].gpu[0, :2],
torch.tensor([1, 2], dtype=torch.int32, device=device),
)
# Group 1 has blocks_per_kv_block == 2, so each KV block expands to two
# kernel block IDs.
assert torch.equal(
block_tables.block_tables[1].gpu[0, :4],
torch.tensor([20, 21, 22, 23], dtype=torch.int32, device=device),
block_tables.block_tables[1].gpu[0, :2],
torch.tensor([10, 11], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[0].gpu[1, :1],
torch.tensor([3], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[1].gpu[1, :2],
torch.tensor([24, 25], dtype=torch.int32, device=device),
block_tables.block_tables[1].gpu[1, :1],
torch.tensor([12], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[2].gpu[1, :2],
torch.tensor([5, 6], dtype=torch.int32, device=device),
)
assert block_tables.num_blocks.np[0, 0] == 2
assert block_tables.num_blocks.np[1, 0] == 4
assert block_tables.num_blocks.np[1, 0] == 2
assert block_tables.num_blocks.np[2, 0] == 0
assert block_tables.num_blocks.np[0, 1] == 1
assert block_tables.num_blocks.np[1, 1] == 2
assert block_tables.num_blocks.np[1, 1] == 1
assert block_tables.num_blocks.np[2, 1] == 2
assert torch.equal(
block_tables.num_blocks.gpu[:, :2],
torch.tensor([[2, 1], [4, 2], [0, 2]], dtype=torch.int32, device=device),
torch.tensor([[2, 1], [2, 1], [0, 2]], dtype=torch.int32, device=device),
)
for block_table in block_tables.block_tables:
@@ -96,15 +93,15 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
torch.tensor([1, 2, 7], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[1].gpu[0, :6],
torch.tensor([20, 21, 22, 23, 26, 27], dtype=torch.int32, device=device),
block_tables.block_tables[1].gpu[0, :3],
torch.tensor([10, 11, 13], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[2].gpu[0, :1],
torch.tensor([8], dtype=torch.int32, device=device),
)
assert block_tables.num_blocks.np[0, 0] == 3
assert block_tables.num_blocks.np[1, 0] == 6
assert block_tables.num_blocks.np[1, 0] == 3
assert block_tables.num_blocks.np[2, 0] == 1
@@ -116,7 +113,6 @@ def test_block_tables_apply_staged_writes_single_group():
max_num_batched_tokens=16,
max_num_blocks_per_group=[4],
device=device,
kernel_block_sizes=[16],
)
block_tables.append_block_ids(
@@ -141,7 +137,6 @@ def test_block_tables_skip_custom_slot_mapping_groups():
max_num_batched_tokens=4,
max_num_blocks_per_group=[1, 1],
device=device,
kernel_block_sizes=[8, 262144],
slot_mapping_enabled=[False, True],
)
block_tables.append_block_ids(
@@ -170,7 +165,7 @@ def test_block_tables_skip_custom_slot_mapping_groups():
@pytest.mark.parametrize("cp_rank", range(4))
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
def test_dcp_slot_mapping_only_shards_sharded_groups(cp_rank: int):
"""Only sharded groups use DCP interleave in logical-block coordinates."""
device = torch.device("cuda")
block_tables = BlockTables(
@@ -179,7 +174,6 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
max_num_batched_tokens=1024,
max_num_blocks_per_group=[2, 8],
device=device,
kernel_block_sizes=[64, 64],
dcp_sharded=[True, False],
cp_size=4,
cp_rank=cp_rank,
@@ -230,7 +224,6 @@ def test_v1_block_table_move_row_clears_vacated_row():
max_num_batched_tokens=64,
pin_memory=False,
device=torch.device("cuda"),
kernel_block_size=16,
cp_kv_cache_interleave_size=1,
)
block_table.add_row([7, 8, 9], row_idx=0)
@@ -260,7 +253,6 @@ def test_get_dummy_block_tables_returns_zeroed_rows():
max_num_batched_tokens=64,
max_num_blocks_per_group=[8],
device=device,
kernel_block_sizes=[16],
)
# Simulate a real step: stage a request's blocks and gather them into
# the persistent input block tables.
@@ -292,7 +284,6 @@ def test_dummy_request_slot_mapping_is_pad():
max_num_batched_tokens=16,
max_num_blocks_per_group=[4],
device=device,
kernel_block_sizes=[4],
)
block_tables.append_block_ids(req_index=0, new_block_ids=([7, 8],), overwrite=True)
block_tables.apply_staged_writes()
-9
View File
@@ -233,7 +233,6 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
device=torch.device(device),
vocab_size=1024,
block_sizes=[1],
kernel_block_sizes=[1],
max_num_blocks_per_req=[1024],
)
reqs: list[CachedRequestState] = []
@@ -327,7 +326,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
device=torch.device(device),
vocab_size=1024,
block_sizes=[1],
kernel_block_sizes=[1],
max_num_blocks_per_req=[1024],
)
ref_input_batch: InputBatch = InputBatch(
@@ -337,7 +335,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
device=torch.device(device),
vocab_size=1024,
block_sizes=[1],
kernel_block_sizes=[1],
max_num_blocks_per_req=[1024],
)
@@ -388,7 +385,6 @@ def test_condense_clears_stale_allowed_token_ids_mask(device: str):
device=torch.device(device),
vocab_size=VOCAB_SIZE,
block_sizes=[1],
kernel_block_sizes=[1],
max_num_blocks_per_req=[1024],
)
@@ -446,7 +442,6 @@ def test_swap_states_preserves_allowed_token_ids_mask(device: str):
device=torch.device(device),
vocab_size=VOCAB_SIZE,
block_sizes=[1],
kernel_block_sizes=[1],
max_num_blocks_per_req=[1024],
)
@@ -516,7 +511,6 @@ def test_pooling_prompt_lens_not_aliased(device: str):
device=torch.device(device),
vocab_size=VOCAB_SIZE,
block_sizes=[16],
kernel_block_sizes=[16],
max_num_blocks_per_req=[64],
is_pooling_model=True,
)
@@ -552,7 +546,6 @@ def test_placeholder_spec_token_ids_written_verbatim():
device=torch.device("cpu"),
vocab_size=VOCAB_SIZE,
block_sizes=[16],
kernel_block_sizes=[16],
max_num_blocks_per_req=[1],
)
req = CachedRequestState(
@@ -599,7 +592,6 @@ def test_pooling_metadata_token_id_buffers(
device=torch.device("cpu"),
vocab_size=VOCAB_SIZE,
block_sizes=[16],
kernel_block_sizes=[16],
max_num_blocks_per_req=[64],
is_pooling_model=True,
)
@@ -630,7 +622,6 @@ def _make_input_batch(is_pooling_model: bool = False) -> InputBatch:
device=torch.device("cpu"),
vocab_size=VOCAB_SIZE,
block_sizes=[16],
kernel_block_sizes=[16],
max_num_blocks_per_req=[64],
is_pooling_model=is_pooling_model,
)
@@ -5,7 +5,7 @@
The kpool tail cache holds one circular block per request, so its block table
row is orders of magnitude shorter than the sequence it serves.
`_compute_slot_mappings_kernel` indexes block tables by
`position // kernel_block_size` with an effectively unmasked load, so
`position // block_size` with an effectively unmasked load, so
this leads to a read of neighboring requests' rows (and for the later requests, outside
the buffer entirely).
"""
@@ -15,7 +15,7 @@ import torch
from vllm.platforms import current_platform
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.kv_cache_interface import KpoolTailSpec
from vllm.v1.kv_cache_interface import CircularBufferSpec
from vllm.v1.worker.block_table import get_block_table_width
from vllm.v1.worker.gpu.block_table import BlockTables
@@ -32,13 +32,12 @@ POISON_BLOCK_ID = 1 << 20
def make_tail_spec():
return KpoolTailSpec(
return CircularBufferSpec(
block_size=KPOOL,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=KPOOL,
)
@@ -67,7 +66,6 @@ def test_kpool_tail_group_never_position_indexes_its_block_table():
max_num_batched_tokens=len(positions),
max_num_blocks_per_group=[mla_width, tail_width],
device=device,
kernel_block_sizes=[MLA_BLOCK_SIZE, spec.block_size],
slot_mapping_enabled=[True, spec.uses_slot_mapping],
)
mla_blocks = list(range(1, mla_width + 1))
+4 -104
View File
@@ -111,9 +111,6 @@ def initialize_kv_cache(runner: GPUModelRunner):
device=runner.device,
vocab_size=runner.model_config.get_vocab_size(),
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
kernel_block_sizes=[
kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
],
max_num_blocks_per_req=[NUM_BLOCKS],
)
runner.initialize_attn_backend(kv_cache_config)
@@ -303,7 +300,7 @@ def _make_mock_backend_for_kernel_block_size(
return "MOCK"
@staticmethod
def get_supported_kernel_block_sizes():
def get_supported_kernel_block_sizes(kv_cache_spec=None):
return supported_sizes
return _MockBackend()
@@ -441,7 +438,7 @@ def _mock_backend(supported: list, *, exact: bool = False):
return "MOCK_EXACT" if exact else "MOCK"
@staticmethod
def get_supported_kernel_block_sizes():
def get_supported_kernel_block_sizes(kv_cache_spec=None):
return list(supported)
if exact:
@@ -1625,7 +1622,6 @@ def test_input_batch_reinitialized_after_late_interleave_adjustment(monkeypatch)
runner.device = torch.device("cpu")
runner.is_pooling_model = False
runner._init_block_sizes = [16]
runner._init_kernel_block_sizes = [16]
runner._init_max_num_blocks = [4]
runner._init_slot_mapping_modes = [
gpu_model_runner_module.SlotMappingMode.TOKEN_TO_KV_SLOT
@@ -1653,7 +1649,7 @@ def test_input_batch_reinitialized_after_late_interleave_adjustment(monkeypatch)
lambda _: gpu_model_runner_module.KVCacheSpecKind.FULL_ATTENTION,
)
runner.may_reinitialize_input_batch(kv_cache_config, [16])
runner.may_reinitialize_input_batch(kv_cache_config)
assert input_batch_cls.call_count == 1
assert input_batch_cls.call_args.kwargs["cp_kv_cache_interleave_size"] == 16
@@ -1681,68 +1677,10 @@ def test_v2_runner_snapshots_late_interleave_adjustment(monkeypatch):
assert runner.cp_interleave == 16
def test_hybrid_block_table_initialization():
"""Test hybrid block table with different kernel and kvcache_manager block
sizes."""
from vllm.v1.worker.block_table import BlockTable
# Test configuration: kvcache_manager block size = 32,
# kernel block size = 16
block_size = 32
kernel_block_sizes = [16]
max_num_reqs = 10
max_num_blocks_per_req = 20
max_num_batched_tokens = 512
cp_kv_cache_interleave_size = 8
block_table = BlockTable(
block_size=block_size,
max_num_reqs=max_num_reqs,
max_num_blocks_per_req=max_num_blocks_per_req,
max_num_batched_tokens=max_num_batched_tokens,
pin_memory=False,
device=torch.device(DEVICE_TYPE),
kernel_block_size=kernel_block_sizes[0],
cp_kv_cache_interleave_size=cp_kv_cache_interleave_size,
)
# Verify hybrid block configuration
assert block_table.use_hybrid_blocks is True
assert block_table.block_size == kernel_block_sizes[0]
assert block_table.blocks_per_kv_block == (
block_size // kernel_block_sizes[0]
) # Changed to use first element
# Test block table conversion logic
# One kvcache_manager block should map to multiple kernel blocks
kvcache_manager_blocks = [0, 1, 2]
# Verify that kvcache_manager blocks can be converted to kernel blocks
# and that block table operations work correctly.
req_index = 0
block_table.append_row(kvcache_manager_blocks, req_index)
# Get expected kernel blocks from the implementation for verification.
expected_kernel_blocks = block_table.map_to_kernel_blocks(
np.array(kvcache_manager_blocks),
block_table.blocks_per_kv_block,
block_table._kernel_block_arange,
)
# Verify block table state
assert block_table.num_blocks_per_row[req_index] == len(expected_kernel_blocks)
assert np.array_equal(
block_table.block_table.np[req_index, : len(expected_kernel_blocks)],
expected_kernel_blocks,
)
def test_get_block_table_width_aligns_to_128_tokens():
assert get_block_table_width(1875, 64) == 1876
def test_get_block_table_width_splits_virtual_blocks():
assert get_block_table_width(235, 256, 64) == 940
def test_mamba_state_table_width_is_not_aligned():
block_tables = MultiGroupBlockTable(
max_num_reqs=1,
@@ -1750,7 +1688,6 @@ def test_mamba_state_table_width_is_not_aligned():
pin_memory=False,
device=torch.device("cpu"),
block_sizes=[39664],
kernel_block_sizes=[39664],
max_num_blocks=[1],
slot_mapping_modes=[SlotMappingMode.NONE],
)
@@ -1758,42 +1695,6 @@ def test_mamba_state_table_width_is_not_aligned():
assert block_tables[0].max_num_blocks_per_req == 1
def test_input_batch_with_kernel_block_sizes():
"""Test InputBatch initialization with kernel_block_sizes parameter."""
max_num_reqs = 10
max_model_len = 512
max_num_batched_tokens = 512
device = torch.device(DEVICE_TYPE)
vocab_size = 50272
# Test with different kernel block sizes
block_sizes = [32, 64]
kernel_block_sizes = [16, 32]
input_batch = InputBatch(
max_num_reqs=max_num_reqs,
max_model_len=max_model_len,
max_num_batched_tokens=max_num_batched_tokens,
device=device,
vocab_size=vocab_size,
block_sizes=block_sizes,
kernel_block_sizes=kernel_block_sizes,
max_num_blocks_per_req=[16, 8],
)
# Verify that block tables were created with kernel block sizes
assert len(input_batch.block_table.block_tables) == len(block_sizes)
for i, (kv_size, kernel_size) in enumerate(zip(block_sizes, kernel_block_sizes)):
block_table = input_batch.block_table.block_tables[i]
if kv_size != kernel_size:
assert block_table.use_hybrid_blocks is True
assert block_table.block_size == kernel_size
else:
assert block_table.use_hybrid_blocks is False
assert block_table.block_size == kernel_size
def test_hybrid_cache_integration(default_vllm_config, dist_init):
"""Test hybrid cache architecture integration with GPUModelRunner."""
# Create a new model runner with hybrid cache configuration
@@ -1843,9 +1744,8 @@ def test_hybrid_cache_integration(default_vllm_config, dist_init):
device=runner.device,
vocab_size=runner.model_config.get_vocab_size(),
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
kernel_block_sizes=[16],
max_num_blocks_per_req=[NUM_BLOCKS],
) # Use kernel block size
)
runner.initialize_attn_backend(kv_cache_config)
@@ -119,9 +119,6 @@ def _initialize_kv_cache(runner: GPUModelRunner) -> None:
device=runner.device,
vocab_size=runner.model_config.get_vocab_size(),
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
kernel_block_sizes=[
kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
],
max_num_blocks_per_req=[NUM_BLOCKS],
)
runner.initialize_attn_backend(kv_cache_config)
@@ -237,7 +237,6 @@ def test_append_block_ids_rejects_write_past_row_capacity():
block_tables = BlockTables.__new__(BlockTables)
block_tables.num_kv_cache_groups = 1
block_tables.blocks_per_kv_block = [1]
block_tables.block_tables = [_BlockTable()]
block_tables.num_blocks = SimpleNamespace(
np=torch.tensor([[0, 3]], dtype=torch.int32)
+1 -2
View File
@@ -47,8 +47,7 @@ def _make_capture_manager(block_table: torch.Tensor):
block_tables = SimpleNamespace(
input_block_tables=(block_table,),
num_kv_cache_groups=1,
kernel_block_sizes=(2,),
blocks_per_kv_block=(1,),
block_sizes=(2,),
)
manager = PCPManager(
pcp_world_size=2,
@@ -20,6 +20,7 @@ from vllm.model_executor.models.mistral_large_3_eagle import (
from vllm.v1.attention.backends import flash_attn as flash_attn_module
from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
@@ -30,6 +31,7 @@ from vllm.v1.worker.gpu.spec_decode.target_dependent_ar import speculator as spe
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
TargetDependentARSpeculator,
)
from vllm.v1.worker.utils import AttentionGroup
class _TestSpeculator(TargetDependentARSpeculator):
@@ -102,6 +104,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
get_metadata_builder=lambda _: SimpleNamespace(
build=build, supports_update_block_table=False
),
build_metadata=lambda cm, ubatch_idx=0, **_: build(0, cm),
layer_names=["draft"],
)
]
@@ -427,6 +430,8 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
query_start_loc=torch.arange(3),
)
speculator.idx_mapping = torch.arange(2)
speculator.attn_groups = []
speculator.block_tables = SimpleNamespace(input_block_tables=[])
generate_draft = Mock()
speculator._generate_draft = generate_draft
run_fullgraph = Mock()
@@ -450,6 +455,52 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
assert run_fullgraph.call_count == expected_graph_replays
def test_fused_full_graph_maps_split_group_block_tables():
"""The fused FULL replay skips build(), so split groups' tables are mapped first."""
spec = MLAAttentionSpec(
block_size=1024, num_kv_heads=1, head_size=128, dtype=torch.uint8
)
group = AttentionGroup(
SimpleNamespace(),
["layer.0"],
spec,
0,
kernel_block_size=256,
kernel_block_table=torch.zeros(1, 4, 32, dtype=torch.int32),
kernel_block_offsets=torch.arange(4, dtype=torch.int32),
)
speculator = object.__new__(_TestSpeculator)
speculator.input_buffers = SimpleNamespace(
positions=torch.arange(2), query_start_loc=torch.arange(3)
)
speculator.idx_mapping = torch.arange(2)
speculator.attn_groups = [[group]]
speculator.block_tables = SimpleNamespace(
input_block_tables=[torch.tensor([[1, 2], [0, 3]], dtype=torch.int32)]
)
replayed_tables = []
speculator.decode_cudagraph_manager = SimpleNamespace(
run_fullgraph=lambda _: replayed_tables.append(
group.kernel_block_table[0, :2, :8].tolist()
)
)
speculator._fused_multi_step_decode(
num_reqs=2,
skip_attn=False,
batch_desc=BatchExecutionDescriptor(
cg_mode=CUDAGraphMode.FULL, num_tokens=2, num_reqs=2
),
num_tokens_across_dp=None,
seq_lens_cpu_upper_bound=None,
num_speculative_steps=4,
)
assert replayed_tables == [
[[4, 5, 6, 7, 8, 9, 10, 11], [0, 1, 2, 3, 12, 13, 14, 15]]
]
def test_update_draft_decode_metadata_updates_fa3_scheduler_metadata(
monkeypatch,
):
+1 -17
View File
@@ -12,40 +12,24 @@ if not current_platform.is_cuda_alike():
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
@pytest.mark.parametrize(
("kv_cache_block_size", "blocks_per_kv_block", "block_size", "block_size_rep"),
[
(256, 1, 256, 16),
(256, 4, 64, 16),
(64, 1, 64, 16),
(8, 1, 8, 2),
(4, 1, 4, 2),
],
)
@pytest.mark.parametrize("kv_cache_block_size", [256, 64, 8, 4])
def test_compute_slot_mapping_warmup_matches_runtime_specializations(
kv_cache_block_size: int,
blocks_per_kv_block: int,
block_size: int,
block_size_rep: int,
) -> None:
kernel = ComputeSlotMappingKernel()
kwargs = dict(
kv_cache_block_size=kv_cache_block_size,
blocks_per_kv_block=blocks_per_kv_block,
total_cp_world_size=1,
total_cp_rank=0,
cp_kv_cache_interleave_size=1,
block_table_stride=32768,
block_size=block_size,
)
expected = kernel.CompileKey(
kv_cache_block_size=kv_cache_block_size,
blocks_per_kv_block=blocks_per_kv_block,
total_cp_world_size=1,
total_cp_rank=0,
cp_kv_cache_interleave_size=1,
block_table_stride=16,
block_size=block_size_rep,
)
assert kernel.dispatch(**kwargs) == expected
@@ -112,6 +112,8 @@ def test_mrv1_kv_pool_only_wraps_backing_allocation(monkeypatch) -> None:
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type="test")),
compilation_config=SimpleNamespace(static_forward_context={}),
kv_caches=[],
attn_groups=[],
speculative_config=None,
)
result = gpu_model_runner.GPUModelRunner.initialize_kv_cache_tensors(
runner,
-1
View File
@@ -659,7 +659,6 @@ def test_v2_align_ctx_binds_request_slot_tables_for_state_copies():
max_num_batched_tokens=64,
max_num_blocks_per_group=[4],
device=device,
kernel_block_sizes=[cfg.block_size],
)
state = object.__new__(MambaHybridModelState)
+37
View File
@@ -443,6 +443,43 @@ def test_copy_cpu_kv_cache_logical_blocks_ignores_storage_padding():
assert (backing[9] == -1).all()
def test_bind_kv_cache_keeps_scheduler_block_views_for_runner():
"""Block copies index scheduler blocks, so the runner keeps the unmapped
views while a packed layer is bound with kernel blocks."""
class _Layer:
def bind_kv_cache(self, kv_cache):
self.kv_cache = kv_cache
num_blocks, row = 4, 64
storage = torch.zeros(num_blocks * row)
# Two layers packed into one block row: an 8x4 page, then a 4x2 page
# split into two 2-row kernel blocks.
mla = storage.as_strided((num_blocks, 8, 4), (row, 4, 1))
indexer = storage.as_strided((num_blocks, 1, 4, 2), (row, 8, 2, 1), 40)
# The indexer in 2-row kernel blocks: block b's kernel block j is b * 16 + j.
mapped = indexer.as_strided((50, 1, 2, 2), (4, 8, 2, 1))
kv_caches = {"layers.0.attn": mla, "layers.1.indexer": indexer}
ctx = {name: _Layer() for name in kv_caches}
runner_kv_caches: list[torch.Tensor] = []
bind_kv_cache(
kv_caches,
ctx,
runner_kv_caches,
layer_kv_caches={**kv_caches, "layers.1.indexer": mapped},
)
assert ctx["layers.1.indexer"].kv_cache is mapped
assert runner_kv_caches == [mla, indexer]
copies = [KVCacheBlockCopy(1, 3)]
with pytest.raises(AssertionError, match="not divisible"):
copy_kv_cache_blocks_inplace([mapped], num_blocks, copies)
storage[row : 2 * row] = 7
copy_kv_cache_blocks_inplace(runner_kv_caches, num_blocks, copies)
assert (mla[3] == 7).all() and (indexer[3] == 7).all()
@pytest.mark.parametrize("shared,rank", [(False, 0), (True, 0), (True, 1)])
@pytest.mark.parametrize("has_copies", [False, True])
def test_hisparse_host_copy_waits_for_writes_on_writer_only(