[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -15,7 +15,11 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.hisparse.binding as attn_utils_module
|
||||
from tests.v1.attention.utils import dense_kv_cache_views
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
dense_kv_cache_views,
|
||||
)
|
||||
from vllm.config.compilation import CompilationConfig, CUDAGraphMode
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
@@ -42,6 +46,8 @@ from vllm.v1.worker.utils import (
|
||||
AttentionGroup,
|
||||
allocate_kv_cache,
|
||||
copy_kv_cache_blocks_inplace,
|
||||
customize_attention_spec,
|
||||
map_kv_caches_to_kernel_blocks,
|
||||
)
|
||||
|
||||
|
||||
@@ -79,7 +85,7 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
|
||||
backend = SimpleNamespace(
|
||||
get_name=lambda name=name: name,
|
||||
customize_spec=AttentionBackend.customize_spec,
|
||||
get_supported_kernel_block_sizes=lambda sizes=sizes: sizes,
|
||||
get_supported_kernel_block_sizes=lambda _=None, sizes=sizes: sizes,
|
||||
)
|
||||
layers[name] = SimpleNamespace(
|
||||
get_kv_cache_spec=lambda _, spec=specs[name]: spec,
|
||||
@@ -99,7 +105,8 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
|
||||
|
||||
resolved = attn_utils.get_kv_cache_spec(config)
|
||||
assert resolved["main"].block_size == resolved["indexer"].block_size == expected
|
||||
assert resolved["dense"] is specs["dense"]
|
||||
dense_backend = layers["dense"].get_attn_backend()
|
||||
assert resolved["dense"] == customize_attention_spec(dense_backend, specs["dense"])
|
||||
assert all(spec.block_size == block_size for spec in specs.values())
|
||||
|
||||
|
||||
@@ -495,20 +502,17 @@ def test_reshape_padded_kv_cache_strides_by_padded_page():
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"kernel_block_sizes",
|
||||
"storage_block_size",
|
||||
"expected_num_blocks",
|
||||
"expected_num_states",
|
||||
),
|
||||
[
|
||||
(None, None, 4, 64),
|
||||
([256], None, 4, 64),
|
||||
([64], None, 16, 16),
|
||||
([64], 256, 4, 64),
|
||||
(None, 4, 64),
|
||||
([256], 4, 64),
|
||||
([64], 16, 16),
|
||||
],
|
||||
)
|
||||
def test_allocate_compressed_mla_cache(
|
||||
kernel_block_sizes: list[int] | None,
|
||||
storage_block_size: int | None,
|
||||
expected_num_blocks: int,
|
||||
expected_num_states: int,
|
||||
):
|
||||
@@ -518,7 +522,6 @@ def test_allocate_compressed_mla_cache(
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=4,
|
||||
storage_block_size=storage_block_size,
|
||||
)
|
||||
num_pages = 4
|
||||
config = KVCacheConfig(
|
||||
@@ -744,3 +747,92 @@ def test_allocate_hisparse_kv_caches_host_pool_and_view_less_specs():
|
||||
backing.untyped_storage().data_ptr()
|
||||
== caches["indexer"].untyped_storage().data_ptr()
|
||||
)
|
||||
|
||||
|
||||
class _TableBuilder:
|
||||
requires_block_table_width = False
|
||||
supports_update_block_table = False
|
||||
|
||||
def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
|
||||
self.kv_cache_spec = kv_cache_spec
|
||||
|
||||
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
|
||||
return common_attn_metadata.block_table_tensor
|
||||
|
||||
|
||||
_VLLM_CONFIG = SimpleNamespace(
|
||||
model_config=SimpleNamespace(max_model_len=8192),
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=4),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
)
|
||||
|
||||
|
||||
def _packed_group(tokens_per_state, row_bytes, supported_sizes):
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.uint8,
|
||||
state_content_bytes=row_bytes,
|
||||
tokens_per_state=tokens_per_state,
|
||||
)
|
||||
backend = SimpleNamespace(
|
||||
get_builder_cls=lambda: _TableBuilder,
|
||||
get_supported_kernel_block_sizes=lambda _=None: supported_sizes,
|
||||
get_name=lambda: "FAKE",
|
||||
customize_spec=lambda spec: spec,
|
||||
)
|
||||
return AttentionGroup(backend, ["layer.0"], spec, 0)
|
||||
|
||||
|
||||
def test_packed_compressed_group_gets_kernel_block_table():
|
||||
"""A packed indexer cache whose kernel needs 64-state pages gets a table and
|
||||
a bound view in kernel blocks that address the same packed-block rows."""
|
||||
row, stride_rows = 132, 5 * 64
|
||||
group = _packed_group(4, row, [128, 256])
|
||||
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=1024)
|
||||
assert group.get_metadata_builder().kv_cache_spec.block_size == 256
|
||||
|
||||
backing = torch.arange(3 * stride_rows).repeat_interleave(row)
|
||||
manager = backing.view(3, 1, stride_rows, row)[:, :, :256]
|
||||
pages = map_kv_caches_to_kernel_blocks({"layer.0": manager}, [group])["layer.0"]
|
||||
metadata = create_common_attn_metadata(BatchSpec([2048], [1]), 1024, "cpu")
|
||||
metadata.block_table_tensor = torch.tensor([[0, 2]], dtype=torch.int32)
|
||||
table = group.build_metadata(metadata)
|
||||
assert table.tolist() == [[0, 1, 2, 3, 10, 11, 12, 13]]
|
||||
assert torch.equal(pages[table[0, 4:], 0].flatten(0, 1), manager[2, 0])
|
||||
|
||||
|
||||
def test_split_group_maps_manager_block_table():
|
||||
"""Groups split in a dense layout keep manager block tables; builders get
|
||||
b * blocks_per_kv_block + j against the view allocated in kernel blocks."""
|
||||
group = _packed_group(4, 132, [128, 256])
|
||||
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=256)
|
||||
kernel_view = torch.zeros(12, 1, 64, 132)
|
||||
bound = map_kv_caches_to_kernel_blocks({"layer.0": kernel_view}, [group])
|
||||
assert bound["layer.0"] is kernel_view
|
||||
|
||||
metadata = create_common_attn_metadata(BatchSpec([2048], [1]), 1024, "cpu")
|
||||
metadata.block_table_tensor = torch.tensor([[0, 2]], dtype=torch.int32)
|
||||
table = group.build_metadata(metadata)
|
||||
assert table.tolist() == [[0, 1, 2, 3, 8, 9, 10, 11]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("block_size", "page_states"), [(1024, 64), (384, 32), (256, 0)]
|
||||
)
|
||||
def test_split_blocks_align_to_kernel_blocks(block_size, page_states):
|
||||
"""Backends taking only fixed kernel blocks align packed blocks to the kernel
|
||||
block the final block size is split into, and not at all when unsplit."""
|
||||
group = _packed_group(4, 132, [128, 256])
|
||||
spec = customize_attention_spec(group.backend, group.kv_cache_spec)
|
||||
spec = spec.copy_with_new_block_size(block_size)
|
||||
assert spec.get_block_stride_alignment() == max(page_states * 132, 1)
|
||||
|
||||
|
||||
def test_packed_token_cache_cannot_be_split():
|
||||
group = _packed_group(1, 1152, [64])
|
||||
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=1024)
|
||||
packed = torch.zeros(3, 1, 2048, 1152, dtype=torch.uint8)[:, :, :1024]
|
||||
with pytest.raises(ValueError, match="cannot be split"):
|
||||
map_kv_caches_to_kernel_blocks({"layer.0": packed}, [group])
|
||||
|
||||
@@ -22,7 +22,6 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
|
||||
max_num_batched_tokens=64,
|
||||
max_num_blocks_per_group=[8, 8, 8],
|
||||
device=device,
|
||||
kernel_block_sizes=[16, 16, 8],
|
||||
)
|
||||
|
||||
def fail_if_apply_write_called():
|
||||
@@ -48,33 +47,31 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
|
||||
block_tables.block_tables[0].gpu[0, :2],
|
||||
torch.tensor([1, 2], dtype=torch.int32, device=device),
|
||||
)
|
||||
# Group 1 has blocks_per_kv_block == 2, so each KV block expands to two
|
||||
# kernel block IDs.
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[1].gpu[0, :4],
|
||||
torch.tensor([20, 21, 22, 23], dtype=torch.int32, device=device),
|
||||
block_tables.block_tables[1].gpu[0, :2],
|
||||
torch.tensor([10, 11], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[0].gpu[1, :1],
|
||||
torch.tensor([3], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[1].gpu[1, :2],
|
||||
torch.tensor([24, 25], dtype=torch.int32, device=device),
|
||||
block_tables.block_tables[1].gpu[1, :1],
|
||||
torch.tensor([12], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[2].gpu[1, :2],
|
||||
torch.tensor([5, 6], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert block_tables.num_blocks.np[0, 0] == 2
|
||||
assert block_tables.num_blocks.np[1, 0] == 4
|
||||
assert block_tables.num_blocks.np[1, 0] == 2
|
||||
assert block_tables.num_blocks.np[2, 0] == 0
|
||||
assert block_tables.num_blocks.np[0, 1] == 1
|
||||
assert block_tables.num_blocks.np[1, 1] == 2
|
||||
assert block_tables.num_blocks.np[1, 1] == 1
|
||||
assert block_tables.num_blocks.np[2, 1] == 2
|
||||
assert torch.equal(
|
||||
block_tables.num_blocks.gpu[:, :2],
|
||||
torch.tensor([[2, 1], [4, 2], [0, 2]], dtype=torch.int32, device=device),
|
||||
torch.tensor([[2, 1], [2, 1], [0, 2]], dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
for block_table in block_tables.block_tables:
|
||||
@@ -96,15 +93,15 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
|
||||
torch.tensor([1, 2, 7], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[1].gpu[0, :6],
|
||||
torch.tensor([20, 21, 22, 23, 26, 27], dtype=torch.int32, device=device),
|
||||
block_tables.block_tables[1].gpu[0, :3],
|
||||
torch.tensor([10, 11, 13], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert torch.equal(
|
||||
block_tables.block_tables[2].gpu[0, :1],
|
||||
torch.tensor([8], dtype=torch.int32, device=device),
|
||||
)
|
||||
assert block_tables.num_blocks.np[0, 0] == 3
|
||||
assert block_tables.num_blocks.np[1, 0] == 6
|
||||
assert block_tables.num_blocks.np[1, 0] == 3
|
||||
assert block_tables.num_blocks.np[2, 0] == 1
|
||||
|
||||
|
||||
@@ -116,7 +113,6 @@ def test_block_tables_apply_staged_writes_single_group():
|
||||
max_num_batched_tokens=16,
|
||||
max_num_blocks_per_group=[4],
|
||||
device=device,
|
||||
kernel_block_sizes=[16],
|
||||
)
|
||||
|
||||
block_tables.append_block_ids(
|
||||
@@ -141,7 +137,6 @@ def test_block_tables_skip_custom_slot_mapping_groups():
|
||||
max_num_batched_tokens=4,
|
||||
max_num_blocks_per_group=[1, 1],
|
||||
device=device,
|
||||
kernel_block_sizes=[8, 262144],
|
||||
slot_mapping_enabled=[False, True],
|
||||
)
|
||||
block_tables.append_block_ids(
|
||||
@@ -170,7 +165,7 @@ def test_block_tables_skip_custom_slot_mapping_groups():
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cp_rank", range(4))
|
||||
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
def test_dcp_slot_mapping_only_shards_sharded_groups(cp_rank: int):
|
||||
"""Only sharded groups use DCP interleave in logical-block coordinates."""
|
||||
device = torch.device("cuda")
|
||||
block_tables = BlockTables(
|
||||
@@ -179,7 +174,6 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
|
||||
max_num_batched_tokens=1024,
|
||||
max_num_blocks_per_group=[2, 8],
|
||||
device=device,
|
||||
kernel_block_sizes=[64, 64],
|
||||
dcp_sharded=[True, False],
|
||||
cp_size=4,
|
||||
cp_rank=cp_rank,
|
||||
@@ -230,7 +224,6 @@ def test_v1_block_table_move_row_clears_vacated_row():
|
||||
max_num_batched_tokens=64,
|
||||
pin_memory=False,
|
||||
device=torch.device("cuda"),
|
||||
kernel_block_size=16,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
)
|
||||
block_table.add_row([7, 8, 9], row_idx=0)
|
||||
@@ -260,7 +253,6 @@ def test_get_dummy_block_tables_returns_zeroed_rows():
|
||||
max_num_batched_tokens=64,
|
||||
max_num_blocks_per_group=[8],
|
||||
device=device,
|
||||
kernel_block_sizes=[16],
|
||||
)
|
||||
# Simulate a real step: stage a request's blocks and gather them into
|
||||
# the persistent input block tables.
|
||||
@@ -292,7 +284,6 @@ def test_dummy_request_slot_mapping_is_pad():
|
||||
max_num_batched_tokens=16,
|
||||
max_num_blocks_per_group=[4],
|
||||
device=device,
|
||||
kernel_block_sizes=[4],
|
||||
)
|
||||
block_tables.append_block_ids(req_index=0, new_block_ids=([7, 8],), overwrite=True)
|
||||
block_tables.apply_staged_writes()
|
||||
|
||||
@@ -233,7 +233,6 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
|
||||
device=torch.device(device),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
reqs: list[CachedRequestState] = []
|
||||
@@ -327,7 +326,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
|
||||
device=torch.device(device),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
ref_input_batch: InputBatch = InputBatch(
|
||||
@@ -337,7 +335,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
|
||||
device=torch.device(device),
|
||||
vocab_size=1024,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
|
||||
@@ -388,7 +385,6 @@ def test_condense_clears_stale_allowed_token_ids_mask(device: str):
|
||||
device=torch.device(device),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
|
||||
@@ -446,7 +442,6 @@ def test_swap_states_preserves_allowed_token_ids_mask(device: str):
|
||||
device=torch.device(device),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[1],
|
||||
kernel_block_sizes=[1],
|
||||
max_num_blocks_per_req=[1024],
|
||||
)
|
||||
|
||||
@@ -516,7 +511,6 @@ def test_pooling_prompt_lens_not_aliased(device: str):
|
||||
device=torch.device(device),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
is_pooling_model=True,
|
||||
)
|
||||
@@ -552,7 +546,6 @@ def test_placeholder_spec_token_ids_written_verbatim():
|
||||
device=torch.device("cpu"),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[1],
|
||||
)
|
||||
req = CachedRequestState(
|
||||
@@ -599,7 +592,6 @@ def test_pooling_metadata_token_id_buffers(
|
||||
device=torch.device("cpu"),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
is_pooling_model=True,
|
||||
)
|
||||
@@ -630,7 +622,6 @@ def _make_input_batch(is_pooling_model: bool = False) -> InputBatch:
|
||||
device=torch.device("cpu"),
|
||||
vocab_size=VOCAB_SIZE,
|
||||
block_sizes=[16],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[64],
|
||||
is_pooling_model=is_pooling_model,
|
||||
)
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
The kpool tail cache holds one circular block per request, so its block table
|
||||
row is orders of magnitude shorter than the sequence it serves.
|
||||
`_compute_slot_mappings_kernel` indexes block tables by
|
||||
`position // kernel_block_size` with an effectively unmasked load, so
|
||||
`position // block_size` with an effectively unmasked load, so
|
||||
this leads to a read of neighboring requests' rows (and for the later requests, outside
|
||||
the buffer entirely).
|
||||
"""
|
||||
@@ -15,7 +15,7 @@ import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec
|
||||
from vllm.v1.kv_cache_interface import CircularBufferSpec
|
||||
from vllm.v1.worker.block_table import get_block_table_width
|
||||
from vllm.v1.worker.gpu.block_table import BlockTables
|
||||
|
||||
@@ -32,13 +32,12 @@ POISON_BLOCK_ID = 1 << 20
|
||||
|
||||
|
||||
def make_tail_spec():
|
||||
return KpoolTailSpec(
|
||||
return CircularBufferSpec(
|
||||
block_size=KPOOL,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=KPOOL,
|
||||
)
|
||||
|
||||
|
||||
@@ -67,7 +66,6 @@ def test_kpool_tail_group_never_position_indexes_its_block_table():
|
||||
max_num_batched_tokens=len(positions),
|
||||
max_num_blocks_per_group=[mla_width, tail_width],
|
||||
device=device,
|
||||
kernel_block_sizes=[MLA_BLOCK_SIZE, spec.block_size],
|
||||
slot_mapping_enabled=[True, spec.uses_slot_mapping],
|
||||
)
|
||||
mla_blocks = list(range(1, mla_width + 1))
|
||||
|
||||
@@ -111,9 +111,6 @@ def initialize_kv_cache(runner: GPUModelRunner):
|
||||
device=runner.device,
|
||||
vocab_size=runner.model_config.get_vocab_size(),
|
||||
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
|
||||
kernel_block_sizes=[
|
||||
kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
|
||||
],
|
||||
max_num_blocks_per_req=[NUM_BLOCKS],
|
||||
)
|
||||
runner.initialize_attn_backend(kv_cache_config)
|
||||
@@ -303,7 +300,7 @@ def _make_mock_backend_for_kernel_block_size(
|
||||
return "MOCK"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None):
|
||||
return supported_sizes
|
||||
|
||||
return _MockBackend()
|
||||
@@ -441,7 +438,7 @@ def _mock_backend(supported: list, *, exact: bool = False):
|
||||
return "MOCK_EXACT" if exact else "MOCK"
|
||||
|
||||
@staticmethod
|
||||
def get_supported_kernel_block_sizes():
|
||||
def get_supported_kernel_block_sizes(kv_cache_spec=None):
|
||||
return list(supported)
|
||||
|
||||
if exact:
|
||||
@@ -1625,7 +1622,6 @@ def test_input_batch_reinitialized_after_late_interleave_adjustment(monkeypatch)
|
||||
runner.device = torch.device("cpu")
|
||||
runner.is_pooling_model = False
|
||||
runner._init_block_sizes = [16]
|
||||
runner._init_kernel_block_sizes = [16]
|
||||
runner._init_max_num_blocks = [4]
|
||||
runner._init_slot_mapping_modes = [
|
||||
gpu_model_runner_module.SlotMappingMode.TOKEN_TO_KV_SLOT
|
||||
@@ -1653,7 +1649,7 @@ def test_input_batch_reinitialized_after_late_interleave_adjustment(monkeypatch)
|
||||
lambda _: gpu_model_runner_module.KVCacheSpecKind.FULL_ATTENTION,
|
||||
)
|
||||
|
||||
runner.may_reinitialize_input_batch(kv_cache_config, [16])
|
||||
runner.may_reinitialize_input_batch(kv_cache_config)
|
||||
|
||||
assert input_batch_cls.call_count == 1
|
||||
assert input_batch_cls.call_args.kwargs["cp_kv_cache_interleave_size"] == 16
|
||||
@@ -1681,68 +1677,10 @@ def test_v2_runner_snapshots_late_interleave_adjustment(monkeypatch):
|
||||
assert runner.cp_interleave == 16
|
||||
|
||||
|
||||
def test_hybrid_block_table_initialization():
|
||||
"""Test hybrid block table with different kernel and kvcache_manager block
|
||||
sizes."""
|
||||
from vllm.v1.worker.block_table import BlockTable
|
||||
|
||||
# Test configuration: kvcache_manager block size = 32,
|
||||
# kernel block size = 16
|
||||
block_size = 32
|
||||
kernel_block_sizes = [16]
|
||||
max_num_reqs = 10
|
||||
max_num_blocks_per_req = 20
|
||||
max_num_batched_tokens = 512
|
||||
cp_kv_cache_interleave_size = 8
|
||||
|
||||
block_table = BlockTable(
|
||||
block_size=block_size,
|
||||
max_num_reqs=max_num_reqs,
|
||||
max_num_blocks_per_req=max_num_blocks_per_req,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
pin_memory=False,
|
||||
device=torch.device(DEVICE_TYPE),
|
||||
kernel_block_size=kernel_block_sizes[0],
|
||||
cp_kv_cache_interleave_size=cp_kv_cache_interleave_size,
|
||||
)
|
||||
|
||||
# Verify hybrid block configuration
|
||||
assert block_table.use_hybrid_blocks is True
|
||||
assert block_table.block_size == kernel_block_sizes[0]
|
||||
assert block_table.blocks_per_kv_block == (
|
||||
block_size // kernel_block_sizes[0]
|
||||
) # Changed to use first element
|
||||
|
||||
# Test block table conversion logic
|
||||
# One kvcache_manager block should map to multiple kernel blocks
|
||||
kvcache_manager_blocks = [0, 1, 2]
|
||||
|
||||
# Verify that kvcache_manager blocks can be converted to kernel blocks
|
||||
# and that block table operations work correctly.
|
||||
req_index = 0
|
||||
block_table.append_row(kvcache_manager_blocks, req_index)
|
||||
# Get expected kernel blocks from the implementation for verification.
|
||||
expected_kernel_blocks = block_table.map_to_kernel_blocks(
|
||||
np.array(kvcache_manager_blocks),
|
||||
block_table.blocks_per_kv_block,
|
||||
block_table._kernel_block_arange,
|
||||
)
|
||||
# Verify block table state
|
||||
assert block_table.num_blocks_per_row[req_index] == len(expected_kernel_blocks)
|
||||
assert np.array_equal(
|
||||
block_table.block_table.np[req_index, : len(expected_kernel_blocks)],
|
||||
expected_kernel_blocks,
|
||||
)
|
||||
|
||||
|
||||
def test_get_block_table_width_aligns_to_128_tokens():
|
||||
assert get_block_table_width(1875, 64) == 1876
|
||||
|
||||
|
||||
def test_get_block_table_width_splits_virtual_blocks():
|
||||
assert get_block_table_width(235, 256, 64) == 940
|
||||
|
||||
|
||||
def test_mamba_state_table_width_is_not_aligned():
|
||||
block_tables = MultiGroupBlockTable(
|
||||
max_num_reqs=1,
|
||||
@@ -1750,7 +1688,6 @@ def test_mamba_state_table_width_is_not_aligned():
|
||||
pin_memory=False,
|
||||
device=torch.device("cpu"),
|
||||
block_sizes=[39664],
|
||||
kernel_block_sizes=[39664],
|
||||
max_num_blocks=[1],
|
||||
slot_mapping_modes=[SlotMappingMode.NONE],
|
||||
)
|
||||
@@ -1758,42 +1695,6 @@ def test_mamba_state_table_width_is_not_aligned():
|
||||
assert block_tables[0].max_num_blocks_per_req == 1
|
||||
|
||||
|
||||
def test_input_batch_with_kernel_block_sizes():
|
||||
"""Test InputBatch initialization with kernel_block_sizes parameter."""
|
||||
max_num_reqs = 10
|
||||
max_model_len = 512
|
||||
max_num_batched_tokens = 512
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
vocab_size = 50272
|
||||
|
||||
# Test with different kernel block sizes
|
||||
block_sizes = [32, 64]
|
||||
kernel_block_sizes = [16, 32]
|
||||
|
||||
input_batch = InputBatch(
|
||||
max_num_reqs=max_num_reqs,
|
||||
max_model_len=max_model_len,
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
device=device,
|
||||
vocab_size=vocab_size,
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
max_num_blocks_per_req=[16, 8],
|
||||
)
|
||||
|
||||
# Verify that block tables were created with kernel block sizes
|
||||
assert len(input_batch.block_table.block_tables) == len(block_sizes)
|
||||
|
||||
for i, (kv_size, kernel_size) in enumerate(zip(block_sizes, kernel_block_sizes)):
|
||||
block_table = input_batch.block_table.block_tables[i]
|
||||
if kv_size != kernel_size:
|
||||
assert block_table.use_hybrid_blocks is True
|
||||
assert block_table.block_size == kernel_size
|
||||
else:
|
||||
assert block_table.use_hybrid_blocks is False
|
||||
assert block_table.block_size == kernel_size
|
||||
|
||||
|
||||
def test_hybrid_cache_integration(default_vllm_config, dist_init):
|
||||
"""Test hybrid cache architecture integration with GPUModelRunner."""
|
||||
# Create a new model runner with hybrid cache configuration
|
||||
@@ -1843,9 +1744,8 @@ def test_hybrid_cache_integration(default_vllm_config, dist_init):
|
||||
device=runner.device,
|
||||
vocab_size=runner.model_config.get_vocab_size(),
|
||||
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
|
||||
kernel_block_sizes=[16],
|
||||
max_num_blocks_per_req=[NUM_BLOCKS],
|
||||
) # Use kernel block size
|
||||
)
|
||||
|
||||
runner.initialize_attn_backend(kv_cache_config)
|
||||
|
||||
|
||||
@@ -119,9 +119,6 @@ def _initialize_kv_cache(runner: GPUModelRunner) -> None:
|
||||
device=runner.device,
|
||||
vocab_size=runner.model_config.get_vocab_size(),
|
||||
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
|
||||
kernel_block_sizes=[
|
||||
kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
|
||||
],
|
||||
max_num_blocks_per_req=[NUM_BLOCKS],
|
||||
)
|
||||
runner.initialize_attn_backend(kv_cache_config)
|
||||
|
||||
@@ -237,7 +237,6 @@ def test_append_block_ids_rejects_write_past_row_capacity():
|
||||
|
||||
block_tables = BlockTables.__new__(BlockTables)
|
||||
block_tables.num_kv_cache_groups = 1
|
||||
block_tables.blocks_per_kv_block = [1]
|
||||
block_tables.block_tables = [_BlockTable()]
|
||||
block_tables.num_blocks = SimpleNamespace(
|
||||
np=torch.tensor([[0, 3]], dtype=torch.int32)
|
||||
|
||||
@@ -47,8 +47,7 @@ def _make_capture_manager(block_table: torch.Tensor):
|
||||
block_tables = SimpleNamespace(
|
||||
input_block_tables=(block_table,),
|
||||
num_kv_cache_groups=1,
|
||||
kernel_block_sizes=(2,),
|
||||
blocks_per_kv_block=(1,),
|
||||
block_sizes=(2,),
|
||||
)
|
||||
manager = PCPManager(
|
||||
pcp_world_size=2,
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.model_executor.models.mistral_large_3_eagle import (
|
||||
from vllm.v1.attention.backends import flash_attn as flash_attn_module
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
|
||||
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
|
||||
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
|
||||
@@ -30,6 +31,7 @@ from vllm.v1.worker.gpu.spec_decode.target_dependent_ar import speculator as spe
|
||||
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
|
||||
TargetDependentARSpeculator,
|
||||
)
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
class _TestSpeculator(TargetDependentARSpeculator):
|
||||
@@ -102,6 +104,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
|
||||
get_metadata_builder=lambda _: SimpleNamespace(
|
||||
build=build, supports_update_block_table=False
|
||||
),
|
||||
build_metadata=lambda cm, ubatch_idx=0, **_: build(0, cm),
|
||||
layer_names=["draft"],
|
||||
)
|
||||
]
|
||||
@@ -427,6 +430,8 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
|
||||
query_start_loc=torch.arange(3),
|
||||
)
|
||||
speculator.idx_mapping = torch.arange(2)
|
||||
speculator.attn_groups = []
|
||||
speculator.block_tables = SimpleNamespace(input_block_tables=[])
|
||||
generate_draft = Mock()
|
||||
speculator._generate_draft = generate_draft
|
||||
run_fullgraph = Mock()
|
||||
@@ -450,6 +455,52 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
|
||||
assert run_fullgraph.call_count == expected_graph_replays
|
||||
|
||||
|
||||
def test_fused_full_graph_maps_split_group_block_tables():
|
||||
"""The fused FULL replay skips build(), so split groups' tables are mapped first."""
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=1024, num_kv_heads=1, head_size=128, dtype=torch.uint8
|
||||
)
|
||||
group = AttentionGroup(
|
||||
SimpleNamespace(),
|
||||
["layer.0"],
|
||||
spec,
|
||||
0,
|
||||
kernel_block_size=256,
|
||||
kernel_block_table=torch.zeros(1, 4, 32, dtype=torch.int32),
|
||||
kernel_block_offsets=torch.arange(4, dtype=torch.int32),
|
||||
)
|
||||
speculator = object.__new__(_TestSpeculator)
|
||||
speculator.input_buffers = SimpleNamespace(
|
||||
positions=torch.arange(2), query_start_loc=torch.arange(3)
|
||||
)
|
||||
speculator.idx_mapping = torch.arange(2)
|
||||
speculator.attn_groups = [[group]]
|
||||
speculator.block_tables = SimpleNamespace(
|
||||
input_block_tables=[torch.tensor([[1, 2], [0, 3]], dtype=torch.int32)]
|
||||
)
|
||||
replayed_tables = []
|
||||
speculator.decode_cudagraph_manager = SimpleNamespace(
|
||||
run_fullgraph=lambda _: replayed_tables.append(
|
||||
group.kernel_block_table[0, :2, :8].tolist()
|
||||
)
|
||||
)
|
||||
|
||||
speculator._fused_multi_step_decode(
|
||||
num_reqs=2,
|
||||
skip_attn=False,
|
||||
batch_desc=BatchExecutionDescriptor(
|
||||
cg_mode=CUDAGraphMode.FULL, num_tokens=2, num_reqs=2
|
||||
),
|
||||
num_tokens_across_dp=None,
|
||||
seq_lens_cpu_upper_bound=None,
|
||||
num_speculative_steps=4,
|
||||
)
|
||||
|
||||
assert replayed_tables == [
|
||||
[[4, 5, 6, 7, 8, 9, 10, 11], [0, 1, 2, 3, 12, 13, 14, 15]]
|
||||
]
|
||||
|
||||
|
||||
def test_update_draft_decode_metadata_updates_fa3_scheduler_metadata(
|
||||
monkeypatch,
|
||||
):
|
||||
|
||||
@@ -12,40 +12,24 @@ if not current_platform.is_cuda_alike():
|
||||
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kv_cache_block_size", "blocks_per_kv_block", "block_size", "block_size_rep"),
|
||||
[
|
||||
(256, 1, 256, 16),
|
||||
(256, 4, 64, 16),
|
||||
(64, 1, 64, 16),
|
||||
(8, 1, 8, 2),
|
||||
(4, 1, 4, 2),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("kv_cache_block_size", [256, 64, 8, 4])
|
||||
def test_compute_slot_mapping_warmup_matches_runtime_specializations(
|
||||
kv_cache_block_size: int,
|
||||
blocks_per_kv_block: int,
|
||||
block_size: int,
|
||||
block_size_rep: int,
|
||||
) -> None:
|
||||
kernel = ComputeSlotMappingKernel()
|
||||
kwargs = dict(
|
||||
kv_cache_block_size=kv_cache_block_size,
|
||||
blocks_per_kv_block=blocks_per_kv_block,
|
||||
total_cp_world_size=1,
|
||||
total_cp_rank=0,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
block_table_stride=32768,
|
||||
block_size=block_size,
|
||||
)
|
||||
expected = kernel.CompileKey(
|
||||
kv_cache_block_size=kv_cache_block_size,
|
||||
blocks_per_kv_block=blocks_per_kv_block,
|
||||
total_cp_world_size=1,
|
||||
total_cp_rank=0,
|
||||
cp_kv_cache_interleave_size=1,
|
||||
block_table_stride=16,
|
||||
block_size=block_size_rep,
|
||||
)
|
||||
|
||||
assert kernel.dispatch(**kwargs) == expected
|
||||
|
||||
@@ -112,6 +112,8 @@ def test_mrv1_kv_pool_only_wraps_backing_allocation(monkeypatch) -> None:
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type="test")),
|
||||
compilation_config=SimpleNamespace(static_forward_context={}),
|
||||
kv_caches=[],
|
||||
attn_groups=[],
|
||||
speculative_config=None,
|
||||
)
|
||||
result = gpu_model_runner.GPUModelRunner.initialize_kv_cache_tensors(
|
||||
runner,
|
||||
|
||||
@@ -659,7 +659,6 @@ def test_v2_align_ctx_binds_request_slot_tables_for_state_copies():
|
||||
max_num_batched_tokens=64,
|
||||
max_num_blocks_per_group=[4],
|
||||
device=device,
|
||||
kernel_block_sizes=[cfg.block_size],
|
||||
)
|
||||
|
||||
state = object.__new__(MambaHybridModelState)
|
||||
|
||||
@@ -443,6 +443,43 @@ def test_copy_cpu_kv_cache_logical_blocks_ignores_storage_padding():
|
||||
assert (backing[9] == -1).all()
|
||||
|
||||
|
||||
def test_bind_kv_cache_keeps_scheduler_block_views_for_runner():
|
||||
"""Block copies index scheduler blocks, so the runner keeps the unmapped
|
||||
views while a packed layer is bound with kernel blocks."""
|
||||
|
||||
class _Layer:
|
||||
def bind_kv_cache(self, kv_cache):
|
||||
self.kv_cache = kv_cache
|
||||
|
||||
num_blocks, row = 4, 64
|
||||
storage = torch.zeros(num_blocks * row)
|
||||
# Two layers packed into one block row: an 8x4 page, then a 4x2 page
|
||||
# split into two 2-row kernel blocks.
|
||||
mla = storage.as_strided((num_blocks, 8, 4), (row, 4, 1))
|
||||
indexer = storage.as_strided((num_blocks, 1, 4, 2), (row, 8, 2, 1), 40)
|
||||
# The indexer in 2-row kernel blocks: block b's kernel block j is b * 16 + j.
|
||||
mapped = indexer.as_strided((50, 1, 2, 2), (4, 8, 2, 1))
|
||||
kv_caches = {"layers.0.attn": mla, "layers.1.indexer": indexer}
|
||||
ctx = {name: _Layer() for name in kv_caches}
|
||||
runner_kv_caches: list[torch.Tensor] = []
|
||||
|
||||
bind_kv_cache(
|
||||
kv_caches,
|
||||
ctx,
|
||||
runner_kv_caches,
|
||||
layer_kv_caches={**kv_caches, "layers.1.indexer": mapped},
|
||||
)
|
||||
|
||||
assert ctx["layers.1.indexer"].kv_cache is mapped
|
||||
assert runner_kv_caches == [mla, indexer]
|
||||
copies = [KVCacheBlockCopy(1, 3)]
|
||||
with pytest.raises(AssertionError, match="not divisible"):
|
||||
copy_kv_cache_blocks_inplace([mapped], num_blocks, copies)
|
||||
storage[row : 2 * row] = 7
|
||||
copy_kv_cache_blocks_inplace(runner_kv_caches, num_blocks, copies)
|
||||
assert (mla[3] == 7).all() and (indexer[3] == 7).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shared,rank", [(False, 0), (True, 0), (True, 1)])
|
||||
@pytest.mark.parametrize("has_copies", [False, True])
|
||||
def test_hisparse_host_copy_waits_for_writes_on_writer_only(
|
||||
|
||||
Reference in New Issue
Block a user