[Core][DSV4] Compact MXFP4 indexer KV cache and packed group overlays (#48993)

This commit is contained in:
Summer Yang
2026-07-22 16:58:44 -07:00
committed by GitHub
parent 149daf0d72
commit f3a920a076
3 changed files with 213 additions and 81 deletions
+168 -20
View File
@@ -9,6 +9,7 @@ import torch
from vllm.v1.core.kv_cache_utils import (
_get_kv_cache_config_packed,
_get_kv_cache_groups_uniform_groups,
get_kv_cache_config_from_groups,
)
from vllm.v1.kv_cache_interface import (
@@ -16,6 +17,7 @@ from vllm.v1.kv_cache_interface import (
KVCacheGroupSpec,
KVCacheTensor,
MLAAttentionSpec,
SlidingWindowMLASpec,
SlidingWindowSpec,
UniformTypeKVCacheSpecs,
)
@@ -109,7 +111,132 @@ def _page_sizes_by_layer(
return page_sizes
def _packing_by_layer(
tensors: list[KVCacheTensor],
) -> dict[str, tuple[int, int]]:
return {
layer_name: (tensor.offset, tensor.block_stride)
for tensor in tensors
for layer_name in tensor.shared_by
}
def _make_views(
groups: list[KVCacheGroupSpec],
num_blocks: int,
tensors: list[KVCacheTensor],
) -> dict[str, torch.Tensor]:
page_sizes = _page_sizes_by_layer(groups)
packing = _packing_by_layer(tensors)
backing = torch.zeros(tensors[0].size, dtype=torch.uint8)
return {
layer_name: torch.as_strided(
backing,
size=(num_blocks, page_size),
stride=(packing[layer_name][1], 1),
storage_offset=packing[layer_name][0],
)
for layer_name, page_size in page_sizes.items()
}
def _make_page_group(prefix: str, page_sizes: list[int]) -> KVCacheGroupSpec:
specs = {
f"{prefix}.{i}": MagicMock(page_size_bytes=page_size)
for i, page_size in enumerate(page_sizes)
}
return KVCacheGroupSpec(
layer_names=list(specs),
kv_cache_spec=UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=specs),
)
class TestInterleavedPacking:
def test_compact_cache_overlays_fp32_state_group(self):
full_specs = {}
state_specs = {}
for i in range(2):
full_specs[f"mla.{i}"] = MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=512,
dtype=torch.uint8,
page_size_padded=32768,
indexes_kv_by_block_stride=True,
compress_ratio=4,
)
full_specs[f"indexer.{i}"] = MLAAttentionSpec(
block_size=256,
num_kv_heads=1,
head_size=68,
dtype=torch.uint8,
page_size_padded=4608,
compress_ratio=4,
)
state_specs[f"mla_state.{i}"] = SlidingWindowMLASpec(
block_size=4,
num_kv_heads=1,
head_size=2048,
dtype=torch.float32,
sliding_window=8,
indexes_kv_by_block_stride=True,
)
state_specs[f"indexer_state.{i}"] = SlidingWindowMLASpec(
block_size=4,
num_kv_heads=1,
head_size=512,
dtype=torch.float32,
sliding_window=8,
indexes_kv_by_block_stride=True,
)
grouped_specs = [
UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=full_specs),
UniformTypeKVCacheSpecs(block_size=4, kv_cache_specs=state_specs),
]
groups = _get_kv_cache_groups_uniform_groups(grouped_specs)
assert len(groups) == 2
assert {full_specs[f"indexer.{i}"].page_size_bytes for i in range(2)} == {4608}
assert {full_specs[f"indexer.{i}"].real_page_size_bytes for i in range(2)} == {
4352
}
assert {
state_specs[f"indexer_state.{i}"].page_size_bytes for i in range(2)
} == {8192}
full_group_bytes = 2 * (32768 + 4608)
state_group_bytes = 2 * (32768 + 8192)
bytes_per_block = max(full_group_bytes, state_group_bytes)
num_blocks, tensors = _get_kv_cache_config_packed(
_mock_vllm_config(), groups, bytes_per_block * 32
)
assert num_blocks == 32
assert {tensor.block_stride for tensor in tensors} == {bytes_per_block}
packing = _packing_by_layer(tensors)
assert packing["mla.0"][0] == packing["mla_state.0"][0] == 0
assert packing["indexer.0"][0] == 32768
assert packing["indexer_state.0"][0] == 32768
def test_deepseek_v4_pro_stride(self):
groups = [
_make_page_group("full", [32768, 4608] * 30 + [1024] * 31),
_make_page_group("c4_state", [32768, 8192] * 30),
_make_page_group("c128_state", [32768] * 31),
_make_page_group("swa.0", [32768] * 31),
_make_page_group("swa.1", [32768] * 30),
]
expected_stride = 1_228_800
num_blocks, tensors = _get_kv_cache_config_packed(
_mock_vllm_config(), groups, expected_stride * 32
)
assert num_blocks == 32
assert {tensor.block_stride for tensor in tensors} == {expected_stride}
assert {tensor.size for tensor in tensors} == {expected_stride * 32}
def test_all_tensors_have_block_stride(self):
_, tensors = _run()
for t in tensors:
@@ -122,9 +249,30 @@ class TestInterleavedPacking:
assert sizes.pop() > 0
def test_offsets_within_one_block(self):
_, tensors = _run()
for t in tensors:
assert t.offset < t.block_stride
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
_, tensors = _get_kv_cache_config_packed(
_mock_vllm_config(), groups, 100 * 1024 * 1024
)
page_sizes = _page_sizes_by_layer(groups)
packing = _packing_by_layer(tensors)
for layer_name, page_size in page_sizes.items():
offset, block_stride = packing[layer_name]
assert offset + page_size <= block_stride
def test_layouts_are_disjoint_within_each_group(self):
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
_, tensors = _get_kv_cache_config_packed(
_mock_vllm_config(), groups, 100 * 1024 * 1024
)
page_sizes = _page_sizes_by_layer(groups)
packing = _packing_by_layer(tensors)
for group in groups:
ranges = sorted(
(packing[name][0], packing[name][0] + page_sizes[name])
for name in group.layer_names
)
assert all(left[1] <= right[0] for left, right in zip(ranges, ranges[1:]))
def test_all_layers_accounted_for(self):
n_c4, n_c128, n_swa = 5, 4, 7
@@ -135,29 +283,29 @@ class TestInterleavedPacking:
expected = n_c4 * 2 + n_c128 + n_swa
assert len(all_names) == expected
def test_strided_views_are_independent(self):
def test_group_owned_blocks_do_not_alias(self):
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
page_sizes = _page_sizes_by_layer(groups)
num_blocks, tensors = _get_kv_cache_config_packed(
_mock_vllm_config(), groups, 100 * 1024 * 1024
)
backing = torch.zeros(tensors[0].size, dtype=torch.uint8)
views = []
for t in tensors:
page_size = page_sizes[t.shared_by[0]]
v = torch.as_strided(
backing,
size=(num_blocks, page_size),
stride=(t.block_stride, 1),
storage_offset=t.offset,
)
views.append(v)
views = _make_views(groups, num_blocks, tensors)
for i, v in enumerate(views):
v.fill_(i + 1)
expected = {}
value = 1
for block_id, group in enumerate(groups):
for layer_name in group.layer_names:
views[layer_name][block_id].fill_(value)
expected[layer_name] = (block_id, value)
value += 1
for i, v in enumerate(views):
assert (v == i + 1).all(), f"View {i} was corrupted"
for layer_name, (block_id, value) in expected.items():
assert (views[layer_name][block_id] == value).all()
# Once the first group releases its block, another group may reuse it.
for layer_name in groups[1].layer_names:
views[layer_name][0].fill_(255)
for layer_name in groups[1].layer_names:
assert (views[layer_name][0] == 255).all()
def test_hma_attention_groups_keep_default_backing(self):
full = _make_full_spec()
+11 -5
View File
@@ -26,6 +26,7 @@ from vllm.models.deepseek_v4.common.ops import (
fused_indexer_q_rope_quant,
fused_q_kv_rmsnorm,
)
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE
if TYPE_CHECKING:
from vllm.v1.attention.backends.mla.sparse_swa import (
@@ -727,11 +728,16 @@ class DeepseekV4Indexer(nn.Module):
)
assert cache_config is not None, "Deepseek V4 indexer requires cache_config"
# NOTE(yifan): FP8 indxer cache use the same layout as V3.2:
# head_dim bytes = 128 fp8 + 4 fp32 scale = 132.
# For FP4 indexer cache, we still allocate the same amount of memory as FP8,
# but only use the first half of the memory.
k_cache_head_dim = self.head_dim + self.head_dim // self.quant_block_size * 4
if self.use_fp4_kv:
# MXFP4 stores two values per byte plus one UE8M0 byte per 32 values.
# head_dim bytes = 64 packed values + 4 UE8M0 scales = 68.
k_cache_head_dim = self.head_dim // 2 + self.head_dim // MXFP4_BLOCK_SIZE
else:
# NOTE(yifan): FP8 indexer cache uses the same layout as V3.2:
# head_dim bytes = 128 fp8 + 4 fp32 scale = 132.
k_cache_head_dim = (
self.head_dim + self.head_dim // self.quant_block_size * 4
)
self.k_cache = DeepseekV4IndexerCache(
head_dim=k_cache_head_dim,
dtype=torch.uint8,
+34 -56
View File
@@ -983,9 +983,8 @@ def _pool_bytes_per_block(
):
return kv_cache_groups[0].kv_cache_spec.page_size_bytes
if _use_packed_kv_cache_config(vllm_config, kv_cache_groups):
# buckets = {page_size: [[layer_names], [layer_names], ...]}
buckets = _bucket_layers_by_page_size(kv_cache_groups)
return sum(ps * len(slots) for ps, slots in buckets.items())
block_stride, _ = _get_packed_kv_cache_layout(kv_cache_groups)
return block_stride
group_size = max(len(g.layer_names) for g in kv_cache_groups)
page_size = get_uniform_page_size([g.kv_cache_spec for g in kv_cache_groups])
return page_size * group_size
@@ -1260,29 +1259,29 @@ def _get_kv_cache_groups_uniform_page_size(
return create_kv_cache_group_specs(kv_cache_spec, grouped_layers)
def _bucket_layers_by_page_size(
def _get_packed_kv_cache_layout(
kv_cache_groups: list[KVCacheGroupSpec],
) -> dict[int, list[list[str]]]:
"""Bucket layers by page size: ``result[ps][slot_idx] = [layer_names]``.
) -> tuple[int, dict[int, list[str]]]:
"""Lay out each cache group densely in one shared block slab.
Layers from different groups at the same ``slot_idx`` share an underlying tensor
(they have independent block tables so block-id namespaces never collide).
A block ID is owned by one cache group at a time, so layouts from different
groups may overlap. Layers within a group remain disjoint.
"""
buckets: dict[int, list[list[str]]] = defaultdict(list)
layers_by_offset: dict[int, list[str]] = defaultdict(list)
block_stride = 0
for group in kv_cache_groups:
spec = group.kv_cache_spec
slot_count: dict[int, int] = defaultdict(int)
byte_offset = 0
for layer_name in group.layer_names:
if isinstance(spec, UniformTypeKVCacheSpecs):
ps = spec.kv_cache_specs[layer_name].page_size_bytes
page_size = spec.kv_cache_specs[layer_name].page_size_bytes
else:
ps = spec.page_size_bytes
slot_idx = slot_count[ps]
slot_count[ps] += 1
if slot_idx == len(buckets[ps]):
buckets[ps].append([])
buckets[ps][slot_idx].append(layer_name)
return buckets
page_size = spec.page_size_bytes
layers_by_offset[byte_offset].append(layer_name)
byte_offset += page_size
block_stride = max(block_stride, byte_offset)
assert block_stride > 0
return block_stride, layers_by_offset
def _use_packed_kv_cache_config(
@@ -1314,33 +1313,26 @@ def _get_kv_cache_config_packed(
) -> tuple[int, list[KVCacheTensor]]:
"""Plan a packed per-block KV cache tensor layout.
Emit one KVCacheTensor per (slot_idx, page_size). Layers from different
groups at the same slot share a tensor (they have independent block
tables so block-id namespaces never collide). Each emitted tensor aliases
one physical backing allocation, with per-block data laid out contiguously.
Cache groups use dense, overlapping layouts within one block slab. Each
emitted tensor aliases the same physical backing allocation.
"""
# buckets = {page_size: [[layer_names], [layer_names], ...]}
buckets = _bucket_layers_by_page_size(kv_cache_groups)
total_num_bytes_per_block = sum(ps * len(slots) for ps, slots in buckets.items())
block_stride, layers_by_offset = _get_packed_kv_cache_layout(kv_cache_groups)
num_blocks = available_memory // total_num_bytes_per_block
num_blocks = available_memory // block_stride
num_blocks = may_override_num_blocks(vllm_config, num_blocks)
total_size = total_num_bytes_per_block * num_blocks
total_size = block_stride * num_blocks
kv_cache_tensors: list[KVCacheTensor] = []
byte_offset = 0
for ps, slots in buckets.items():
for slot in slots:
kv_cache_tensors.append(
KVCacheTensor(
size=total_size,
shared_by=slot,
offset=byte_offset,
block_stride=total_num_bytes_per_block,
)
for byte_offset in sorted(layers_by_offset):
kv_cache_tensors.append(
KVCacheTensor(
size=total_size,
shared_by=layers_by_offset[byte_offset],
offset=byte_offset,
block_stride=block_stride,
)
byte_offset += ps
)
return num_blocks, kv_cache_tensors
@@ -1651,29 +1643,15 @@ def _get_kv_cache_groups_uniform_groups(
for spec in group.kv_cache_specs.values()
)
# Split each SWA UniformKV group into smaller groups to align their #(layer tuples)
# Possibly padding layer tuples for this.
# Additionally, we also pad KV blocks in each SWA layer, to align the page size
# with the corresponding layer in the full-MLA group.
all_page_sizes = full_mla_spec.get_page_sizes()
# Split each SWA UniformKV group into smaller groups to align their
# numbers of layer tuples. The packed block planner overlays groups, so
# their page sizes do not need to match.
swa_mla_groups = []
for sm_spec in swa_mla_specs:
sm_page_sizes = sm_spec.get_page_sizes()
layers_per_size: dict[int, list[str]] = defaultdict(list)
assert max(sm_page_sizes) <= max(all_page_sizes)
# Unify page size by padding layers' page_size to the nearest larger page_size.
# Compute candidate (nearest larger page_size) for each unique page size.
size_to_candidate: dict[int, int] = {}
for ps in sm_page_sizes:
size_to_candidate[ps] = min(x for x in all_page_sizes if x >= ps)
# Pad and collect layer names per page size.
for layer_name, layer_spec in sm_spec.kv_cache_specs.items():
current_size = layer_spec.page_size_bytes
candidate = size_to_candidate[current_size]
if current_size < candidate:
object.__setattr__(layer_spec, "page_size_padded", candidate)
layers_per_size[candidate].append(layer_name)
layers_per_size[layer_spec.page_size_bytes].append(layer_name)
# NOTE(yifan): for now, inside a UniformKV group, each page_size should
# have the same number of layers. This also means we don't need to pad layers
# inside a partial-full layer tuple.