[Core][DSV4] Compact MXFP4 indexer KV cache and packed group overlays (#48993)
This commit is contained in:
@@ -9,6 +9,7 @@ import torch
|
||||
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
_get_kv_cache_config_packed,
|
||||
_get_kv_cache_groups_uniform_groups,
|
||||
get_kv_cache_config_from_groups,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
@@ -16,6 +17,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
@@ -109,7 +111,132 @@ def _page_sizes_by_layer(
|
||||
return page_sizes
|
||||
|
||||
|
||||
def _packing_by_layer(
|
||||
tensors: list[KVCacheTensor],
|
||||
) -> dict[str, tuple[int, int]]:
|
||||
return {
|
||||
layer_name: (tensor.offset, tensor.block_stride)
|
||||
for tensor in tensors
|
||||
for layer_name in tensor.shared_by
|
||||
}
|
||||
|
||||
|
||||
def _make_views(
|
||||
groups: list[KVCacheGroupSpec],
|
||||
num_blocks: int,
|
||||
tensors: list[KVCacheTensor],
|
||||
) -> dict[str, torch.Tensor]:
|
||||
page_sizes = _page_sizes_by_layer(groups)
|
||||
packing = _packing_by_layer(tensors)
|
||||
backing = torch.zeros(tensors[0].size, dtype=torch.uint8)
|
||||
return {
|
||||
layer_name: torch.as_strided(
|
||||
backing,
|
||||
size=(num_blocks, page_size),
|
||||
stride=(packing[layer_name][1], 1),
|
||||
storage_offset=packing[layer_name][0],
|
||||
)
|
||||
for layer_name, page_size in page_sizes.items()
|
||||
}
|
||||
|
||||
|
||||
def _make_page_group(prefix: str, page_sizes: list[int]) -> KVCacheGroupSpec:
|
||||
specs = {
|
||||
f"{prefix}.{i}": MagicMock(page_size_bytes=page_size)
|
||||
for i, page_size in enumerate(page_sizes)
|
||||
}
|
||||
return KVCacheGroupSpec(
|
||||
layer_names=list(specs),
|
||||
kv_cache_spec=UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=specs),
|
||||
)
|
||||
|
||||
|
||||
class TestInterleavedPacking:
|
||||
def test_compact_cache_overlays_fp32_state_group(self):
|
||||
full_specs = {}
|
||||
state_specs = {}
|
||||
for i in range(2):
|
||||
full_specs[f"mla.{i}"] = MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.uint8,
|
||||
page_size_padded=32768,
|
||||
indexes_kv_by_block_stride=True,
|
||||
compress_ratio=4,
|
||||
)
|
||||
full_specs[f"indexer.{i}"] = MLAAttentionSpec(
|
||||
block_size=256,
|
||||
num_kv_heads=1,
|
||||
head_size=68,
|
||||
dtype=torch.uint8,
|
||||
page_size_padded=4608,
|
||||
compress_ratio=4,
|
||||
)
|
||||
state_specs[f"mla_state.{i}"] = SlidingWindowMLASpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=2048,
|
||||
dtype=torch.float32,
|
||||
sliding_window=8,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
state_specs[f"indexer_state.{i}"] = SlidingWindowMLASpec(
|
||||
block_size=4,
|
||||
num_kv_heads=1,
|
||||
head_size=512,
|
||||
dtype=torch.float32,
|
||||
sliding_window=8,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
|
||||
grouped_specs = [
|
||||
UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=full_specs),
|
||||
UniformTypeKVCacheSpecs(block_size=4, kv_cache_specs=state_specs),
|
||||
]
|
||||
groups = _get_kv_cache_groups_uniform_groups(grouped_specs)
|
||||
|
||||
assert len(groups) == 2
|
||||
assert {full_specs[f"indexer.{i}"].page_size_bytes for i in range(2)} == {4608}
|
||||
assert {full_specs[f"indexer.{i}"].real_page_size_bytes for i in range(2)} == {
|
||||
4352
|
||||
}
|
||||
assert {
|
||||
state_specs[f"indexer_state.{i}"].page_size_bytes for i in range(2)
|
||||
} == {8192}
|
||||
|
||||
full_group_bytes = 2 * (32768 + 4608)
|
||||
state_group_bytes = 2 * (32768 + 8192)
|
||||
bytes_per_block = max(full_group_bytes, state_group_bytes)
|
||||
num_blocks, tensors = _get_kv_cache_config_packed(
|
||||
_mock_vllm_config(), groups, bytes_per_block * 32
|
||||
)
|
||||
assert num_blocks == 32
|
||||
assert {tensor.block_stride for tensor in tensors} == {bytes_per_block}
|
||||
|
||||
packing = _packing_by_layer(tensors)
|
||||
assert packing["mla.0"][0] == packing["mla_state.0"][0] == 0
|
||||
assert packing["indexer.0"][0] == 32768
|
||||
assert packing["indexer_state.0"][0] == 32768
|
||||
|
||||
def test_deepseek_v4_pro_stride(self):
|
||||
groups = [
|
||||
_make_page_group("full", [32768, 4608] * 30 + [1024] * 31),
|
||||
_make_page_group("c4_state", [32768, 8192] * 30),
|
||||
_make_page_group("c128_state", [32768] * 31),
|
||||
_make_page_group("swa.0", [32768] * 31),
|
||||
_make_page_group("swa.1", [32768] * 30),
|
||||
]
|
||||
expected_stride = 1_228_800
|
||||
|
||||
num_blocks, tensors = _get_kv_cache_config_packed(
|
||||
_mock_vllm_config(), groups, expected_stride * 32
|
||||
)
|
||||
|
||||
assert num_blocks == 32
|
||||
assert {tensor.block_stride for tensor in tensors} == {expected_stride}
|
||||
assert {tensor.size for tensor in tensors} == {expected_stride * 32}
|
||||
|
||||
def test_all_tensors_have_block_stride(self):
|
||||
_, tensors = _run()
|
||||
for t in tensors:
|
||||
@@ -122,9 +249,30 @@ class TestInterleavedPacking:
|
||||
assert sizes.pop() > 0
|
||||
|
||||
def test_offsets_within_one_block(self):
|
||||
_, tensors = _run()
|
||||
for t in tensors:
|
||||
assert t.offset < t.block_stride
|
||||
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
|
||||
_, tensors = _get_kv_cache_config_packed(
|
||||
_mock_vllm_config(), groups, 100 * 1024 * 1024
|
||||
)
|
||||
page_sizes = _page_sizes_by_layer(groups)
|
||||
packing = _packing_by_layer(tensors)
|
||||
for layer_name, page_size in page_sizes.items():
|
||||
offset, block_stride = packing[layer_name]
|
||||
assert offset + page_size <= block_stride
|
||||
|
||||
def test_layouts_are_disjoint_within_each_group(self):
|
||||
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
|
||||
_, tensors = _get_kv_cache_config_packed(
|
||||
_mock_vllm_config(), groups, 100 * 1024 * 1024
|
||||
)
|
||||
page_sizes = _page_sizes_by_layer(groups)
|
||||
packing = _packing_by_layer(tensors)
|
||||
|
||||
for group in groups:
|
||||
ranges = sorted(
|
||||
(packing[name][0], packing[name][0] + page_sizes[name])
|
||||
for name in group.layer_names
|
||||
)
|
||||
assert all(left[1] <= right[0] for left, right in zip(ranges, ranges[1:]))
|
||||
|
||||
def test_all_layers_accounted_for(self):
|
||||
n_c4, n_c128, n_swa = 5, 4, 7
|
||||
@@ -135,29 +283,29 @@ class TestInterleavedPacking:
|
||||
expected = n_c4 * 2 + n_c128 + n_swa
|
||||
assert len(all_names) == expected
|
||||
|
||||
def test_strided_views_are_independent(self):
|
||||
def test_group_owned_blocks_do_not_alias(self):
|
||||
groups = _make_groups(n_c4=3, n_c128=2, n_swa=5)
|
||||
page_sizes = _page_sizes_by_layer(groups)
|
||||
num_blocks, tensors = _get_kv_cache_config_packed(
|
||||
_mock_vllm_config(), groups, 100 * 1024 * 1024
|
||||
)
|
||||
backing = torch.zeros(tensors[0].size, dtype=torch.uint8)
|
||||
views = []
|
||||
for t in tensors:
|
||||
page_size = page_sizes[t.shared_by[0]]
|
||||
v = torch.as_strided(
|
||||
backing,
|
||||
size=(num_blocks, page_size),
|
||||
stride=(t.block_stride, 1),
|
||||
storage_offset=t.offset,
|
||||
)
|
||||
views.append(v)
|
||||
views = _make_views(groups, num_blocks, tensors)
|
||||
|
||||
for i, v in enumerate(views):
|
||||
v.fill_(i + 1)
|
||||
expected = {}
|
||||
value = 1
|
||||
for block_id, group in enumerate(groups):
|
||||
for layer_name in group.layer_names:
|
||||
views[layer_name][block_id].fill_(value)
|
||||
expected[layer_name] = (block_id, value)
|
||||
value += 1
|
||||
|
||||
for i, v in enumerate(views):
|
||||
assert (v == i + 1).all(), f"View {i} was corrupted"
|
||||
for layer_name, (block_id, value) in expected.items():
|
||||
assert (views[layer_name][block_id] == value).all()
|
||||
|
||||
# Once the first group releases its block, another group may reuse it.
|
||||
for layer_name in groups[1].layer_names:
|
||||
views[layer_name][0].fill_(255)
|
||||
for layer_name in groups[1].layer_names:
|
||||
assert (views[layer_name][0] == 255).all()
|
||||
|
||||
def test_hma_attention_groups_keep_default_backing(self):
|
||||
full = _make_full_spec()
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.models.deepseek_v4.common.ops import (
|
||||
fused_indexer_q_rope_quant,
|
||||
fused_q_kv_rmsnorm,
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.attention.backends.mla.sparse_swa import (
|
||||
@@ -727,11 +728,16 @@ class DeepseekV4Indexer(nn.Module):
|
||||
)
|
||||
|
||||
assert cache_config is not None, "Deepseek V4 indexer requires cache_config"
|
||||
# NOTE(yifan): FP8 indxer cache use the same layout as V3.2:
|
||||
# head_dim bytes = 128 fp8 + 4 fp32 scale = 132.
|
||||
# For FP4 indexer cache, we still allocate the same amount of memory as FP8,
|
||||
# but only use the first half of the memory.
|
||||
k_cache_head_dim = self.head_dim + self.head_dim // self.quant_block_size * 4
|
||||
if self.use_fp4_kv:
|
||||
# MXFP4 stores two values per byte plus one UE8M0 byte per 32 values.
|
||||
# head_dim bytes = 64 packed values + 4 UE8M0 scales = 68.
|
||||
k_cache_head_dim = self.head_dim // 2 + self.head_dim // MXFP4_BLOCK_SIZE
|
||||
else:
|
||||
# NOTE(yifan): FP8 indexer cache uses the same layout as V3.2:
|
||||
# head_dim bytes = 128 fp8 + 4 fp32 scale = 132.
|
||||
k_cache_head_dim = (
|
||||
self.head_dim + self.head_dim // self.quant_block_size * 4
|
||||
)
|
||||
self.k_cache = DeepseekV4IndexerCache(
|
||||
head_dim=k_cache_head_dim,
|
||||
dtype=torch.uint8,
|
||||
|
||||
@@ -983,9 +983,8 @@ def _pool_bytes_per_block(
|
||||
):
|
||||
return kv_cache_groups[0].kv_cache_spec.page_size_bytes
|
||||
if _use_packed_kv_cache_config(vllm_config, kv_cache_groups):
|
||||
# buckets = {page_size: [[layer_names], [layer_names], ...]}
|
||||
buckets = _bucket_layers_by_page_size(kv_cache_groups)
|
||||
return sum(ps * len(slots) for ps, slots in buckets.items())
|
||||
block_stride, _ = _get_packed_kv_cache_layout(kv_cache_groups)
|
||||
return block_stride
|
||||
group_size = max(len(g.layer_names) for g in kv_cache_groups)
|
||||
page_size = get_uniform_page_size([g.kv_cache_spec for g in kv_cache_groups])
|
||||
return page_size * group_size
|
||||
@@ -1260,29 +1259,29 @@ def _get_kv_cache_groups_uniform_page_size(
|
||||
return create_kv_cache_group_specs(kv_cache_spec, grouped_layers)
|
||||
|
||||
|
||||
def _bucket_layers_by_page_size(
|
||||
def _get_packed_kv_cache_layout(
|
||||
kv_cache_groups: list[KVCacheGroupSpec],
|
||||
) -> dict[int, list[list[str]]]:
|
||||
"""Bucket layers by page size: ``result[ps][slot_idx] = [layer_names]``.
|
||||
) -> tuple[int, dict[int, list[str]]]:
|
||||
"""Lay out each cache group densely in one shared block slab.
|
||||
|
||||
Layers from different groups at the same ``slot_idx`` share an underlying tensor
|
||||
(they have independent block tables so block-id namespaces never collide).
|
||||
A block ID is owned by one cache group at a time, so layouts from different
|
||||
groups may overlap. Layers within a group remain disjoint.
|
||||
"""
|
||||
buckets: dict[int, list[list[str]]] = defaultdict(list)
|
||||
layers_by_offset: dict[int, list[str]] = defaultdict(list)
|
||||
block_stride = 0
|
||||
for group in kv_cache_groups:
|
||||
spec = group.kv_cache_spec
|
||||
slot_count: dict[int, int] = defaultdict(int)
|
||||
byte_offset = 0
|
||||
for layer_name in group.layer_names:
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
ps = spec.kv_cache_specs[layer_name].page_size_bytes
|
||||
page_size = spec.kv_cache_specs[layer_name].page_size_bytes
|
||||
else:
|
||||
ps = spec.page_size_bytes
|
||||
slot_idx = slot_count[ps]
|
||||
slot_count[ps] += 1
|
||||
if slot_idx == len(buckets[ps]):
|
||||
buckets[ps].append([])
|
||||
buckets[ps][slot_idx].append(layer_name)
|
||||
return buckets
|
||||
page_size = spec.page_size_bytes
|
||||
layers_by_offset[byte_offset].append(layer_name)
|
||||
byte_offset += page_size
|
||||
block_stride = max(block_stride, byte_offset)
|
||||
assert block_stride > 0
|
||||
return block_stride, layers_by_offset
|
||||
|
||||
|
||||
def _use_packed_kv_cache_config(
|
||||
@@ -1314,33 +1313,26 @@ def _get_kv_cache_config_packed(
|
||||
) -> tuple[int, list[KVCacheTensor]]:
|
||||
"""Plan a packed per-block KV cache tensor layout.
|
||||
|
||||
Emit one KVCacheTensor per (slot_idx, page_size). Layers from different
|
||||
groups at the same slot share a tensor (they have independent block
|
||||
tables so block-id namespaces never collide). Each emitted tensor aliases
|
||||
one physical backing allocation, with per-block data laid out contiguously.
|
||||
Cache groups use dense, overlapping layouts within one block slab. Each
|
||||
emitted tensor aliases the same physical backing allocation.
|
||||
"""
|
||||
# buckets = {page_size: [[layer_names], [layer_names], ...]}
|
||||
buckets = _bucket_layers_by_page_size(kv_cache_groups)
|
||||
total_num_bytes_per_block = sum(ps * len(slots) for ps, slots in buckets.items())
|
||||
block_stride, layers_by_offset = _get_packed_kv_cache_layout(kv_cache_groups)
|
||||
|
||||
num_blocks = available_memory // total_num_bytes_per_block
|
||||
num_blocks = available_memory // block_stride
|
||||
num_blocks = may_override_num_blocks(vllm_config, num_blocks)
|
||||
|
||||
total_size = total_num_bytes_per_block * num_blocks
|
||||
total_size = block_stride * num_blocks
|
||||
|
||||
kv_cache_tensors: list[KVCacheTensor] = []
|
||||
byte_offset = 0
|
||||
for ps, slots in buckets.items():
|
||||
for slot in slots:
|
||||
kv_cache_tensors.append(
|
||||
KVCacheTensor(
|
||||
size=total_size,
|
||||
shared_by=slot,
|
||||
offset=byte_offset,
|
||||
block_stride=total_num_bytes_per_block,
|
||||
)
|
||||
for byte_offset in sorted(layers_by_offset):
|
||||
kv_cache_tensors.append(
|
||||
KVCacheTensor(
|
||||
size=total_size,
|
||||
shared_by=layers_by_offset[byte_offset],
|
||||
offset=byte_offset,
|
||||
block_stride=block_stride,
|
||||
)
|
||||
byte_offset += ps
|
||||
)
|
||||
|
||||
return num_blocks, kv_cache_tensors
|
||||
|
||||
@@ -1651,29 +1643,15 @@ def _get_kv_cache_groups_uniform_groups(
|
||||
for spec in group.kv_cache_specs.values()
|
||||
)
|
||||
|
||||
# Split each SWA UniformKV group into smaller groups to align their #(layer tuples)
|
||||
# Possibly padding layer tuples for this.
|
||||
# Additionally, we also pad KV blocks in each SWA layer, to align the page size
|
||||
# with the corresponding layer in the full-MLA group.
|
||||
all_page_sizes = full_mla_spec.get_page_sizes()
|
||||
# Split each SWA UniformKV group into smaller groups to align their
|
||||
# numbers of layer tuples. The packed block planner overlays groups, so
|
||||
# their page sizes do not need to match.
|
||||
swa_mla_groups = []
|
||||
for sm_spec in swa_mla_specs:
|
||||
sm_page_sizes = sm_spec.get_page_sizes()
|
||||
layers_per_size: dict[int, list[str]] = defaultdict(list)
|
||||
assert max(sm_page_sizes) <= max(all_page_sizes)
|
||||
|
||||
# Unify page size by padding layers' page_size to the nearest larger page_size.
|
||||
# Compute candidate (nearest larger page_size) for each unique page size.
|
||||
size_to_candidate: dict[int, int] = {}
|
||||
for ps in sm_page_sizes:
|
||||
size_to_candidate[ps] = min(x for x in all_page_sizes if x >= ps)
|
||||
# Pad and collect layer names per page size.
|
||||
for layer_name, layer_spec in sm_spec.kv_cache_specs.items():
|
||||
current_size = layer_spec.page_size_bytes
|
||||
candidate = size_to_candidate[current_size]
|
||||
if current_size < candidate:
|
||||
object.__setattr__(layer_spec, "page_size_padded", candidate)
|
||||
layers_per_size[candidate].append(layer_name)
|
||||
layers_per_size[layer_spec.page_size_bytes].append(layer_name)
|
||||
# NOTE(yifan): for now, inside a UniformKV group, each page_size should
|
||||
# have the same number of layers. This also means we don't need to pad layers
|
||||
# inside a partial-full layer tuple.
|
||||
|
||||
Reference in New Issue
Block a user