diff --git a/tests/v1/core/test_contiguous_kv_packing.py b/tests/v1/core/test_contiguous_kv_packing.py index 647241ce73..88d17e9acd 100644 --- a/tests/v1/core/test_contiguous_kv_packing.py +++ b/tests/v1/core/test_contiguous_kv_packing.py @@ -9,6 +9,7 @@ import torch from vllm.v1.core.kv_cache_utils import ( _get_kv_cache_config_packed, + _get_kv_cache_groups_uniform_groups, get_kv_cache_config_from_groups, ) from vllm.v1.kv_cache_interface import ( @@ -16,6 +17,7 @@ from vllm.v1.kv_cache_interface import ( KVCacheGroupSpec, KVCacheTensor, MLAAttentionSpec, + SlidingWindowMLASpec, SlidingWindowSpec, UniformTypeKVCacheSpecs, ) @@ -109,7 +111,132 @@ def _page_sizes_by_layer( return page_sizes +def _packing_by_layer( + tensors: list[KVCacheTensor], +) -> dict[str, tuple[int, int]]: + return { + layer_name: (tensor.offset, tensor.block_stride) + for tensor in tensors + for layer_name in tensor.shared_by + } + + +def _make_views( + groups: list[KVCacheGroupSpec], + num_blocks: int, + tensors: list[KVCacheTensor], +) -> dict[str, torch.Tensor]: + page_sizes = _page_sizes_by_layer(groups) + packing = _packing_by_layer(tensors) + backing = torch.zeros(tensors[0].size, dtype=torch.uint8) + return { + layer_name: torch.as_strided( + backing, + size=(num_blocks, page_size), + stride=(packing[layer_name][1], 1), + storage_offset=packing[layer_name][0], + ) + for layer_name, page_size in page_sizes.items() + } + + +def _make_page_group(prefix: str, page_sizes: list[int]) -> KVCacheGroupSpec: + specs = { + f"{prefix}.{i}": MagicMock(page_size_bytes=page_size) + for i, page_size in enumerate(page_sizes) + } + return KVCacheGroupSpec( + layer_names=list(specs), + kv_cache_spec=UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=specs), + ) + + class TestInterleavedPacking: + def test_compact_cache_overlays_fp32_state_group(self): + full_specs = {} + state_specs = {} + for i in range(2): + full_specs[f"mla.{i}"] = MLAAttentionSpec( + block_size=256, + num_kv_heads=1, + head_size=512, + dtype=torch.uint8, + page_size_padded=32768, + indexes_kv_by_block_stride=True, + compress_ratio=4, + ) + full_specs[f"indexer.{i}"] = MLAAttentionSpec( + block_size=256, + num_kv_heads=1, + head_size=68, + dtype=torch.uint8, + page_size_padded=4608, + compress_ratio=4, + ) + state_specs[f"mla_state.{i}"] = SlidingWindowMLASpec( + block_size=4, + num_kv_heads=1, + head_size=2048, + dtype=torch.float32, + sliding_window=8, + indexes_kv_by_block_stride=True, + ) + state_specs[f"indexer_state.{i}"] = SlidingWindowMLASpec( + block_size=4, + num_kv_heads=1, + head_size=512, + dtype=torch.float32, + sliding_window=8, + indexes_kv_by_block_stride=True, + ) + + grouped_specs = [ + UniformTypeKVCacheSpecs(block_size=256, kv_cache_specs=full_specs), + UniformTypeKVCacheSpecs(block_size=4, kv_cache_specs=state_specs), + ] + groups = _get_kv_cache_groups_uniform_groups(grouped_specs) + + assert len(groups) == 2 + assert {full_specs[f"indexer.{i}"].page_size_bytes for i in range(2)} == {4608} + assert {full_specs[f"indexer.{i}"].real_page_size_bytes for i in range(2)} == { + 4352 + } + assert { + state_specs[f"indexer_state.{i}"].page_size_bytes for i in range(2) + } == {8192} + + full_group_bytes = 2 * (32768 + 4608) + state_group_bytes = 2 * (32768 + 8192) + bytes_per_block = max(full_group_bytes, state_group_bytes) + num_blocks, tensors = _get_kv_cache_config_packed( + _mock_vllm_config(), groups, bytes_per_block * 32 + ) + assert num_blocks == 32 + assert {tensor.block_stride for tensor in tensors} == {bytes_per_block} + + packing = _packing_by_layer(tensors) + assert packing["mla.0"][0] == packing["mla_state.0"][0] == 0 + assert packing["indexer.0"][0] == 32768 + assert packing["indexer_state.0"][0] == 32768 + + def test_deepseek_v4_pro_stride(self): + groups = [ + _make_page_group("full", [32768, 4608] * 30 + [1024] * 31), + _make_page_group("c4_state", [32768, 8192] * 30), + _make_page_group("c128_state", [32768] * 31), + _make_page_group("swa.0", [32768] * 31), + _make_page_group("swa.1", [32768] * 30), + ] + expected_stride = 1_228_800 + + num_blocks, tensors = _get_kv_cache_config_packed( + _mock_vllm_config(), groups, expected_stride * 32 + ) + + assert num_blocks == 32 + assert {tensor.block_stride for tensor in tensors} == {expected_stride} + assert {tensor.size for tensor in tensors} == {expected_stride * 32} + def test_all_tensors_have_block_stride(self): _, tensors = _run() for t in tensors: @@ -122,9 +249,30 @@ class TestInterleavedPacking: assert sizes.pop() > 0 def test_offsets_within_one_block(self): - _, tensors = _run() - for t in tensors: - assert t.offset < t.block_stride + groups = _make_groups(n_c4=3, n_c128=2, n_swa=5) + _, tensors = _get_kv_cache_config_packed( + _mock_vllm_config(), groups, 100 * 1024 * 1024 + ) + page_sizes = _page_sizes_by_layer(groups) + packing = _packing_by_layer(tensors) + for layer_name, page_size in page_sizes.items(): + offset, block_stride = packing[layer_name] + assert offset + page_size <= block_stride + + def test_layouts_are_disjoint_within_each_group(self): + groups = _make_groups(n_c4=3, n_c128=2, n_swa=5) + _, tensors = _get_kv_cache_config_packed( + _mock_vllm_config(), groups, 100 * 1024 * 1024 + ) + page_sizes = _page_sizes_by_layer(groups) + packing = _packing_by_layer(tensors) + + for group in groups: + ranges = sorted( + (packing[name][0], packing[name][0] + page_sizes[name]) + for name in group.layer_names + ) + assert all(left[1] <= right[0] for left, right in zip(ranges, ranges[1:])) def test_all_layers_accounted_for(self): n_c4, n_c128, n_swa = 5, 4, 7 @@ -135,29 +283,29 @@ class TestInterleavedPacking: expected = n_c4 * 2 + n_c128 + n_swa assert len(all_names) == expected - def test_strided_views_are_independent(self): + def test_group_owned_blocks_do_not_alias(self): groups = _make_groups(n_c4=3, n_c128=2, n_swa=5) - page_sizes = _page_sizes_by_layer(groups) num_blocks, tensors = _get_kv_cache_config_packed( _mock_vllm_config(), groups, 100 * 1024 * 1024 ) - backing = torch.zeros(tensors[0].size, dtype=torch.uint8) - views = [] - for t in tensors: - page_size = page_sizes[t.shared_by[0]] - v = torch.as_strided( - backing, - size=(num_blocks, page_size), - stride=(t.block_stride, 1), - storage_offset=t.offset, - ) - views.append(v) + views = _make_views(groups, num_blocks, tensors) - for i, v in enumerate(views): - v.fill_(i + 1) + expected = {} + value = 1 + for block_id, group in enumerate(groups): + for layer_name in group.layer_names: + views[layer_name][block_id].fill_(value) + expected[layer_name] = (block_id, value) + value += 1 - for i, v in enumerate(views): - assert (v == i + 1).all(), f"View {i} was corrupted" + for layer_name, (block_id, value) in expected.items(): + assert (views[layer_name][block_id] == value).all() + + # Once the first group releases its block, another group may reuse it. + for layer_name in groups[1].layer_names: + views[layer_name][0].fill_(255) + for layer_name in groups[1].layer_names: + assert (views[layer_name][0] == 255).all() def test_hma_attention_groups_keep_default_backing(self): full = _make_full_spec() diff --git a/vllm/models/deepseek_v4/attention.py b/vllm/models/deepseek_v4/attention.py index 5628a6d0d7..913b506dc6 100644 --- a/vllm/models/deepseek_v4/attention.py +++ b/vllm/models/deepseek_v4/attention.py @@ -26,6 +26,7 @@ from vllm.models.deepseek_v4.common.ops import ( fused_indexer_q_rope_quant, fused_q_kv_rmsnorm, ) +from vllm.models.deepseek_v4.common.ops.fused_indexer_q import MXFP4_BLOCK_SIZE if TYPE_CHECKING: from vllm.v1.attention.backends.mla.sparse_swa import ( @@ -727,11 +728,16 @@ class DeepseekV4Indexer(nn.Module): ) assert cache_config is not None, "Deepseek V4 indexer requires cache_config" - # NOTE(yifan): FP8 indxer cache use the same layout as V3.2: - # head_dim bytes = 128 fp8 + 4 fp32 scale = 132. - # For FP4 indexer cache, we still allocate the same amount of memory as FP8, - # but only use the first half of the memory. - k_cache_head_dim = self.head_dim + self.head_dim // self.quant_block_size * 4 + if self.use_fp4_kv: + # MXFP4 stores two values per byte plus one UE8M0 byte per 32 values. + # head_dim bytes = 64 packed values + 4 UE8M0 scales = 68. + k_cache_head_dim = self.head_dim // 2 + self.head_dim // MXFP4_BLOCK_SIZE + else: + # NOTE(yifan): FP8 indexer cache uses the same layout as V3.2: + # head_dim bytes = 128 fp8 + 4 fp32 scale = 132. + k_cache_head_dim = ( + self.head_dim + self.head_dim // self.quant_block_size * 4 + ) self.k_cache = DeepseekV4IndexerCache( head_dim=k_cache_head_dim, dtype=torch.uint8, diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index f86f8db2b1..0db1aa136b 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -983,9 +983,8 @@ def _pool_bytes_per_block( ): return kv_cache_groups[0].kv_cache_spec.page_size_bytes if _use_packed_kv_cache_config(vllm_config, kv_cache_groups): - # buckets = {page_size: [[layer_names], [layer_names], ...]} - buckets = _bucket_layers_by_page_size(kv_cache_groups) - return sum(ps * len(slots) for ps, slots in buckets.items()) + block_stride, _ = _get_packed_kv_cache_layout(kv_cache_groups) + return block_stride group_size = max(len(g.layer_names) for g in kv_cache_groups) page_size = get_uniform_page_size([g.kv_cache_spec for g in kv_cache_groups]) return page_size * group_size @@ -1260,29 +1259,29 @@ def _get_kv_cache_groups_uniform_page_size( return create_kv_cache_group_specs(kv_cache_spec, grouped_layers) -def _bucket_layers_by_page_size( +def _get_packed_kv_cache_layout( kv_cache_groups: list[KVCacheGroupSpec], -) -> dict[int, list[list[str]]]: - """Bucket layers by page size: ``result[ps][slot_idx] = [layer_names]``. +) -> tuple[int, dict[int, list[str]]]: + """Lay out each cache group densely in one shared block slab. - Layers from different groups at the same ``slot_idx`` share an underlying tensor - (they have independent block tables so block-id namespaces never collide). + A block ID is owned by one cache group at a time, so layouts from different + groups may overlap. Layers within a group remain disjoint. """ - buckets: dict[int, list[list[str]]] = defaultdict(list) + layers_by_offset: dict[int, list[str]] = defaultdict(list) + block_stride = 0 for group in kv_cache_groups: spec = group.kv_cache_spec - slot_count: dict[int, int] = defaultdict(int) + byte_offset = 0 for layer_name in group.layer_names: if isinstance(spec, UniformTypeKVCacheSpecs): - ps = spec.kv_cache_specs[layer_name].page_size_bytes + page_size = spec.kv_cache_specs[layer_name].page_size_bytes else: - ps = spec.page_size_bytes - slot_idx = slot_count[ps] - slot_count[ps] += 1 - if slot_idx == len(buckets[ps]): - buckets[ps].append([]) - buckets[ps][slot_idx].append(layer_name) - return buckets + page_size = spec.page_size_bytes + layers_by_offset[byte_offset].append(layer_name) + byte_offset += page_size + block_stride = max(block_stride, byte_offset) + assert block_stride > 0 + return block_stride, layers_by_offset def _use_packed_kv_cache_config( @@ -1314,33 +1313,26 @@ def _get_kv_cache_config_packed( ) -> tuple[int, list[KVCacheTensor]]: """Plan a packed per-block KV cache tensor layout. - Emit one KVCacheTensor per (slot_idx, page_size). Layers from different - groups at the same slot share a tensor (they have independent block - tables so block-id namespaces never collide). Each emitted tensor aliases - one physical backing allocation, with per-block data laid out contiguously. + Cache groups use dense, overlapping layouts within one block slab. Each + emitted tensor aliases the same physical backing allocation. """ - # buckets = {page_size: [[layer_names], [layer_names], ...]} - buckets = _bucket_layers_by_page_size(kv_cache_groups) - total_num_bytes_per_block = sum(ps * len(slots) for ps, slots in buckets.items()) + block_stride, layers_by_offset = _get_packed_kv_cache_layout(kv_cache_groups) - num_blocks = available_memory // total_num_bytes_per_block + num_blocks = available_memory // block_stride num_blocks = may_override_num_blocks(vllm_config, num_blocks) - total_size = total_num_bytes_per_block * num_blocks + total_size = block_stride * num_blocks kv_cache_tensors: list[KVCacheTensor] = [] - byte_offset = 0 - for ps, slots in buckets.items(): - for slot in slots: - kv_cache_tensors.append( - KVCacheTensor( - size=total_size, - shared_by=slot, - offset=byte_offset, - block_stride=total_num_bytes_per_block, - ) + for byte_offset in sorted(layers_by_offset): + kv_cache_tensors.append( + KVCacheTensor( + size=total_size, + shared_by=layers_by_offset[byte_offset], + offset=byte_offset, + block_stride=block_stride, ) - byte_offset += ps + ) return num_blocks, kv_cache_tensors @@ -1651,29 +1643,15 @@ def _get_kv_cache_groups_uniform_groups( for spec in group.kv_cache_specs.values() ) - # Split each SWA UniformKV group into smaller groups to align their #(layer tuples) - # Possibly padding layer tuples for this. - # Additionally, we also pad KV blocks in each SWA layer, to align the page size - # with the corresponding layer in the full-MLA group. - all_page_sizes = full_mla_spec.get_page_sizes() + # Split each SWA UniformKV group into smaller groups to align their + # numbers of layer tuples. The packed block planner overlays groups, so + # their page sizes do not need to match. swa_mla_groups = [] for sm_spec in swa_mla_specs: - sm_page_sizes = sm_spec.get_page_sizes() layers_per_size: dict[int, list[str]] = defaultdict(list) - assert max(sm_page_sizes) <= max(all_page_sizes) - # Unify page size by padding layers' page_size to the nearest larger page_size. - # Compute candidate (nearest larger page_size) for each unique page size. - size_to_candidate: dict[int, int] = {} - for ps in sm_page_sizes: - size_to_candidate[ps] = min(x for x in all_page_sizes if x >= ps) - # Pad and collect layer names per page size. for layer_name, layer_spec in sm_spec.kv_cache_specs.items(): - current_size = layer_spec.page_size_bytes - candidate = size_to_candidate[current_size] - if current_size < candidate: - object.__setattr__(layer_spec, "page_size_padded", candidate) - layers_per_size[candidate].append(layer_name) + layers_per_size[layer_spec.page_size_bytes].append(layer_name) # NOTE(yifan): for now, inside a UniformKV group, each page_size should # have the same number of layers. This also means we don't need to pad layers # inside a partial-full layer tuple.