[Spec Decode] Support mixed KV page sizes for DFlash (#45181)

Signed-off-by: Alex Steiner <asteiner@nvidia.com>
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Alex Steiner
2026-06-21 22:45:14 +08:00
committed by GitHub
co-authored by Claude Opus 4.8 Giancarlo Delfin Yifan Qiao
parent 3e6e33526d
commit 2cac89f9da
8 changed files with 511 additions and 141 deletions
+102 -7
View File
@@ -117,6 +117,7 @@ def new_kv_cache_spec(
page_size_padded=None,
sliding_window=None,
attention_chunk_size=None,
indexes_kv_by_block_stride=False,
):
return FullAttentionSpec(
block_size=block_size,
@@ -126,6 +127,7 @@ def new_kv_cache_spec(
page_size_padded=page_size_padded,
sliding_window=sliding_window,
attention_chunk_size=attention_chunk_size,
indexes_kv_by_block_stride=indexes_kv_by_block_stride,
)
@@ -136,6 +138,7 @@ def new_sliding_window_spec(
dtype=torch.float32,
page_size_padded=None,
sliding_window=1,
indexes_kv_by_block_stride=False,
):
return SlidingWindowSpec(
block_size=block_size,
@@ -144,6 +147,7 @@ def new_sliding_window_spec(
dtype=dtype,
page_size_padded=page_size_padded,
sliding_window=sliding_window,
indexes_kv_by_block_stride=indexes_kv_by_block_stride,
)
@@ -1799,16 +1803,38 @@ def test_get_kv_cache_config_one_worker():
],
)
# different hidden size that cannot be aligned by using different block size
# different hidden size that cannot be aligned by using different block size,
# but can be aligned by padding the smaller physical page.
swa_spec = new_sliding_window_spec(head_size=96, indexes_kv_by_block_stride=True)
kv_cache_specs_hybrid = {
"layer_1": new_kv_cache_spec(head_size=64),
"layer_2": new_sliding_window_spec(head_size=96),
"layer_1": new_kv_cache_spec(head_size=64, indexes_kv_by_block_stride=True),
"layer_2": swa_spec,
}
with pytest.raises(NotImplementedError):
get_kv_cache_configs(
vllm_config, [kv_cache_specs_hybrid], [mem_per_block_per_layer * 2 * 32]
)[0]
kv_cache_config_hybrid = get_kv_cache_configs(
vllm_config, [kv_cache_specs_hybrid], [mem_per_block_per_layer * 2 * 32]
)[0]
padded_page_size = swa_spec.page_size_bytes
assert kv_cache_config_hybrid == KVCacheConfig(
num_blocks=42,
kv_cache_tensors=[
KVCacheTensor(size=padded_page_size * 42, shared_by=["layer_1", "layer_2"]),
],
kv_cache_groups=[
KVCacheGroupSpec(
["layer_1"],
new_kv_cache_spec(
head_size=64,
page_size_padded=padded_page_size,
indexes_kv_by_block_stride=True,
),
),
KVCacheGroupSpec(
["layer_2"],
new_sliding_window_spec(head_size=96, indexes_kv_by_block_stride=True),
),
],
)
# Test num_gpu_blocks_override
vllm_config.cache_config.num_gpu_blocks_override = 16
@@ -2322,6 +2348,75 @@ def test_check_enough_kv_cache_memory_respects_num_gpu_blocks_override():
get_kv_cache_configs(vllm_config, [kv_cache_specs], [large_available_memory])
def test_unify_kv_cache_page_size_uses_padding_for_non_divisible_sizes():
"""DFlash drafters can have a smaller head size than the target model.
For example, MiMo uses 192-dim target KV heads while its DFlash draft uses
128-dim KV heads. The resulting page sizes are 3:2 rather than an integer
block-size multiple, so the smaller page must be padded instead.
"""
# Both layers' backends opt into the padded-page strided view (e.g.
# FlashAttention / its DiffKV subclass), so padding is allowed.
target_spec = new_kv_cache_spec(
block_size=16,
num_kv_heads=1,
head_size=192,
dtype=torch.bfloat16,
indexes_kv_by_block_stride=True,
)
draft_spec = new_sliding_window_spec(
block_size=16,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
sliding_window=1024,
indexes_kv_by_block_stride=True,
)
unified_specs = kv_cache_utils.unify_kv_cache_spec_page_size(
{
"target_attn": target_spec,
"draft_attn": draft_spec,
}
)
assert unified_specs["target_attn"] == target_spec
unified_draft_spec = unified_specs["draft_attn"]
assert unified_draft_spec.block_size == draft_spec.block_size
assert unified_draft_spec.real_page_size_bytes == draft_spec.real_page_size_bytes
assert unified_draft_spec.page_size_padded == target_spec.page_size_bytes
assert unified_draft_spec.page_size_bytes == target_spec.page_size_bytes
def test_unify_kv_cache_page_size_padding_requires_backend_support():
"""Padding is gated on the backend declaring ``indexes_kv_by_block_stride``.
A backend that does not support the strided padded-page view must raise
rather than silently padding (and misreading KV at runtime).
"""
target_spec = new_kv_cache_spec(
block_size=16,
num_kv_heads=1,
head_size=192,
dtype=torch.bfloat16,
indexes_kv_by_block_stride=True,
)
# The non-divisible draft layer needs padding but its backend does not
# support the strided padded-page view -> must raise, not silently pad.
draft_spec = new_sliding_window_spec(
block_size=16,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
sliding_window=1024,
indexes_kv_by_block_stride=False,
)
specs = {"target_attn": target_spec, "draft_attn": draft_spec}
with pytest.raises(NotImplementedError):
kv_cache_utils.unify_kv_cache_spec_page_size(specs)
def test_unify_hybrid_kv_cache_specs():
# 1. has_full_attention and has_sliding_window
before_spec_1 = new_kv_cache_spec()
+242
View File
@@ -0,0 +1,242 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVQuantMode
from vllm.v1.worker.gpu.attn_utils import _reshape_kv_cache
from vllm.v1.worker.utils import AttentionGroup
class FakeFlashAttentionBackend:
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
return (num_blocks, 2, block_size, num_kv_heads, head_size)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
assert not include_num_layers_dimension
return (0, 1, 2, 3, 4)
class FakeHNDFlashAttentionBackend(FakeFlashAttentionBackend):
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
assert not include_num_layers_dimension
return (0, 1, 3, 2, 4)
def test_reshape_padded_flash_attention_kv_cache_strides_by_page():
num_blocks = 3
spec = FullAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=2,
dtype=torch.float32,
page_size_padded=384,
)
assert spec.real_page_size_bytes == 256
raw_tensors = {
"layer": torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
}
attn_groups = [
AttentionGroup(
backend=FakeFlashAttentionBackend,
layer_names=["layer"],
kv_cache_spec=spec,
kv_cache_group_id=0,
)
]
kv_cache = _reshape_kv_cache(
attn_groups,
raw_tensors,
"auto",
[spec.block_size],
{},
)["layer"]
assert kv_cache.shape == (num_blocks, 2, 16, 1, 2)
assert kv_cache.stride(0) == spec.page_size_bytes // 4
assert kv_cache.stride(1) == spec.real_page_size_bytes // 2 // 4
assert kv_cache[1, 0].storage_offset() == spec.page_size_bytes // 4
assert (
kv_cache[1, 1].storage_offset()
== (spec.page_size_bytes + spec.real_page_size_bytes // 2) // 4
)
def test_reshape_padded_hnd_flash_attention_kv_cache_strides_by_page():
num_blocks = 3
spec = FullAttentionSpec(
block_size=16,
num_kv_heads=3,
head_size=2,
dtype=torch.float32,
page_size_padded=1024,
)
assert spec.real_page_size_bytes == 768
raw_tensors = {
"layer": torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
}
attn_groups = [
AttentionGroup(
backend=FakeHNDFlashAttentionBackend,
layer_names=["layer"],
kv_cache_spec=spec,
kv_cache_group_id=0,
)
]
kv_cache = _reshape_kv_cache(
attn_groups,
raw_tensors,
"auto",
[spec.block_size],
{},
)["layer"]
assert kv_cache.shape == (num_blocks, 2, 16, 3, 2)
assert kv_cache.stride(0) == spec.page_size_bytes // 4
assert kv_cache.stride(1) == spec.real_page_size_bytes // 2 // 4
assert kv_cache.stride(2) == 2
assert kv_cache.stride(3) == spec.block_size * spec.head_size
assert kv_cache[1, 0].storage_offset() == spec.page_size_bytes // 4
assert (
kv_cache[1, 1].storage_offset()
== (spec.page_size_bytes + spec.real_page_size_bytes // 2) // 4
)
assert (
kv_cache[1, 1, 3, 2].storage_offset()
== (
spec.page_size_bytes
+ spec.real_page_size_bytes // 2
+ 3 * spec.head_size * 4
+ 2 * spec.block_size * spec.head_size * 4
)
// 4
)
class FakeDiffKVBackend:
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
return (num_blocks, block_size, num_kv_heads, head_size * 2)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
assert not include_num_layers_dimension
return (0, 1, 2, 3)
def test_reshape_padded_diff_kv_cache_does_not_infer_kv_dim():
num_blocks = 3
spec = FullAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=2,
dtype=torch.float32,
page_size_padded=384,
)
raw_tensors = {
"layer": torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
}
attn_groups = [
AttentionGroup(
backend=FakeDiffKVBackend,
layer_names=["layer"],
kv_cache_spec=spec,
kv_cache_group_id=0,
)
]
kv_cache = _reshape_kv_cache(
attn_groups,
raw_tensors,
"auto",
[spec.block_size],
{},
)["layer"]
assert kv_cache.shape == (num_blocks, 16, 1, 4)
assert kv_cache.stride(0) == spec.page_size_bytes // 4
assert kv_cache.stride(1) == 4
class FakePerTokenScaleBackend:
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
cache_dtype_str: str = "auto",
) -> tuple[int, ...]:
return (num_blocks, 2, block_size, num_kv_heads, head_size + 4)
@staticmethod
def get_kv_cache_stride_order(
include_num_layers_dimension: bool = False,
) -> tuple[int, ...]:
assert not include_num_layers_dimension
return (0, 1, 2, 3, 4)
def test_reshape_padded_quantized_kv_cache_preserves_scale_stride():
num_blocks = 3
spec = FullAttentionSpec(
block_size=16,
num_kv_heads=1,
head_size=4,
dtype=torch.int8,
kv_quant_mode=KVQuantMode.INT8_PER_TOKEN_HEAD,
page_size_padded=384,
)
assert spec.real_page_size_bytes == 128
assert spec.page_size_bytes == 384
raw_tensors = {
"layer": torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
}
attn_groups = [
AttentionGroup(
backend=FakePerTokenScaleBackend,
layer_names=["layer"],
kv_cache_spec=spec,
kv_cache_group_id=0,
)
]
kv_cache = _reshape_kv_cache(
attn_groups,
raw_tensors,
"int8_per_token_head",
[spec.block_size],
{},
)["layer"]
assert kv_cache.shape == (num_blocks, 2, 16, 1, 8)
assert kv_cache.stride(0) == spec.page_size_bytes
assert kv_cache.stride(1) == 16 * 1 * 8
assert kv_cache[1, 1].storage_offset() == spec.page_size_bytes + 16 * 1 * 8