[Spec Decode] Support mixed KV page sizes for DFlash (#45181)
Signed-off-by: Alex Steiner <asteiner@nvidia.com> Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Claude Opus 4.8
Giancarlo Delfin
Yifan Qiao
parent
3e6e33526d
commit
2cac89f9da
@@ -117,6 +117,7 @@ def new_kv_cache_spec(
|
||||
page_size_padded=None,
|
||||
sliding_window=None,
|
||||
attention_chunk_size=None,
|
||||
indexes_kv_by_block_stride=False,
|
||||
):
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
@@ -126,6 +127,7 @@ def new_kv_cache_spec(
|
||||
page_size_padded=page_size_padded,
|
||||
sliding_window=sliding_window,
|
||||
attention_chunk_size=attention_chunk_size,
|
||||
indexes_kv_by_block_stride=indexes_kv_by_block_stride,
|
||||
)
|
||||
|
||||
|
||||
@@ -136,6 +138,7 @@ def new_sliding_window_spec(
|
||||
dtype=torch.float32,
|
||||
page_size_padded=None,
|
||||
sliding_window=1,
|
||||
indexes_kv_by_block_stride=False,
|
||||
):
|
||||
return SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
@@ -144,6 +147,7 @@ def new_sliding_window_spec(
|
||||
dtype=dtype,
|
||||
page_size_padded=page_size_padded,
|
||||
sliding_window=sliding_window,
|
||||
indexes_kv_by_block_stride=indexes_kv_by_block_stride,
|
||||
)
|
||||
|
||||
|
||||
@@ -1799,16 +1803,38 @@ def test_get_kv_cache_config_one_worker():
|
||||
],
|
||||
)
|
||||
|
||||
# different hidden size that cannot be aligned by using different block size
|
||||
# different hidden size that cannot be aligned by using different block size,
|
||||
# but can be aligned by padding the smaller physical page.
|
||||
swa_spec = new_sliding_window_spec(head_size=96, indexes_kv_by_block_stride=True)
|
||||
kv_cache_specs_hybrid = {
|
||||
"layer_1": new_kv_cache_spec(head_size=64),
|
||||
"layer_2": new_sliding_window_spec(head_size=96),
|
||||
"layer_1": new_kv_cache_spec(head_size=64, indexes_kv_by_block_stride=True),
|
||||
"layer_2": swa_spec,
|
||||
}
|
||||
|
||||
with pytest.raises(NotImplementedError):
|
||||
get_kv_cache_configs(
|
||||
vllm_config, [kv_cache_specs_hybrid], [mem_per_block_per_layer * 2 * 32]
|
||||
)[0]
|
||||
kv_cache_config_hybrid = get_kv_cache_configs(
|
||||
vllm_config, [kv_cache_specs_hybrid], [mem_per_block_per_layer * 2 * 32]
|
||||
)[0]
|
||||
padded_page_size = swa_spec.page_size_bytes
|
||||
assert kv_cache_config_hybrid == KVCacheConfig(
|
||||
num_blocks=42,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=padded_page_size * 42, shared_by=["layer_1", "layer_2"]),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer_1"],
|
||||
new_kv_cache_spec(
|
||||
head_size=64,
|
||||
page_size_padded=padded_page_size,
|
||||
indexes_kv_by_block_stride=True,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer_2"],
|
||||
new_sliding_window_spec(head_size=96, indexes_kv_by_block_stride=True),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# Test num_gpu_blocks_override
|
||||
vllm_config.cache_config.num_gpu_blocks_override = 16
|
||||
@@ -2322,6 +2348,75 @@ def test_check_enough_kv_cache_memory_respects_num_gpu_blocks_override():
|
||||
get_kv_cache_configs(vllm_config, [kv_cache_specs], [large_available_memory])
|
||||
|
||||
|
||||
def test_unify_kv_cache_page_size_uses_padding_for_non_divisible_sizes():
|
||||
"""DFlash drafters can have a smaller head size than the target model.
|
||||
|
||||
For example, MiMo uses 192-dim target KV heads while its DFlash draft uses
|
||||
128-dim KV heads. The resulting page sizes are 3:2 rather than an integer
|
||||
block-size multiple, so the smaller page must be padded instead.
|
||||
"""
|
||||
# Both layers' backends opt into the padded-page strided view (e.g.
|
||||
# FlashAttention / its DiffKV subclass), so padding is allowed.
|
||||
target_spec = new_kv_cache_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=192,
|
||||
dtype=torch.bfloat16,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
draft_spec = new_sliding_window_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=1024,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
|
||||
unified_specs = kv_cache_utils.unify_kv_cache_spec_page_size(
|
||||
{
|
||||
"target_attn": target_spec,
|
||||
"draft_attn": draft_spec,
|
||||
}
|
||||
)
|
||||
|
||||
assert unified_specs["target_attn"] == target_spec
|
||||
unified_draft_spec = unified_specs["draft_attn"]
|
||||
assert unified_draft_spec.block_size == draft_spec.block_size
|
||||
assert unified_draft_spec.real_page_size_bytes == draft_spec.real_page_size_bytes
|
||||
assert unified_draft_spec.page_size_padded == target_spec.page_size_bytes
|
||||
assert unified_draft_spec.page_size_bytes == target_spec.page_size_bytes
|
||||
|
||||
|
||||
def test_unify_kv_cache_page_size_padding_requires_backend_support():
|
||||
"""Padding is gated on the backend declaring ``indexes_kv_by_block_stride``.
|
||||
|
||||
A backend that does not support the strided padded-page view must raise
|
||||
rather than silently padding (and misreading KV at runtime).
|
||||
"""
|
||||
target_spec = new_kv_cache_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=192,
|
||||
dtype=torch.bfloat16,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
# The non-divisible draft layer needs padding but its backend does not
|
||||
# support the strided padded-page view -> must raise, not silently pad.
|
||||
draft_spec = new_sliding_window_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=1024,
|
||||
indexes_kv_by_block_stride=False,
|
||||
)
|
||||
specs = {"target_attn": target_spec, "draft_attn": draft_spec}
|
||||
|
||||
with pytest.raises(NotImplementedError):
|
||||
kv_cache_utils.unify_kv_cache_spec_page_size(specs)
|
||||
|
||||
|
||||
def test_unify_hybrid_kv_cache_specs():
|
||||
# 1. has_full_attention and has_sliding_window
|
||||
before_spec_1 = new_kv_cache_spec()
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVQuantMode
|
||||
from vllm.v1.worker.gpu.attn_utils import _reshape_kv_cache
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
class FakeFlashAttentionBackend:
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
assert not include_num_layers_dimension
|
||||
return (0, 1, 2, 3, 4)
|
||||
|
||||
|
||||
class FakeHNDFlashAttentionBackend(FakeFlashAttentionBackend):
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
assert not include_num_layers_dimension
|
||||
return (0, 1, 3, 2, 4)
|
||||
|
||||
|
||||
def test_reshape_padded_flash_attention_kv_cache_strides_by_page():
|
||||
num_blocks = 3
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
page_size_padded=384,
|
||||
)
|
||||
assert spec.real_page_size_bytes == 256
|
||||
|
||||
raw_tensors = {
|
||||
"layer": torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
|
||||
}
|
||||
attn_groups = [
|
||||
AttentionGroup(
|
||||
backend=FakeFlashAttentionBackend,
|
||||
layer_names=["layer"],
|
||||
kv_cache_spec=spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
|
||||
kv_cache = _reshape_kv_cache(
|
||||
attn_groups,
|
||||
raw_tensors,
|
||||
"auto",
|
||||
[spec.block_size],
|
||||
{},
|
||||
)["layer"]
|
||||
|
||||
assert kv_cache.shape == (num_blocks, 2, 16, 1, 2)
|
||||
assert kv_cache.stride(0) == spec.page_size_bytes // 4
|
||||
assert kv_cache.stride(1) == spec.real_page_size_bytes // 2 // 4
|
||||
assert kv_cache[1, 0].storage_offset() == spec.page_size_bytes // 4
|
||||
assert (
|
||||
kv_cache[1, 1].storage_offset()
|
||||
== (spec.page_size_bytes + spec.real_page_size_bytes // 2) // 4
|
||||
)
|
||||
|
||||
|
||||
def test_reshape_padded_hnd_flash_attention_kv_cache_strides_by_page():
|
||||
num_blocks = 3
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=3,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
page_size_padded=1024,
|
||||
)
|
||||
assert spec.real_page_size_bytes == 768
|
||||
|
||||
raw_tensors = {
|
||||
"layer": torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
|
||||
}
|
||||
attn_groups = [
|
||||
AttentionGroup(
|
||||
backend=FakeHNDFlashAttentionBackend,
|
||||
layer_names=["layer"],
|
||||
kv_cache_spec=spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
|
||||
kv_cache = _reshape_kv_cache(
|
||||
attn_groups,
|
||||
raw_tensors,
|
||||
"auto",
|
||||
[spec.block_size],
|
||||
{},
|
||||
)["layer"]
|
||||
|
||||
assert kv_cache.shape == (num_blocks, 2, 16, 3, 2)
|
||||
assert kv_cache.stride(0) == spec.page_size_bytes // 4
|
||||
assert kv_cache.stride(1) == spec.real_page_size_bytes // 2 // 4
|
||||
assert kv_cache.stride(2) == 2
|
||||
assert kv_cache.stride(3) == spec.block_size * spec.head_size
|
||||
assert kv_cache[1, 0].storage_offset() == spec.page_size_bytes // 4
|
||||
assert (
|
||||
kv_cache[1, 1].storage_offset()
|
||||
== (spec.page_size_bytes + spec.real_page_size_bytes // 2) // 4
|
||||
)
|
||||
assert (
|
||||
kv_cache[1, 1, 3, 2].storage_offset()
|
||||
== (
|
||||
spec.page_size_bytes
|
||||
+ spec.real_page_size_bytes // 2
|
||||
+ 3 * spec.head_size * 4
|
||||
+ 2 * spec.block_size * spec.head_size * 4
|
||||
)
|
||||
// 4
|
||||
)
|
||||
|
||||
|
||||
class FakeDiffKVBackend:
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, block_size, num_kv_heads, head_size * 2)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
assert not include_num_layers_dimension
|
||||
return (0, 1, 2, 3)
|
||||
|
||||
|
||||
def test_reshape_padded_diff_kv_cache_does_not_infer_kv_dim():
|
||||
num_blocks = 3
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=2,
|
||||
dtype=torch.float32,
|
||||
page_size_padded=384,
|
||||
)
|
||||
|
||||
raw_tensors = {
|
||||
"layer": torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
|
||||
}
|
||||
attn_groups = [
|
||||
AttentionGroup(
|
||||
backend=FakeDiffKVBackend,
|
||||
layer_names=["layer"],
|
||||
kv_cache_spec=spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
|
||||
kv_cache = _reshape_kv_cache(
|
||||
attn_groups,
|
||||
raw_tensors,
|
||||
"auto",
|
||||
[spec.block_size],
|
||||
{},
|
||||
)["layer"]
|
||||
|
||||
assert kv_cache.shape == (num_blocks, 16, 1, 4)
|
||||
assert kv_cache.stride(0) == spec.page_size_bytes // 4
|
||||
assert kv_cache.stride(1) == 4
|
||||
|
||||
|
||||
class FakePerTokenScaleBackend:
|
||||
@staticmethod
|
||||
def get_kv_cache_shape(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype_str: str = "auto",
|
||||
) -> tuple[int, ...]:
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size + 4)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order(
|
||||
include_num_layers_dimension: bool = False,
|
||||
) -> tuple[int, ...]:
|
||||
assert not include_num_layers_dimension
|
||||
return (0, 1, 2, 3, 4)
|
||||
|
||||
|
||||
def test_reshape_padded_quantized_kv_cache_preserves_scale_stride():
|
||||
num_blocks = 3
|
||||
spec = FullAttentionSpec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=4,
|
||||
dtype=torch.int8,
|
||||
kv_quant_mode=KVQuantMode.INT8_PER_TOKEN_HEAD,
|
||||
page_size_padded=384,
|
||||
)
|
||||
assert spec.real_page_size_bytes == 128
|
||||
assert spec.page_size_bytes == 384
|
||||
|
||||
raw_tensors = {
|
||||
"layer": torch.zeros(spec.page_size_bytes * num_blocks, dtype=torch.int8)
|
||||
}
|
||||
attn_groups = [
|
||||
AttentionGroup(
|
||||
backend=FakePerTokenScaleBackend,
|
||||
layer_names=["layer"],
|
||||
kv_cache_spec=spec,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
]
|
||||
|
||||
kv_cache = _reshape_kv_cache(
|
||||
attn_groups,
|
||||
raw_tensors,
|
||||
"int8_per_token_head",
|
||||
[spec.block_size],
|
||||
{},
|
||||
)["layer"]
|
||||
|
||||
assert kv_cache.shape == (num_blocks, 2, 16, 1, 8)
|
||||
assert kv_cache.stride(0) == spec.page_size_bytes
|
||||
assert kv_cache.stride(1) == 16 * 1 * 8
|
||||
assert kv_cache[1, 1].storage_offset() == spec.page_size_bytes + 16 * 1 * 8
|
||||
Reference in New Issue
Block a user