[Spec Decode] Support mixed KV page sizes for DFlash (#45181)
Signed-off-by: Alex Steiner <asteiner@nvidia.com> Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Claude Opus 4.8
Giancarlo Delfin
Yifan Qiao
parent
3e6e33526d
commit
2cac89f9da
@@ -117,6 +117,7 @@ def new_kv_cache_spec(
|
||||
page_size_padded=None,
|
||||
sliding_window=None,
|
||||
attention_chunk_size=None,
|
||||
indexes_kv_by_block_stride=False,
|
||||
):
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
@@ -126,6 +127,7 @@ def new_kv_cache_spec(
|
||||
page_size_padded=page_size_padded,
|
||||
sliding_window=sliding_window,
|
||||
attention_chunk_size=attention_chunk_size,
|
||||
indexes_kv_by_block_stride=indexes_kv_by_block_stride,
|
||||
)
|
||||
|
||||
|
||||
@@ -136,6 +138,7 @@ def new_sliding_window_spec(
|
||||
dtype=torch.float32,
|
||||
page_size_padded=None,
|
||||
sliding_window=1,
|
||||
indexes_kv_by_block_stride=False,
|
||||
):
|
||||
return SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
@@ -144,6 +147,7 @@ def new_sliding_window_spec(
|
||||
dtype=dtype,
|
||||
page_size_padded=page_size_padded,
|
||||
sliding_window=sliding_window,
|
||||
indexes_kv_by_block_stride=indexes_kv_by_block_stride,
|
||||
)
|
||||
|
||||
|
||||
@@ -1799,16 +1803,38 @@ def test_get_kv_cache_config_one_worker():
|
||||
],
|
||||
)
|
||||
|
||||
# different hidden size that cannot be aligned by using different block size
|
||||
# different hidden size that cannot be aligned by using different block size,
|
||||
# but can be aligned by padding the smaller physical page.
|
||||
swa_spec = new_sliding_window_spec(head_size=96, indexes_kv_by_block_stride=True)
|
||||
kv_cache_specs_hybrid = {
|
||||
"layer_1": new_kv_cache_spec(head_size=64),
|
||||
"layer_2": new_sliding_window_spec(head_size=96),
|
||||
"layer_1": new_kv_cache_spec(head_size=64, indexes_kv_by_block_stride=True),
|
||||
"layer_2": swa_spec,
|
||||
}
|
||||
|
||||
with pytest.raises(NotImplementedError):
|
||||
get_kv_cache_configs(
|
||||
vllm_config, [kv_cache_specs_hybrid], [mem_per_block_per_layer * 2 * 32]
|
||||
)[0]
|
||||
kv_cache_config_hybrid = get_kv_cache_configs(
|
||||
vllm_config, [kv_cache_specs_hybrid], [mem_per_block_per_layer * 2 * 32]
|
||||
)[0]
|
||||
padded_page_size = swa_spec.page_size_bytes
|
||||
assert kv_cache_config_hybrid == KVCacheConfig(
|
||||
num_blocks=42,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(size=padded_page_size * 42, shared_by=["layer_1", "layer_2"]),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["layer_1"],
|
||||
new_kv_cache_spec(
|
||||
head_size=64,
|
||||
page_size_padded=padded_page_size,
|
||||
indexes_kv_by_block_stride=True,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["layer_2"],
|
||||
new_sliding_window_spec(head_size=96, indexes_kv_by_block_stride=True),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
# Test num_gpu_blocks_override
|
||||
vllm_config.cache_config.num_gpu_blocks_override = 16
|
||||
@@ -2322,6 +2348,75 @@ def test_check_enough_kv_cache_memory_respects_num_gpu_blocks_override():
|
||||
get_kv_cache_configs(vllm_config, [kv_cache_specs], [large_available_memory])
|
||||
|
||||
|
||||
def test_unify_kv_cache_page_size_uses_padding_for_non_divisible_sizes():
|
||||
"""DFlash drafters can have a smaller head size than the target model.
|
||||
|
||||
For example, MiMo uses 192-dim target KV heads while its DFlash draft uses
|
||||
128-dim KV heads. The resulting page sizes are 3:2 rather than an integer
|
||||
block-size multiple, so the smaller page must be padded instead.
|
||||
"""
|
||||
# Both layers' backends opt into the padded-page strided view (e.g.
|
||||
# FlashAttention / its DiffKV subclass), so padding is allowed.
|
||||
target_spec = new_kv_cache_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=192,
|
||||
dtype=torch.bfloat16,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
draft_spec = new_sliding_window_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=1024,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
|
||||
unified_specs = kv_cache_utils.unify_kv_cache_spec_page_size(
|
||||
{
|
||||
"target_attn": target_spec,
|
||||
"draft_attn": draft_spec,
|
||||
}
|
||||
)
|
||||
|
||||
assert unified_specs["target_attn"] == target_spec
|
||||
unified_draft_spec = unified_specs["draft_attn"]
|
||||
assert unified_draft_spec.block_size == draft_spec.block_size
|
||||
assert unified_draft_spec.real_page_size_bytes == draft_spec.real_page_size_bytes
|
||||
assert unified_draft_spec.page_size_padded == target_spec.page_size_bytes
|
||||
assert unified_draft_spec.page_size_bytes == target_spec.page_size_bytes
|
||||
|
||||
|
||||
def test_unify_kv_cache_page_size_padding_requires_backend_support():
|
||||
"""Padding is gated on the backend declaring ``indexes_kv_by_block_stride``.
|
||||
|
||||
A backend that does not support the strided padded-page view must raise
|
||||
rather than silently padding (and misreading KV at runtime).
|
||||
"""
|
||||
target_spec = new_kv_cache_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=192,
|
||||
dtype=torch.bfloat16,
|
||||
indexes_kv_by_block_stride=True,
|
||||
)
|
||||
# The non-divisible draft layer needs padding but its backend does not
|
||||
# support the strided padded-page view -> must raise, not silently pad.
|
||||
draft_spec = new_sliding_window_spec(
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=1024,
|
||||
indexes_kv_by_block_stride=False,
|
||||
)
|
||||
specs = {"target_attn": target_spec, "draft_attn": draft_spec}
|
||||
|
||||
with pytest.raises(NotImplementedError):
|
||||
kv_cache_utils.unify_kv_cache_spec_page_size(specs)
|
||||
|
||||
|
||||
def test_unify_hybrid_kv_cache_specs():
|
||||
# 1. has_full_attention and has_sliding_window
|
||||
before_spec_1 = new_kv_cache_spec()
|
||||
|
||||
Reference in New Issue
Block a user