[Spec Decode] Support mixed KV page sizes for DFlash (#45181)

Signed-off-by: Alex Steiner <asteiner@nvidia.com>
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Alex Steiner
2026-06-21 22:45:14 +08:00
committed by GitHub
co-authored by Claude Opus 4.8 Giancarlo Delfin Yifan Qiao
parent 3e6e33526d
commit 2cac89f9da
8 changed files with 511 additions and 141 deletions
+13 -3
View File
@@ -163,6 +163,7 @@ class AttentionSpec(KVCacheSpec):
dtype: torch.dtype
kv_quant_mode: KVQuantMode = KVQuantMode.NONE
page_size_padded: int | None = None
indexes_kv_by_block_stride: bool = False
@property
def page_size_bytes(self) -> int:
@@ -283,6 +284,7 @@ class FullAttentionSpec(AttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride,
sliding_window=cls.merge_window_sizes(sliding_window),
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),
# If any layer in the group is non-causal, treat the group as
@@ -403,13 +405,16 @@ class MLAAttentionSpec(FullAttentionSpec):
cache_dtype_str_set = set(spec.cache_dtype_str for spec in specs)
compress_ratio_set = set(spec.compress_ratio for spec in specs)
model_version_set = set(spec.model_version for spec in specs)
block_stride_set = set(spec.indexes_kv_by_block_stride for spec in specs)
assert (
len(cache_dtype_str_set) == 1
and len(compress_ratio_set) == 1
and len(model_version_set) == 1
and len(block_stride_set) == 1
), (
"All attention layers in the same KV cache group must use the same "
"quantization method, compress ratio, and model version."
"quantization method, compress ratio, model version, and KV block "
"stride indexing."
)
return cls(
block_size=specs[0].block_size,
@@ -418,6 +423,7 @@ class MLAAttentionSpec(FullAttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
indexes_kv_by_block_stride=block_stride_set.pop(),
cache_dtype_str=cache_dtype_str_set.pop(),
compress_ratio=compress_ratio_set.pop(),
model_version=model_version_set.pop(),
@@ -584,15 +590,17 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
compress_ratio_set = set(spec.compress_ratio for spec in specs)
model_version_set = set(spec.model_version for spec in specs)
sliding_window_set = set(spec.sliding_window for spec in specs)
block_stride_set = set(spec.indexes_kv_by_block_stride for spec in specs)
assert (
len(cache_dtype_str_set) == 1
and len(compress_ratio_set) == 1
and len(model_version_set) == 1
and len(sliding_window_set) == 1
and len(block_stride_set) == 1
), (
"All attention layers in the same KV cache group must use the same "
"quantization method, compress ratio, model version and sliding "
"window size."
"quantization method, compress ratio, model version, sliding "
"window size, and KV block stride indexing."
)
return cls(
block_size=specs[0].block_size,
@@ -600,6 +608,7 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
head_size=specs[0].head_size,
dtype=specs[0].dtype,
page_size_padded=specs[0].page_size_padded,
indexes_kv_by_block_stride=block_stride_set.pop(),
sliding_window=sliding_window_set.pop(),
cache_dtype_str=cache_dtype_str_set.pop(),
compress_ratio=compress_ratio_set.pop(),
@@ -711,6 +720,7 @@ class SinkFullAttentionSpec(FullAttentionSpec):
dtype=specs[0].dtype,
kv_quant_mode=specs[0].kv_quant_mode,
page_size_padded=specs[0].page_size_padded,
indexes_kv_by_block_stride=specs[0].indexes_kv_by_block_stride,
sliding_window=cls.merge_window_sizes(sliding_window),
attention_chunk_size=cls.merge_window_sizes(attention_chunk_size),
non_causal=any(spec.non_causal for spec in specs),