[Spec Decode] Support mixed KV page sizes for DFlash (#45181)

Signed-off-by: Alex Steiner <asteiner@nvidia.com>
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Giancarlo Delfin <gdelfin@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Alex Steiner
2026-06-21 22:45:14 +08:00
committed by GitHub
co-authored by Claude Opus 4.8 Giancarlo Delfin Yifan Qiao
parent 3e6e33526d
commit 2cac89f9da
8 changed files with 511 additions and 141 deletions
+17 -52
View File
@@ -12,7 +12,6 @@ from contextlib import contextmanager
from copy import copy, deepcopy
from dataclasses import dataclass, replace
from functools import reduce
from math import prod
from typing import TYPE_CHECKING, Any, NamedTuple, TypeAlias, cast
import numpy as np
@@ -203,6 +202,7 @@ from vllm.v1.worker.cp_utils import (
)
from vllm.v1.worker.dp_utils import coordinate_batch_across_dp
from vllm.v1.worker.ec_connector_model_runner_mixin import ECConnectorModelRunnerMixin
from vllm.v1.worker.gpu.attn_utils import _reshape_attention_kv_cache
from vllm.v1.worker.gpu.pool.late_interaction_runner import LateInteractionRunner
from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
from vllm.v1.worker.gpu_ubatch_wrapper import UBatchWrapper
@@ -7125,62 +7125,20 @@ class GPUModelRunner(
kv_cache_spec.head_size,
cache_dtype_str=self.cache_config.cache_dtype,
)
dtype = kv_cache_spec.dtype
try:
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order()
assert len(kv_cache_stride_order) == len(kv_cache_shape)
except (AttributeError, NotImplementedError):
kv_cache_stride_order = tuple(range(len(kv_cache_shape)))
# The allocation respects the backend-defined stride order
# to ensure the semantic remains consistent for each
# backend. We first obtain the generic kv cache shape and
# then permute it according to the stride order which could
# result in a non-contiguous tensor.
kv_cache_shape = tuple(
kv_cache_shape[i] for i in kv_cache_stride_order
raw_tensor = kv_cache_raw_tensors[layer_name]
kv_caches[layer_name] = _reshape_attention_kv_cache(
raw_tensor,
kv_cache_spec,
kv_cache_shape,
kv_cache_stride_order,
kernel_num_blocks,
packing,
)
# Maintain original KV shape view.
inv_order = [
kv_cache_stride_order.index(i)
for i in range(len(kv_cache_stride_order))
]
if packing is not None:
offset, block_stride = packing
assert inv_order[0] == 0
page_bytes = prod(kv_cache_shape[1:]) * get_dtype_size(dtype)
kv_cache = (
kv_cache_raw_tensors[layer_name]
.view(-1, block_stride)[:, offset : offset + page_bytes]
.view(dtype)
.view(kv_cache_shape)
)
elif kv_cache_spec.page_size_padded is not None:
# Use strided view to handle page_size_bytes that
# include padding. This follows
# the same pattern as MambaSpec handling below.
# NOTE: This assumes kv_cache_shape[0] == num_blocks
# (i.e. the first physical dimension is the block
# index), which holds for MLA backends but NOT for
# standard attention backends whose shape starts with
# a K/V dimension of size 2.
dtype_size = get_dtype_size(dtype)
page_stride = kv_cache_spec.page_size_bytes // dtype_size
strides = list(torch.empty(kv_cache_shape).stride())
strides[inv_order[0]] = page_stride
kv_cache = torch.as_strided(
kv_cache_raw_tensors[layer_name].view(dtype),
size=kv_cache_shape,
stride=tuple(strides),
)
else:
# No padding — safe to use a contiguous view.
kv_cache = (
kv_cache_raw_tensors[layer_name]
.view(dtype)
.view(kv_cache_shape)
)
kv_caches[layer_name] = kv_cache.permute(*inv_order)
elif isinstance(kv_cache_spec, MambaSpec):
has_mamba = True
@@ -7265,7 +7223,7 @@ class GPUModelRunner(
# Try creating KV caches optimized for kv-connector transfers
cache_dtype = self.cache_config.cache_dtype
if self.use_uniform_kv_cache(self.attn_groups, cache_dtype):
if self.use_uniform_kv_cache(self.attn_groups):
kv_caches, cross_layers_kv_cache, attn_backend = (
self.allocate_uniform_kv_caches(
kv_cache_config,
@@ -7515,6 +7473,13 @@ class GPUModelRunner(
continue
# Skip modules that don't need KV cache (eg encoder-only attention)
if spec := attn_module.get_kv_cache_spec(self.vllm_config):
if isinstance(spec, AttentionSpec):
backend = attn_module.get_attn_backend()
# indexes_kv_by_block_stride() -> get_kv_cache_stride_order()
# -> get_kv_cache_layout() needs the current vLLM config.
with set_current_vllm_config(self.vllm_config):
indexes = backend.indexes_kv_by_block_stride()
spec = replace(spec, indexes_kv_by_block_stride=indexes)
kv_cache_spec[layer_name] = spec
return kv_cache_spec