[KV Connector] Fix DecodeBench fp8 fill values and add a startup fill mode (#58472)

Signed-off-by: Zijing Liu <liuzijing2014@gmail.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Zijing Liu
2026-09-24 23:55:47 +00:00
committed by GitHub
co-authored by Yifan Qiao Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 71891bdb94
commit 7a9406690e
3 changed files with 549 additions and 38 deletions
@@ -28,6 +28,16 @@ Example:
- fill_mean (float): Mean value for random normal fill (default: 0.015)
- fill_std (float): Standard deviation for random fill (default: 0.0)
Set to 0 for constant values, >0 for random sampling
- startup_fill (bool): Fill the whole KV cache once at startup instead
of filling each new request's blocks in the step that schedules it
(default: False). Circular-buffer caches are still zeroed per
request, and attention caches whose new blocks the engine zeroes
before use (hybrid Mamba or mixed-precision KV caches) are still
filled per request. The fill is not redone after sleep mode
discards the KV cache.
The dummy KV cache is only meant for performance measurement; outputs are
not meaningful for accuracy evaluation.
"""
@@ -45,13 +55,21 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
SupportsHMA,
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_utils import (
dcp_world_size_for_kv_cache_spec,
resolve_dcp_kv_block_size,
)
from vllm.v1.kv_cache_interface import CircularBufferSpec, iter_layer_specs
from vllm.v1.kv_cache_interface import (
AttentionSpec,
CircularBufferSpec,
KVCacheSpec,
KVQuantMode,
UniformTypeKVCacheSpecs,
iter_layer_specs,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
@@ -64,6 +82,45 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _get_fp8_dtype(spec: KVCacheSpec, cache_dtype: str) -> torch.dtype | None:
"""The fp8 dtype of a KV cache that stores plain fp8 values as uint8.
Returns None for other layouts, including packed ones that embed scales
or other metadata in the uint8 page.
"""
if (
not isinstance(spec, AttentionSpec)
or spec.dtype != torch.uint8
or spec.kv_quant_mode != KVQuantMode.FP8_PER_TENSOR
or spec.state_content_bytes is not None
):
return None
cache_dtype = getattr(spec, "cache_dtype_str", None) or cache_dtype
if cache_dtype in ("fp8", "fp8_e4m3"):
return current_platform.fp8_dtype()
if cache_dtype == "fp8_e5m2":
return torch.float8_e5m2
return None
def _get_startup_fill_group_ids(kv_cache_config: "KVCacheConfig") -> set[int]:
"""KV cache groups that startup fill covers.
Circular buffers are zero-filled per request. Attention groups whose new
blocks the engine zeroes before use are filled per request, since the
zeroing would wipe the startup fill.
"""
group_ids = set()
for group_idx, group in enumerate(kv_cache_config.kv_cache_groups):
spec = group.kv_cache_spec
if any(isinstance(s, CircularBufferSpec) for s in iter_layer_specs(spec)):
continue
if kv_cache_config.needs_kv_cache_zeroing and isinstance(spec, AttentionSpec):
continue
group_ids.add(group_idx)
return group_ids
@dataclass
class DecodeBenchConnectorMetadata(KVConnectorMetadata):
"""Metadata for DecodeBenchConnector.
@@ -191,6 +248,14 @@ class DecodeBenchConnectorScheduler:
"""Scheduler-side implementation for DecodeBenchConnector."""
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
kv_transfer_config = vllm_config.kv_transfer_config
assert kv_transfer_config is not None
self.startup_fill = kv_transfer_config.get_from_extra_config(
"startup_fill", False
)
self._startup_fill_group_ids = (
_get_startup_fill_group_ids(kv_cache_config) if self.startup_fill else set()
)
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
self.kv_cache_groups = kv_cache_config.kv_cache_groups
self.group_block_sizes = tuple(
@@ -275,6 +340,9 @@ class DecodeBenchConnectorScheduler:
strict=True,
)
):
if group_idx in self._startup_fill_group_ids:
block_ids_per_group_list.append([])
continue
is_circular_buffer = all(
isinstance(spec, CircularBufferSpec)
for spec in iter_layer_specs(group.kv_cache_spec)
@@ -310,6 +378,9 @@ class DecodeBenchConnectorScheduler:
)
block_ids_per_group_list.append(block_ids)
block_ids_per_group = tuple(block_ids_per_group_list)
self._filled_requests.add(req_id)
if self.startup_fill and not any(block_ids_per_group):
return
# Store the blocks to fill for all group. _pending_fills doesn't need cleanup
# as it's cleared after build_connector_meta
@@ -317,7 +388,6 @@ class DecodeBenchConnectorScheduler:
block_ids_per_group,
num_external_tokens,
)
self._filled_requests.add(req_id)
block_counts = tuple(len(group) for group in block_ids_per_group)
logger.debug(
@@ -356,10 +426,27 @@ class DecodeBenchConnectorWorker:
assert kv_transfer_config is not None
self.fill_mean = kv_transfer_config.get_from_extra_config("fill_mean", 0.015)
self.fill_std = kv_transfer_config.get_from_extra_config("fill_std", 0.0)
self.startup_fill = kv_transfer_config.get_from_extra_config(
"startup_fill", False
)
self._startup_fill_group_ids = (
_get_startup_fill_group_ids(kv_cache_config) if self.startup_fill else set()
)
# Will be populated via register_kv_caches
self.kv_caches: dict[str, torch.Tensor] | None = None
# Layers whose uint8 KV cache stores plain fp8 values
self._fp8_dtypes: dict[str, torch.dtype] = {}
for group in kv_cache_config.kv_cache_groups:
for layer_name in group.layer_names:
spec = group.kv_cache_spec
if isinstance(spec, UniformTypeKVCacheSpecs):
spec = spec.kv_cache_specs[layer_name]
fp8_dtype = _get_fp8_dtype(spec, vllm_config.cache_config.cache_dtype)
if fp8_dtype is not None:
self._fp8_dtypes[layer_name] = fp8_dtype
# Mapping from KV cache group index to list of layer names in that group
self.group_to_layers = {
group_idx: list(group.layer_names)
@@ -373,6 +460,17 @@ class DecodeBenchConnectorWorker:
for spec in iter_layer_specs(group.kv_cache_spec)
)
}
num_zeroed_groups = len(
self.group_to_layers.keys()
- self._startup_fill_group_ids
- self._zero_fill_group_ids
)
if self.startup_fill and num_zeroed_groups:
logger.info(
"DecodeBenchConnector: the engine zeroes new blocks of %d KV "
"cache groups before use, so they are still filled per request.",
num_zeroed_groups,
)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Store references to the KV cache tensors."""
@@ -383,6 +481,48 @@ class DecodeBenchConnectorWorker:
len(kv_caches),
)
if self.startup_fill:
self._fill_all_kv_caches()
def _fill_all_kv_caches(self):
"""Fill the KV cache tensors of the startup-filled groups once, in
place, with dummy values, so that no per-request fill is needed.
"""
assert self.kv_caches is not None
num_layers = 0
for group_idx in sorted(self._startup_fill_group_ids):
for layer_name in self.group_to_layers[group_idx]:
kv_cache = self.kv_caches.get(layer_name)
if isinstance(kv_cache, torch.Tensor):
tensors = [kv_cache]
elif isinstance(kv_cache, (list, tuple)) and all(
isinstance(t, torch.Tensor) for t in kv_cache
):
tensors = list(kv_cache)
else:
logger.warning_once(
"DecodeBenchConnector: skipping startup fill for layer "
"%s whose KV cache is %s, not a tensor or a list/tuple "
"of tensors.",
layer_name,
type(kv_cache).__name__,
)
continue
for tensor in tensors:
fill_dtype = self._fp8_dtypes.get(layer_name, tensor.dtype)
self._fill_tensor(tensor, self.fill_mean, self.fill_std, fill_dtype)
num_layers += 1
logger.info(
"DecodeBenchConnector: Filled %d KV cache layers at startup with %s "
"values (mean=%.3f, std=%.3f)",
num_layers,
"random" if self.fill_std > 0 else "constant",
self.fill_mean,
self.fill_std,
)
def start_fill_kv(self, metadata: DecodeBenchConnectorMetadata):
"""Fill the allocated KV cache blocks with dummy values.
@@ -455,12 +595,17 @@ class DecodeBenchConnectorWorker:
# dimension — so fill each tensor in its entirety with the same
# dummy values.
if isinstance(kv_cache, torch.Tensor):
self._fill_block_tensor(kv_cache, block_ids, fill_mean, fill_std)
fill_dtype = self._fp8_dtypes.get(layer_name, kv_cache.dtype)
self._fill_block_tensor(
kv_cache, block_ids, fill_mean, fill_std, fill_dtype
)
elif isinstance(kv_cache, (list, tuple)) and all(
isinstance(t, torch.Tensor) for t in kv_cache
):
for state_tensor in kv_cache:
self._fill_state_tensor(state_tensor, fill_mean, fill_std)
self._fill_tensor(
state_tensor, fill_mean, fill_std, state_tensor.dtype
)
else:
logger.warning_once(
"DecodeBenchConnector: skipping fill for layer %s whose KV "
@@ -486,6 +631,7 @@ class DecodeBenchConnectorWorker:
block_ids: list[int],
fill_mean: float,
fill_std: float,
fill_dtype: torch.dtype,
):
"""Fill the requested block rows of a block-indexed KV cache tensor.
@@ -495,6 +641,8 @@ class DecodeBenchConnectorWorker:
tensor's first dim are ignored.
fill_mean: Mean value for the fill.
fill_std: Standard deviation for the fill.
fill_dtype: Dtype the values are encoded as, e.g. the fp8 dtype
of a uint8 fp8 cache.
"""
# Convert block_ids to tensor on device
@@ -509,46 +657,85 @@ class DecodeBenchConnectorWorker:
if len(valid_block_ids) == 0:
return
# Create fill values - either constant or random
block_shape = kv_cache.shape[1:]
if fill_std > 0:
# Random normal sampling
fill_values = torch.normal(
mean=fill_mean,
std=fill_std,
size=(len(valid_block_ids),) + block_shape,
dtype=kv_cache.dtype,
device=kv_cache.device,
)
else:
# Constant fill value
fill_values = torch.full(
(len(valid_block_ids),) + block_shape,
fill_mean,
dtype=kv_cache.dtype,
device=kv_cache.device,
)
fill_values = self._make_fill_values(
(len(valid_block_ids),) + kv_cache.shape[1:],
kv_cache.device,
fill_dtype,
fill_mean,
fill_std,
)
# Batch fill operation
kv_cache[valid_block_ids] = fill_values
kv_cache[valid_block_ids] = fill_values.view(kv_cache.dtype)
def _fill_state_tensor(
self, kv_cache: torch.Tensor, fill_mean: float, fill_std: float
def _make_fill_values(
self,
size: tuple[int, ...],
device: torch.device,
dtype: torch.dtype,
fill_mean: float,
fill_std: float,
) -> torch.Tensor:
"""Create constant or random fill values of ``dtype``, clamped to its
finite range. Non-floating dtypes (packed layouts) are filled with
zeros.
"""
if not dtype.is_floating_point:
if fill_mean or fill_std:
logger.warning_once(
"DecodeBenchConnector: %s KV caches do not hold plain "
"floating-point values; filling them with zeros.",
dtype,
)
return torch.zeros(size, dtype=dtype, device=device)
finfo = torch.finfo(dtype)
if fill_std > 0:
sample_dtype = torch.float32
if dtype in (torch.float16, torch.bfloat16, torch.float32, torch.float64):
sample_dtype = dtype
values = torch.normal(
fill_mean, fill_std, size=size, device=device, dtype=sample_dtype
)
return values.clamp_(finfo.min, finfo.max).to(dtype)
fill_mean = min(max(fill_mean, finfo.min), finfo.max)
return torch.full(size, fill_mean, dtype=dtype, device=device)
def _fill_tensor(
self,
kv_cache: torch.Tensor,
fill_mean: float,
fill_std: float,
fill_dtype: torch.dtype,
):
"""Fill an entire non-block-indexed state tensor with dummy values.
"""Fill an entire tensor in place with dummy values.
Hybrid / linear-attention layers (e.g. Mamba, Kimi Delta Attention)
store their per-layer state as tensors with no num_blocks dimension,
so the whole tensor is filled with the same constant or random values
used for block fills, rather than selected block rows.
Used for startup fills, and for hybrid / linear-attention layers (e.g.
Mamba, Kimi Delta Attention) whose per-layer state tensors are filled
in their entirety with the same constant or random values used for
block fills, rather than selected block rows.
Args:
kv_cache: A state tensor to fill in its entirety.
kv_cache: A tensor to fill in its entirety.
fill_mean: Mean value for the fill.
fill_std: Standard deviation for the fill.
fill_dtype: Dtype the values are encoded as, e.g. the fp8 dtype
of a uint8 fp8 cache.
"""
if fill_std > 0:
kv_cache.normal_(mean=fill_mean, std=fill_std)
else:
kv_cache.fill_(fill_mean)
view = kv_cache.view(fill_dtype)
if fill_std == 0 or not fill_dtype.is_floating_point:
view.fill_(
self._make_fill_values(
(), torch.device("cpu"), fill_dtype, fill_mean, fill_std
)
)
return
# Sample in chunks to bound the temporary memory.
rows = max(1, (1 << 24) // max(1, view[0].numel()))
for chunk in view.split(rows):
chunk.copy_(
self._make_fill_values(
chunk.shape, chunk.device, fill_dtype, fill_mean, fill_std
)
)