[KV Connector] Fix DecodeBench fp8 fill values and add a startup fill mode (#58472)
Signed-off-by: Zijing Liu <liuzijing2014@gmail.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
71891bdb94
commit
7a9406690e
@@ -28,6 +28,16 @@ Example:
|
||||
- fill_mean (float): Mean value for random normal fill (default: 0.015)
|
||||
- fill_std (float): Standard deviation for random fill (default: 0.0)
|
||||
Set to 0 for constant values, >0 for random sampling
|
||||
- startup_fill (bool): Fill the whole KV cache once at startup instead
|
||||
of filling each new request's blocks in the step that schedules it
|
||||
(default: False). Circular-buffer caches are still zeroed per
|
||||
request, and attention caches whose new blocks the engine zeroes
|
||||
before use (hybrid Mamba or mixed-precision KV caches) are still
|
||||
filled per request. The fill is not redone after sleep mode
|
||||
discards the KV cache.
|
||||
|
||||
The dummy KV cache is only meant for performance measurement; outputs are
|
||||
not meaningful for accuracy evaluation.
|
||||
|
||||
"""
|
||||
|
||||
@@ -45,13 +55,21 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
dcp_world_size_for_kv_cache_spec,
|
||||
resolve_dcp_kv_block_size,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import CircularBufferSpec, iter_layer_specs
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
CircularBufferSpec,
|
||||
KVCacheSpec,
|
||||
KVQuantMode,
|
||||
UniformTypeKVCacheSpecs,
|
||||
iter_layer_specs,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -64,6 +82,45 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_fp8_dtype(spec: KVCacheSpec, cache_dtype: str) -> torch.dtype | None:
|
||||
"""The fp8 dtype of a KV cache that stores plain fp8 values as uint8.
|
||||
|
||||
Returns None for other layouts, including packed ones that embed scales
|
||||
or other metadata in the uint8 page.
|
||||
"""
|
||||
if (
|
||||
not isinstance(spec, AttentionSpec)
|
||||
or spec.dtype != torch.uint8
|
||||
or spec.kv_quant_mode != KVQuantMode.FP8_PER_TENSOR
|
||||
or spec.state_content_bytes is not None
|
||||
):
|
||||
return None
|
||||
cache_dtype = getattr(spec, "cache_dtype_str", None) or cache_dtype
|
||||
if cache_dtype in ("fp8", "fp8_e4m3"):
|
||||
return current_platform.fp8_dtype()
|
||||
if cache_dtype == "fp8_e5m2":
|
||||
return torch.float8_e5m2
|
||||
return None
|
||||
|
||||
|
||||
def _get_startup_fill_group_ids(kv_cache_config: "KVCacheConfig") -> set[int]:
|
||||
"""KV cache groups that startup fill covers.
|
||||
|
||||
Circular buffers are zero-filled per request. Attention groups whose new
|
||||
blocks the engine zeroes before use are filled per request, since the
|
||||
zeroing would wipe the startup fill.
|
||||
"""
|
||||
group_ids = set()
|
||||
for group_idx, group in enumerate(kv_cache_config.kv_cache_groups):
|
||||
spec = group.kv_cache_spec
|
||||
if any(isinstance(s, CircularBufferSpec) for s in iter_layer_specs(spec)):
|
||||
continue
|
||||
if kv_cache_config.needs_kv_cache_zeroing and isinstance(spec, AttentionSpec):
|
||||
continue
|
||||
group_ids.add(group_idx)
|
||||
return group_ids
|
||||
|
||||
|
||||
@dataclass
|
||||
class DecodeBenchConnectorMetadata(KVConnectorMetadata):
|
||||
"""Metadata for DecodeBenchConnector.
|
||||
@@ -191,6 +248,14 @@ class DecodeBenchConnectorScheduler:
|
||||
"""Scheduler-side implementation for DecodeBenchConnector."""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
assert kv_transfer_config is not None
|
||||
self.startup_fill = kv_transfer_config.get_from_extra_config(
|
||||
"startup_fill", False
|
||||
)
|
||||
self._startup_fill_group_ids = (
|
||||
_get_startup_fill_group_ids(kv_cache_config) if self.startup_fill else set()
|
||||
)
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
self.kv_cache_groups = kv_cache_config.kv_cache_groups
|
||||
self.group_block_sizes = tuple(
|
||||
@@ -275,6 +340,9 @@ class DecodeBenchConnectorScheduler:
|
||||
strict=True,
|
||||
)
|
||||
):
|
||||
if group_idx in self._startup_fill_group_ids:
|
||||
block_ids_per_group_list.append([])
|
||||
continue
|
||||
is_circular_buffer = all(
|
||||
isinstance(spec, CircularBufferSpec)
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
@@ -310,6 +378,9 @@ class DecodeBenchConnectorScheduler:
|
||||
)
|
||||
block_ids_per_group_list.append(block_ids)
|
||||
block_ids_per_group = tuple(block_ids_per_group_list)
|
||||
self._filled_requests.add(req_id)
|
||||
if self.startup_fill and not any(block_ids_per_group):
|
||||
return
|
||||
|
||||
# Store the blocks to fill for all group. _pending_fills doesn't need cleanup
|
||||
# as it's cleared after build_connector_meta
|
||||
@@ -317,7 +388,6 @@ class DecodeBenchConnectorScheduler:
|
||||
block_ids_per_group,
|
||||
num_external_tokens,
|
||||
)
|
||||
self._filled_requests.add(req_id)
|
||||
|
||||
block_counts = tuple(len(group) for group in block_ids_per_group)
|
||||
logger.debug(
|
||||
@@ -356,10 +426,27 @@ class DecodeBenchConnectorWorker:
|
||||
assert kv_transfer_config is not None
|
||||
self.fill_mean = kv_transfer_config.get_from_extra_config("fill_mean", 0.015)
|
||||
self.fill_std = kv_transfer_config.get_from_extra_config("fill_std", 0.0)
|
||||
self.startup_fill = kv_transfer_config.get_from_extra_config(
|
||||
"startup_fill", False
|
||||
)
|
||||
self._startup_fill_group_ids = (
|
||||
_get_startup_fill_group_ids(kv_cache_config) if self.startup_fill else set()
|
||||
)
|
||||
|
||||
# Will be populated via register_kv_caches
|
||||
self.kv_caches: dict[str, torch.Tensor] | None = None
|
||||
|
||||
# Layers whose uint8 KV cache stores plain fp8 values
|
||||
self._fp8_dtypes: dict[str, torch.dtype] = {}
|
||||
for group in kv_cache_config.kv_cache_groups:
|
||||
for layer_name in group.layer_names:
|
||||
spec = group.kv_cache_spec
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
spec = spec.kv_cache_specs[layer_name]
|
||||
fp8_dtype = _get_fp8_dtype(spec, vllm_config.cache_config.cache_dtype)
|
||||
if fp8_dtype is not None:
|
||||
self._fp8_dtypes[layer_name] = fp8_dtype
|
||||
|
||||
# Mapping from KV cache group index to list of layer names in that group
|
||||
self.group_to_layers = {
|
||||
group_idx: list(group.layer_names)
|
||||
@@ -373,6 +460,17 @@ class DecodeBenchConnectorWorker:
|
||||
for spec in iter_layer_specs(group.kv_cache_spec)
|
||||
)
|
||||
}
|
||||
num_zeroed_groups = len(
|
||||
self.group_to_layers.keys()
|
||||
- self._startup_fill_group_ids
|
||||
- self._zero_fill_group_ids
|
||||
)
|
||||
if self.startup_fill and num_zeroed_groups:
|
||||
logger.info(
|
||||
"DecodeBenchConnector: the engine zeroes new blocks of %d KV "
|
||||
"cache groups before use, so they are still filled per request.",
|
||||
num_zeroed_groups,
|
||||
)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Store references to the KV cache tensors."""
|
||||
@@ -383,6 +481,48 @@ class DecodeBenchConnectorWorker:
|
||||
len(kv_caches),
|
||||
)
|
||||
|
||||
if self.startup_fill:
|
||||
self._fill_all_kv_caches()
|
||||
|
||||
def _fill_all_kv_caches(self):
|
||||
"""Fill the KV cache tensors of the startup-filled groups once, in
|
||||
place, with dummy values, so that no per-request fill is needed.
|
||||
"""
|
||||
assert self.kv_caches is not None
|
||||
|
||||
num_layers = 0
|
||||
for group_idx in sorted(self._startup_fill_group_ids):
|
||||
for layer_name in self.group_to_layers[group_idx]:
|
||||
kv_cache = self.kv_caches.get(layer_name)
|
||||
if isinstance(kv_cache, torch.Tensor):
|
||||
tensors = [kv_cache]
|
||||
elif isinstance(kv_cache, (list, tuple)) and all(
|
||||
isinstance(t, torch.Tensor) for t in kv_cache
|
||||
):
|
||||
tensors = list(kv_cache)
|
||||
else:
|
||||
logger.warning_once(
|
||||
"DecodeBenchConnector: skipping startup fill for layer "
|
||||
"%s whose KV cache is %s, not a tensor or a list/tuple "
|
||||
"of tensors.",
|
||||
layer_name,
|
||||
type(kv_cache).__name__,
|
||||
)
|
||||
continue
|
||||
for tensor in tensors:
|
||||
fill_dtype = self._fp8_dtypes.get(layer_name, tensor.dtype)
|
||||
self._fill_tensor(tensor, self.fill_mean, self.fill_std, fill_dtype)
|
||||
num_layers += 1
|
||||
|
||||
logger.info(
|
||||
"DecodeBenchConnector: Filled %d KV cache layers at startup with %s "
|
||||
"values (mean=%.3f, std=%.3f)",
|
||||
num_layers,
|
||||
"random" if self.fill_std > 0 else "constant",
|
||||
self.fill_mean,
|
||||
self.fill_std,
|
||||
)
|
||||
|
||||
def start_fill_kv(self, metadata: DecodeBenchConnectorMetadata):
|
||||
"""Fill the allocated KV cache blocks with dummy values.
|
||||
|
||||
@@ -455,12 +595,17 @@ class DecodeBenchConnectorWorker:
|
||||
# dimension — so fill each tensor in its entirety with the same
|
||||
# dummy values.
|
||||
if isinstance(kv_cache, torch.Tensor):
|
||||
self._fill_block_tensor(kv_cache, block_ids, fill_mean, fill_std)
|
||||
fill_dtype = self._fp8_dtypes.get(layer_name, kv_cache.dtype)
|
||||
self._fill_block_tensor(
|
||||
kv_cache, block_ids, fill_mean, fill_std, fill_dtype
|
||||
)
|
||||
elif isinstance(kv_cache, (list, tuple)) and all(
|
||||
isinstance(t, torch.Tensor) for t in kv_cache
|
||||
):
|
||||
for state_tensor in kv_cache:
|
||||
self._fill_state_tensor(state_tensor, fill_mean, fill_std)
|
||||
self._fill_tensor(
|
||||
state_tensor, fill_mean, fill_std, state_tensor.dtype
|
||||
)
|
||||
else:
|
||||
logger.warning_once(
|
||||
"DecodeBenchConnector: skipping fill for layer %s whose KV "
|
||||
@@ -486,6 +631,7 @@ class DecodeBenchConnectorWorker:
|
||||
block_ids: list[int],
|
||||
fill_mean: float,
|
||||
fill_std: float,
|
||||
fill_dtype: torch.dtype,
|
||||
):
|
||||
"""Fill the requested block rows of a block-indexed KV cache tensor.
|
||||
|
||||
@@ -495,6 +641,8 @@ class DecodeBenchConnectorWorker:
|
||||
tensor's first dim are ignored.
|
||||
fill_mean: Mean value for the fill.
|
||||
fill_std: Standard deviation for the fill.
|
||||
fill_dtype: Dtype the values are encoded as, e.g. the fp8 dtype
|
||||
of a uint8 fp8 cache.
|
||||
|
||||
"""
|
||||
# Convert block_ids to tensor on device
|
||||
@@ -509,46 +657,85 @@ class DecodeBenchConnectorWorker:
|
||||
if len(valid_block_ids) == 0:
|
||||
return
|
||||
|
||||
# Create fill values - either constant or random
|
||||
block_shape = kv_cache.shape[1:]
|
||||
if fill_std > 0:
|
||||
# Random normal sampling
|
||||
fill_values = torch.normal(
|
||||
mean=fill_mean,
|
||||
std=fill_std,
|
||||
size=(len(valid_block_ids),) + block_shape,
|
||||
dtype=kv_cache.dtype,
|
||||
device=kv_cache.device,
|
||||
)
|
||||
else:
|
||||
# Constant fill value
|
||||
fill_values = torch.full(
|
||||
(len(valid_block_ids),) + block_shape,
|
||||
fill_mean,
|
||||
dtype=kv_cache.dtype,
|
||||
device=kv_cache.device,
|
||||
)
|
||||
fill_values = self._make_fill_values(
|
||||
(len(valid_block_ids),) + kv_cache.shape[1:],
|
||||
kv_cache.device,
|
||||
fill_dtype,
|
||||
fill_mean,
|
||||
fill_std,
|
||||
)
|
||||
|
||||
# Batch fill operation
|
||||
kv_cache[valid_block_ids] = fill_values
|
||||
kv_cache[valid_block_ids] = fill_values.view(kv_cache.dtype)
|
||||
|
||||
def _fill_state_tensor(
|
||||
self, kv_cache: torch.Tensor, fill_mean: float, fill_std: float
|
||||
def _make_fill_values(
|
||||
self,
|
||||
size: tuple[int, ...],
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
fill_mean: float,
|
||||
fill_std: float,
|
||||
) -> torch.Tensor:
|
||||
"""Create constant or random fill values of ``dtype``, clamped to its
|
||||
finite range. Non-floating dtypes (packed layouts) are filled with
|
||||
zeros.
|
||||
"""
|
||||
if not dtype.is_floating_point:
|
||||
if fill_mean or fill_std:
|
||||
logger.warning_once(
|
||||
"DecodeBenchConnector: %s KV caches do not hold plain "
|
||||
"floating-point values; filling them with zeros.",
|
||||
dtype,
|
||||
)
|
||||
return torch.zeros(size, dtype=dtype, device=device)
|
||||
|
||||
finfo = torch.finfo(dtype)
|
||||
if fill_std > 0:
|
||||
sample_dtype = torch.float32
|
||||
if dtype in (torch.float16, torch.bfloat16, torch.float32, torch.float64):
|
||||
sample_dtype = dtype
|
||||
values = torch.normal(
|
||||
fill_mean, fill_std, size=size, device=device, dtype=sample_dtype
|
||||
)
|
||||
return values.clamp_(finfo.min, finfo.max).to(dtype)
|
||||
fill_mean = min(max(fill_mean, finfo.min), finfo.max)
|
||||
return torch.full(size, fill_mean, dtype=dtype, device=device)
|
||||
|
||||
def _fill_tensor(
|
||||
self,
|
||||
kv_cache: torch.Tensor,
|
||||
fill_mean: float,
|
||||
fill_std: float,
|
||||
fill_dtype: torch.dtype,
|
||||
):
|
||||
"""Fill an entire non-block-indexed state tensor with dummy values.
|
||||
"""Fill an entire tensor in place with dummy values.
|
||||
|
||||
Hybrid / linear-attention layers (e.g. Mamba, Kimi Delta Attention)
|
||||
store their per-layer state as tensors with no num_blocks dimension,
|
||||
so the whole tensor is filled with the same constant or random values
|
||||
used for block fills, rather than selected block rows.
|
||||
Used for startup fills, and for hybrid / linear-attention layers (e.g.
|
||||
Mamba, Kimi Delta Attention) whose per-layer state tensors are filled
|
||||
in their entirety with the same constant or random values used for
|
||||
block fills, rather than selected block rows.
|
||||
|
||||
Args:
|
||||
kv_cache: A state tensor to fill in its entirety.
|
||||
kv_cache: A tensor to fill in its entirety.
|
||||
fill_mean: Mean value for the fill.
|
||||
fill_std: Standard deviation for the fill.
|
||||
fill_dtype: Dtype the values are encoded as, e.g. the fp8 dtype
|
||||
of a uint8 fp8 cache.
|
||||
|
||||
"""
|
||||
if fill_std > 0:
|
||||
kv_cache.normal_(mean=fill_mean, std=fill_std)
|
||||
else:
|
||||
kv_cache.fill_(fill_mean)
|
||||
view = kv_cache.view(fill_dtype)
|
||||
if fill_std == 0 or not fill_dtype.is_floating_point:
|
||||
view.fill_(
|
||||
self._make_fill_values(
|
||||
(), torch.device("cpu"), fill_dtype, fill_mean, fill_std
|
||||
)
|
||||
)
|
||||
return
|
||||
# Sample in chunks to bound the temporary memory.
|
||||
rows = max(1, (1 << 24) // max(1, view[0].numel()))
|
||||
for chunk in view.split(rows):
|
||||
chunk.copy_(
|
||||
self._make_fill_values(
|
||||
chunk.shape, chunk.device, fill_dtype, fill_mean, fill_std
|
||||
)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user