[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-08-21 17:47:49 -07:00
committed by GitHub
parent 9ff7041b55
commit 8bdc70ec7b
150 changed files with 4006 additions and 5815 deletions
@@ -12,7 +12,7 @@ import torch
from vllm.config import (
VllmConfig,
get_current_vllm_config,
get_current_vllm_config_or_none,
get_layers_from_vllm_config,
set_current_vllm_config,
)
@@ -35,20 +35,20 @@ EngineId = str
BlockIds = tuple[list[int], ...] | list[list[int]]
def get_kv_connector_cache_layout():
# NOTE (NickLucche) When running disaggregated PD with NIXL, HND layout is
def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
# NOTE (NickLucche) When running disaggregated PD with NIXL, LBHNC layout is
# used for faster transfer.
vllm_config = get_current_vllm_config()
if vllm_config is None:
vllm_config = get_current_vllm_config_or_none()
if vllm_config is None:
return None
kv_config = vllm_config.kv_transfer_config
if kv_config is not None:
connector_cls = KVConnectorFactory.get_connector_class(kv_config)
required_kvcache_layout = connector_cls.get_required_kvcache_layout(vllm_config)
if required_kvcache_layout is not None:
return required_kvcache_layout
logger.info_once(
"Connectors do not specify a kv cache layout, defaulting to NHD."
)
return "NHD"
return None
class KVOutputAggregator:
@@ -288,11 +288,11 @@ def kv_postprocess_layout_on_receive(cache, indices):
def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_ratio):
"""
Transforms the layout of received KV cache to the local block_size and HND.
Transforms the layout of received KV cache to the local block_size and LBHNC.
(Only works for local blocksize > remote blocksize)
prefill is HND, smaller block_size
decode(local) is NHD, larger block_size
prefill is LBHNC, smaller block_size
decode(local) is LBNHC, larger block_size
"""
blocks_to_update = cache.index_select(0, indices)
@@ -427,48 +427,6 @@ class TransferTopology:
self._engines: dict[tuple[EngineId, int], EngineTransferInfo] = {}
# Figure out whether the first dimension of the cache is K/V
# or num_blocks.
attn_backend = self.attn_backends[0]
if not self.is_mamba:
_MOCK_BLOCK_SIZE = 16
kv_cache_shape: tuple[int, ...] = attn_backend.get_kv_cache_shape(
num_blocks=1,
block_size=_MOCK_BLOCK_SIZE,
num_kv_heads=1,
head_size=1,
)
logger.debug("Test kv_cache_shape: %s", kv_cache_shape)
assert kv_cache_shape[0] == 1, (
"KV cache layout must be blocks-first; expected mocked "
f"num_blocks=1 in leading dim, got shape {kv_cache_shape}."
)
if not self.is_mla:
assert len(kv_cache_shape) == 4, (
"Attention KV cache layout must be standardized as "
"[num_blocks, num_kv_heads, block_size, content_size], "
f"got shape {kv_cache_shape}."
)
self._cross_layers_blocks = False
if self.tensor_shape is not None:
self._cross_layers_blocks = (
len(self.tensor_shape) == len(kv_cache_shape) + 1
)
if self._cross_layers_blocks:
logger.debug("Using cross-layer KV cache")
_MOCK_NUM_LAYERS = 80
kv_cache_shape = (_MOCK_NUM_LAYERS,) + kv_cache_shape
try:
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order(
include_num_layers_dimension=self._cross_layers_blocks
)
except (AttributeError, NotImplementedError):
assert self.tensor_shape is not None
kv_cache_stride_order = tuple(range(len(self.tensor_shape)))
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
# ============================================================
# Engine registration
# ============================================================
@@ -503,24 +461,6 @@ class TransferTopology:
for key in [k for k in self._engines if k[0] == remote_engine_id]:
del self._engines[key]
# ============================================================
# Layout properties
# ============================================================
@property
def cross_layers_blocks(self) -> bool:
return self._cross_layers_blocks
@property
def virtually_split_kv_in_blocks(self) -> bool:
# Whether to logically split each block into two separately-indexable
# sub-regions. With K and V packed into the content dim, an attention
# block transfers as a single unit — no K/V sub-split is needed. Only
# Mamba still needs this, to index its two state regions (conv/ssm)
# separately. Not applicable to cross-layer blocks (per-layer
# interleaving means a simple half-split does not separate the parts).
return self.is_mamba and not self._cross_layers_blocks
# ============================================================
# Common methods
# ============================================================
@@ -48,7 +48,7 @@ from typing import TYPE_CHECKING, Any, Literal
import torch
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
@@ -182,14 +182,6 @@ class KVConnectorBase_V1(ABC):
"""
return False
@property
def prefer_cross_layer_blocks(self) -> bool:
"""
Indicates whether this connector prefers KV blocks that hold KV data for all
layers, which can speed up KV data transfers. Defaults to False.
"""
return False
@property
def requires_kv_delivery(self) -> bool:
"""Whether this connector hands off KV that must be reliably delivered.
@@ -279,23 +271,6 @@ class KVConnectorBase_V1(ABC):
"""
return
def register_cross_layers_kv_cache(
self, kv_cache: torch.Tensor, attn_backend: type["AttentionBackend"]
):
"""
Initialize with a single KV cache tensor used by all layers.
The first dimension should be num_layers.
This function will only be called for models with uniform layers,
and only if the prefers_cross_layer_blocks is set to True.
Only one of the functions
{register_kv_caches, register_cross_layers_kv_cache} will be called.
Args:
kv_cache: a cross-layers kv cache tensor
attn_backend: The attention backend that corresponds to all layers
"""
return
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
"""
Set the xPU-specific ops for copying KV between host and device.
@@ -129,20 +129,17 @@ class ExampleConnector(KVConnectorBase_V1):
"""Inject the KV cache into the layer.
Args:
dst_kv_cache_layer (torch.Tensor): the destination KV cache
layer. In shape [num_pages, page_size, xxx] for MLA,
[num_pages, 2, page_size, xxx] otherwise.
dst_kv_cache_layer (torch.Tensor): the destination KV cache layer,
a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
src_kv_cache (torch.Tensor): the source KV cache.
slot_mapping (torch.Tensor): the slot mapping. In shape
[num_tokens].
"""
slot_mapping = slot_mapping.to(dst_kv_cache_layer.device, non_blocking=True)
if isinstance(attn_metadata, MLACommonMetadata):
dst_kv_cache_layer_shape = dst_kv_cache_layer.shape
num_pages = dst_kv_cache_layer_shape[0]
page_size = dst_kv_cache_layer_shape[1]
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
dst_kv_cache_layer = dst_kv_cache_layer.reshape(
num_pages * page_size, -1
-1, dst_kv_cache_layer.shape[-1]
)
dst_kv_cache_layer[slot_mapping, ...] = src_kv_cache
else:
@@ -225,13 +222,12 @@ class ExampleConnector(KVConnectorBase_V1):
) -> torch.Tensor:
"""Extract the KV cache from the layer.
Assume the shape of the layer is (num_pages, page_size, xxx)
for MLA, and (num_pages, 2, page_size, xxx) otherwise.
The layer is a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
"""
slot_mapping = slot_mapping.to(layer.device, non_blocking=True)
if isinstance(attn_metadata, MLACommonMetadata):
num_pages, page_size = layer.shape[0], layer.shape[1]
return layer.reshape(num_pages * page_size, -1)[slot_mapping, ...]
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
return layer.reshape(-1, layer.shape[-1])[slot_mapping, ...]
block_idxs = slot_mapping // self._block_size
offsets = slot_mapping % self._block_size
return layer[block_idxs, :, offsets]
@@ -38,8 +38,10 @@ def extract_from_kv_cache(
num_tokens: int,
) -> torch.Tensor:
"""Extract data from KV cache."""
block_size = kv_cache.shape[1]
return kv_cache[slot_mapping // block_size, slot_mapping % block_size][:num_tokens]
block_size = kv_cache.shape[2]
return kv_cache[slot_mapping // block_size, :, slot_mapping % block_size][
:num_tokens
]
def load_hidden_states(path: str) -> dict[str, torch.Tensor]:
@@ -100,15 +102,6 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
Must be used in conjunction with the `extract_hidden_states` spec decoding method.
"""
@property
def prefer_cross_layer_blocks(self) -> bool:
"""
Indicates whether this connector prefers KV blocks that hold KV data for all
layers, which can speed up KV data transfers. Defaults to False.
"""
# Must be False so that drafter kv cache isn't merged with verifier's
return False
@classmethod
def _find_cache_kv_group_id(cls, kv_cache_config: "KVCacheConfig | None") -> int:
"""Index of the KV cache group holding the extracted hidden states.
@@ -310,10 +303,12 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
# Block size must match the indexed buffer, else reads hit the wrong
# slots. Raise (not assert) so the check survives `python -O`.
if self._block_size != self._kv_cache.shape[1]:
# Views are [num_blocks, num_heads, block_size, head_size], matching what
# extract_from_kv_cache() indexes.
if self._block_size != self._kv_cache.shape[2]:
raise ValueError(
f"Hidden-states block-size mismatch: derived {self._block_size} "
f"but buffer block size is {self._kv_cache.shape[1]}; read slots "
f"but buffer block size is {self._kv_cache.shape[2]}; read slots "
"would be wrong (likely a hybrid block-size resolution bug)."
)
@@ -605,9 +600,9 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
"get_required_kvcache_layout should not be called "
"on the abstract base class"
)
# NHD means we have (num_tokens, num_heads)
# HND means we have (num_heads, num_tokens)
# For now, we only support NHD layout since this keeps the
# LBNHC means we have (num_tokens, num_heads)
# LBHNC means we have (num_heads, num_tokens)
# For now, we only support LBNHC layout since this keeps the
# hidden states for each token together in memory.
# HND is primarily used when sharding heads across devices.
return "NHD"
# LBHNC is primarily used when sharding heads across devices.
return "LBNHC"
@@ -541,35 +541,32 @@ class HF3FSKVConnector(KVConnectorBase_V1):
self._dtype = first_cache.dtype
element_size = first_cache.element_size()
if self._use_mla:
assert len(first_cache.shape) == 3, "MLA format should have 3 dimensions"
# MLA format: [num_blocks, block_size, head_size]
num_blocks, block_size, head_size = first_cache.shape
num_heads = 1
else:
# MHA format: [2, num_blocks, block_size, num_heads, head_size]
_, num_blocks, block_size, num_heads, head_size = first_cache.shape
# Standardized per-layer [B, H, N, C] view; MLA is just H == 1 with the
# latent vector as the content dim.
assert len(first_cache.shape) == 4, (
f"expected a [B, H, N, C] KV cache view, got {tuple(first_cache.shape)}"
)
num_blocks, num_heads, block_size, content_dim = first_cache.shape
cache_strides = first_cache.stride()
assert all(
cache.shape == first_cache.shape and cache.stride() == cache_strides
for cache in self._kv_caches.values()
), "HF3FS requires uniform KV cache shapes and strides."
self._local_total_tokens = num_blocks * block_size
self._local_block_size = block_size
self._num_heads = num_heads
self._content_dim = content_dim
self._kv_cache_strides = cache_strides
if self._use_mla:
layer_block_size = block_size * head_size * element_size
self._bytes_per_page = layer_block_size * len(self._kv_caches)
self._shape_per_page = [
len(self._kv_caches),
block_size,
head_size,
]
else:
layer_block_size = 2 * block_size * num_heads * head_size * element_size
self._bytes_per_page = layer_block_size * len(self._kv_caches)
self._shape_per_page = [
len(self._kv_caches),
2,
block_size,
num_heads * head_size,
]
layer_block_size = num_heads * block_size * content_dim * element_size
self._bytes_per_page = layer_block_size * len(self._kv_caches)
self._shape_per_page = [
len(self._kv_caches),
num_heads,
block_size,
content_dim,
]
self._kvcache_ptrs = torch.tensor(
[cache.data_ptr() for cache in self._kv_caches.values()],
@@ -993,7 +990,10 @@ class HF3FSKVConnector(KVConnectorBase_V1):
self._local_total_tokens,
buffer_tensor,
token_indices,
is_mla=self._use_mla,
self._local_block_size,
self._num_heads,
self._content_dim,
self._kv_cache_strides,
)
else:
gather_scatter_helper.scatter_kv_caches(
@@ -1001,7 +1001,10 @@ class HF3FSKVConnector(KVConnectorBase_V1):
self._local_total_tokens,
buffer_tensor,
token_indices,
is_mla=self._use_mla,
self._local_block_size,
self._num_heads,
self._content_dim,
self._kv_cache_strides,
)
def _compute_prefix_hash(
@@ -12,10 +12,15 @@ def kv_cache_scatter_kernel(
source_ptr,
token_indices_ptr,
num_tokens_in_block,
hidden_size,
total_token_in_kvcache,
num_layers,
is_mla,
tokens_per_block,
block_stride,
head_stride,
token_stride,
content_stride,
num_heads: tl.constexpr,
content_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
layer_idx = tl.program_id(0)
@@ -30,39 +35,25 @@ def kv_cache_scatter_kernel(
if token_idx >= total_token_in_kvcache:
return
if is_mla:
# MLA format: source [num_layers, num_tokens_in_block, hidden_size]
# MLA format: target [total_token_in_kvcache, hidden_size] (per layer)
source_offset = (layer_idx * num_tokens_in_block + token_pos) * hidden_size
target_offset = token_idx * hidden_size
for i in range(0, hidden_size, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < hidden_size
val = tl.load(source_ptr + source_offset + offset, mask=mask)
tl.store(kv_cache_ptr + target_offset + offset, val, mask=mask)
else:
# MHA format: source [num_layers, 2, num_tokens_in_block, hidden_size]
# MHA format: target [2, total_token_in_kvcache, hidden_size]
source_offset_k = (
layer_idx * num_tokens_in_block * 2 + token_pos
) * hidden_size
source_offset_v = (
layer_idx * num_tokens_in_block * 2 + num_tokens_in_block + token_pos
) * hidden_size
target_offset_k = token_idx * hidden_size
target_offset_v = (total_token_in_kvcache + token_idx) * hidden_size
for i in range(0, hidden_size, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < hidden_size
val_k = tl.load(source_ptr + source_offset_k + offset, mask=mask)
val_v = tl.load(source_ptr + source_offset_v + offset, mask=mask)
tl.store(kv_cache_ptr + target_offset_k + offset, val_k, mask=mask)
tl.store(kv_cache_ptr + target_offset_v + offset, val_v, mask=mask)
block_idx = token_idx // tokens_per_block
state_idx = token_idx % tokens_per_block
hidden_size = num_heads * content_size
for i in range(0, hidden_size, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < hidden_size
head_idx = offset // content_size
content_idx = offset % content_size
source_offset = (
(layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos
) * content_size + content_idx
target_offset = (
block_idx * block_stride
+ head_idx * head_stride
+ state_idx * token_stride
+ content_idx * content_stride
)
value = tl.load(source_ptr + source_offset, mask=mask)
tl.store(kv_cache_ptr + target_offset, value, mask=mask)
@triton.jit
@@ -71,10 +62,15 @@ def kv_cache_gather_kernel(
dst_ptr,
token_indices_ptr,
num_tokens_in_block,
hidden_size,
total_token_in_kvcache,
num_layers,
is_mla,
tokens_per_block,
block_stride,
head_stride,
token_stride,
content_stride,
num_heads: tl.constexpr,
content_size: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
):
layer_idx = tl.program_id(0)
@@ -89,37 +85,25 @@ def kv_cache_gather_kernel(
if token_idx >= total_token_in_kvcache:
return
if is_mla:
# MLA format: source [total_token_in_kvcache, hidden_size] (per layer)
# MLA format: dst [num_layers, num_tokens_in_block, hidden_size]
kvcache_offset = token_idx * hidden_size
dst_offset = (layer_idx * num_tokens_in_block + token_pos) * hidden_size
for i in range(0, hidden_size, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < hidden_size
val = tl.load(kv_cache_ptr + kvcache_offset + offset, mask=mask)
tl.store(dst_ptr + dst_offset + offset, val, mask=mask)
else:
# MHA format: source [2, total_token_in_kvcache, hidden_size]
# MHA format: dst [num_layers, 2, num_tokens_in_block, hidden_size]
dst_offset_k = (layer_idx * num_tokens_in_block * 2 + token_pos) * hidden_size
dst_offset_v = (
layer_idx * num_tokens_in_block * 2 + num_tokens_in_block + token_pos
) * hidden_size
kvcache_offset_k = token_idx * hidden_size
kvcache_offset_v = (total_token_in_kvcache + token_idx) * hidden_size
for i in range(0, hidden_size, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < hidden_size
val_k = tl.load(kv_cache_ptr + kvcache_offset_k + offset, mask=mask)
val_v = tl.load(kv_cache_ptr + kvcache_offset_v + offset, mask=mask)
tl.store(dst_ptr + dst_offset_k + offset, val_k, mask=mask)
tl.store(dst_ptr + dst_offset_v + offset, val_v, mask=mask)
block_idx = token_idx // tokens_per_block
state_idx = token_idx % tokens_per_block
hidden_size = num_heads * content_size
for i in range(0, hidden_size, BLOCK_SIZE):
offset = i + tl.arange(0, BLOCK_SIZE)
mask = offset < hidden_size
head_idx = offset // content_size
content_idx = offset % content_size
kvcache_offset = (
block_idx * block_stride
+ head_idx * head_stride
+ state_idx * token_stride
+ content_idx * content_stride
)
dst_offset = (
(layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos
) * content_size + content_idx
value = tl.load(kv_cache_ptr + kvcache_offset, mask=mask)
tl.store(dst_ptr + dst_offset, value, mask=mask)
def scatter_kv_caches(
@@ -127,34 +111,33 @@ def scatter_kv_caches(
total_token_in_kvcache: int,
src_tensor: torch.Tensor,
token_indices: list[int],
is_mla: bool = False,
tokens_per_block: int,
num_heads: int,
content_size: int,
kv_cache_strides: tuple[int, ...],
) -> None:
"""Scatter KV cache data from source tensor to KV cache storage.
Args:
kv_caches_ptrs: Tensor of KV cache pointers (one per layer)
total_token_in_kvcache: Total number of tokens in KV cache
src_tensor: Source tensor containing data to scatter
- MHA format: [num_layers, 2, num_tokens_in_block, hidden_size]
- MLA format: [num_layers, num_tokens_in_block, hidden_size]
src_tensor: Source ``[L, H, N, C]`` tensor containing data to scatter
token_indices: List of token positions to update
is_mla: Whether using MLA model format
tokens_per_block: Number of stored states in each cache block
num_heads: Size of the H axis
content_size: Size of the C axis
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
"""
num_layers = len(kv_caches_ptrs)
num_tokens_in_block = len(token_indices)
if is_mla:
# MLA: src_tensor is [num_layers, num_tokens_in_block, hidden_size]
assert len(src_tensor.shape) == 3, (
f"MLA src_tensor should be 3D, got {src_tensor.shape}"
)
hidden_size = src_tensor.shape[2]
else:
# MHA: src_tensor is [num_layers, 2, num_tokens_in_block, hidden_size]
assert len(src_tensor.shape) == 4, (
f"MHA src_tensor should be 4D, got {src_tensor.shape}"
)
hidden_size = src_tensor.shape[3]
assert src_tensor.shape == (
num_layers,
num_heads,
num_tokens_in_block,
content_size,
)
assert len(kv_cache_strides) == 4
device = src_tensor.device
token_indices_tensor = torch.tensor(
@@ -169,10 +152,12 @@ def scatter_kv_caches(
src_tensor,
token_indices_tensor,
num_tokens_in_block,
hidden_size,
total_token_in_kvcache,
num_layers,
is_mla,
tokens_per_block,
*kv_cache_strides,
num_heads=num_heads,
content_size=content_size,
BLOCK_SIZE=BLOCK_SIZE,
)
@@ -182,49 +167,33 @@ def gather_kv_caches(
total_token_in_kvcache: int,
dst_tensor: torch.Tensor,
token_indices: list[int],
is_mla: bool = False,
tokens_per_block: int,
num_heads: int,
content_size: int,
kv_cache_strides: tuple[int, ...],
) -> None:
"""Gather KV cache data from KV cache storage to destination tensor.
Args:
kv_caches_ptrs: Tensor of KV cache pointers (one per layer)
total_token_in_kvcache: Total number of tokens in KV cache
dst_tensor: Destination tensor to store gathered data
- MHA format: [num_layers, 2, num_tokens_in_block, hidden_size]
- MLA format: [num_layers, num_tokens_in_block, hidden_size]
dst_tensor: Destination ``[L, H, N, C]`` tensor
token_indices: List of token positions to gather
is_mla: Whether using MLA model format
tokens_per_block: Number of stored states in each cache block
num_heads: Size of the H axis
content_size: Size of the C axis
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
"""
num_layers = kv_caches_ptrs.shape[0]
num_tokens_in_block = len(token_indices)
if is_mla:
# MLA: dst_tensor is [num_layers, num_tokens_in_block, hidden_size]
assert len(dst_tensor.shape) == 3, (
f"MLA dst_tensor should be 3D, got {dst_tensor.shape}"
)
assert dst_tensor.shape[0] == num_layers, (
f"Layer count mismatch: {dst_tensor.shape[0]} vs {num_layers}"
)
assert dst_tensor.shape[1] == num_tokens_in_block, (
f"Token count mismatch: {dst_tensor.shape[1]} vs {num_tokens_in_block}"
)
hidden_size = dst_tensor.shape[2]
else:
# MHA: dst_tensor is [num_layers, 2, num_tokens_in_block, hidden_size]
assert len(dst_tensor.shape) == 4, (
f"MHA dst_tensor should be 4D, got {dst_tensor.shape}"
)
assert dst_tensor.shape[0] == num_layers, (
f"Layer count mismatch: {dst_tensor.shape[0]} vs {num_layers}"
)
assert dst_tensor.shape[1] == 2, (
f"MHA should have 2 (K,V) components, got {dst_tensor.shape[1]}"
)
assert dst_tensor.shape[2] == num_tokens_in_block, (
f"Token count mismatch: {dst_tensor.shape[2]} vs {num_tokens_in_block}"
)
hidden_size = dst_tensor.shape[3]
assert dst_tensor.shape == (
num_layers,
num_heads,
num_tokens_in_block,
content_size,
)
assert len(kv_cache_strides) == 4
device = dst_tensor.device
token_indices_tensor = torch.tensor(
@@ -239,10 +208,12 @@ def gather_kv_caches(
dst_tensor,
token_indices_tensor,
num_tokens_in_block,
hidden_size,
total_token_in_kvcache,
num_layers,
is_mla,
tokens_per_block,
*kv_cache_strides,
num_heads=num_heads,
content_size=content_size,
BLOCK_SIZE=BLOCK_SIZE,
)
@@ -49,15 +49,15 @@ from vllm.model_executor.models.utils import extract_layer_index
from vllm.platforms import current_platform
from vllm.utils.math_utils import cdiv
from vllm.utils.network_utils import get_ip, make_zmq_path, make_zmq_socket
from vllm.utils.torch_utils import is_non_overlapping_and_dense
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID, get_kv_cache_layout
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
AttentionSpec,
FullAttentionSpec,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
SlidingWindowSpec,
)
from vllm.v1.request import RequestStatus
@@ -122,9 +122,7 @@ def _expand_transfer_regions(
kv_block_lens: list[int],
layer_names: list[str],
layer_indices: list[int],
is_kv_layout_blocks_first: bool,
group_indices: list[int] | None = None,
split_kv_regions: list[bool] | None = None,
) -> list[TransferRegion]:
"""Expand registered KV tensors into the regions transferred by Mooncake."""
assert (
@@ -146,13 +144,6 @@ def _expand_transfer_regions(
"Mooncake transfer regions require matching group metadata lengths, "
f"got group_indices={len(group_indices)}, layer_names={len(layer_names)}."
)
if split_kv_regions is None:
split_kv_regions = [is_kv_layout_blocks_first] * len(layer_names)
assert len(split_kv_regions) == len(layer_names), (
"Mooncake transfer regions require matching split metadata, "
f"got split_kv_regions={len(split_kv_regions)}, "
f"layer_names={len(layer_names)}."
)
regions: list[TransferRegion] = []
for (
base_addr,
@@ -161,7 +152,6 @@ def _expand_transfer_regions(
layer_name,
layer_index,
group_index,
split_kv_region,
) in zip(
base_addrs,
block_lens,
@@ -169,7 +159,6 @@ def _expand_transfer_regions(
layer_names,
layer_indices,
group_indices,
split_kv_regions,
):
regions.append(
TransferRegion(
@@ -181,17 +170,6 @@ def _expand_transfer_regions(
group_index=group_index,
)
)
if split_kv_region:
regions.append(
TransferRegion(
layer_name=layer_name,
layer_index=layer_index,
base_addr=base_addr + kv_block_len,
block_len=block_len,
kv_block_len=kv_block_len,
group_index=group_index,
)
)
return regions
@@ -506,10 +484,10 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
if vllm_config.model_config.use_mla:
return None
logger.info_once(
"MooncakeConnector setting KV cache layout to HND for "
"MooncakeConnector setting KV cache layout to LBHNC for "
"heterogeneous TP-safe KV transfer."
)
return "HND"
return "LBHNC"
############################################################
# Scheduler Side Methods
@@ -1023,12 +1001,10 @@ class MooncakeConnectorWorker:
self._sync_block_size_with_kernel()
self.attn_backends = get_current_attn_backends(vllm_config)
self.kv_cache_layout = get_kv_cache_layout()
logger.debug(
"Detected attention backends %s",
[backend.get_name() for backend in self.attn_backends],
)
logger.debug("Detected kv cache layout %s", self.kv_cache_layout)
self._tp_size: dict[EngineId, int] = {self.engine_id: self.tp_size}
self._layer_specs: dict[str, KVCacheSpec] = {}
@@ -1665,7 +1641,7 @@ class MooncakeConnectorWorker:
self.registered_layer_indices = []
self.registered_group_indices = []
for layer_name, cache_or_caches in kv_caches.items():
for layer_name, cache in kv_caches.items():
layer_index = extract_layer_index(layer_name)
layer_spec = self._layer_specs.get(layer_name)
if layer_spec is None:
@@ -1674,34 +1650,30 @@ class MooncakeConnectorWorker:
layer_name,
)
continue
if isinstance(layer_spec, MambaSpec):
conv, _ = cache_or_caches
cache_list = [conv]
# One raw page tensor per layer; for Mamba that page holds all the
# recurrent states, unpacked only when binding the cache for execution.
self._log_debug_cache_registration(layer_name, cache)
block_is_contiguous = is_non_overlapping_and_dense(cache[0])
if not block_is_contiguous:
# Non-block-compact layouts scatter a block across per-head
# regions; each region's blocks are contiguous.
region_caches = [cache[:, head] for head in range(cache.shape[1])]
assert all(
is_non_overlapping_and_dense(region[0]) for region in region_caches
)
else:
# K and V are packed into one blocks-first tensor per layer,
# so each layer registers as a single region.
cache_list = [cache_or_caches]
region_caches = [cache]
logger.debug(
"registering layer %s with %d cache tensor(s)",
layer_name,
len(cache_list),
)
for cache in cache_list:
self._log_debug_cache_registration(layer_name, cache)
base_addr = cache.data_ptr()
block_len = cache.stride(0) * cache.element_size()
for region_cache in region_caches:
base_addr = region_cache.data_ptr()
block_len = region_cache.stride(0) * region_cache.element_size()
region_base_addresses.append(base_addr)
if isinstance(layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)):
kv_block_len = layer_spec.page_size_bytes
elif self.transfer_topo.virtually_split_kv_in_blocks and not isinstance(
layer_spec, MambaSpec
):
kv_block_len = block_len // 2
else:
kv_block_len = block_len
kv_block_len = (
layer_spec.page_size_bytes
if isinstance(layer_spec, AttentionSpec) and block_is_contiguous
else block_len
)
self.block_len_per_layer.append(block_len)
self.kv_block_len_per_layer.append(kv_block_len)
self.registered_layer_names.append(layer_name)
@@ -1709,12 +1681,12 @@ class MooncakeConnectorWorker:
self.registered_group_indices.append(
self._layer_group_indices[layer_name]
)
storage = cache.untyped_storage()
storage_addr = storage.data_ptr()
if storage_addr not in seen_storage_ptrs:
seen_storage_ptrs.add(storage_addr)
kv_data_ptrs.append(storage_addr)
kv_data_lens.append(storage.nbytes())
storage = cache.untyped_storage()
storage_addr = storage.data_ptr()
if storage_addr not in seen_storage_ptrs:
seen_storage_ptrs.add(storage_addr)
kv_data_ptrs.append(storage_addr)
kv_data_lens.append(storage.nbytes())
self.kv_caches_base_addr = region_base_addresses
self.seen_base_addresses = kv_data_ptrs
@@ -2055,24 +2027,13 @@ class MooncakeConnectorWorker:
self._layer_group_indices.get(layer_name, 0)
for layer_name in layer_names
]
split_kv_regions = None
if self.transfer_topo.virtually_split_kv_in_blocks:
split_kv_regions = [
not isinstance(
self._layer_specs[layer_name],
(MambaSpec, MLAAttentionSpec, SlidingWindowMLASpec),
)
for layer_name in layer_names
]
return _expand_transfer_regions(
base_addrs=base_addrs,
block_lens=block_lens,
kv_block_lens=kv_block_lens,
layer_names=layer_names,
layer_indices=layer_indices,
is_kv_layout_blocks_first=self.transfer_topo.virtually_split_kv_in_blocks,
group_indices=group_indices,
split_kv_regions=split_kv_regions,
)
def _get_sender_transfer_plan(
@@ -89,14 +89,6 @@ class MooncakeStoreKVEvents(KVConnectorKVEvents):
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
"""KV connector using MooncakeDistributedStore as shared KV pool."""
@property
def prefer_cross_layer_blocks(self) -> bool:
extra_config = self._kv_transfer_config.kv_connector_extra_config
return (
str(extra_config.get("enable_cross_layers_blocks", "False")).lower()
== "true"
)
@staticmethod
def _validate_kv_cache_config(
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
@@ -290,16 +282,6 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def register_cross_layers_kv_cache(
self, kv_cache: torch.Tensor, attn_backend: type
):
assert self.connector_worker is not None
assert (
self._kv_cache_config is not None
and len(self._kv_cache_config.kv_cache_groups) == 1
), "Cross-layer KV cache does not supported with hybrid models"
self.connector_worker.register_cross_layers_kv_caches(kv_cache)
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
# No-op: loads are issued in get_finished() for compute overlap.
pass
@@ -58,6 +58,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.utils.network_utils import get_ip, make_zmq_socket
from vllm.utils.torch_utils import is_non_overlapping_and_dense
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.core.kv_cache_utils import (
BlockHash,
@@ -72,6 +73,7 @@ from vllm.v1.kv_cache_interface import (
MLAAttentionSpec,
SlidingWindowMLASpec,
UniformTypeKVCacheSpecs,
group_kernel_blocks,
)
from .metrics import MooncakeStoreConnectorStats
@@ -1617,18 +1619,9 @@ class MooncakeStoreWorker:
for g_idx, db in enumerate(self.token_dbs)
)
def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None:
"""Register a cross-layers KV cache tensor.
Wraps the unified tensor in a single-entry dict so that the
existing stride-based logic in register_kv_caches() produces
the correct single-segment result (block_len = page_size * num_layers).
"""
self.register_kv_caches({"__cross_layer__": kv_cache})
def register_kv_caches(
self,
kv_caches: dict[str, torch.Tensor | list[torch.Tensor]],
kv_caches: dict[str, torch.Tensor],
) -> None:
"""Register KV cache tensors and start transfer threads."""
if self._capacity_only:
@@ -1637,58 +1630,62 @@ class MooncakeStoreWorker:
logger.warning("No KV caches to offload.")
return
# Resolve each entry to a representative tensor for storage
# deduplication. For attention layers the value is already a tensor;
# for Mamba layers it is a list of tensors that all share the same
# underlying raw storage, so we take the first one.
def _repr_tensor(v: torch.Tensor | list[torch.Tensor]) -> torch.Tensor:
assert isinstance(v, torch.Tensor | list)
return v if isinstance(v, torch.Tensor) else v[0]
assert self.cache_config.num_gpu_blocks is not None
self.num_blocks = self.cache_config.num_gpu_blocks
seen_ptrs: set[int] = set()
seen_storage_ptrs: set[int] = set()
seen_region_ptrs: set[int] = set()
addrs: list[int] = []
block_lens: list[int] = []
for value in kv_caches.values():
cache = _repr_tensor(value)
for cache in kv_caches.values():
cache = group_kernel_blocks(cache, self.num_blocks)
cache_storage = cache.untyped_storage()
base_addr = cache_storage.data_ptr()
if base_addr in seen_ptrs:
continue
seen_ptrs.add(base_addr)
region_len = cache_storage.nbytes()
ret = self.store.register_buffer(base_addr, region_len)
if ret != 0:
logger.error(
"register_buffer failed for addr %#x len %d: %d",
base_addr,
region_len,
ret,
)
if base_addr not in seen_storage_ptrs:
seen_storage_ptrs.add(base_addr)
ret = self.store.register_buffer(base_addr, region_len)
if ret != 0:
logger.error(
"register_buffer failed for addr %#x len %d: %d",
base_addr,
region_len,
ret,
)
# Detect layout via stride: a dim whose byte-stride exceeds
# page_size_bytes is an outer segment dim (e.g. the K/V dim of
# FlashAttn's (2, num_blocks, ...)). FlashInfer/MLA's blocks-
# outermost layout has no such dim and yields a single segment.
el = cache.element_size()
page_size_bytes = region_len // self.num_blocks
outer_dims = [
d for d in range(cache.ndim) if cache.stride(d) * el > page_size_bytes
]
if not outer_dims:
# Blocks-first layout (FlashInfer / MLA): one segment.
if not is_non_overlapping_and_dense(cache[0]):
# A block is scattered across per-head regions; each region's
# blocks are contiguous.
for head_idx in range(cache.shape[1]):
head_cache = cache[:, head_idx]
assert is_non_overlapping_and_dense(head_cache[0])
region_addr = head_cache.data_ptr()
if region_addr in seen_region_ptrs:
continue
seen_region_ptrs.add(region_addr)
addrs.append(region_addr)
block_lens.append(head_cache.stride(0) * head_cache.element_size())
elif cache.stride(0) * cache.element_size() * self.num_blocks == region_len:
# The block stride spans the whole per-block window
# (block-outermost and packed layouts, and single-layer
# tensors), which may hold other layers' pages at higher
# offsets. Register the storage once as one whole-window
# region; per-layer regions would copy overlapping windows and
# run past the storage for offset layers.
if base_addr in seen_region_ptrs:
continue
seen_region_ptrs.add(base_addr)
addrs.append(base_addr)
block_lens.append(page_size_bytes)
block_lens.append(region_len // self.num_blocks)
else:
# K/V-first layout (FlashAttn / ROCm): split segments.
seg_stride = cache.stride(outer_dims[0]) * el
for idx in range(cache.shape[outer_dims[0]]):
addrs.append(base_addr + idx * seg_stride)
block_lens.append(seg_stride // self.num_blocks)
region_addr = cache.data_ptr()
if region_addr in seen_region_ptrs:
continue
seen_region_ptrs.add(region_addr)
addrs.append(region_addr)
block_lens.append(cache.stride(0) * cache.element_size())
logger.info(
"Registered KV caches: num_groups=%d, num_segments=%d, num_blocks=%d",
@@ -57,11 +57,6 @@ def is_mla_cache_layer(
return isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec))
def _content_packed_dim(spec: AttentionSpec) -> int:
head_size_v = getattr(spec, "head_size_v", spec.head_size)
return spec.head_size + head_size_v
def _spec_dim_matches(value: int, expected: int | None) -> bool:
return expected is None or value == expected
@@ -110,23 +105,6 @@ def get_layer_transfer_geometry(
spec = layer_to_spec[layer_name]
is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name)
if is_mla_cache and len(shape) == 3:
num_blocks, block_size, latent_dim = shape
slot_size_bytes = latent_dim * element_size
block_len = block_size * slot_size_bytes
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[0],
local_kv_stride=None,
remote_kv_stride=None,
transfers_per_block=1,
regions_per_block=1,
split_kv_regions=False,
)
if not is_mla_cache and len(shape) == 5 and shape[0] == 2:
_, num_blocks = shape[:2]
kernel_blocks_per_block = 1
@@ -230,19 +208,19 @@ def get_layer_transfer_geometry(
)
if (
not is_mla_cache
and isinstance(spec, AttentionSpec)
isinstance(spec, AttentionSpec)
and len(shape) == 4
and shape[1] == spec.num_kv_heads
and shape[2] == spec.block_size
and shape[3] == _content_packed_dim(spec)
and shape[1] == spec.num_heads
and shape[2] == spec.num_states
and shape[3] * element_size == spec.state_content_size_bytes
):
num_blocks, num_kv_heads, block_size, packed_dim = shape
slot_size_bytes = num_kv_heads * packed_dim * element_size
block_len = block_size * slot_size_bytes
# Standardized per-layer [B, H, N, C] view (MLA is just H == 1).
num_blocks, num_heads, num_states, content_dim = shape
slot_size_bytes = num_heads * content_dim * element_size
block_len = num_states * slot_size_bytes
return LayerTransferGeometry(
num_blocks=num_blocks,
block_size=block_size,
block_size=spec.block_size,
block_len=block_len,
slot_size_bytes=slot_size_bytes,
block_stride=stride[0],
@@ -267,6 +245,16 @@ def iter_layer_registration_regions(
) -> list[tuple[torch.Tensor, int]]:
geometry = get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec)
region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len
if geometry.regions_per_block == 1:
# With padded or interleaved pages the block stride exceeds the
# meaningful block_len; register the strided span. The span ends with
# the last block's meaningful bytes, not a whole stride, so it never
# runs past the backing allocation of a strided layer view.
block_stride_bytes = geometry.block_stride * kv_cache.element_size()
region_len = max(
region_len,
(geometry.num_blocks - 1) * block_stride_bytes + geometry.block_len,
)
if geometry.split_kv_regions:
return [(cache, region_len) for cache in kv_cache]
return [(kv_cache, region_len)]
@@ -27,7 +27,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
PromMetricT,
)
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import KVConnectorOutput
@@ -209,12 +209,6 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
c.supports_divergent_local_hybrid_hits for c in self._connectors
)
@property
def prefer_cross_layer_blocks(self) -> bool:
if not self._connectors:
return False
return all(c.prefer_cross_layer_blocks for c in self._connectors)
@property
def requires_kv_delivery(self) -> bool:
return any(c.requires_kv_delivery for c in self._connectors)
@@ -245,13 +239,6 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
)
return ret
def register_cross_layers_kv_cache(
self, kv_cache: torch.Tensor, attn_backend: type[AttentionBackend]
):
# Register on all connectors
for c in self._connectors:
c.register_cross_layers_kv_cache(kv_cache, attn_backend)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
for c in self._connectors:
c.register_kv_caches(kv_caches)
@@ -71,9 +71,9 @@ from vllm.platforms import current_platform
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.network_utils import make_zmq_path
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backends.utils import get_kv_cache_layout
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheLayout,
MambaSpec,
MLAAttentionSpec,
SlidingWindowMLASpec,
@@ -89,6 +89,13 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _share_storage_and_block_stride(caches: list[torch.Tensor]) -> bool:
"""Return whether all views share storage and a block stride."""
block_strides = {cache.stride(0) * cache.element_size() for cache in caches}
storage_ptrs = {cache.untyped_storage().data_ptr() for cache in caches}
return len(block_strides) == len(storage_ptrs) == 1
class NixlBaseConnectorWorker:
"""Base implementation of Worker side methods shared by pull and push."""
@@ -302,7 +309,6 @@ class NixlBaseConnectorWorker:
for group in kv_cache_config.kv_cache_groups
for layer in group.layer_names
}
self.hma_group_size = len(kv_cache_config.kv_cache_tensors)
# ---- Model state (derived from model config) ----
mamba_ssm_size = (0, 0)
@@ -523,7 +529,9 @@ class NixlBaseConnectorWorker:
self.attn_backends = get_current_attn_backends(vllm_config)
self.backend_name = self.attn_backends[0].get_name()
self.kv_cache_layout = get_kv_cache_layout()
self.kv_cache_layout = (
vllm_config.cache_config.get_resolved_kv_cache_layout().name
)
self.host_buffer_kv_cache_layout = self.kv_cache_layout
logger.info(
"Detected attention backend(s) %s",
@@ -558,6 +566,11 @@ class NixlBaseConnectorWorker:
# This is not used for SSM layers, which use the counterpart `mamba_ssm_size`.
self.block_len_per_layer = list[int]()
# Per-region block stride in bytes. Taken from the registered tensor's
# stride(0) so it stays correct under layouts that interleave layers
# within a block (BLHNC/BHLNC), where stride > block_len.
self.block_stride_per_layer = list[int]()
# Per-engine TP mappings. Generated during handshake.
self.tp_mappings: dict[EngineId, TPMapping] = {}
@@ -788,16 +801,16 @@ class NixlBaseConnectorWorker:
if not self.use_mla:
assert kv_cache.ndim == 4
if self.kv_cache_layout == "NHD":
if self.kv_cache_layout == "LBNHC":
if self.kv_transfer_config.enable_permute_local_kv:
logger.info_once(
"'enable_permute_local_kv' flag is enabled while "
"device KV Layout is NHD. Init host buffer with"
" HND to better support Decode/Prefill TP_ratio > 1."
"device KV Layout is LBNHC. Init host buffer with"
" LBHNC to better support Decode/Prefill TP_ratio > 1."
)
# Since NHD will not support Decode/Prefill TP_ratio > 1,
# Since LBNHC will not support Decode/Prefill TP_ratio > 1,
# we can leverage host_buffer for permute.
self.host_buffer_kv_cache_layout = "HND"
self.host_buffer_kv_cache_layout = "LBHNC"
else:
# Packed KV layout is logical (B, H, N, 2*D). Allocate
# (B, N, H, 2*D) and view it as logical (B, H, N, 2*D)
@@ -969,120 +982,9 @@ class NixlBaseConnectorWorker:
fut.add_done_callback(request_ready)
def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None:
"""Register a cross-layers KV cache tensor with NIXL.
`use_uniform_kv_cache()` guarantees a single KV cache group whose
layers all share the same `AttentionSpec`, so any layer name from
`_layer_specs` yields the correct per-layer spec for `page_size_bytes`.
"""
first_layer = next(iter(self._layer_specs))
# Forwarding a real layer name rather than a synthetic key
self.register_kv_caches({first_layer: kv_cache})
def _register_packed_kv_cache(
self,
storage: torch.UntypedStorage,
) -> None:
"""Register a packed KV cache as a single NIXL region.
The packed allocation interleaves all layers per block, so each
block_stride-byte chunk is one logical block. We register 1
NIXL region and create 1 descriptor per block.
"""
self.transfer_topo = TransferTopology(
tp_rank=self.tp_rank,
tp_size=self.world_size,
block_size=self.block_size,
engine_id=self.engine_id,
is_mla=self.use_mla,
total_num_kv_heads=self.model_config.get_total_num_kv_heads(),
attn_backends=self.attn_backends,
tensor_shape=None,
is_mamba=self._has_mamba,
)
self.compat_hash = compute_nixl_compatibility_hash(
self.vllm_config,
self.backend_name,
self.transfer_topo.cross_layers_blocks,
transfer_mode=self._TRANSFER_MODE,
)
total_size = storage.nbytes()
block_stride = total_size // self.num_blocks
base_addr = storage.data_ptr()
device_id = storage.device.index
assert device_id is not None
logger.info(
"Registering packed KV cache: total_size=%s, block_stride=%s, "
"num_blocks=%s, num_regions=1",
total_size,
block_stride,
self.num_blocks,
)
self.device_id = device_id
caches_data = [(base_addr, total_size, self.device_id, "")]
self.block_len_per_layer = [block_stride]
self.num_regions = 1
self.num_descs = self.num_blocks
self.kv_caches_base_addr[self.engine_id][self.tp_rank] = [base_addr]
descs = self.nixl_wrapper.get_reg_descs(caches_data, self.nixl_memory_type)
self.nixl_wrapper.register_memory(descs, backends=self.nixl_backends)
self._registered_descs.append(descs)
self.dst_num_blocks[self.engine_id] = self.num_blocks
self.src_xfer_handles_by_block_size[self.block_size], (self.src_blocks_data) = (
self.register_local_xfer_handler(self.block_size)
)
agent_metadata = NixlAgentMetadata(
engine_id=self.engine_id,
agent_metadata=self.nixl_wrapper.get_agent_metadata(),
device_id=self.device_id,
kv_caches_base_addr=(
self.kv_caches_base_addr[self.engine_id][self.tp_rank]
),
num_blocks=self.num_blocks,
block_lens=self.block_len_per_layer,
kv_cache_layout=self.kv_cache_layout,
block_size=self.block_size,
ssm_sizes=self._mamba_ssm_size,
attn_backend_name=self.backend_name,
physical_blocks_per_logical_kv_block=(
self._physical_blocks_per_logical_kv_block
),
dcp_size=self.dcp_size,
pcp_size=self.pcp_size,
)
assert self.compat_hash is not None
encoder = msgspec.msgpack.Encoder()
self.xfer_handshake_metadata = NixlHandshakePayload(
compatibility_hash=self.compat_hash,
agent_metadata_bytes=encoder.encode(agent_metadata),
)
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
"""Register the KV Cache data in nixl."""
# Detect packed allocation: all tensors are strided views into the
# same backing storage (different data_ptr but same storage).
# This happens with DSv4-style contiguous per-block packing.
if len(kv_caches) > 1 and not self._has_mamba:
storage = next(iter(kv_caches.values())).untyped_storage()
storage_ptrs = {
cache.untyped_storage().data_ptr() for cache in kv_caches.values()
}
data_ptrs = {cache.data_ptr() for cache in kv_caches.values()}
if len(storage_ptrs) == 1 and len(data_ptrs) > 1:
self._register_packed_kv_cache(storage)
self.device_kv_caches = kv_caches
return
self.transfer_topo = TransferTopology(
tp_rank=self.tp_rank,
tp_size=self.world_size,
@@ -1100,7 +1002,6 @@ class NixlBaseConnectorWorker:
self.compat_hash = compute_nixl_compatibility_hash(
self.vllm_config,
self.backend_name,
self.transfer_topo.cross_layers_blocks,
transfer_mode=self._TRANSFER_MODE,
)
@@ -1127,14 +1028,14 @@ class NixlBaseConnectorWorker:
)
caches_data = []
# With hybrid allocator, layers can share a kv cache tensor
seen_storage_addresses: set[int] = set()
seen_base_addresses: list[int] = []
packed_storage = _share_storage_and_block_stride(list(xfer_buffers.values()))
# K and V are packed into the content dim, so each attention layer is a
# single NIXL region whose block transfers as one unit. Mamba layers instead
# register separate conv/ssm sub-regions (see `_build_mamba_local`).
tensor_size_bytes = None
for layer_name, cache in xfer_buffers.items():
# NOTE (NickLucche) Hybrid SSM mamba/FA physical page_size may differ when
# kernel requires a specific block size. This leads to SSM and FA layers
@@ -1159,55 +1060,98 @@ class NixlBaseConnectorWorker:
else layer_spec.page_size_bytes
// self._physical_blocks_per_logical_kv_block
)
if self.transfer_topo._cross_layers_blocks:
# When cross-layers blocks are used, multiply by number of layers
physical_page_size = physical_page_size * len(
self.kv_cache_config.kv_cache_tensors
)
num_blocks = (
self._logical_num_blocks
if isinstance(layer_spec, MambaSpec)
else self.num_blocks
)
# `page_size` accounts for physical blocks, st KVCache is always
# [`num_blocks` * `page_size`]
curr_tensor_size_bytes = num_blocks * physical_page_size
base_addr = cache.data_ptr()
is_mla_region = isinstance(
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
)
if base_addr in seen_base_addresses:
# NOTE (NickLucche) HMA employs memory pooling to share tensors
# across groups. This results in skipping all tensors but the ones
# pointed to by group0. Also, generally we will have more blocks
# per tensor but fewer regions.
# A shared tensor may back both SSM and attention layers (e.g.
# KDA+MLA in KimiLinear); the region's FA view is MLA whichever
# layer registered it first.
idx = seen_base_addresses.index(base_addr)
self._region_is_mla[idx] |= is_mla_region
logger.debug("Skipping %s because it's already seen", layer_name)
continue
logger.debug(
"Registering layer %s with cache shape: %s", layer_name, cache.shape
)
seen_base_addresses.append(base_addr)
# Only record non-Mamba page sizes.
if isinstance(layer_spec, MambaSpec):
self.block_len_per_layer.append(
physical_page_size // self._physical_blocks_per_logical_kv_block
)
else:
self.block_len_per_layer.append(physical_page_size)
self._region_is_mla.append(is_mla_region)
storage = cache.untyped_storage()
storage_addr = storage.data_ptr()
# Memory registration follows allocations, while transfer regions follow
# logical layers (or contiguous head segments). This keeps strided
# cross-layer views inside their registered allocation.
if storage_addr not in seen_storage_addresses:
seen_storage_addresses.add(storage_addr)
self.device_id = max(cache.get_device(), 0)
caches_data.append((storage_addr, storage.nbytes(), self.device_id, ""))
if not is_mla_region:
if tensor_size_bytes is None:
tensor_size_bytes = curr_tensor_size_bytes
assert tensor_size_bytes == curr_tensor_size_bytes, (
"All non-MLA kv cache tensors must have the same size"
is_mla_region = isinstance(
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
)
if isinstance(layer_spec, MambaSpec):
region_specs = [
(
cache.data_ptr(),
physical_page_size
// self._physical_blocks_per_logical_kv_block,
physical_page_size
// self._physical_blocks_per_logical_kv_block,
)
]
else:
if cache.ndim == 1:
# Flat byte view: HMA tensors shared between layer types carry
# no block dimension, so stride(0) is 1 byte. Blocks abut, so
# the stride is the page size.
block_stride = physical_page_size
else:
block_stride = cache.stride(0) * cache.element_size()
storage_is_block_major = num_blocks * block_stride == storage.nbytes()
# A layer whose [H, N, C] interior is dense addresses its own page
# as one chunk, so it can own a region. When it does not, or when
# every layer is packed into this one storage, the block's whole
# row is the only contiguous unit.
hnc_contiguous = (
cache.ndim == 4
and cache.stride(2) == cache.shape[3]
and cache.stride(1) == cache.shape[2] * cache.shape[3]
)
if storage_is_block_major and (
(packed_storage and is_mla_region) or not hnc_contiguous
):
# TODO(Lucas): handle TP slicing for packed_storage; for now
# restrict to MLA (DSv4) where kv is replicated.
storage_block_len = storage.nbytes() // num_blocks
region_specs = [
(storage_addr, storage_block_len, storage_block_len)
]
elif storage_is_block_major:
region_specs = [
(cache.data_ptr(), physical_page_size, block_stride)
]
else:
segment_bytes = num_blocks * block_stride
assert cache.nbytes % segment_bytes == 0
num_segments = cache.nbytes // segment_bytes
region_block_len = (
block_stride if num_segments > 1 else physical_page_size
)
region_specs = [
(
cache.data_ptr() + segment_idx * segment_bytes,
region_block_len,
block_stride,
)
for segment_idx in range(num_segments)
]
for base_addr, block_len, block_stride in region_specs:
if base_addr in seen_base_addresses:
if is_mla_region:
# Dual-purpose HMA tensor: an MLA layer shares a region
# that a non-MLA layer registered first. MLA is not
# head-sharded, so the region must be flagged MLA.
idx = seen_base_addresses.index(base_addr)
self._region_is_mla[idx] = True
continue
seen_base_addresses.append(base_addr)
self.block_len_per_layer.append(block_len)
self.block_stride_per_layer.append(block_stride)
self._region_is_mla.append(is_mla_region)
# When there's a mismatch between kbs<>bs, we rely on HMA to ensure
# caches are either [NB, PS] or [NB*r, PS/r] where r is bs/kbs.
@@ -1228,11 +1172,6 @@ class NixlBaseConnectorWorker:
f"kv_cache_layout={self.kv_cache_layout}"
)
# Need to make sure the device ID is non-negative for NIXL,
# Torch uses -1 to indicate CPU tensors.
self.device_id = max(cache.get_device(), 0)
caches_data.append((base_addr, curr_tensor_size_bytes, self.device_id, ""))
logger.debug(
"Different block lengths collected: %s", set(self.block_len_per_layer)
)
@@ -1240,10 +1179,11 @@ class NixlBaseConnectorWorker:
len(self.block_len_per_layer)
== len(seen_base_addresses)
== len(self._region_is_mla)
== len(self.block_stride_per_layer)
)
self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses
self.num_regions = len(caches_data)
self.num_regions = len(seen_base_addresses)
if self.pp_size > 1:
start_layer, end_layer = self.model_config.get_layers_start_end_indices(
@@ -1293,6 +1233,7 @@ class NixlBaseConnectorWorker:
kv_caches_base_addr=self.kv_caches_base_addr[self.engine_id][self.tp_rank],
num_blocks=self.num_blocks,
block_lens=self.block_len_per_layer,
block_strides=self.block_stride_per_layer,
kv_cache_layout=self.kv_cache_layout
if not self.use_host_buffer
else self.host_buffer_kv_cache_layout,
@@ -1431,10 +1372,9 @@ class NixlBaseConnectorWorker:
block_arange = np.arange(num_blocks, dtype=np.uint64)
parts: list[np.ndarray] = []
for i, base_addr in enumerate(base_addresses):
# K/V are packed into the content dim, so the whole block transfers
# as one unit: desc length equals the block stride.
block_len = self.block_len_per_layer[i] // block_size_ratio
addrs = base_addr + block_arange * block_len
block_stride = self.block_stride_per_layer[i] // block_size_ratio
addrs = base_addr + block_arange * block_stride
parts.append(self._stack_descs(addrs, block_len, device_id))
return np.concatenate(parts)
@@ -1476,7 +1416,7 @@ class NixlBaseConnectorWorker:
)
local_block_len = local_block_len // num_reads
page_size = nixl_agent_meta.block_lens[i]
page_size = nixl_agent_meta.block_strides[i]
addrs = base_addr + rank_offset + block_arange * page_size
parts.append(self._stack_descs(addrs, local_block_len, device_id))
return np.concatenate(parts)
@@ -1560,7 +1500,7 @@ class NixlBaseConnectorWorker:
tp_ratio = 4 // 2 = 2
Considering the KV Caches, if P-Worker_i has cache size [2, num_blocksP, kv_heads, block_size, head_dim]
then D-Worker_j has [2, num_blocksD, kv_heads//tp_ratio, block_size, head_dim]. Mind the "HND" layout format.
then D-Worker_j has [2, num_blocksD, kv_heads//tp_ratio, block_size, head_dim]. Mind the "LBHNC" layout format.
Assuming num_blocksD >= num_blocksP, D-Worker0 reads from P-Worker0 by preparing the kv_heads//tp_ratio
first heads from all the slots of all the blocks. D-Worker1 will do the same, but reading the second split
along the kv_heads dimension, and so forth until "tp_ratio" D TP workers have pulled from P-Worker0.
@@ -1600,6 +1540,7 @@ class NixlBaseConnectorWorker:
start:end
]
nixl_agent_meta.block_lens = nixl_agent_meta.block_lens[start:end]
nixl_agent_meta.block_strides = nixl_agent_meta.block_strides[start:end]
### Register remote engine in TransferTopology (idempotent).
assert self.transfer_topo is not None
@@ -1790,10 +1731,10 @@ class NixlBaseConnectorWorker:
if not self.use_mla and nixl_agent_meta.kv_cache_layout != kv_cache_layout:
if (
self.kv_transfer_config.enable_permute_local_kv
and nixl_agent_meta.kv_cache_layout == "HND"
and nixl_agent_meta.kv_cache_layout == "LBHNC"
):
logger.info(
"Remote is HND and local is NHD, enabled additional permute "
"Remote is LBHNC and local is LBNHC, enabled additional permute "
"on local device KV."
)
assert not self._is_hma_required, (
@@ -1823,18 +1764,18 @@ class NixlBaseConnectorWorker:
self.enable_heterogeneous_attn_post_process = True
# Heterogeneous TP requires head-splitting, which only works with
# HND layout. MLA and replicated-KV cases don't split on heads.
# Mamba doesn't support heterogeneous TP.
# block-contiguous layouts (e.g. LBHNC). MLA and replicated-KV cases
# don't split on heads. Mamba doesn't support heterogeneous TP.
if (
abs(tp_ratio) != 1
and not self.use_mla
and not self.transfer_topo.is_kv_replicated(remote_engine_id)
and kv_cache_layout != "HND"
and not KVCacheLayout[kv_cache_layout].is_block_contiguous
and not self.enable_permute_local_kv
):
raise RuntimeError(
"Heterogeneous TP head-dimension splitting requires contiguous heads. "
"Use HND layout on the prefill side."
"Use a block-contiguous layout (e.g. LBHNC) on the prefill side."
)
# Per-region block_len validation enforcing the P/D invariant.
@@ -19,7 +19,6 @@ import torch
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.utils import (
EngineId,
get_current_attn_backend,
)
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
CopyBlocksOp,
@@ -56,10 +55,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
)
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
from vllm.v1.attention.backends.utils import get_kv_cache_layout
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import MambaSpec
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
@@ -83,36 +80,6 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
def supports_divergent_local_hybrid_hits(self) -> bool:
return True
@property
def prefer_cross_layer_blocks(self) -> bool:
if any(
[
isinstance(group.kv_cache_spec, MambaSpec)
for group in self.kv_cache_config.kv_cache_groups
]
):
# Hybrid SSM models do not yet support cross-layer layout
return False
backend = get_current_attn_backend(self._vllm_config)
if backend.get_name() not in (
"FLASH_ATTN",
"FLASHINFER",
"TRITON_ATTN",
):
return False
# For now there is no benefit to run cross layers when backend
# does not support on HND
if get_kv_cache_layout() != "HND":
return False
extra_config = self.kv_transfer_config.kv_connector_extra_config
return (
str(extra_config.get("enable_cross_layers_blocks", "False")).lower()
== "true"
)
def __init__(
self,
vllm_config: VllmConfig,
@@ -179,9 +146,10 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
# which fallback to the default behavior.
return None
logger.info_once(
"NixlConnector setting KV cache layout to HND for better xfer performance."
"NixlConnector setting KV cache layout to LBHNC for "
"better xfer performance."
)
return "HND"
return "LBHNC"
############################################################
# Scheduler Side Methods
@@ -266,12 +234,6 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def register_cross_layers_kv_cache(
self, kv_cache: torch.Tensor, attn_backend: type[AttentionBackend]
):
assert self.connector_worker is not None
self.connector_worker.register_cross_layers_kv_caches(kv_cache)
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
assert self.connector_worker is not None
self.connector_worker.set_host_xfer_buffer_ops(copy_operation)
@@ -44,8 +44,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
# 6: Validate EAGLE/MTP speculative configuration compatibility
# 7: Include NIXL transfer mode (push vs pull) in the compatibility hash
# 8: Add dcp_size and pcp_size to NixlAgentMetadata
# 9: Add block_strides
#
NIXL_CONNECTOR_VERSION: int = 8
NIXL_CONNECTOR_VERSION: int = 9
@dataclass
@@ -56,6 +57,7 @@ class NixlAgentMetadata:
device_id: int
num_blocks: int
block_lens: list[int]
block_strides: list[int]
kv_cache_layout: str
block_size: int
ssm_sizes: tuple[int, int]
@@ -129,7 +131,6 @@ def _get_speculative_compatibility_factors(
def compute_nixl_compatibility_hash(
vllm_config: VllmConfig,
attn_backend_name: str,
cross_layers_blocks: bool,
transfer_mode: str = "pull",
) -> str:
"""
@@ -180,7 +181,6 @@ def compute_nixl_compatibility_hash(
# Attention backend and KV cache dtype affect memory layout
"attn_backend_name": attn_backend_name,
"cache_dtype": str(cache_config.cache_dtype),
"cross_layers_blocks": cross_layers_blocks,
"is_hma_enabled": is_hma_enabled,
"speculative_config": _get_speculative_compatibility_factors(vllm_config),
# push (WRITE) and pull (READ) connectors are protocol-incompatible
@@ -17,6 +17,7 @@ import torch
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheConfig,
KVCacheLayout,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
@@ -31,13 +32,16 @@ if TYPE_CHECKING:
CANONICAL_FORMAT_VERSION = 1
def canonical_format_id() -> str:
def canonical_format_id(kv_cache_layout: str) -> str:
"""Identity of the canonical byte format, for namespacing persisted KV.
Canonical pages keep the worker's KV layout family, so the id couples the
format version with that family; consumers must match it exactly."""
from vllm.v1.attention.backends.utils import get_kv_cache_layout
return f"v{CANONICAL_FORMAT_VERSION}-{get_kv_cache_layout().lower()}"
format version with that family; consumers must match it exactly. The family
keeps its historical NHD/HND spelling so ids stay stable for KV persisted
before the layout enum existed."""
layout = KVCacheLayout[kv_cache_layout]
legacy = {KVCacheLayout.LBNHC: "nhd", KVCacheLayout.LBHNC: "hnd"}
family = legacy.get(layout, layout.name.lower())
return f"v{CANONICAL_FORMAT_VERSION}-{family}"
@dataclass(frozen=True)
@@ -278,7 +282,7 @@ def _layer_mapping(
if isinstance(spec, MLAAttentionSpec):
# TP-replicated latent; CP shards its tokens across the DCP groups
if (
spec.compress_ratio != 1
spec.tokens_per_state != 1
or page % bs
or ctx.tp_size % ctx.dcp_size
or spec.kv_quant_mode.is_per_token_head
@@ -20,12 +20,7 @@ from vllm.v1.kv_offload.config import (
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheTensor
def is_kv_cache_tensor_packed(kv_cache_tensor: "KVCacheTensor") -> bool:
"""Return whether a KV cache tensor uses a packed block stride."""
return bool(kv_cache_tensor.block_stride)
from vllm.v1.kv_cache_interface import KVCacheConfig
def build_offloading_config(
@@ -96,18 +91,10 @@ def build_offloading_config(
blocks_per_chunk = tokens_per_chunk_int // tokens_per_block
worker_kv_bytes_per_block = 0
if kv_cache_config.num_blocks > 0:
packed_tensors = tuple(
is_kv_cache_tensor_packed(tensor)
for tensor in kv_cache_config.kv_cache_tensors
)
is_packed = any(packed_tensors)
assert not is_packed or all(packed_tensors)
total_gpu_kv_bytes = (
kv_cache_config.kv_cache_tensors[0].size
if is_packed
else sum(tensor.size for tensor in kv_cache_config.kv_cache_tensors)
)
if kv_cache_config.num_blocks > 0 and kv_cache_config.kv_cache_tensors:
# Every KVCacheTensor describes placement within the same backing allocation,
# so its size is the total, not a per-tensor share.
total_gpu_kv_bytes = kv_cache_config.kv_cache_tensors[0].size
worker_kv_bytes_per_block = total_gpu_kv_bytes // kv_cache_config.num_blocks
single_group_spec = (
@@ -164,7 +151,7 @@ def build_offloading_config(
if isinstance(single_group_spec, FullAttentionSpec):
if type(single_group_spec) is MLAAttentionSpec:
spec_certifiable = (
single_group_spec.compress_ratio == 1
single_group_spec.tokens_per_state == 1
and single_group_spec.real_page_size_bytes
% single_group_spec.block_size
== 0
@@ -219,4 +206,5 @@ def build_offloading_config(
),
replicated_layout=replicated_layout,
canonical_layout=canonical_layout,
kv_cache_layout=vllm_config.cache_config.kv_cache_layout,
)
@@ -14,16 +14,13 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingWorkerMetadata,
ReqId,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
is_kv_cache_tensor_packed,
)
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheConfig,
MambaSpec,
UniformTypeKVCacheSpecs,
group_kernel_blocks,
)
from vllm.v1.kv_offload.base import (
CanonicalKVCacheRef,
@@ -73,16 +70,6 @@ class OffloadingConnectorWorker:
self.vllm_config, kv_cache_config, kv_caches
)
# Packed layouts (e.g. DSv4) set block_stride > 0; their tensors use
# stride(0) as the manager-block stride (equals total_num_bytes_per_block).
# General (non-packed) layouts size the tensor at page_size_bytes per
# manager block, so page_size_bytes is the correct offloading stride.
layer_is_packed: dict[str, bool] = {
ln: is_kv_cache_tensor_packed(kv_tensor)
for kv_tensor in kv_cache_config.kv_cache_tensors
for ln in kv_tensor.shared_by
}
# layer_name -> (num_blocks, page_size_bytes) tensor
tensors_per_block: dict[str, tuple[torch.Tensor, ...]] = {}
# layer_name -> size of (un-padded) page in bytes
@@ -100,63 +87,52 @@ class OffloadingConnectorWorker:
layer_kv_cache_spec = per_layer_specs.get(
layer_name, group_kv_cache_spec
)
if isinstance(layer_kv_cache_spec, AttentionSpec):
layer_kv_cache = kv_caches[layer_name]
assert isinstance(layer_kv_cache, torch.Tensor)
ref = group_kernel_blocks(kv_caches[layer_name], num_blocks)
page = layer_kv_cache_spec.page_size_bytes
elem_size = ref.element_size()
byte_offset = ref.storage_offset() * elem_size
# Packed layouts (e.g. DSv4) interleave layers per block, so the
# attention tensor's stride(0) (the manager-block stride) exceeds
# page_size_bytes; other layouts have stride(0) == page_size_bytes.
block_stride_bytes = (
ref.stride(0) * elem_size
if isinstance(layer_kv_cache_spec, AttentionSpec)
else page
)
tensors_per_block[layer_name] = (
torch.tensor([], dtype=torch.int8, device=ref.device).set_(
ref.untyped_storage(),
byte_offset,
(num_blocks, page),
(block_stride_bytes, 1),
),
)
page_size_bytes[layer_name] = page
page = layer_kv_cache_spec.page_size_bytes
elem_size = layer_kv_cache.element_size()
byte_offset = layer_kv_cache.storage_offset() * elem_size
block_stride_bytes = (
layer_kv_cache.stride(0) * elem_size
if layer_is_packed[layer_name]
else page
)
raw = torch.empty(
0,
dtype=torch.int8,
device=layer_kv_cache.device,
).set_(layer_kv_cache.untyped_storage())
tensors_per_block[layer_name] = (
torch.as_strided(
raw,
(num_blocks, page),
(block_stride_bytes, 1),
byte_offset,
),
)
page_size_bytes[layer_name] = layer_kv_cache_spec.page_size_bytes
if isinstance(layer_kv_cache_spec, AttentionSpec):
unpadded_page_size_bytes[layer_name] = (
layer_kv_cache_spec.unpadded_page_size_bytes
)
elif isinstance(layer_kv_cache_spec, MambaSpec):
layer_kv_cache = kv_caches[layer_name]
assert layer_kv_cache.dtype == torch.int8
tensors_per_block[layer_name] = (
layer_kv_cache.view(
num_blocks, layer_kv_cache_spec.page_size_bytes
),
)
page_size_bytes[layer_name] = layer_kv_cache_spec.page_size_bytes
unpadded_page_size_bytes[layer_name] = replace(
layer_kv_cache_spec, page_size_padded=None
).page_size_bytes
else:
raise NotImplementedError
packed_kv_cache_tensor = next(
# Packed layouts (e.g. DSv4) interleave all layers within each manager
# block: a layer view's block stride exceeds its page size. Offload the
# whole packed block as a single transfer region.
packed_layer_name = next(
(
t
for t in kv_cache_config.kv_cache_tensors
if is_kv_cache_tensor_packed(t) and t.shared_by
layer_name
for layer_name, (tensor,) in tensors_per_block.items()
if tensor.stride(0) != tensor.shape[1]
),
None,
)
if packed_kv_cache_tensor is not None:
(tensor,) = tensors_per_block[packed_kv_cache_tensor.shared_by[0]]
if packed_layer_name is not None:
(tensor,) = tensors_per_block[packed_layer_name]
block_stride = tensor.stride(0)
packed_tensor = tensor.as_strided(
(num_blocks, block_stride),
@@ -176,29 +152,16 @@ class OffloadingConnectorWorker:
block_tensors: list[CanonicalKVCacheTensor] = []
block_data_refs: dict[str, list[CanonicalKVCacheRef]] = defaultdict(list)
for kv_cache_tensor in kv_cache_config.kv_cache_tensors:
# Filter to layers that were actually processed above.
# Packed KV allocation emits KVCacheTensor entries for
# every (tuple_idx, page_size) slot; slots where no group has a
# layer at that index produce an empty shared_by (reserved memory
# with no corresponding model layer).
tensor_layer_names = [
n for n in kv_cache_tensor.shared_by if n in tensors_per_block
]
if not tensor_layer_names:
continue
# Layers that alias the same bytes (cache groups overlay each other) share
# one transfer region: exactly the views with equal address and strides.
aliased_layers: dict[tuple[int, tuple[int, ...]], list[str]] = defaultdict(list)
for layer_name, layer_tensors in tensors_per_block.items():
view = layer_tensors[0]
aliased_layers[(view.data_ptr(), view.stride())].append(layer_name)
# verify all layers in the group reference the exact same tensors
for tensor_layer_names in aliased_layers.values():
assert len({len(tensors_per_block[n]) for n in tensor_layer_names}) == 1
assert (
len({tensors_per_block[n][0].data_ptr() for n in tensor_layer_names})
== 1
)
assert (
len({tensors_per_block[n][0].stride() for n in tensor_layer_names}) == 1
)
# pick the first layer to represent the group
first_layer_name = tensor_layer_names[0]
for tensor in tensors_per_block[first_layer_name]:
block_tensors.append(
@@ -242,53 +205,6 @@ class OffloadingConnectorWorker:
self._init_worker(canonical_kv_caches)
def register_cross_layers_kv_cache(
self, kv_cache: torch.Tensor, attn_backend: type[AttentionBackend]
):
# verify that num_blocks is at physical position 0 in the cross-layers
# tensor layout.
test_shape = attn_backend.get_kv_cache_shape(
num_blocks=1234, block_size=16, num_kv_heads=1, head_size=256
)
num_blocks_logical_dim = test_shape.index(1234) + 1
physical_to_logical = attn_backend.get_kv_cache_stride_order(
include_num_layers_dimension=True
)
num_blocks_physical_dim = physical_to_logical.index(num_blocks_logical_dim)
assert num_blocks_physical_dim == 0
kv_cache_groups = self.kv_cache_config.kv_cache_groups
assert len(kv_cache_groups) == 1
kv_cache_spec = kv_cache_groups[0].kv_cache_spec
num_layers = len(kv_cache_groups[0].layer_names)
page_size_bytes = kv_cache_spec.page_size_bytes * num_layers
assert kv_cache.storage_offset() == 0
storage = kv_cache.untyped_storage()
assert len(storage) % page_size_bytes == 0
num_blocks = len(storage) // page_size_bytes
tensor = (
torch.tensor(
[],
dtype=torch.int8,
device=kv_cache.device,
)
.set_(storage)
.view(num_blocks, page_size_bytes)
)
kv_cache_tensor = CanonicalKVCacheTensor(
tensor=tensor, page_size_bytes=page_size_bytes
)
# in cross layers layout, there's currently only a single group
kv_cache_data_ref = CanonicalKVCacheRef(
tensor_idx=0, page_size_bytes=page_size_bytes
)
canonical_kv_caches = CanonicalKVCaches(
tensors=[kv_cache_tensor], group_data_refs=[[kv_cache_data_ref]]
)
self._init_worker(canonical_kv_caches)
def handle_preemptions(self, kv_connector_metadata: OffloadingConnectorMetadata):
assert self.worker is not None
@@ -37,7 +37,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.worker import (
OffloadingConnectorWorker,
)
from vllm.forward_context import ForwardContext
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
@@ -47,11 +47,6 @@ from vllm.v1.request import Request
class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
@property
def prefer_cross_layer_blocks(self) -> bool:
# Cross-layer slabs have no per-layer refs to certify canonical mappings
return not self._canonical_layout
@property
def requires_kv_delivery(self) -> bool:
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
@@ -91,12 +86,6 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_worker is not None
self.connector_worker.register_kv_caches(kv_caches)
def register_cross_layers_kv_cache(
self, kv_cache: torch.Tensor, attn_backend: type[AttentionBackend]
):
assert self.connector_worker is not None
self.connector_worker.register_cross_layers_kv_cache(kv_cache, attn_backend)
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata):
assert self.connector_worker is not None
assert isinstance(kv_connector_metadata, OffloadingConnectorMetadata)
@@ -196,7 +185,7 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
@classmethod
def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str | None:
return "HND"
return "LBHNC"
def reset_cache(self) -> bool | None:
assert self.connector_scheduler is not None