[6/N][KV-Cache Layout Refactor] Standardize KV cache layout (#51718)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -12,7 +12,7 @@ import torch
|
||||
|
||||
from vllm.config import (
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
get_current_vllm_config_or_none,
|
||||
get_layers_from_vllm_config,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
@@ -35,20 +35,20 @@ EngineId = str
|
||||
BlockIds = tuple[list[int], ...] | list[list[int]]
|
||||
|
||||
|
||||
def get_kv_connector_cache_layout():
|
||||
# NOTE (NickLucche) When running disaggregated PD with NIXL, HND layout is
|
||||
def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
|
||||
# NOTE (NickLucche) When running disaggregated PD with NIXL, LBHNC layout is
|
||||
# used for faster transfer.
|
||||
vllm_config = get_current_vllm_config()
|
||||
if vllm_config is None:
|
||||
vllm_config = get_current_vllm_config_or_none()
|
||||
if vllm_config is None:
|
||||
return None
|
||||
kv_config = vllm_config.kv_transfer_config
|
||||
if kv_config is not None:
|
||||
connector_cls = KVConnectorFactory.get_connector_class(kv_config)
|
||||
required_kvcache_layout = connector_cls.get_required_kvcache_layout(vllm_config)
|
||||
if required_kvcache_layout is not None:
|
||||
return required_kvcache_layout
|
||||
logger.info_once(
|
||||
"Connectors do not specify a kv cache layout, defaulting to NHD."
|
||||
)
|
||||
return "NHD"
|
||||
return None
|
||||
|
||||
|
||||
class KVOutputAggregator:
|
||||
@@ -288,11 +288,11 @@ def kv_postprocess_layout_on_receive(cache, indices):
|
||||
|
||||
def kv_postprocess_blksize_and_layout_on_receive(cache, indices, block_size_ratio):
|
||||
"""
|
||||
Transforms the layout of received KV cache to the local block_size and HND.
|
||||
Transforms the layout of received KV cache to the local block_size and LBHNC.
|
||||
(Only works for local blocksize > remote blocksize)
|
||||
|
||||
prefill is HND, smaller block_size
|
||||
decode(local) is NHD, larger block_size
|
||||
prefill is LBHNC, smaller block_size
|
||||
decode(local) is LBNHC, larger block_size
|
||||
"""
|
||||
blocks_to_update = cache.index_select(0, indices)
|
||||
|
||||
@@ -427,48 +427,6 @@ class TransferTopology:
|
||||
|
||||
self._engines: dict[tuple[EngineId, int], EngineTransferInfo] = {}
|
||||
|
||||
# Figure out whether the first dimension of the cache is K/V
|
||||
# or num_blocks.
|
||||
attn_backend = self.attn_backends[0]
|
||||
if not self.is_mamba:
|
||||
_MOCK_BLOCK_SIZE = 16
|
||||
kv_cache_shape: tuple[int, ...] = attn_backend.get_kv_cache_shape(
|
||||
num_blocks=1,
|
||||
block_size=_MOCK_BLOCK_SIZE,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
)
|
||||
logger.debug("Test kv_cache_shape: %s", kv_cache_shape)
|
||||
assert kv_cache_shape[0] == 1, (
|
||||
"KV cache layout must be blocks-first; expected mocked "
|
||||
f"num_blocks=1 in leading dim, got shape {kv_cache_shape}."
|
||||
)
|
||||
if not self.is_mla:
|
||||
assert len(kv_cache_shape) == 4, (
|
||||
"Attention KV cache layout must be standardized as "
|
||||
"[num_blocks, num_kv_heads, block_size, content_size], "
|
||||
f"got shape {kv_cache_shape}."
|
||||
)
|
||||
|
||||
self._cross_layers_blocks = False
|
||||
if self.tensor_shape is not None:
|
||||
self._cross_layers_blocks = (
|
||||
len(self.tensor_shape) == len(kv_cache_shape) + 1
|
||||
)
|
||||
|
||||
if self._cross_layers_blocks:
|
||||
logger.debug("Using cross-layer KV cache")
|
||||
_MOCK_NUM_LAYERS = 80
|
||||
kv_cache_shape = (_MOCK_NUM_LAYERS,) + kv_cache_shape
|
||||
try:
|
||||
kv_cache_stride_order = attn_backend.get_kv_cache_stride_order(
|
||||
include_num_layers_dimension=self._cross_layers_blocks
|
||||
)
|
||||
except (AttributeError, NotImplementedError):
|
||||
assert self.tensor_shape is not None
|
||||
kv_cache_stride_order = tuple(range(len(self.tensor_shape)))
|
||||
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
|
||||
|
||||
# ============================================================
|
||||
# Engine registration
|
||||
# ============================================================
|
||||
@@ -503,24 +461,6 @@ class TransferTopology:
|
||||
for key in [k for k in self._engines if k[0] == remote_engine_id]:
|
||||
del self._engines[key]
|
||||
|
||||
# ============================================================
|
||||
# Layout properties
|
||||
# ============================================================
|
||||
|
||||
@property
|
||||
def cross_layers_blocks(self) -> bool:
|
||||
return self._cross_layers_blocks
|
||||
|
||||
@property
|
||||
def virtually_split_kv_in_blocks(self) -> bool:
|
||||
# Whether to logically split each block into two separately-indexable
|
||||
# sub-regions. With K and V packed into the content dim, an attention
|
||||
# block transfers as a single unit — no K/V sub-split is needed. Only
|
||||
# Mamba still needs this, to index its two state regions (conv/ssm)
|
||||
# separately. Not applicable to cross-layer blocks (per-layer
|
||||
# interleaving means a simple half-split does not separate the parts).
|
||||
return self.is_mamba and not self._cross_layers_blocks
|
||||
|
||||
# ============================================================
|
||||
# Common methods
|
||||
# ============================================================
|
||||
|
||||
@@ -48,7 +48,7 @@ from typing import TYPE_CHECKING, Any, Literal
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
@@ -182,14 +182,6 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return False
|
||||
|
||||
@property
|
||||
def prefer_cross_layer_blocks(self) -> bool:
|
||||
"""
|
||||
Indicates whether this connector prefers KV blocks that hold KV data for all
|
||||
layers, which can speed up KV data transfers. Defaults to False.
|
||||
"""
|
||||
return False
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
"""Whether this connector hands off KV that must be reliably delivered.
|
||||
@@ -279,23 +271,6 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def register_cross_layers_kv_cache(
|
||||
self, kv_cache: torch.Tensor, attn_backend: type["AttentionBackend"]
|
||||
):
|
||||
"""
|
||||
Initialize with a single KV cache tensor used by all layers.
|
||||
The first dimension should be num_layers.
|
||||
This function will only be called for models with uniform layers,
|
||||
and only if the prefers_cross_layer_blocks is set to True.
|
||||
Only one of the functions
|
||||
{register_kv_caches, register_cross_layers_kv_cache} will be called.
|
||||
|
||||
Args:
|
||||
kv_cache: a cross-layers kv cache tensor
|
||||
attn_backend: The attention backend that corresponds to all layers
|
||||
"""
|
||||
return
|
||||
|
||||
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
|
||||
"""
|
||||
Set the xPU-specific ops for copying KV between host and device.
|
||||
|
||||
@@ -129,20 +129,17 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
"""Inject the KV cache into the layer.
|
||||
|
||||
Args:
|
||||
dst_kv_cache_layer (torch.Tensor): the destination KV cache
|
||||
layer. In shape [num_pages, page_size, xxx] for MLA,
|
||||
[num_pages, 2, page_size, xxx] otherwise.
|
||||
dst_kv_cache_layer (torch.Tensor): the destination KV cache layer,
|
||||
a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
|
||||
src_kv_cache (torch.Tensor): the source KV cache.
|
||||
slot_mapping (torch.Tensor): the slot mapping. In shape
|
||||
[num_tokens].
|
||||
"""
|
||||
slot_mapping = slot_mapping.to(dst_kv_cache_layer.device, non_blocking=True)
|
||||
if isinstance(attn_metadata, MLACommonMetadata):
|
||||
dst_kv_cache_layer_shape = dst_kv_cache_layer.shape
|
||||
num_pages = dst_kv_cache_layer_shape[0]
|
||||
page_size = dst_kv_cache_layer_shape[1]
|
||||
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
|
||||
dst_kv_cache_layer = dst_kv_cache_layer.reshape(
|
||||
num_pages * page_size, -1
|
||||
-1, dst_kv_cache_layer.shape[-1]
|
||||
)
|
||||
dst_kv_cache_layer[slot_mapping, ...] = src_kv_cache
|
||||
else:
|
||||
@@ -225,13 +222,12 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
) -> torch.Tensor:
|
||||
"""Extract the KV cache from the layer.
|
||||
|
||||
Assume the shape of the layer is (num_pages, page_size, xxx)
|
||||
for MLA, and (num_pages, 2, page_size, xxx) otherwise.
|
||||
The layer is a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
|
||||
"""
|
||||
slot_mapping = slot_mapping.to(layer.device, non_blocking=True)
|
||||
if isinstance(attn_metadata, MLACommonMetadata):
|
||||
num_pages, page_size = layer.shape[0], layer.shape[1]
|
||||
return layer.reshape(num_pages * page_size, -1)[slot_mapping, ...]
|
||||
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
|
||||
return layer.reshape(-1, layer.shape[-1])[slot_mapping, ...]
|
||||
block_idxs = slot_mapping // self._block_size
|
||||
offsets = slot_mapping % self._block_size
|
||||
return layer[block_idxs, :, offsets]
|
||||
|
||||
@@ -38,8 +38,10 @@ def extract_from_kv_cache(
|
||||
num_tokens: int,
|
||||
) -> torch.Tensor:
|
||||
"""Extract data from KV cache."""
|
||||
block_size = kv_cache.shape[1]
|
||||
return kv_cache[slot_mapping // block_size, slot_mapping % block_size][:num_tokens]
|
||||
block_size = kv_cache.shape[2]
|
||||
return kv_cache[slot_mapping // block_size, :, slot_mapping % block_size][
|
||||
:num_tokens
|
||||
]
|
||||
|
||||
|
||||
def load_hidden_states(path: str) -> dict[str, torch.Tensor]:
|
||||
@@ -100,15 +102,6 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
Must be used in conjunction with the `extract_hidden_states` spec decoding method.
|
||||
"""
|
||||
|
||||
@property
|
||||
def prefer_cross_layer_blocks(self) -> bool:
|
||||
"""
|
||||
Indicates whether this connector prefers KV blocks that hold KV data for all
|
||||
layers, which can speed up KV data transfers. Defaults to False.
|
||||
"""
|
||||
# Must be False so that drafter kv cache isn't merged with verifier's
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _find_cache_kv_group_id(cls, kv_cache_config: "KVCacheConfig | None") -> int:
|
||||
"""Index of the KV cache group holding the extracted hidden states.
|
||||
@@ -310,10 +303,12 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
# Block size must match the indexed buffer, else reads hit the wrong
|
||||
# slots. Raise (not assert) so the check survives `python -O`.
|
||||
if self._block_size != self._kv_cache.shape[1]:
|
||||
# Views are [num_blocks, num_heads, block_size, head_size], matching what
|
||||
# extract_from_kv_cache() indexes.
|
||||
if self._block_size != self._kv_cache.shape[2]:
|
||||
raise ValueError(
|
||||
f"Hidden-states block-size mismatch: derived {self._block_size} "
|
||||
f"but buffer block size is {self._kv_cache.shape[1]}; read slots "
|
||||
f"but buffer block size is {self._kv_cache.shape[2]}; read slots "
|
||||
"would be wrong (likely a hybrid block-size resolution bug)."
|
||||
)
|
||||
|
||||
@@ -605,9 +600,9 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"get_required_kvcache_layout should not be called "
|
||||
"on the abstract base class"
|
||||
)
|
||||
# NHD means we have (num_tokens, num_heads)
|
||||
# HND means we have (num_heads, num_tokens)
|
||||
# For now, we only support NHD layout since this keeps the
|
||||
# LBNHC means we have (num_tokens, num_heads)
|
||||
# LBHNC means we have (num_heads, num_tokens)
|
||||
# For now, we only support LBNHC layout since this keeps the
|
||||
# hidden states for each token together in memory.
|
||||
# HND is primarily used when sharding heads across devices.
|
||||
return "NHD"
|
||||
# LBHNC is primarily used when sharding heads across devices.
|
||||
return "LBNHC"
|
||||
|
||||
@@ -541,35 +541,32 @@ class HF3FSKVConnector(KVConnectorBase_V1):
|
||||
self._dtype = first_cache.dtype
|
||||
element_size = first_cache.element_size()
|
||||
|
||||
if self._use_mla:
|
||||
assert len(first_cache.shape) == 3, "MLA format should have 3 dimensions"
|
||||
# MLA format: [num_blocks, block_size, head_size]
|
||||
num_blocks, block_size, head_size = first_cache.shape
|
||||
num_heads = 1
|
||||
else:
|
||||
# MHA format: [2, num_blocks, block_size, num_heads, head_size]
|
||||
_, num_blocks, block_size, num_heads, head_size = first_cache.shape
|
||||
# Standardized per-layer [B, H, N, C] view; MLA is just H == 1 with the
|
||||
# latent vector as the content dim.
|
||||
assert len(first_cache.shape) == 4, (
|
||||
f"expected a [B, H, N, C] KV cache view, got {tuple(first_cache.shape)}"
|
||||
)
|
||||
num_blocks, num_heads, block_size, content_dim = first_cache.shape
|
||||
cache_strides = first_cache.stride()
|
||||
assert all(
|
||||
cache.shape == first_cache.shape and cache.stride() == cache_strides
|
||||
for cache in self._kv_caches.values()
|
||||
), "HF3FS requires uniform KV cache shapes and strides."
|
||||
|
||||
self._local_total_tokens = num_blocks * block_size
|
||||
self._local_block_size = block_size
|
||||
self._num_heads = num_heads
|
||||
self._content_dim = content_dim
|
||||
self._kv_cache_strides = cache_strides
|
||||
|
||||
if self._use_mla:
|
||||
layer_block_size = block_size * head_size * element_size
|
||||
self._bytes_per_page = layer_block_size * len(self._kv_caches)
|
||||
self._shape_per_page = [
|
||||
len(self._kv_caches),
|
||||
block_size,
|
||||
head_size,
|
||||
]
|
||||
else:
|
||||
layer_block_size = 2 * block_size * num_heads * head_size * element_size
|
||||
self._bytes_per_page = layer_block_size * len(self._kv_caches)
|
||||
self._shape_per_page = [
|
||||
len(self._kv_caches),
|
||||
2,
|
||||
block_size,
|
||||
num_heads * head_size,
|
||||
]
|
||||
layer_block_size = num_heads * block_size * content_dim * element_size
|
||||
self._bytes_per_page = layer_block_size * len(self._kv_caches)
|
||||
self._shape_per_page = [
|
||||
len(self._kv_caches),
|
||||
num_heads,
|
||||
block_size,
|
||||
content_dim,
|
||||
]
|
||||
|
||||
self._kvcache_ptrs = torch.tensor(
|
||||
[cache.data_ptr() for cache in self._kv_caches.values()],
|
||||
@@ -993,7 +990,10 @@ class HF3FSKVConnector(KVConnectorBase_V1):
|
||||
self._local_total_tokens,
|
||||
buffer_tensor,
|
||||
token_indices,
|
||||
is_mla=self._use_mla,
|
||||
self._local_block_size,
|
||||
self._num_heads,
|
||||
self._content_dim,
|
||||
self._kv_cache_strides,
|
||||
)
|
||||
else:
|
||||
gather_scatter_helper.scatter_kv_caches(
|
||||
@@ -1001,7 +1001,10 @@ class HF3FSKVConnector(KVConnectorBase_V1):
|
||||
self._local_total_tokens,
|
||||
buffer_tensor,
|
||||
token_indices,
|
||||
is_mla=self._use_mla,
|
||||
self._local_block_size,
|
||||
self._num_heads,
|
||||
self._content_dim,
|
||||
self._kv_cache_strides,
|
||||
)
|
||||
|
||||
def _compute_prefix_hash(
|
||||
|
||||
+92
-121
@@ -12,10 +12,15 @@ def kv_cache_scatter_kernel(
|
||||
source_ptr,
|
||||
token_indices_ptr,
|
||||
num_tokens_in_block,
|
||||
hidden_size,
|
||||
total_token_in_kvcache,
|
||||
num_layers,
|
||||
is_mla,
|
||||
tokens_per_block,
|
||||
block_stride,
|
||||
head_stride,
|
||||
token_stride,
|
||||
content_stride,
|
||||
num_heads: tl.constexpr,
|
||||
content_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
layer_idx = tl.program_id(0)
|
||||
@@ -30,39 +35,25 @@ def kv_cache_scatter_kernel(
|
||||
if token_idx >= total_token_in_kvcache:
|
||||
return
|
||||
|
||||
if is_mla:
|
||||
# MLA format: source [num_layers, num_tokens_in_block, hidden_size]
|
||||
# MLA format: target [total_token_in_kvcache, hidden_size] (per layer)
|
||||
source_offset = (layer_idx * num_tokens_in_block + token_pos) * hidden_size
|
||||
target_offset = token_idx * hidden_size
|
||||
|
||||
for i in range(0, hidden_size, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < hidden_size
|
||||
val = tl.load(source_ptr + source_offset + offset, mask=mask)
|
||||
tl.store(kv_cache_ptr + target_offset + offset, val, mask=mask)
|
||||
else:
|
||||
# MHA format: source [num_layers, 2, num_tokens_in_block, hidden_size]
|
||||
# MHA format: target [2, total_token_in_kvcache, hidden_size]
|
||||
source_offset_k = (
|
||||
layer_idx * num_tokens_in_block * 2 + token_pos
|
||||
) * hidden_size
|
||||
source_offset_v = (
|
||||
layer_idx * num_tokens_in_block * 2 + num_tokens_in_block + token_pos
|
||||
) * hidden_size
|
||||
|
||||
target_offset_k = token_idx * hidden_size
|
||||
target_offset_v = (total_token_in_kvcache + token_idx) * hidden_size
|
||||
|
||||
for i in range(0, hidden_size, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < hidden_size
|
||||
|
||||
val_k = tl.load(source_ptr + source_offset_k + offset, mask=mask)
|
||||
val_v = tl.load(source_ptr + source_offset_v + offset, mask=mask)
|
||||
|
||||
tl.store(kv_cache_ptr + target_offset_k + offset, val_k, mask=mask)
|
||||
tl.store(kv_cache_ptr + target_offset_v + offset, val_v, mask=mask)
|
||||
block_idx = token_idx // tokens_per_block
|
||||
state_idx = token_idx % tokens_per_block
|
||||
hidden_size = num_heads * content_size
|
||||
for i in range(0, hidden_size, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < hidden_size
|
||||
head_idx = offset // content_size
|
||||
content_idx = offset % content_size
|
||||
source_offset = (
|
||||
(layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos
|
||||
) * content_size + content_idx
|
||||
target_offset = (
|
||||
block_idx * block_stride
|
||||
+ head_idx * head_stride
|
||||
+ state_idx * token_stride
|
||||
+ content_idx * content_stride
|
||||
)
|
||||
value = tl.load(source_ptr + source_offset, mask=mask)
|
||||
tl.store(kv_cache_ptr + target_offset, value, mask=mask)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@@ -71,10 +62,15 @@ def kv_cache_gather_kernel(
|
||||
dst_ptr,
|
||||
token_indices_ptr,
|
||||
num_tokens_in_block,
|
||||
hidden_size,
|
||||
total_token_in_kvcache,
|
||||
num_layers,
|
||||
is_mla,
|
||||
tokens_per_block,
|
||||
block_stride,
|
||||
head_stride,
|
||||
token_stride,
|
||||
content_stride,
|
||||
num_heads: tl.constexpr,
|
||||
content_size: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
layer_idx = tl.program_id(0)
|
||||
@@ -89,37 +85,25 @@ def kv_cache_gather_kernel(
|
||||
if token_idx >= total_token_in_kvcache:
|
||||
return
|
||||
|
||||
if is_mla:
|
||||
# MLA format: source [total_token_in_kvcache, hidden_size] (per layer)
|
||||
# MLA format: dst [num_layers, num_tokens_in_block, hidden_size]
|
||||
kvcache_offset = token_idx * hidden_size
|
||||
dst_offset = (layer_idx * num_tokens_in_block + token_pos) * hidden_size
|
||||
|
||||
for i in range(0, hidden_size, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < hidden_size
|
||||
val = tl.load(kv_cache_ptr + kvcache_offset + offset, mask=mask)
|
||||
tl.store(dst_ptr + dst_offset + offset, val, mask=mask)
|
||||
else:
|
||||
# MHA format: source [2, total_token_in_kvcache, hidden_size]
|
||||
# MHA format: dst [num_layers, 2, num_tokens_in_block, hidden_size]
|
||||
dst_offset_k = (layer_idx * num_tokens_in_block * 2 + token_pos) * hidden_size
|
||||
dst_offset_v = (
|
||||
layer_idx * num_tokens_in_block * 2 + num_tokens_in_block + token_pos
|
||||
) * hidden_size
|
||||
|
||||
kvcache_offset_k = token_idx * hidden_size
|
||||
kvcache_offset_v = (total_token_in_kvcache + token_idx) * hidden_size
|
||||
|
||||
for i in range(0, hidden_size, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < hidden_size
|
||||
|
||||
val_k = tl.load(kv_cache_ptr + kvcache_offset_k + offset, mask=mask)
|
||||
val_v = tl.load(kv_cache_ptr + kvcache_offset_v + offset, mask=mask)
|
||||
|
||||
tl.store(dst_ptr + dst_offset_k + offset, val_k, mask=mask)
|
||||
tl.store(dst_ptr + dst_offset_v + offset, val_v, mask=mask)
|
||||
block_idx = token_idx // tokens_per_block
|
||||
state_idx = token_idx % tokens_per_block
|
||||
hidden_size = num_heads * content_size
|
||||
for i in range(0, hidden_size, BLOCK_SIZE):
|
||||
offset = i + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offset < hidden_size
|
||||
head_idx = offset // content_size
|
||||
content_idx = offset % content_size
|
||||
kvcache_offset = (
|
||||
block_idx * block_stride
|
||||
+ head_idx * head_stride
|
||||
+ state_idx * token_stride
|
||||
+ content_idx * content_stride
|
||||
)
|
||||
dst_offset = (
|
||||
(layer_idx * num_heads + head_idx) * num_tokens_in_block + token_pos
|
||||
) * content_size + content_idx
|
||||
value = tl.load(kv_cache_ptr + kvcache_offset, mask=mask)
|
||||
tl.store(dst_ptr + dst_offset, value, mask=mask)
|
||||
|
||||
|
||||
def scatter_kv_caches(
|
||||
@@ -127,34 +111,33 @@ def scatter_kv_caches(
|
||||
total_token_in_kvcache: int,
|
||||
src_tensor: torch.Tensor,
|
||||
token_indices: list[int],
|
||||
is_mla: bool = False,
|
||||
tokens_per_block: int,
|
||||
num_heads: int,
|
||||
content_size: int,
|
||||
kv_cache_strides: tuple[int, ...],
|
||||
) -> None:
|
||||
"""Scatter KV cache data from source tensor to KV cache storage.
|
||||
|
||||
Args:
|
||||
kv_caches_ptrs: Tensor of KV cache pointers (one per layer)
|
||||
total_token_in_kvcache: Total number of tokens in KV cache
|
||||
src_tensor: Source tensor containing data to scatter
|
||||
- MHA format: [num_layers, 2, num_tokens_in_block, hidden_size]
|
||||
- MLA format: [num_layers, num_tokens_in_block, hidden_size]
|
||||
src_tensor: Source ``[L, H, N, C]`` tensor containing data to scatter
|
||||
token_indices: List of token positions to update
|
||||
is_mla: Whether using MLA model format
|
||||
tokens_per_block: Number of stored states in each cache block
|
||||
num_heads: Size of the H axis
|
||||
content_size: Size of the C axis
|
||||
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
|
||||
"""
|
||||
num_layers = len(kv_caches_ptrs)
|
||||
num_tokens_in_block = len(token_indices)
|
||||
|
||||
if is_mla:
|
||||
# MLA: src_tensor is [num_layers, num_tokens_in_block, hidden_size]
|
||||
assert len(src_tensor.shape) == 3, (
|
||||
f"MLA src_tensor should be 3D, got {src_tensor.shape}"
|
||||
)
|
||||
hidden_size = src_tensor.shape[2]
|
||||
else:
|
||||
# MHA: src_tensor is [num_layers, 2, num_tokens_in_block, hidden_size]
|
||||
assert len(src_tensor.shape) == 4, (
|
||||
f"MHA src_tensor should be 4D, got {src_tensor.shape}"
|
||||
)
|
||||
hidden_size = src_tensor.shape[3]
|
||||
assert src_tensor.shape == (
|
||||
num_layers,
|
||||
num_heads,
|
||||
num_tokens_in_block,
|
||||
content_size,
|
||||
)
|
||||
assert len(kv_cache_strides) == 4
|
||||
|
||||
device = src_tensor.device
|
||||
token_indices_tensor = torch.tensor(
|
||||
@@ -169,10 +152,12 @@ def scatter_kv_caches(
|
||||
src_tensor,
|
||||
token_indices_tensor,
|
||||
num_tokens_in_block,
|
||||
hidden_size,
|
||||
total_token_in_kvcache,
|
||||
num_layers,
|
||||
is_mla,
|
||||
tokens_per_block,
|
||||
*kv_cache_strides,
|
||||
num_heads=num_heads,
|
||||
content_size=content_size,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
@@ -182,49 +167,33 @@ def gather_kv_caches(
|
||||
total_token_in_kvcache: int,
|
||||
dst_tensor: torch.Tensor,
|
||||
token_indices: list[int],
|
||||
is_mla: bool = False,
|
||||
tokens_per_block: int,
|
||||
num_heads: int,
|
||||
content_size: int,
|
||||
kv_cache_strides: tuple[int, ...],
|
||||
) -> None:
|
||||
"""Gather KV cache data from KV cache storage to destination tensor.
|
||||
|
||||
Args:
|
||||
kv_caches_ptrs: Tensor of KV cache pointers (one per layer)
|
||||
total_token_in_kvcache: Total number of tokens in KV cache
|
||||
dst_tensor: Destination tensor to store gathered data
|
||||
- MHA format: [num_layers, 2, num_tokens_in_block, hidden_size]
|
||||
- MLA format: [num_layers, num_tokens_in_block, hidden_size]
|
||||
dst_tensor: Destination ``[L, H, N, C]`` tensor
|
||||
token_indices: List of token positions to gather
|
||||
is_mla: Whether using MLA model format
|
||||
tokens_per_block: Number of stored states in each cache block
|
||||
num_heads: Size of the H axis
|
||||
content_size: Size of the C axis
|
||||
kv_cache_strides: Element strides of each ``[B, H, N, C]`` layer view
|
||||
"""
|
||||
num_layers = kv_caches_ptrs.shape[0]
|
||||
num_tokens_in_block = len(token_indices)
|
||||
|
||||
if is_mla:
|
||||
# MLA: dst_tensor is [num_layers, num_tokens_in_block, hidden_size]
|
||||
assert len(dst_tensor.shape) == 3, (
|
||||
f"MLA dst_tensor should be 3D, got {dst_tensor.shape}"
|
||||
)
|
||||
assert dst_tensor.shape[0] == num_layers, (
|
||||
f"Layer count mismatch: {dst_tensor.shape[0]} vs {num_layers}"
|
||||
)
|
||||
assert dst_tensor.shape[1] == num_tokens_in_block, (
|
||||
f"Token count mismatch: {dst_tensor.shape[1]} vs {num_tokens_in_block}"
|
||||
)
|
||||
hidden_size = dst_tensor.shape[2]
|
||||
else:
|
||||
# MHA: dst_tensor is [num_layers, 2, num_tokens_in_block, hidden_size]
|
||||
assert len(dst_tensor.shape) == 4, (
|
||||
f"MHA dst_tensor should be 4D, got {dst_tensor.shape}"
|
||||
)
|
||||
assert dst_tensor.shape[0] == num_layers, (
|
||||
f"Layer count mismatch: {dst_tensor.shape[0]} vs {num_layers}"
|
||||
)
|
||||
assert dst_tensor.shape[1] == 2, (
|
||||
f"MHA should have 2 (K,V) components, got {dst_tensor.shape[1]}"
|
||||
)
|
||||
assert dst_tensor.shape[2] == num_tokens_in_block, (
|
||||
f"Token count mismatch: {dst_tensor.shape[2]} vs {num_tokens_in_block}"
|
||||
)
|
||||
hidden_size = dst_tensor.shape[3]
|
||||
assert dst_tensor.shape == (
|
||||
num_layers,
|
||||
num_heads,
|
||||
num_tokens_in_block,
|
||||
content_size,
|
||||
)
|
||||
assert len(kv_cache_strides) == 4
|
||||
|
||||
device = dst_tensor.device
|
||||
token_indices_tensor = torch.tensor(
|
||||
@@ -239,10 +208,12 @@ def gather_kv_caches(
|
||||
dst_tensor,
|
||||
token_indices_tensor,
|
||||
num_tokens_in_block,
|
||||
hidden_size,
|
||||
total_token_in_kvcache,
|
||||
num_layers,
|
||||
is_mla,
|
||||
tokens_per_block,
|
||||
*kv_cache_strides,
|
||||
num_heads=num_heads,
|
||||
content_size=content_size,
|
||||
BLOCK_SIZE=BLOCK_SIZE,
|
||||
)
|
||||
|
||||
|
||||
@@ -49,15 +49,15 @@ from vllm.model_executor.models.utils import extract_layer_index
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.network_utils import get_ip, make_zmq_path, make_zmq_socket
|
||||
from vllm.utils.torch_utils import is_non_overlapping_and_dense
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID, get_kv_cache_layout
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.request import RequestStatus
|
||||
@@ -122,9 +122,7 @@ def _expand_transfer_regions(
|
||||
kv_block_lens: list[int],
|
||||
layer_names: list[str],
|
||||
layer_indices: list[int],
|
||||
is_kv_layout_blocks_first: bool,
|
||||
group_indices: list[int] | None = None,
|
||||
split_kv_regions: list[bool] | None = None,
|
||||
) -> list[TransferRegion]:
|
||||
"""Expand registered KV tensors into the regions transferred by Mooncake."""
|
||||
assert (
|
||||
@@ -146,13 +144,6 @@ def _expand_transfer_regions(
|
||||
"Mooncake transfer regions require matching group metadata lengths, "
|
||||
f"got group_indices={len(group_indices)}, layer_names={len(layer_names)}."
|
||||
)
|
||||
if split_kv_regions is None:
|
||||
split_kv_regions = [is_kv_layout_blocks_first] * len(layer_names)
|
||||
assert len(split_kv_regions) == len(layer_names), (
|
||||
"Mooncake transfer regions require matching split metadata, "
|
||||
f"got split_kv_regions={len(split_kv_regions)}, "
|
||||
f"layer_names={len(layer_names)}."
|
||||
)
|
||||
regions: list[TransferRegion] = []
|
||||
for (
|
||||
base_addr,
|
||||
@@ -161,7 +152,6 @@ def _expand_transfer_regions(
|
||||
layer_name,
|
||||
layer_index,
|
||||
group_index,
|
||||
split_kv_region,
|
||||
) in zip(
|
||||
base_addrs,
|
||||
block_lens,
|
||||
@@ -169,7 +159,6 @@ def _expand_transfer_regions(
|
||||
layer_names,
|
||||
layer_indices,
|
||||
group_indices,
|
||||
split_kv_regions,
|
||||
):
|
||||
regions.append(
|
||||
TransferRegion(
|
||||
@@ -181,17 +170,6 @@ def _expand_transfer_regions(
|
||||
group_index=group_index,
|
||||
)
|
||||
)
|
||||
if split_kv_region:
|
||||
regions.append(
|
||||
TransferRegion(
|
||||
layer_name=layer_name,
|
||||
layer_index=layer_index,
|
||||
base_addr=base_addr + kv_block_len,
|
||||
block_len=block_len,
|
||||
kv_block_len=kv_block_len,
|
||||
group_index=group_index,
|
||||
)
|
||||
)
|
||||
return regions
|
||||
|
||||
|
||||
@@ -506,10 +484,10 @@ class MooncakeConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
if vllm_config.model_config.use_mla:
|
||||
return None
|
||||
logger.info_once(
|
||||
"MooncakeConnector setting KV cache layout to HND for "
|
||||
"MooncakeConnector setting KV cache layout to LBHNC for "
|
||||
"heterogeneous TP-safe KV transfer."
|
||||
)
|
||||
return "HND"
|
||||
return "LBHNC"
|
||||
|
||||
############################################################
|
||||
# Scheduler Side Methods
|
||||
@@ -1023,12 +1001,10 @@ class MooncakeConnectorWorker:
|
||||
self._sync_block_size_with_kernel()
|
||||
|
||||
self.attn_backends = get_current_attn_backends(vllm_config)
|
||||
self.kv_cache_layout = get_kv_cache_layout()
|
||||
logger.debug(
|
||||
"Detected attention backends %s",
|
||||
[backend.get_name() for backend in self.attn_backends],
|
||||
)
|
||||
logger.debug("Detected kv cache layout %s", self.kv_cache_layout)
|
||||
|
||||
self._tp_size: dict[EngineId, int] = {self.engine_id: self.tp_size}
|
||||
self._layer_specs: dict[str, KVCacheSpec] = {}
|
||||
@@ -1665,7 +1641,7 @@ class MooncakeConnectorWorker:
|
||||
self.registered_layer_indices = []
|
||||
self.registered_group_indices = []
|
||||
|
||||
for layer_name, cache_or_caches in kv_caches.items():
|
||||
for layer_name, cache in kv_caches.items():
|
||||
layer_index = extract_layer_index(layer_name)
|
||||
layer_spec = self._layer_specs.get(layer_name)
|
||||
if layer_spec is None:
|
||||
@@ -1674,34 +1650,30 @@ class MooncakeConnectorWorker:
|
||||
layer_name,
|
||||
)
|
||||
continue
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
conv, _ = cache_or_caches
|
||||
cache_list = [conv]
|
||||
# One raw page tensor per layer; for Mamba that page holds all the
|
||||
# recurrent states, unpacked only when binding the cache for execution.
|
||||
self._log_debug_cache_registration(layer_name, cache)
|
||||
block_is_contiguous = is_non_overlapping_and_dense(cache[0])
|
||||
if not block_is_contiguous:
|
||||
# Non-block-compact layouts scatter a block across per-head
|
||||
# regions; each region's blocks are contiguous.
|
||||
region_caches = [cache[:, head] for head in range(cache.shape[1])]
|
||||
assert all(
|
||||
is_non_overlapping_and_dense(region[0]) for region in region_caches
|
||||
)
|
||||
else:
|
||||
# K and V are packed into one blocks-first tensor per layer,
|
||||
# so each layer registers as a single region.
|
||||
cache_list = [cache_or_caches]
|
||||
region_caches = [cache]
|
||||
|
||||
logger.debug(
|
||||
"registering layer %s with %d cache tensor(s)",
|
||||
layer_name,
|
||||
len(cache_list),
|
||||
)
|
||||
|
||||
for cache in cache_list:
|
||||
self._log_debug_cache_registration(layer_name, cache)
|
||||
base_addr = cache.data_ptr()
|
||||
block_len = cache.stride(0) * cache.element_size()
|
||||
for region_cache in region_caches:
|
||||
base_addr = region_cache.data_ptr()
|
||||
block_len = region_cache.stride(0) * region_cache.element_size()
|
||||
region_base_addresses.append(base_addr)
|
||||
|
||||
if isinstance(layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)):
|
||||
kv_block_len = layer_spec.page_size_bytes
|
||||
elif self.transfer_topo.virtually_split_kv_in_blocks and not isinstance(
|
||||
layer_spec, MambaSpec
|
||||
):
|
||||
kv_block_len = block_len // 2
|
||||
else:
|
||||
kv_block_len = block_len
|
||||
kv_block_len = (
|
||||
layer_spec.page_size_bytes
|
||||
if isinstance(layer_spec, AttentionSpec) and block_is_contiguous
|
||||
else block_len
|
||||
)
|
||||
self.block_len_per_layer.append(block_len)
|
||||
self.kv_block_len_per_layer.append(kv_block_len)
|
||||
self.registered_layer_names.append(layer_name)
|
||||
@@ -1709,12 +1681,12 @@ class MooncakeConnectorWorker:
|
||||
self.registered_group_indices.append(
|
||||
self._layer_group_indices[layer_name]
|
||||
)
|
||||
storage = cache.untyped_storage()
|
||||
storage_addr = storage.data_ptr()
|
||||
if storage_addr not in seen_storage_ptrs:
|
||||
seen_storage_ptrs.add(storage_addr)
|
||||
kv_data_ptrs.append(storage_addr)
|
||||
kv_data_lens.append(storage.nbytes())
|
||||
storage = cache.untyped_storage()
|
||||
storage_addr = storage.data_ptr()
|
||||
if storage_addr not in seen_storage_ptrs:
|
||||
seen_storage_ptrs.add(storage_addr)
|
||||
kv_data_ptrs.append(storage_addr)
|
||||
kv_data_lens.append(storage.nbytes())
|
||||
|
||||
self.kv_caches_base_addr = region_base_addresses
|
||||
self.seen_base_addresses = kv_data_ptrs
|
||||
@@ -2055,24 +2027,13 @@ class MooncakeConnectorWorker:
|
||||
self._layer_group_indices.get(layer_name, 0)
|
||||
for layer_name in layer_names
|
||||
]
|
||||
split_kv_regions = None
|
||||
if self.transfer_topo.virtually_split_kv_in_blocks:
|
||||
split_kv_regions = [
|
||||
not isinstance(
|
||||
self._layer_specs[layer_name],
|
||||
(MambaSpec, MLAAttentionSpec, SlidingWindowMLASpec),
|
||||
)
|
||||
for layer_name in layer_names
|
||||
]
|
||||
return _expand_transfer_regions(
|
||||
base_addrs=base_addrs,
|
||||
block_lens=block_lens,
|
||||
kv_block_lens=kv_block_lens,
|
||||
layer_names=layer_names,
|
||||
layer_indices=layer_indices,
|
||||
is_kv_layout_blocks_first=self.transfer_topo.virtually_split_kv_in_blocks,
|
||||
group_indices=group_indices,
|
||||
split_kv_regions=split_kv_regions,
|
||||
)
|
||||
|
||||
def _get_sender_transfer_plan(
|
||||
|
||||
@@ -89,14 +89,6 @@ class MooncakeStoreKVEvents(KVConnectorKVEvents):
|
||||
class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""KV connector using MooncakeDistributedStore as shared KV pool."""
|
||||
|
||||
@property
|
||||
def prefer_cross_layer_blocks(self) -> bool:
|
||||
extra_config = self._kv_transfer_config.kv_connector_extra_config
|
||||
return (
|
||||
str(extra_config.get("enable_cross_layers_blocks", "False")).lower()
|
||||
== "true"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _validate_kv_cache_config(
|
||||
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
|
||||
@@ -290,16 +282,6 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def register_cross_layers_kv_cache(
|
||||
self, kv_cache: torch.Tensor, attn_backend: type
|
||||
):
|
||||
assert self.connector_worker is not None
|
||||
assert (
|
||||
self._kv_cache_config is not None
|
||||
and len(self._kv_cache_config.kv_cache_groups) == 1
|
||||
), "Cross-layer KV cache does not supported with hybrid models"
|
||||
self.connector_worker.register_cross_layers_kv_caches(kv_cache)
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
# No-op: loads are issued in get_finished() for compute overlap.
|
||||
pass
|
||||
|
||||
@@ -58,6 +58,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.network_utils import get_ip, make_zmq_socket
|
||||
from vllm.utils.torch_utils import is_non_overlapping_and_dense
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
@@ -72,6 +73,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
group_kernel_blocks,
|
||||
)
|
||||
|
||||
from .metrics import MooncakeStoreConnectorStats
|
||||
@@ -1617,18 +1619,9 @@ class MooncakeStoreWorker:
|
||||
for g_idx, db in enumerate(self.token_dbs)
|
||||
)
|
||||
|
||||
def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None:
|
||||
"""Register a cross-layers KV cache tensor.
|
||||
|
||||
Wraps the unified tensor in a single-entry dict so that the
|
||||
existing stride-based logic in register_kv_caches() produces
|
||||
the correct single-segment result (block_len = page_size * num_layers).
|
||||
"""
|
||||
self.register_kv_caches({"__cross_layer__": kv_cache})
|
||||
|
||||
def register_kv_caches(
|
||||
self,
|
||||
kv_caches: dict[str, torch.Tensor | list[torch.Tensor]],
|
||||
kv_caches: dict[str, torch.Tensor],
|
||||
) -> None:
|
||||
"""Register KV cache tensors and start transfer threads."""
|
||||
if self._capacity_only:
|
||||
@@ -1637,58 +1630,62 @@ class MooncakeStoreWorker:
|
||||
logger.warning("No KV caches to offload.")
|
||||
return
|
||||
|
||||
# Resolve each entry to a representative tensor for storage
|
||||
# deduplication. For attention layers the value is already a tensor;
|
||||
# for Mamba layers it is a list of tensors that all share the same
|
||||
# underlying raw storage, so we take the first one.
|
||||
def _repr_tensor(v: torch.Tensor | list[torch.Tensor]) -> torch.Tensor:
|
||||
assert isinstance(v, torch.Tensor | list)
|
||||
return v if isinstance(v, torch.Tensor) else v[0]
|
||||
|
||||
assert self.cache_config.num_gpu_blocks is not None
|
||||
self.num_blocks = self.cache_config.num_gpu_blocks
|
||||
|
||||
seen_ptrs: set[int] = set()
|
||||
seen_storage_ptrs: set[int] = set()
|
||||
seen_region_ptrs: set[int] = set()
|
||||
addrs: list[int] = []
|
||||
block_lens: list[int] = []
|
||||
|
||||
for value in kv_caches.values():
|
||||
cache = _repr_tensor(value)
|
||||
for cache in kv_caches.values():
|
||||
cache = group_kernel_blocks(cache, self.num_blocks)
|
||||
cache_storage = cache.untyped_storage()
|
||||
base_addr = cache_storage.data_ptr()
|
||||
if base_addr in seen_ptrs:
|
||||
continue
|
||||
seen_ptrs.add(base_addr)
|
||||
region_len = cache_storage.nbytes()
|
||||
|
||||
ret = self.store.register_buffer(base_addr, region_len)
|
||||
if ret != 0:
|
||||
logger.error(
|
||||
"register_buffer failed for addr %#x len %d: %d",
|
||||
base_addr,
|
||||
region_len,
|
||||
ret,
|
||||
)
|
||||
if base_addr not in seen_storage_ptrs:
|
||||
seen_storage_ptrs.add(base_addr)
|
||||
ret = self.store.register_buffer(base_addr, region_len)
|
||||
if ret != 0:
|
||||
logger.error(
|
||||
"register_buffer failed for addr %#x len %d: %d",
|
||||
base_addr,
|
||||
region_len,
|
||||
ret,
|
||||
)
|
||||
|
||||
# Detect layout via stride: a dim whose byte-stride exceeds
|
||||
# page_size_bytes is an outer segment dim (e.g. the K/V dim of
|
||||
# FlashAttn's (2, num_blocks, ...)). FlashInfer/MLA's blocks-
|
||||
# outermost layout has no such dim and yields a single segment.
|
||||
el = cache.element_size()
|
||||
page_size_bytes = region_len // self.num_blocks
|
||||
outer_dims = [
|
||||
d for d in range(cache.ndim) if cache.stride(d) * el > page_size_bytes
|
||||
]
|
||||
if not outer_dims:
|
||||
# Blocks-first layout (FlashInfer / MLA): one segment.
|
||||
if not is_non_overlapping_and_dense(cache[0]):
|
||||
# A block is scattered across per-head regions; each region's
|
||||
# blocks are contiguous.
|
||||
for head_idx in range(cache.shape[1]):
|
||||
head_cache = cache[:, head_idx]
|
||||
assert is_non_overlapping_and_dense(head_cache[0])
|
||||
region_addr = head_cache.data_ptr()
|
||||
if region_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(region_addr)
|
||||
addrs.append(region_addr)
|
||||
block_lens.append(head_cache.stride(0) * head_cache.element_size())
|
||||
elif cache.stride(0) * cache.element_size() * self.num_blocks == region_len:
|
||||
# The block stride spans the whole per-block window
|
||||
# (block-outermost and packed layouts, and single-layer
|
||||
# tensors), which may hold other layers' pages at higher
|
||||
# offsets. Register the storage once as one whole-window
|
||||
# region; per-layer regions would copy overlapping windows and
|
||||
# run past the storage for offset layers.
|
||||
if base_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(base_addr)
|
||||
addrs.append(base_addr)
|
||||
block_lens.append(page_size_bytes)
|
||||
block_lens.append(region_len // self.num_blocks)
|
||||
else:
|
||||
# K/V-first layout (FlashAttn / ROCm): split segments.
|
||||
seg_stride = cache.stride(outer_dims[0]) * el
|
||||
for idx in range(cache.shape[outer_dims[0]]):
|
||||
addrs.append(base_addr + idx * seg_stride)
|
||||
block_lens.append(seg_stride // self.num_blocks)
|
||||
region_addr = cache.data_ptr()
|
||||
if region_addr in seen_region_ptrs:
|
||||
continue
|
||||
seen_region_ptrs.add(region_addr)
|
||||
addrs.append(region_addr)
|
||||
block_lens.append(cache.stride(0) * cache.element_size())
|
||||
|
||||
logger.info(
|
||||
"Registered KV caches: num_groups=%d, num_segments=%d, num_blocks=%d",
|
||||
|
||||
@@ -57,11 +57,6 @@ def is_mla_cache_layer(
|
||||
return isinstance(spec, (MLAAttentionSpec, SlidingWindowMLASpec))
|
||||
|
||||
|
||||
def _content_packed_dim(spec: AttentionSpec) -> int:
|
||||
head_size_v = getattr(spec, "head_size_v", spec.head_size)
|
||||
return spec.head_size + head_size_v
|
||||
|
||||
|
||||
def _spec_dim_matches(value: int, expected: int | None) -> bool:
|
||||
return expected is None or value == expected
|
||||
|
||||
@@ -110,23 +105,6 @@ def get_layer_transfer_geometry(
|
||||
spec = layer_to_spec[layer_name]
|
||||
is_mla_cache = is_mla_cache_layer(layer_to_spec, layer_name)
|
||||
|
||||
if is_mla_cache and len(shape) == 3:
|
||||
num_blocks, block_size, latent_dim = shape
|
||||
slot_size_bytes = latent_dim * element_size
|
||||
block_len = block_size * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[0],
|
||||
local_kv_stride=None,
|
||||
remote_kv_stride=None,
|
||||
transfers_per_block=1,
|
||||
regions_per_block=1,
|
||||
split_kv_regions=False,
|
||||
)
|
||||
|
||||
if not is_mla_cache and len(shape) == 5 and shape[0] == 2:
|
||||
_, num_blocks = shape[:2]
|
||||
kernel_blocks_per_block = 1
|
||||
@@ -230,19 +208,19 @@ def get_layer_transfer_geometry(
|
||||
)
|
||||
|
||||
if (
|
||||
not is_mla_cache
|
||||
and isinstance(spec, AttentionSpec)
|
||||
isinstance(spec, AttentionSpec)
|
||||
and len(shape) == 4
|
||||
and shape[1] == spec.num_kv_heads
|
||||
and shape[2] == spec.block_size
|
||||
and shape[3] == _content_packed_dim(spec)
|
||||
and shape[1] == spec.num_heads
|
||||
and shape[2] == spec.num_states
|
||||
and shape[3] * element_size == spec.state_content_size_bytes
|
||||
):
|
||||
num_blocks, num_kv_heads, block_size, packed_dim = shape
|
||||
slot_size_bytes = num_kv_heads * packed_dim * element_size
|
||||
block_len = block_size * slot_size_bytes
|
||||
# Standardized per-layer [B, H, N, C] view (MLA is just H == 1).
|
||||
num_blocks, num_heads, num_states, content_dim = shape
|
||||
slot_size_bytes = num_heads * content_dim * element_size
|
||||
block_len = num_states * slot_size_bytes
|
||||
return LayerTransferGeometry(
|
||||
num_blocks=num_blocks,
|
||||
block_size=block_size,
|
||||
block_size=spec.block_size,
|
||||
block_len=block_len,
|
||||
slot_size_bytes=slot_size_bytes,
|
||||
block_stride=stride[0],
|
||||
@@ -267,6 +245,16 @@ def iter_layer_registration_regions(
|
||||
) -> list[tuple[torch.Tensor, int]]:
|
||||
geometry = get_layer_transfer_geometry(layer_name, kv_cache, layer_to_spec)
|
||||
region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len
|
||||
if geometry.regions_per_block == 1:
|
||||
# With padded or interleaved pages the block stride exceeds the
|
||||
# meaningful block_len; register the strided span. The span ends with
|
||||
# the last block's meaningful bytes, not a whole stride, so it never
|
||||
# runs past the backing allocation of a strided layer view.
|
||||
block_stride_bytes = geometry.block_stride * kv_cache.element_size()
|
||||
region_len = max(
|
||||
region_len,
|
||||
(geometry.num_blocks - 1) * block_stride_bytes + geometry.block_len,
|
||||
)
|
||||
if geometry.split_kv_regions:
|
||||
return [(cache, region_len) for cache in kv_cache]
|
||||
return [(kv_cache, region_len)]
|
||||
|
||||
@@ -27,7 +27,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
PromMetricT,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
@@ -209,12 +209,6 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
c.supports_divergent_local_hybrid_hits for c in self._connectors
|
||||
)
|
||||
|
||||
@property
|
||||
def prefer_cross_layer_blocks(self) -> bool:
|
||||
if not self._connectors:
|
||||
return False
|
||||
return all(c.prefer_cross_layer_blocks for c in self._connectors)
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
return any(c.requires_kv_delivery for c in self._connectors)
|
||||
@@ -245,13 +239,6 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
)
|
||||
return ret
|
||||
|
||||
def register_cross_layers_kv_cache(
|
||||
self, kv_cache: torch.Tensor, attn_backend: type[AttentionBackend]
|
||||
):
|
||||
# Register on all connectors
|
||||
for c in self._connectors:
|
||||
c.register_cross_layers_kv_cache(kv_cache, attn_backend)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
for c in self._connectors:
|
||||
c.register_kv_caches(kv_caches)
|
||||
|
||||
@@ -71,9 +71,9 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.network_utils import make_zmq_path
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backends.utils import get_kv_cache_layout
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheLayout,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
@@ -89,6 +89,13 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _share_storage_and_block_stride(caches: list[torch.Tensor]) -> bool:
|
||||
"""Return whether all views share storage and a block stride."""
|
||||
block_strides = {cache.stride(0) * cache.element_size() for cache in caches}
|
||||
storage_ptrs = {cache.untyped_storage().data_ptr() for cache in caches}
|
||||
return len(block_strides) == len(storage_ptrs) == 1
|
||||
|
||||
|
||||
class NixlBaseConnectorWorker:
|
||||
"""Base implementation of Worker side methods shared by pull and push."""
|
||||
|
||||
@@ -302,7 +309,6 @@ class NixlBaseConnectorWorker:
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
for layer in group.layer_names
|
||||
}
|
||||
self.hma_group_size = len(kv_cache_config.kv_cache_tensors)
|
||||
|
||||
# ---- Model state (derived from model config) ----
|
||||
mamba_ssm_size = (0, 0)
|
||||
@@ -523,7 +529,9 @@ class NixlBaseConnectorWorker:
|
||||
self.attn_backends = get_current_attn_backends(vllm_config)
|
||||
self.backend_name = self.attn_backends[0].get_name()
|
||||
|
||||
self.kv_cache_layout = get_kv_cache_layout()
|
||||
self.kv_cache_layout = (
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout().name
|
||||
)
|
||||
self.host_buffer_kv_cache_layout = self.kv_cache_layout
|
||||
logger.info(
|
||||
"Detected attention backend(s) %s",
|
||||
@@ -558,6 +566,11 @@ class NixlBaseConnectorWorker:
|
||||
# This is not used for SSM layers, which use the counterpart `mamba_ssm_size`.
|
||||
self.block_len_per_layer = list[int]()
|
||||
|
||||
# Per-region block stride in bytes. Taken from the registered tensor's
|
||||
# stride(0) so it stays correct under layouts that interleave layers
|
||||
# within a block (BLHNC/BHLNC), where stride > block_len.
|
||||
self.block_stride_per_layer = list[int]()
|
||||
|
||||
# Per-engine TP mappings. Generated during handshake.
|
||||
self.tp_mappings: dict[EngineId, TPMapping] = {}
|
||||
|
||||
@@ -788,16 +801,16 @@ class NixlBaseConnectorWorker:
|
||||
if not self.use_mla:
|
||||
assert kv_cache.ndim == 4
|
||||
|
||||
if self.kv_cache_layout == "NHD":
|
||||
if self.kv_cache_layout == "LBNHC":
|
||||
if self.kv_transfer_config.enable_permute_local_kv:
|
||||
logger.info_once(
|
||||
"'enable_permute_local_kv' flag is enabled while "
|
||||
"device KV Layout is NHD. Init host buffer with"
|
||||
" HND to better support Decode/Prefill TP_ratio > 1."
|
||||
"device KV Layout is LBNHC. Init host buffer with"
|
||||
" LBHNC to better support Decode/Prefill TP_ratio > 1."
|
||||
)
|
||||
# Since NHD will not support Decode/Prefill TP_ratio > 1,
|
||||
# Since LBNHC will not support Decode/Prefill TP_ratio > 1,
|
||||
# we can leverage host_buffer for permute.
|
||||
self.host_buffer_kv_cache_layout = "HND"
|
||||
self.host_buffer_kv_cache_layout = "LBHNC"
|
||||
else:
|
||||
# Packed KV layout is logical (B, H, N, 2*D). Allocate
|
||||
# (B, N, H, 2*D) and view it as logical (B, H, N, 2*D)
|
||||
@@ -969,120 +982,9 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
fut.add_done_callback(request_ready)
|
||||
|
||||
def register_cross_layers_kv_caches(self, kv_cache: torch.Tensor) -> None:
|
||||
"""Register a cross-layers KV cache tensor with NIXL.
|
||||
|
||||
`use_uniform_kv_cache()` guarantees a single KV cache group whose
|
||||
layers all share the same `AttentionSpec`, so any layer name from
|
||||
`_layer_specs` yields the correct per-layer spec for `page_size_bytes`.
|
||||
"""
|
||||
first_layer = next(iter(self._layer_specs))
|
||||
# Forwarding a real layer name rather than a synthetic key
|
||||
self.register_kv_caches({first_layer: kv_cache})
|
||||
|
||||
def _register_packed_kv_cache(
|
||||
self,
|
||||
storage: torch.UntypedStorage,
|
||||
) -> None:
|
||||
"""Register a packed KV cache as a single NIXL region.
|
||||
|
||||
The packed allocation interleaves all layers per block, so each
|
||||
block_stride-byte chunk is one logical block. We register 1
|
||||
NIXL region and create 1 descriptor per block.
|
||||
"""
|
||||
self.transfer_topo = TransferTopology(
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.world_size,
|
||||
block_size=self.block_size,
|
||||
engine_id=self.engine_id,
|
||||
is_mla=self.use_mla,
|
||||
total_num_kv_heads=self.model_config.get_total_num_kv_heads(),
|
||||
attn_backends=self.attn_backends,
|
||||
tensor_shape=None,
|
||||
is_mamba=self._has_mamba,
|
||||
)
|
||||
self.compat_hash = compute_nixl_compatibility_hash(
|
||||
self.vllm_config,
|
||||
self.backend_name,
|
||||
self.transfer_topo.cross_layers_blocks,
|
||||
transfer_mode=self._TRANSFER_MODE,
|
||||
)
|
||||
|
||||
total_size = storage.nbytes()
|
||||
block_stride = total_size // self.num_blocks
|
||||
base_addr = storage.data_ptr()
|
||||
device_id = storage.device.index
|
||||
assert device_id is not None
|
||||
|
||||
logger.info(
|
||||
"Registering packed KV cache: total_size=%s, block_stride=%s, "
|
||||
"num_blocks=%s, num_regions=1",
|
||||
total_size,
|
||||
block_stride,
|
||||
self.num_blocks,
|
||||
)
|
||||
|
||||
self.device_id = device_id
|
||||
caches_data = [(base_addr, total_size, self.device_id, "")]
|
||||
|
||||
self.block_len_per_layer = [block_stride]
|
||||
self.num_regions = 1
|
||||
self.num_descs = self.num_blocks
|
||||
self.kv_caches_base_addr[self.engine_id][self.tp_rank] = [base_addr]
|
||||
|
||||
descs = self.nixl_wrapper.get_reg_descs(caches_data, self.nixl_memory_type)
|
||||
self.nixl_wrapper.register_memory(descs, backends=self.nixl_backends)
|
||||
self._registered_descs.append(descs)
|
||||
|
||||
self.dst_num_blocks[self.engine_id] = self.num_blocks
|
||||
|
||||
self.src_xfer_handles_by_block_size[self.block_size], (self.src_blocks_data) = (
|
||||
self.register_local_xfer_handler(self.block_size)
|
||||
)
|
||||
|
||||
agent_metadata = NixlAgentMetadata(
|
||||
engine_id=self.engine_id,
|
||||
agent_metadata=self.nixl_wrapper.get_agent_metadata(),
|
||||
device_id=self.device_id,
|
||||
kv_caches_base_addr=(
|
||||
self.kv_caches_base_addr[self.engine_id][self.tp_rank]
|
||||
),
|
||||
num_blocks=self.num_blocks,
|
||||
block_lens=self.block_len_per_layer,
|
||||
kv_cache_layout=self.kv_cache_layout,
|
||||
block_size=self.block_size,
|
||||
ssm_sizes=self._mamba_ssm_size,
|
||||
attn_backend_name=self.backend_name,
|
||||
physical_blocks_per_logical_kv_block=(
|
||||
self._physical_blocks_per_logical_kv_block
|
||||
),
|
||||
dcp_size=self.dcp_size,
|
||||
pcp_size=self.pcp_size,
|
||||
)
|
||||
assert self.compat_hash is not None
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
self.xfer_handshake_metadata = NixlHandshakePayload(
|
||||
compatibility_hash=self.compat_hash,
|
||||
agent_metadata_bytes=encoder.encode(agent_metadata),
|
||||
)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Register the KV Cache data in nixl."""
|
||||
|
||||
# Detect packed allocation: all tensors are strided views into the
|
||||
# same backing storage (different data_ptr but same storage).
|
||||
# This happens with DSv4-style contiguous per-block packing.
|
||||
if len(kv_caches) > 1 and not self._has_mamba:
|
||||
storage = next(iter(kv_caches.values())).untyped_storage()
|
||||
storage_ptrs = {
|
||||
cache.untyped_storage().data_ptr() for cache in kv_caches.values()
|
||||
}
|
||||
data_ptrs = {cache.data_ptr() for cache in kv_caches.values()}
|
||||
if len(storage_ptrs) == 1 and len(data_ptrs) > 1:
|
||||
self._register_packed_kv_cache(storage)
|
||||
self.device_kv_caches = kv_caches
|
||||
return
|
||||
|
||||
self.transfer_topo = TransferTopology(
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.world_size,
|
||||
@@ -1100,7 +1002,6 @@ class NixlBaseConnectorWorker:
|
||||
self.compat_hash = compute_nixl_compatibility_hash(
|
||||
self.vllm_config,
|
||||
self.backend_name,
|
||||
self.transfer_topo.cross_layers_blocks,
|
||||
transfer_mode=self._TRANSFER_MODE,
|
||||
)
|
||||
|
||||
@@ -1127,14 +1028,14 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
|
||||
caches_data = []
|
||||
# With hybrid allocator, layers can share a kv cache tensor
|
||||
seen_storage_addresses: set[int] = set()
|
||||
seen_base_addresses: list[int] = []
|
||||
|
||||
packed_storage = _share_storage_and_block_stride(list(xfer_buffers.values()))
|
||||
|
||||
# K and V are packed into the content dim, so each attention layer is a
|
||||
# single NIXL region whose block transfers as one unit. Mamba layers instead
|
||||
# register separate conv/ssm sub-regions (see `_build_mamba_local`).
|
||||
tensor_size_bytes = None
|
||||
|
||||
for layer_name, cache in xfer_buffers.items():
|
||||
# NOTE (NickLucche) Hybrid SSM mamba/FA physical page_size may differ when
|
||||
# kernel requires a specific block size. This leads to SSM and FA layers
|
||||
@@ -1159,55 +1060,98 @@ class NixlBaseConnectorWorker:
|
||||
else layer_spec.page_size_bytes
|
||||
// self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
if self.transfer_topo._cross_layers_blocks:
|
||||
# When cross-layers blocks are used, multiply by number of layers
|
||||
physical_page_size = physical_page_size * len(
|
||||
self.kv_cache_config.kv_cache_tensors
|
||||
)
|
||||
num_blocks = (
|
||||
self._logical_num_blocks
|
||||
if isinstance(layer_spec, MambaSpec)
|
||||
else self.num_blocks
|
||||
)
|
||||
# `page_size` accounts for physical blocks, st KVCache is always
|
||||
# [`num_blocks` * `page_size`]
|
||||
curr_tensor_size_bytes = num_blocks * physical_page_size
|
||||
|
||||
base_addr = cache.data_ptr()
|
||||
is_mla_region = isinstance(
|
||||
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
|
||||
)
|
||||
if base_addr in seen_base_addresses:
|
||||
# NOTE (NickLucche) HMA employs memory pooling to share tensors
|
||||
# across groups. This results in skipping all tensors but the ones
|
||||
# pointed to by group0. Also, generally we will have more blocks
|
||||
# per tensor but fewer regions.
|
||||
# A shared tensor may back both SSM and attention layers (e.g.
|
||||
# KDA+MLA in KimiLinear); the region's FA view is MLA whichever
|
||||
# layer registered it first.
|
||||
idx = seen_base_addresses.index(base_addr)
|
||||
self._region_is_mla[idx] |= is_mla_region
|
||||
logger.debug("Skipping %s because it's already seen", layer_name)
|
||||
continue
|
||||
logger.debug(
|
||||
"Registering layer %s with cache shape: %s", layer_name, cache.shape
|
||||
)
|
||||
seen_base_addresses.append(base_addr)
|
||||
# Only record non-Mamba page sizes.
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
self.block_len_per_layer.append(
|
||||
physical_page_size // self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
else:
|
||||
self.block_len_per_layer.append(physical_page_size)
|
||||
self._region_is_mla.append(is_mla_region)
|
||||
storage = cache.untyped_storage()
|
||||
storage_addr = storage.data_ptr()
|
||||
# Memory registration follows allocations, while transfer regions follow
|
||||
# logical layers (or contiguous head segments). This keeps strided
|
||||
# cross-layer views inside their registered allocation.
|
||||
if storage_addr not in seen_storage_addresses:
|
||||
seen_storage_addresses.add(storage_addr)
|
||||
self.device_id = max(cache.get_device(), 0)
|
||||
caches_data.append((storage_addr, storage.nbytes(), self.device_id, ""))
|
||||
|
||||
if not is_mla_region:
|
||||
if tensor_size_bytes is None:
|
||||
tensor_size_bytes = curr_tensor_size_bytes
|
||||
assert tensor_size_bytes == curr_tensor_size_bytes, (
|
||||
"All non-MLA kv cache tensors must have the same size"
|
||||
is_mla_region = isinstance(
|
||||
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
|
||||
)
|
||||
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
region_specs = [
|
||||
(
|
||||
cache.data_ptr(),
|
||||
physical_page_size
|
||||
// self._physical_blocks_per_logical_kv_block,
|
||||
physical_page_size
|
||||
// self._physical_blocks_per_logical_kv_block,
|
||||
)
|
||||
]
|
||||
else:
|
||||
if cache.ndim == 1:
|
||||
# Flat byte view: HMA tensors shared between layer types carry
|
||||
# no block dimension, so stride(0) is 1 byte. Blocks abut, so
|
||||
# the stride is the page size.
|
||||
block_stride = physical_page_size
|
||||
else:
|
||||
block_stride = cache.stride(0) * cache.element_size()
|
||||
storage_is_block_major = num_blocks * block_stride == storage.nbytes()
|
||||
# A layer whose [H, N, C] interior is dense addresses its own page
|
||||
# as one chunk, so it can own a region. When it does not, or when
|
||||
# every layer is packed into this one storage, the block's whole
|
||||
# row is the only contiguous unit.
|
||||
hnc_contiguous = (
|
||||
cache.ndim == 4
|
||||
and cache.stride(2) == cache.shape[3]
|
||||
and cache.stride(1) == cache.shape[2] * cache.shape[3]
|
||||
)
|
||||
if storage_is_block_major and (
|
||||
(packed_storage and is_mla_region) or not hnc_contiguous
|
||||
):
|
||||
# TODO(Lucas): handle TP slicing for packed_storage; for now
|
||||
# restrict to MLA (DSv4) where kv is replicated.
|
||||
storage_block_len = storage.nbytes() // num_blocks
|
||||
region_specs = [
|
||||
(storage_addr, storage_block_len, storage_block_len)
|
||||
]
|
||||
elif storage_is_block_major:
|
||||
region_specs = [
|
||||
(cache.data_ptr(), physical_page_size, block_stride)
|
||||
]
|
||||
else:
|
||||
segment_bytes = num_blocks * block_stride
|
||||
assert cache.nbytes % segment_bytes == 0
|
||||
num_segments = cache.nbytes // segment_bytes
|
||||
region_block_len = (
|
||||
block_stride if num_segments > 1 else physical_page_size
|
||||
)
|
||||
region_specs = [
|
||||
(
|
||||
cache.data_ptr() + segment_idx * segment_bytes,
|
||||
region_block_len,
|
||||
block_stride,
|
||||
)
|
||||
for segment_idx in range(num_segments)
|
||||
]
|
||||
|
||||
for base_addr, block_len, block_stride in region_specs:
|
||||
if base_addr in seen_base_addresses:
|
||||
if is_mla_region:
|
||||
# Dual-purpose HMA tensor: an MLA layer shares a region
|
||||
# that a non-MLA layer registered first. MLA is not
|
||||
# head-sharded, so the region must be flagged MLA.
|
||||
idx = seen_base_addresses.index(base_addr)
|
||||
self._region_is_mla[idx] = True
|
||||
continue
|
||||
seen_base_addresses.append(base_addr)
|
||||
self.block_len_per_layer.append(block_len)
|
||||
self.block_stride_per_layer.append(block_stride)
|
||||
self._region_is_mla.append(is_mla_region)
|
||||
|
||||
# When there's a mismatch between kbs<>bs, we rely on HMA to ensure
|
||||
# caches are either [NB, PS] or [NB*r, PS/r] where r is bs/kbs.
|
||||
@@ -1228,11 +1172,6 @@ class NixlBaseConnectorWorker:
|
||||
f"kv_cache_layout={self.kv_cache_layout}"
|
||||
)
|
||||
|
||||
# Need to make sure the device ID is non-negative for NIXL,
|
||||
# Torch uses -1 to indicate CPU tensors.
|
||||
self.device_id = max(cache.get_device(), 0)
|
||||
caches_data.append((base_addr, curr_tensor_size_bytes, self.device_id, ""))
|
||||
|
||||
logger.debug(
|
||||
"Different block lengths collected: %s", set(self.block_len_per_layer)
|
||||
)
|
||||
@@ -1240,10 +1179,11 @@ class NixlBaseConnectorWorker:
|
||||
len(self.block_len_per_layer)
|
||||
== len(seen_base_addresses)
|
||||
== len(self._region_is_mla)
|
||||
== len(self.block_stride_per_layer)
|
||||
)
|
||||
|
||||
self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses
|
||||
self.num_regions = len(caches_data)
|
||||
self.num_regions = len(seen_base_addresses)
|
||||
|
||||
if self.pp_size > 1:
|
||||
start_layer, end_layer = self.model_config.get_layers_start_end_indices(
|
||||
@@ -1293,6 +1233,7 @@ class NixlBaseConnectorWorker:
|
||||
kv_caches_base_addr=self.kv_caches_base_addr[self.engine_id][self.tp_rank],
|
||||
num_blocks=self.num_blocks,
|
||||
block_lens=self.block_len_per_layer,
|
||||
block_strides=self.block_stride_per_layer,
|
||||
kv_cache_layout=self.kv_cache_layout
|
||||
if not self.use_host_buffer
|
||||
else self.host_buffer_kv_cache_layout,
|
||||
@@ -1431,10 +1372,9 @@ class NixlBaseConnectorWorker:
|
||||
block_arange = np.arange(num_blocks, dtype=np.uint64)
|
||||
parts: list[np.ndarray] = []
|
||||
for i, base_addr in enumerate(base_addresses):
|
||||
# K/V are packed into the content dim, so the whole block transfers
|
||||
# as one unit: desc length equals the block stride.
|
||||
block_len = self.block_len_per_layer[i] // block_size_ratio
|
||||
addrs = base_addr + block_arange * block_len
|
||||
block_stride = self.block_stride_per_layer[i] // block_size_ratio
|
||||
addrs = base_addr + block_arange * block_stride
|
||||
parts.append(self._stack_descs(addrs, block_len, device_id))
|
||||
return np.concatenate(parts)
|
||||
|
||||
@@ -1476,7 +1416,7 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
local_block_len = local_block_len // num_reads
|
||||
|
||||
page_size = nixl_agent_meta.block_lens[i]
|
||||
page_size = nixl_agent_meta.block_strides[i]
|
||||
addrs = base_addr + rank_offset + block_arange * page_size
|
||||
parts.append(self._stack_descs(addrs, local_block_len, device_id))
|
||||
return np.concatenate(parts)
|
||||
@@ -1560,7 +1500,7 @@ class NixlBaseConnectorWorker:
|
||||
tp_ratio = 4 // 2 = 2
|
||||
|
||||
Considering the KV Caches, if P-Worker_i has cache size [2, num_blocksP, kv_heads, block_size, head_dim]
|
||||
then D-Worker_j has [2, num_blocksD, kv_heads//tp_ratio, block_size, head_dim]. Mind the "HND" layout format.
|
||||
then D-Worker_j has [2, num_blocksD, kv_heads//tp_ratio, block_size, head_dim]. Mind the "LBHNC" layout format.
|
||||
Assuming num_blocksD >= num_blocksP, D-Worker0 reads from P-Worker0 by preparing the kv_heads//tp_ratio
|
||||
first heads from all the slots of all the blocks. D-Worker1 will do the same, but reading the second split
|
||||
along the kv_heads dimension, and so forth until "tp_ratio" D TP workers have pulled from P-Worker0.
|
||||
@@ -1600,6 +1540,7 @@ class NixlBaseConnectorWorker:
|
||||
start:end
|
||||
]
|
||||
nixl_agent_meta.block_lens = nixl_agent_meta.block_lens[start:end]
|
||||
nixl_agent_meta.block_strides = nixl_agent_meta.block_strides[start:end]
|
||||
|
||||
### Register remote engine in TransferTopology (idempotent).
|
||||
assert self.transfer_topo is not None
|
||||
@@ -1790,10 +1731,10 @@ class NixlBaseConnectorWorker:
|
||||
if not self.use_mla and nixl_agent_meta.kv_cache_layout != kv_cache_layout:
|
||||
if (
|
||||
self.kv_transfer_config.enable_permute_local_kv
|
||||
and nixl_agent_meta.kv_cache_layout == "HND"
|
||||
and nixl_agent_meta.kv_cache_layout == "LBHNC"
|
||||
):
|
||||
logger.info(
|
||||
"Remote is HND and local is NHD, enabled additional permute "
|
||||
"Remote is LBHNC and local is LBNHC, enabled additional permute "
|
||||
"on local device KV."
|
||||
)
|
||||
assert not self._is_hma_required, (
|
||||
@@ -1823,18 +1764,18 @@ class NixlBaseConnectorWorker:
|
||||
self.enable_heterogeneous_attn_post_process = True
|
||||
|
||||
# Heterogeneous TP requires head-splitting, which only works with
|
||||
# HND layout. MLA and replicated-KV cases don't split on heads.
|
||||
# Mamba doesn't support heterogeneous TP.
|
||||
# block-contiguous layouts (e.g. LBHNC). MLA and replicated-KV cases
|
||||
# don't split on heads. Mamba doesn't support heterogeneous TP.
|
||||
if (
|
||||
abs(tp_ratio) != 1
|
||||
and not self.use_mla
|
||||
and not self.transfer_topo.is_kv_replicated(remote_engine_id)
|
||||
and kv_cache_layout != "HND"
|
||||
and not KVCacheLayout[kv_cache_layout].is_block_contiguous
|
||||
and not self.enable_permute_local_kv
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Heterogeneous TP head-dimension splitting requires contiguous heads. "
|
||||
"Use HND layout on the prefill side."
|
||||
"Use a block-contiguous layout (e.g. LBHNC) on the prefill side."
|
||||
)
|
||||
|
||||
# Per-region block_len validation enforcing the P/D invariant.
|
||||
|
||||
@@ -19,7 +19,6 @@ import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineId,
|
||||
get_current_attn_backend,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
CopyBlocksOp,
|
||||
@@ -56,10 +55,8 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl.stats import (
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
|
||||
from vllm.v1.attention.backends.utils import get_kv_cache_layout
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -83,36 +80,6 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def prefer_cross_layer_blocks(self) -> bool:
|
||||
if any(
|
||||
[
|
||||
isinstance(group.kv_cache_spec, MambaSpec)
|
||||
for group in self.kv_cache_config.kv_cache_groups
|
||||
]
|
||||
):
|
||||
# Hybrid SSM models do not yet support cross-layer layout
|
||||
return False
|
||||
|
||||
backend = get_current_attn_backend(self._vllm_config)
|
||||
if backend.get_name() not in (
|
||||
"FLASH_ATTN",
|
||||
"FLASHINFER",
|
||||
"TRITON_ATTN",
|
||||
):
|
||||
return False
|
||||
|
||||
# For now there is no benefit to run cross layers when backend
|
||||
# does not support on HND
|
||||
if get_kv_cache_layout() != "HND":
|
||||
return False
|
||||
|
||||
extra_config = self.kv_transfer_config.kv_connector_extra_config
|
||||
return (
|
||||
str(extra_config.get("enable_cross_layers_blocks", "False")).lower()
|
||||
== "true"
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
@@ -179,9 +146,10 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# which fallback to the default behavior.
|
||||
return None
|
||||
logger.info_once(
|
||||
"NixlConnector setting KV cache layout to HND for better xfer performance."
|
||||
"NixlConnector setting KV cache layout to LBHNC for "
|
||||
"better xfer performance."
|
||||
)
|
||||
return "HND"
|
||||
return "LBHNC"
|
||||
|
||||
############################################################
|
||||
# Scheduler Side Methods
|
||||
@@ -266,12 +234,6 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def register_cross_layers_kv_cache(
|
||||
self, kv_cache: torch.Tensor, attn_backend: type[AttentionBackend]
|
||||
):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_cross_layers_kv_caches(kv_cache)
|
||||
|
||||
def set_host_xfer_buffer_ops(self, copy_operation: CopyBlocksOp):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.set_host_xfer_buffer_ops(copy_operation)
|
||||
|
||||
@@ -44,8 +44,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
|
||||
# 6: Validate EAGLE/MTP speculative configuration compatibility
|
||||
# 7: Include NIXL transfer mode (push vs pull) in the compatibility hash
|
||||
# 8: Add dcp_size and pcp_size to NixlAgentMetadata
|
||||
# 9: Add block_strides
|
||||
#
|
||||
NIXL_CONNECTOR_VERSION: int = 8
|
||||
NIXL_CONNECTOR_VERSION: int = 9
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -56,6 +57,7 @@ class NixlAgentMetadata:
|
||||
device_id: int
|
||||
num_blocks: int
|
||||
block_lens: list[int]
|
||||
block_strides: list[int]
|
||||
kv_cache_layout: str
|
||||
block_size: int
|
||||
ssm_sizes: tuple[int, int]
|
||||
@@ -129,7 +131,6 @@ def _get_speculative_compatibility_factors(
|
||||
def compute_nixl_compatibility_hash(
|
||||
vllm_config: VllmConfig,
|
||||
attn_backend_name: str,
|
||||
cross_layers_blocks: bool,
|
||||
transfer_mode: str = "pull",
|
||||
) -> str:
|
||||
"""
|
||||
@@ -180,7 +181,6 @@ def compute_nixl_compatibility_hash(
|
||||
# Attention backend and KV cache dtype affect memory layout
|
||||
"attn_backend_name": attn_backend_name,
|
||||
"cache_dtype": str(cache_config.cache_dtype),
|
||||
"cross_layers_blocks": cross_layers_blocks,
|
||||
"is_hma_enabled": is_hma_enabled,
|
||||
"speculative_config": _get_speculative_compatibility_factors(vllm_config),
|
||||
# push (WRITE) and pull (READ) connectors are protocol-incompatible
|
||||
|
||||
@@ -17,6 +17,7 @@ import torch
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
@@ -31,13 +32,16 @@ if TYPE_CHECKING:
|
||||
CANONICAL_FORMAT_VERSION = 1
|
||||
|
||||
|
||||
def canonical_format_id() -> str:
|
||||
def canonical_format_id(kv_cache_layout: str) -> str:
|
||||
"""Identity of the canonical byte format, for namespacing persisted KV.
|
||||
Canonical pages keep the worker's KV layout family, so the id couples the
|
||||
format version with that family; consumers must match it exactly."""
|
||||
from vllm.v1.attention.backends.utils import get_kv_cache_layout
|
||||
|
||||
return f"v{CANONICAL_FORMAT_VERSION}-{get_kv_cache_layout().lower()}"
|
||||
format version with that family; consumers must match it exactly. The family
|
||||
keeps its historical NHD/HND spelling so ids stay stable for KV persisted
|
||||
before the layout enum existed."""
|
||||
layout = KVCacheLayout[kv_cache_layout]
|
||||
legacy = {KVCacheLayout.LBNHC: "nhd", KVCacheLayout.LBHNC: "hnd"}
|
||||
family = legacy.get(layout, layout.name.lower())
|
||||
return f"v{CANONICAL_FORMAT_VERSION}-{family}"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -278,7 +282,7 @@ def _layer_mapping(
|
||||
if isinstance(spec, MLAAttentionSpec):
|
||||
# TP-replicated latent; CP shards its tokens across the DCP groups
|
||||
if (
|
||||
spec.compress_ratio != 1
|
||||
spec.tokens_per_state != 1
|
||||
or page % bs
|
||||
or ctx.tp_size % ctx.dcp_size
|
||||
or spec.kv_quant_mode.is_per_token_head
|
||||
|
||||
@@ -20,12 +20,7 @@ from vllm.v1.kv_offload.config import (
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheTensor
|
||||
|
||||
|
||||
def is_kv_cache_tensor_packed(kv_cache_tensor: "KVCacheTensor") -> bool:
|
||||
"""Return whether a KV cache tensor uses a packed block stride."""
|
||||
return bool(kv_cache_tensor.block_stride)
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
|
||||
def build_offloading_config(
|
||||
@@ -96,18 +91,10 @@ def build_offloading_config(
|
||||
blocks_per_chunk = tokens_per_chunk_int // tokens_per_block
|
||||
|
||||
worker_kv_bytes_per_block = 0
|
||||
if kv_cache_config.num_blocks > 0:
|
||||
packed_tensors = tuple(
|
||||
is_kv_cache_tensor_packed(tensor)
|
||||
for tensor in kv_cache_config.kv_cache_tensors
|
||||
)
|
||||
is_packed = any(packed_tensors)
|
||||
assert not is_packed or all(packed_tensors)
|
||||
total_gpu_kv_bytes = (
|
||||
kv_cache_config.kv_cache_tensors[0].size
|
||||
if is_packed
|
||||
else sum(tensor.size for tensor in kv_cache_config.kv_cache_tensors)
|
||||
)
|
||||
if kv_cache_config.num_blocks > 0 and kv_cache_config.kv_cache_tensors:
|
||||
# Every KVCacheTensor describes placement within the same backing allocation,
|
||||
# so its size is the total, not a per-tensor share.
|
||||
total_gpu_kv_bytes = kv_cache_config.kv_cache_tensors[0].size
|
||||
worker_kv_bytes_per_block = total_gpu_kv_bytes // kv_cache_config.num_blocks
|
||||
|
||||
single_group_spec = (
|
||||
@@ -164,7 +151,7 @@ def build_offloading_config(
|
||||
if isinstance(single_group_spec, FullAttentionSpec):
|
||||
if type(single_group_spec) is MLAAttentionSpec:
|
||||
spec_certifiable = (
|
||||
single_group_spec.compress_ratio == 1
|
||||
single_group_spec.tokens_per_state == 1
|
||||
and single_group_spec.real_page_size_bytes
|
||||
% single_group_spec.block_size
|
||||
== 0
|
||||
@@ -219,4 +206,5 @@ def build_offloading_config(
|
||||
),
|
||||
replicated_layout=replicated_layout,
|
||||
canonical_layout=canonical_layout,
|
||||
kv_cache_layout=vllm_config.cache_config.kv_cache_layout,
|
||||
)
|
||||
|
||||
@@ -14,16 +14,13 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingWorkerMetadata,
|
||||
ReqId,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
|
||||
is_kv_cache_tensor_packed,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheConfig,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
group_kernel_blocks,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCacheRef,
|
||||
@@ -73,16 +70,6 @@ class OffloadingConnectorWorker:
|
||||
self.vllm_config, kv_cache_config, kv_caches
|
||||
)
|
||||
|
||||
# Packed layouts (e.g. DSv4) set block_stride > 0; their tensors use
|
||||
# stride(0) as the manager-block stride (equals total_num_bytes_per_block).
|
||||
# General (non-packed) layouts size the tensor at page_size_bytes per
|
||||
# manager block, so page_size_bytes is the correct offloading stride.
|
||||
layer_is_packed: dict[str, bool] = {
|
||||
ln: is_kv_cache_tensor_packed(kv_tensor)
|
||||
for kv_tensor in kv_cache_config.kv_cache_tensors
|
||||
for ln in kv_tensor.shared_by
|
||||
}
|
||||
|
||||
# layer_name -> (num_blocks, page_size_bytes) tensor
|
||||
tensors_per_block: dict[str, tuple[torch.Tensor, ...]] = {}
|
||||
# layer_name -> size of (un-padded) page in bytes
|
||||
@@ -100,63 +87,52 @@ class OffloadingConnectorWorker:
|
||||
layer_kv_cache_spec = per_layer_specs.get(
|
||||
layer_name, group_kv_cache_spec
|
||||
)
|
||||
if isinstance(layer_kv_cache_spec, AttentionSpec):
|
||||
layer_kv_cache = kv_caches[layer_name]
|
||||
assert isinstance(layer_kv_cache, torch.Tensor)
|
||||
ref = group_kernel_blocks(kv_caches[layer_name], num_blocks)
|
||||
page = layer_kv_cache_spec.page_size_bytes
|
||||
elem_size = ref.element_size()
|
||||
byte_offset = ref.storage_offset() * elem_size
|
||||
# Packed layouts (e.g. DSv4) interleave layers per block, so the
|
||||
# attention tensor's stride(0) (the manager-block stride) exceeds
|
||||
# page_size_bytes; other layouts have stride(0) == page_size_bytes.
|
||||
block_stride_bytes = (
|
||||
ref.stride(0) * elem_size
|
||||
if isinstance(layer_kv_cache_spec, AttentionSpec)
|
||||
else page
|
||||
)
|
||||
tensors_per_block[layer_name] = (
|
||||
torch.tensor([], dtype=torch.int8, device=ref.device).set_(
|
||||
ref.untyped_storage(),
|
||||
byte_offset,
|
||||
(num_blocks, page),
|
||||
(block_stride_bytes, 1),
|
||||
),
|
||||
)
|
||||
page_size_bytes[layer_name] = page
|
||||
|
||||
page = layer_kv_cache_spec.page_size_bytes
|
||||
elem_size = layer_kv_cache.element_size()
|
||||
byte_offset = layer_kv_cache.storage_offset() * elem_size
|
||||
block_stride_bytes = (
|
||||
layer_kv_cache.stride(0) * elem_size
|
||||
if layer_is_packed[layer_name]
|
||||
else page
|
||||
)
|
||||
raw = torch.empty(
|
||||
0,
|
||||
dtype=torch.int8,
|
||||
device=layer_kv_cache.device,
|
||||
).set_(layer_kv_cache.untyped_storage())
|
||||
tensors_per_block[layer_name] = (
|
||||
torch.as_strided(
|
||||
raw,
|
||||
(num_blocks, page),
|
||||
(block_stride_bytes, 1),
|
||||
byte_offset,
|
||||
),
|
||||
)
|
||||
page_size_bytes[layer_name] = layer_kv_cache_spec.page_size_bytes
|
||||
if isinstance(layer_kv_cache_spec, AttentionSpec):
|
||||
unpadded_page_size_bytes[layer_name] = (
|
||||
layer_kv_cache_spec.unpadded_page_size_bytes
|
||||
)
|
||||
|
||||
elif isinstance(layer_kv_cache_spec, MambaSpec):
|
||||
layer_kv_cache = kv_caches[layer_name]
|
||||
assert layer_kv_cache.dtype == torch.int8
|
||||
tensors_per_block[layer_name] = (
|
||||
layer_kv_cache.view(
|
||||
num_blocks, layer_kv_cache_spec.page_size_bytes
|
||||
),
|
||||
)
|
||||
|
||||
page_size_bytes[layer_name] = layer_kv_cache_spec.page_size_bytes
|
||||
unpadded_page_size_bytes[layer_name] = replace(
|
||||
layer_kv_cache_spec, page_size_padded=None
|
||||
).page_size_bytes
|
||||
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
packed_kv_cache_tensor = next(
|
||||
# Packed layouts (e.g. DSv4) interleave all layers within each manager
|
||||
# block: a layer view's block stride exceeds its page size. Offload the
|
||||
# whole packed block as a single transfer region.
|
||||
packed_layer_name = next(
|
||||
(
|
||||
t
|
||||
for t in kv_cache_config.kv_cache_tensors
|
||||
if is_kv_cache_tensor_packed(t) and t.shared_by
|
||||
layer_name
|
||||
for layer_name, (tensor,) in tensors_per_block.items()
|
||||
if tensor.stride(0) != tensor.shape[1]
|
||||
),
|
||||
None,
|
||||
)
|
||||
if packed_kv_cache_tensor is not None:
|
||||
(tensor,) = tensors_per_block[packed_kv_cache_tensor.shared_by[0]]
|
||||
if packed_layer_name is not None:
|
||||
(tensor,) = tensors_per_block[packed_layer_name]
|
||||
block_stride = tensor.stride(0)
|
||||
packed_tensor = tensor.as_strided(
|
||||
(num_blocks, block_stride),
|
||||
@@ -176,29 +152,16 @@ class OffloadingConnectorWorker:
|
||||
|
||||
block_tensors: list[CanonicalKVCacheTensor] = []
|
||||
block_data_refs: dict[str, list[CanonicalKVCacheRef]] = defaultdict(list)
|
||||
for kv_cache_tensor in kv_cache_config.kv_cache_tensors:
|
||||
# Filter to layers that were actually processed above.
|
||||
# Packed KV allocation emits KVCacheTensor entries for
|
||||
# every (tuple_idx, page_size) slot; slots where no group has a
|
||||
# layer at that index produce an empty shared_by (reserved memory
|
||||
# with no corresponding model layer).
|
||||
tensor_layer_names = [
|
||||
n for n in kv_cache_tensor.shared_by if n in tensors_per_block
|
||||
]
|
||||
if not tensor_layer_names:
|
||||
continue
|
||||
# Layers that alias the same bytes (cache groups overlay each other) share
|
||||
# one transfer region: exactly the views with equal address and strides.
|
||||
aliased_layers: dict[tuple[int, tuple[int, ...]], list[str]] = defaultdict(list)
|
||||
for layer_name, layer_tensors in tensors_per_block.items():
|
||||
view = layer_tensors[0]
|
||||
aliased_layers[(view.data_ptr(), view.stride())].append(layer_name)
|
||||
|
||||
# verify all layers in the group reference the exact same tensors
|
||||
for tensor_layer_names in aliased_layers.values():
|
||||
assert len({len(tensors_per_block[n]) for n in tensor_layer_names}) == 1
|
||||
assert (
|
||||
len({tensors_per_block[n][0].data_ptr() for n in tensor_layer_names})
|
||||
== 1
|
||||
)
|
||||
assert (
|
||||
len({tensors_per_block[n][0].stride() for n in tensor_layer_names}) == 1
|
||||
)
|
||||
|
||||
# pick the first layer to represent the group
|
||||
first_layer_name = tensor_layer_names[0]
|
||||
for tensor in tensors_per_block[first_layer_name]:
|
||||
block_tensors.append(
|
||||
@@ -242,53 +205,6 @@ class OffloadingConnectorWorker:
|
||||
|
||||
self._init_worker(canonical_kv_caches)
|
||||
|
||||
def register_cross_layers_kv_cache(
|
||||
self, kv_cache: torch.Tensor, attn_backend: type[AttentionBackend]
|
||||
):
|
||||
# verify that num_blocks is at physical position 0 in the cross-layers
|
||||
# tensor layout.
|
||||
test_shape = attn_backend.get_kv_cache_shape(
|
||||
num_blocks=1234, block_size=16, num_kv_heads=1, head_size=256
|
||||
)
|
||||
num_blocks_logical_dim = test_shape.index(1234) + 1
|
||||
physical_to_logical = attn_backend.get_kv_cache_stride_order(
|
||||
include_num_layers_dimension=True
|
||||
)
|
||||
num_blocks_physical_dim = physical_to_logical.index(num_blocks_logical_dim)
|
||||
assert num_blocks_physical_dim == 0
|
||||
|
||||
kv_cache_groups = self.kv_cache_config.kv_cache_groups
|
||||
assert len(kv_cache_groups) == 1
|
||||
kv_cache_spec = kv_cache_groups[0].kv_cache_spec
|
||||
num_layers = len(kv_cache_groups[0].layer_names)
|
||||
page_size_bytes = kv_cache_spec.page_size_bytes * num_layers
|
||||
|
||||
assert kv_cache.storage_offset() == 0
|
||||
storage = kv_cache.untyped_storage()
|
||||
assert len(storage) % page_size_bytes == 0
|
||||
num_blocks = len(storage) // page_size_bytes
|
||||
tensor = (
|
||||
torch.tensor(
|
||||
[],
|
||||
dtype=torch.int8,
|
||||
device=kv_cache.device,
|
||||
)
|
||||
.set_(storage)
|
||||
.view(num_blocks, page_size_bytes)
|
||||
)
|
||||
kv_cache_tensor = CanonicalKVCacheTensor(
|
||||
tensor=tensor, page_size_bytes=page_size_bytes
|
||||
)
|
||||
# in cross layers layout, there's currently only a single group
|
||||
kv_cache_data_ref = CanonicalKVCacheRef(
|
||||
tensor_idx=0, page_size_bytes=page_size_bytes
|
||||
)
|
||||
canonical_kv_caches = CanonicalKVCaches(
|
||||
tensors=[kv_cache_tensor], group_data_refs=[[kv_cache_data_ref]]
|
||||
)
|
||||
|
||||
self._init_worker(canonical_kv_caches)
|
||||
|
||||
def handle_preemptions(self, kv_connector_metadata: OffloadingConnectorMetadata):
|
||||
assert self.worker is not None
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.worker import (
|
||||
OffloadingConnectorWorker,
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionMetadata
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
@@ -47,11 +47,6 @@ from vllm.v1.request import Request
|
||||
|
||||
|
||||
class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
@property
|
||||
def prefer_cross_layer_blocks(self) -> bool:
|
||||
# Cross-layer slabs have no per-layer refs to certify canonical mappings
|
||||
return not self._canonical_layout
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
|
||||
@@ -91,12 +86,6 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def register_cross_layers_kv_cache(
|
||||
self, kv_cache: torch.Tensor, attn_backend: type[AttentionBackend]
|
||||
):
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_cross_layers_kv_cache(kv_cache, attn_backend)
|
||||
|
||||
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata):
|
||||
assert self.connector_worker is not None
|
||||
assert isinstance(kv_connector_metadata, OffloadingConnectorMetadata)
|
||||
@@ -196,7 +185,7 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str | None:
|
||||
return "HND"
|
||||
return "LBHNC"
|
||||
|
||||
def reset_cache(self) -> bool | None:
|
||||
assert self.connector_scheduler is not None
|
||||
|
||||
Reference in New Issue
Block a user