[3/N][KV-Cache Layout Refactor] Standardize Mamba cache; drop get_transfer_cache_regions (#44456)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Claude
OpenAI Codex
parent
eb44b3aaa4
commit
6700813f86
@@ -21,12 +21,10 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBase
|
||||
from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -594,32 +592,6 @@ class TransferTopology:
|
||||
abs_ratio = -tp_ratio
|
||||
return [self.tp_rank * abs_ratio + i for i in range(abs_ratio)]
|
||||
|
||||
def get_transfer_cache_regions(
|
||||
self, cache: torch.Tensor, layer_spec: "KVCacheSpec"
|
||||
) -> list[torch.Tensor] | torch.Tensor:
|
||||
"""Return the cache tensor(s) to register as NIXL memory regions,
|
||||
also accounting for hybrid SSM models specificities.
|
||||
"""
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
# Register the whole kv cache shared tensor, including
|
||||
# SSM/Conv.
|
||||
conv, ssm = cache
|
||||
return [conv]
|
||||
|
||||
# Check may be hacky but it's matching
|
||||
# `_update_hybrid_attention_mamba_layout`.
|
||||
if self.is_mamba and cache.shape[0] == 2:
|
||||
# When MAMBA is present, all backends are blocks first, so
|
||||
# that blocks can be shared between attention layers and mamba
|
||||
# layers. Runner already adjusted strides for FlashAttn-like
|
||||
# backends so its num_blocks first.
|
||||
# Swap [2<>num_blocks] dims for hybrid SSM layout.
|
||||
cache = cache.transpose(0, 1)
|
||||
|
||||
# K and V are packed into one tensor (content dim), so each layer
|
||||
# registers as a single region.
|
||||
return [cache]
|
||||
|
||||
def describe(self, remote_engine_id: EngineId, remote_pp_rank: int = 0) -> str:
|
||||
"""One-line summary of transfer config for logging."""
|
||||
info = self._engines[(remote_engine_id, remote_pp_rank)]
|
||||
|
||||
@@ -1678,9 +1678,9 @@ class MooncakeConnectorWorker:
|
||||
conv, _ = cache_or_caches
|
||||
cache_list = [conv]
|
||||
else:
|
||||
cache_list = self.transfer_topo.get_transfer_cache_regions(
|
||||
cache_or_caches, layer_spec
|
||||
)
|
||||
# K and V are packed into one blocks-first tensor per layer,
|
||||
# so each layer registers as a single region.
|
||||
cache_list = [cache_or_caches]
|
||||
|
||||
logger.debug(
|
||||
"registering layer %s with %d cache tensor(s)",
|
||||
|
||||
@@ -1092,7 +1092,7 @@ class NixlBaseConnectorWorker:
|
||||
# to better exploit the memory layout (ie num_blocks is the first dim).
|
||||
tensor_size_bytes = None
|
||||
|
||||
for layer_name, cache_or_caches in xfer_buffers.items():
|
||||
for layer_name, cache in xfer_buffers.items():
|
||||
# NOTE (NickLucche) Hybrid SSM models assume a layout that is similar to
|
||||
# that of FI, with block laid out as in `get_backend_aware_kv_block_len`.
|
||||
# However, physical page_size may differ when kernel requires a specific
|
||||
@@ -1109,9 +1109,6 @@ class NixlBaseConnectorWorker:
|
||||
if isinstance(layer_spec, UniformTypeKVCacheSpecs):
|
||||
# MLA DSv32 Indexer case: UniformTypeKVCacheSpecs merges kv_cache_specs
|
||||
layer_spec = layer_spec.kv_cache_specs[layer_name]
|
||||
cache_list = self.transfer_topo.get_transfer_cache_regions(
|
||||
cache_or_caches, layer_spec
|
||||
)
|
||||
# `layer_spec.page_size_bytes` only accounts for logical page_size, that is
|
||||
# the page_size assuming constant `self._logical_num_blocks`.
|
||||
physical_page_size = (
|
||||
@@ -1120,8 +1117,6 @@ class NixlBaseConnectorWorker:
|
||||
else layer_spec.page_size_bytes
|
||||
// self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
# For when registering multiple tensors eg K/V in separate regions.
|
||||
physical_page_size = physical_page_size // len(cache_list)
|
||||
if self.transfer_topo._cross_layers_blocks:
|
||||
# When cross-layers blocks are used, multiply by number of layers
|
||||
physical_page_size = physical_page_size * len(
|
||||
@@ -1136,66 +1131,61 @@ class NixlBaseConnectorWorker:
|
||||
# [`num_blocks` * `page_size`]
|
||||
curr_tensor_size_bytes = num_blocks * physical_page_size
|
||||
|
||||
# TODO (NickLucche) we could eventually unify how we handle FA/FI regions,
|
||||
# registering a single tensor for both K/V and splitting logically like FI.
|
||||
for cache in cache_list:
|
||||
base_addr = cache.data_ptr()
|
||||
if base_addr in seen_base_addresses:
|
||||
# NOTE (NickLucche) HMA employs memory pooling to share tensors
|
||||
# across groups. This results in skipping all tensors but the ones
|
||||
# pointed to by group0. Also, generally we will have more blocks
|
||||
# per tensor but fewer regions.
|
||||
logger.debug("Skipping %s because it's already seen", layer_name)
|
||||
continue
|
||||
logger.debug(
|
||||
"Registering layer %s with cache shape: %s", layer_name, cache.shape
|
||||
base_addr = cache.data_ptr()
|
||||
if base_addr in seen_base_addresses:
|
||||
# NOTE (NickLucche) HMA employs memory pooling to share tensors
|
||||
# across groups. This results in skipping all tensors but the ones
|
||||
# pointed to by group0. Also, generally we will have more blocks
|
||||
# per tensor but fewer regions.
|
||||
logger.debug("Skipping %s because it's already seen", layer_name)
|
||||
continue
|
||||
logger.debug(
|
||||
"Registering layer %s with cache shape: %s", layer_name, cache.shape
|
||||
)
|
||||
seen_base_addresses.append(base_addr)
|
||||
# Only record non-Mamba page sizes.
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
self.block_len_per_layer.append(
|
||||
physical_page_size // self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
seen_base_addresses.append(base_addr)
|
||||
# Only record non-Mamba page sizes.
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
self.block_len_per_layer.append(
|
||||
physical_page_size // self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
else:
|
||||
self.block_len_per_layer.append(physical_page_size)
|
||||
is_mla_region = isinstance(
|
||||
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
|
||||
else:
|
||||
self.block_len_per_layer.append(physical_page_size)
|
||||
is_mla_region = isinstance(
|
||||
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
|
||||
)
|
||||
self._region_is_mla.append(is_mla_region)
|
||||
|
||||
if not is_mla_region:
|
||||
if tensor_size_bytes is None:
|
||||
tensor_size_bytes = curr_tensor_size_bytes
|
||||
assert tensor_size_bytes == curr_tensor_size_bytes, (
|
||||
"All non-MLA kv cache tensors must have the same size"
|
||||
)
|
||||
self._region_is_mla.append(is_mla_region)
|
||||
|
||||
if not is_mla_region:
|
||||
if tensor_size_bytes is None:
|
||||
tensor_size_bytes = curr_tensor_size_bytes
|
||||
assert tensor_size_bytes == curr_tensor_size_bytes, (
|
||||
"All non-MLA kv cache tensors must have the same size"
|
||||
)
|
||||
|
||||
# When there's a mismatch between kbs<>bs, we rely on HMA to ensure
|
||||
# caches are either [NB, PS] or [NB*r, PS/r] where r is bs/kbs.
|
||||
if (
|
||||
self._physical_blocks_per_logical_kv_block == 1
|
||||
and cache.shape[0] != num_blocks
|
||||
):
|
||||
raise AssertionError(
|
||||
"All kv cache tensors must have the same number of "
|
||||
f"blocks; layer={layer_name}, "
|
||||
f"expected_num_blocks={num_blocks}, "
|
||||
f"cache_shape={tuple(cache.shape)}, "
|
||||
f"cache_stride={tuple(cache.stride())}, "
|
||||
f"layer_spec={type(layer_spec).__name__}, "
|
||||
f"backend={self.backend_name}, "
|
||||
"all_backends="
|
||||
f"{[backend.get_name() for backend in self.attn_backends]}, "
|
||||
f"kv_cache_layout={self.kv_cache_layout}"
|
||||
)
|
||||
|
||||
# Need to make sure the device ID is non-negative for NIXL,
|
||||
# Torch uses -1 to indicate CPU tensors.
|
||||
self.device_id = max(cache.get_device(), 0)
|
||||
caches_data.append(
|
||||
(base_addr, curr_tensor_size_bytes, self.device_id, "")
|
||||
# When there's a mismatch between kbs<>bs, we rely on HMA to ensure
|
||||
# caches are either [NB, PS] or [NB*r, PS/r] where r is bs/kbs.
|
||||
if (
|
||||
self._physical_blocks_per_logical_kv_block == 1
|
||||
and cache.shape[0] != num_blocks
|
||||
):
|
||||
raise AssertionError(
|
||||
"All kv cache tensors must have the same number of "
|
||||
f"blocks; layer={layer_name}, "
|
||||
f"expected_num_blocks={num_blocks}, "
|
||||
f"cache_shape={tuple(cache.shape)}, "
|
||||
f"cache_stride={tuple(cache.stride())}, "
|
||||
f"layer_spec={type(layer_spec).__name__}, "
|
||||
f"backend={self.backend_name}, "
|
||||
"all_backends="
|
||||
f"{[backend.get_name() for backend in self.attn_backends]}, "
|
||||
f"kv_cache_layout={self.kv_cache_layout}"
|
||||
)
|
||||
|
||||
# Need to make sure the device ID is non-negative for NIXL,
|
||||
# Torch uses -1 to indicate CPU tensors.
|
||||
self.device_id = max(cache.get_device(), 0)
|
||||
caches_data.append((base_addr, curr_tensor_size_bytes, self.device_id, ""))
|
||||
|
||||
logger.debug(
|
||||
"Different block lengths collected: %s", set(self.block_len_per_layer)
|
||||
)
|
||||
|
||||
@@ -56,9 +56,7 @@ class OffloadingConnectorWorker:
|
||||
def _init_worker(self, kv_caches: CanonicalKVCaches) -> None:
|
||||
self.worker = self.spec.get_worker(kv_caches)
|
||||
|
||||
def register_kv_caches(
|
||||
self, kv_caches: dict[str, torch.Tensor | list[torch.Tensor]]
|
||||
):
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
kv_cache_config = self.kv_cache_config
|
||||
num_blocks = kv_cache_config.num_blocks
|
||||
|
||||
@@ -120,24 +118,13 @@ class OffloadingConnectorWorker:
|
||||
)
|
||||
|
||||
elif isinstance(layer_kv_cache_spec, MambaSpec):
|
||||
state_tensors = kv_caches[layer_name]
|
||||
assert isinstance(state_tensors, list)
|
||||
|
||||
# re-construct the raw (num_blocks, page_size) tensor
|
||||
# from the first state tensor
|
||||
assert len(state_tensors) > 0
|
||||
first_state_tensor = state_tensors[0]
|
||||
assert first_state_tensor.storage_offset() == 0
|
||||
tensor = (
|
||||
torch.tensor(
|
||||
[],
|
||||
dtype=torch.int8,
|
||||
device=first_state_tensor.device,
|
||||
)
|
||||
.set_(first_state_tensor.untyped_storage())
|
||||
.view((num_blocks, layer_kv_cache_spec.page_size_bytes))
|
||||
layer_kv_cache = kv_caches[layer_name]
|
||||
assert layer_kv_cache.dtype == torch.int8
|
||||
tensors_per_block[layer_name] = (
|
||||
layer_kv_cache.view(
|
||||
num_blocks, layer_kv_cache_spec.page_size_bytes
|
||||
),
|
||||
)
|
||||
tensors_per_block[layer_name] = (tensor,)
|
||||
|
||||
page_size_bytes[layer_name] = layer_kv_cache_spec.page_size_bytes
|
||||
unpadded_page_size_bytes[layer_name] = replace(
|
||||
|
||||
Reference in New Issue
Block a user