[DSv4] Pack KV caches into contiguous per-block allocations for DeepSeek V4 (#44577)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
Matthew Bonanni
Lucas Wilkinson
Lucas Wilkinson
OpenAI Codex
parent
b9a7cd464c
commit
01192139bf
@@ -841,8 +841,106 @@ class NixlBaseConnectorWorker:
|
||||
# Forwarding a real layer name rather than a synthetic key
|
||||
self.register_kv_caches({first_layer: kv_cache})
|
||||
|
||||
def _register_packed_kv_cache(
|
||||
self,
|
||||
storage: torch.UntypedStorage,
|
||||
) -> None:
|
||||
"""Register a packed KV cache as a single NIXL region.
|
||||
|
||||
The packed allocation interleaves all layers per block, so each
|
||||
block_stride-byte chunk is one logical block. We register 1
|
||||
NIXL region and create 1 descriptor per block.
|
||||
"""
|
||||
self.transfer_topo = TransferTopology(
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.world_size,
|
||||
block_size=self.block_size,
|
||||
engine_id=self.engine_id,
|
||||
is_mla=self.use_mla,
|
||||
total_num_kv_heads=self.model_config.get_total_num_kv_heads(),
|
||||
attn_backends=self.attn_backends,
|
||||
tensor_shape=None,
|
||||
is_mamba=self._has_mamba,
|
||||
)
|
||||
self.compat_hash = compute_nixl_compatibility_hash(
|
||||
self.vllm_config,
|
||||
self.backend_name,
|
||||
self.transfer_topo.cross_layers_blocks,
|
||||
)
|
||||
|
||||
total_size = storage.nbytes()
|
||||
block_stride = total_size // self.num_blocks
|
||||
base_addr = storage.data_ptr()
|
||||
device_id = storage.device.index
|
||||
assert device_id is not None
|
||||
|
||||
logger.info(
|
||||
"Registering packed KV cache: total_size=%s, block_stride=%s, "
|
||||
"num_blocks=%s, num_regions=1",
|
||||
total_size,
|
||||
block_stride,
|
||||
self.num_blocks,
|
||||
)
|
||||
|
||||
self.device_id = device_id
|
||||
caches_data = [(base_addr, total_size, self.device_id, "")]
|
||||
|
||||
self.block_len_per_layer = [block_stride]
|
||||
self.num_regions = 1
|
||||
self.num_descs = self.num_blocks
|
||||
self.kv_caches_base_addr[self.engine_id][self.tp_rank] = [base_addr]
|
||||
|
||||
descs = self.nixl_wrapper.get_reg_descs(caches_data, self.nixl_memory_type)
|
||||
self.nixl_wrapper.register_memory(descs, backends=self.nixl_backends)
|
||||
self._registered_descs.append(descs)
|
||||
|
||||
self.dst_num_blocks[self.engine_id] = self.num_blocks
|
||||
|
||||
self.src_xfer_handles_by_block_size[self.block_size], (self.src_blocks_data) = (
|
||||
self.register_local_xfer_handler(self.block_size)
|
||||
)
|
||||
|
||||
agent_metadata = NixlAgentMetadata(
|
||||
engine_id=self.engine_id,
|
||||
agent_metadata=self.nixl_wrapper.get_agent_metadata(),
|
||||
device_id=self.device_id,
|
||||
kv_caches_base_addr=(
|
||||
self.kv_caches_base_addr[self.engine_id][self.tp_rank]
|
||||
),
|
||||
num_blocks=self.num_blocks,
|
||||
block_lens=self.block_len_per_layer,
|
||||
kv_cache_layout=self.kv_cache_layout,
|
||||
block_size=self.block_size,
|
||||
ssm_sizes=self._mamba_ssm_size,
|
||||
attn_backend_name=self.backend_name,
|
||||
physical_blocks_per_logical_kv_block=(
|
||||
self._physical_blocks_per_logical_kv_block
|
||||
),
|
||||
)
|
||||
assert self.compat_hash is not None
|
||||
encoder = msgspec.msgpack.Encoder()
|
||||
self.xfer_handshake_metadata = NixlHandshakePayload(
|
||||
compatibility_hash=self.compat_hash,
|
||||
agent_metadata_bytes=encoder.encode(agent_metadata),
|
||||
)
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
"""Register the KV Cache data in nixl."""
|
||||
|
||||
# Detect packed allocation: all tensors are strided views into the
|
||||
# same backing storage (different data_ptr but same storage).
|
||||
# This happens with DSv4-style contiguous per-block packing.
|
||||
if len(kv_caches) > 1 and not self._has_mamba:
|
||||
storage = next(iter(kv_caches.values())).untyped_storage()
|
||||
storage_ptrs = {
|
||||
cache.untyped_storage().data_ptr() for cache in kv_caches.values()
|
||||
}
|
||||
data_ptrs = {cache.data_ptr() for cache in kv_caches.values()}
|
||||
if len(storage_ptrs) == 1 and len(data_ptrs) > 1:
|
||||
self._register_packed_kv_cache(storage)
|
||||
self.device_kv_caches = kv_caches
|
||||
return
|
||||
|
||||
self.transfer_topo = TransferTopology(
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.world_size,
|
||||
|
||||
@@ -72,18 +72,22 @@ class OffloadingConnectorWorker:
|
||||
if isinstance(layer_kv_cache_spec, AttentionSpec):
|
||||
layer_kv_cache = kv_caches[layer_name]
|
||||
assert isinstance(layer_kv_cache, torch.Tensor)
|
||||
assert layer_kv_cache.storage_offset() == 0
|
||||
|
||||
storage = layer_kv_cache.untyped_storage()
|
||||
page = layer_kv_cache_spec.page_size_bytes
|
||||
elem_size = layer_kv_cache.element_size()
|
||||
byte_offset = layer_kv_cache.storage_offset() * elem_size
|
||||
block_stride_bytes = layer_kv_cache.stride(0) * elem_size
|
||||
tensors_per_block[layer_name] = (
|
||||
torch.tensor(
|
||||
[],
|
||||
dtype=torch.int8,
|
||||
device=layer_kv_cache.device,
|
||||
)
|
||||
.set_(storage)
|
||||
.view(num_blocks, page),
|
||||
).set_(
|
||||
layer_kv_cache.untyped_storage(),
|
||||
byte_offset,
|
||||
(num_blocks, page),
|
||||
(block_stride_bytes, 1),
|
||||
),
|
||||
)
|
||||
page_size_bytes[layer_name] = layer_kv_cache_spec.page_size_bytes
|
||||
unpadded_page_size_bytes[layer_name] = (
|
||||
|
||||
Reference in New Issue
Block a user