[KV Offload] Enable single-copy MLA layout for CPUOffloadingSpec (#50301)

Signed-off-by: Change72 <changg@nvidia.com>
Co-authored-by: Cursor Agent <cursor-agent@cursor.com>
This commit is contained in:
Chang Guo
2026-07-31 06:20:47 +03:00
committed by GitHub
co-authored by Cursor Agent
parent 1d8be5cb44
commit 541128bdeb
3 changed files with 119 additions and 17 deletions
+14 -11
View File
@@ -26,7 +26,6 @@ from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
class CPUOffloadingSpec(OffloadingSpec):
BLOCK_SIZE_ALIGNMENT = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT
SUPPORTS_REPLICATED_LAYOUT = False
@classmethod
def build_metric_definitions(
@@ -88,9 +87,7 @@ class CPUOffloadingSpec(OffloadingSpec):
self.num_blocks = 0
self.kv_bytes_per_chunk = 0
self.cpu_page_size_per_worker = 0
self.replicated_layout = (
config.replicated_layout and self.SUPPORTS_REPLICATED_LAYOUT
)
self.replicated_layout = config.replicated_layout and self._uses_shared_region()
if config.worker_kv_bytes_per_block > 0 and world_size > 0:
num_copies = 1 if self.replicated_layout else world_size
kv_bytes_per_block = config.worker_kv_bytes_per_block * num_copies
@@ -144,17 +141,23 @@ class CPUOffloadingSpec(OffloadingSpec):
)
return self._manager
def _uses_shared_region(self) -> bool:
"""Whether the worker CPU buffer is the shared mmap region (vs a private
per-rank tensor); replicated-layout dedup is gated on this being True."""
return current_platform.is_cuda_alike()
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
mmap_region: SharedOffloadRegion | None = None
# num_blocks == 0 would size the region to zero bytes, which cannot be
# mmap'd; fall back to the tensor path (empty tensors) as before.
if current_platform.is_cuda_alike() and self.num_blocks > 0:
# Back each worker's CPU buffer with a private slot in a single
# shared mmap region instead of a per-rank pinned tensor. Fold the
# global physical device index into this replica's
# [0, world_size) slot range.
world_size = self.config.parallel.world_size
rank = torch.accelerator.current_device_index() % world_size
if self._uses_shared_region() and self.num_blocks > 0:
# Replicated layout puts all ranks on slot 0 (single MLA copy);
# otherwise each rank takes its own slot by physical device index.
if self.replicated_layout:
rank = 0
else:
world_size = self.config.parallel.world_size
rank = torch.accelerator.current_device_index() % world_size
mmap_region = SharedOffloadRegion(
engine_id=self.config.engine_id,
num_blocks=self.num_blocks,
+7 -1
View File
@@ -76,7 +76,6 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
"""
BLOCK_SIZE_ALIGNMENT = SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT
SUPPORTS_REPLICATED_LAYOUT = True
@classmethod
@override
@@ -234,6 +233,13 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
return self._manager
@override
def _uses_shared_region(self) -> bool:
# Tiering always allocates on the shared region (every platform), so the
# replicated-layout gate must not be narrowed by the CPU spec's
# CUDA-alike check.
return True
@override
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
world_size = self.config.parallel.world_size