[KV Offload] Split replicated-layout pre-faulting across workers (#59545)
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c068db0e1d
commit
483e0c24c7
@@ -14,6 +14,7 @@ from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
check_shm_free_space,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv, round_up
|
||||
from vllm.v1.kv_offload.cpu.host_register import host_unregister
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -82,6 +83,8 @@ class SharedOffloadRegion:
|
||||
|
||||
Creator-only population pre-faults the entire region before the barrier
|
||||
and requires that barrier to keep joiners from using unpopulated pages.
|
||||
``populate_shard=(index, count)`` pre-faults only this worker's contiguous
|
||||
share, for layouts where every worker maps the same bytes.
|
||||
"""
|
||||
|
||||
BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE
|
||||
@@ -97,6 +100,7 @@ class SharedOffloadRegion:
|
||||
*,
|
||||
creator_memory_check: Callable[[int], None] | None = None,
|
||||
populate_only_on_creator: bool = False,
|
||||
populate_shard: tuple[int, int] | None = None,
|
||||
) -> None:
|
||||
if populate_only_on_creator and barrier is None:
|
||||
raise ValueError("Creator-only population requires a barrier.")
|
||||
@@ -212,7 +216,14 @@ class SharedOffloadRegion:
|
||||
|
||||
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
|
||||
|
||||
if rank is not None:
|
||||
if populate_shard is not None:
|
||||
index, count = populate_shard
|
||||
share = round_up(cdiv(self.total_size_bytes, count), self.page_size)
|
||||
start = min(index * share, self.total_size_bytes)
|
||||
if start < self.total_size_bytes:
|
||||
length = min(share, self.total_size_bytes - start)
|
||||
populate_write_fn(self.mmap_obj, start, length)
|
||||
elif rank is not None:
|
||||
# Populate only this worker's pages (one slot per chunk row).
|
||||
worker_offset = rank * cpu_page_size
|
||||
_t0 = time.perf_counter()
|
||||
|
||||
@@ -170,16 +170,21 @@ class CPUOffloadingSpec(OffloadingSpec):
|
||||
# num_chunks == 0 would size the region to zero bytes, which cannot be
|
||||
# mmap'd; fall back to the tensor path (empty tensors) as before.
|
||||
if self._uses_shared_region() and self.num_chunks > 0:
|
||||
# Replicated layout puts all ranks on slot 0 (single MLA copy);
|
||||
# otherwise each worker uses its own rank-indexed slot.
|
||||
rank = 0 if self.replicated_layout else self.config.parallel.rank
|
||||
# Use the logical worker rank for each slot, and distribute the
|
||||
# replicated region's pre-faulting work across the worker group.
|
||||
world_size = self.config.parallel.world_size
|
||||
worker_index = self.config.parallel.rank
|
||||
|
||||
mmap_region = SharedOffloadRegion(
|
||||
engine_id=self.config.engine_id,
|
||||
num_chunks=self.num_chunks,
|
||||
rank=rank,
|
||||
rank=0 if self.replicated_layout else worker_index,
|
||||
kv_bytes_per_chunk=self.kv_bytes_per_chunk,
|
||||
cpu_page_size=self.cpu_page_size_per_worker,
|
||||
barrier=_all_workers_barrier,
|
||||
populate_shard=(
|
||||
(worker_index, world_size) if self.replicated_layout else None
|
||||
),
|
||||
)
|
||||
try:
|
||||
return CPUOffloadingWorker(
|
||||
|
||||
Reference in New Issue
Block a user