[KV Offload] Split replicated-layout pre-faulting across workers (#59545)
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
c068db0e1d
commit
483e0c24c7
@@ -45,6 +45,7 @@ def _make_region(
|
|||||||
num_workers: int = 1,
|
num_workers: int = 1,
|
||||||
rank: int = 0,
|
rank: int = 0,
|
||||||
barrier=None,
|
barrier=None,
|
||||||
|
populate_shard: tuple[int, int] | None = None,
|
||||||
) -> SharedOffloadRegion:
|
) -> SharedOffloadRegion:
|
||||||
assert cpu_page_size % PAGE_SIZE == 0
|
assert cpu_page_size % PAGE_SIZE == 0
|
||||||
return SharedOffloadRegion(
|
return SharedOffloadRegion(
|
||||||
@@ -54,6 +55,7 @@ def _make_region(
|
|||||||
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
kv_bytes_per_chunk=num_workers * cpu_page_size,
|
||||||
cpu_page_size=cpu_page_size,
|
cpu_page_size=cpu_page_size,
|
||||||
barrier=barrier,
|
barrier=barrier,
|
||||||
|
populate_shard=populate_shard,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -526,6 +528,25 @@ def test_madvise_success_selects_madvise_population(iid, monkeypatch):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"shard, expected",
|
||||||
|
[((1, 2), [(3 * PAGE_SIZE, 3 * PAGE_SIZE)]), ((3, 4), [])],
|
||||||
|
)
|
||||||
|
def test_populate_shard_prefaults_only_its_share(iid, monkeypatch, shard, expected):
|
||||||
|
"""Workers that map the same bytes each pre-fault one contiguous share
|
||||||
|
(page-rounded; a trailing share may be empty) instead of the whole region."""
|
||||||
|
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
||||||
|
|
||||||
|
madvise_calls: list[tuple[int, int]] = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
sor,
|
||||||
|
"_madvise_populate_write",
|
||||||
|
lambda mm, off, ln: madvise_calls.append((off, ln)),
|
||||||
|
)
|
||||||
|
with _region(iid, num_chunks=3, num_workers=2, populate_shard=shard):
|
||||||
|
assert madvise_calls == [(0, PAGE_SIZE), *expected]
|
||||||
|
|
||||||
|
|
||||||
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
|
def test_madvise_einval_selects_fallback_for_ranked_region(iid, monkeypatch):
|
||||||
"""An EINVAL probe must select fallback for every ranked chunk."""
|
"""An EINVAL probe must select fallback for every ranked chunk."""
|
||||||
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
from vllm.v1.kv_offload.cpu import shared_offload_region as sor
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from vllm.distributed.device_communicators.shm_broadcast import (
|
|||||||
check_shm_free_space,
|
check_shm_free_space,
|
||||||
)
|
)
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
|
from vllm.utils.math_utils import cdiv, round_up
|
||||||
from vllm.v1.kv_offload.cpu.host_register import host_unregister
|
from vllm.v1.kv_offload.cpu.host_register import host_unregister
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
@@ -82,6 +83,8 @@ class SharedOffloadRegion:
|
|||||||
|
|
||||||
Creator-only population pre-faults the entire region before the barrier
|
Creator-only population pre-faults the entire region before the barrier
|
||||||
and requires that barrier to keep joiners from using unpopulated pages.
|
and requires that barrier to keep joiners from using unpopulated pages.
|
||||||
|
``populate_shard=(index, count)`` pre-faults only this worker's contiguous
|
||||||
|
share, for layouts where every worker maps the same bytes.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE
|
BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE
|
||||||
@@ -97,6 +100,7 @@ class SharedOffloadRegion:
|
|||||||
*,
|
*,
|
||||||
creator_memory_check: Callable[[int], None] | None = None,
|
creator_memory_check: Callable[[int], None] | None = None,
|
||||||
populate_only_on_creator: bool = False,
|
populate_only_on_creator: bool = False,
|
||||||
|
populate_shard: tuple[int, int] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
if populate_only_on_creator and barrier is None:
|
if populate_only_on_creator and barrier is None:
|
||||||
raise ValueError("Creator-only population requires a barrier.")
|
raise ValueError("Creator-only population requires a barrier.")
|
||||||
@@ -212,7 +216,14 @@ class SharedOffloadRegion:
|
|||||||
|
|
||||||
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
|
populate_write_fn = _get_populate_write_fn(self.mmap_obj)
|
||||||
|
|
||||||
if rank is not None:
|
if populate_shard is not None:
|
||||||
|
index, count = populate_shard
|
||||||
|
share = round_up(cdiv(self.total_size_bytes, count), self.page_size)
|
||||||
|
start = min(index * share, self.total_size_bytes)
|
||||||
|
if start < self.total_size_bytes:
|
||||||
|
length = min(share, self.total_size_bytes - start)
|
||||||
|
populate_write_fn(self.mmap_obj, start, length)
|
||||||
|
elif rank is not None:
|
||||||
# Populate only this worker's pages (one slot per chunk row).
|
# Populate only this worker's pages (one slot per chunk row).
|
||||||
worker_offset = rank * cpu_page_size
|
worker_offset = rank * cpu_page_size
|
||||||
_t0 = time.perf_counter()
|
_t0 = time.perf_counter()
|
||||||
|
|||||||
@@ -170,16 +170,21 @@ class CPUOffloadingSpec(OffloadingSpec):
|
|||||||
# num_chunks == 0 would size the region to zero bytes, which cannot be
|
# num_chunks == 0 would size the region to zero bytes, which cannot be
|
||||||
# mmap'd; fall back to the tensor path (empty tensors) as before.
|
# mmap'd; fall back to the tensor path (empty tensors) as before.
|
||||||
if self._uses_shared_region() and self.num_chunks > 0:
|
if self._uses_shared_region() and self.num_chunks > 0:
|
||||||
# Replicated layout puts all ranks on slot 0 (single MLA copy);
|
# Use the logical worker rank for each slot, and distribute the
|
||||||
# otherwise each worker uses its own rank-indexed slot.
|
# replicated region's pre-faulting work across the worker group.
|
||||||
rank = 0 if self.replicated_layout else self.config.parallel.rank
|
world_size = self.config.parallel.world_size
|
||||||
|
worker_index = self.config.parallel.rank
|
||||||
|
|
||||||
mmap_region = SharedOffloadRegion(
|
mmap_region = SharedOffloadRegion(
|
||||||
engine_id=self.config.engine_id,
|
engine_id=self.config.engine_id,
|
||||||
num_chunks=self.num_chunks,
|
num_chunks=self.num_chunks,
|
||||||
rank=rank,
|
rank=0 if self.replicated_layout else worker_index,
|
||||||
kv_bytes_per_chunk=self.kv_bytes_per_chunk,
|
kv_bytes_per_chunk=self.kv_bytes_per_chunk,
|
||||||
cpu_page_size=self.cpu_page_size_per_worker,
|
cpu_page_size=self.cpu_page_size_per_worker,
|
||||||
barrier=_all_workers_barrier,
|
barrier=_all_workers_barrier,
|
||||||
|
populate_shard=(
|
||||||
|
(worker_index, world_size) if self.replicated_layout else None
|
||||||
|
),
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
return CPUOffloadingWorker(
|
return CPUOffloadingWorker(
|
||||||
|
|||||||
Reference in New Issue
Block a user