Make tiering offload region DP-replica aware (#47987)

Signed-off-by: Liran Schour <lirans@il.ibm.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
liranschour
2026-07-12 13:10:21 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 5f8e73cb8b
commit 5c0c987c03
4 changed files with 28 additions and 18 deletions
+1 -1
View File
@@ -96,7 +96,7 @@ def test_transfer(
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
)
mmap_region = SharedOffloadRegion(
instance_id=str(uuid.uuid4()),
engine_id=str(uuid.uuid4()),
num_blocks=num_cpu_blocks,
rank=0,
kv_bytes_per_block=cpu_page_size,
@@ -34,7 +34,7 @@ def _set_spawn_method(monkeypatch):
def _make_region(
instance_id: str,
engine_id: str,
num_blocks: int = 4,
cpu_page_size: int = PAGE_SIZE,
num_workers: int = 1,
@@ -42,7 +42,7 @@ def _make_region(
) -> SharedOffloadRegion:
assert cpu_page_size % PAGE_SIZE == 0
return SharedOffloadRegion(
instance_id=instance_id,
engine_id=engine_id,
num_blocks=num_blocks,
rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size,
@@ -57,9 +57,9 @@ def _cleanup_file(path: str) -> None:
@contextlib.contextmanager
def _region(instance_id: str, **kwargs):
def _region(engine_id: str, **kwargs):
"""Context manager: create one region, clean up on exit."""
r = _make_region(instance_id, **kwargs)
r = _make_region(engine_id, **kwargs)
try:
yield r
finally:
@@ -69,7 +69,7 @@ def _region(instance_id: str, **kwargs):
@contextlib.contextmanager
def _multi_region(
instance_id: str,
engine_id: str,
num_workers: int,
num_blocks: int = 4,
cpu_page_size: int = PAGE_SIZE,
@@ -77,7 +77,7 @@ def _multi_region(
"""Context manager: create one SharedOffloadRegion per rank, clean up on exit."""
regions = [
SharedOffloadRegion(
instance_id=instance_id,
engine_id=engine_id,
num_blocks=num_blocks,
rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size,
@@ -94,7 +94,7 @@ def _multi_region(
def _race_construct(
instance_id: str,
engine_id: str,
num_workers: int,
num_blocks: int = 4,
cpu_page_size: int = PAGE_SIZE,
@@ -108,7 +108,7 @@ def _race_construct(
barrier.wait() # all threads start at the same instant
try:
regions[rank] = SharedOffloadRegion(
instance_id=instance_id,
engine_id=engine_id,
num_blocks=num_blocks,
rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size,
@@ -127,7 +127,7 @@ def _race_construct(
def _mp_race_construct_and_write(
instance_id: str,
engine_id: str,
num_blocks: int,
rank: int,
num_workers: int,
@@ -141,7 +141,7 @@ def _mp_race_construct_and_write(
parent a window to read the raw mmap before the creator removes the file."""
try:
region = SharedOffloadRegion(
instance_id=instance_id,
engine_id=engine_id,
num_blocks=num_blocks,
rank=rank,
kv_bytes_per_block=num_workers * cpu_page_size,
@@ -308,7 +308,7 @@ def test_create_next_view_multiprocess_slots(iid):
# Parent is rank 0 (creator); child is rank 1 (joiner).
region = SharedOffloadRegion(
instance_id=iid,
engine_id=iid,
num_blocks=num_blocks,
rank=0,
kv_bytes_per_block=num_workers * PAGE_SIZE,
@@ -33,14 +33,14 @@ class SharedOffloadRegion:
the rest open the existing file and wait until it reaches the expected
size. Each worker then mmap()s the full file.
File path: /dev/shm/vllm_offload_{instance_id}.mmap
File path: /dev/shm/vllm_offload_{engine_id}.mmap
"""
BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE
def __init__(
self,
instance_id: str,
engine_id: str,
num_blocks: int,
rank: int | None,
kv_bytes_per_block: int,
@@ -53,7 +53,7 @@ class SharedOffloadRegion:
self._row_stride = kv_bytes_per_block
self.total_size_bytes = self.num_blocks * self._row_stride
self.mmap_path = f"/dev/shm/vllm_offload_{instance_id}.mmap"
self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
self._creator = False # set True only if this worker creates the file
self.rank = rank
if rank is not None:
+13 -3
View File
@@ -108,6 +108,13 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
# Scheduler-side mmap (rank=None); kept for cleanup
self._scheduler_mmap: SharedOffloadRegion | None = None
# engine_id is unique per DP replica (suffixed with _dp{rank} in both
# the Ray and multiprocessing paths), so it names a per-replica offload
# region. Non-None is guaranteed by OffloadingSpec.__init__.
assert vllm_config.kv_transfer_config is not None
assert vllm_config.kv_transfer_config.engine_id is not None
self._engine_id: str = vllm_config.kv_transfer_config.engine_id
@override
def get_manager(self) -> OffloadingManager:
"""
@@ -124,7 +131,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
# Create scheduler-side SharedOffloadRegion (rank=None) so the
# primary tier can eagerly create a memoryview over _base.
scheduler_mmap = SharedOffloadRegion(
instance_id=self.vllm_config.instance_id,
engine_id=self._engine_id,
num_blocks=self.num_blocks,
rank=None,
kv_bytes_per_block=self.kv_bytes_per_offloaded_block,
@@ -187,9 +194,12 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
@override
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
rank = torch.accelerator.current_device_index()
# Fold the global physical device index into the replica-local
# [0, world_size) slot range.
world_size = self.vllm_config.parallel_config.world_size
rank = torch.accelerator.current_device_index() % world_size
worker_mmap = SharedOffloadRegion(
instance_id=self.vllm_config.instance_id,
engine_id=self._engine_id,
num_blocks=self.num_blocks,
rank=rank,
kv_bytes_per_block=self.kv_bytes_per_offloaded_block,