Make tiering offload region DP-replica aware (#47987)
Signed-off-by: Liran Schour <lirans@il.ibm.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -96,7 +96,7 @@ def test_transfer(
|
||||
SharedOffloadRegion.BLOCK_SIZE_ALIGNMENT,
|
||||
)
|
||||
mmap_region = SharedOffloadRegion(
|
||||
instance_id=str(uuid.uuid4()),
|
||||
engine_id=str(uuid.uuid4()),
|
||||
num_blocks=num_cpu_blocks,
|
||||
rank=0,
|
||||
kv_bytes_per_block=cpu_page_size,
|
||||
|
||||
@@ -34,7 +34,7 @@ def _set_spawn_method(monkeypatch):
|
||||
|
||||
|
||||
def _make_region(
|
||||
instance_id: str,
|
||||
engine_id: str,
|
||||
num_blocks: int = 4,
|
||||
cpu_page_size: int = PAGE_SIZE,
|
||||
num_workers: int = 1,
|
||||
@@ -42,7 +42,7 @@ def _make_region(
|
||||
) -> SharedOffloadRegion:
|
||||
assert cpu_page_size % PAGE_SIZE == 0
|
||||
return SharedOffloadRegion(
|
||||
instance_id=instance_id,
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
@@ -57,9 +57,9 @@ def _cleanup_file(path: str) -> None:
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _region(instance_id: str, **kwargs):
|
||||
def _region(engine_id: str, **kwargs):
|
||||
"""Context manager: create one region, clean up on exit."""
|
||||
r = _make_region(instance_id, **kwargs)
|
||||
r = _make_region(engine_id, **kwargs)
|
||||
try:
|
||||
yield r
|
||||
finally:
|
||||
@@ -69,7 +69,7 @@ def _region(instance_id: str, **kwargs):
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _multi_region(
|
||||
instance_id: str,
|
||||
engine_id: str,
|
||||
num_workers: int,
|
||||
num_blocks: int = 4,
|
||||
cpu_page_size: int = PAGE_SIZE,
|
||||
@@ -77,7 +77,7 @@ def _multi_region(
|
||||
"""Context manager: create one SharedOffloadRegion per rank, clean up on exit."""
|
||||
regions = [
|
||||
SharedOffloadRegion(
|
||||
instance_id=instance_id,
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
@@ -94,7 +94,7 @@ def _multi_region(
|
||||
|
||||
|
||||
def _race_construct(
|
||||
instance_id: str,
|
||||
engine_id: str,
|
||||
num_workers: int,
|
||||
num_blocks: int = 4,
|
||||
cpu_page_size: int = PAGE_SIZE,
|
||||
@@ -108,7 +108,7 @@ def _race_construct(
|
||||
barrier.wait() # all threads start at the same instant
|
||||
try:
|
||||
regions[rank] = SharedOffloadRegion(
|
||||
instance_id=instance_id,
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
@@ -127,7 +127,7 @@ def _race_construct(
|
||||
|
||||
|
||||
def _mp_race_construct_and_write(
|
||||
instance_id: str,
|
||||
engine_id: str,
|
||||
num_blocks: int,
|
||||
rank: int,
|
||||
num_workers: int,
|
||||
@@ -141,7 +141,7 @@ def _mp_race_construct_and_write(
|
||||
parent a window to read the raw mmap before the creator removes the file."""
|
||||
try:
|
||||
region = SharedOffloadRegion(
|
||||
instance_id=instance_id,
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=num_workers * cpu_page_size,
|
||||
@@ -308,7 +308,7 @@ def test_create_next_view_multiprocess_slots(iid):
|
||||
|
||||
# Parent is rank 0 (creator); child is rank 1 (joiner).
|
||||
region = SharedOffloadRegion(
|
||||
instance_id=iid,
|
||||
engine_id=iid,
|
||||
num_blocks=num_blocks,
|
||||
rank=0,
|
||||
kv_bytes_per_block=num_workers * PAGE_SIZE,
|
||||
|
||||
@@ -33,14 +33,14 @@ class SharedOffloadRegion:
|
||||
the rest open the existing file and wait until it reaches the expected
|
||||
size. Each worker then mmap()s the full file.
|
||||
|
||||
File path: /dev/shm/vllm_offload_{instance_id}.mmap
|
||||
File path: /dev/shm/vllm_offload_{engine_id}.mmap
|
||||
"""
|
||||
|
||||
BLOCK_SIZE_ALIGNMENT: int = mmap.PAGESIZE
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
instance_id: str,
|
||||
engine_id: str,
|
||||
num_blocks: int,
|
||||
rank: int | None,
|
||||
kv_bytes_per_block: int,
|
||||
@@ -53,7 +53,7 @@ class SharedOffloadRegion:
|
||||
self._row_stride = kv_bytes_per_block
|
||||
self.total_size_bytes = self.num_blocks * self._row_stride
|
||||
|
||||
self.mmap_path = f"/dev/shm/vllm_offload_{instance_id}.mmap"
|
||||
self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
|
||||
self._creator = False # set True only if this worker creates the file
|
||||
self.rank = rank
|
||||
if rank is not None:
|
||||
|
||||
@@ -108,6 +108,13 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
# Scheduler-side mmap (rank=None); kept for cleanup
|
||||
self._scheduler_mmap: SharedOffloadRegion | None = None
|
||||
|
||||
# engine_id is unique per DP replica (suffixed with _dp{rank} in both
|
||||
# the Ray and multiprocessing paths), so it names a per-replica offload
|
||||
# region. Non-None is guaranteed by OffloadingSpec.__init__.
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
assert vllm_config.kv_transfer_config.engine_id is not None
|
||||
self._engine_id: str = vllm_config.kv_transfer_config.engine_id
|
||||
|
||||
@override
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
"""
|
||||
@@ -124,7 +131,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
# Create scheduler-side SharedOffloadRegion (rank=None) so the
|
||||
# primary tier can eagerly create a memoryview over _base.
|
||||
scheduler_mmap = SharedOffloadRegion(
|
||||
instance_id=self.vllm_config.instance_id,
|
||||
engine_id=self._engine_id,
|
||||
num_blocks=self.num_blocks,
|
||||
rank=None,
|
||||
kv_bytes_per_block=self.kv_bytes_per_offloaded_block,
|
||||
@@ -187,9 +194,12 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
|
||||
@override
|
||||
def create_worker(self, kv_caches: CanonicalKVCaches) -> CPUOffloadingWorker:
|
||||
rank = torch.accelerator.current_device_index()
|
||||
# Fold the global physical device index into the replica-local
|
||||
# [0, world_size) slot range.
|
||||
world_size = self.vllm_config.parallel_config.world_size
|
||||
rank = torch.accelerator.current_device_index() % world_size
|
||||
worker_mmap = SharedOffloadRegion(
|
||||
instance_id=self.vllm_config.instance_id,
|
||||
engine_id=self._engine_id,
|
||||
num_blocks=self.num_blocks,
|
||||
rank=rank,
|
||||
kv_bytes_per_block=self.kv_bytes_per_offloaded_block,
|
||||
|
||||
Reference in New Issue
Block a user