[EC Connector] CPU Offloading EC Connector (#47423)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Signed-off-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
omerpaz95
2026-07-13 20:09:41 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 62286308c9
commit 43c8cbf79b
18 changed files with 3008 additions and 0 deletions
@@ -0,0 +1,89 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared types for the ECCPUConnector scheduler and worker delegates."""
from dataclasses import dataclass, field
from typing import TYPE_CHECKING
import torch
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
@dataclass
class ECCPUConnectorMetadata(ECConnectorMetadata):
"""Per-step scheduler → worker payload for the ECCPUConnector.
Populated by `ECCPUScheduler.build_connector_meta`; consumed by
`ECCPUWorker` via the mixin's `bind_connector_metadata`.
"""
# Producer role: mm_hashes the scheduler has just allocated CPU
# blocks for this step; the worker's save_caches copies
# encoder_cache[mm_hash] → mmap at these block IDs.
saves: dict[str, list[int]] = field(default_factory=dict)
# Consumer role: mm_hashes whose bytes are available in the local mmap;
# the worker's start_load_caches copies mmap[block_ids] → GPU
# encoder_cache.
loads: dict[str, list[int]] = field(default_factory=dict)
def _get_encoder_cache_hidden_dim(vllm_config: "VllmConfig") -> int:
"""Return the per-token hidden dimension for encoder cache entries.
For most models this equals the LLM's hidden size. Qwen3-VL (and any
future model with deepstack visual encoding) is an exception: the ViT
concatenates its own output with features from N decoder layers before
storing in encoder_cache, producing a tensor of width
``out_hidden_size * (1 + N)`` per visual token. Using the plain LLM
hidden size would under-allocate EC blocks and silently truncate the
transferred data, leading to a shape mismatch on the consumer.
"""
model_config = vllm_config.model_config
hf_config = getattr(model_config, "hf_config", None)
vision_config = (
getattr(hf_config, "vision_config", None) if hf_config is not None else None
)
if vision_config is not None:
out_hidden_size = getattr(vision_config, "out_hidden_size", None)
deepstack_indexes = getattr(vision_config, "deepstack_visual_indexes", None)
if out_hidden_size is not None and deepstack_indexes:
return out_hidden_size * (1 + len(deepstack_indexes))
return model_config.get_inputs_embeds_size()
def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion:
"""Build the EC mmap region from `vllm_config`.
Both `ECCPUScheduler` and `ECCPUWorker` call this to get the same
shared region (same engine_id, same block_size_bytes).
"""
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None, "ec_transfer_config required to build region"
dp_rank = vllm_config.parallel_config.data_parallel_rank
engine_id = f"{vllm_config.instance_id}_dp{dp_rank}"
dtype = vllm_config.model_config.dtype
hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
element_size = torch.empty(0, dtype=dtype).element_size()
block_size_bytes = hidden_dim * element_size
cpu_bytes = ec_config.ec_connector_extra_config.get("ec_cpu_bytes")
if not cpu_bytes:
raise ValueError("ec_cpu_bytes must be specified in ec_connector_extra_config")
cpu_bytes = int(cpu_bytes)
num_blocks = cpu_bytes // block_size_bytes
return ECSharedRegion(
engine_id=engine_id,
num_blocks=num_blocks,
block_size_bytes=block_size_bytes,
)
@@ -0,0 +1,121 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ECCPUConnector — CPU encoder-cache offloading.
A role-routed shell: one instance per process. The scheduler delegate owns the
mmap region and offload bookkeeping; the worker delegate owns the mmap view and
GPU copy plumbing. An ec_both instance reuses encoder outputs it has already
offloaded to CPU instead of recomputing them.
"""
from typing import TYPE_CHECKING
import torch
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorBase,
ECConnectorRole,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
class ECCPUConnector(ECConnectorBase):
"""EC connector that offloads encoder cache to a shared CPU mmap region."""
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole) -> None:
super().__init__(vllm_config=vllm_config, role=role)
self.connector_worker = None
self.connector_scheduler = None
if role == ECConnectorRole.WORKER:
self.connector_worker = self._make_worker(vllm_config)
elif role == ECConnectorRole.SCHEDULER:
self.connector_scheduler = self._make_scheduler(vllm_config)
else:
raise ValueError(f"Unknown ECConnectorRole: {role}")
# Construction seams.
def _make_worker(self, vllm_config: "VllmConfig"):
# Deferred import: the worker module touches torch/CUDA at import time
# via the region, so keep that cost off the scheduler path.
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import (
ECCPUWorker,
)
return ECCPUWorker(vllm_config)
def _make_scheduler(self, vllm_config: "VllmConfig"):
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
ECCPUScheduler,
)
return ECCPUScheduler(vllm_config)
# Worker-side forwarders.
def start_load_caches(
self, encoder_cache: dict[str, torch.Tensor], **kwargs
) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, ECCPUConnectorMetadata)
self.connector_worker.start_load_caches(
encoder_cache, connector_metadata=metadata
)
def save_caches(
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs
) -> None:
assert self.connector_worker is not None
metadata = self._get_connector_metadata()
assert isinstance(metadata, ECCPUConnectorMetadata)
self.connector_worker.save_caches(
encoder_cache, mm_hash, connector_metadata=metadata
)
# Scheduler-side forwarders.
def has_cache_item(self, identifier: str) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.has_cache_item(identifier)
def ensure_cache_available(
self, request: "Request", num_computed_tokens: int
) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.ensure_cache_available(
request, num_computed_tokens
)
def update_state_after_alloc(self, request: "Request", index: int) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_state_after_alloc(request, index)
def build_connector_meta(
self, scheduler_output: "SchedulerOutput"
) -> ECCPUConnectorMetadata:
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def get_finished(
self, finished_req_ids: set[str]
) -> tuple[set[str] | None, set[str] | None]:
if self.connector_worker is not None:
self.connector_worker.flush_saves()
return None, None
# Shared.
def shutdown(self) -> None:
if self.connector_scheduler is not None:
self.connector_scheduler.shutdown()
if self.connector_worker is not None:
self.connector_worker.shutdown()
@@ -0,0 +1,159 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Lightweight mmap-backed shared memory region for encoder cache (EC) data.
Modeled after SharedOffloadRegion (vllm/v1/kv_offload/cpu/) but simplified
for EC: flat shared layout, no multi-tensor cursor, no block_size_factor.
"""
import mmap
import os
import time
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0):
"""Spin-wait until the file reaches expected_size (creator truncated it)."""
deadline = time.monotonic() + timeout
while True:
if os.fstat(fd).st_size >= expected_size:
return
if time.monotonic() > deadline:
raise TimeoutError(
f"Timed out waiting for EC mmap file to reach {expected_size} bytes"
)
time.sleep(0.005)
class ECSharedRegion:
"""Flat mmap-backed memory region shared across TP workers for
encoder cache blocks.
Layout: (num_blocks, block_size_bytes) — contiguous, no per-worker
interleaving. All workers map the same file and see identical data.
File path: /dev/shm/vllm_ec_{engine_id}.mmap
This class owns only the shared memory substrate (mmap lifecycle, the
`blocks` view, CUDA host registration). Block allocation and eviction
are tracked by `EmbeddingCache` in the scheduler process.
"""
def __init__(
self,
engine_id: str,
num_blocks: int,
block_size_bytes: int,
) -> None:
self.num_blocks = num_blocks
self.block_size_bytes = block_size_bytes
total_size_bytes = num_blocks * block_size_bytes
# Path in /dev/shm (tmpfs); unique per engine instance.
self._mmap_path = f"/dev/shm/vllm_ec_{engine_id}.mmap"
# True for the process that created the file (responsible for unlink).
self._is_creator = False
# True after successful cudaHostRegister (cleanup must unregister).
self._is_pinned = False
# File descriptor for the shared memory backing file.
try:
self._fd: int | None = os.open(
self._mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600
)
os.ftruncate(self._fd, total_size_bytes)
self._is_creator = True
logger.info(
"Created EC mmap file %s (%.2f MiB)",
self._mmap_path,
total_size_bytes / (1 << 20),
)
except FileExistsError:
self._fd = os.open(self._mmap_path, os.O_RDWR)
try:
_wait_for_file_size(self._fd, total_size_bytes)
except Exception:
os.close(self._fd)
self._fd = None
raise
logger.info("Opened existing EC mmap file %s", self._mmap_path)
# MAP_SHARED mmap over _fd; all processes see the same pages.
self._mmap_obj: mmap.mmap | None = mmap.mmap(
self._fd,
total_size_bytes,
flags=mmap.MAP_SHARED,
prot=mmap.PROT_READ | mmap.PROT_WRITE,
)
if self._is_creator:
_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23)
self._mmap_obj.madvise(_MADV_POPULATE_WRITE, 0, total_size_bytes)
# (num_blocks, block_size_bytes) int8 tensor over the mmap buffer.
self.blocks: torch.Tensor = torch.frombuffer(
memoryview(self._mmap_obj), dtype=torch.int8
).view(num_blocks, block_size_bytes)
# Cached for cudaHostRegister/Unregister and pointer math.
self._blocks_ptr: int = self.blocks.data_ptr()
self._blocks_nbytes: int = self.blocks.nbytes
def pin_memory(self) -> None:
"""Register the entire mmap as CUDA pinned memory for fast DMA.
Each TP worker process owns its own virtual address for the same
shared physical pages, so every process must register independently.
No-op when CUDA is not available or already pinned.
"""
if self._is_pinned or not torch.cuda.is_available():
return
result = torch.cuda.cudart().cudaHostRegister(
self._blocks_ptr, self._blocks_nbytes, 0
)
if result.value != 0:
logger.warning(
"cudaHostRegister failed (code=%d) — "
"transfers will still work but may be slower (unpinned DMA)",
result.value,
)
else:
logger.debug("cudaHostRegister %.2f MB", self._blocks_nbytes / 1e6)
self._is_pinned = True
def cleanup(self) -> None:
"""Tear down the region. Lifecycle method; no concurrent access."""
logger.info("Starting ECSharedRegion cleanup...")
if self._is_creator:
try:
os.unlink(self._mmap_path)
logger.info("Removed EC mmap file %s", self._mmap_path)
except Exception:
logger.warning(
"Failed to unlink path %s", self._mmap_path, exc_info=True
)
self._is_creator = False
if self._is_pinned:
result = torch.cuda.cudart().cudaHostUnregister(self._blocks_ptr)
if result.value != 0:
logger.warning("cudaHostUnregister failed (code=%d)", result)
self._is_pinned = False
if hasattr(self, "blocks"):
del self.blocks
if self._mmap_obj:
try:
self._mmap_obj.close()
except Exception:
logger.warning("Failed to close mmap_obj", exc_info=True)
self._mmap_obj = None
if self._fd is not None:
try:
os.close(self._fd)
except Exception:
logger.warning("Failed to close fd %s", self._fd, exc_info=True)
self._fd = None
@@ -0,0 +1,122 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ECCPUScheduler — CPU offload scheduler delegate.
Owns the mmap region and the embedding cache, and handles the producer
(GPU->CPU offload) and consumer (CPU->GPU reload) scheduler-side logic
for the ECCPUConnector.
"""
from typing import TYPE_CHECKING
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
create_ec_shared_region,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import (
StepTracker,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
class ECCPUScheduler:
"""Scheduler delegate for the ECCPUConnector."""
def __init__(self, vllm_config: "VllmConfig") -> None:
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None
self._is_producer: bool = ec_config.is_ec_producer
self._is_consumer: bool = ec_config.is_ec_consumer
self._region = create_ec_shared_region(vllm_config)
# Block allocator + LRU eviction policy for the shared region.
self._cache = EmbeddingCache(self._region.num_blocks)
max_batches = vllm_config.max_concurrent_batches
# Delays mark_ready until the GPU→mmap DMA is guaranteed complete.
self._ready_tracker = StepTracker(max_batches)
# Delays unpin until the mmap→GPU DMA is guaranteed complete.
self._unpin_tracker = StepTracker(max_batches)
# mm_hash → block IDs allocated this step for GPU→mmap saves.
self._pending_saves: dict[str, list[int]] = {}
# mm_hash → block IDs to load from mmap→GPU this step.
self._pending_loads: dict[str, list[int]] = {}
def has_cache_item(self, identifier: str) -> bool:
if not self._is_consumer:
return False
entry = self._cache.get(identifier)
return entry is not None and entry.ready
def ensure_cache_available(
self, request: "Request", num_computed_tokens: int
) -> bool:
return True
def update_state_after_alloc(self, request: "Request", index: int) -> None:
feature = request.mm_features[index]
mm_hash = feature.identifier
if self._is_producer and self._cache.get(mm_hash) is None:
entry = self._cache.alloc(mm_hash, feature.mm_position.length)
if entry is not None:
self._pending_saves[mm_hash] = list(entry.block_ids)
self._ready_tracker.add(mm_hash, request.request_id)
if self._is_consumer and mm_hash not in self._pending_loads:
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
self._cache.pin(mm_hash)
self._pending_loads[mm_hash] = list(entry.block_ids)
self._unpin_tracker.add(mm_hash, request.request_id)
def build_connector_meta(
self, scheduler_output: "SchedulerOutput"
) -> ECCPUConnectorMetadata:
finished = (
scheduler_output.finished_req_ids if scheduler_output is not None else set()
)
for key in self._ready_tracker.step(finished):
entry = self._cache.get(key)
if entry is not None and not entry.ready:
self._cache.mark_ready(key)
for key in self._unpin_tracker.step(finished):
self._cache.unpin(key)
meta = ECCPUConnectorMetadata()
if self._is_producer:
meta.saves = self._pending_saves
self._pending_saves = {}
if self._is_consumer:
meta.loads = self._pending_loads
self._pending_loads = {}
return meta
def shutdown(self) -> None:
# drain_all() covers both entries still in _current (never
# consumed by build_connector_meta) and entries in slots.
self._pending_loads.clear()
for mm_hash in self._unpin_tracker.drain_all():
self._cache.unpin(mm_hash)
self._ready_tracker.drain_all()
self._is_producer = False
self._is_consumer = False
try:
self._region.cleanup()
except Exception:
logger.debug("ec: region cleanup failed", exc_info=True)
@@ -0,0 +1,131 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""EmbeddingCache — named-entry block cache with FIFO eviction.
Manages a fixed pool of block IDs keyed by content identity (mm_hash).
Entries transition through: not-ready → ready (evictable) → pinned.
Eviction targets ready + unpinned entries in FIFO order.
All public methods are thread-safe.
"""
import threading
from collections import OrderedDict
class CacheEntry:
"""A single cache entry. Read `.block_ids` and `.ready` freely;
mutations only through EmbeddingCache methods."""
__slots__ = ("block_ids", "_pin_count")
def __init__(self, block_ids: tuple[int, ...]) -> None:
self.block_ids = block_ids
self._pin_count = -1 # not ready
@property
def ready(self) -> bool:
return self._pin_count >= 0
@property
def evictable(self) -> bool:
return self._pin_count == 0
def mark_ready(self):
self._pin_count = 0
def pin(self):
self._pin_count += 1
class EmbeddingCache:
"""Fixed-size block cache with FIFO eviction.
Entries are keyed by content identity (e.g. mm_hash). Blocks are
allocated from a free-set; when space is needed, ready + unpinned
entries are evicted oldest-first.
The caller is responsible for deciding *when* to call ``mark_ready``
(e.g. after enough engine steps have elapsed for the worker to have
completed the write).
"""
def __init__(self, num_blocks: int) -> None:
self._num_blocks = num_blocks
self._lock = threading.Lock()
# Available block IDs, LIFO stack.
self._free_blocks: list[int] = list(range(num_blocks))
# Key → live entry (all states: not-ready, ready, pinned).
self._entries: dict[str, CacheEntry] = {}
# Ready + unpinned entries in insertion order (FIFO eviction).
self._entries_free_list: OrderedDict[str, None] = OrderedDict()
# Total blocks reclaimable via eviction (fail-fast check).
self._evictable_block_count: int = 0
def get(self, key: str) -> CacheEntry | None:
"""Return the entry for *key*, or None if not present."""
with self._lock:
return self._entries.get(key)
def alloc(self, key: str, n_blocks: int) -> CacheEntry | None:
"""Allocate *n_blocks* for *key*, evicting as needed.
The entry starts not-ready. Returns None if there is not enough
space even after evicting all evictable entries.
"""
with self._lock:
assert key not in self._entries, (
f"EmbeddingCache: duplicate alloc for {key!r}"
)
assert n_blocks <= self._num_blocks, (
f"EmbeddingCache: {n_blocks} blocks requested but capacity "
f"is {self._num_blocks}"
)
if len(self._free_blocks) + self._evictable_block_count < n_blocks:
return None
self._evict_until(n_blocks)
block_ids = tuple(self._free_blocks.pop() for _ in range(n_blocks))
entry = CacheEntry(block_ids)
self._entries[key] = entry
return entry
def mark_ready(self, key: str) -> None:
"""Mark an entry as ready (data is CPU-visible)."""
with self._lock:
entry = self._entries[key]
assert entry._pin_count == -1, (
f"EmbeddingCache: mark_ready on already-ready entry {key!r}"
)
entry.mark_ready()
self._entries_free_list[key] = None
self._evictable_block_count += len(entry.block_ids)
def pin(self, key: str) -> None:
"""Pin an entry (prevent eviction)."""
with self._lock:
entry = self._entries[key]
assert entry.ready, f"EmbeddingCache: pin of not-ready entry {key!r}"
if entry.evictable:
del self._entries_free_list[key]
self._evictable_block_count -= len(entry.block_ids)
entry.pin()
def unpin(self, key: str) -> None:
"""Unpin an entry. Asserts currently pinned."""
with self._lock:
entry = self._entries[key]
assert entry._pin_count > 0, (
f"EmbeddingCache: unpin of unpinned entry {key!r}"
)
entry._pin_count -= 1
if entry._pin_count == 0:
self._entries_free_list[key] = None
self._evictable_block_count += len(entry.block_ids)
def _evict_until(self, n_blocks: int) -> None:
"""Evict ready+unpinned entries FIFO until enough space. Lock held."""
while len(self._free_blocks) < n_blocks:
key, _ = self._entries_free_list.popitem(last=False)
entry = self._entries.pop(key)
self._evictable_block_count -= len(entry.block_ids)
self._free_blocks.extend(entry.block_ids)
@@ -0,0 +1,115 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""StepTracker — step-counting delay with first-finish fast-path.
Manages a collection of (mm_hash, request_id) entries that need to be
processed after a configurable number of engine steps have elapsed.
Additionally, if the originating request finishes before the step-count
expires, the entry is processed immediately (the GPU transfer is
guaranteed complete by request finish time).
Used by ECCPUScheduler: one instance for readiness marking, one for
unpins.
"""
from collections import deque
from dataclasses import dataclass
@dataclass(slots=True)
class _PendingEntry:
mm_hash: str
request_id: str
processed: bool = False
class StepTracker:
"""Track pending operations with step-count delay and first-finish.
Each ``add(mm_hash, request_id)`` registers an operation that will be
returned from ``step()`` either when:
(a) ``max_concurrent_batches`` steps elapse (deque expiry), OR
(b) ``request_id`` appears in ``finished_req_ids`` (first-finish).
Each add is processed exactly once — never both (a) and (b).
"""
def __init__(self, max_concurrent_batches: int) -> None:
# One slot per past step; oldest at the right. When the deque is
# full the rightmost slot is popped and its entries expire.
self._slots: deque[list[_PendingEntry]] = deque(maxlen=max_concurrent_batches)
# Reverse index: request_id → entries across all slots + _current.
# Enables O(1) lookup for the first-finish fast-path.
self._req_index: dict[str, list[_PendingEntry]] = {}
# Entries added this step (via add()), committed to a new slot at
# the start of the next step() call.
self._current: list[_PendingEntry] = []
def add(self, mm_hash: str, request_id: str) -> None:
"""Register an mm_hash needing processing after a delay."""
entry = _PendingEntry(mm_hash=mm_hash, request_id=request_id)
self._current.append(entry)
self._req_index.setdefault(request_id, []).append(entry)
def step(self, finished_req_ids: set[str]) -> list[str]:
"""Advance one step. Return mm_hashes ready for processing.
Returns:
List of mm_hashes to process. May contain duplicates if the
same mm_hash was added multiple times (from different requests).
"""
result: list[str] = []
# Phase 1: first-finish fast-path.
for req_id in finished_req_ids:
entries = self._req_index.pop(req_id, None)
if entries is None:
continue
for entry in entries:
if not entry.processed:
entry.processed = True
result.append(entry.mm_hash)
# Phase 2: step-count expiry.
if len(self._slots) == self._slots.maxlen:
expired_slot = self._slots.pop()
for entry in expired_slot:
if not entry.processed:
entry.processed = True
result.append(entry.mm_hash)
self._cleanup_expired(expired_slot)
# Phase 3: commit current entries to new slot.
self._slots.appendleft(self._current)
self._current = []
return result
def drain_all(self) -> list[str]:
"""Return all unprocessed entries and clear internal state."""
result: list[str] = []
for entry in self._current:
if not entry.processed:
entry.processed = True
result.append(entry.mm_hash)
self._current = []
for slot in self._slots:
for entry in slot:
if not entry.processed:
entry.processed = True
result.append(entry.mm_hash)
self._slots.clear()
self._req_index.clear()
return result
def _cleanup_expired(self, slot: list[_PendingEntry]) -> None:
"""Prune reverse index for entries in an expired slot."""
for entry in slot:
entries_for_req = self._req_index.get(entry.request_id)
if entries_for_req is None:
# Already removed by fast-path (pop in step()).
continue
entries_for_req.remove(entry)
if not entries_for_req:
del self._req_index[entry.request_id]
@@ -0,0 +1,202 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Worker-side of the ECCPUConnector.
Thin, stateless across steps: opens the shared mmap region and uses the
per-step connector metadata (`ECCPUConnectorMetadata`) to decide which
blocks to copy in each direction.
"""
from typing import TYPE_CHECKING
import torch
from vllm._custom_ops import swap_blocks_batch
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
create_ec_shared_region,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.worker.descriptor_buffers import (
DescriptorBufferPool,
DescriptorBuffers,
)
from vllm.distributed.parallel_state import (
get_pcp_group,
get_tensor_model_parallel_rank,
)
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.platform_utils import is_pin_memory_available
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
class ECCPUWorker:
"""Worker-side delegate for the ECCPUConnector.
- Producer role: copies `encoder_cache[mm_hash]` → `mmap[block_ids]`
for each entry in `metadata.saves`. Descriptor buffers are filled
directly in `save_caches`; the actual DMA is issued as a single
batched call in `flush_saves`.
- Consumer role: copies `mmap[block_ids]` → `encoder_cache[mm_hash]`
for all entries in `metadata.loads` via a single `swap_blocks_batch`
call on the load stream.
- On `ec_both` nodes both paths run back-to-back in a single step.
"""
def __init__(self, vllm_config: "VllmConfig") -> None:
self._region = create_ec_shared_region(vllm_config)
# Model dtype; used to reinterpret raw int8 blocks on load.
self._dtype = vllm_config.model_config.dtype
if is_pin_memory_available():
self._region.pin_memory()
# All TP/PCP ranks hold identical encoder output. Only one rank
# per mmap needs to write — saves host memory bandwidth.
# DCP is a subdivision of TP, so tp_rank==0 covers it.
self._is_save_rank = (
get_tensor_model_parallel_rank() == 0 and get_pcp_group().rank_in_group == 0
)
# Dedicated stream for async mmap→GPU loads (overlaps with compute).
self._load_stream = current_platform.Stream()
# Descriptor buffer pool (recycled across steps, shared by both paths).
self._buf_pool = DescriptorBufferPool()
# Active save buffer being filled during save_caches calls this step.
self._save_bufs: DescriptorBuffers | None = None
self._save_count: int = 0
def save_caches(
self,
encoder_cache: dict[str, torch.Tensor],
mm_hash: str,
connector_metadata: ECCPUConnectorMetadata,
) -> None:
"""Fill descriptor buffers directly for batched flush."""
if not self._is_save_rank:
return
block_ids = connector_metadata.saves.get(mm_hash)
if block_ids is None:
return
src = encoder_cache[mm_hash]
total_bytes = src.numel() * src.element_size()
block_size = self._region.block_size_bytes
allocated_bytes = len(block_ids) * block_size
assert total_bytes <= allocated_bytes, (
f"EC: encoder output exceeds allocated blocks for mm_hash={mm_hash}: "
f"{total_bytes} bytes but only {allocated_bytes} allocated "
f"({len(block_ids)} blocks × {block_size}). "
f"shape={list(src.shape)}"
)
if self._save_bufs is None:
total = sum(len(v) for v in connector_metadata.saves.values())
self._save_bufs = self._buf_pool.acquire(total)
assert self._save_count + len(block_ids) <= self._save_bufs.src_ptrs.numel()
src_ptrs, dst_ptrs, sizes = self._save_bufs
src_base = src.view(-1).view(torch.uint8).data_ptr()
dst_base = self._region.blocks.data_ptr()
idx = self._save_count
for i, block_idx in enumerate(block_ids):
start = i * block_size
src_ptrs[idx] = src_base + start
dst_ptrs[idx] = dst_base + block_idx * block_size
sizes[idx] = min(block_size, total_bytes - start)
idx += 1
self._save_count = idx
def flush_saves(self) -> None:
"""Flush all accumulated saves in a single swap_blocks_batch call."""
if self._save_count == 0:
return
bufs = self._save_bufs
assert bufs is not None
src_ptrs, dst_ptrs, sizes = bufs
n = self._save_count
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
self._buf_pool.release(bufs)
self._save_bufs = None
self._save_count = 0
def start_load_caches(
self,
encoder_cache: dict[str, torch.Tensor],
connector_metadata: ECCPUConnectorMetadata,
) -> None:
"""Consumer path: single batched copy of all loads from mmap→GPU."""
if not connector_metadata.loads:
return
block_size = self._region.block_size_bytes
blocks = self._region.blocks
dtype = self._dtype
device_type = current_platform.device_type
src_base = blocks.data_ptr()
# Pre-filter: only hashes not already in encoder_cache.
load_items = {
h: idxs
for h, idxs in connector_metadata.loads.items()
if h not in encoder_cache
}
if not load_items:
return
total_blocks = sum(len(idxs) for idxs in load_items.values())
with current_platform.stream(self._load_stream):
# Single contiguous destination buffer for all loads.
dst_buf = torch.empty(
total_blocks, block_size, dtype=torch.int8, device=device_type
)
dst_buf_base = dst_buf.data_ptr()
bufs = self._buf_pool.acquire(total_blocks)
src_ptrs = bufs.src_ptrs[:total_blocks]
dst_ptrs = bufs.dst_ptrs[:total_blocks]
sizes = bufs.sizes[:total_blocks]
sizes[:] = block_size
op_idx = 0
for block_ids in load_items.values():
for block_idx in block_ids:
src_ptrs[op_idx] = src_base + block_idx * block_size
dst_ptrs[op_idx] = dst_buf_base + op_idx * block_size
op_idx += 1
swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True)
self._buf_pool.release(bufs)
# Slice contiguous buffer into per-hash views.
offset = 0
for mm_hash, block_ids in load_items.items():
n = len(block_ids)
encoder_cache[mm_hash] = (
dst_buf[offset : offset + n].view(dtype).reshape(n, -1)
)
offset += n
current_platform.current_stream().wait_stream(self._load_stream)
def shutdown(self) -> None:
self._load_stream.synchronize()
self._save_bufs = None
self._save_count = 0
try:
self._region.cleanup()
except Exception:
logger.debug("EC: worker region cleanup failed", exc_info=True)
@@ -0,0 +1,46 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Reusable pool of (src_ptrs, dst_ptrs, sizes) int64 tensor triples.
Used by ECCPUWorker to batch swap_blocks_batch descriptors without
per-step allocation overhead.
"""
from typing import NamedTuple
import torch
class DescriptorBuffers(NamedTuple):
src_ptrs: torch.Tensor
dst_ptrs: torch.Tensor
sizes: torch.Tensor
class DescriptorBufferPool:
"""Pool of descriptor buffer triples for swap_blocks_batch.
Each buffer is a `DescriptorBuffers` namedtuple of three 1-D int64
tensors of equal length. Buffers are recycled across steps; if a
returned buffer is too small it is discarded and a fresh one allocated.
"""
def __init__(self) -> None:
# LIFO stack of idle buffer triples.
self._pool: list[DescriptorBuffers] = []
def acquire(self, n: int) -> DescriptorBuffers:
"""Get a buffer triple with capacity >= *n*."""
if self._pool:
bufs = self._pool.pop()
if bufs.src_ptrs.numel() >= n:
return bufs
return DescriptorBuffers(
torch.empty(n, dtype=torch.int64),
torch.empty(n, dtype=torch.int64),
torch.empty(n, dtype=torch.int64),
)
def release(self, bufs: DescriptorBuffers) -> None:
"""Return a buffer triple to the pool for reuse."""
self._pool.append(bufs)
@@ -83,3 +83,9 @@ ECConnectorFactory.register_connector(
"vllm.distributed.ec_transfer.ec_connector.example_connector",
"ECExampleConnector",
)
ECConnectorFactory.register_connector(
"ECCPUConnector",
"vllm.distributed.ec_transfer.ec_connector.cpu.connector",
"ECCPUConnector",
)