[EC Connector] CPU Offloading EC Connector (#47423)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Signed-off-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -0,0 +1,89 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared types for the ECCPUConnector scheduler and worker delegates."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECCPUConnectorMetadata(ECConnectorMetadata):
|
||||
"""Per-step scheduler → worker payload for the ECCPUConnector.
|
||||
|
||||
Populated by `ECCPUScheduler.build_connector_meta`; consumed by
|
||||
`ECCPUWorker` via the mixin's `bind_connector_metadata`.
|
||||
"""
|
||||
|
||||
# Producer role: mm_hashes the scheduler has just allocated CPU
|
||||
# blocks for this step; the worker's save_caches copies
|
||||
# encoder_cache[mm_hash] → mmap at these block IDs.
|
||||
saves: dict[str, list[int]] = field(default_factory=dict)
|
||||
|
||||
# Consumer role: mm_hashes whose bytes are available in the local mmap;
|
||||
# the worker's start_load_caches copies mmap[block_ids] → GPU
|
||||
# encoder_cache.
|
||||
loads: dict[str, list[int]] = field(default_factory=dict)
|
||||
|
||||
|
||||
def _get_encoder_cache_hidden_dim(vllm_config: "VllmConfig") -> int:
|
||||
"""Return the per-token hidden dimension for encoder cache entries.
|
||||
|
||||
For most models this equals the LLM's hidden size. Qwen3-VL (and any
|
||||
future model with deepstack visual encoding) is an exception: the ViT
|
||||
concatenates its own output with features from N decoder layers before
|
||||
storing in encoder_cache, producing a tensor of width
|
||||
``out_hidden_size * (1 + N)`` per visual token. Using the plain LLM
|
||||
hidden size would under-allocate EC blocks and silently truncate the
|
||||
transferred data, leading to a shape mismatch on the consumer.
|
||||
"""
|
||||
model_config = vllm_config.model_config
|
||||
hf_config = getattr(model_config, "hf_config", None)
|
||||
vision_config = (
|
||||
getattr(hf_config, "vision_config", None) if hf_config is not None else None
|
||||
)
|
||||
if vision_config is not None:
|
||||
out_hidden_size = getattr(vision_config, "out_hidden_size", None)
|
||||
deepstack_indexes = getattr(vision_config, "deepstack_visual_indexes", None)
|
||||
if out_hidden_size is not None and deepstack_indexes:
|
||||
return out_hidden_size * (1 + len(deepstack_indexes))
|
||||
return model_config.get_inputs_embeds_size()
|
||||
|
||||
|
||||
def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion:
|
||||
"""Build the EC mmap region from `vllm_config`.
|
||||
|
||||
Both `ECCPUScheduler` and `ECCPUWorker` call this to get the same
|
||||
shared region (same engine_id, same block_size_bytes).
|
||||
"""
|
||||
ec_config = vllm_config.ec_transfer_config
|
||||
assert ec_config is not None, "ec_transfer_config required to build region"
|
||||
|
||||
dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
engine_id = f"{vllm_config.instance_id}_dp{dp_rank}"
|
||||
|
||||
dtype = vllm_config.model_config.dtype
|
||||
hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
|
||||
element_size = torch.empty(0, dtype=dtype).element_size()
|
||||
block_size_bytes = hidden_dim * element_size
|
||||
|
||||
cpu_bytes = ec_config.ec_connector_extra_config.get("ec_cpu_bytes")
|
||||
if not cpu_bytes:
|
||||
raise ValueError("ec_cpu_bytes must be specified in ec_connector_extra_config")
|
||||
cpu_bytes = int(cpu_bytes)
|
||||
num_blocks = cpu_bytes // block_size_bytes
|
||||
|
||||
return ECSharedRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
block_size_bytes=block_size_bytes,
|
||||
)
|
||||
@@ -0,0 +1,121 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ECCPUConnector — CPU encoder-cache offloading.
|
||||
|
||||
A role-routed shell: one instance per process. The scheduler delegate owns the
|
||||
mmap region and offload bookkeeping; the worker delegate owns the mmap view and
|
||||
GPU copy plumbing. An ec_both instance reuses encoder outputs it has already
|
||||
offloaded to CPU instead of recomputing them.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorBase,
|
||||
ECConnectorRole,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ECCPUConnector(ECConnectorBase):
|
||||
"""EC connector that offloads encoder cache to a shared CPU mmap region."""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole) -> None:
|
||||
super().__init__(vllm_config=vllm_config, role=role)
|
||||
|
||||
self.connector_worker = None
|
||||
self.connector_scheduler = None
|
||||
|
||||
if role == ECConnectorRole.WORKER:
|
||||
self.connector_worker = self._make_worker(vllm_config)
|
||||
elif role == ECConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = self._make_scheduler(vllm_config)
|
||||
else:
|
||||
raise ValueError(f"Unknown ECConnectorRole: {role}")
|
||||
|
||||
# Construction seams.
|
||||
def _make_worker(self, vllm_config: "VllmConfig"):
|
||||
# Deferred import: the worker module touches torch/CUDA at import time
|
||||
# via the region, so keep that cost off the scheduler path.
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import (
|
||||
ECCPUWorker,
|
||||
)
|
||||
|
||||
return ECCPUWorker(vllm_config)
|
||||
|
||||
def _make_scheduler(self, vllm_config: "VllmConfig"):
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
|
||||
ECCPUScheduler,
|
||||
)
|
||||
|
||||
return ECCPUScheduler(vllm_config)
|
||||
|
||||
# Worker-side forwarders.
|
||||
def start_load_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], **kwargs
|
||||
) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, ECCPUConnectorMetadata)
|
||||
self.connector_worker.start_load_caches(
|
||||
encoder_cache, connector_metadata=metadata
|
||||
)
|
||||
|
||||
def save_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs
|
||||
) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, ECCPUConnectorMetadata)
|
||||
self.connector_worker.save_caches(
|
||||
encoder_cache, mm_hash, connector_metadata=metadata
|
||||
)
|
||||
|
||||
# Scheduler-side forwarders.
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.has_cache_item(identifier)
|
||||
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.ensure_cache_available(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
|
||||
def update_state_after_alloc(self, request: "Request", index: int) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_state_after_alloc(request, index)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> ECCPUConnectorMetadata:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.flush_saves()
|
||||
return None, None
|
||||
|
||||
# Shared.
|
||||
def shutdown(self) -> None:
|
||||
if self.connector_scheduler is not None:
|
||||
self.connector_scheduler.shutdown()
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.shutdown()
|
||||
@@ -0,0 +1,159 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Lightweight mmap-backed shared memory region for encoder cache (EC) data.
|
||||
|
||||
Modeled after SharedOffloadRegion (vllm/v1/kv_offload/cpu/) but simplified
|
||||
for EC: flat shared layout, no multi-tensor cursor, no block_size_factor.
|
||||
"""
|
||||
|
||||
import mmap
|
||||
import os
|
||||
import time
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0):
|
||||
"""Spin-wait until the file reaches expected_size (creator truncated it)."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
if os.fstat(fd).st_size >= expected_size:
|
||||
return
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for EC mmap file to reach {expected_size} bytes"
|
||||
)
|
||||
time.sleep(0.005)
|
||||
|
||||
|
||||
class ECSharedRegion:
|
||||
"""Flat mmap-backed memory region shared across TP workers for
|
||||
encoder cache blocks.
|
||||
|
||||
Layout: (num_blocks, block_size_bytes) — contiguous, no per-worker
|
||||
interleaving. All workers map the same file and see identical data.
|
||||
|
||||
File path: /dev/shm/vllm_ec_{engine_id}.mmap
|
||||
|
||||
This class owns only the shared memory substrate (mmap lifecycle, the
|
||||
`blocks` view, CUDA host registration). Block allocation and eviction
|
||||
are tracked by `EmbeddingCache` in the scheduler process.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_id: str,
|
||||
num_blocks: int,
|
||||
block_size_bytes: int,
|
||||
) -> None:
|
||||
self.num_blocks = num_blocks
|
||||
self.block_size_bytes = block_size_bytes
|
||||
|
||||
total_size_bytes = num_blocks * block_size_bytes
|
||||
# Path in /dev/shm (tmpfs); unique per engine instance.
|
||||
self._mmap_path = f"/dev/shm/vllm_ec_{engine_id}.mmap"
|
||||
# True for the process that created the file (responsible for unlink).
|
||||
self._is_creator = False
|
||||
# True after successful cudaHostRegister (cleanup must unregister).
|
||||
self._is_pinned = False
|
||||
|
||||
# File descriptor for the shared memory backing file.
|
||||
try:
|
||||
self._fd: int | None = os.open(
|
||||
self._mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600
|
||||
)
|
||||
os.ftruncate(self._fd, total_size_bytes)
|
||||
self._is_creator = True
|
||||
logger.info(
|
||||
"Created EC mmap file %s (%.2f MiB)",
|
||||
self._mmap_path,
|
||||
total_size_bytes / (1 << 20),
|
||||
)
|
||||
except FileExistsError:
|
||||
self._fd = os.open(self._mmap_path, os.O_RDWR)
|
||||
try:
|
||||
_wait_for_file_size(self._fd, total_size_bytes)
|
||||
except Exception:
|
||||
os.close(self._fd)
|
||||
self._fd = None
|
||||
raise
|
||||
logger.info("Opened existing EC mmap file %s", self._mmap_path)
|
||||
|
||||
# MAP_SHARED mmap over _fd; all processes see the same pages.
|
||||
self._mmap_obj: mmap.mmap | None = mmap.mmap(
|
||||
self._fd,
|
||||
total_size_bytes,
|
||||
flags=mmap.MAP_SHARED,
|
||||
prot=mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
)
|
||||
|
||||
if self._is_creator:
|
||||
_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23)
|
||||
self._mmap_obj.madvise(_MADV_POPULATE_WRITE, 0, total_size_bytes)
|
||||
|
||||
# (num_blocks, block_size_bytes) int8 tensor over the mmap buffer.
|
||||
self.blocks: torch.Tensor = torch.frombuffer(
|
||||
memoryview(self._mmap_obj), dtype=torch.int8
|
||||
).view(num_blocks, block_size_bytes)
|
||||
# Cached for cudaHostRegister/Unregister and pointer math.
|
||||
self._blocks_ptr: int = self.blocks.data_ptr()
|
||||
self._blocks_nbytes: int = self.blocks.nbytes
|
||||
|
||||
def pin_memory(self) -> None:
|
||||
"""Register the entire mmap as CUDA pinned memory for fast DMA.
|
||||
|
||||
Each TP worker process owns its own virtual address for the same
|
||||
shared physical pages, so every process must register independently.
|
||||
No-op when CUDA is not available or already pinned.
|
||||
"""
|
||||
if self._is_pinned or not torch.cuda.is_available():
|
||||
return
|
||||
result = torch.cuda.cudart().cudaHostRegister(
|
||||
self._blocks_ptr, self._blocks_nbytes, 0
|
||||
)
|
||||
if result.value != 0:
|
||||
logger.warning(
|
||||
"cudaHostRegister failed (code=%d) — "
|
||||
"transfers will still work but may be slower (unpinned DMA)",
|
||||
result.value,
|
||||
)
|
||||
else:
|
||||
logger.debug("cudaHostRegister %.2f MB", self._blocks_nbytes / 1e6)
|
||||
self._is_pinned = True
|
||||
|
||||
def cleanup(self) -> None:
|
||||
"""Tear down the region. Lifecycle method; no concurrent access."""
|
||||
logger.info("Starting ECSharedRegion cleanup...")
|
||||
if self._is_creator:
|
||||
try:
|
||||
os.unlink(self._mmap_path)
|
||||
logger.info("Removed EC mmap file %s", self._mmap_path)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to unlink path %s", self._mmap_path, exc_info=True
|
||||
)
|
||||
self._is_creator = False
|
||||
if self._is_pinned:
|
||||
result = torch.cuda.cudart().cudaHostUnregister(self._blocks_ptr)
|
||||
if result.value != 0:
|
||||
logger.warning("cudaHostUnregister failed (code=%d)", result)
|
||||
self._is_pinned = False
|
||||
if hasattr(self, "blocks"):
|
||||
del self.blocks
|
||||
if self._mmap_obj:
|
||||
try:
|
||||
self._mmap_obj.close()
|
||||
except Exception:
|
||||
logger.warning("Failed to close mmap_obj", exc_info=True)
|
||||
self._mmap_obj = None
|
||||
if self._fd is not None:
|
||||
try:
|
||||
os.close(self._fd)
|
||||
except Exception:
|
||||
logger.warning("Failed to close fd %s", self._fd, exc_info=True)
|
||||
self._fd = None
|
||||
@@ -0,0 +1,122 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ECCPUScheduler — CPU offload scheduler delegate.
|
||||
|
||||
Owns the mmap region and the embedding cache, and handles the producer
|
||||
(GPU->CPU offload) and consumer (CPU->GPU reload) scheduler-side logic
|
||||
for the ECCPUConnector.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
create_ec_shared_region,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import (
|
||||
StepTracker,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ECCPUScheduler:
|
||||
"""Scheduler delegate for the ECCPUConnector."""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig") -> None:
|
||||
ec_config = vllm_config.ec_transfer_config
|
||||
assert ec_config is not None
|
||||
self._is_producer: bool = ec_config.is_ec_producer
|
||||
self._is_consumer: bool = ec_config.is_ec_consumer
|
||||
|
||||
self._region = create_ec_shared_region(vllm_config)
|
||||
# Block allocator + LRU eviction policy for the shared region.
|
||||
self._cache = EmbeddingCache(self._region.num_blocks)
|
||||
|
||||
max_batches = vllm_config.max_concurrent_batches
|
||||
# Delays mark_ready until the GPU→mmap DMA is guaranteed complete.
|
||||
self._ready_tracker = StepTracker(max_batches)
|
||||
# Delays unpin until the mmap→GPU DMA is guaranteed complete.
|
||||
self._unpin_tracker = StepTracker(max_batches)
|
||||
|
||||
# mm_hash → block IDs allocated this step for GPU→mmap saves.
|
||||
self._pending_saves: dict[str, list[int]] = {}
|
||||
# mm_hash → block IDs to load from mmap→GPU this step.
|
||||
self._pending_loads: dict[str, list[int]] = {}
|
||||
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
if not self._is_consumer:
|
||||
return False
|
||||
entry = self._cache.get(identifier)
|
||||
return entry is not None and entry.ready
|
||||
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def update_state_after_alloc(self, request: "Request", index: int) -> None:
|
||||
feature = request.mm_features[index]
|
||||
mm_hash = feature.identifier
|
||||
|
||||
if self._is_producer and self._cache.get(mm_hash) is None:
|
||||
entry = self._cache.alloc(mm_hash, feature.mm_position.length)
|
||||
if entry is not None:
|
||||
self._pending_saves[mm_hash] = list(entry.block_ids)
|
||||
self._ready_tracker.add(mm_hash, request.request_id)
|
||||
|
||||
if self._is_consumer and mm_hash not in self._pending_loads:
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and entry.ready:
|
||||
self._cache.pin(mm_hash)
|
||||
self._pending_loads[mm_hash] = list(entry.block_ids)
|
||||
self._unpin_tracker.add(mm_hash, request.request_id)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> ECCPUConnectorMetadata:
|
||||
finished = (
|
||||
scheduler_output.finished_req_ids if scheduler_output is not None else set()
|
||||
)
|
||||
|
||||
for key in self._ready_tracker.step(finished):
|
||||
entry = self._cache.get(key)
|
||||
if entry is not None and not entry.ready:
|
||||
self._cache.mark_ready(key)
|
||||
|
||||
for key in self._unpin_tracker.step(finished):
|
||||
self._cache.unpin(key)
|
||||
|
||||
meta = ECCPUConnectorMetadata()
|
||||
if self._is_producer:
|
||||
meta.saves = self._pending_saves
|
||||
self._pending_saves = {}
|
||||
if self._is_consumer:
|
||||
meta.loads = self._pending_loads
|
||||
self._pending_loads = {}
|
||||
return meta
|
||||
|
||||
def shutdown(self) -> None:
|
||||
# drain_all() covers both entries still in _current (never
|
||||
# consumed by build_connector_meta) and entries in slots.
|
||||
self._pending_loads.clear()
|
||||
for mm_hash in self._unpin_tracker.drain_all():
|
||||
self._cache.unpin(mm_hash)
|
||||
self._ready_tracker.drain_all()
|
||||
|
||||
self._is_producer = False
|
||||
self._is_consumer = False
|
||||
|
||||
try:
|
||||
self._region.cleanup()
|
||||
except Exception:
|
||||
logger.debug("ec: region cleanup failed", exc_info=True)
|
||||
@@ -0,0 +1,131 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""EmbeddingCache — named-entry block cache with FIFO eviction.
|
||||
|
||||
Manages a fixed pool of block IDs keyed by content identity (mm_hash).
|
||||
Entries transition through: not-ready → ready (evictable) → pinned.
|
||||
Eviction targets ready + unpinned entries in FIFO order.
|
||||
|
||||
All public methods are thread-safe.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
class CacheEntry:
|
||||
"""A single cache entry. Read `.block_ids` and `.ready` freely;
|
||||
mutations only through EmbeddingCache methods."""
|
||||
|
||||
__slots__ = ("block_ids", "_pin_count")
|
||||
|
||||
def __init__(self, block_ids: tuple[int, ...]) -> None:
|
||||
self.block_ids = block_ids
|
||||
self._pin_count = -1 # not ready
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
return self._pin_count >= 0
|
||||
|
||||
@property
|
||||
def evictable(self) -> bool:
|
||||
return self._pin_count == 0
|
||||
|
||||
def mark_ready(self):
|
||||
self._pin_count = 0
|
||||
|
||||
def pin(self):
|
||||
self._pin_count += 1
|
||||
|
||||
|
||||
class EmbeddingCache:
|
||||
"""Fixed-size block cache with FIFO eviction.
|
||||
|
||||
Entries are keyed by content identity (e.g. mm_hash). Blocks are
|
||||
allocated from a free-set; when space is needed, ready + unpinned
|
||||
entries are evicted oldest-first.
|
||||
|
||||
The caller is responsible for deciding *when* to call ``mark_ready``
|
||||
(e.g. after enough engine steps have elapsed for the worker to have
|
||||
completed the write).
|
||||
"""
|
||||
|
||||
def __init__(self, num_blocks: int) -> None:
|
||||
self._num_blocks = num_blocks
|
||||
self._lock = threading.Lock()
|
||||
# Available block IDs, LIFO stack.
|
||||
self._free_blocks: list[int] = list(range(num_blocks))
|
||||
# Key → live entry (all states: not-ready, ready, pinned).
|
||||
self._entries: dict[str, CacheEntry] = {}
|
||||
# Ready + unpinned entries in insertion order (FIFO eviction).
|
||||
self._entries_free_list: OrderedDict[str, None] = OrderedDict()
|
||||
# Total blocks reclaimable via eviction (fail-fast check).
|
||||
self._evictable_block_count: int = 0
|
||||
|
||||
def get(self, key: str) -> CacheEntry | None:
|
||||
"""Return the entry for *key*, or None if not present."""
|
||||
with self._lock:
|
||||
return self._entries.get(key)
|
||||
|
||||
def alloc(self, key: str, n_blocks: int) -> CacheEntry | None:
|
||||
"""Allocate *n_blocks* for *key*, evicting as needed.
|
||||
|
||||
The entry starts not-ready. Returns None if there is not enough
|
||||
space even after evicting all evictable entries.
|
||||
"""
|
||||
with self._lock:
|
||||
assert key not in self._entries, (
|
||||
f"EmbeddingCache: duplicate alloc for {key!r}"
|
||||
)
|
||||
assert n_blocks <= self._num_blocks, (
|
||||
f"EmbeddingCache: {n_blocks} blocks requested but capacity "
|
||||
f"is {self._num_blocks}"
|
||||
)
|
||||
if len(self._free_blocks) + self._evictable_block_count < n_blocks:
|
||||
return None
|
||||
self._evict_until(n_blocks)
|
||||
block_ids = tuple(self._free_blocks.pop() for _ in range(n_blocks))
|
||||
entry = CacheEntry(block_ids)
|
||||
self._entries[key] = entry
|
||||
return entry
|
||||
|
||||
def mark_ready(self, key: str) -> None:
|
||||
"""Mark an entry as ready (data is CPU-visible)."""
|
||||
with self._lock:
|
||||
entry = self._entries[key]
|
||||
assert entry._pin_count == -1, (
|
||||
f"EmbeddingCache: mark_ready on already-ready entry {key!r}"
|
||||
)
|
||||
entry.mark_ready()
|
||||
self._entries_free_list[key] = None
|
||||
self._evictable_block_count += len(entry.block_ids)
|
||||
|
||||
def pin(self, key: str) -> None:
|
||||
"""Pin an entry (prevent eviction)."""
|
||||
with self._lock:
|
||||
entry = self._entries[key]
|
||||
assert entry.ready, f"EmbeddingCache: pin of not-ready entry {key!r}"
|
||||
if entry.evictable:
|
||||
del self._entries_free_list[key]
|
||||
self._evictable_block_count -= len(entry.block_ids)
|
||||
entry.pin()
|
||||
|
||||
def unpin(self, key: str) -> None:
|
||||
"""Unpin an entry. Asserts currently pinned."""
|
||||
with self._lock:
|
||||
entry = self._entries[key]
|
||||
assert entry._pin_count > 0, (
|
||||
f"EmbeddingCache: unpin of unpinned entry {key!r}"
|
||||
)
|
||||
entry._pin_count -= 1
|
||||
if entry._pin_count == 0:
|
||||
self._entries_free_list[key] = None
|
||||
self._evictable_block_count += len(entry.block_ids)
|
||||
|
||||
def _evict_until(self, n_blocks: int) -> None:
|
||||
"""Evict ready+unpinned entries FIFO until enough space. Lock held."""
|
||||
while len(self._free_blocks) < n_blocks:
|
||||
key, _ = self._entries_free_list.popitem(last=False)
|
||||
entry = self._entries.pop(key)
|
||||
self._evictable_block_count -= len(entry.block_ids)
|
||||
self._free_blocks.extend(entry.block_ids)
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""StepTracker — step-counting delay with first-finish fast-path.
|
||||
|
||||
Manages a collection of (mm_hash, request_id) entries that need to be
|
||||
processed after a configurable number of engine steps have elapsed.
|
||||
Additionally, if the originating request finishes before the step-count
|
||||
expires, the entry is processed immediately (the GPU transfer is
|
||||
guaranteed complete by request finish time).
|
||||
|
||||
Used by ECCPUScheduler: one instance for readiness marking, one for
|
||||
unpins.
|
||||
"""
|
||||
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _PendingEntry:
|
||||
mm_hash: str
|
||||
request_id: str
|
||||
processed: bool = False
|
||||
|
||||
|
||||
class StepTracker:
|
||||
"""Track pending operations with step-count delay and first-finish.
|
||||
|
||||
Each ``add(mm_hash, request_id)`` registers an operation that will be
|
||||
returned from ``step()`` either when:
|
||||
(a) ``max_concurrent_batches`` steps elapse (deque expiry), OR
|
||||
(b) ``request_id`` appears in ``finished_req_ids`` (first-finish).
|
||||
|
||||
Each add is processed exactly once — never both (a) and (b).
|
||||
"""
|
||||
|
||||
def __init__(self, max_concurrent_batches: int) -> None:
|
||||
# One slot per past step; oldest at the right. When the deque is
|
||||
# full the rightmost slot is popped and its entries expire.
|
||||
self._slots: deque[list[_PendingEntry]] = deque(maxlen=max_concurrent_batches)
|
||||
# Reverse index: request_id → entries across all slots + _current.
|
||||
# Enables O(1) lookup for the first-finish fast-path.
|
||||
self._req_index: dict[str, list[_PendingEntry]] = {}
|
||||
# Entries added this step (via add()), committed to a new slot at
|
||||
# the start of the next step() call.
|
||||
self._current: list[_PendingEntry] = []
|
||||
|
||||
def add(self, mm_hash: str, request_id: str) -> None:
|
||||
"""Register an mm_hash needing processing after a delay."""
|
||||
entry = _PendingEntry(mm_hash=mm_hash, request_id=request_id)
|
||||
self._current.append(entry)
|
||||
self._req_index.setdefault(request_id, []).append(entry)
|
||||
|
||||
def step(self, finished_req_ids: set[str]) -> list[str]:
|
||||
"""Advance one step. Return mm_hashes ready for processing.
|
||||
|
||||
Returns:
|
||||
List of mm_hashes to process. May contain duplicates if the
|
||||
same mm_hash was added multiple times (from different requests).
|
||||
"""
|
||||
result: list[str] = []
|
||||
|
||||
# Phase 1: first-finish fast-path.
|
||||
for req_id in finished_req_ids:
|
||||
entries = self._req_index.pop(req_id, None)
|
||||
if entries is None:
|
||||
continue
|
||||
for entry in entries:
|
||||
if not entry.processed:
|
||||
entry.processed = True
|
||||
result.append(entry.mm_hash)
|
||||
|
||||
# Phase 2: step-count expiry.
|
||||
if len(self._slots) == self._slots.maxlen:
|
||||
expired_slot = self._slots.pop()
|
||||
for entry in expired_slot:
|
||||
if not entry.processed:
|
||||
entry.processed = True
|
||||
result.append(entry.mm_hash)
|
||||
self._cleanup_expired(expired_slot)
|
||||
|
||||
# Phase 3: commit current entries to new slot.
|
||||
self._slots.appendleft(self._current)
|
||||
self._current = []
|
||||
|
||||
return result
|
||||
|
||||
def drain_all(self) -> list[str]:
|
||||
"""Return all unprocessed entries and clear internal state."""
|
||||
result: list[str] = []
|
||||
for entry in self._current:
|
||||
if not entry.processed:
|
||||
entry.processed = True
|
||||
result.append(entry.mm_hash)
|
||||
self._current = []
|
||||
|
||||
for slot in self._slots:
|
||||
for entry in slot:
|
||||
if not entry.processed:
|
||||
entry.processed = True
|
||||
result.append(entry.mm_hash)
|
||||
self._slots.clear()
|
||||
self._req_index.clear()
|
||||
return result
|
||||
|
||||
def _cleanup_expired(self, slot: list[_PendingEntry]) -> None:
|
||||
"""Prune reverse index for entries in an expired slot."""
|
||||
for entry in slot:
|
||||
entries_for_req = self._req_index.get(entry.request_id)
|
||||
if entries_for_req is None:
|
||||
# Already removed by fast-path (pop in step()).
|
||||
continue
|
||||
entries_for_req.remove(entry)
|
||||
if not entries_for_req:
|
||||
del self._req_index[entry.request_id]
|
||||
@@ -0,0 +1,202 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Worker-side of the ECCPUConnector.
|
||||
|
||||
Thin, stateless across steps: opens the shared mmap region and uses the
|
||||
per-step connector metadata (`ECCPUConnectorMetadata`) to decide which
|
||||
blocks to copy in each direction.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._custom_ops import swap_blocks_batch
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
create_ec_shared_region,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.worker.descriptor_buffers import (
|
||||
DescriptorBufferPool,
|
||||
DescriptorBuffers,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pcp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ECCPUWorker:
|
||||
"""Worker-side delegate for the ECCPUConnector.
|
||||
|
||||
- Producer role: copies `encoder_cache[mm_hash]` → `mmap[block_ids]`
|
||||
for each entry in `metadata.saves`. Descriptor buffers are filled
|
||||
directly in `save_caches`; the actual DMA is issued as a single
|
||||
batched call in `flush_saves`.
|
||||
- Consumer role: copies `mmap[block_ids]` → `encoder_cache[mm_hash]`
|
||||
for all entries in `metadata.loads` via a single `swap_blocks_batch`
|
||||
call on the load stream.
|
||||
- On `ec_both` nodes both paths run back-to-back in a single step.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig") -> None:
|
||||
self._region = create_ec_shared_region(vllm_config)
|
||||
# Model dtype; used to reinterpret raw int8 blocks on load.
|
||||
self._dtype = vllm_config.model_config.dtype
|
||||
|
||||
if is_pin_memory_available():
|
||||
self._region.pin_memory()
|
||||
|
||||
# All TP/PCP ranks hold identical encoder output. Only one rank
|
||||
# per mmap needs to write — saves host memory bandwidth.
|
||||
# DCP is a subdivision of TP, so tp_rank==0 covers it.
|
||||
self._is_save_rank = (
|
||||
get_tensor_model_parallel_rank() == 0 and get_pcp_group().rank_in_group == 0
|
||||
)
|
||||
|
||||
# Dedicated stream for async mmap→GPU loads (overlaps with compute).
|
||||
self._load_stream = current_platform.Stream()
|
||||
|
||||
# Descriptor buffer pool (recycled across steps, shared by both paths).
|
||||
self._buf_pool = DescriptorBufferPool()
|
||||
|
||||
# Active save buffer being filled during save_caches calls this step.
|
||||
self._save_bufs: DescriptorBuffers | None = None
|
||||
self._save_count: int = 0
|
||||
|
||||
def save_caches(
|
||||
self,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
mm_hash: str,
|
||||
connector_metadata: ECCPUConnectorMetadata,
|
||||
) -> None:
|
||||
"""Fill descriptor buffers directly for batched flush."""
|
||||
if not self._is_save_rank:
|
||||
return
|
||||
block_ids = connector_metadata.saves.get(mm_hash)
|
||||
if block_ids is None:
|
||||
return
|
||||
|
||||
src = encoder_cache[mm_hash]
|
||||
total_bytes = src.numel() * src.element_size()
|
||||
block_size = self._region.block_size_bytes
|
||||
allocated_bytes = len(block_ids) * block_size
|
||||
assert total_bytes <= allocated_bytes, (
|
||||
f"EC: encoder output exceeds allocated blocks for mm_hash={mm_hash}: "
|
||||
f"{total_bytes} bytes but only {allocated_bytes} allocated "
|
||||
f"({len(block_ids)} blocks × {block_size}). "
|
||||
f"shape={list(src.shape)}"
|
||||
)
|
||||
|
||||
if self._save_bufs is None:
|
||||
total = sum(len(v) for v in connector_metadata.saves.values())
|
||||
self._save_bufs = self._buf_pool.acquire(total)
|
||||
|
||||
assert self._save_count + len(block_ids) <= self._save_bufs.src_ptrs.numel()
|
||||
|
||||
src_ptrs, dst_ptrs, sizes = self._save_bufs
|
||||
src_base = src.view(-1).view(torch.uint8).data_ptr()
|
||||
dst_base = self._region.blocks.data_ptr()
|
||||
idx = self._save_count
|
||||
|
||||
for i, block_idx in enumerate(block_ids):
|
||||
start = i * block_size
|
||||
src_ptrs[idx] = src_base + start
|
||||
dst_ptrs[idx] = dst_base + block_idx * block_size
|
||||
sizes[idx] = min(block_size, total_bytes - start)
|
||||
idx += 1
|
||||
|
||||
self._save_count = idx
|
||||
|
||||
def flush_saves(self) -> None:
|
||||
"""Flush all accumulated saves in a single swap_blocks_batch call."""
|
||||
if self._save_count == 0:
|
||||
return
|
||||
|
||||
bufs = self._save_bufs
|
||||
assert bufs is not None
|
||||
src_ptrs, dst_ptrs, sizes = bufs
|
||||
n = self._save_count
|
||||
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
|
||||
|
||||
self._buf_pool.release(bufs)
|
||||
self._save_bufs = None
|
||||
self._save_count = 0
|
||||
|
||||
def start_load_caches(
|
||||
self,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
connector_metadata: ECCPUConnectorMetadata,
|
||||
) -> None:
|
||||
"""Consumer path: single batched copy of all loads from mmap→GPU."""
|
||||
if not connector_metadata.loads:
|
||||
return
|
||||
|
||||
block_size = self._region.block_size_bytes
|
||||
blocks = self._region.blocks
|
||||
dtype = self._dtype
|
||||
device_type = current_platform.device_type
|
||||
src_base = blocks.data_ptr()
|
||||
|
||||
# Pre-filter: only hashes not already in encoder_cache.
|
||||
load_items = {
|
||||
h: idxs
|
||||
for h, idxs in connector_metadata.loads.items()
|
||||
if h not in encoder_cache
|
||||
}
|
||||
if not load_items:
|
||||
return
|
||||
|
||||
total_blocks = sum(len(idxs) for idxs in load_items.values())
|
||||
|
||||
with current_platform.stream(self._load_stream):
|
||||
# Single contiguous destination buffer for all loads.
|
||||
dst_buf = torch.empty(
|
||||
total_blocks, block_size, dtype=torch.int8, device=device_type
|
||||
)
|
||||
dst_buf_base = dst_buf.data_ptr()
|
||||
|
||||
bufs = self._buf_pool.acquire(total_blocks)
|
||||
src_ptrs = bufs.src_ptrs[:total_blocks]
|
||||
dst_ptrs = bufs.dst_ptrs[:total_blocks]
|
||||
sizes = bufs.sizes[:total_blocks]
|
||||
sizes[:] = block_size
|
||||
|
||||
op_idx = 0
|
||||
for block_ids in load_items.values():
|
||||
for block_idx in block_ids:
|
||||
src_ptrs[op_idx] = src_base + block_idx * block_size
|
||||
dst_ptrs[op_idx] = dst_buf_base + op_idx * block_size
|
||||
op_idx += 1
|
||||
|
||||
swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True)
|
||||
|
||||
self._buf_pool.release(bufs)
|
||||
|
||||
# Slice contiguous buffer into per-hash views.
|
||||
offset = 0
|
||||
for mm_hash, block_ids in load_items.items():
|
||||
n = len(block_ids)
|
||||
encoder_cache[mm_hash] = (
|
||||
dst_buf[offset : offset + n].view(dtype).reshape(n, -1)
|
||||
)
|
||||
offset += n
|
||||
|
||||
current_platform.current_stream().wait_stream(self._load_stream)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._load_stream.synchronize()
|
||||
self._save_bufs = None
|
||||
self._save_count = 0
|
||||
try:
|
||||
self._region.cleanup()
|
||||
except Exception:
|
||||
logger.debug("EC: worker region cleanup failed", exc_info=True)
|
||||
@@ -0,0 +1,46 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reusable pool of (src_ptrs, dst_ptrs, sizes) int64 tensor triples.
|
||||
|
||||
Used by ECCPUWorker to batch swap_blocks_batch descriptors without
|
||||
per-step allocation overhead.
|
||||
"""
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class DescriptorBuffers(NamedTuple):
|
||||
src_ptrs: torch.Tensor
|
||||
dst_ptrs: torch.Tensor
|
||||
sizes: torch.Tensor
|
||||
|
||||
|
||||
class DescriptorBufferPool:
|
||||
"""Pool of descriptor buffer triples for swap_blocks_batch.
|
||||
|
||||
Each buffer is a `DescriptorBuffers` namedtuple of three 1-D int64
|
||||
tensors of equal length. Buffers are recycled across steps; if a
|
||||
returned buffer is too small it is discarded and a fresh one allocated.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# LIFO stack of idle buffer triples.
|
||||
self._pool: list[DescriptorBuffers] = []
|
||||
|
||||
def acquire(self, n: int) -> DescriptorBuffers:
|
||||
"""Get a buffer triple with capacity >= *n*."""
|
||||
if self._pool:
|
||||
bufs = self._pool.pop()
|
||||
if bufs.src_ptrs.numel() >= n:
|
||||
return bufs
|
||||
return DescriptorBuffers(
|
||||
torch.empty(n, dtype=torch.int64),
|
||||
torch.empty(n, dtype=torch.int64),
|
||||
torch.empty(n, dtype=torch.int64),
|
||||
)
|
||||
|
||||
def release(self, bufs: DescriptorBuffers) -> None:
|
||||
"""Return a buffer triple to the pool for reuse."""
|
||||
self._pool.append(bufs)
|
||||
@@ -83,3 +83,9 @@ ECConnectorFactory.register_connector(
|
||||
"vllm.distributed.ec_transfer.ec_connector.example_connector",
|
||||
"ECExampleConnector",
|
||||
)
|
||||
|
||||
ECConnectorFactory.register_connector(
|
||||
"ECCPUConnector",
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.connector",
|
||||
"ECCPUConnector",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user