[EC Connector] EC Offloading Connector use events instead of StepTracker (#49994)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
94d96e2446
commit
f8d38fbc87
@@ -7,7 +7,10 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorMetadata,
|
||||
ECConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
@@ -29,10 +32,38 @@ class ECCPUConnectorMetadata(ECConnectorMetadata):
|
||||
# encoder_cache[mm_hash] → mmap at these block IDs.
|
||||
saves: dict[str, list[int]] = field(default_factory=dict)
|
||||
|
||||
# Consumer role: mm_hashes whose bytes are available in the local mmap;
|
||||
# the worker's start_load_caches copies mmap[block_ids] → GPU
|
||||
# encoder_cache.
|
||||
loads: dict[str, list[int]] = field(default_factory=dict)
|
||||
# Consumer role: mm_hashes whose bytes are available in the local mmap,
|
||||
# mapped to (transfer_id, block_ids); the worker's start_load_caches
|
||||
# copies mmap[block_ids] → GPU encoder_cache and reports the transfer_id
|
||||
# once its copy lands. The id is minted per dispatch, so two loads of the
|
||||
# same mm_hash in different steps are distinct and a late report from the
|
||||
# earlier one cannot release the later one's pin.
|
||||
loads: dict[str, tuple[int, list[int]]] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECCPUWorkerMetadata(ECConnectorWorkerMetadata):
|
||||
"""Per-step worker → scheduler payload for the ECCPUConnector.
|
||||
|
||||
Reports the GPU copies that completed this step: saved mm_hashes become
|
||||
safe to mark ready, and loaded transfers become safe to unpin once every
|
||||
participating rank has reported them. Built by
|
||||
`ECCPUWorker.build_connector_worker_meta`; consumed by
|
||||
`ECCPUScheduler.update_connector_output`.
|
||||
|
||||
Loads are reported by transfer id rather than mm_hash because every rank
|
||||
copies the same blocks: `aggregate` concatenates, so an id appears once
|
||||
per reporting rank and the scheduler can count participants off the list.
|
||||
"""
|
||||
|
||||
completed_saves: list[str] = field(default_factory=list)
|
||||
completed_loads: list[int] = field(default_factory=list)
|
||||
|
||||
def aggregate(self, other: ECConnectorWorkerMetadata) -> ECConnectorWorkerMetadata:
|
||||
assert isinstance(other, ECCPUWorkerMetadata)
|
||||
self.completed_saves.extend(other.completed_saves)
|
||||
self.completed_loads.extend(other.completed_loads)
|
||||
return self
|
||||
|
||||
|
||||
def _get_encoder_cache_hidden_dim(vllm_config: "VllmConfig") -> int:
|
||||
|
||||
@@ -15,6 +15,7 @@ import torch
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorBase,
|
||||
ECConnectorRole,
|
||||
ECConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
@@ -24,6 +25,7 @@ from vllm.logger import init_logger
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ECConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -113,6 +115,20 @@ class ECCPUConnector(ECConnectorBase):
|
||||
self.connector_worker.flush_saves()
|
||||
return None, None
|
||||
|
||||
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
|
||||
if self.connector_worker is not None:
|
||||
return self.connector_worker.build_connector_worker_meta()
|
||||
return None
|
||||
|
||||
def update_connector_output(self, connector_output: "ECConnectorOutput") -> None:
|
||||
if self.connector_scheduler is not None:
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
if self.connector_scheduler is not None:
|
||||
return self.connector_scheduler.has_pending_push_work()
|
||||
return False
|
||||
|
||||
# Shared.
|
||||
def shutdown(self) -> None:
|
||||
if self.connector_scheduler is not None:
|
||||
|
||||
@@ -11,19 +11,18 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
ECCPUWorkerMetadata,
|
||||
create_ec_shared_region,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import (
|
||||
StepTracker,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ECConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -42,16 +41,27 @@ class ECCPUScheduler:
|
||||
# Block allocator + LRU eviction policy for the shared region.
|
||||
self._cache = EmbeddingCache(self._region.num_blocks)
|
||||
|
||||
max_batches = vllm_config.max_concurrent_batches
|
||||
# Delays mark_ready until the GPU→mmap DMA is guaranteed complete.
|
||||
self._ready_tracker = StepTracker(max_batches)
|
||||
# Delays unpin until the mmap→GPU DMA is guaranteed complete.
|
||||
self._unpin_tracker = StepTracker(max_batches)
|
||||
|
||||
# mm_hash → block IDs allocated this step for GPU→mmap saves.
|
||||
self._pending_saves: dict[str, list[int]] = {}
|
||||
# mm_hash → block IDs to load from mmap→GPU this step.
|
||||
self._pending_loads: dict[str, list[int]] = {}
|
||||
# mm_hash → (transfer_id, block IDs) to load from mmap→GPU this step.
|
||||
self._pending_loads: dict[str, tuple[int, list[int]]] = {}
|
||||
|
||||
# Dispatched loads awaiting completion reports, keyed by transfer id:
|
||||
# transfer_id → (mm_hash, reports still outstanding). The pin taken at
|
||||
# dispatch is released only once the count reaches zero.
|
||||
self._load_acks: dict[int, tuple[str, int]] = {}
|
||||
self._next_transfer_id = 0
|
||||
|
||||
pc = vllm_config.parallel_config
|
||||
# Reports to expect per load. Only pipeline stage 0 runs the EC
|
||||
# connector, so the participants are the tp × pcp ranks of that stage.
|
||||
# Other executors deliver a single rank's output, and expecting reports
|
||||
# that cannot arrive would hold the pin forever.
|
||||
self._expected_load_reports = (
|
||||
pc.tensor_parallel_size * pc.prefill_context_parallel_size
|
||||
if pc.distributed_executor_backend == "mp"
|
||||
else 1
|
||||
)
|
||||
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
if not self._is_consumer:
|
||||
@@ -72,30 +82,19 @@ class ECCPUScheduler:
|
||||
entry = self._cache.alloc(mm_hash, feature.mm_position.length)
|
||||
if entry is not None:
|
||||
self._pending_saves[mm_hash] = list(entry.block_ids)
|
||||
self._ready_tracker.add(mm_hash, request.request_id)
|
||||
|
||||
if self._is_consumer and mm_hash not in self._pending_loads:
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and entry.ready:
|
||||
self._cache.pin(mm_hash)
|
||||
self._pending_loads[mm_hash] = list(entry.block_ids)
|
||||
self._unpin_tracker.add(mm_hash, request.request_id)
|
||||
transfer_id = self._next_transfer_id
|
||||
self._next_transfer_id += 1
|
||||
self._pending_loads[mm_hash] = (transfer_id, list(entry.block_ids))
|
||||
self._load_acks[transfer_id] = (mm_hash, self._expected_load_reports)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> ECCPUConnectorMetadata:
|
||||
finished = (
|
||||
scheduler_output.finished_req_ids if scheduler_output is not None else set()
|
||||
)
|
||||
|
||||
for key in self._ready_tracker.step(finished):
|
||||
entry = self._cache.get(key)
|
||||
if entry is not None and not entry.ready:
|
||||
self._cache.mark_ready(key)
|
||||
|
||||
for key in self._unpin_tracker.step(finished):
|
||||
self._cache.unpin(key)
|
||||
|
||||
meta = ECCPUConnectorMetadata()
|
||||
if self._is_producer:
|
||||
meta.saves = self._pending_saves
|
||||
@@ -105,13 +104,46 @@ class ECCPUScheduler:
|
||||
self._pending_loads = {}
|
||||
return meta
|
||||
|
||||
def shutdown(self) -> None:
|
||||
# drain_all() covers both entries still in _current (never
|
||||
# consumed by build_connector_meta) and entries in slots.
|
||||
self._pending_loads.clear()
|
||||
for mm_hash in self._unpin_tracker.drain_all():
|
||||
def update_connector_output(self, connector_output: "ECConnectorOutput") -> None:
|
||||
"""Apply the worker's memcpy-completion report to the cache.
|
||||
|
||||
Completed saves become safe to mark ready. A load is unpinned once
|
||||
every participating rank has reported its transfer id; reports for a
|
||||
transfer that has already been released, or for one this scheduler
|
||||
never dispatched, are ignored.
|
||||
"""
|
||||
meta = connector_output.ec_connector_worker_meta
|
||||
if not isinstance(meta, ECCPUWorkerMetadata):
|
||||
return
|
||||
for mm_hash in meta.completed_saves:
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and not entry.ready:
|
||||
self._cache.mark_ready(mm_hash)
|
||||
for transfer_id in meta.completed_loads:
|
||||
pending = self._load_acks.get(transfer_id)
|
||||
if pending is None:
|
||||
continue
|
||||
mm_hash, outstanding = pending
|
||||
if outstanding > 1:
|
||||
self._load_acks[transfer_id] = (mm_hash, outstanding - 1)
|
||||
continue
|
||||
# Drop the entry before unpinning so a replayed report is treated
|
||||
# as stale rather than releasing the pin a second time.
|
||||
del self._load_acks[transfer_id]
|
||||
self._cache.unpin(mm_hash)
|
||||
self._ready_tracker.drain_all()
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
"""True while any dispatched save or load has not been confirmed done.
|
||||
|
||||
Keeps the engine stepping so the worker's completion reports are polled
|
||||
even when no requests are otherwise runnable.
|
||||
"""
|
||||
return self._cache.has_held_entries()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._pending_saves.clear()
|
||||
self._pending_loads.clear()
|
||||
self._load_acks.clear()
|
||||
|
||||
self._is_producer = False
|
||||
self._is_consumer = False
|
||||
|
||||
@@ -67,6 +67,15 @@ class EmbeddingCache:
|
||||
with self._lock:
|
||||
return self._entries.get(key)
|
||||
|
||||
def has_held_entries(self) -> bool:
|
||||
"""True if any entry is not-ready or pinned (an in-flight transfer).
|
||||
|
||||
Held entries are exactly those absent from the eviction free list, so
|
||||
they represent saves awaiting mark_ready or loads awaiting unpin.
|
||||
"""
|
||||
with self._lock:
|
||||
return len(self._entries) != len(self._entries_free_list)
|
||||
|
||||
def alloc(self, key: str, n_blocks: int) -> CacheEntry | None:
|
||||
"""Allocate *n_blocks* for *key*, evicting as needed.
|
||||
|
||||
@@ -84,7 +93,10 @@ class EmbeddingCache:
|
||||
if len(self._free_blocks) + self._evictable_block_count < n_blocks:
|
||||
return None
|
||||
self._evict_until(n_blocks)
|
||||
block_ids = tuple(self._free_blocks.pop() for _ in range(n_blocks))
|
||||
# Canonical ascending order: keeps a freed entry's blocks contiguous
|
||||
# in the free list when it is evicted, and lets consumers detect
|
||||
# which of an entry's blocks are adjacent.
|
||||
block_ids = tuple(sorted(self._free_blocks.pop() for _ in range(n_blocks)))
|
||||
entry = CacheEntry(block_ids)
|
||||
self._entries[key] = entry
|
||||
return entry
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""StepTracker — step-counting delay with first-finish fast-path.
|
||||
|
||||
Manages a collection of (mm_hash, request_id) entries that need to be
|
||||
processed after a configurable number of engine steps have elapsed.
|
||||
Additionally, if the originating request finishes before the step-count
|
||||
expires, the entry is processed immediately (the GPU transfer is
|
||||
guaranteed complete by request finish time).
|
||||
|
||||
Used by ECCPUScheduler: one instance for readiness marking, one for
|
||||
unpins.
|
||||
"""
|
||||
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _PendingEntry:
|
||||
mm_hash: str
|
||||
request_id: str
|
||||
processed: bool = False
|
||||
|
||||
|
||||
class StepTracker:
|
||||
"""Track pending operations with step-count delay and first-finish.
|
||||
|
||||
Each ``add(mm_hash, request_id)`` registers an operation that will be
|
||||
returned from ``step()`` either when:
|
||||
(a) ``max_concurrent_batches`` steps elapse (deque expiry), OR
|
||||
(b) ``request_id`` appears in ``finished_req_ids`` (first-finish).
|
||||
|
||||
Each add is processed exactly once — never both (a) and (b).
|
||||
"""
|
||||
|
||||
def __init__(self, max_concurrent_batches: int) -> None:
|
||||
# One slot per past step; oldest at the right. When the deque is
|
||||
# full the rightmost slot is popped and its entries expire.
|
||||
self._slots: deque[list[_PendingEntry]] = deque(maxlen=max_concurrent_batches)
|
||||
# Reverse index: request_id → entries across all slots + _current.
|
||||
# Enables O(1) lookup for the first-finish fast-path.
|
||||
self._req_index: dict[str, list[_PendingEntry]] = {}
|
||||
# Entries added this step (via add()), committed to a new slot at
|
||||
# the start of the next step() call.
|
||||
self._current: list[_PendingEntry] = []
|
||||
|
||||
def add(self, mm_hash: str, request_id: str) -> None:
|
||||
"""Register an mm_hash needing processing after a delay."""
|
||||
entry = _PendingEntry(mm_hash=mm_hash, request_id=request_id)
|
||||
self._current.append(entry)
|
||||
self._req_index.setdefault(request_id, []).append(entry)
|
||||
|
||||
def step(self, finished_req_ids: set[str]) -> list[str]:
|
||||
"""Advance one step. Return mm_hashes ready for processing.
|
||||
|
||||
Returns:
|
||||
List of mm_hashes to process. May contain duplicates if the
|
||||
same mm_hash was added multiple times (from different requests).
|
||||
"""
|
||||
result: list[str] = []
|
||||
|
||||
# Phase 1: first-finish fast-path.
|
||||
for req_id in finished_req_ids:
|
||||
entries = self._req_index.pop(req_id, None)
|
||||
if entries is None:
|
||||
continue
|
||||
for entry in entries:
|
||||
if not entry.processed:
|
||||
entry.processed = True
|
||||
result.append(entry.mm_hash)
|
||||
|
||||
# Phase 2: step-count expiry.
|
||||
if len(self._slots) == self._slots.maxlen:
|
||||
expired_slot = self._slots.pop()
|
||||
for entry in expired_slot:
|
||||
if not entry.processed:
|
||||
entry.processed = True
|
||||
result.append(entry.mm_hash)
|
||||
self._cleanup_expired(expired_slot)
|
||||
|
||||
# Phase 3: commit current entries to new slot.
|
||||
self._slots.appendleft(self._current)
|
||||
self._current = []
|
||||
|
||||
return result
|
||||
|
||||
def drain_all(self) -> list[str]:
|
||||
"""Return all unprocessed entries and clear internal state."""
|
||||
result: list[str] = []
|
||||
for entry in self._current:
|
||||
if not entry.processed:
|
||||
entry.processed = True
|
||||
result.append(entry.mm_hash)
|
||||
self._current = []
|
||||
|
||||
for slot in self._slots:
|
||||
for entry in slot:
|
||||
if not entry.processed:
|
||||
entry.processed = True
|
||||
result.append(entry.mm_hash)
|
||||
self._slots.clear()
|
||||
self._req_index.clear()
|
||||
return result
|
||||
|
||||
def _cleanup_expired(self, slot: list[_PendingEntry]) -> None:
|
||||
"""Prune reverse index for entries in an expired slot."""
|
||||
for entry in slot:
|
||||
entries_for_req = self._req_index.get(entry.request_id)
|
||||
if entries_for_req is None:
|
||||
# Already removed by fast-path (pop in step()).
|
||||
continue
|
||||
entries_for_req.remove(entry)
|
||||
if not entries_for_req:
|
||||
del self._req_index[entry.request_id]
|
||||
@@ -7,13 +7,19 @@ per-step connector metadata (`ECCPUConnectorMetadata`) to decide which
|
||||
blocks to copy in each direction.
|
||||
"""
|
||||
|
||||
from collections import deque
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm._custom_ops import swap_blocks_batch
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
ECCPUWorkerMetadata,
|
||||
create_ec_shared_region,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.worker.descriptor_buffers import (
|
||||
@@ -34,6 +40,50 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _coalesce_runs(
|
||||
block_ids: Sequence[int],
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""Split block ids into runs that are consecutive in the region.
|
||||
|
||||
Returns `(slots, first_blocks, num_blocks)`, one element per run, where
|
||||
`slots` is the run's offset in the flat block sequence. Per-descriptor
|
||||
overhead dominates these copies, so describing a run with one descriptor
|
||||
rather than one per block is worth an order of magnitude.
|
||||
|
||||
Both callers derive the non-region side of a run from its slot, so that
|
||||
side advances in lockstep with the region and only the region has to be
|
||||
split. `EmbeddingCache` hands out ascending ids, which is what makes runs
|
||||
findable at all.
|
||||
"""
|
||||
|
||||
ids = np.fromiter(block_ids, dtype=np.int64, count=len(block_ids))
|
||||
breaks = np.flatnonzero(np.diff(ids) != 1) + 1
|
||||
slots = np.concatenate(([0], breaks))
|
||||
ends = np.concatenate((breaks, [ids.size]))
|
||||
return slots, ids[slots], ends - slots
|
||||
|
||||
|
||||
@dataclass
|
||||
class Transfer:
|
||||
"""A batched copy in flight on a dedicated GPU stream.
|
||||
|
||||
Its descriptor buffers and stream are held until `end_event` fires; only
|
||||
then are the buffers and stream safe to recycle and the `completions` safe
|
||||
to report. `start_event`/`end_event` bracket the copy so its elapsed time
|
||||
can be measured on completion.
|
||||
|
||||
`completions` is what the scheduler is told about this copy: mm_hashes for
|
||||
saves, transfer ids for loads.
|
||||
"""
|
||||
|
||||
start_event: torch.Event
|
||||
end_event: torch.Event
|
||||
completions: list[str] | list[int]
|
||||
bufs: DescriptorBuffers
|
||||
stream: torch.Stream
|
||||
num_bytes: int
|
||||
|
||||
|
||||
class ECCPUWorker:
|
||||
"""Worker-side delegate for the ECCPUConnector.
|
||||
|
||||
@@ -43,8 +93,18 @@ class ECCPUWorker:
|
||||
batched call in `flush_saves`.
|
||||
- Consumer role: copies `mmap[block_ids]` → `encoder_cache[mm_hash]`
|
||||
for all entries in `metadata.loads` via a single `swap_blocks_batch`
|
||||
call on the load stream.
|
||||
call on a dedicated stream.
|
||||
- On `ec_both` nodes both paths run back-to-back in a single step.
|
||||
|
||||
Every batched copy runs on a stream drawn from a pool so it overlaps the
|
||||
compute stream, bracketed by a start/end event pair. The stream, events,
|
||||
and descriptor buffers are recycled once the end event fires.
|
||||
|
||||
Both paths move memory between the two streams, so each buffer is registered
|
||||
against the stream that is not its own: save sources against the save stream,
|
||||
load destinations against the compute stream. Without that, the caching
|
||||
allocator would be free to reissue memory a copy or a queued read still
|
||||
needs, since it only tracks the stream a buffer was allocated on.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig") -> None:
|
||||
@@ -62,15 +122,72 @@ class ECCPUWorker:
|
||||
get_tensor_model_parallel_rank() == 0 and get_pcp_group().rank_in_group == 0
|
||||
)
|
||||
|
||||
# Dedicated stream for async mmap→GPU loads (overlaps with compute).
|
||||
self._load_stream = current_platform.Stream()
|
||||
|
||||
# Descriptor buffer pool (recycled across steps, shared by both paths).
|
||||
self._buf_pool = DescriptorBufferPool()
|
||||
|
||||
# Active save buffer being filled during save_caches calls this step.
|
||||
self._save_bufs: DescriptorBuffers | None = None
|
||||
self._save_count: int = 0
|
||||
# Bytes accumulated into the active save buffer this step, for logging.
|
||||
self._save_bytes: int = 0
|
||||
# mm_hashes accumulated into the active save buffer this step.
|
||||
self._save_mm_hashes: list[str] = []
|
||||
# Stream the active save buffer's copy will run on, held from the moment
|
||||
# the buffer opens so `save_caches` can register its sources against it.
|
||||
self._save_stream: torch.Stream | None = None
|
||||
|
||||
# Batched copies whose end event has not yet fired. Buffers and stream
|
||||
# stay held until the event fires; the completions are reported to the
|
||||
# scheduler only then. Separate per direction because saves and loads
|
||||
# can be in flight concurrently.
|
||||
self._inflight_saves: deque[Transfer] = deque()
|
||||
self._inflight_loads: deque[Transfer] = deque()
|
||||
|
||||
# Streams and timing events recycled across steps to avoid per-step
|
||||
# create/destroy churn. Shared by both directions.
|
||||
self._stream_pool: list[torch.Stream] = []
|
||||
self._event_pool: list[torch.Event] = []
|
||||
|
||||
def _acquire_stream(self) -> torch.Stream:
|
||||
if self._stream_pool:
|
||||
return self._stream_pool.pop()
|
||||
return current_platform.Stream()
|
||||
|
||||
def _acquire_event(self) -> torch.Event:
|
||||
return (
|
||||
self._event_pool.pop()
|
||||
if self._event_pool
|
||||
else torch.Event(enable_timing=True)
|
||||
)
|
||||
|
||||
def _collect_finished(self, inflight: deque[Transfer], direction: str) -> list:
|
||||
"""Pop transfers whose end event has fired, recycle their stream,
|
||||
events, and buffers, and return their completions.
|
||||
|
||||
The front check is conservative: a transfer is reported only once its
|
||||
own end event fires, so completions are always genuinely done, and each
|
||||
is reported exactly once because the transfer is popped as it is
|
||||
reported. A later transfer that happens to finish first simply waits
|
||||
behind the front and is reported on a subsequent poll.
|
||||
"""
|
||||
done: list = []
|
||||
while inflight and inflight[0].end_event.query():
|
||||
transfer = inflight.popleft()
|
||||
done.extend(transfer.completions)
|
||||
elapsed_ms = transfer.start_event.elapsed_time(transfer.end_event)
|
||||
logger.debug(
|
||||
"EC %s: %d entr%s (%d bytes) took %.3f ms",
|
||||
direction,
|
||||
len(transfer.completions),
|
||||
"y" if len(transfer.completions) == 1 else "ies",
|
||||
transfer.num_bytes,
|
||||
elapsed_ms,
|
||||
)
|
||||
self._buf_pool.release(transfer.bufs)
|
||||
self._stream_pool.append(transfer.stream)
|
||||
self._event_pool.append(transfer.start_event)
|
||||
self._event_pool.append(transfer.end_event)
|
||||
return done
|
||||
|
||||
def save_caches(
|
||||
self,
|
||||
@@ -78,7 +195,14 @@ class ECCPUWorker:
|
||||
mm_hash: str,
|
||||
connector_metadata: ECCPUConnectorMetadata,
|
||||
) -> None:
|
||||
"""Fill descriptor buffers directly for batched flush."""
|
||||
"""Fill descriptor buffers directly for batched flush.
|
||||
|
||||
Registers `encoder_cache[mm_hash]` against the stream the copy will run
|
||||
on, so the caching allocator will not hand that memory to another
|
||||
allocation before the copy has read it. The descriptor buffers hold raw
|
||||
addresses, and the caller is free to evict the entry as soon as this
|
||||
step's saves are dispatched.
|
||||
"""
|
||||
if not self._is_save_rank:
|
||||
return
|
||||
block_ids = connector_metadata.saves.get(mm_hash)
|
||||
@@ -99,43 +223,95 @@ class ECCPUWorker:
|
||||
if self._save_bufs is None:
|
||||
total = sum(len(v) for v in connector_metadata.saves.values())
|
||||
self._save_bufs = self._buf_pool.acquire(total)
|
||||
self._save_stream = self._acquire_stream()
|
||||
self._save_mm_hashes = []
|
||||
|
||||
assert self._save_count + len(block_ids) <= self._save_bufs.src_ptrs.numel()
|
||||
|
||||
bufs = self._save_bufs
|
||||
src_base = src.view(-1).view(torch.uint8).data_ptr()
|
||||
assert self._save_stream is not None
|
||||
src.record_stream(self._save_stream)
|
||||
# Device addresses are unsigned and can have the high bit set (XPU USM),
|
||||
# so they are added up in uint64; signed arithmetic rejects them.
|
||||
src_base = np.uint64(src.view(-1).view(torch.uint8).data_ptr())
|
||||
dst_base = self._region.blocks.data_ptr()
|
||||
idx = self._save_count
|
||||
# Entries are sized by placeholder count, which can exceed the number
|
||||
# of embeddings, so blocks the output never reaches get no descriptor.
|
||||
n_used = -(-total_bytes // block_size)
|
||||
if n_used:
|
||||
slots, first_blocks, num_blocks = _coalesce_runs(block_ids[:n_used])
|
||||
run_bytes = num_blocks * block_size
|
||||
# The output can stop partway into the final run.
|
||||
run_bytes[-1] = total_bytes - slots[-1] * block_size
|
||||
bufs.add_copies(
|
||||
self._save_count,
|
||||
src_base + slots.astype(np.uint64) * np.uint64(block_size),
|
||||
dst_base + first_blocks * block_size,
|
||||
run_bytes,
|
||||
)
|
||||
self._save_count += slots.size
|
||||
|
||||
for i, block_idx in enumerate(block_ids):
|
||||
start = i * block_size
|
||||
bufs.set_ptrs(idx, src_base + start, dst_base + block_idx * block_size)
|
||||
bufs.sizes[idx] = min(block_size, total_bytes - start)
|
||||
idx += 1
|
||||
|
||||
self._save_count = idx
|
||||
self._save_bytes += total_bytes
|
||||
self._save_mm_hashes.append(mm_hash)
|
||||
|
||||
def flush_saves(self) -> None:
|
||||
"""Flush all accumulated saves in a single swap_blocks_batch call."""
|
||||
"""Flush all accumulated saves in a single swap_blocks_batch call.
|
||||
|
||||
Runs the copy on the stream the batch acquired when it opened, gated
|
||||
behind the compute stream that produced the encoder outputs, brackets it
|
||||
with start/end events, and enqueues the batch as in-flight. The stream,
|
||||
events, and descriptor buffers are recycled once the end event fires (see
|
||||
`_collect_finished`), which is also when the saved mm_hashes become
|
||||
safe to mark ready.
|
||||
"""
|
||||
if self._save_count == 0:
|
||||
return
|
||||
|
||||
bufs = self._save_bufs
|
||||
assert bufs is not None
|
||||
stream = self._save_stream
|
||||
assert bufs is not None and stream is not None
|
||||
src_ptrs, dst_ptrs, sizes = bufs.src_ptrs, bufs.dst_ptrs, bufs.sizes
|
||||
n = self._save_count
|
||||
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
|
||||
num_bytes = self._save_bytes
|
||||
|
||||
# Gate the GPU→CPU copy behind the compute stream: it reads the encoder
|
||||
# outputs the model just produced there.
|
||||
stream.wait_stream(current_platform.current_stream())
|
||||
start_event = self._acquire_event()
|
||||
end_event = self._acquire_event()
|
||||
with current_platform.stream(stream):
|
||||
start_event.record(stream)
|
||||
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
|
||||
end_event.record(stream)
|
||||
|
||||
self._inflight_saves.append(
|
||||
Transfer(
|
||||
start_event=start_event,
|
||||
end_event=end_event,
|
||||
completions=self._save_mm_hashes,
|
||||
bufs=bufs,
|
||||
stream=stream,
|
||||
num_bytes=num_bytes,
|
||||
)
|
||||
)
|
||||
|
||||
self._buf_pool.release(bufs)
|
||||
self._save_bufs = None
|
||||
self._save_stream = None
|
||||
self._save_count = 0
|
||||
self._save_bytes = 0
|
||||
self._save_mm_hashes = []
|
||||
|
||||
def start_load_caches(
|
||||
self,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
connector_metadata: ECCPUConnectorMetadata,
|
||||
) -> None:
|
||||
"""Consumer path: single batched copy of all loads from mmap→GPU."""
|
||||
"""Consumer path: single batched copy of all loads from mmap→GPU.
|
||||
|
||||
The destination is registered against the compute stream that consumes
|
||||
it, so the caching allocator will not hand that memory to a later load
|
||||
while reads of it are still queued there.
|
||||
"""
|
||||
if not connector_metadata.loads:
|
||||
return
|
||||
|
||||
@@ -145,59 +321,99 @@ class ECCPUWorker:
|
||||
device_type = current_platform.device_type
|
||||
src_base = blocks.data_ptr()
|
||||
|
||||
# Pre-filter: only hashes not already in encoder_cache.
|
||||
load_items = {
|
||||
h: idxs
|
||||
for h, idxs in connector_metadata.loads.items()
|
||||
if h not in encoder_cache
|
||||
}
|
||||
if not load_items:
|
||||
return
|
||||
# Copy every dispatched load. The scheduler routes an input here only
|
||||
# after missing the GPU encoder cache, so a resident hit is not expected;
|
||||
# copying unconditionally means every participating rank reports each
|
||||
# transfer exactly once, which is what the scheduler counts to release
|
||||
# the pin.
|
||||
# Ordered by first block so entries that neighbour each other in the
|
||||
# region land next to each other in the destination and coalesce into
|
||||
# one descriptor. Every loop below consumes this same order.
|
||||
load_items = sorted(
|
||||
connector_metadata.loads.items(), key=lambda kv: kv[1][1][0]
|
||||
)
|
||||
total_blocks = sum(len(block_ids) for _, (_, block_ids) in load_items)
|
||||
|
||||
total_blocks = sum(len(idxs) for idxs in load_items.values())
|
||||
|
||||
with current_platform.stream(self._load_stream):
|
||||
stream = self._acquire_stream()
|
||||
compute_stream = current_platform.current_stream()
|
||||
start_event = self._acquire_event()
|
||||
end_event = self._acquire_event()
|
||||
with current_platform.stream(stream):
|
||||
# Single contiguous destination buffer for all loads.
|
||||
dst_buf = torch.empty(
|
||||
total_blocks, block_size, dtype=torch.int8, device=device_type
|
||||
)
|
||||
dst_buf_base = dst_buf.data_ptr()
|
||||
dst_buf.record_stream(compute_stream)
|
||||
# See save_caches: device addresses are added up in uint64.
|
||||
dst_buf_base = np.uint64(dst_buf.data_ptr())
|
||||
|
||||
bufs = self._buf_pool.acquire(total_blocks)
|
||||
src_ptrs = bufs.src_ptrs[:total_blocks]
|
||||
dst_ptrs = bufs.dst_ptrs[:total_blocks]
|
||||
sizes = bufs.sizes[:total_blocks]
|
||||
sizes[:] = block_size
|
||||
|
||||
op_idx = 0
|
||||
for block_ids in load_items.values():
|
||||
for block_idx in block_ids:
|
||||
bufs.set_ptrs(
|
||||
op_idx,
|
||||
src_base + block_idx * block_size,
|
||||
dst_buf_base + op_idx * block_size,
|
||||
)
|
||||
op_idx += 1
|
||||
slots, first_blocks, num_blocks = _coalesce_runs(
|
||||
list(chain.from_iterable(ids for _, (_, ids) in load_items))
|
||||
)
|
||||
bufs.add_copies(
|
||||
0,
|
||||
src_base + first_blocks * block_size,
|
||||
dst_buf_base + slots.astype(np.uint64) * np.uint64(block_size),
|
||||
num_blocks * block_size,
|
||||
)
|
||||
op_idx = slots.size
|
||||
src_ptrs = bufs.src_ptrs[:op_idx]
|
||||
dst_ptrs = bufs.dst_ptrs[:op_idx]
|
||||
sizes = bufs.sizes[:op_idx]
|
||||
|
||||
start_event.record(stream)
|
||||
swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True)
|
||||
|
||||
self._buf_pool.release(bufs)
|
||||
end_event.record(stream)
|
||||
self._inflight_loads.append(
|
||||
Transfer(
|
||||
start_event=start_event,
|
||||
end_event=end_event,
|
||||
completions=[tid for _, (tid, _) in load_items],
|
||||
bufs=bufs,
|
||||
stream=stream,
|
||||
num_bytes=total_blocks * block_size,
|
||||
)
|
||||
)
|
||||
|
||||
# Slice contiguous buffer into per-hash views.
|
||||
offset = 0
|
||||
for mm_hash, block_ids in load_items.items():
|
||||
for mm_hash, (_, block_ids) in load_items:
|
||||
n = len(block_ids)
|
||||
encoder_cache[mm_hash] = (
|
||||
dst_buf[offset : offset + n].view(dtype).reshape(n, -1)
|
||||
)
|
||||
offset += n
|
||||
|
||||
current_platform.current_stream().wait_stream(self._load_stream)
|
||||
current_platform.current_stream().wait_stream(stream)
|
||||
|
||||
def build_connector_worker_meta(self) -> ECCPUWorkerMetadata | None:
|
||||
"""Report the GPU copies that completed on this rank this step: saved
|
||||
mm_hashes and loaded transfer ids.
|
||||
|
||||
Returns None when nothing finished, so the scheduler sees no payload.
|
||||
"""
|
||||
completed_saves = self._collect_finished(self._inflight_saves, "save")
|
||||
completed_loads = self._collect_finished(self._inflight_loads, "load")
|
||||
if not completed_saves and not completed_loads:
|
||||
return None
|
||||
return ECCPUWorkerMetadata(
|
||||
completed_saves=completed_saves,
|
||||
completed_loads=completed_loads,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._load_stream.synchronize()
|
||||
for transfer in (*self._inflight_saves, *self._inflight_loads):
|
||||
transfer.end_event.synchronize()
|
||||
self._save_bufs = None
|
||||
self._save_stream = None
|
||||
self._save_count = 0
|
||||
self._save_bytes = 0
|
||||
self._save_mm_hashes = []
|
||||
self._inflight_saves.clear()
|
||||
self._inflight_loads.clear()
|
||||
self._stream_pool.clear()
|
||||
self._event_pool.clear()
|
||||
try:
|
||||
self._region.cleanup()
|
||||
except Exception:
|
||||
|
||||
@@ -22,24 +22,43 @@ class DescriptorBuffers(NamedTuple):
|
||||
src_ptrs: torch.Tensor
|
||||
dst_ptrs: torch.Tensor
|
||||
sizes: torch.Tensor
|
||||
# Numpy aliases of src_ptrs/dst_ptrs, written via set_ptrs().
|
||||
# Numpy aliases of the three tensors, written via add_copies().
|
||||
src_np: np.ndarray
|
||||
dst_np: np.ndarray
|
||||
sizes_np: np.ndarray
|
||||
|
||||
def set_ptrs(self, idx: int, src: int, dst: int) -> None:
|
||||
"""Record the source and destination address of descriptor *idx*.
|
||||
def add_copies(
|
||||
self,
|
||||
first: int,
|
||||
sources: np.ndarray,
|
||||
destinations: np.ndarray,
|
||||
byte_counts: np.ndarray,
|
||||
) -> None:
|
||||
"""Store a batch of copies in these buffers, the first at *first*.
|
||||
|
||||
TODO(torch>=2.14): drop this indirection and assign the tensors
|
||||
directly once the minimum supported torch is 2.14. The numpy detour
|
||||
exists only because torch's setitem unpacks the value as a signed
|
||||
long long before 2.14 (pytorch#191458), rejecting XPU USM pointers
|
||||
>= 2**63, while the two's-complement rewrite those versions accept is
|
||||
in turn rejected by uint64 from 2.14 on. Numpy casts against the
|
||||
array dtype and so works on either. `sizes` holds byte counts and
|
||||
needs no such care.
|
||||
One copy is one source address, one destination address, and a number
|
||||
of bytes. `swap_blocks_batch` performs every copy held in the buffers
|
||||
in a single call, so a caller fills in all of a step's copies before
|
||||
asking for any of them to run, and passes *first* to say where its own
|
||||
copies begin.
|
||||
|
||||
Whole batches go in at once because storing a single number into a
|
||||
torch tensor costs microseconds. Adding copies one at a time can take
|
||||
longer than performing them.
|
||||
|
||||
TODO(torch>=2.14): write the tensors directly and drop these numpy
|
||||
aliases once the minimum supported torch is 2.14. They exist only
|
||||
because torch's setitem unpacks the value as a signed long long before
|
||||
2.14 (pytorch#191458), rejecting XPU USM addresses >= 2**63, while the
|
||||
two's-complement rewrite those versions accept is in turn rejected by
|
||||
uint64 from 2.14 on. Numpy casts against the array's own type and so
|
||||
works on either. Byte counts are small enough to need none of this
|
||||
care, and use their alias only to match the addresses.
|
||||
"""
|
||||
self.src_np[idx] = src
|
||||
self.dst_np[idx] = dst
|
||||
last = first + len(sources)
|
||||
self.src_np[first:last] = sources
|
||||
self.dst_np[first:last] = destinations
|
||||
self.sizes_np[first:last] = byte_counts
|
||||
|
||||
|
||||
class DescriptorBufferPool:
|
||||
@@ -62,7 +81,9 @@ class DescriptorBufferPool:
|
||||
if bufs.src_ptrs.numel() >= n:
|
||||
return bufs
|
||||
src, dst, sizes = (torch.empty(n, dtype=_PTR_DTYPE) for _ in range(3))
|
||||
return DescriptorBuffers(src, dst, sizes, src.numpy(), dst.numpy())
|
||||
return DescriptorBuffers(
|
||||
src, dst, sizes, src.numpy(), dst.numpy(), sizes.numpy()
|
||||
)
|
||||
|
||||
def release(self, bufs: DescriptorBuffers) -> None:
|
||||
"""Return a buffer triple to the pool for reuse."""
|
||||
|
||||
Reference in New Issue
Block a user