[EC Connector] EC Offloading Connector use events instead of StepTracker (#49994)

Signed-off-by: omerpaz95 <omerpaz95@gmail.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
omerpaz95
2026-08-27 03:41:46 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 94d96e2446
commit f8d38fbc87
15 changed files with 1054 additions and 650 deletions
@@ -7,7 +7,10 @@ from typing import TYPE_CHECKING
import torch
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
ECConnectorWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
@@ -29,10 +32,38 @@ class ECCPUConnectorMetadata(ECConnectorMetadata):
# encoder_cache[mm_hash] → mmap at these block IDs.
saves: dict[str, list[int]] = field(default_factory=dict)
# Consumer role: mm_hashes whose bytes are available in the local mmap;
# the worker's start_load_caches copies mmap[block_ids] → GPU
# encoder_cache.
loads: dict[str, list[int]] = field(default_factory=dict)
# Consumer role: mm_hashes whose bytes are available in the local mmap,
# mapped to (transfer_id, block_ids); the worker's start_load_caches
# copies mmap[block_ids] → GPU encoder_cache and reports the transfer_id
# once its copy lands. The id is minted per dispatch, so two loads of the
# same mm_hash in different steps are distinct and a late report from the
# earlier one cannot release the later one's pin.
loads: dict[str, tuple[int, list[int]]] = field(default_factory=dict)
@dataclass
class ECCPUWorkerMetadata(ECConnectorWorkerMetadata):
"""Per-step worker → scheduler payload for the ECCPUConnector.
Reports the GPU copies that completed this step: saved mm_hashes become
safe to mark ready, and loaded transfers become safe to unpin once every
participating rank has reported them. Built by
`ECCPUWorker.build_connector_worker_meta`; consumed by
`ECCPUScheduler.update_connector_output`.
Loads are reported by transfer id rather than mm_hash because every rank
copies the same blocks: `aggregate` concatenates, so an id appears once
per reporting rank and the scheduler can count participants off the list.
"""
completed_saves: list[str] = field(default_factory=list)
completed_loads: list[int] = field(default_factory=list)
def aggregate(self, other: ECConnectorWorkerMetadata) -> ECConnectorWorkerMetadata:
assert isinstance(other, ECCPUWorkerMetadata)
self.completed_saves.extend(other.completed_saves)
self.completed_loads.extend(other.completed_loads)
return self
def _get_encoder_cache_hidden_dim(vllm_config: "VllmConfig") -> int:
@@ -15,6 +15,7 @@ import torch
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorBase,
ECConnectorRole,
ECConnectorWorkerMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
@@ -24,6 +25,7 @@ from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ECConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
@@ -113,6 +115,20 @@ class ECCPUConnector(ECConnectorBase):
self.connector_worker.flush_saves()
return None, None
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
if self.connector_worker is not None:
return self.connector_worker.build_connector_worker_meta()
return None
def update_connector_output(self, connector_output: "ECConnectorOutput") -> None:
if self.connector_scheduler is not None:
self.connector_scheduler.update_connector_output(connector_output)
def has_pending_push_work(self) -> bool:
if self.connector_scheduler is not None:
return self.connector_scheduler.has_pending_push_work()
return False
# Shared.
def shutdown(self) -> None:
if self.connector_scheduler is not None:
@@ -11,19 +11,18 @@ from typing import TYPE_CHECKING
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
ECCPUWorkerMetadata,
create_ec_shared_region,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import (
StepTracker,
)
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ECConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
@@ -42,16 +41,27 @@ class ECCPUScheduler:
# Block allocator + LRU eviction policy for the shared region.
self._cache = EmbeddingCache(self._region.num_blocks)
max_batches = vllm_config.max_concurrent_batches
# Delays mark_ready until the GPU→mmap DMA is guaranteed complete.
self._ready_tracker = StepTracker(max_batches)
# Delays unpin until the mmap→GPU DMA is guaranteed complete.
self._unpin_tracker = StepTracker(max_batches)
# mm_hash → block IDs allocated this step for GPU→mmap saves.
self._pending_saves: dict[str, list[int]] = {}
# mm_hash → block IDs to load from mmap→GPU this step.
self._pending_loads: dict[str, list[int]] = {}
# mm_hash → (transfer_id, block IDs) to load from mmap→GPU this step.
self._pending_loads: dict[str, tuple[int, list[int]]] = {}
# Dispatched loads awaiting completion reports, keyed by transfer id:
# transfer_id → (mm_hash, reports still outstanding). The pin taken at
# dispatch is released only once the count reaches zero.
self._load_acks: dict[int, tuple[str, int]] = {}
self._next_transfer_id = 0
pc = vllm_config.parallel_config
# Reports to expect per load. Only pipeline stage 0 runs the EC
# connector, so the participants are the tp × pcp ranks of that stage.
# Other executors deliver a single rank's output, and expecting reports
# that cannot arrive would hold the pin forever.
self._expected_load_reports = (
pc.tensor_parallel_size * pc.prefill_context_parallel_size
if pc.distributed_executor_backend == "mp"
else 1
)
def has_cache_item(self, identifier: str) -> bool:
if not self._is_consumer:
@@ -72,30 +82,19 @@ class ECCPUScheduler:
entry = self._cache.alloc(mm_hash, feature.mm_position.length)
if entry is not None:
self._pending_saves[mm_hash] = list(entry.block_ids)
self._ready_tracker.add(mm_hash, request.request_id)
if self._is_consumer and mm_hash not in self._pending_loads:
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
self._cache.pin(mm_hash)
self._pending_loads[mm_hash] = list(entry.block_ids)
self._unpin_tracker.add(mm_hash, request.request_id)
transfer_id = self._next_transfer_id
self._next_transfer_id += 1
self._pending_loads[mm_hash] = (transfer_id, list(entry.block_ids))
self._load_acks[transfer_id] = (mm_hash, self._expected_load_reports)
def build_connector_meta(
self, scheduler_output: "SchedulerOutput"
) -> ECCPUConnectorMetadata:
finished = (
scheduler_output.finished_req_ids if scheduler_output is not None else set()
)
for key in self._ready_tracker.step(finished):
entry = self._cache.get(key)
if entry is not None and not entry.ready:
self._cache.mark_ready(key)
for key in self._unpin_tracker.step(finished):
self._cache.unpin(key)
meta = ECCPUConnectorMetadata()
if self._is_producer:
meta.saves = self._pending_saves
@@ -105,13 +104,46 @@ class ECCPUScheduler:
self._pending_loads = {}
return meta
def shutdown(self) -> None:
# drain_all() covers both entries still in _current (never
# consumed by build_connector_meta) and entries in slots.
self._pending_loads.clear()
for mm_hash in self._unpin_tracker.drain_all():
def update_connector_output(self, connector_output: "ECConnectorOutput") -> None:
"""Apply the worker's memcpy-completion report to the cache.
Completed saves become safe to mark ready. A load is unpinned once
every participating rank has reported its transfer id; reports for a
transfer that has already been released, or for one this scheduler
never dispatched, are ignored.
"""
meta = connector_output.ec_connector_worker_meta
if not isinstance(meta, ECCPUWorkerMetadata):
return
for mm_hash in meta.completed_saves:
entry = self._cache.get(mm_hash)
if entry is not None and not entry.ready:
self._cache.mark_ready(mm_hash)
for transfer_id in meta.completed_loads:
pending = self._load_acks.get(transfer_id)
if pending is None:
continue
mm_hash, outstanding = pending
if outstanding > 1:
self._load_acks[transfer_id] = (mm_hash, outstanding - 1)
continue
# Drop the entry before unpinning so a replayed report is treated
# as stale rather than releasing the pin a second time.
del self._load_acks[transfer_id]
self._cache.unpin(mm_hash)
self._ready_tracker.drain_all()
def has_pending_push_work(self) -> bool:
"""True while any dispatched save or load has not been confirmed done.
Keeps the engine stepping so the worker's completion reports are polled
even when no requests are otherwise runnable.
"""
return self._cache.has_held_entries()
def shutdown(self) -> None:
self._pending_saves.clear()
self._pending_loads.clear()
self._load_acks.clear()
self._is_producer = False
self._is_consumer = False
@@ -67,6 +67,15 @@ class EmbeddingCache:
with self._lock:
return self._entries.get(key)
def has_held_entries(self) -> bool:
"""True if any entry is not-ready or pinned (an in-flight transfer).
Held entries are exactly those absent from the eviction free list, so
they represent saves awaiting mark_ready or loads awaiting unpin.
"""
with self._lock:
return len(self._entries) != len(self._entries_free_list)
def alloc(self, key: str, n_blocks: int) -> CacheEntry | None:
"""Allocate *n_blocks* for *key*, evicting as needed.
@@ -84,7 +93,10 @@ class EmbeddingCache:
if len(self._free_blocks) + self._evictable_block_count < n_blocks:
return None
self._evict_until(n_blocks)
block_ids = tuple(self._free_blocks.pop() for _ in range(n_blocks))
# Canonical ascending order: keeps a freed entry's blocks contiguous
# in the free list when it is evicted, and lets consumers detect
# which of an entry's blocks are adjacent.
block_ids = tuple(sorted(self._free_blocks.pop() for _ in range(n_blocks)))
entry = CacheEntry(block_ids)
self._entries[key] = entry
return entry
@@ -1,115 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""StepTracker — step-counting delay with first-finish fast-path.
Manages a collection of (mm_hash, request_id) entries that need to be
processed after a configurable number of engine steps have elapsed.
Additionally, if the originating request finishes before the step-count
expires, the entry is processed immediately (the GPU transfer is
guaranteed complete by request finish time).
Used by ECCPUScheduler: one instance for readiness marking, one for
unpins.
"""
from collections import deque
from dataclasses import dataclass
@dataclass(slots=True)
class _PendingEntry:
mm_hash: str
request_id: str
processed: bool = False
class StepTracker:
"""Track pending operations with step-count delay and first-finish.
Each ``add(mm_hash, request_id)`` registers an operation that will be
returned from ``step()`` either when:
(a) ``max_concurrent_batches`` steps elapse (deque expiry), OR
(b) ``request_id`` appears in ``finished_req_ids`` (first-finish).
Each add is processed exactly once — never both (a) and (b).
"""
def __init__(self, max_concurrent_batches: int) -> None:
# One slot per past step; oldest at the right. When the deque is
# full the rightmost slot is popped and its entries expire.
self._slots: deque[list[_PendingEntry]] = deque(maxlen=max_concurrent_batches)
# Reverse index: request_id → entries across all slots + _current.
# Enables O(1) lookup for the first-finish fast-path.
self._req_index: dict[str, list[_PendingEntry]] = {}
# Entries added this step (via add()), committed to a new slot at
# the start of the next step() call.
self._current: list[_PendingEntry] = []
def add(self, mm_hash: str, request_id: str) -> None:
"""Register an mm_hash needing processing after a delay."""
entry = _PendingEntry(mm_hash=mm_hash, request_id=request_id)
self._current.append(entry)
self._req_index.setdefault(request_id, []).append(entry)
def step(self, finished_req_ids: set[str]) -> list[str]:
"""Advance one step. Return mm_hashes ready for processing.
Returns:
List of mm_hashes to process. May contain duplicates if the
same mm_hash was added multiple times (from different requests).
"""
result: list[str] = []
# Phase 1: first-finish fast-path.
for req_id in finished_req_ids:
entries = self._req_index.pop(req_id, None)
if entries is None:
continue
for entry in entries:
if not entry.processed:
entry.processed = True
result.append(entry.mm_hash)
# Phase 2: step-count expiry.
if len(self._slots) == self._slots.maxlen:
expired_slot = self._slots.pop()
for entry in expired_slot:
if not entry.processed:
entry.processed = True
result.append(entry.mm_hash)
self._cleanup_expired(expired_slot)
# Phase 3: commit current entries to new slot.
self._slots.appendleft(self._current)
self._current = []
return result
def drain_all(self) -> list[str]:
"""Return all unprocessed entries and clear internal state."""
result: list[str] = []
for entry in self._current:
if not entry.processed:
entry.processed = True
result.append(entry.mm_hash)
self._current = []
for slot in self._slots:
for entry in slot:
if not entry.processed:
entry.processed = True
result.append(entry.mm_hash)
self._slots.clear()
self._req_index.clear()
return result
def _cleanup_expired(self, slot: list[_PendingEntry]) -> None:
"""Prune reverse index for entries in an expired slot."""
for entry in slot:
entries_for_req = self._req_index.get(entry.request_id)
if entries_for_req is None:
# Already removed by fast-path (pop in step()).
continue
entries_for_req.remove(entry)
if not entries_for_req:
del self._req_index[entry.request_id]
@@ -7,13 +7,19 @@ per-step connector metadata (`ECCPUConnectorMetadata`) to decide which
blocks to copy in each direction.
"""
from collections import deque
from collections.abc import Sequence
from dataclasses import dataclass
from itertools import chain
from typing import TYPE_CHECKING
import numpy as np
import torch
from vllm._custom_ops import swap_blocks_batch
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
ECCPUWorkerMetadata,
create_ec_shared_region,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.worker.descriptor_buffers import (
@@ -34,6 +40,50 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _coalesce_runs(
block_ids: Sequence[int],
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
"""Split block ids into runs that are consecutive in the region.
Returns `(slots, first_blocks, num_blocks)`, one element per run, where
`slots` is the run's offset in the flat block sequence. Per-descriptor
overhead dominates these copies, so describing a run with one descriptor
rather than one per block is worth an order of magnitude.
Both callers derive the non-region side of a run from its slot, so that
side advances in lockstep with the region and only the region has to be
split. `EmbeddingCache` hands out ascending ids, which is what makes runs
findable at all.
"""
ids = np.fromiter(block_ids, dtype=np.int64, count=len(block_ids))
breaks = np.flatnonzero(np.diff(ids) != 1) + 1
slots = np.concatenate(([0], breaks))
ends = np.concatenate((breaks, [ids.size]))
return slots, ids[slots], ends - slots
@dataclass
class Transfer:
"""A batched copy in flight on a dedicated GPU stream.
Its descriptor buffers and stream are held until `end_event` fires; only
then are the buffers and stream safe to recycle and the `completions` safe
to report. `start_event`/`end_event` bracket the copy so its elapsed time
can be measured on completion.
`completions` is what the scheduler is told about this copy: mm_hashes for
saves, transfer ids for loads.
"""
start_event: torch.Event
end_event: torch.Event
completions: list[str] | list[int]
bufs: DescriptorBuffers
stream: torch.Stream
num_bytes: int
class ECCPUWorker:
"""Worker-side delegate for the ECCPUConnector.
@@ -43,8 +93,18 @@ class ECCPUWorker:
batched call in `flush_saves`.
- Consumer role: copies `mmap[block_ids]` → `encoder_cache[mm_hash]`
for all entries in `metadata.loads` via a single `swap_blocks_batch`
call on the load stream.
call on a dedicated stream.
- On `ec_both` nodes both paths run back-to-back in a single step.
Every batched copy runs on a stream drawn from a pool so it overlaps the
compute stream, bracketed by a start/end event pair. The stream, events,
and descriptor buffers are recycled once the end event fires.
Both paths move memory between the two streams, so each buffer is registered
against the stream that is not its own: save sources against the save stream,
load destinations against the compute stream. Without that, the caching
allocator would be free to reissue memory a copy or a queued read still
needs, since it only tracks the stream a buffer was allocated on.
"""
def __init__(self, vllm_config: "VllmConfig") -> None:
@@ -62,15 +122,72 @@ class ECCPUWorker:
get_tensor_model_parallel_rank() == 0 and get_pcp_group().rank_in_group == 0
)
# Dedicated stream for async mmap→GPU loads (overlaps with compute).
self._load_stream = current_platform.Stream()
# Descriptor buffer pool (recycled across steps, shared by both paths).
self._buf_pool = DescriptorBufferPool()
# Active save buffer being filled during save_caches calls this step.
self._save_bufs: DescriptorBuffers | None = None
self._save_count: int = 0
# Bytes accumulated into the active save buffer this step, for logging.
self._save_bytes: int = 0
# mm_hashes accumulated into the active save buffer this step.
self._save_mm_hashes: list[str] = []
# Stream the active save buffer's copy will run on, held from the moment
# the buffer opens so `save_caches` can register its sources against it.
self._save_stream: torch.Stream | None = None
# Batched copies whose end event has not yet fired. Buffers and stream
# stay held until the event fires; the completions are reported to the
# scheduler only then. Separate per direction because saves and loads
# can be in flight concurrently.
self._inflight_saves: deque[Transfer] = deque()
self._inflight_loads: deque[Transfer] = deque()
# Streams and timing events recycled across steps to avoid per-step
# create/destroy churn. Shared by both directions.
self._stream_pool: list[torch.Stream] = []
self._event_pool: list[torch.Event] = []
def _acquire_stream(self) -> torch.Stream:
if self._stream_pool:
return self._stream_pool.pop()
return current_platform.Stream()
def _acquire_event(self) -> torch.Event:
return (
self._event_pool.pop()
if self._event_pool
else torch.Event(enable_timing=True)
)
def _collect_finished(self, inflight: deque[Transfer], direction: str) -> list:
"""Pop transfers whose end event has fired, recycle their stream,
events, and buffers, and return their completions.
The front check is conservative: a transfer is reported only once its
own end event fires, so completions are always genuinely done, and each
is reported exactly once because the transfer is popped as it is
reported. A later transfer that happens to finish first simply waits
behind the front and is reported on a subsequent poll.
"""
done: list = []
while inflight and inflight[0].end_event.query():
transfer = inflight.popleft()
done.extend(transfer.completions)
elapsed_ms = transfer.start_event.elapsed_time(transfer.end_event)
logger.debug(
"EC %s: %d entr%s (%d bytes) took %.3f ms",
direction,
len(transfer.completions),
"y" if len(transfer.completions) == 1 else "ies",
transfer.num_bytes,
elapsed_ms,
)
self._buf_pool.release(transfer.bufs)
self._stream_pool.append(transfer.stream)
self._event_pool.append(transfer.start_event)
self._event_pool.append(transfer.end_event)
return done
def save_caches(
self,
@@ -78,7 +195,14 @@ class ECCPUWorker:
mm_hash: str,
connector_metadata: ECCPUConnectorMetadata,
) -> None:
"""Fill descriptor buffers directly for batched flush."""
"""Fill descriptor buffers directly for batched flush.
Registers `encoder_cache[mm_hash]` against the stream the copy will run
on, so the caching allocator will not hand that memory to another
allocation before the copy has read it. The descriptor buffers hold raw
addresses, and the caller is free to evict the entry as soon as this
step's saves are dispatched.
"""
if not self._is_save_rank:
return
block_ids = connector_metadata.saves.get(mm_hash)
@@ -99,43 +223,95 @@ class ECCPUWorker:
if self._save_bufs is None:
total = sum(len(v) for v in connector_metadata.saves.values())
self._save_bufs = self._buf_pool.acquire(total)
self._save_stream = self._acquire_stream()
self._save_mm_hashes = []
assert self._save_count + len(block_ids) <= self._save_bufs.src_ptrs.numel()
bufs = self._save_bufs
src_base = src.view(-1).view(torch.uint8).data_ptr()
assert self._save_stream is not None
src.record_stream(self._save_stream)
# Device addresses are unsigned and can have the high bit set (XPU USM),
# so they are added up in uint64; signed arithmetic rejects them.
src_base = np.uint64(src.view(-1).view(torch.uint8).data_ptr())
dst_base = self._region.blocks.data_ptr()
idx = self._save_count
# Entries are sized by placeholder count, which can exceed the number
# of embeddings, so blocks the output never reaches get no descriptor.
n_used = -(-total_bytes // block_size)
if n_used:
slots, first_blocks, num_blocks = _coalesce_runs(block_ids[:n_used])
run_bytes = num_blocks * block_size
# The output can stop partway into the final run.
run_bytes[-1] = total_bytes - slots[-1] * block_size
bufs.add_copies(
self._save_count,
src_base + slots.astype(np.uint64) * np.uint64(block_size),
dst_base + first_blocks * block_size,
run_bytes,
)
self._save_count += slots.size
for i, block_idx in enumerate(block_ids):
start = i * block_size
bufs.set_ptrs(idx, src_base + start, dst_base + block_idx * block_size)
bufs.sizes[idx] = min(block_size, total_bytes - start)
idx += 1
self._save_count = idx
self._save_bytes += total_bytes
self._save_mm_hashes.append(mm_hash)
def flush_saves(self) -> None:
"""Flush all accumulated saves in a single swap_blocks_batch call."""
"""Flush all accumulated saves in a single swap_blocks_batch call.
Runs the copy on the stream the batch acquired when it opened, gated
behind the compute stream that produced the encoder outputs, brackets it
with start/end events, and enqueues the batch as in-flight. The stream,
events, and descriptor buffers are recycled once the end event fires (see
`_collect_finished`), which is also when the saved mm_hashes become
safe to mark ready.
"""
if self._save_count == 0:
return
bufs = self._save_bufs
assert bufs is not None
stream = self._save_stream
assert bufs is not None and stream is not None
src_ptrs, dst_ptrs, sizes = bufs.src_ptrs, bufs.dst_ptrs, bufs.sizes
n = self._save_count
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
num_bytes = self._save_bytes
# Gate the GPU→CPU copy behind the compute stream: it reads the encoder
# outputs the model just produced there.
stream.wait_stream(current_platform.current_stream())
start_event = self._acquire_event()
end_event = self._acquire_event()
with current_platform.stream(stream):
start_event.record(stream)
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
end_event.record(stream)
self._inflight_saves.append(
Transfer(
start_event=start_event,
end_event=end_event,
completions=self._save_mm_hashes,
bufs=bufs,
stream=stream,
num_bytes=num_bytes,
)
)
self._buf_pool.release(bufs)
self._save_bufs = None
self._save_stream = None
self._save_count = 0
self._save_bytes = 0
self._save_mm_hashes = []
def start_load_caches(
self,
encoder_cache: dict[str, torch.Tensor],
connector_metadata: ECCPUConnectorMetadata,
) -> None:
"""Consumer path: single batched copy of all loads from mmap→GPU."""
"""Consumer path: single batched copy of all loads from mmap→GPU.
The destination is registered against the compute stream that consumes
it, so the caching allocator will not hand that memory to a later load
while reads of it are still queued there.
"""
if not connector_metadata.loads:
return
@@ -145,59 +321,99 @@ class ECCPUWorker:
device_type = current_platform.device_type
src_base = blocks.data_ptr()
# Pre-filter: only hashes not already in encoder_cache.
load_items = {
h: idxs
for h, idxs in connector_metadata.loads.items()
if h not in encoder_cache
}
if not load_items:
return
# Copy every dispatched load. The scheduler routes an input here only
# after missing the GPU encoder cache, so a resident hit is not expected;
# copying unconditionally means every participating rank reports each
# transfer exactly once, which is what the scheduler counts to release
# the pin.
# Ordered by first block so entries that neighbour each other in the
# region land next to each other in the destination and coalesce into
# one descriptor. Every loop below consumes this same order.
load_items = sorted(
connector_metadata.loads.items(), key=lambda kv: kv[1][1][0]
)
total_blocks = sum(len(block_ids) for _, (_, block_ids) in load_items)
total_blocks = sum(len(idxs) for idxs in load_items.values())
with current_platform.stream(self._load_stream):
stream = self._acquire_stream()
compute_stream = current_platform.current_stream()
start_event = self._acquire_event()
end_event = self._acquire_event()
with current_platform.stream(stream):
# Single contiguous destination buffer for all loads.
dst_buf = torch.empty(
total_blocks, block_size, dtype=torch.int8, device=device_type
)
dst_buf_base = dst_buf.data_ptr()
dst_buf.record_stream(compute_stream)
# See save_caches: device addresses are added up in uint64.
dst_buf_base = np.uint64(dst_buf.data_ptr())
bufs = self._buf_pool.acquire(total_blocks)
src_ptrs = bufs.src_ptrs[:total_blocks]
dst_ptrs = bufs.dst_ptrs[:total_blocks]
sizes = bufs.sizes[:total_blocks]
sizes[:] = block_size
op_idx = 0
for block_ids in load_items.values():
for block_idx in block_ids:
bufs.set_ptrs(
op_idx,
src_base + block_idx * block_size,
dst_buf_base + op_idx * block_size,
)
op_idx += 1
slots, first_blocks, num_blocks = _coalesce_runs(
list(chain.from_iterable(ids for _, (_, ids) in load_items))
)
bufs.add_copies(
0,
src_base + first_blocks * block_size,
dst_buf_base + slots.astype(np.uint64) * np.uint64(block_size),
num_blocks * block_size,
)
op_idx = slots.size
src_ptrs = bufs.src_ptrs[:op_idx]
dst_ptrs = bufs.dst_ptrs[:op_idx]
sizes = bufs.sizes[:op_idx]
start_event.record(stream)
swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True)
self._buf_pool.release(bufs)
end_event.record(stream)
self._inflight_loads.append(
Transfer(
start_event=start_event,
end_event=end_event,
completions=[tid for _, (tid, _) in load_items],
bufs=bufs,
stream=stream,
num_bytes=total_blocks * block_size,
)
)
# Slice contiguous buffer into per-hash views.
offset = 0
for mm_hash, block_ids in load_items.items():
for mm_hash, (_, block_ids) in load_items:
n = len(block_ids)
encoder_cache[mm_hash] = (
dst_buf[offset : offset + n].view(dtype).reshape(n, -1)
)
offset += n
current_platform.current_stream().wait_stream(self._load_stream)
current_platform.current_stream().wait_stream(stream)
def build_connector_worker_meta(self) -> ECCPUWorkerMetadata | None:
"""Report the GPU copies that completed on this rank this step: saved
mm_hashes and loaded transfer ids.
Returns None when nothing finished, so the scheduler sees no payload.
"""
completed_saves = self._collect_finished(self._inflight_saves, "save")
completed_loads = self._collect_finished(self._inflight_loads, "load")
if not completed_saves and not completed_loads:
return None
return ECCPUWorkerMetadata(
completed_saves=completed_saves,
completed_loads=completed_loads,
)
def shutdown(self) -> None:
self._load_stream.synchronize()
for transfer in (*self._inflight_saves, *self._inflight_loads):
transfer.end_event.synchronize()
self._save_bufs = None
self._save_stream = None
self._save_count = 0
self._save_bytes = 0
self._save_mm_hashes = []
self._inflight_saves.clear()
self._inflight_loads.clear()
self._stream_pool.clear()
self._event_pool.clear()
try:
self._region.cleanup()
except Exception:
@@ -22,24 +22,43 @@ class DescriptorBuffers(NamedTuple):
src_ptrs: torch.Tensor
dst_ptrs: torch.Tensor
sizes: torch.Tensor
# Numpy aliases of src_ptrs/dst_ptrs, written via set_ptrs().
# Numpy aliases of the three tensors, written via add_copies().
src_np: np.ndarray
dst_np: np.ndarray
sizes_np: np.ndarray
def set_ptrs(self, idx: int, src: int, dst: int) -> None:
"""Record the source and destination address of descriptor *idx*.
def add_copies(
self,
first: int,
sources: np.ndarray,
destinations: np.ndarray,
byte_counts: np.ndarray,
) -> None:
"""Store a batch of copies in these buffers, the first at *first*.
TODO(torch>=2.14): drop this indirection and assign the tensors
directly once the minimum supported torch is 2.14. The numpy detour
exists only because torch's setitem unpacks the value as a signed
long long before 2.14 (pytorch#191458), rejecting XPU USM pointers
>= 2**63, while the two's-complement rewrite those versions accept is
in turn rejected by uint64 from 2.14 on. Numpy casts against the
array dtype and so works on either. `sizes` holds byte counts and
needs no such care.
One copy is one source address, one destination address, and a number
of bytes. `swap_blocks_batch` performs every copy held in the buffers
in a single call, so a caller fills in all of a step's copies before
asking for any of them to run, and passes *first* to say where its own
copies begin.
Whole batches go in at once because storing a single number into a
torch tensor costs microseconds. Adding copies one at a time can take
longer than performing them.
TODO(torch>=2.14): write the tensors directly and drop these numpy
aliases once the minimum supported torch is 2.14. They exist only
because torch's setitem unpacks the value as a signed long long before
2.14 (pytorch#191458), rejecting XPU USM addresses >= 2**63, while the
two's-complement rewrite those versions accept is in turn rejected by
uint64 from 2.14 on. Numpy casts against the array's own type and so
works on either. Byte counts are small enough to need none of this
care, and use their alias only to match the addresses.
"""
self.src_np[idx] = src
self.dst_np[idx] = dst
last = first + len(sources)
self.src_np[first:last] = sources
self.dst_np[first:last] = destinations
self.sizes_np[first:last] = byte_counts
class DescriptorBufferPool:
@@ -62,7 +81,9 @@ class DescriptorBufferPool:
if bufs.src_ptrs.numel() >= n:
return bufs
src, dst, sizes = (torch.empty(n, dtype=_PTR_DTYPE) for _ in range(3))
return DescriptorBuffers(src, dst, sizes, src.numpy(), dst.numpy())
return DescriptorBuffers(
src, dst, sizes, src.numpy(), dst.numpy(), sizes.numpy()
)
def release(self, bufs: DescriptorBuffers) -> None:
"""Return a buffer triple to the pool for reuse."""