[Refactor][EC Connector] Add backend extension points to ECCPUWorker (#54033)

Signed-off-by: Akine-Ko <2923601304@qq.com>
This commit is contained in:
Akine-Ko
2026-09-08 20:13:47 -07:00
committed by GitHub
parent f1aeff6007
commit 41541636fd
2 changed files with 117 additions and 9 deletions
@@ -10,6 +10,7 @@ blocks to copy in each direction.
from collections import deque
from collections.abc import Sequence
from dataclasses import dataclass
from enum import Enum, auto
from itertools import chain
from typing import TYPE_CHECKING
@@ -40,6 +41,11 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
class ECCPUTransferDirection(Enum):
HOST_TO_DEVICE = auto()
DEVICE_TO_HOST = auto()
def _coalesce_runs(
block_ids: Sequence[int],
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
@@ -111,8 +117,7 @@ class ECCPUWorker:
# Model dtype; used to reinterpret raw int8 blocks on load.
self._dtype = vllm_config.model_config.dtype
if is_pin_memory_available():
self._region.pin_memory()
self._pin_shared_region()
# All TP/PCP ranks hold identical encoder output. Only one rank
# per mmap needs to write — saves host memory bandwidth.
@@ -188,6 +193,34 @@ class ECCPUWorker:
self._event_pool.append(transfer.end_event)
return done
def _pin_shared_region(self) -> None:
"""Prepare the shared region for device transfers."""
if is_pin_memory_available():
self._region.pin_memory()
def _submit_transfer(
self,
bufs: DescriptorBuffers,
count: int,
direction: ECCPUTransferDirection,
) -> None:
"""Submit a transfer on the current backend stream."""
src_ptrs = bufs.src_ptrs[:count]
dst_ptrs = bufs.dst_ptrs[:count]
sizes = bufs.sizes[:count]
if direction == ECCPUTransferDirection.HOST_TO_DEVICE:
swap_blocks_batch(
src_ptrs,
dst_ptrs,
sizes,
is_src_access_order_any=True,
)
else:
swap_blocks_batch(src_ptrs, dst_ptrs, sizes)
def _shutdown_transfer_backend(self) -> None:
"""Release backend-specific transfer resources."""
def save_caches(
self,
encoder_cache: dict[str, torch.Tensor],
@@ -269,7 +302,6 @@ class ECCPUWorker:
bufs = self._save_bufs
stream = self._save_stream
assert bufs is not None and stream is not None
src_ptrs, dst_ptrs, sizes = bufs.src_ptrs, bufs.dst_ptrs, bufs.sizes
n = self._save_count
num_bytes = self._save_bytes
@@ -280,7 +312,11 @@ class ECCPUWorker:
end_event = self._acquire_event()
with current_platform.stream(stream):
start_event.record(stream)
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
self._submit_transfer(
bufs,
n,
ECCPUTransferDirection.DEVICE_TO_HOST,
)
end_event.record(stream)
self._inflight_saves.append(
@@ -357,12 +393,13 @@ class ECCPUWorker:
num_blocks * block_size,
)
op_idx = slots.size
src_ptrs = bufs.src_ptrs[:op_idx]
dst_ptrs = bufs.dst_ptrs[:op_idx]
sizes = bufs.sizes[:op_idx]
start_event.record(stream)
swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True)
self._submit_transfer(
bufs,
op_idx,
ECCPUTransferDirection.HOST_TO_DEVICE,
)
end_event.record(stream)
self._inflight_loads.append(
Transfer(
@@ -404,6 +441,7 @@ class ECCPUWorker:
def shutdown(self) -> None:
for transfer in (*self._inflight_saves, *self._inflight_loads):
transfer.end_event.synchronize()
self._shutdown_transfer_backend()
self._save_bufs = None
self._save_stream = None
self._save_count = 0