[Refactor][EC Connector] Add backend extension points to ECCPUWorker (#54033)
Signed-off-by: Akine-Ko <2923601304@qq.com>
This commit is contained in:
@@ -10,6 +10,7 @@ blocks to copy in each direction.
|
||||
from collections import deque
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -40,6 +41,11 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ECCPUTransferDirection(Enum):
|
||||
HOST_TO_DEVICE = auto()
|
||||
DEVICE_TO_HOST = auto()
|
||||
|
||||
|
||||
def _coalesce_runs(
|
||||
block_ids: Sequence[int],
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
@@ -111,8 +117,7 @@ class ECCPUWorker:
|
||||
# Model dtype; used to reinterpret raw int8 blocks on load.
|
||||
self._dtype = vllm_config.model_config.dtype
|
||||
|
||||
if is_pin_memory_available():
|
||||
self._region.pin_memory()
|
||||
self._pin_shared_region()
|
||||
|
||||
# All TP/PCP ranks hold identical encoder output. Only one rank
|
||||
# per mmap needs to write — saves host memory bandwidth.
|
||||
@@ -188,6 +193,34 @@ class ECCPUWorker:
|
||||
self._event_pool.append(transfer.end_event)
|
||||
return done
|
||||
|
||||
def _pin_shared_region(self) -> None:
|
||||
"""Prepare the shared region for device transfers."""
|
||||
if is_pin_memory_available():
|
||||
self._region.pin_memory()
|
||||
|
||||
def _submit_transfer(
|
||||
self,
|
||||
bufs: DescriptorBuffers,
|
||||
count: int,
|
||||
direction: ECCPUTransferDirection,
|
||||
) -> None:
|
||||
"""Submit a transfer on the current backend stream."""
|
||||
src_ptrs = bufs.src_ptrs[:count]
|
||||
dst_ptrs = bufs.dst_ptrs[:count]
|
||||
sizes = bufs.sizes[:count]
|
||||
if direction == ECCPUTransferDirection.HOST_TO_DEVICE:
|
||||
swap_blocks_batch(
|
||||
src_ptrs,
|
||||
dst_ptrs,
|
||||
sizes,
|
||||
is_src_access_order_any=True,
|
||||
)
|
||||
else:
|
||||
swap_blocks_batch(src_ptrs, dst_ptrs, sizes)
|
||||
|
||||
def _shutdown_transfer_backend(self) -> None:
|
||||
"""Release backend-specific transfer resources."""
|
||||
|
||||
def save_caches(
|
||||
self,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
@@ -269,7 +302,6 @@ class ECCPUWorker:
|
||||
bufs = self._save_bufs
|
||||
stream = self._save_stream
|
||||
assert bufs is not None and stream is not None
|
||||
src_ptrs, dst_ptrs, sizes = bufs.src_ptrs, bufs.dst_ptrs, bufs.sizes
|
||||
n = self._save_count
|
||||
num_bytes = self._save_bytes
|
||||
|
||||
@@ -280,7 +312,11 @@ class ECCPUWorker:
|
||||
end_event = self._acquire_event()
|
||||
with current_platform.stream(stream):
|
||||
start_event.record(stream)
|
||||
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
|
||||
self._submit_transfer(
|
||||
bufs,
|
||||
n,
|
||||
ECCPUTransferDirection.DEVICE_TO_HOST,
|
||||
)
|
||||
end_event.record(stream)
|
||||
|
||||
self._inflight_saves.append(
|
||||
@@ -357,12 +393,13 @@ class ECCPUWorker:
|
||||
num_blocks * block_size,
|
||||
)
|
||||
op_idx = slots.size
|
||||
src_ptrs = bufs.src_ptrs[:op_idx]
|
||||
dst_ptrs = bufs.dst_ptrs[:op_idx]
|
||||
sizes = bufs.sizes[:op_idx]
|
||||
|
||||
start_event.record(stream)
|
||||
swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True)
|
||||
self._submit_transfer(
|
||||
bufs,
|
||||
op_idx,
|
||||
ECCPUTransferDirection.HOST_TO_DEVICE,
|
||||
)
|
||||
end_event.record(stream)
|
||||
self._inflight_loads.append(
|
||||
Transfer(
|
||||
@@ -404,6 +441,7 @@ class ECCPUWorker:
|
||||
def shutdown(self) -> None:
|
||||
for transfer in (*self._inflight_saves, *self._inflight_loads):
|
||||
transfer.end_event.synchronize()
|
||||
self._shutdown_transfer_backend()
|
||||
self._save_bufs = None
|
||||
self._save_stream = None
|
||||
self._save_count = 0
|
||||
|
||||
Reference in New Issue
Block a user