[Mini-VLLM] Remove orphan subsystems: snapshot, CPU workers, dead decoders
- Delete the snapshot package, CLI command, and its launch tests - Delete CPU workers, cpu_communicator, and the hw_agnostic layer package - Delete the EC CPU connector, keeping the EC base glue intact - Delete dead composite/triton_flashinfer backends and third_party/flashmla - Delete dead video decoders, video-bound tests, and orphan utils - Remove the deep_gemm_mega_moe option and dead allreduce fusion pair Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,339 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch.distributed import ProcessGroup
|
||||
|
||||
from vllm.distributed.utils import pickle
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.interface import CpuArchEnum
|
||||
|
||||
from .base_device_communicator import DeviceCommunicatorBase
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CpuCommunicator(DeviceCommunicatorBase):
|
||||
def __init__(
|
||||
self,
|
||||
cpu_group: ProcessGroup,
|
||||
device: torch.device | None = None,
|
||||
device_group: ProcessGroup | None = None,
|
||||
unique_name: str = "",
|
||||
use_all2all: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
cpu_group, device, device_group, unique_name, use_all2all=use_all2all
|
||||
)
|
||||
self.dist_module = torch.distributed
|
||||
|
||||
if (
|
||||
(
|
||||
current_platform.get_cpu_architecture() == CpuArchEnum.X86
|
||||
or current_platform.get_cpu_architecture() == CpuArchEnum.ARM
|
||||
or current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC
|
||||
or current_platform.get_cpu_architecture() == CpuArchEnum.S390X
|
||||
)
|
||||
and hasattr(torch.ops._C, "init_shm_manager")
|
||||
and (unique_name.startswith("tp") or unique_name.startswith("pp"))
|
||||
and self._all_group_ranks_share_shm_group_name()
|
||||
):
|
||||
self.dist_module = _CPUSHMDistributed(self)
|
||||
elif unique_name.startswith("tp") or unique_name.startswith("pp"):
|
||||
logger.info(
|
||||
"CPU SHM communicator disabled for group %s: ranks do not share "
|
||||
"the same SHM group name, falling back to torch.distributed.",
|
||||
unique_name,
|
||||
)
|
||||
|
||||
# send/recv tensor_dict is only supported through the SHM communicator backend
|
||||
self.supports_tensor_dict = isinstance(self.dist_module, _CPUSHMDistributed)
|
||||
|
||||
if self.use_all2all:
|
||||
if self.all2all_backend not in (
|
||||
"naive",
|
||||
"allgather_reducescatter",
|
||||
): # type: ignore[has-type]
|
||||
logger.warning(
|
||||
"`%s` all2all manager is not supported on CPU. "
|
||||
"Falling back to `allgather_reducescatter` manager.",
|
||||
self.all2all_backend, # type: ignore[has-type]
|
||||
)
|
||||
from .all2all import AgRsAll2AllManager
|
||||
|
||||
self.all2all_manager = AgRsAll2AllManager(self.cpu_group)
|
||||
logger.info("Using allgather_reducescatter all2all manager.")
|
||||
|
||||
def _all_group_ranks_share_shm_group_name(self) -> bool:
|
||||
"""CPUSHM requires all ranks in this group to agree on one SHM group name.
|
||||
This is a lightweight consistency check for VLLM_DIST_IDENT/name inputs.
|
||||
"""
|
||||
local_name = _CPUSHMDistributed.make_group_name(self)
|
||||
names: list[str] = [""] * self.world_size
|
||||
torch.distributed.all_gather_object(
|
||||
names,
|
||||
local_name,
|
||||
group=self.device_group,
|
||||
)
|
||||
return len(set(names)) == 1
|
||||
|
||||
def all_reduce(self, input_):
|
||||
self.dist_module.all_reduce(input_, group=self.device_group)
|
||||
return input_
|
||||
|
||||
def gather(
|
||||
self, input_: torch.Tensor, dst: int = 0, dim: int = -1
|
||||
) -> torch.Tensor | None:
|
||||
"""NOTE: We assume that the input tensor is on the same device across
|
||||
all the ranks.
|
||||
NOTE: `dst` is the local rank of the destination rank.
|
||||
"""
|
||||
world_size = self.world_size
|
||||
assert -input_.dim() <= dim < input_.dim(), (
|
||||
f"Invalid dim ({dim}) for input tensor with shape {input_.size()}"
|
||||
)
|
||||
if dim < 0:
|
||||
# Convert negative dim to positive.
|
||||
dim += input_.dim()
|
||||
|
||||
# Allocate output tensor.
|
||||
if self.rank_in_group == dst:
|
||||
gather_list = [torch.empty_like(input_) for _ in range(world_size)]
|
||||
else:
|
||||
gather_list = None
|
||||
|
||||
# Gather.
|
||||
self.dist_module.gather(
|
||||
input_, gather_list, dst=self.ranks[dst], group=self.device_group
|
||||
)
|
||||
|
||||
if self.rank_in_group == dst:
|
||||
output_tensor = torch.cat(gather_list, dim=dim)
|
||||
else:
|
||||
output_tensor = None
|
||||
return output_tensor
|
||||
|
||||
def all_gather(self, input_: torch.Tensor, dim: int = -1) -> torch.Tensor:
|
||||
if dim < 0:
|
||||
# Convert negative dim to positive.
|
||||
dim += input_.dim()
|
||||
input_size = input_.size()
|
||||
# NOTE: we have to use concat-style all-gather here,
|
||||
# stack-style all-gather has compatibility issues with
|
||||
# torch.compile . see https://github.com/pytorch/pytorch/issues/138795
|
||||
output_size = (input_size[0] * self.world_size,) + input_size[1:]
|
||||
# Allocate output tensor.
|
||||
output_tensor = torch.empty(
|
||||
output_size, dtype=input_.dtype, device=input_.device
|
||||
)
|
||||
# All-gather.
|
||||
self.dist_module.all_gather_into_tensor(
|
||||
output_tensor, input_, group=self.device_group
|
||||
)
|
||||
|
||||
# Reshape
|
||||
output_tensor = output_tensor.reshape((self.world_size,) + input_size)
|
||||
output_tensor = output_tensor.movedim(0, dim)
|
||||
output_tensor = output_tensor.reshape(
|
||||
input_size[:dim]
|
||||
+ (self.world_size * input_size[dim],)
|
||||
+ input_size[dim + 1 :]
|
||||
)
|
||||
return output_tensor
|
||||
|
||||
def send_tensor_dict(
|
||||
self,
|
||||
tensor_dict: dict[str, torch.Tensor | Any],
|
||||
dst: int,
|
||||
) -> None:
|
||||
if not self.supports_tensor_dict:
|
||||
raise NotImplementedError(
|
||||
"CpuCommunicator does not support tensor dict fastpath with "
|
||||
"torch.distributed backend."
|
||||
)
|
||||
return self.dist_module.send_tensor_dict(tensor_dict, dst)
|
||||
|
||||
def recv_tensor_dict(
|
||||
self,
|
||||
src: int,
|
||||
) -> dict[str, torch.Tensor | Any]:
|
||||
if not self.supports_tensor_dict:
|
||||
raise NotImplementedError(
|
||||
"CpuCommunicator does not support tensor dict fastpath with "
|
||||
"torch.distributed backend."
|
||||
)
|
||||
return self.dist_module.recv_tensor_dict(src)
|
||||
|
||||
def dispatch_router_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
router_logits: torch.Tensor,
|
||||
is_sequence_parallel: bool = False,
|
||||
extra_tensors: list[torch.Tensor] | None = None,
|
||||
) -> (
|
||||
tuple[torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""Dispatch the hidden states and router logits to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch_router_logits(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
is_sequence_parallel,
|
||||
extra_tensors,
|
||||
)
|
||||
|
||||
def dispatch(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
is_sequence_parallel: bool = False,
|
||||
extra_tensors: list[torch.Tensor] | None = None,
|
||||
) -> (
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
||||
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, list[torch.Tensor]]
|
||||
):
|
||||
"""Dispatch the hidden states and topk weights/ids to the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.dispatch(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
is_sequence_parallel,
|
||||
extra_tensors=extra_tensors,
|
||||
)
|
||||
|
||||
def combine(
|
||||
self, hidden_states: torch.Tensor, is_sequence_parallel: bool = False
|
||||
) -> torch.Tensor:
|
||||
"""Combine the hidden states and router logits from the appropriate device.
|
||||
This is a no-op in the base class.
|
||||
"""
|
||||
assert self.all2all_manager is not None
|
||||
return self.all2all_manager.combine(
|
||||
hidden_states,
|
||||
is_sequence_parallel,
|
||||
)
|
||||
|
||||
|
||||
class _CPUSHMDistributed:
|
||||
def __init__(self, communicator: CpuCommunicator):
|
||||
self.communicator = communicator
|
||||
|
||||
self.group_name = self.make_group_name(communicator)
|
||||
|
||||
self.handle = self._init_cpu_shm()
|
||||
|
||||
@staticmethod
|
||||
def make_group_name(communicator: CpuCommunicator) -> str:
|
||||
instance_identifier = os.environ["VLLM_DIST_IDENT"]
|
||||
unique_name = communicator.unique_name
|
||||
instance_identifier = f"{instance_identifier}-{unique_name}"
|
||||
group_ranks = [str(rank) for rank in communicator.ranks]
|
||||
shm_group_identifier = f"[{'-'.join(group_ranks)}]"
|
||||
return f"{instance_identifier}-{shm_group_identifier}-cpushm"
|
||||
|
||||
def _init_cpu_shm(self) -> int:
|
||||
thread_num_tensor = torch.tensor(
|
||||
[torch.get_num_threads()],
|
||||
dtype=torch.int64,
|
||||
)
|
||||
torch.distributed.all_reduce(
|
||||
thread_num_tensor,
|
||||
op=torch.distributed.ReduceOp.MIN,
|
||||
group=self.communicator.device_group,
|
||||
)
|
||||
thread_num = thread_num_tensor.item()
|
||||
|
||||
handle = torch.ops._C.init_shm_manager(
|
||||
self.group_name,
|
||||
self.communicator.world_size,
|
||||
self.communicator.rank,
|
||||
thread_num,
|
||||
)
|
||||
torch.distributed.barrier(self.communicator.device_group)
|
||||
torch.ops._C.join_shm_manager(
|
||||
handle,
|
||||
self.group_name,
|
||||
)
|
||||
torch.distributed.barrier(self.communicator.device_group)
|
||||
|
||||
return handle
|
||||
|
||||
def all_reduce(
|
||||
self, input: torch.Tensor, group: ProcessGroup | None = None
|
||||
) -> None:
|
||||
torch.ops._C.shm_allreduce(self.handle, input)
|
||||
|
||||
def gather(
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
gather_list: list[torch.Tensor] | None,
|
||||
dst: int = -1,
|
||||
group: ProcessGroup | None = None,
|
||||
) -> None:
|
||||
# Note: different from the torch gather, here we use local dst rank.
|
||||
torch.ops._C.shm_gather(
|
||||
self.handle,
|
||||
input,
|
||||
gather_list,
|
||||
torch.distributed.get_group_rank(group, dst),
|
||||
)
|
||||
|
||||
def all_gather_into_tensor(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
group: ProcessGroup | None = None,
|
||||
) -> None:
|
||||
torch.ops._C.shm_all_gather(self.handle, input, output)
|
||||
|
||||
def send_tensor_dict(
|
||||
self,
|
||||
tensor_dict: dict[str, torch.Tensor | Any],
|
||||
dst: int,
|
||||
) -> None:
|
||||
key_list = list(tensor_dict.keys())
|
||||
value_list = list(tensor_dict.values())
|
||||
size_list = []
|
||||
for v in value_list:
|
||||
if not isinstance(v, torch.Tensor):
|
||||
raise RuntimeError("CpuCommunicator only supports sending tensors.")
|
||||
size_list.append(v.size())
|
||||
key_size_tensor = torch.frombuffer(
|
||||
pickle.dumps([key_list, size_list]), dtype=torch.uint8
|
||||
)
|
||||
value_list.append(key_size_tensor)
|
||||
|
||||
torch.ops._C.shm_send_tensor_list(self.handle, value_list, dst)
|
||||
|
||||
return None
|
||||
|
||||
def recv_tensor_dict(
|
||||
self,
|
||||
src: int,
|
||||
) -> dict[str, torch.Tensor | Any]:
|
||||
tensor_list = torch.ops._C.shm_recv_tensor_list(self.handle, src)
|
||||
|
||||
value_list: list[torch.Tensor] = tensor_list[:-1]
|
||||
key_size_tensor = tensor_list[-1]
|
||||
|
||||
key_size = pickle.loads(key_size_tensor.numpy().tobytes())
|
||||
key_list = key_size[0]
|
||||
size_list = key_size[1]
|
||||
assert len(key_list) == len(size_list)
|
||||
assert len(key_list) == len(value_list)
|
||||
|
||||
tensor_dict: dict[str, torch.Tensor] = {}
|
||||
for key, size, t in zip(key_list, size_list, value_list):
|
||||
tensor_dict[key] = t.view(size)
|
||||
return tensor_dict
|
||||
@@ -1,98 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared types for the ECCPUConnector scheduler and worker delegates."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorMetadata,
|
||||
ECConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECCPUConnectorMetadata(ECConnectorMetadata):
|
||||
"""Per-step scheduler → worker payload for the ECCPUConnector.
|
||||
|
||||
Populated by `ECCPUScheduler.build_connector_meta`; consumed by
|
||||
`ECCPUWorker` via the mixin's `bind_connector_metadata`.
|
||||
"""
|
||||
|
||||
# Producer role: mm_hashes the scheduler has just allocated CPU
|
||||
# blocks for this step; the worker's save_caches copies
|
||||
# encoder_cache[mm_hash] → mmap at these block IDs.
|
||||
saves: dict[str, list[int]] = field(default_factory=dict)
|
||||
|
||||
# Consumer role: mm_hashes whose bytes are available in the local mmap,
|
||||
# mapped to (transfer_id, block_ids, shape); the worker's start_load_caches
|
||||
# copies mmap[block_ids] → GPU encoder_cache and reports the transfer_id
|
||||
# once its copy lands. The id is minted per dispatch, so two loads of the
|
||||
# same mm_hash in different steps are distinct and a late report from the
|
||||
# earlier one cannot release the later one's pin.
|
||||
loads: dict[str, tuple[int, list[int], tuple[int, int]]] = field(
|
||||
default_factory=dict
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECCPUWorkerMetadata(ECConnectorWorkerMetadata):
|
||||
"""Per-step worker → scheduler payload for the ECCPUConnector.
|
||||
|
||||
Reports the GPU copies that completed this step: saved mm_hashes become
|
||||
safe to mark ready, and loaded transfers become safe to unpin once every
|
||||
participating rank has reported them. Built by
|
||||
`ECCPUWorker.build_connector_worker_meta`; consumed by
|
||||
`ECCPUScheduler.update_connector_output`.
|
||||
|
||||
Loads are reported by transfer id rather than mm_hash because every rank
|
||||
copies the same blocks: `aggregate` concatenates, so an id appears once
|
||||
per reporting rank and the scheduler can count participants off the list.
|
||||
"""
|
||||
|
||||
completed_saves: list[str] = field(default_factory=list)
|
||||
completed_loads: list[int] = field(default_factory=list)
|
||||
|
||||
def aggregate(self, other: ECConnectorWorkerMetadata) -> ECConnectorWorkerMetadata:
|
||||
assert isinstance(other, ECCPUWorkerMetadata)
|
||||
self.completed_saves.extend(other.completed_saves)
|
||||
self.completed_loads.extend(other.completed_loads)
|
||||
return self
|
||||
|
||||
|
||||
def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion:
|
||||
"""Build the EC mmap region from `vllm_config`.
|
||||
|
||||
Both `ECCPUScheduler` and `ECCPUWorker` call this to get the same
|
||||
shared region (same engine_id, same block_size_bytes).
|
||||
"""
|
||||
ec_config = vllm_config.ec_transfer_config
|
||||
assert ec_config is not None, "ec_transfer_config required to build region"
|
||||
|
||||
dp_index = vllm_config.parallel_config.data_parallel_index
|
||||
engine_id = f"{vllm_config.instance_id}_dp{dp_index}"
|
||||
|
||||
dtype = vllm_config.model_config.dtype
|
||||
hidden_dim = vllm_config.model_config.get_inputs_embeds_size()
|
||||
element_size = torch.empty(0, dtype=dtype).element_size()
|
||||
block_size_bytes = hidden_dim * element_size
|
||||
|
||||
cpu_bytes = ec_config.ec_connector_extra_config.get("ec_cpu_bytes")
|
||||
if not cpu_bytes:
|
||||
raise ValueError("ec_cpu_bytes must be specified in ec_connector_extra_config")
|
||||
cpu_bytes = int(cpu_bytes)
|
||||
num_blocks = cpu_bytes // block_size_bytes
|
||||
|
||||
return ECSharedRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
block_size_bytes=block_size_bytes,
|
||||
)
|
||||
@@ -1,153 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ECCPUConnector — CPU encoder-cache offloading.
|
||||
|
||||
A role-routed shell: one instance per process. The scheduler delegate owns the
|
||||
mmap region and offload bookkeeping; the worker delegate owns the mmap view and
|
||||
GPU copy plumbing. An ec_both instance reuses encoder outputs it has already
|
||||
offloaded to CPU instead of recomputing them.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorBase,
|
||||
ECConnectorRole,
|
||||
ECConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ECConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ECCPUConnector(ECConnectorBase):
|
||||
"""EC connector that offloads encoder cache to a shared CPU mmap region."""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole) -> None:
|
||||
super().__init__(vllm_config=vllm_config, role=role)
|
||||
|
||||
if not vllm_config.use_v2_model_runner:
|
||||
raise ValueError(
|
||||
"ECCPUConnector requires the V2 model runner. "
|
||||
"Set VLLM_USE_V2_MODEL_RUNNER=1."
|
||||
)
|
||||
|
||||
self.connector_worker = None
|
||||
self.connector_scheduler = None
|
||||
|
||||
if role == ECConnectorRole.WORKER:
|
||||
self.connector_worker = self._make_worker(vllm_config)
|
||||
elif role == ECConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = self._make_scheduler(vllm_config)
|
||||
else:
|
||||
raise ValueError(f"Unknown ECConnectorRole: {role}")
|
||||
|
||||
# Construction seams.
|
||||
def _make_worker(self, vllm_config: "VllmConfig"):
|
||||
# Deferred import: the worker module touches torch/CUDA at import time
|
||||
# via the region, so keep that cost off the scheduler path.
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import (
|
||||
ECCPUWorker,
|
||||
)
|
||||
|
||||
return ECCPUWorker(vllm_config)
|
||||
|
||||
def _make_scheduler(self, vllm_config: "VllmConfig"):
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
|
||||
ECCPUScheduler,
|
||||
)
|
||||
|
||||
return ECCPUScheduler(vllm_config)
|
||||
|
||||
# Worker-side forwarders.
|
||||
def start_load_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], **kwargs
|
||||
) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, ECCPUConnectorMetadata)
|
||||
self.connector_worker.start_load_caches(
|
||||
encoder_cache, connector_metadata=metadata
|
||||
)
|
||||
|
||||
def save_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs
|
||||
) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, ECCPUConnectorMetadata)
|
||||
self.connector_worker.save_caches(
|
||||
encoder_cache, mm_hash, connector_metadata=metadata
|
||||
)
|
||||
|
||||
# Scheduler-side forwarders.
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.has_cache_item(identifier)
|
||||
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.ensure_cache_available(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
|
||||
def take_unavailable_requests(self) -> set[str]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.take_unavailable_requests()
|
||||
|
||||
def update_state_after_alloc(self, request: "Request", index: int) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_state_after_alloc(request, index)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> ECCPUConnectorMetadata:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def request_finished(
|
||||
self, request: "Request"
|
||||
) -> tuple[bool, "dict[str, Any] | None"]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.flush_saves()
|
||||
return None, None
|
||||
|
||||
def build_connector_worker_meta(self) -> ECConnectorWorkerMetadata | None:
|
||||
if self.connector_worker is not None:
|
||||
return self.connector_worker.build_connector_worker_meta()
|
||||
return None
|
||||
|
||||
def update_connector_output(self, connector_output: "ECConnectorOutput") -> None:
|
||||
if self.connector_scheduler is not None:
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
if self.connector_scheduler is not None:
|
||||
return self.connector_scheduler.has_pending_push_work()
|
||||
return False
|
||||
|
||||
# Shared.
|
||||
def shutdown(self) -> None:
|
||||
if self.connector_scheduler is not None:
|
||||
self.connector_scheduler.shutdown()
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.shutdown()
|
||||
@@ -1,37 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Abstract base classes for the EC control plane.
|
||||
|
||||
ControlConnection wraps a single peer channel (one DEALER socket).
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
|
||||
class ControlConnection(ABC):
|
||||
"""Bidirectional message channel to a single remote peer.
|
||||
|
||||
All I/O is non-blocking: send enqueues without blocking; recv drains
|
||||
all buffered inbound messages since the last call.
|
||||
"""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def alive(self) -> bool:
|
||||
"""True if the connection is still usable."""
|
||||
|
||||
@abstractmethod
|
||||
def send(self, msg: bytes) -> None:
|
||||
"""Enqueue msg for delivery. Must not block."""
|
||||
|
||||
@abstractmethod
|
||||
def recv(self) -> list[bytes]:
|
||||
"""Drain and return all buffered inbound messages."""
|
||||
|
||||
@abstractmethod
|
||||
def mark_dead(self) -> None:
|
||||
"""Signal that the peer disconnected. Sets alive to False."""
|
||||
|
||||
@abstractmethod
|
||||
def close(self) -> None:
|
||||
"""Release all resources. Idempotent."""
|
||||
@@ -1,233 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ZMQ control-plane implementations for ECCPUConnector.
|
||||
|
||||
ZmqClientConnection — one DEALER socket to a single producer peer.
|
||||
ZmqClientTransport — pool of ZmqClientConnection objects (consumer side).
|
||||
ZmqServerTransport — ROUTER socket (producer side).
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
import zmq
|
||||
from zmq.utils.monitor import recv_monitor_message
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.base import (
|
||||
ControlConnection,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.network_utils import make_zmq_path, make_zmq_socket
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_HEARTBEAT_IVL_MS = 2000
|
||||
_HEARTBEAT_TIMEOUT_MS = 4000
|
||||
_HEARTBEAT_TTL_MS = 8000
|
||||
|
||||
|
||||
# ── Client side ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ZmqClientConnection(ControlConnection):
|
||||
"""One DEALER socket to one producer peer.
|
||||
|
||||
Owns the socket and its optional monitor socket. NIXL state is NOT
|
||||
stored here — it belongs in ECSession above this layer.
|
||||
"""
|
||||
|
||||
dealer: zmq.Socket
|
||||
monitor: zmq.Socket | None
|
||||
_alive: bool = field(default=True, init=False)
|
||||
|
||||
@property
|
||||
def alive(self) -> bool:
|
||||
return self._alive
|
||||
|
||||
def send(self, msg: bytes) -> None:
|
||||
self.dealer.send_multipart([b"", msg])
|
||||
|
||||
def recv(self) -> list[bytes]:
|
||||
msgs: list[bytes] = []
|
||||
while True:
|
||||
try:
|
||||
frames = self.dealer.recv_multipart(flags=zmq.NOBLOCK)
|
||||
except zmq.Again:
|
||||
break
|
||||
except Exception:
|
||||
logger.exception("ec: DEALER recv failed")
|
||||
break
|
||||
if len(frames) != 2 or frames[0] != b"":
|
||||
logger.warning(
|
||||
"ec: dropped malformed frame envelope (%d frames)", len(frames)
|
||||
)
|
||||
continue
|
||||
msgs.append(frames[1])
|
||||
return msgs
|
||||
|
||||
def mark_dead(self) -> None:
|
||||
self._alive = False
|
||||
|
||||
def close(self) -> None:
|
||||
if self.monitor is not None:
|
||||
with contextlib.suppress(Exception):
|
||||
self.dealer.disable_monitor()
|
||||
with contextlib.suppress(Exception):
|
||||
self.monitor.close(linger=0)
|
||||
with contextlib.suppress(Exception):
|
||||
self.dealer.close(linger=0)
|
||||
self._alive = False
|
||||
|
||||
|
||||
class ZmqClientTransport:
|
||||
"""Pool of ZmqClientConnection objects — one per producer peer.
|
||||
|
||||
Creates DEALER sockets lazily on first contact. Does not touch NIXL;
|
||||
NIXL registration is handled by ECCPUConsumer once an XferAck arrives.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._ctx = zmq.Context()
|
||||
self._connections: dict[PeerAddr, ZmqClientConnection] = {}
|
||||
|
||||
def connect(self, addr: PeerAddr) -> ZmqClientConnection:
|
||||
"""Return the existing connection for addr, or create a new one."""
|
||||
existing = self._connections.get(addr)
|
||||
if existing is not None:
|
||||
return existing
|
||||
|
||||
host, port = addr
|
||||
dealer = make_zmq_socket(
|
||||
ctx=self._ctx,
|
||||
path=make_zmq_path(scheme="tcp", host=host, port=port),
|
||||
socket_type=zmq.DEALER,
|
||||
bind=False,
|
||||
)
|
||||
dealer.setsockopt(zmq.HEARTBEAT_IVL, _HEARTBEAT_IVL_MS)
|
||||
dealer.setsockopt(zmq.HEARTBEAT_TIMEOUT, _HEARTBEAT_TIMEOUT_MS)
|
||||
dealer.setsockopt(zmq.HEARTBEAT_TTL, _HEARTBEAT_TTL_MS)
|
||||
|
||||
monitor_addr = f"inproc://ec-peer-mon-{host}-{port}"
|
||||
dealer.monitor(monitor_addr, zmq.EVENT_DISCONNECTED)
|
||||
monitor = self._ctx.socket(zmq.PAIR)
|
||||
monitor.connect(monitor_addr)
|
||||
|
||||
conn = ZmqClientConnection(dealer=dealer, monitor=monitor)
|
||||
self._connections[addr] = conn
|
||||
return conn
|
||||
|
||||
def poll(self) -> dict[PeerAddr, list[bytes]]:
|
||||
"""Non-blocking drain of messages from every peer's DEALER."""
|
||||
msgs: dict[PeerAddr, list[bytes]] = {}
|
||||
for addr, conn in list(self._connections.items()):
|
||||
conn_msgs = conn.recv()
|
||||
msgs[addr] = conn_msgs
|
||||
return msgs
|
||||
|
||||
def poll_dead(self) -> list[PeerAddr]:
|
||||
"""Non-blocking sweep of peer monitors; return addresses that died."""
|
||||
dead: list[PeerAddr] = []
|
||||
for addr, conn in list(self._connections.items()):
|
||||
if conn.monitor is None:
|
||||
continue
|
||||
try:
|
||||
while True:
|
||||
evt = recv_monitor_message(conn.monitor, flags=zmq.NOBLOCK)
|
||||
if evt["event"] == zmq.EVENT_DISCONNECTED:
|
||||
conn.mark_dead()
|
||||
dead.append(addr)
|
||||
break
|
||||
except zmq.Again:
|
||||
pass
|
||||
except Exception:
|
||||
logger.warning("ec: monitor poll failed for %s", addr, exc_info=True)
|
||||
return dead
|
||||
|
||||
def remove(self, addr: PeerAddr) -> ZmqClientConnection | None:
|
||||
"""Remove and close the connection for addr, returning it (or None)."""
|
||||
conn = self._connections.pop(addr, None)
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
return conn
|
||||
|
||||
def close(self) -> None:
|
||||
for conn in list(self._connections.values()):
|
||||
conn.close()
|
||||
self._connections.clear()
|
||||
try:
|
||||
self._ctx.destroy(linger=0)
|
||||
except Exception:
|
||||
logger.debug("ec: zmq context destroy failed", exc_info=True)
|
||||
|
||||
|
||||
# ── Server side ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class ZmqServerTransport:
|
||||
"""ROUTER socket for the producer side. Byte-level only, driven by
|
||||
ProducerSession.
|
||||
|
||||
poll(timeout_ms) waits up to timeout_ms for messages, then drains all that
|
||||
are available; timeout_ms=0 drains whatever has already arrived. send()
|
||||
routes a reply back to a specific peer by ZMQ identity.
|
||||
"""
|
||||
|
||||
def __init__(self, host: str, port: int) -> None:
|
||||
self._ctx = zmq.Context()
|
||||
self._router = make_zmq_socket(
|
||||
ctx=self._ctx,
|
||||
path=make_zmq_path(scheme="tcp", host=host, port=port),
|
||||
socket_type=zmq.ROUTER,
|
||||
bind=True,
|
||||
)
|
||||
self._poller = zmq.Poller()
|
||||
self._poller.register(self._router, zmq.POLLIN)
|
||||
|
||||
def poll(self, timeout_ms: int = 0) -> list[tuple[bytes, bytes]]:
|
||||
"""Wait up to timeout_ms for incoming messages, then drain all pending.
|
||||
|
||||
Returns a list of (identity, payload) pairs.
|
||||
"""
|
||||
if timeout_ms > 0:
|
||||
try:
|
||||
events = dict(self._poller.poll(timeout=timeout_ms))
|
||||
except zmq.ContextTerminated:
|
||||
return []
|
||||
except Exception:
|
||||
logger.exception("ec: router poller failed")
|
||||
return []
|
||||
if self._router not in events:
|
||||
return []
|
||||
|
||||
msgs: list[tuple[bytes, bytes]] = []
|
||||
while True:
|
||||
try:
|
||||
identity, _, payload = self._router.recv_multipart(flags=zmq.NOBLOCK)
|
||||
msgs.append((identity, payload))
|
||||
except zmq.Again:
|
||||
break
|
||||
except zmq.ContextTerminated:
|
||||
break
|
||||
except Exception:
|
||||
logger.exception("ec: router recv failed")
|
||||
break
|
||||
return msgs
|
||||
|
||||
def send(self, identity: bytes, payload: bytes) -> None:
|
||||
"""Send a reply payload to the peer identified by ZMQ identity."""
|
||||
try:
|
||||
self._router.send_multipart([identity, b"", payload])
|
||||
except Exception:
|
||||
logger.exception("ec: router send failed")
|
||||
|
||||
def close(self) -> None:
|
||||
try:
|
||||
self._router.close(linger=0)
|
||||
except Exception:
|
||||
logger.debug("ec: router close failed", exc_info=True)
|
||||
try:
|
||||
self._ctx.destroy(linger=0)
|
||||
except Exception:
|
||||
logger.debug("ec: zmq context destroy failed", exc_info=True)
|
||||
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Abstract base class for the EC data-plane transport (NIXL/UCX)."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class DataTransport(ABC):
|
||||
"""Interface wrapping the NIXL data-plane for EC block transfers.
|
||||
|
||||
Peer lifecycle:
|
||||
add_remote_peer() — register a peer and store its dlist handle internally
|
||||
remove_remote_peer() — deregister and release the stored handle
|
||||
post_read() — issue a READ using the stored handle for agent_name
|
||||
|
||||
The dlist handle (prepped xfer descriptor list) is an implementation detail
|
||||
of the transport and is never exposed to callers.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
"""Return this agent's serialized NIXL metadata blob."""
|
||||
|
||||
@abstractmethod
|
||||
def get_mem_descriptor(self) -> bytes:
|
||||
"""Return the msgpack-encoded block descriptor list for this agent."""
|
||||
|
||||
@abstractmethod
|
||||
def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str:
|
||||
"""Register a remote peer, prep its READ-source dlist, and return agent_name.
|
||||
|
||||
The dlist handle is stored internally and used by post_read().
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def remove_remote_peer(self, agent_name: str) -> None:
|
||||
"""Deregister a previously registered remote peer and release its handle."""
|
||||
|
||||
@abstractmethod
|
||||
def post_read(
|
||||
self,
|
||||
local_indices: list[int],
|
||||
agent_name: str,
|
||||
remote_indices: list[int],
|
||||
notif_msg: bytes,
|
||||
) -> Any:
|
||||
"""Issue a consumer-initiated READ using the stored dlist for agent_name.
|
||||
|
||||
Returns a transfer handle for use with check_xfer_state / release_xfer_handle.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def get_new_notifs(self) -> dict[str, list[bytes]]:
|
||||
"""Drain NIXL completion notifications addressed to this agent."""
|
||||
|
||||
@abstractmethod
|
||||
def check_xfer_state(self, handle: Any) -> str:
|
||||
"""Return the NIXL state string: 'DONE', 'PROC', or an error string."""
|
||||
|
||||
@abstractmethod
|
||||
def release_xfer_handle(self, handle: Any) -> None:
|
||||
"""Release a transfer handle returned by post_read."""
|
||||
|
||||
@abstractmethod
|
||||
def deregister(self) -> None:
|
||||
"""Deregister all memory and tear down the NIXL agent."""
|
||||
@@ -1,162 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NIXL-backed data-plane transport for ECCPUConnector."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
|
||||
DataTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import (
|
||||
build_block_descs,
|
||||
deserialize_mem_descriptor,
|
||||
serialize_mem_descriptor,
|
||||
)
|
||||
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_NIXL_DRAM = "DRAM"
|
||||
|
||||
|
||||
class NixlDataTransport(DataTransport):
|
||||
"""Adapts NixlWrapper to the DataTransport interface.
|
||||
|
||||
Dlist handles (prepped xfer descriptor lists for remote peers) are stored
|
||||
internally keyed by agent_name. Callers never touch raw NIXL handles —
|
||||
post_read() resolves the handle from agent_name internally.
|
||||
|
||||
NixlWrapper is not thread-safe, and neither is this class. Every call site
|
||||
runs on the scheduler thread, which serializes the transport by
|
||||
construction; transfers are async inside NIXL and are drained from that
|
||||
same thread via get_new_notifs() / check_xfer_state().
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
agent_name: str,
|
||||
base_ptr: int,
|
||||
num_blocks: int,
|
||||
block_size_bytes: int,
|
||||
total_size_bytes: int,
|
||||
num_threads: int = 1,
|
||||
) -> None:
|
||||
if NixlWrapper is None or nixl_agent_config is None:
|
||||
raise RuntimeError(
|
||||
"ECCPUConnector requires NIXL; "
|
||||
"install the `nixl` package or set a different ec_connector."
|
||||
)
|
||||
# agent_name → remote dlist handle (from add_remote_peer, used by post_read)
|
||||
self._peer_handles: dict[str, int] = {}
|
||||
# None until the local region is registered, so deregister() is safe on
|
||||
# a transport whose registration failed part-way.
|
||||
self._reg_descs: Any = None
|
||||
|
||||
self._nixl = NixlWrapper(
|
||||
agent_name,
|
||||
nixl_agent_config(num_threads=num_threads, capture_telemetry=True),
|
||||
)
|
||||
self._register_local_memory(
|
||||
base_ptr, num_blocks, block_size_bytes, total_size_bytes
|
||||
)
|
||||
|
||||
def _register_local_memory(
|
||||
self,
|
||||
base_ptr: int,
|
||||
num_blocks: int,
|
||||
block_size_bytes: int,
|
||||
total_size_bytes: int,
|
||||
) -> None:
|
||||
"""Register the local region with NIXL and prep its transfer dlist.
|
||||
|
||||
Unwinds the registration if a later step fails: __init__ propagates the
|
||||
error, so the caller never receives an object it could deregister with.
|
||||
"""
|
||||
block_descs = build_block_descs(base_ptr, num_blocks, block_size_bytes)
|
||||
self._reg_descs = self._nixl.get_reg_descs(
|
||||
[(base_ptr, total_size_bytes, 0, "")], _NIXL_DRAM
|
||||
)
|
||||
self._nixl.register_memory(self._reg_descs, backends=["UCX"])
|
||||
try:
|
||||
xfer_descs = self._nixl.get_xfer_descs(block_descs, _NIXL_DRAM)
|
||||
self._local_xfer_handle = self._nixl.prep_xfer_dlist(
|
||||
"NIXL_INIT_AGENT", xfer_descs
|
||||
)
|
||||
self._agent_metadata = self._nixl.get_agent_metadata()
|
||||
self._mem_descriptor_bytes = serialize_mem_descriptor(block_descs)
|
||||
except Exception:
|
||||
self.deregister()
|
||||
raise
|
||||
|
||||
def get_agent_metadata(self) -> bytes:
|
||||
return self._agent_metadata
|
||||
|
||||
def get_mem_descriptor(self) -> bytes:
|
||||
return self._mem_descriptor_bytes
|
||||
|
||||
def add_remote_peer(self, metadata: bytes, mem_descriptor: bytes) -> str:
|
||||
"""Register remote peer, prep its dlist, store handle internally.
|
||||
|
||||
Returns agent_name for use in post_read() and remove_remote_peer().
|
||||
"""
|
||||
agent_name = self._nixl.add_remote_agent(metadata)
|
||||
remote_blocks = deserialize_mem_descriptor(mem_descriptor)
|
||||
remote_xfer_descs = self._nixl.get_xfer_descs(remote_blocks, _NIXL_DRAM)
|
||||
self._peer_handles[agent_name] = self._nixl.prep_xfer_dlist(
|
||||
agent_name, remote_xfer_descs
|
||||
)
|
||||
return agent_name
|
||||
|
||||
def remove_remote_peer(self, agent_name: str) -> None:
|
||||
self._peer_handles.pop(agent_name, None)
|
||||
try:
|
||||
self._nixl.remove_remote_agent(agent_name)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"EC: remove_remote_peer failed for %s", agent_name, exc_info=True
|
||||
)
|
||||
|
||||
def post_read(
|
||||
self,
|
||||
local_indices: list[int],
|
||||
agent_name: str,
|
||||
remote_indices: list[int],
|
||||
notif_msg: bytes,
|
||||
) -> Any:
|
||||
if len(local_indices) != len(remote_indices):
|
||||
raise ValueError(
|
||||
f"EC: local/remote block count mismatch "
|
||||
f"({len(local_indices)} vs {len(remote_indices)})"
|
||||
)
|
||||
remote_handle = self._peer_handles[agent_name]
|
||||
handle = self._nixl.make_prepped_xfer(
|
||||
"READ",
|
||||
self._local_xfer_handle,
|
||||
local_indices,
|
||||
remote_handle,
|
||||
remote_indices,
|
||||
notif_msg=notif_msg,
|
||||
)
|
||||
self._nixl.transfer(handle)
|
||||
return handle
|
||||
|
||||
def get_new_notifs(self) -> dict[str, list[bytes]]:
|
||||
return self._nixl.get_new_notifs()
|
||||
|
||||
def check_xfer_state(self, handle: Any) -> str:
|
||||
return self._nixl.check_xfer_state(handle)
|
||||
|
||||
def release_xfer_handle(self, handle: Any) -> None:
|
||||
try:
|
||||
self._nixl.release_xfer_handle(handle)
|
||||
except Exception:
|
||||
logger.warning("EC: release_xfer_handle failed", exc_info=True)
|
||||
|
||||
def deregister(self) -> None:
|
||||
if self._reg_descs is None:
|
||||
return
|
||||
try:
|
||||
self._nixl.deregister_memory(self._reg_descs)
|
||||
except Exception:
|
||||
logger.warning("EC: deregister failed", exc_info=True)
|
||||
self._reg_descs = None
|
||||
@@ -1,193 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Lightweight mmap-backed shared memory region for encoder cache (EC) data.
|
||||
|
||||
Modeled after SharedOffloadRegion (vllm/v1/kv_offload/cpu/) but simplified
|
||||
for EC: flat shared layout, no multi-tensor cursor, no block_size_factor.
|
||||
"""
|
||||
|
||||
import errno
|
||||
import mmap
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# MADV_POPULATE_WRITE was added in Linux 5.14 (value 23).
|
||||
_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23)
|
||||
|
||||
|
||||
def _madvise_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None:
|
||||
mmap_obj.madvise(_MADV_POPULATE_WRITE, offset, length)
|
||||
|
||||
|
||||
def _fallback_populate_write(mmap_obj: mmap.mmap, offset: int, length: int) -> None:
|
||||
# Touch one byte per page via a read-modify-write so existing bytes are
|
||||
# preserved — a peer worker may have already written EC data into this
|
||||
# shared mmap by the time we run on a kernel without MADV_POPULATE_WRITE.
|
||||
arr = np.frombuffer(mmap_obj, dtype=np.uint8)
|
||||
arr[offset : offset + length : mmap.PAGESIZE] |= 0
|
||||
|
||||
|
||||
def _get_populate_write_fn(
|
||||
mmap_obj: mmap.mmap,
|
||||
) -> Callable[[mmap.mmap, int, int], None]:
|
||||
"""Select the pre-faulting method once for this mmap."""
|
||||
try:
|
||||
_madvise_populate_write(mmap_obj, 0, mmap.PAGESIZE)
|
||||
except OSError as e:
|
||||
if e.errno != errno.EINVAL:
|
||||
raise
|
||||
logger.warning(
|
||||
"MADV_POPULATE_WRITE is not supported; falling back to per-page "
|
||||
"writes for EC mmap pre-population. Startup may be slower."
|
||||
)
|
||||
return _fallback_populate_write
|
||||
return _madvise_populate_write
|
||||
|
||||
|
||||
def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0):
|
||||
"""Spin-wait until the file reaches expected_size (creator truncated it)."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
if os.fstat(fd).st_size >= expected_size:
|
||||
return
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for EC mmap file to reach {expected_size} bytes"
|
||||
)
|
||||
time.sleep(0.005)
|
||||
|
||||
|
||||
class ECSharedRegion:
|
||||
"""Flat mmap-backed memory region shared across TP workers for
|
||||
encoder cache blocks.
|
||||
|
||||
Layout: (num_blocks, block_size_bytes) — contiguous, no per-worker
|
||||
interleaving. All workers map the same file and see identical data.
|
||||
|
||||
File path: /dev/shm/vllm_ec_{engine_id}.mmap
|
||||
|
||||
This class owns only the shared memory substrate (mmap lifecycle, the
|
||||
`blocks` view, CUDA host registration). Block allocation and eviction
|
||||
are tracked by `EmbeddingCache` in the scheduler process.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_id: str,
|
||||
num_blocks: int,
|
||||
block_size_bytes: int,
|
||||
) -> None:
|
||||
self.num_blocks = num_blocks
|
||||
self.block_size_bytes = block_size_bytes
|
||||
|
||||
total_size_bytes = num_blocks * block_size_bytes
|
||||
# Path in /dev/shm (tmpfs); unique per engine instance.
|
||||
self._mmap_path = f"/dev/shm/vllm_ec_{engine_id}.mmap"
|
||||
# True for the process that created the file (responsible for unlink).
|
||||
self._is_creator = False
|
||||
# True after successful cudaHostRegister (cleanup must unregister).
|
||||
self._is_pinned = False
|
||||
|
||||
# File descriptor for the shared memory backing file.
|
||||
try:
|
||||
self._fd: int | None = os.open(
|
||||
self._mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600
|
||||
)
|
||||
os.ftruncate(self._fd, total_size_bytes)
|
||||
self._is_creator = True
|
||||
logger.info(
|
||||
"Created EC mmap file %s (%.2f MiB)",
|
||||
self._mmap_path,
|
||||
total_size_bytes / (1 << 20),
|
||||
)
|
||||
except FileExistsError:
|
||||
self._fd = os.open(self._mmap_path, os.O_RDWR)
|
||||
try:
|
||||
_wait_for_file_size(self._fd, total_size_bytes)
|
||||
except Exception:
|
||||
os.close(self._fd)
|
||||
self._fd = None
|
||||
raise
|
||||
logger.info("Opened existing EC mmap file %s", self._mmap_path)
|
||||
|
||||
# MAP_SHARED mmap over _fd; all processes see the same pages.
|
||||
self._mmap_obj: mmap.mmap | None = mmap.mmap(
|
||||
self._fd,
|
||||
total_size_bytes,
|
||||
flags=mmap.MAP_SHARED,
|
||||
prot=mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
)
|
||||
|
||||
if self._is_creator:
|
||||
populate_write_fn = _get_populate_write_fn(self._mmap_obj)
|
||||
populate_write_fn(self._mmap_obj, 0, total_size_bytes)
|
||||
|
||||
# (num_blocks, block_size_bytes) int8 tensor over the mmap buffer.
|
||||
self.blocks: torch.Tensor = torch.frombuffer(
|
||||
memoryview(self._mmap_obj), dtype=torch.int8
|
||||
).view(num_blocks, block_size_bytes)
|
||||
# Cached for cudaHostRegister/Unregister and pointer math.
|
||||
self._blocks_ptr: int = self.blocks.data_ptr()
|
||||
self._blocks_nbytes: int = self.blocks.nbytes
|
||||
|
||||
def pin_memory(self) -> None:
|
||||
"""Register the entire mmap as CUDA pinned memory for fast DMA.
|
||||
|
||||
Each TP worker process owns its own virtual address for the same
|
||||
shared physical pages, so every process must register independently.
|
||||
No-op when CUDA is not available or already pinned.
|
||||
"""
|
||||
if self._is_pinned or not torch.cuda.is_available():
|
||||
return
|
||||
result = torch.cuda.cudart().cudaHostRegister(
|
||||
self._blocks_ptr, self._blocks_nbytes, 0
|
||||
)
|
||||
if result.value != 0:
|
||||
logger.warning(
|
||||
"cudaHostRegister failed (code=%d) — "
|
||||
"transfers will still work but may be slower (unpinned DMA)",
|
||||
result.value,
|
||||
)
|
||||
else:
|
||||
logger.debug("cudaHostRegister %.2f MB", self._blocks_nbytes / 1e6)
|
||||
self._is_pinned = True
|
||||
|
||||
def cleanup(self) -> None:
|
||||
"""Tear down the region. Lifecycle method; no concurrent access."""
|
||||
logger.info("Starting ECSharedRegion cleanup...")
|
||||
if self._is_creator:
|
||||
try:
|
||||
os.unlink(self._mmap_path)
|
||||
logger.info("Removed EC mmap file %s", self._mmap_path)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to unlink path %s", self._mmap_path, exc_info=True
|
||||
)
|
||||
self._is_creator = False
|
||||
if self._is_pinned:
|
||||
result = torch.cuda.cudart().cudaHostUnregister(self._blocks_ptr)
|
||||
if result.value != 0:
|
||||
logger.warning("cudaHostUnregister failed (code=%d)", result)
|
||||
self._is_pinned = False
|
||||
if hasattr(self, "blocks"):
|
||||
del self.blocks
|
||||
if self._mmap_obj:
|
||||
try:
|
||||
self._mmap_obj.close()
|
||||
except Exception:
|
||||
logger.warning("Failed to close mmap_obj", exc_info=True)
|
||||
self._mmap_obj = None
|
||||
if self._fd is not None:
|
||||
try:
|
||||
os.close(self._fd)
|
||||
except Exception:
|
||||
logger.warning("Failed to close fd %s", self._fd, exc_info=True)
|
||||
self._fd = None
|
||||
@@ -1,91 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Scheduler-side wire types for the ECCPUConnector.
|
||||
|
||||
Two peer-to-peer message types exchanged between consumer and producer over
|
||||
ZMQ (XferReq, XferAck) plus a peer-compatibility fingerprint
|
||||
(compute_ec_compatibility_hash) sent on every XferReq so a producer refuses
|
||||
to serve a mismatched peer.
|
||||
"""
|
||||
|
||||
import enum
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
import msgspec
|
||||
|
||||
# Bump when the wire format or tensor/block layout changes in a
|
||||
# backward-incompatible way. Peers reject mismatched versions.
|
||||
EC_CONNECTOR_VERSION: int = 2
|
||||
|
||||
|
||||
class XferStatus(enum.IntEnum):
|
||||
"""Outcome the producer reports for an XferReq."""
|
||||
|
||||
OK = 0
|
||||
NACK_MISSING = 1 # producer no longer holds the encoding
|
||||
NACK_INCOMPAT = 2 # peer-compatibility hash mismatch
|
||||
NACK_VERSION = 3 # connector wire-version mismatch
|
||||
NACK_INTERNAL = 4 # producer hit an unexpected error
|
||||
NACK_NOT_READY = 5 # announced, but its GPU->mmap save has not landed yet
|
||||
|
||||
|
||||
# The producer expects a retryable condition to clear on its own, so the
|
||||
# consumer re-requests the read rather than giving up on the encoding.
|
||||
RETRYABLE_NACKS = frozenset((XferStatus.NACK_NOT_READY,))
|
||||
# Ordinary outcomes rather than signs of a broken peer.
|
||||
EXPECTED_NACKS = RETRYABLE_NACKS | {XferStatus.NACK_MISSING}
|
||||
|
||||
|
||||
class XferReq( # type: ignore[call-arg]
|
||||
msgspec.Struct,
|
||||
tag="req",
|
||||
omit_defaults=True,
|
||||
):
|
||||
"""Consumer → Producer: I want to READ mm_hash; pin it and tell me where."""
|
||||
|
||||
mm_hash: str
|
||||
compatibility_hash: str
|
||||
# Consumer's session identity. Together with mm_hash it uniquely identifies
|
||||
# this transfer on both sides: the producer keys its active xfers by
|
||||
# (session_id, mm_hash), and the NIXL notif_msg carries the same pair.
|
||||
session_id: str = ""
|
||||
connector_version: int = EC_CONNECTOR_VERSION
|
||||
|
||||
|
||||
class XferAck( # type: ignore[call-arg]
|
||||
msgspec.Struct,
|
||||
tag="ack",
|
||||
omit_defaults=True,
|
||||
):
|
||||
"""Producer → Consumer: response to an XferReq.
|
||||
|
||||
On status == OK: producer has pinned source blocks and returns their
|
||||
indices plus fresh NIXL metadata for the consumer to register and READ.
|
||||
On any NACK_*: optional fields are empty; consumer falls back to local encode.
|
||||
"""
|
||||
|
||||
mm_hash: str
|
||||
status: XferStatus
|
||||
# Consumer's session_id echoed back from XferReq. Together with mm_hash it
|
||||
# identifies the exact transfer; the consumer uses both to build notif_msg.
|
||||
session_id: str = ""
|
||||
src_block_indices: list[int] = []
|
||||
agent_metadata: bytes = b""
|
||||
mem_descriptor: bytes = b""
|
||||
|
||||
|
||||
def compute_ec_compatibility_hash(
|
||||
vllm_version: str,
|
||||
model: str,
|
||||
dtype: str,
|
||||
block_size_bytes: int,
|
||||
) -> str:
|
||||
"""Peer-compatibility fingerprint over the four factors that determine
|
||||
byte-layout compatibility. Producer NACKs any XferReq whose hash differs.
|
||||
"""
|
||||
factors = json.dumps(
|
||||
[vllm_version, model, dtype, block_size_bytes],
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return hashlib.sha256(factors.encode("utf-8")).hexdigest()
|
||||
@@ -1,700 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ECCPUScheduler — CPU offload scheduler delegate.
|
||||
|
||||
Owns the mmap region and the embedding cache, and handles the producer
|
||||
(GPU->CPU offload) and consumer (CPU->GPU reload) scheduler-side logic
|
||||
for the ECCPUConnector.
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Collection
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
ECCPUWorkerMetadata,
|
||||
create_ec_shared_region,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
|
||||
get_encoder_output_width,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ECConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Total remote wait budget per request item, including all retries.
|
||||
_ADMIT_DEFER_TIMEOUT_S = 60.0
|
||||
|
||||
|
||||
class _RemoteUnavailable(Exception):
|
||||
"""An item cannot be fetched within its request's remote wait budget."""
|
||||
|
||||
|
||||
class ECCPUScheduler:
|
||||
"""Scheduler delegate for the ECCPUConnector."""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig") -> None:
|
||||
self._vllm_config = vllm_config
|
||||
ec_config = vllm_config.ec_transfer_config
|
||||
assert ec_config is not None
|
||||
self._is_producer: bool = ec_config.is_ec_producer
|
||||
self._is_consumer: bool = ec_config.is_ec_consumer
|
||||
|
||||
self._region = create_ec_shared_region(vllm_config)
|
||||
# Block allocator + LRU eviction policy for the shared region.
|
||||
self._cache = EmbeddingCache(self._region.num_blocks)
|
||||
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
|
||||
|
||||
# mm_hash → block IDs allocated this step for GPU→mmap saves.
|
||||
self._pending_saves: dict[str, list[int]] = {}
|
||||
# mm_hash → (transfer_id, block IDs, shape) to load from mmap→GPU.
|
||||
self._pending_loads: dict[str, tuple[int, list[int], tuple[int, int]]] = {}
|
||||
|
||||
# Dispatched loads awaiting completion reports, keyed by transfer id:
|
||||
# transfer_id → (mm_hash, reports still outstanding). The pin taken at
|
||||
# dispatch is released only once the count reaches zero.
|
||||
self._load_acks: dict[int, tuple[str, int]] = {}
|
||||
self._next_transfer_id = 0
|
||||
|
||||
pc = vllm_config.parallel_config
|
||||
# Reports to expect per load. Only pipeline stage 0 runs the EC
|
||||
# connector, so the participants are the tp × pcp ranks of that stage.
|
||||
# Other executors deliver a single rank's output, and expecting reports
|
||||
# that cannot arrive would hold the pin forever.
|
||||
self._expected_load_reports = (
|
||||
pc.tensor_parallel_size * pc.prefill_context_parallel_size
|
||||
if pc.distributed_executor_backend == "mp"
|
||||
else 1
|
||||
)
|
||||
|
||||
self._ec_config = ec_config
|
||||
# NIXL p2p is an option of this connector, not engine-wide behavior, so
|
||||
# it lives in extra config. Extra config is not type coerced, so a value
|
||||
# that arrived as a JSON/CLI string is parsed, not truth-tested.
|
||||
raw_nixl = ec_config.get_from_extra_config("ec_enable_nixl", False)
|
||||
self._nixl_enabled = (
|
||||
raw_nixl
|
||||
if isinstance(raw_nixl, bool)
|
||||
else str(raw_nixl).strip().lower() in ("true", "1", "yes")
|
||||
)
|
||||
# NIXL fields default to None/empty so the gate-off path is untouched.
|
||||
self._data: Any = None
|
||||
self._compat_hash: str | None = None
|
||||
self._first_in_batch = True
|
||||
self._transport: Any = None
|
||||
self._producer_session: Any = None
|
||||
self._sessions: dict = {}
|
||||
self._in_flight: set[str] = set()
|
||||
self._tombstones: set[str] = set()
|
||||
self._step_completed: set[str] = set()
|
||||
# Per-request clocks survive retries and are cleared on completion.
|
||||
self._deferred_since: dict[tuple[str, str], float] = {}
|
||||
# Requests needing a remote encoding that will not arrive. Drained by
|
||||
# take_unavailable_requests(); the scheduler aborts them.
|
||||
self._unrecoverable: set[str] = set()
|
||||
self._peer_host: str | None = None
|
||||
self._peer_port: int | None = None
|
||||
dtype = vllm_config.model_config.dtype
|
||||
assert isinstance(dtype, torch.dtype)
|
||||
self._dtype = dtype
|
||||
self._ack_timeout_s: float = 0.0
|
||||
if self._nixl_enabled:
|
||||
try:
|
||||
self._setup_nixl(vllm_config)
|
||||
except Exception:
|
||||
self.shutdown()
|
||||
raise
|
||||
|
||||
def _setup_nixl(self, vllm_config: "VllmConfig") -> None:
|
||||
# Lazy imports keep nixl/zmq off the gate-off path.
|
||||
from vllm import envs
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import (
|
||||
ZmqClientTransport,
|
||||
ZmqServerTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.nixl import (
|
||||
NixlDataTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
|
||||
compute_ec_compatibility_hash,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
|
||||
PRODUCER_PIN_LEASE_S,
|
||||
ProducerSession,
|
||||
)
|
||||
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
if NixlWrapper is None or nixl_agent_config is None:
|
||||
raise RuntimeError(
|
||||
"ec_enable_nixl requires NIXL; install the `nixl` package or "
|
||||
"remove ec_enable_nixl from ec_connector_extra_config."
|
||||
)
|
||||
engine_id = self._ec_config.engine_id
|
||||
assert engine_id is not None
|
||||
# How long a consumer waits for an XferAck. The producer answers from
|
||||
# its scheduler step, so its reply latency scales with the encoder's
|
||||
# batch size: a deployment whose steps run longer must raise this.
|
||||
self._ack_timeout_s = float(
|
||||
self._ec_config.get_from_extra_config(
|
||||
"consumer_ack_timeout_s", PRODUCER_PIN_LEASE_S
|
||||
)
|
||||
)
|
||||
self._compat_hash = compute_ec_compatibility_hash(
|
||||
vllm_version=VLLM_VERSION,
|
||||
model=str(vllm_config.model_config.model),
|
||||
dtype=str(self._dtype),
|
||||
block_size_bytes=self._region.block_size_bytes,
|
||||
)
|
||||
if self._is_producer:
|
||||
self._peer_host = envs.VLLM_EC_SIDE_CHANNEL_HOST
|
||||
self._peer_port = envs.VLLM_EC_SIDE_CHANNEL_PORT
|
||||
|
||||
self._data = NixlDataTransport(
|
||||
agent_name=engine_id,
|
||||
base_ptr=self._region.blocks.data_ptr(),
|
||||
num_blocks=self._region.num_blocks,
|
||||
block_size_bytes=self._region.block_size_bytes,
|
||||
total_size_bytes=self._region.num_blocks * self._region.block_size_bytes,
|
||||
)
|
||||
if self._is_producer:
|
||||
assert self._peer_host is not None
|
||||
assert self._peer_port is not None
|
||||
self._producer_session = ProducerSession(
|
||||
transport=ZmqServerTransport(
|
||||
host=self._peer_host, port=self._peer_port
|
||||
),
|
||||
data=self._data,
|
||||
cache=self._cache,
|
||||
compat_hash=self._compat_hash,
|
||||
)
|
||||
if self._is_consumer:
|
||||
self._transport = ZmqClientTransport()
|
||||
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
if not self._is_consumer:
|
||||
return False
|
||||
entry = self._cache.get(identifier)
|
||||
return entry is not None and entry.ready
|
||||
|
||||
def ensure_cache_available(
|
||||
self,
|
||||
request: "Request",
|
||||
num_computed_tokens: int,
|
||||
local_cache_hashes: Collection[str] | None = None,
|
||||
) -> bool:
|
||||
if not self._nixl_enabled:
|
||||
return True # CPU offload never blocks.
|
||||
first = self._first_in_batch
|
||||
self._first_in_batch = False
|
||||
if not self._is_consumer:
|
||||
return True
|
||||
if first:
|
||||
self._poll_step()
|
||||
return self._nixl_consumer_admit(request, num_computed_tokens)
|
||||
|
||||
def _nixl_consumer_admit(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
"""Admit a request once its required remote encodings are ready.
|
||||
|
||||
Each item has a per-request wait budget that survives transfer retries.
|
||||
Unavailable items fall back to local input; without it, the request is
|
||||
reported by take_unavailable_requests() for the scheduler to abort.
|
||||
|
||||
Returns:
|
||||
True if no item needs to wait for a remote encoding. False if any
|
||||
item is pending or the request must be aborted.
|
||||
|
||||
"""
|
||||
if not request.ec_transfer_params:
|
||||
return True
|
||||
now = time.monotonic()
|
||||
pending = False
|
||||
for feature in request.mm_features:
|
||||
try:
|
||||
if not self._admit_item(request, feature, num_computed_tokens, now):
|
||||
pending = True
|
||||
except _RemoteUnavailable as error:
|
||||
if not self._fail_or_fallback(request, feature, str(error)):
|
||||
return False
|
||||
return not pending
|
||||
|
||||
def _admit_item(
|
||||
self,
|
||||
request: "Request",
|
||||
feature: "MultiModalFeatureSpec",
|
||||
num_computed_tokens: int,
|
||||
now: float,
|
||||
) -> bool:
|
||||
"""Check one encoding, starting a remote read if needed.
|
||||
|
||||
Returns:
|
||||
True if already computed, cached, or not remotely sourced. False
|
||||
while waiting for a read or cache space, including across retries.
|
||||
|
||||
Raises:
|
||||
_RemoteUnavailable: The remote source is invalid, the read failed,
|
||||
or the item's total wait budget expired.
|
||||
|
||||
"""
|
||||
pos = feature.mm_position
|
||||
mm_hash = feature.identifier
|
||||
if pos.offset + pos.length <= num_computed_tokens:
|
||||
self._deferred_since.pop((request.request_id, mm_hash), None)
|
||||
return True
|
||||
announced = (request.ec_transfer_params or {}).get(mm_hash)
|
||||
# Without a producer address there is nothing to fetch: the request
|
||||
# carries what the model needs and the encoder runs locally.
|
||||
# `ec_transfer_params` reaches us from the request, so its shape is
|
||||
# checked rather than assumed.
|
||||
remote: dict[str, Any] | None = (
|
||||
announced
|
||||
if isinstance(announced, dict) and "peer_host" in announced
|
||||
else None
|
||||
)
|
||||
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and entry.ready:
|
||||
# Local hit: upstream's update_state_after_alloc pins and
|
||||
# loads it through the same path as a natively cached entry.
|
||||
self._deferred_since.pop((request.request_id, mm_hash), None)
|
||||
return True
|
||||
if remote is None:
|
||||
return True
|
||||
since = self._deferred_since.setdefault((request.request_id, mm_hash), now)
|
||||
if now - since > _ADMIT_DEFER_TIMEOUT_S:
|
||||
raise _RemoteUnavailable(f"the remote wait exceeded {now - since:.0f}s")
|
||||
if mm_hash in self._in_flight or mm_hash in self._step_completed:
|
||||
return False
|
||||
|
||||
if mm_hash in self._tombstones:
|
||||
self._tombstones.discard(mm_hash)
|
||||
raise _RemoteUnavailable("the remote read failed")
|
||||
|
||||
if entry is not None:
|
||||
# Present but not ready and not being fetched: its blocks are
|
||||
# held by a quarantined/settling DMA and cannot be reused.
|
||||
return False
|
||||
|
||||
try:
|
||||
size = int(remote["size_bytes"])
|
||||
shape = tuple(int(dim) for dim in remote["shape"])
|
||||
except (KeyError, TypeError, ValueError) as error:
|
||||
raise _RemoteUnavailable("the announced size was unusable") from error
|
||||
# The width is the producer's to report; the row count is not.
|
||||
num_embeds = pos.get_num_embeds()
|
||||
if (
|
||||
len(shape) != 2
|
||||
or shape[0] != num_embeds
|
||||
or size != self._num_blocks(shape) * self._region.block_size_bytes
|
||||
):
|
||||
logger.warning(
|
||||
"EC consumer: size mismatch mm_hash=%s announced=%d shape=%s "
|
||||
"expected %d rows",
|
||||
mm_hash,
|
||||
size,
|
||||
shape,
|
||||
num_embeds,
|
||||
)
|
||||
raise _RemoteUnavailable("the announced size was wrong")
|
||||
|
||||
try:
|
||||
started = self._start_xfer(mm_hash, remote, size, (shape[0], shape[1]))
|
||||
except Exception as error:
|
||||
logger.exception(
|
||||
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
|
||||
)
|
||||
raise _RemoteUnavailable("the read could not be started") from error
|
||||
if started:
|
||||
self._in_flight.add(mm_hash)
|
||||
return False
|
||||
|
||||
def _fail_or_fallback(
|
||||
self, request: "Request", feature: "MultiModalFeatureSpec", why: str
|
||||
) -> bool:
|
||||
"""Resolve an unavailable remote encoding and clear its wait budget.
|
||||
|
||||
Returns:
|
||||
True if local media or embeddings are available, removing this
|
||||
item's remote announcement so subsequent steps do not retry it.
|
||||
False if only placeholder metadata (or no input) is available,
|
||||
recording the request for take_unavailable_requests() to drain.
|
||||
|
||||
"""
|
||||
mm_hash = feature.identifier
|
||||
self._deferred_since.pop((request.request_id, mm_hash), None)
|
||||
data = feature.data
|
||||
metadata = (
|
||||
self._metadata_resolver.fields_for(feature.modality) if data else set()
|
||||
)
|
||||
if data and any(
|
||||
value.data is not None for key, value in data.items() if key not in metadata
|
||||
):
|
||||
# Passthrough embeddings bypass the processor cache. SHM address
|
||||
# items therefore refer to media that the worker can encode.
|
||||
request.ec_transfer_params = dict(request.ec_transfer_params or {})
|
||||
request.ec_transfer_params.pop(mm_hash, None)
|
||||
logger.warning(
|
||||
"EC consumer: request %s mm_hash=%s: %s; using local input",
|
||||
request.request_id,
|
||||
mm_hash,
|
||||
why,
|
||||
)
|
||||
return True
|
||||
self._unrecoverable.add(request.request_id)
|
||||
logger.error(
|
||||
"EC consumer: request %s needs remote encoding mm_hash=%s but %s. "
|
||||
"No local model input is available; failing the request.",
|
||||
request.request_id,
|
||||
mm_hash,
|
||||
why,
|
||||
)
|
||||
return False
|
||||
|
||||
def take_unavailable_requests(self) -> set[str]:
|
||||
"""Return and clear IDs of requests that cannot obtain their encodings.
|
||||
|
||||
The scheduler must abort these requests rather than leave them waiting.
|
||||
Requests that can fall back to local input are not included, and each
|
||||
recorded failure is returned only once.
|
||||
"""
|
||||
if not self._unrecoverable:
|
||||
return set()
|
||||
failed = self._unrecoverable
|
||||
self._unrecoverable = set()
|
||||
return failed
|
||||
|
||||
def _start_xfer(
|
||||
self,
|
||||
mm_hash: str,
|
||||
info: "dict[str, Any]",
|
||||
size_bytes: int,
|
||||
shape: tuple[int, int],
|
||||
) -> bool:
|
||||
"""Allocate a not-ready cache entry and start a NIXL READ into it.
|
||||
|
||||
Returns True if the transfer was started. Returns False when the
|
||||
cache cannot accommodate the encoding; admission will retry it.
|
||||
"""
|
||||
from math import ceil
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
|
||||
ConsumerSession,
|
||||
)
|
||||
|
||||
n_blocks = max(1, ceil(size_bytes / self._region.block_size_bytes))
|
||||
entry = self._cache.alloc(mm_hash, n_blocks, shape)
|
||||
if entry is None:
|
||||
logger.debug(
|
||||
"EC consumer: cache full for mm_hash=%s (%d blocks); deferring",
|
||||
mm_hash,
|
||||
n_blocks,
|
||||
)
|
||||
return False
|
||||
indices = list(entry.block_ids)
|
||||
addr = (info["peer_host"], int(info["peer_port"]))
|
||||
if addr not in self._sessions:
|
||||
zmq_conn = self._transport.connect(addr)
|
||||
assert self._compat_hash is not None
|
||||
self._sessions[addr] = ConsumerSession(
|
||||
addr=addr,
|
||||
zmq_conn=zmq_conn,
|
||||
transport=self._transport,
|
||||
data=self._data,
|
||||
compat_hash=self._compat_hash,
|
||||
)
|
||||
deadline = time.monotonic() + self._ack_timeout_s
|
||||
try:
|
||||
self._sessions[addr].start_xfer(mm_hash, indices, deadline)
|
||||
except Exception:
|
||||
self._cache.discard(mm_hash)
|
||||
raise
|
||||
logger.debug(
|
||||
"EC consumer: starting NIXL xfer mm_hash=%s from %s:%d blocks=%d",
|
||||
mm_hash,
|
||||
addr[0],
|
||||
addr[1],
|
||||
n_blocks,
|
||||
)
|
||||
return True
|
||||
|
||||
def _poll_step(self) -> None:
|
||||
now = time.monotonic()
|
||||
all_messages = self._transport.poll()
|
||||
for addr, session in list(self._sessions.items()):
|
||||
session.poll(all_messages.get(addr, []), now)
|
||||
for addr in self._transport.poll_dead():
|
||||
self._on_peer_down(addr)
|
||||
for session in self._sessions.values():
|
||||
self._process_session_results(session)
|
||||
|
||||
def _process_session_results(self, session: Any) -> None:
|
||||
r = session.take_results()
|
||||
for mm_hash in r.completed:
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._cache.mark_ready(mm_hash)
|
||||
self._step_completed.add(mm_hash)
|
||||
logger.debug("EC consumer: NIXL xfer complete mm_hash=%s", mm_hash)
|
||||
for mm_hash in r.tombstoned:
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._cache.discard(mm_hash)
|
||||
self._tombstones.add(mm_hash)
|
||||
logger.debug("EC consumer: NIXL xfer failed mm_hash=%s", mm_hash)
|
||||
for mm_hash in r.quarantined:
|
||||
# DMA still running: keep the blocks reserved (entry stays
|
||||
# not-ready, hence non-evictable) until the xfer settles.
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._tombstones.add(mm_hash)
|
||||
logger.debug(
|
||||
"EC consumer: NIXL xfer mm_hash=%s quarantined; DMA still running",
|
||||
mm_hash,
|
||||
)
|
||||
for mm_hash in r.retryable:
|
||||
# Release this attempt's blocks, retaining the request wait budget.
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._cache.discard(mm_hash)
|
||||
logger.debug(
|
||||
"EC consumer: NIXL xfer mm_hash=%s retryable; will re-request",
|
||||
mm_hash,
|
||||
)
|
||||
for mm_hash, _block_indices in r.settled:
|
||||
self._cache.discard(mm_hash)
|
||||
logger.debug(
|
||||
"EC consumer: quarantined NIXL xfer mm_hash=%s settled", mm_hash
|
||||
)
|
||||
|
||||
def _on_peer_down(self, addr: Any) -> None:
|
||||
session = self._sessions.pop(addr, None)
|
||||
if session is None:
|
||||
return
|
||||
session.on_peer_down()
|
||||
self._process_session_results(session)
|
||||
session.close()
|
||||
logger.info("EC consumer: producer peer down addr=%s", addr)
|
||||
|
||||
def _promote_completed_reads(self) -> None:
|
||||
"""Drain the just-completed set built during ``_poll_step``.
|
||||
|
||||
Reads are marked ready in ``_process_session_results``, so a deferred
|
||||
request is re-admitted by ``ensure_cache_available`` and loaded through
|
||||
the same local path as a natively cached entry
|
||||
(``update_state_after_alloc`` -> ``_pending_loads``). No explicit
|
||||
pin/load bookkeeping is needed here beyond clearing the set.
|
||||
"""
|
||||
self._step_completed.clear()
|
||||
|
||||
def _num_blocks(self, shape: tuple[int, ...]) -> int:
|
||||
size_bytes = shape[0] * shape[1] * self._dtype.itemsize
|
||||
block_size = self._region.block_size_bytes
|
||||
return (size_bytes + block_size - 1) // block_size
|
||||
|
||||
def update_state_after_alloc(self, request: "Request", index: int) -> None:
|
||||
feature = request.mm_features[index]
|
||||
mm_hash = feature.identifier
|
||||
|
||||
if self._is_producer and self._cache.get(mm_hash) is None:
|
||||
shape = (
|
||||
feature.mm_position.get_num_embeds(),
|
||||
get_encoder_output_width(self._vllm_config, feature.modality),
|
||||
)
|
||||
entry = self._cache.alloc(mm_hash, self._num_blocks(shape), shape)
|
||||
if entry is not None:
|
||||
self._pending_saves[mm_hash] = list(entry.block_ids)
|
||||
|
||||
if self._is_consumer and mm_hash not in self._pending_loads:
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and entry.ready:
|
||||
assert entry.shape is not None
|
||||
self._cache.pin(mm_hash)
|
||||
transfer_id = self._next_transfer_id
|
||||
self._next_transfer_id += 1
|
||||
self._pending_loads[mm_hash] = (
|
||||
transfer_id,
|
||||
list(entry.block_ids),
|
||||
entry.shape,
|
||||
)
|
||||
self._load_acks[transfer_id] = (mm_hash, self._expected_load_reports)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> ECCPUConnectorMetadata:
|
||||
meta = ECCPUConnectorMetadata()
|
||||
if self._is_producer:
|
||||
if self._nixl_enabled and self._producer_session is not None:
|
||||
self._producer_session.poll_step()
|
||||
meta.saves = self._pending_saves
|
||||
self._pending_saves = {}
|
||||
if self._is_consumer:
|
||||
if self._nixl_enabled:
|
||||
self._promote_completed_reads()
|
||||
meta.loads = self._pending_loads
|
||||
self._pending_loads = {}
|
||||
if self._nixl_enabled:
|
||||
self._first_in_batch = True
|
||||
return meta
|
||||
|
||||
def update_connector_output(self, connector_output: "ECConnectorOutput") -> None:
|
||||
"""Apply the worker's memcpy-completion report to the cache.
|
||||
|
||||
Completed saves become safe to mark ready. A load is unpinned once
|
||||
every participating rank has reported its transfer id; reports for a
|
||||
transfer that has already been released, or for one this scheduler
|
||||
never dispatched, are ignored.
|
||||
"""
|
||||
meta = connector_output.ec_connector_worker_meta
|
||||
if not isinstance(meta, ECCPUWorkerMetadata):
|
||||
return
|
||||
for mm_hash in meta.completed_saves:
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is None:
|
||||
logger.debug(
|
||||
"EC producer: worker reported completed save for unknown "
|
||||
"mm_hash=%s (already discarded/evicted?)",
|
||||
mm_hash,
|
||||
)
|
||||
elif not entry.ready:
|
||||
self._cache.mark_ready(mm_hash)
|
||||
logger.debug("EC producer: mm_hash=%s marked ready", mm_hash)
|
||||
for transfer_id in meta.completed_loads:
|
||||
pending = self._load_acks.get(transfer_id)
|
||||
if pending is None:
|
||||
continue
|
||||
mm_hash, outstanding = pending
|
||||
if outstanding > 1:
|
||||
self._load_acks[transfer_id] = (mm_hash, outstanding - 1)
|
||||
continue
|
||||
# Drop the entry before unpinning so a replayed report is treated
|
||||
# as stale rather than releasing the pin a second time.
|
||||
del self._load_acks[transfer_id]
|
||||
self._cache.unpin(mm_hash)
|
||||
logger.debug("EC consumer: mm_hash=%s unpinned after load", mm_hash)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
"""Keep the engine stepping so this connector's polls keep running.
|
||||
|
||||
With NIXL enabled the engine tick is the only driver of
|
||||
``ProducerSession.poll_step()``, and a producer's work arrives from a
|
||||
remote consumer, so nothing local would otherwise wake it. Without NIXL
|
||||
this only has to outlive dispatched saves and loads, so the engine can
|
||||
quiesce once the worker has confirmed them.
|
||||
"""
|
||||
return self._nixl_enabled or self._cache.has_held_entries()
|
||||
|
||||
def request_finished(
|
||||
self, request: "Request"
|
||||
) -> tuple[bool, "dict[str, Any] | None"]:
|
||||
for feature in request.mm_features:
|
||||
self._deferred_since.pop((request.request_id, feature.identifier), None)
|
||||
if not (self._nixl_enabled and self._is_producer):
|
||||
return False, None
|
||||
if not request.mm_features:
|
||||
return False, None
|
||||
|
||||
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
|
||||
|
||||
for feature in request.mm_features:
|
||||
mm_hash = feature.identifier
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is None:
|
||||
# Never saved, or evicted since. Publishing placeholder
|
||||
# metadata without an address invites an orchestrator to
|
||||
# rewrite the media into a reference to an encoding no
|
||||
# consumer can fetch, leaving the decoder nothing to embed.
|
||||
# Publish neither, so the media stays on the request.
|
||||
items[mm_hash]["metadata"] = {}
|
||||
logger.debug(
|
||||
"EC producer: mm_hash=%s absent at request_finished; "
|
||||
"announcing no metadata so the media is not rewritten away",
|
||||
mm_hash,
|
||||
)
|
||||
continue
|
||||
# Announce even if the save's GPU->mmap copy hasn't been
|
||||
# confirmed complete yet: a not-ready entry can't be evicted, so
|
||||
# it will still be here by the time a consumer's XferReq arrives.
|
||||
# A read arriving before the save lands is NACKed NACK_NOT_READY,
|
||||
# which the consumer retries on a later step rather than treating
|
||||
# as a miss.
|
||||
assert entry.shape is not None
|
||||
size_bytes = len(entry.block_ids) * self._region.block_size_bytes
|
||||
items[mm_hash].update(
|
||||
peer_host=self._peer_host,
|
||||
peer_port=self._peer_port,
|
||||
size_bytes=size_bytes,
|
||||
shape=list(entry.shape),
|
||||
)
|
||||
logger.debug(
|
||||
"EC producer: announcing NIXL-readable encodings req_id=%s items=%s",
|
||||
request.request_id,
|
||||
items,
|
||||
)
|
||||
return False, items
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._pending_saves.clear()
|
||||
self._pending_loads.clear()
|
||||
self._load_acks.clear()
|
||||
self._deferred_since.clear()
|
||||
|
||||
self._is_producer = False
|
||||
self._is_consumer = False
|
||||
|
||||
if self._nixl_enabled:
|
||||
self._teardown_nixl()
|
||||
|
||||
try:
|
||||
self._region.cleanup()
|
||||
except Exception:
|
||||
logger.debug("ec: region cleanup failed", exc_info=True)
|
||||
|
||||
def _teardown_nixl(self) -> None:
|
||||
"""Close the control sockets and release the NIXL agent.
|
||||
|
||||
Guards each step and clears what it releases, so it is safe on a
|
||||
scheduler whose NIXL setup failed part-way and safe to call twice.
|
||||
The shared region is not touched — shutdown() owns that.
|
||||
"""
|
||||
if self._producer_session is not None:
|
||||
try:
|
||||
self._producer_session.close()
|
||||
except Exception:
|
||||
logger.debug("ec: producer session close failed", exc_info=True)
|
||||
self._producer_session = None
|
||||
|
||||
for session in list(self._sessions.values()):
|
||||
try:
|
||||
session.close()
|
||||
except Exception:
|
||||
logger.debug("ec: consumer session close failed", exc_info=True)
|
||||
self._sessions.clear()
|
||||
|
||||
if self._transport is not None:
|
||||
try:
|
||||
self._transport.close()
|
||||
except Exception:
|
||||
logger.debug("ec: client transport close failed", exc_info=True)
|
||||
self._transport = None
|
||||
|
||||
if self._data is not None:
|
||||
try:
|
||||
self._data.deregister()
|
||||
except Exception:
|
||||
logger.debug("ec: deregister failed", exc_info=True)
|
||||
self._data = None
|
||||
@@ -1,184 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""EmbeddingCache — named-entry block cache with FIFO eviction.
|
||||
|
||||
Manages a fixed pool of block IDs keyed by content identity (mm_hash).
|
||||
Entries transition through: not-ready → ready (evictable) → pinned.
|
||||
Eviction targets ready + unpinned entries in FIFO order.
|
||||
|
||||
All public methods are thread-safe.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
class CacheEntry:
|
||||
"""A single cache entry. Read `.block_ids`, `.shape` and `.ready` freely;
|
||||
mutations only through EmbeddingCache methods.
|
||||
|
||||
`shape` is the encoder output the blocks hold, so a load can restore it
|
||||
without knowing how wide this model's encoder output is.
|
||||
"""
|
||||
|
||||
__slots__ = ("block_ids", "shape", "_pin_count")
|
||||
|
||||
def __init__(
|
||||
self, block_ids: tuple[int, ...], shape: tuple[int, int] | None = None
|
||||
) -> None:
|
||||
self.block_ids = block_ids
|
||||
self.shape = shape
|
||||
self._pin_count = -1 # not ready
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
return self._pin_count >= 0
|
||||
|
||||
@property
|
||||
def evictable(self) -> bool:
|
||||
return self._pin_count == 0
|
||||
|
||||
def mark_ready(self):
|
||||
self._pin_count = 0
|
||||
|
||||
def pin(self):
|
||||
self._pin_count += 1
|
||||
|
||||
|
||||
class EmbeddingCache:
|
||||
"""Fixed-size block cache with FIFO eviction.
|
||||
|
||||
Entries are keyed by content identity (e.g. mm_hash). Blocks are
|
||||
allocated from a free-set; when space is needed, ready + unpinned
|
||||
entries are evicted oldest-first.
|
||||
|
||||
The caller is responsible for deciding *when* to call ``mark_ready``
|
||||
(e.g. after enough engine steps have elapsed for the worker to have
|
||||
completed the write).
|
||||
"""
|
||||
|
||||
def __init__(self, num_blocks: int) -> None:
|
||||
self._num_blocks = num_blocks
|
||||
self._lock = threading.Lock()
|
||||
# Available block IDs, LIFO stack.
|
||||
self._free_blocks: list[int] = list(range(num_blocks))
|
||||
# Key → live entry (all states: not-ready, ready, pinned).
|
||||
self._entries: dict[str, CacheEntry] = {}
|
||||
# Ready + unpinned entries in insertion order (FIFO eviction).
|
||||
self._entries_free_list: OrderedDict[str, None] = OrderedDict()
|
||||
# Total blocks reclaimable via eviction (fail-fast check).
|
||||
self._evictable_block_count: int = 0
|
||||
|
||||
def get(self, key: str) -> CacheEntry | None:
|
||||
"""Return the entry for *key*, or None if not present."""
|
||||
with self._lock:
|
||||
return self._entries.get(key)
|
||||
|
||||
def has_held_entries(self) -> bool:
|
||||
"""True if any entry is not-ready or pinned (an in-flight transfer).
|
||||
|
||||
Held entries are exactly those absent from the eviction free list, so
|
||||
they represent saves awaiting mark_ready or loads awaiting unpin.
|
||||
"""
|
||||
with self._lock:
|
||||
return len(self._entries) != len(self._entries_free_list)
|
||||
|
||||
def alloc(
|
||||
self, key: str, n_blocks: int, shape: tuple[int, int] | None = None
|
||||
) -> CacheEntry | None:
|
||||
"""Allocate *n_blocks* for *key*, evicting as needed.
|
||||
|
||||
The entry starts not-ready. Returns None if there is not enough
|
||||
space even after evicting all evictable entries.
|
||||
"""
|
||||
with self._lock:
|
||||
assert key not in self._entries, (
|
||||
f"EmbeddingCache: duplicate alloc for {key!r}"
|
||||
)
|
||||
assert n_blocks <= self._num_blocks, (
|
||||
f"EmbeddingCache: {n_blocks} blocks requested but capacity "
|
||||
f"is {self._num_blocks}"
|
||||
)
|
||||
if len(self._free_blocks) + self._evictable_block_count < n_blocks:
|
||||
return None
|
||||
self._evict_until(n_blocks)
|
||||
# Canonical ascending order: keeps a freed entry's blocks contiguous
|
||||
# in the free list when it is evicted, and lets consumers detect
|
||||
# which of an entry's blocks are adjacent.
|
||||
block_ids = tuple(sorted(self._free_blocks.pop() for _ in range(n_blocks)))
|
||||
entry = CacheEntry(block_ids, shape)
|
||||
self._entries[key] = entry
|
||||
return entry
|
||||
|
||||
def mark_ready(self, key: str) -> None:
|
||||
"""Mark an entry as ready (data is CPU-visible)."""
|
||||
with self._lock:
|
||||
entry = self._entries[key]
|
||||
assert entry._pin_count == -1, (
|
||||
f"EmbeddingCache: mark_ready on already-ready entry {key!r}"
|
||||
)
|
||||
entry.mark_ready()
|
||||
self._entries_free_list[key] = None
|
||||
self._evictable_block_count += len(entry.block_ids)
|
||||
|
||||
def pin(self, key: str) -> None:
|
||||
"""Pin an entry (prevent eviction)."""
|
||||
with self._lock:
|
||||
entry = self._entries[key]
|
||||
assert entry.ready, f"EmbeddingCache: pin of not-ready entry {key!r}"
|
||||
if entry.evictable:
|
||||
del self._entries_free_list[key]
|
||||
self._evictable_block_count -= len(entry.block_ids)
|
||||
entry.pin()
|
||||
|
||||
def pin_if_ready(self, key: str) -> CacheEntry | None:
|
||||
"""Atomically pin *key* if present and ready; return its entry.
|
||||
|
||||
Returns None when the key is absent. An entry returned with `ready`
|
||||
False was not pinned: its save is still in flight, which the producer
|
||||
reports differently from a miss because such a read can be retried.
|
||||
"""
|
||||
with self._lock:
|
||||
entry = self._entries.get(key)
|
||||
if entry is None or not entry.ready:
|
||||
return entry
|
||||
if entry.evictable:
|
||||
del self._entries_free_list[key]
|
||||
self._evictable_block_count -= len(entry.block_ids)
|
||||
entry.pin()
|
||||
return entry
|
||||
|
||||
def unpin(self, key: str) -> None:
|
||||
"""Unpin an entry. Asserts currently pinned."""
|
||||
with self._lock:
|
||||
entry = self._entries[key]
|
||||
assert entry._pin_count > 0, (
|
||||
f"EmbeddingCache: unpin of unpinned entry {key!r}"
|
||||
)
|
||||
entry._pin_count -= 1
|
||||
if entry._pin_count == 0:
|
||||
self._entries_free_list[key] = None
|
||||
self._evictable_block_count += len(entry.block_ids)
|
||||
|
||||
def discard(self, key: str) -> None:
|
||||
"""Remove a not-ready in-flight entry, returning its blocks to the pool.
|
||||
|
||||
Used when an in-flight fill (e.g. a NIXL READ) fails before the entry
|
||||
is marked ready. Asserts the entry is present and not ready — ready or
|
||||
pinned entries are reclaimed through eviction/unpin, not discard.
|
||||
"""
|
||||
with self._lock:
|
||||
entry = self._entries[key]
|
||||
assert entry._pin_count == -1, (
|
||||
f"EmbeddingCache: discard of ready/pinned entry {key!r}"
|
||||
)
|
||||
del self._entries[key]
|
||||
self._free_blocks.extend(entry.block_ids)
|
||||
|
||||
def _evict_until(self, n_blocks: int) -> None:
|
||||
"""Evict ready+unpinned entries FIFO until enough space. Lock held."""
|
||||
while len(self._free_blocks) < n_blocks:
|
||||
key, _ = self._entries_free_list.popitem(last=False)
|
||||
entry = self._entries.pop(key)
|
||||
self._evictable_block_count -= len(entry.block_ids)
|
||||
self._free_blocks.extend(entry.block_ids)
|
||||
@@ -1,644 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Session objects for the ECCPUConnector.
|
||||
|
||||
Layer hierarchy: connector → session → transport (connection)
|
||||
|
||||
Two long-lived sessions:
|
||||
|
||||
ProducerSession — owns ZmqServerTransport. The connector calls poll_step()
|
||||
once per engine step, which drains the transport without blocking and
|
||||
feeds the raw bytes to poll(). poll() decodes XferReqs, grants or NACKs
|
||||
them, pins source blocks, sends XferAck replies, drains NIXL completion
|
||||
notifications, and sweeps expired xfers.
|
||||
|
||||
ConsumerSession — owns one ZmqClientConnection and the NIXL agent
|
||||
registration for that peer. The connector calls poll(messages, now)
|
||||
once per engine step with the raw bytes collected for this peer.
|
||||
poll() decodes XferAcks, dispatches them to ConsumerXfer objects,
|
||||
and advances all active xfer state machines.
|
||||
|
||||
Internal data classes:
|
||||
|
||||
ProducerXfer — one pinned read grant, keyed by session_id.
|
||||
ConsumerXfer — state machine for one NIXL READ, keyed by mm_hash.
|
||||
"""
|
||||
|
||||
import enum
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import msgspec
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.control.zmq import (
|
||||
ZmqClientConnection,
|
||||
ZmqClientTransport,
|
||||
ZmqServerTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.data.base import (
|
||||
DataTransport,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
|
||||
EC_CONNECTOR_VERSION,
|
||||
EXPECTED_NACKS,
|
||||
RETRYABLE_NACKS,
|
||||
XferAck,
|
||||
XferReq,
|
||||
XferStatus,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.utils import PeerAddr
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
PRODUCER_PIN_LEASE_S = 30.0
|
||||
_CONSUMER_READ_TIMEOUT_S = 20.0
|
||||
_CONSUMER_QUARANTINE_TIMEOUT_S = 60.0
|
||||
|
||||
|
||||
# ── XferState ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class XferState(enum.Enum):
|
||||
"""Lifecycle states of a single ConsumerXfer."""
|
||||
|
||||
WAITING_ACK = "waiting_ack" # XferReq sent, awaiting XferAck
|
||||
READING = "reading" # NIXL READ in flight
|
||||
DONE = "done" # READ completed; promote to local cache
|
||||
ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + retry
|
||||
READ_FAILED = "read_failed" # Unexpected NIXL state; free blocks + tombstone
|
||||
QUARANTINED = "quarantined" # Read timed out, NIXL unabortable; keep blocks
|
||||
SETTLED = "settled" # Quarantined DMA done; safe to free blocks
|
||||
|
||||
|
||||
# ── ProducerXfer ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProducerXfer:
|
||||
"""One pinned read grant on the producer side.
|
||||
|
||||
Pure data — lifecycle operations (pin/unpin) are performed by ProducerSession.
|
||||
Keyed by (consumer_session_id, mm_hash) in ProducerSession._active_xfers.
|
||||
"""
|
||||
|
||||
mm_hash: str
|
||||
block_indices: list[int]
|
||||
deadline: float
|
||||
|
||||
def is_expired(self) -> bool:
|
||||
return time.monotonic() > self.deadline
|
||||
|
||||
|
||||
# ── ConsumerXfer ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConsumerXfer:
|
||||
"""State machine for one NIXL READ on the consumer side.
|
||||
|
||||
transfer_handle is the in-flight NIXL handle returned by post_read() —
|
||||
used for check_xfer_state() / release_xfer_handle(). The dlist handle
|
||||
for the remote peer lives inside DataTransport, not here.
|
||||
"""
|
||||
|
||||
mm_hash: str
|
||||
block_indices: list[int]
|
||||
addr: PeerAddr
|
||||
deadline: float
|
||||
data: DataTransport
|
||||
consumer_session_id: str # session identity; combined with mm_hash for notif_msg
|
||||
|
||||
transfer_handle: Any | None = field(default=None)
|
||||
_quarantined: bool = field(default=False, init=False)
|
||||
|
||||
def handle_ack(self, ack: XferAck, agent_name: str) -> bool:
|
||||
"""Issue the NIXL READ on an OK XferAck. Returns False on any NACK."""
|
||||
if ack.status != XferStatus.OK:
|
||||
return False
|
||||
# notif_msg encodes both session_id and mm_hash so the producer can
|
||||
# do a direct (session_id, mm_hash) → ProducerXfer lookup on completion.
|
||||
notif_msg = f"{self.consumer_session_id}:{self.mm_hash}".encode()
|
||||
self.transfer_handle = self.data.post_read(
|
||||
self.block_indices,
|
||||
agent_name,
|
||||
ack.src_block_indices,
|
||||
notif_msg=notif_msg,
|
||||
)
|
||||
self.deadline = time.monotonic() + _CONSUMER_READ_TIMEOUT_S
|
||||
logger.debug(
|
||||
"EC consumer: NIXL READ posted mm_hash=%s agent=%s local_blocks=%d "
|
||||
"remote_blocks=%d",
|
||||
self.mm_hash,
|
||||
agent_name,
|
||||
len(self.block_indices),
|
||||
len(ack.src_block_indices),
|
||||
)
|
||||
return True
|
||||
|
||||
def poll(self, now: float) -> XferState:
|
||||
if self._quarantined:
|
||||
return self._poll_quarantined(now)
|
||||
if self.transfer_handle is None:
|
||||
return (
|
||||
XferState.ACK_TIMEOUT if now > self.deadline else XferState.WAITING_ACK
|
||||
)
|
||||
return self._poll_reading(now)
|
||||
|
||||
def _poll_reading(self, now: float) -> XferState:
|
||||
assert self.transfer_handle is not None
|
||||
try:
|
||||
state = self.data.check_xfer_state(self.transfer_handle)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"EC Connector: check_xfer_state failed for mm_hash=%s", self.mm_hash
|
||||
)
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
return XferState.READ_FAILED
|
||||
if state == "DONE":
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
logger.debug("EC consumer: NIXL READ completed mm_hash=%s", self.mm_hash)
|
||||
return XferState.DONE
|
||||
if state == "PROC":
|
||||
if now > self.deadline:
|
||||
self._quarantined = True
|
||||
self.deadline = now + _CONSUMER_QUARANTINE_TIMEOUT_S
|
||||
logger.warning(
|
||||
"EC Connector: READ for mm_hash=%s from %s:%d timed out; "
|
||||
"quarantining",
|
||||
self.mm_hash,
|
||||
self.addr[0],
|
||||
self.addr[1],
|
||||
)
|
||||
return XferState.QUARANTINED
|
||||
return XferState.READING
|
||||
logger.warning(
|
||||
"EC Connector: READ for mm_hash=%s unexpected NIXL state %r",
|
||||
self.mm_hash,
|
||||
state,
|
||||
)
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
return XferState.READ_FAILED
|
||||
|
||||
def _poll_quarantined(self, now: float) -> XferState:
|
||||
if self.transfer_handle is None:
|
||||
return XferState.SETTLED
|
||||
# If quarantine has been too long, assume NIXL is stuck and evict blocks.
|
||||
if now > self.deadline:
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
logger.warning(
|
||||
"EC Connector: quarantined mm_hash=%s from %s:%d timed out; "
|
||||
"evicting blocks",
|
||||
self.mm_hash,
|
||||
self.addr[0],
|
||||
self.addr[1],
|
||||
)
|
||||
return XferState.READ_FAILED
|
||||
try:
|
||||
state = self.data.check_xfer_state(self.transfer_handle)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"EC Connector: check_xfer_state failed for quarantined mm_hash=%s",
|
||||
self.mm_hash,
|
||||
)
|
||||
state = None
|
||||
if state == "PROC":
|
||||
return XferState.QUARANTINED
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
logger.debug(
|
||||
"EC Connector: quarantined mm_hash=%s settled (state=%s)",
|
||||
self.mm_hash,
|
||||
state,
|
||||
)
|
||||
return XferState.SETTLED
|
||||
|
||||
def cancel(self) -> None:
|
||||
"""Cancel a WAITING_ACK xfer (peer down). No tombstone — caller allows retry."""
|
||||
assert self.transfer_handle is None, "cancel() requires WAITING_ACK state"
|
||||
|
||||
def release(self) -> None:
|
||||
"""Force-release the transfer handle. Shutdown only."""
|
||||
if self.transfer_handle is not None:
|
||||
self.data.release_xfer_handle(self.transfer_handle)
|
||||
self.transfer_handle = None
|
||||
|
||||
|
||||
# ── ProducerSession ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class ProducerSession:
|
||||
"""Long-lived producer session. Owns ZmqServerTransport.
|
||||
|
||||
poll_step() runs once per engine step: drains the transport without
|
||||
blocking, decodes XferReqs, grants or NACKs each one, drains NIXL completion
|
||||
notifications, and sweeps expired xfers.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
transport: ZmqServerTransport,
|
||||
data: DataTransport,
|
||||
cache: EmbeddingCache,
|
||||
compat_hash: str,
|
||||
) -> None:
|
||||
self._transport = transport
|
||||
self._data = data
|
||||
self._cache = cache # EmbeddingCache — single block owner
|
||||
self._compat_hash = compat_hash
|
||||
|
||||
# Key: "{consumer_session_id}:{mm_hash}" — matches the NIXL notif_msg exactly,
|
||||
# so completion notification lookup is a direct dict pop with no parsing.
|
||||
self._active_xfers: dict[str, ProducerXfer] = {}
|
||||
|
||||
self._req_decoder = msgspec.msgpack.Decoder(XferReq)
|
||||
self._req_encoder = msgspec.msgpack.Encoder()
|
||||
|
||||
def close(self) -> None:
|
||||
for xfer in self._active_xfers.values():
|
||||
self._cache.unpin(xfer.mm_hash) # release any still-held pins
|
||||
self._active_xfers.clear()
|
||||
self._transport.close()
|
||||
|
||||
def poll_step(self) -> None:
|
||||
"""Drain the transport without blocking and process what arrived."""
|
||||
self.poll(self._transport.poll(timeout_ms=0))
|
||||
|
||||
def poll(self, messages: list[tuple[bytes, bytes]]) -> None:
|
||||
"""Decode and process inbound XferReqs, drain NIXL notifs, sweep timeouts.
|
||||
|
||||
messages: (identity, payload) pairs from ZmqServerTransport.poll().
|
||||
"""
|
||||
for identity, payload in messages:
|
||||
try:
|
||||
req = self._req_decoder.decode(payload)
|
||||
except (msgspec.DecodeError, msgspec.ValidationError):
|
||||
logger.warning("EC Connector: dropped malformed XferReq")
|
||||
continue
|
||||
ack = self._grant_or_nack(req)
|
||||
self._transport.send(identity, self._req_encoder.encode(ack))
|
||||
|
||||
self._drain_notifs()
|
||||
self._sweep_timeouts()
|
||||
|
||||
def _grant_or_nack(self, req: XferReq) -> XferAck:
|
||||
if req.connector_version != EC_CONNECTOR_VERSION:
|
||||
logger.warning(
|
||||
"EC Connector: incompatible version for mm_hash=%s", req.mm_hash
|
||||
)
|
||||
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_VERSION)
|
||||
if req.compatibility_hash != self._compat_hash:
|
||||
logger.warning(
|
||||
"EC Connector: incompatible compat hash for mm_hash=%s", req.mm_hash
|
||||
)
|
||||
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INCOMPAT)
|
||||
if not req.session_id:
|
||||
# Grants are keyed by (session_id, mm_hash), so an omitted id from
|
||||
# one peer aliases another's: the first completion would unpin
|
||||
# blocks the other peer is still reading.
|
||||
logger.warning(
|
||||
"EC Connector: XferReq without session_id for mm_hash=%s", req.mm_hash
|
||||
)
|
||||
return XferAck(mm_hash=req.mm_hash, status=XferStatus.NACK_INTERNAL)
|
||||
entry = self._cache.pin_if_ready(req.mm_hash)
|
||||
if entry is None or not entry.ready:
|
||||
status = (
|
||||
XferStatus.NACK_MISSING if entry is None else XferStatus.NACK_NOT_READY
|
||||
)
|
||||
logger.log(
|
||||
logging.INFO if status in EXPECTED_NACKS else logging.WARNING,
|
||||
"EC Connector: mm_hash=%s not serveable in local cache (%s); NACKing",
|
||||
req.mm_hash,
|
||||
status.name,
|
||||
)
|
||||
return XferAck(mm_hash=req.mm_hash, status=status)
|
||||
block_indices = list(entry.block_ids)
|
||||
key = f"{req.session_id}:{req.mm_hash}"
|
||||
if key in self._active_xfers:
|
||||
# This peer is re-requesting a grant it still holds, having timed
|
||||
# out waiting. Overwriting it drops the pin taken for it, which
|
||||
# neither a completion nor the sweep would ever release.
|
||||
self._cache.unpin(req.mm_hash)
|
||||
self._active_xfers[key] = ProducerXfer(
|
||||
mm_hash=req.mm_hash,
|
||||
block_indices=block_indices,
|
||||
deadline=time.monotonic() + PRODUCER_PIN_LEASE_S,
|
||||
)
|
||||
logger.debug(
|
||||
"EC producer: serving NIXL READ request mm_hash=%s key=%s blocks=%d",
|
||||
req.mm_hash,
|
||||
key,
|
||||
len(block_indices),
|
||||
)
|
||||
return XferAck(
|
||||
mm_hash=req.mm_hash,
|
||||
status=XferStatus.OK,
|
||||
session_id=req.session_id, # echo consumer's session_id back
|
||||
src_block_indices=block_indices,
|
||||
agent_metadata=self._data.get_agent_metadata(),
|
||||
mem_descriptor=self._data.get_mem_descriptor(),
|
||||
)
|
||||
|
||||
def _drain_notifs(self) -> None:
|
||||
try:
|
||||
notifs = self._data.get_new_notifs()
|
||||
except Exception:
|
||||
logger.exception("EC Connector: get_new_notifs failed")
|
||||
return
|
||||
for msgs in notifs.values():
|
||||
for msg in msgs:
|
||||
try:
|
||||
key = msg.decode("utf-8") # "{session_id}:{mm_hash}"
|
||||
except Exception:
|
||||
logger.exception("EC Connector: Dropped malformed NIXL notif")
|
||||
continue
|
||||
xfer = self._active_xfers.pop(key, None)
|
||||
if xfer is not None:
|
||||
self._cache.unpin(xfer.mm_hash)
|
||||
logger.debug(
|
||||
"EC producer: NIXL READ completed mm_hash=%s key=%s",
|
||||
xfer.mm_hash,
|
||||
key,
|
||||
)
|
||||
|
||||
def _sweep_timeouts(self) -> None:
|
||||
for key, xfer in list(self._active_xfers.items()):
|
||||
if xfer.is_expired():
|
||||
logger.warning(
|
||||
"EC producer: consumer did not complete NIXL READ "
|
||||
"mm_hash=%s key=%s within the pin lease; releasing "
|
||||
"pinned blocks",
|
||||
xfer.mm_hash,
|
||||
key,
|
||||
)
|
||||
self._cache.unpin(xfer.mm_hash)
|
||||
del self._active_xfers[key]
|
||||
|
||||
|
||||
# ── ConsumerSession ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class ConsumerSessionResults:
|
||||
"""Results from ConsumerSession.take_results()."""
|
||||
|
||||
completed: set[str] # promote to local cache
|
||||
tombstoned: set[str] # free blocks + prevent retry this step
|
||||
quarantined: set[str] # timed-out, DMA still running: prevent retry
|
||||
# but DO NOT free blocks — session holds them
|
||||
retryable: set[str] # free blocks, allow another attempt (not ready, peer down)
|
||||
settled: list[tuple[str, list[int]]] # (mm_hash, block_indices) quarantine cleared
|
||||
|
||||
|
||||
class ConsumerSession:
|
||||
"""Long-lived per-producer session on the consumer side.
|
||||
|
||||
Owns the ZMQ DEALER connection and the NIXL agent registration for one
|
||||
producer peer. The connector calls poll(messages, now) once per engine
|
||||
step with the raw bytes collected from this peer's DEALER socket.
|
||||
|
||||
poll() decodes XferAcks, dispatches them to the right ConsumerXfer,
|
||||
advances all active xfer state machines, and drains quarantined xfers.
|
||||
Results accumulate in internal sets; call take_results() to collect them.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
addr: PeerAddr,
|
||||
zmq_conn: ZmqClientConnection,
|
||||
transport: ZmqClientTransport,
|
||||
data: DataTransport,
|
||||
compat_hash: str,
|
||||
) -> None:
|
||||
self._addr = addr
|
||||
self._zmq = zmq_conn
|
||||
self._transport = transport
|
||||
self._data = data
|
||||
self._compat_hash = compat_hash
|
||||
self._session_id = str(
|
||||
uuid.uuid4()
|
||||
) # stable identity for (session_id:mm_hash) keys
|
||||
|
||||
self._nixl_agent_name: str | None = None
|
||||
self._nixl_metadata_bytes: bytes | None = None
|
||||
|
||||
self._xfers: dict[str, ConsumerXfer] = {} # mm_hash → xfer
|
||||
self._quarantined: list[ConsumerXfer] = [] # DMA pending, to poll
|
||||
self._newly_quarantined: set[str] = set() # quarantined since last take_results
|
||||
|
||||
self._completed: set[str] = set()
|
||||
self._tombstoned: set[str] = set()
|
||||
self._retryable: set[str] = set()
|
||||
self._settled: list[tuple[str, list[int]]] = []
|
||||
|
||||
self._encoder = msgspec.msgpack.Encoder()
|
||||
self._decoder = msgspec.msgpack.Decoder(XferAck)
|
||||
|
||||
@property
|
||||
def addr(self) -> PeerAddr:
|
||||
return self._addr
|
||||
|
||||
def start_xfer(
|
||||
self, mm_hash: str, block_indices: list[int], deadline: float
|
||||
) -> None:
|
||||
"""Create a ConsumerXfer and send the XferReq over the connection."""
|
||||
xfer = ConsumerXfer(
|
||||
mm_hash=mm_hash,
|
||||
block_indices=block_indices,
|
||||
addr=self._addr,
|
||||
deadline=deadline,
|
||||
data=self._data,
|
||||
consumer_session_id=self._session_id,
|
||||
)
|
||||
self._zmq.send(
|
||||
self._encoder.encode(
|
||||
XferReq(
|
||||
mm_hash=mm_hash,
|
||||
compatibility_hash=self._compat_hash,
|
||||
session_id=self._session_id,
|
||||
)
|
||||
)
|
||||
)
|
||||
self._xfers[mm_hash] = xfer
|
||||
logger.debug(
|
||||
"EC Connector: XferReq sent mm_hash=%s to %s:%d",
|
||||
mm_hash,
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
)
|
||||
|
||||
def poll(self, messages: list[bytes], now: float) -> None:
|
||||
"""Decode inbound XferAcks, dispatch to xfers, and advance all state machines.
|
||||
|
||||
messages: raw bytes from ZmqClientTransport.poll() for this peer.
|
||||
Results accumulate until take_results() is called.
|
||||
"""
|
||||
for raw in messages:
|
||||
try:
|
||||
ack = self._decoder.decode(raw)
|
||||
except (msgspec.DecodeError, msgspec.ValidationError):
|
||||
logger.warning(
|
||||
"EC Connector: dropped malformed XferAck from %s:%d",
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
)
|
||||
continue
|
||||
self._dispatch_ack(ack)
|
||||
|
||||
for mm_hash, xfer in list(self._xfers.items()):
|
||||
state = xfer.poll(now)
|
||||
if state == XferState.DONE:
|
||||
self._completed.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
elif state == XferState.ACK_TIMEOUT:
|
||||
# The scheduler bounds retries with a per-request deadline.
|
||||
logger.warning(
|
||||
"EC consumer: no XferAck for mm_hash=%s from %s:%d in time; "
|
||||
"will re-request",
|
||||
mm_hash,
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
)
|
||||
self._retryable.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
elif state == XferState.READ_FAILED:
|
||||
self._tombstoned.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
elif state == XferState.QUARANTINED:
|
||||
# DMA may still be writing — caller must NOT free blocks.
|
||||
self._quarantined.append(xfer)
|
||||
self._newly_quarantined.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
|
||||
self._drain_quarantine(now)
|
||||
|
||||
def _dispatch_ack(self, ack: XferAck) -> None:
|
||||
xfer = self._xfers.get(ack.mm_hash)
|
||||
if xfer is None or xfer.transfer_handle is not None:
|
||||
return # unknown mm_hash or duplicate ack
|
||||
|
||||
if ack.status != XferStatus.OK:
|
||||
logger.log(
|
||||
logging.INFO if ack.status in EXPECTED_NACKS else logging.WARNING,
|
||||
"EC Connector: NACK %s from %s:%d for mm_hash=%s",
|
||||
ack.status.name,
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
ack.mm_hash,
|
||||
)
|
||||
del self._xfers[ack.mm_hash]
|
||||
# The producer announced this encoding and its save is still in
|
||||
# flight, so another attempt on a later step can be granted. Every
|
||||
# other NACK is terminal for this mm_hash.
|
||||
if ack.status in RETRYABLE_NACKS:
|
||||
self._retryable.add(ack.mm_hash)
|
||||
else:
|
||||
self._tombstoned.add(ack.mm_hash)
|
||||
return
|
||||
|
||||
try:
|
||||
agent_name = self._ensure_registered(ack.agent_metadata, ack.mem_descriptor)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"EC Connector: failed to register peer %s:%d for mm_hash=%s",
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
ack.mm_hash,
|
||||
)
|
||||
del self._xfers[ack.mm_hash]
|
||||
self._tombstoned.add(ack.mm_hash)
|
||||
return
|
||||
|
||||
if not xfer.handle_ack(ack, agent_name):
|
||||
del self._xfers[ack.mm_hash]
|
||||
self._tombstoned.add(ack.mm_hash)
|
||||
|
||||
def _ensure_registered(self, metadata: bytes, mem_descriptor: bytes) -> str:
|
||||
"""Register or re-register the peer in DataTransport. Returns agent_name."""
|
||||
if self._nixl_agent_name is not None and self._nixl_metadata_bytes == metadata:
|
||||
return self._nixl_agent_name
|
||||
if self._nixl_agent_name is not None:
|
||||
logger.info(
|
||||
"EC Connector: producer %s:%d restarted; re-registering",
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
)
|
||||
self._data.remove_remote_peer(self._nixl_agent_name)
|
||||
agent_name = self._data.add_remote_peer(metadata, mem_descriptor)
|
||||
self._nixl_agent_name = agent_name
|
||||
self._nixl_metadata_bytes = metadata
|
||||
logger.debug(
|
||||
"EC Connector: registered peer %s:%d agent=%s",
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
agent_name,
|
||||
)
|
||||
return agent_name
|
||||
|
||||
def _drain_quarantine(self, now: float) -> None:
|
||||
still_pending: list[ConsumerXfer] = []
|
||||
for xfer in list(self._quarantined):
|
||||
state = xfer.poll(now)
|
||||
if state == XferState.SETTLED:
|
||||
self._settled.append((xfer.mm_hash, xfer.block_indices))
|
||||
else:
|
||||
still_pending.append(xfer)
|
||||
self._quarantined = still_pending
|
||||
|
||||
def on_peer_down(self) -> None:
|
||||
"""WAITING_ACK xfers → cancel (retry allowed); READING xfers → quarantine."""
|
||||
for mm_hash, xfer in list(self._xfers.items()):
|
||||
if xfer.transfer_handle is None:
|
||||
xfer.cancel()
|
||||
self._retryable.add(mm_hash)
|
||||
else:
|
||||
self._quarantined.append(xfer)
|
||||
self._newly_quarantined.add(mm_hash)
|
||||
self._xfers.clear()
|
||||
|
||||
def take_results(self) -> ConsumerSessionResults:
|
||||
"""Return and clear all accumulated results since the last call."""
|
||||
results = ConsumerSessionResults(
|
||||
completed=self._completed,
|
||||
tombstoned=self._tombstoned,
|
||||
quarantined=self._newly_quarantined,
|
||||
retryable=self._retryable,
|
||||
settled=self._settled,
|
||||
)
|
||||
self._completed = set()
|
||||
self._tombstoned = set()
|
||||
self._newly_quarantined = set()
|
||||
self._retryable = set()
|
||||
self._settled = []
|
||||
return results
|
||||
|
||||
def has_in_flight(self) -> bool:
|
||||
return bool(self._xfers) or bool(self._quarantined)
|
||||
|
||||
def close(self) -> None:
|
||||
"""Release all transfer handles, deregister NIXL peer, close connection.
|
||||
|
||||
Connection teardown is routed through the transport so the pooled
|
||||
connection is dropped in lock-step with the session — a session and its
|
||||
DEALER connection have a 1:1 lifecycle, so a later connect() to the same
|
||||
peer builds a fresh socket instead of handing back this dead one.
|
||||
"""
|
||||
for xfer in self._xfers.values():
|
||||
xfer.release()
|
||||
self._xfers.clear()
|
||||
for xfer in self._quarantined:
|
||||
xfer.release()
|
||||
self._quarantined.clear()
|
||||
if self._nixl_agent_name is not None:
|
||||
self._data.remove_remote_peer(self._nixl_agent_name)
|
||||
self._transport.remove(self._addr)
|
||||
@@ -1,47 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Supporting utilities for the ECCPUConnector scheduler."""
|
||||
|
||||
import msgspec
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
# (host, port) pair identifying a remote peer's ZMQ side-channel endpoint.
|
||||
PeerAddr = tuple[str, int]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Msgpack (de)serialization for the list-of-(addr, size, device_id)
|
||||
# descriptor form that NIXL's `get_xfer_descs` expects.
|
||||
_MemDescList = list[tuple[int, int, int]]
|
||||
|
||||
_mem_desc_encoder = msgspec.msgpack.Encoder()
|
||||
_mem_desc_decoder = msgspec.msgpack.Decoder(_MemDescList)
|
||||
|
||||
|
||||
def serialize_mem_descriptor(descs: _MemDescList) -> bytes:
|
||||
return _mem_desc_encoder.encode(descs)
|
||||
|
||||
|
||||
def deserialize_mem_descriptor(payload: bytes) -> _MemDescList:
|
||||
return _mem_desc_decoder.decode(payload)
|
||||
|
||||
|
||||
def build_block_descs(
|
||||
base_ptr: int,
|
||||
num_blocks: int,
|
||||
block_size_bytes: int,
|
||||
device_id: int = 0,
|
||||
) -> _MemDescList:
|
||||
"""Per-block `(addr, size, device_id)` tuples for an `ECSharedRegion`.
|
||||
|
||||
Handed to `nixl.get_xfer_descs` to build a dlist where descriptor `i`
|
||||
addresses block `i` in the mmap. This makes a WRITE to block indices
|
||||
`[i1, i2, ...]` a simple `make_prepped_xfer(..., [i1, i2])` on both
|
||||
ends — no per-request address math.
|
||||
"""
|
||||
return [
|
||||
(base_ptr + i * block_size_bytes, block_size_bytes, device_id)
|
||||
for i in range(num_blocks)
|
||||
]
|
||||
@@ -1,455 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Worker-side of the ECCPUConnector.
|
||||
|
||||
Thin, stateless across steps: opens the shared mmap region and uses the
|
||||
per-step connector metadata (`ECCPUConnectorMetadata`) to decide which
|
||||
blocks to copy in each direction.
|
||||
"""
|
||||
|
||||
from collections import deque
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum, auto
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm._custom_ops import swap_blocks_batch
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
ECCPUWorkerMetadata,
|
||||
create_ec_shared_region,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.worker.descriptor_buffers import (
|
||||
DescriptorBufferPool,
|
||||
DescriptorBuffers,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pcp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ECCPUTransferDirection(Enum):
|
||||
HOST_TO_DEVICE = auto()
|
||||
DEVICE_TO_HOST = auto()
|
||||
|
||||
|
||||
def _coalesce_runs(
|
||||
block_ids: Sequence[int],
|
||||
) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""Split block ids into runs that are consecutive in the region.
|
||||
|
||||
Returns `(slots, first_blocks, num_blocks)`, one element per run, where
|
||||
`slots` is the run's offset in the flat block sequence. Per-descriptor
|
||||
overhead dominates these copies, so describing a run with one descriptor
|
||||
rather than one per block is worth an order of magnitude.
|
||||
|
||||
Both callers derive the non-region side of a run from its slot, so that
|
||||
side advances in lockstep with the region and only the region has to be
|
||||
split. `EmbeddingCache` hands out ascending ids, which is what makes runs
|
||||
findable at all.
|
||||
"""
|
||||
ids = np.fromiter(block_ids, dtype=np.int64, count=len(block_ids))
|
||||
breaks = np.flatnonzero(np.diff(ids) != 1) + 1
|
||||
slots = np.concatenate(([0], breaks))
|
||||
ends = np.concatenate((breaks, [ids.size]))
|
||||
return slots, ids[slots], ends - slots
|
||||
|
||||
|
||||
@dataclass
|
||||
class Transfer:
|
||||
"""A batched copy in flight on a dedicated GPU stream.
|
||||
|
||||
Its descriptor buffers and stream are held until `end_event` fires; only
|
||||
then are the buffers and stream safe to recycle and the `completions` safe
|
||||
to report. `start_event`/`end_event` bracket the copy so its elapsed time
|
||||
can be measured on completion.
|
||||
|
||||
`completions` is what the scheduler is told about this copy: mm_hashes for
|
||||
saves, transfer ids for loads.
|
||||
"""
|
||||
|
||||
start_event: torch.Event
|
||||
end_event: torch.Event
|
||||
completions: list[str] | list[int]
|
||||
bufs: DescriptorBuffers
|
||||
stream: torch.Stream
|
||||
num_bytes: int
|
||||
|
||||
|
||||
class ECCPUWorker:
|
||||
"""Worker-side delegate for the ECCPUConnector.
|
||||
|
||||
- Producer role: copies `encoder_cache[mm_hash]` → `mmap[block_ids]`
|
||||
for each entry in `metadata.saves`. Descriptor buffers are filled
|
||||
directly in `save_caches`; the actual DMA is issued as a single
|
||||
batched call in `flush_saves`.
|
||||
- Consumer role: copies `mmap[block_ids]` → `encoder_cache[mm_hash]`
|
||||
for all entries in `metadata.loads` via a single `swap_blocks_batch`
|
||||
call on a dedicated stream.
|
||||
- On `ec_both` nodes both paths run back-to-back in a single step.
|
||||
|
||||
Every batched copy runs on a stream drawn from a pool so it overlaps the
|
||||
compute stream, bracketed by a start/end event pair. The stream, events,
|
||||
and descriptor buffers are recycled once the end event fires.
|
||||
|
||||
Both paths move memory between the two streams, so each buffer is registered
|
||||
against the stream that is not its own: save sources against the save stream,
|
||||
load destinations against the compute stream. Without that, the caching
|
||||
allocator would be free to reissue memory a copy or a queued read still
|
||||
needs, since it only tracks the stream a buffer was allocated on.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig") -> None:
|
||||
self._region = create_ec_shared_region(vllm_config)
|
||||
# Model dtype; used to reinterpret raw int8 blocks on load.
|
||||
self._dtype = vllm_config.model_config.dtype
|
||||
|
||||
self._pin_shared_region()
|
||||
|
||||
# All TP/PCP ranks hold identical encoder output. Only one rank
|
||||
# per mmap needs to write — saves host memory bandwidth.
|
||||
# DCP is a subdivision of TP, so tp_rank==0 covers it.
|
||||
self._is_save_rank = (
|
||||
get_tensor_model_parallel_rank() == 0 and get_pcp_group().rank_in_group == 0
|
||||
)
|
||||
|
||||
# Descriptor buffer pool (recycled across steps, shared by both paths).
|
||||
self._buf_pool = DescriptorBufferPool()
|
||||
|
||||
# Active save buffer being filled during save_caches calls this step.
|
||||
self._save_bufs: DescriptorBuffers | None = None
|
||||
self._save_count: int = 0
|
||||
# Bytes accumulated into the active save buffer this step, for logging.
|
||||
self._save_bytes: int = 0
|
||||
# mm_hashes accumulated into the active save buffer this step.
|
||||
self._save_mm_hashes: list[str] = []
|
||||
# Stream the active save buffer's copy will run on, held from the moment
|
||||
# the buffer opens so `save_caches` can register its sources against it.
|
||||
self._save_stream: torch.Stream | None = None
|
||||
|
||||
# Batched copies whose end event has not yet fired. Buffers and stream
|
||||
# stay held until the event fires; the completions are reported to the
|
||||
# scheduler only then. Separate per direction because saves and loads
|
||||
# can be in flight concurrently.
|
||||
self._inflight_saves: deque[Transfer] = deque()
|
||||
self._inflight_loads: deque[Transfer] = deque()
|
||||
|
||||
# Streams and timing events recycled across steps to avoid per-step
|
||||
# create/destroy churn. Shared by both directions.
|
||||
self._stream_pool: list[torch.Stream] = []
|
||||
self._event_pool: list[torch.Event] = []
|
||||
|
||||
def _acquire_stream(self) -> torch.Stream:
|
||||
if self._stream_pool:
|
||||
return self._stream_pool.pop()
|
||||
return current_platform.Stream()
|
||||
|
||||
def _acquire_event(self) -> torch.Event:
|
||||
return (
|
||||
self._event_pool.pop()
|
||||
if self._event_pool
|
||||
else torch.Event(enable_timing=True)
|
||||
)
|
||||
|
||||
def _collect_finished(self, inflight: deque[Transfer], direction: str) -> list:
|
||||
"""Pop transfers whose end event has fired, recycle their stream,
|
||||
events, and buffers, and return their completions.
|
||||
|
||||
The front check is conservative: a transfer is reported only once its
|
||||
own end event fires, so completions are always genuinely done, and each
|
||||
is reported exactly once because the transfer is popped as it is
|
||||
reported. A later transfer that happens to finish first simply waits
|
||||
behind the front and is reported on a subsequent poll.
|
||||
"""
|
||||
done: list = []
|
||||
while inflight and inflight[0].end_event.query():
|
||||
transfer = inflight.popleft()
|
||||
done.extend(transfer.completions)
|
||||
elapsed_ms = transfer.start_event.elapsed_time(transfer.end_event)
|
||||
logger.debug(
|
||||
"EC %s: %d entr%s (%d bytes) took %.3f ms",
|
||||
direction,
|
||||
len(transfer.completions),
|
||||
"y" if len(transfer.completions) == 1 else "ies",
|
||||
transfer.num_bytes,
|
||||
elapsed_ms,
|
||||
)
|
||||
self._buf_pool.release(transfer.bufs)
|
||||
self._stream_pool.append(transfer.stream)
|
||||
self._event_pool.append(transfer.start_event)
|
||||
self._event_pool.append(transfer.end_event)
|
||||
return done
|
||||
|
||||
def _pin_shared_region(self) -> None:
|
||||
"""Prepare the shared region for device transfers."""
|
||||
if is_pin_memory_available():
|
||||
self._region.pin_memory()
|
||||
|
||||
def _submit_transfer(
|
||||
self,
|
||||
bufs: DescriptorBuffers,
|
||||
count: int,
|
||||
direction: ECCPUTransferDirection,
|
||||
) -> None:
|
||||
"""Submit a transfer on the current backend stream."""
|
||||
src_ptrs = bufs.src_ptrs[:count]
|
||||
dst_ptrs = bufs.dst_ptrs[:count]
|
||||
sizes = bufs.sizes[:count]
|
||||
if direction == ECCPUTransferDirection.HOST_TO_DEVICE:
|
||||
swap_blocks_batch(
|
||||
src_ptrs,
|
||||
dst_ptrs,
|
||||
sizes,
|
||||
is_src_access_order_any=True,
|
||||
)
|
||||
else:
|
||||
swap_blocks_batch(src_ptrs, dst_ptrs, sizes)
|
||||
|
||||
def _shutdown_transfer_backend(self) -> None:
|
||||
"""Release backend-specific transfer resources."""
|
||||
|
||||
def save_caches(
|
||||
self,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
mm_hash: str,
|
||||
connector_metadata: ECCPUConnectorMetadata,
|
||||
) -> None:
|
||||
"""Fill descriptor buffers directly for batched flush.
|
||||
|
||||
Registers `encoder_cache[mm_hash]` against the stream the copy will run
|
||||
on, so the caching allocator will not hand that memory to another
|
||||
allocation before the copy has read it. The descriptor buffers hold raw
|
||||
addresses, and the caller is free to evict the entry as soon as this
|
||||
step's saves are dispatched.
|
||||
"""
|
||||
if not self._is_save_rank:
|
||||
return
|
||||
block_ids = connector_metadata.saves.get(mm_hash)
|
||||
if block_ids is None:
|
||||
return
|
||||
|
||||
src = encoder_cache[mm_hash]
|
||||
total_bytes = src.numel() * src.element_size()
|
||||
block_size = self._region.block_size_bytes
|
||||
allocated_bytes = len(block_ids) * block_size
|
||||
assert total_bytes <= allocated_bytes, (
|
||||
f"EC: encoder output exceeds allocated blocks for mm_hash={mm_hash}: "
|
||||
f"{total_bytes} bytes but only {allocated_bytes} allocated "
|
||||
f"({len(block_ids)} blocks × {block_size}). "
|
||||
f"shape={list(src.shape)}"
|
||||
)
|
||||
|
||||
if self._save_bufs is None:
|
||||
total = sum(len(v) for v in connector_metadata.saves.values())
|
||||
self._save_bufs = self._buf_pool.acquire(total)
|
||||
self._save_stream = self._acquire_stream()
|
||||
self._save_mm_hashes = []
|
||||
|
||||
assert self._save_count + len(block_ids) <= self._save_bufs.src_ptrs.numel()
|
||||
|
||||
bufs = self._save_bufs
|
||||
assert self._save_stream is not None
|
||||
src.record_stream(self._save_stream)
|
||||
# Device addresses are unsigned and can have the high bit set (XPU USM),
|
||||
# so they are added up in uint64; signed arithmetic rejects them.
|
||||
src_base = np.uint64(src.view(-1).view(torch.uint8).data_ptr())
|
||||
dst_base = self._region.blocks.data_ptr()
|
||||
# Copy only the tensor bytes, excluding any block padding.
|
||||
n_used = -(-total_bytes // block_size)
|
||||
if n_used:
|
||||
slots, first_blocks, num_blocks = _coalesce_runs(block_ids[:n_used])
|
||||
run_bytes = num_blocks * block_size
|
||||
# The output can stop partway into the final run.
|
||||
run_bytes[-1] = total_bytes - slots[-1] * block_size
|
||||
bufs.add_copies(
|
||||
self._save_count,
|
||||
src_base + slots.astype(np.uint64) * np.uint64(block_size),
|
||||
dst_base + first_blocks * block_size,
|
||||
run_bytes,
|
||||
)
|
||||
self._save_count += slots.size
|
||||
|
||||
self._save_bytes += total_bytes
|
||||
self._save_mm_hashes.append(mm_hash)
|
||||
|
||||
def flush_saves(self) -> None:
|
||||
"""Flush all accumulated saves in a single swap_blocks_batch call.
|
||||
|
||||
Runs the copy on the stream the batch acquired when it opened, gated
|
||||
behind the compute stream that produced the encoder outputs, brackets it
|
||||
with start/end events, and enqueues the batch as in-flight. The stream,
|
||||
events, and descriptor buffers are recycled once the end event fires (see
|
||||
`_collect_finished`), which is also when the saved mm_hashes become
|
||||
safe to mark ready.
|
||||
"""
|
||||
if self._save_count == 0:
|
||||
return
|
||||
|
||||
bufs = self._save_bufs
|
||||
stream = self._save_stream
|
||||
assert bufs is not None and stream is not None
|
||||
n = self._save_count
|
||||
num_bytes = self._save_bytes
|
||||
|
||||
# Gate the GPU→CPU copy behind the compute stream: it reads the encoder
|
||||
# outputs the model just produced there.
|
||||
stream.wait_stream(current_platform.current_stream())
|
||||
start_event = self._acquire_event()
|
||||
end_event = self._acquire_event()
|
||||
with current_platform.stream(stream):
|
||||
start_event.record(stream)
|
||||
self._submit_transfer(
|
||||
bufs,
|
||||
n,
|
||||
ECCPUTransferDirection.DEVICE_TO_HOST,
|
||||
)
|
||||
end_event.record(stream)
|
||||
|
||||
self._inflight_saves.append(
|
||||
Transfer(
|
||||
start_event=start_event,
|
||||
end_event=end_event,
|
||||
completions=self._save_mm_hashes,
|
||||
bufs=bufs,
|
||||
stream=stream,
|
||||
num_bytes=num_bytes,
|
||||
)
|
||||
)
|
||||
|
||||
self._save_bufs = None
|
||||
self._save_stream = None
|
||||
self._save_count = 0
|
||||
self._save_bytes = 0
|
||||
self._save_mm_hashes = []
|
||||
|
||||
def start_load_caches(
|
||||
self,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
connector_metadata: ECCPUConnectorMetadata,
|
||||
) -> None:
|
||||
"""Consumer path: single batched copy of all loads from mmap→GPU.
|
||||
|
||||
The destination is registered against the compute stream that consumes
|
||||
it, so the caching allocator will not hand that memory to a later load
|
||||
while reads of it are still queued there.
|
||||
"""
|
||||
if not connector_metadata.loads:
|
||||
return
|
||||
|
||||
block_size = self._region.block_size_bytes
|
||||
blocks = self._region.blocks
|
||||
dtype = self._dtype
|
||||
device_type = current_platform.device_type
|
||||
src_base = blocks.data_ptr()
|
||||
|
||||
# Copy every dispatched load. The scheduler routes an input here only
|
||||
# after missing the GPU encoder cache, so a resident hit is not expected;
|
||||
# copying unconditionally means every participating rank reports each
|
||||
# transfer exactly once, which is what the scheduler counts to release
|
||||
# the pin.
|
||||
# Ordered by first block so entries that neighbour each other in the
|
||||
# region land next to each other in the destination and coalesce into
|
||||
# one descriptor. Every loop below consumes this same order.
|
||||
load_items = sorted(
|
||||
connector_metadata.loads.items(), key=lambda kv: kv[1][1][0]
|
||||
)
|
||||
total_blocks = sum(len(block_ids) for _, (_, block_ids, _) in load_items)
|
||||
|
||||
stream = self._acquire_stream()
|
||||
compute_stream = current_platform.current_stream()
|
||||
start_event = self._acquire_event()
|
||||
end_event = self._acquire_event()
|
||||
with current_platform.stream(stream):
|
||||
# Single contiguous destination buffer for all loads.
|
||||
dst_buf = torch.empty(
|
||||
total_blocks, block_size, dtype=torch.int8, device=device_type
|
||||
)
|
||||
dst_buf.record_stream(compute_stream)
|
||||
# See save_caches: device addresses are added up in uint64.
|
||||
dst_buf_base = np.uint64(dst_buf.data_ptr())
|
||||
|
||||
bufs = self._buf_pool.acquire(total_blocks)
|
||||
slots, first_blocks, num_blocks = _coalesce_runs(
|
||||
list(chain.from_iterable(ids for _, (_, ids, _) in load_items))
|
||||
)
|
||||
bufs.add_copies(
|
||||
0,
|
||||
src_base + first_blocks * block_size,
|
||||
dst_buf_base + slots.astype(np.uint64) * np.uint64(block_size),
|
||||
num_blocks * block_size,
|
||||
)
|
||||
op_idx = slots.size
|
||||
|
||||
start_event.record(stream)
|
||||
self._submit_transfer(
|
||||
bufs,
|
||||
op_idx,
|
||||
ECCPUTransferDirection.HOST_TO_DEVICE,
|
||||
)
|
||||
end_event.record(stream)
|
||||
self._inflight_loads.append(
|
||||
Transfer(
|
||||
start_event=start_event,
|
||||
end_event=end_event,
|
||||
completions=[tid for _, (tid, _, _) in load_items],
|
||||
bufs=bufs,
|
||||
stream=stream,
|
||||
num_bytes=total_blocks * block_size,
|
||||
)
|
||||
)
|
||||
|
||||
# Slice contiguous buffer into per-hash views.
|
||||
offset = 0
|
||||
for mm_hash, (_, block_ids, shape) in load_items:
|
||||
n = len(block_ids)
|
||||
flat = dst_buf[offset : offset + n].view(dtype).reshape(-1)
|
||||
encoder_cache[mm_hash] = flat[: shape[0] * shape[1]].view(shape)
|
||||
offset += n
|
||||
|
||||
current_platform.current_stream().wait_stream(stream)
|
||||
|
||||
def build_connector_worker_meta(self) -> ECCPUWorkerMetadata | None:
|
||||
"""Report the GPU copies that completed on this rank this step: saved
|
||||
mm_hashes and loaded transfer ids.
|
||||
|
||||
Returns None when nothing finished, so the scheduler sees no payload.
|
||||
"""
|
||||
completed_saves = self._collect_finished(self._inflight_saves, "save")
|
||||
completed_loads = self._collect_finished(self._inflight_loads, "load")
|
||||
if not completed_saves and not completed_loads:
|
||||
return None
|
||||
return ECCPUWorkerMetadata(
|
||||
completed_saves=completed_saves,
|
||||
completed_loads=completed_loads,
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
for transfer in (*self._inflight_saves, *self._inflight_loads):
|
||||
transfer.end_event.synchronize()
|
||||
self._shutdown_transfer_backend()
|
||||
self._save_bufs = None
|
||||
self._save_stream = None
|
||||
self._save_count = 0
|
||||
self._save_bytes = 0
|
||||
self._save_mm_hashes = []
|
||||
self._inflight_saves.clear()
|
||||
self._inflight_loads.clear()
|
||||
self._stream_pool.clear()
|
||||
self._event_pool.clear()
|
||||
try:
|
||||
self._region.cleanup()
|
||||
except Exception:
|
||||
logger.debug("EC: worker region cleanup failed", exc_info=True)
|
||||
@@ -1,90 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reusable pool of (src_ptrs, dst_ptrs, sizes) tensor triples.
|
||||
|
||||
Used by ECCPUWorker to batch swap_blocks_batch descriptors without
|
||||
per-step allocation overhead.
|
||||
"""
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# CUDA/ROCm cache_kernels.cu requires int64 pointers; the XPU DMA engine
|
||||
# requires uint64 (see vllm._custom_ops.swap_blocks_batch).
|
||||
_PTR_DTYPE = torch.uint64 if current_platform.is_xpu() else torch.int64
|
||||
|
||||
|
||||
class DescriptorBuffers(NamedTuple):
|
||||
src_ptrs: torch.Tensor
|
||||
dst_ptrs: torch.Tensor
|
||||
sizes: torch.Tensor
|
||||
# Numpy aliases of the three tensors, written via add_copies().
|
||||
src_np: np.ndarray
|
||||
dst_np: np.ndarray
|
||||
sizes_np: np.ndarray
|
||||
|
||||
def add_copies(
|
||||
self,
|
||||
first: int,
|
||||
sources: np.ndarray,
|
||||
destinations: np.ndarray,
|
||||
byte_counts: np.ndarray,
|
||||
) -> None:
|
||||
"""Store a batch of copies in these buffers, the first at *first*.
|
||||
|
||||
One copy is one source address, one destination address, and a number
|
||||
of bytes. `swap_blocks_batch` performs every copy held in the buffers
|
||||
in a single call, so a caller fills in all of a step's copies before
|
||||
asking for any of them to run, and passes *first* to say where its own
|
||||
copies begin.
|
||||
|
||||
Whole batches go in at once because storing a single number into a
|
||||
torch tensor costs microseconds. Adding copies one at a time can take
|
||||
longer than performing them.
|
||||
|
||||
TODO(torch>=2.14): write the tensors directly and drop these numpy
|
||||
aliases once the minimum supported torch is 2.14. They exist only
|
||||
because torch's setitem unpacks the value as a signed long long before
|
||||
2.14 (pytorch#191458), rejecting XPU USM addresses >= 2**63, while the
|
||||
two's-complement rewrite those versions accept is in turn rejected by
|
||||
uint64 from 2.14 on. Numpy casts against the array's own type and so
|
||||
works on either. Byte counts are small enough to need none of this
|
||||
care, and use their alias only to match the addresses.
|
||||
"""
|
||||
last = first + len(sources)
|
||||
self.src_np[first:last] = sources
|
||||
self.dst_np[first:last] = destinations
|
||||
self.sizes_np[first:last] = byte_counts
|
||||
|
||||
|
||||
class DescriptorBufferPool:
|
||||
"""Pool of descriptor buffer triples for swap_blocks_batch.
|
||||
|
||||
Each buffer is a `DescriptorBuffers` namedtuple of three 1-D tensors
|
||||
(dtype `_PTR_DTYPE`, platform-dependent) of equal length, paired with
|
||||
numpy aliases used to fill them. Buffers are recycled across steps; if a
|
||||
returned buffer is too small it is discarded and a fresh one allocated.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# LIFO stack of idle buffer triples.
|
||||
self._pool: list[DescriptorBuffers] = []
|
||||
|
||||
def acquire(self, n: int) -> DescriptorBuffers:
|
||||
"""Get a buffer triple with capacity >= *n*."""
|
||||
if self._pool:
|
||||
bufs = self._pool.pop()
|
||||
if bufs.src_ptrs.numel() >= n:
|
||||
return bufs
|
||||
src, dst, sizes = (torch.empty(n, dtype=_PTR_DTYPE) for _ in range(3))
|
||||
return DescriptorBuffers(
|
||||
src, dst, sizes, src.numpy(), dst.numpy(), sizes.numpy()
|
||||
)
|
||||
|
||||
def release(self, bufs: DescriptorBuffers) -> None:
|
||||
"""Return a buffer triple to the pool for reuse."""
|
||||
self._pool.append(bufs)
|
||||
@@ -1,231 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import safetensors
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorBase,
|
||||
ECConnectorMetadata,
|
||||
ECConnectorRole,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MMMeta:
|
||||
mm_hash: str
|
||||
num_token: int
|
||||
|
||||
@staticmethod
|
||||
def make_meta(mm_hash, num_token) -> "MMMeta":
|
||||
return MMMeta(mm_hash=mm_hash, num_token=num_token)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECExampleConnectorMetadata(ECConnectorMetadata):
|
||||
mm_datas: list[MMMeta]
|
||||
|
||||
def __init__(self):
|
||||
self.mm_datas = []
|
||||
|
||||
def add_mm_data(self, mm_data: MMMeta):
|
||||
self.mm_datas.append(mm_data)
|
||||
|
||||
|
||||
class ECExampleConnector(ECConnectorBase):
|
||||
# NOTE: This is Simple debug implementation of the EC connector.
|
||||
# It save / load the EC cache to / from the disk.
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole):
|
||||
super().__init__(vllm_config=vllm_config, role=role)
|
||||
# req_id -> index
|
||||
self._mm_datas_need_loads: dict[str, int] = {}
|
||||
self._metadata_resolver = PlaceholderMetadataResolver(vllm_config.model_config)
|
||||
transfer_config = vllm_config.ec_transfer_config
|
||||
if transfer_config is not None:
|
||||
self._storage_path = transfer_config.get_from_extra_config(
|
||||
"shared_storage_path", "/tmp"
|
||||
)
|
||||
logger.debug(transfer_config)
|
||||
logger.debug("Shared storage path is %s", self._storage_path)
|
||||
else:
|
||||
raise ValueError("ec_transfer_config must be set for ECConnectorBase")
|
||||
|
||||
def start_load_caches(self, encoder_cache, **kwargs) -> None:
|
||||
"""Start loading the cache from the connector into vLLM's encoder cache.
|
||||
|
||||
This method loads the encoder cache based on metadata provided by the scheduler.
|
||||
It is called before `_gather_mm_embeddings` for the EC Connector. For EC,
|
||||
the `encoder_cache` and `mm_hash` are stored in `kwargs`.
|
||||
|
||||
Args:
|
||||
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# Get the metadata
|
||||
metadata: ECConnectorMetadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, ECExampleConnectorMetadata)
|
||||
assert encoder_cache is not None
|
||||
if metadata is None:
|
||||
logger.warning(
|
||||
"In connector.start_load_caches, but the connector metadata is None"
|
||||
)
|
||||
return
|
||||
# A bare "cuda" makes safetensors load onto cuda:0. Pin the load to the
|
||||
# selected device so tensor-parallel workers load the cache locally.
|
||||
device = current_platform.device_type
|
||||
if current_platform.is_cuda_alike():
|
||||
device = f"{device}:{current_platform.current_device()}"
|
||||
# Load the EC for each mm data
|
||||
for mm_data in metadata.mm_datas:
|
||||
if mm_data.mm_hash in encoder_cache:
|
||||
continue
|
||||
filename = self._generate_filename_debug(mm_data.mm_hash)
|
||||
with gpu_sync_allowed():
|
||||
ec_cache = safetensors.torch.load_file(filename, device=device)[
|
||||
"ec_cache"
|
||||
]
|
||||
encoder_cache[mm_data.mm_hash] = ec_cache
|
||||
logger.debug("Success load encoder cache for hash %s", mm_data.mm_hash)
|
||||
|
||||
def save_caches(self, encoder_cache, mm_hash, **kwargs) -> None:
|
||||
"""Save the encoder cache to the connector.
|
||||
|
||||
This method saves the encoder cache from the worker's local storage
|
||||
to shared storage or another external connector.
|
||||
|
||||
Args:
|
||||
encoder_cache (dict[str, torch.Tensor]): A dictionary mapping multimodal
|
||||
data hashes (`mm_hash`) to encoder cache tensors.
|
||||
mm_hash (str): The hash of the multimodal data whose cache is being saved.
|
||||
kwargs (dict): Additional keyword arguments for the connector.
|
||||
|
||||
"""
|
||||
# Return if it is PD Instance
|
||||
if not self.is_producer:
|
||||
return
|
||||
filename = self._generate_filename_debug(mm_hash)
|
||||
ec_cache = encoder_cache[mm_hash]
|
||||
with gpu_sync_allowed():
|
||||
tensors = {"ec_cache": ec_cache.detach().cpu()}
|
||||
safetensors.torch.save_file(tensors, filename)
|
||||
logger.debug("Save cache successful for mm_hash %s", mm_hash)
|
||||
|
||||
def has_cache_item(
|
||||
self,
|
||||
identifier: str,
|
||||
) -> bool:
|
||||
"""Check if cache exist externally for the media.
|
||||
|
||||
Args:
|
||||
identifier (str): the identifier of the media.
|
||||
|
||||
Returns:
|
||||
Bool indicate that media exists in cache or not
|
||||
|
||||
"""
|
||||
return self._found_match_for_mm_data(identifier)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: "Request",
|
||||
index: int,
|
||||
) -> None:
|
||||
"""Update ECConnector state after encoder cache allocation."""
|
||||
mm_hash = request.mm_features[index].identifier
|
||||
# Only load cache if it is consumer and cache exists
|
||||
if not self.is_consumer or not self.has_cache_item(mm_hash):
|
||||
return
|
||||
num_encoder_token = request.get_num_encoder_embeds(index)
|
||||
self._mm_datas_need_loads[mm_hash] = num_encoder_token
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> ECConnectorMetadata:
|
||||
"""Build the connector metadata for this step.
|
||||
|
||||
This function should NOT modify any fields in the scheduler_output.
|
||||
Also, calling this function will reset the state of the connector.
|
||||
This only build for load mm_data only
|
||||
Args:
|
||||
scheduler_output (SchedulerOutput): the scheduler output object.
|
||||
"""
|
||||
meta = ECExampleConnectorMetadata()
|
||||
for mm_hash, num_encoder_token in self._mm_datas_need_loads.items():
|
||||
meta.add_mm_data(MMMeta.make_meta(mm_hash, num_encoder_token))
|
||||
self._mm_datas_need_loads.clear()
|
||||
return meta
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: "Request",
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Report each item's cache key and grid so a consumer can skip the
|
||||
image transform.
|
||||
|
||||
A consumer only needs the grid to size the prompt's placeholder range;
|
||||
the embedding itself arrives through this connector. Reporting the grid
|
||||
the producer actually computed keeps the two sides in agreement without
|
||||
the caller re-deriving it from the raw media.
|
||||
"""
|
||||
if not self.is_producer:
|
||||
return False, None
|
||||
|
||||
items = collect_ec_item_metadata(request.mm_features, self._metadata_resolver)
|
||||
if not items:
|
||||
return False, None
|
||||
return False, items
|
||||
|
||||
# ==============================
|
||||
# Helper functions
|
||||
# ==============================
|
||||
|
||||
def _found_match_for_mm_data(self, mm_hash) -> bool:
|
||||
"""Check if the cache is hit for the request."""
|
||||
filename = self._generate_filename_debug(mm_hash, create_folder=False)
|
||||
return os.path.exists(filename)
|
||||
|
||||
def _generate_foldername_debug(
|
||||
self,
|
||||
mm_hash: str,
|
||||
create_folder: bool = True, # <- now defaults to True
|
||||
) -> str:
|
||||
"""Return the folder in which the cache for this mm_hash lives.
|
||||
If `create_folder` is True (default) the directory is created
|
||||
recursively the first time it is needed.
|
||||
"""
|
||||
foldername = os.path.join(self._storage_path, mm_hash)
|
||||
if create_folder:
|
||||
os.makedirs(foldername, exist_ok=True)
|
||||
return foldername
|
||||
|
||||
def _generate_filename_debug(
|
||||
self,
|
||||
mm_hash: str,
|
||||
create_folder: bool = True,
|
||||
) -> str:
|
||||
"""Return the full path of the safetensors file for this mm_hash.
|
||||
If `create_folder` is True (default) the parent directory is created.
|
||||
"""
|
||||
foldername = self._generate_foldername_debug(mm_hash, create_folder)
|
||||
return os.path.join(foldername, "encoder_cache.safetensors")
|
||||
Reference in New Issue
Block a user