[Bugfix][EC Connector] Fail the request, not the engine, when a remote encoding cannot arrive (#55290)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Tianyu Guo
2026-09-08 20:13:33 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 050f29caf1
commit f1aeff6007
8 changed files with 536 additions and 104 deletions
@@ -104,6 +104,10 @@ class ECCPUConnector(ECConnectorBase):
request, num_computed_tokens
)
def take_unavailable_requests(self) -> set[str]:
assert self.connector_scheduler is not None
return self.connector_scheduler.take_unavailable_requests()
def update_state_after_alloc(self, request: "Request", index: int) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.update_state_after_alloc(request, index)
@@ -7,6 +7,7 @@ Owns the mmap region and the embedding cache, and handles the producer
for the ECCPUConnector.
"""
import time
from collections.abc import Collection
from typing import TYPE_CHECKING, Any
@@ -29,12 +30,20 @@ from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ECConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
# Total remote wait budget per request item, including all retries.
_ADMIT_DEFER_TIMEOUT_S = 60.0
class _RemoteUnavailable(Exception):
"""An item cannot be fetched within its request's remote wait budget."""
class ECCPUScheduler:
"""Scheduler delegate for the ECCPUConnector."""
@@ -92,6 +101,11 @@ class ECCPUScheduler:
self._in_flight: set[str] = set()
self._tombstones: set[str] = set()
self._step_completed: set[str] = set()
# Per-request clocks survive retries and are cleared on completion.
self._deferred_since: dict[tuple[str, str], float] = {}
# Requests needing a remote encoding that will not arrive. Drained by
# take_unavailable_requests(); the scheduler aborts them.
self._unrecoverable: set[str] = set()
self._peer_host: str | None = None
self._peer_port: int | None = None
# Model shape for size checks + compat hash; only set by
@@ -117,6 +131,7 @@ class ECCPUScheduler:
compute_ec_compatibility_hash,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
PRODUCER_PIN_LEASE_S,
ProducerSession,
)
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
@@ -136,7 +151,9 @@ class ECCPUScheduler:
# its scheduler step, so its reply latency scales with the encoder's
# batch size: a deployment whose steps run longer must raise this.
self._ack_timeout_s = float(
self._ec_config.get_from_extra_config("consumer_ack_timeout_s", 2.0)
self._ec_config.get_from_extra_config(
"consumer_ack_timeout_s", PRODUCER_PIN_LEASE_S
)
)
self._compat_hash = compute_ec_compatibility_hash(
vllm_version=VLLM_VERSION,
@@ -203,79 +220,172 @@ class ECCPUScheduler:
def _nixl_consumer_admit(
self, request: "Request", num_computed_tokens: int
) -> bool:
"""Admit a request only once all its remote encodings are cached.
"""Admit a request once its required remote encodings are ready.
Returns True when every encoding the request needs is either already
ready in the local cache or has no remote source. Any encoding whose
NIXL READ is still in flight (or was just started this step) defers
the request, which the scheduler re-presents on a later step.
Each item has a per-request wait budget that survives transfer retries.
Unavailable items fall back to local input; without it, the request is
reported by take_unavailable_requests() for the scheduler to abort.
Returns:
True if no item needs to wait for a remote encoding. False if any
item is pending or the request must be aborted.
"""
params: dict[str, dict[str, Any]] = (
getattr(request, "ec_transfer_params", None) or {}
)
if not params:
if not request.ec_transfer_params:
return True
now = time.monotonic()
pending = False
for feature in request.mm_features:
pos = feature.mm_position
if pos.offset + pos.length <= num_computed_tokens:
continue
mm_hash = feature.identifier
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
# Local hit: upstream's update_state_after_alloc pins and
# loads it through the same path as a natively cached entry.
continue
if mm_hash in self._in_flight:
pending = True
continue
if mm_hash in self._step_completed:
pending = True
continue
if mm_hash in self._tombstones:
self._tombstones.discard(mm_hash)
continue
if entry is not None:
# Present but not ready and not being fetched: its blocks are
# held by a quarantined/settling DMA and cannot be reused.
# Fall back to local compute this step; never re-alloc a
# mm_hash already in the cache.
continue
info = params.get(mm_hash)
# Absent entirely, or present with only placeholder metadata (no
# cache entry existed on the producer side at request_finished
# time): nothing to fetch, fall back to local compute.
if not info or "peer_host" not in info:
continue
expected = pos.length * self._hidden_dim * self._element_size
if int(info.get("size_bytes", -1)) != expected:
logger.warning(
"EC consumer: size mismatch mm_hash=%s; local encode", mm_hash
)
continue
try:
started = self._start_xfer(mm_hash, info, expected)
except Exception:
logger.exception(
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
)
continue
if not started:
continue
self._in_flight.add(mm_hash)
pending = True
if not self._admit_item(request, feature, num_computed_tokens, now):
pending = True
except _RemoteUnavailable as error:
if not self._fail_or_fallback(request, feature, str(error)):
return False
return not pending
def _admit_item(
self,
request: "Request",
feature: "MultiModalFeatureSpec",
num_computed_tokens: int,
now: float,
) -> bool:
"""Check one encoding, starting a remote read if needed.
Returns:
True if already computed, cached, or not remotely sourced. False
while waiting for a read or cache space, including across retries.
Raises:
_RemoteUnavailable: The remote source is invalid, the read failed,
or the item's total wait budget expired.
"""
pos = feature.mm_position
mm_hash = feature.identifier
if pos.offset + pos.length <= num_computed_tokens:
self._deferred_since.pop((request.request_id, mm_hash), None)
return True
announced = (request.ec_transfer_params or {}).get(mm_hash)
# Without a producer address there is nothing to fetch: the request
# carries what the model needs and the encoder runs locally.
# `ec_transfer_params` reaches us from the request, so its shape is
# checked rather than assumed.
remote: dict[str, Any] | None = (
announced
if isinstance(announced, dict) and "peer_host" in announced
else None
)
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
# Local hit: upstream's update_state_after_alloc pins and
# loads it through the same path as a natively cached entry.
self._deferred_since.pop((request.request_id, mm_hash), None)
return True
if remote is None:
return True
since = self._deferred_since.setdefault((request.request_id, mm_hash), now)
if now - since > _ADMIT_DEFER_TIMEOUT_S:
raise _RemoteUnavailable(f"the remote wait exceeded {now - since:.0f}s")
if mm_hash in self._in_flight or mm_hash in self._step_completed:
return False
if mm_hash in self._tombstones:
self._tombstones.discard(mm_hash)
raise _RemoteUnavailable("the remote read failed")
if entry is not None:
# Present but not ready and not being fetched: its blocks are
# held by a quarantined/settling DMA and cannot be reused.
return False
expected = pos.length * self._hidden_dim * self._element_size
try:
size = int(remote["size_bytes"])
except (KeyError, TypeError, ValueError) as error:
raise _RemoteUnavailable("the announced size was unusable") from error
if size != expected:
logger.warning(
"EC consumer: size mismatch mm_hash=%s announced=%d expected=%d",
mm_hash,
size,
expected,
)
raise _RemoteUnavailable("the announced size was wrong")
try:
started = self._start_xfer(mm_hash, remote, expected)
except Exception as error:
logger.exception(
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
)
raise _RemoteUnavailable("the read could not be started") from error
if started:
self._in_flight.add(mm_hash)
return False
def _fail_or_fallback(
self, request: "Request", feature: "MultiModalFeatureSpec", why: str
) -> bool:
"""Resolve an unavailable remote encoding and clear its wait budget.
Returns:
True if local media or embeddings are available, removing this
item's remote announcement so subsequent steps do not retry it.
False if only placeholder metadata (or no input) is available,
recording the request for take_unavailable_requests() to drain.
"""
mm_hash = feature.identifier
self._deferred_since.pop((request.request_id, mm_hash), None)
data = feature.data
metadata = (
self._metadata_resolver.fields_for(feature.modality) if data else set()
)
if data and any(
value.data is not None for key, value in data.items() if key not in metadata
):
# Passthrough embeddings bypass the processor cache. SHM address
# items therefore refer to media that the worker can encode.
request.ec_transfer_params = dict(request.ec_transfer_params or {})
request.ec_transfer_params.pop(mm_hash, None)
logger.warning(
"EC consumer: request %s mm_hash=%s: %s; using local input",
request.request_id,
mm_hash,
why,
)
return True
self._unrecoverable.add(request.request_id)
logger.error(
"EC consumer: request %s needs remote encoding mm_hash=%s but %s. "
"No local model input is available; failing the request.",
request.request_id,
mm_hash,
why,
)
return False
def take_unavailable_requests(self) -> set[str]:
"""Return and clear IDs of requests that cannot obtain their encodings.
The scheduler must abort these requests rather than leave them waiting.
Requests that can fall back to local input are not included, and each
recorded failure is returned only once.
"""
if not self._unrecoverable:
return set()
failed = self._unrecoverable
self._unrecoverable = set()
return failed
def _start_xfer(
self, mm_hash: str, info: "dict[str, Any]", size_bytes: int
) -> bool:
"""Allocate a not-ready cache entry and start a NIXL READ into it.
Returns True if the transfer was started. Returns False when the
cache cannot accommodate the encoding, in which case the request
falls back to local recomputation.
cache cannot accommodate the encoding; admission will retry it.
"""
import time
from math import ceil
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
@@ -286,7 +396,7 @@ class ECCPUScheduler:
entry = self._cache.alloc(mm_hash, n_blocks)
if entry is None:
logger.debug(
"EC consumer: cache full for mm_hash=%s (%d blocks); local encode",
"EC consumer: cache full for mm_hash=%s (%d blocks); deferring",
mm_hash,
n_blocks,
)
@@ -319,8 +429,6 @@ class ECCPUScheduler:
return True
def _poll_step(self) -> None:
import time
now = time.monotonic()
all_messages = self._transport.poll()
for addr, session in list(self._sessions.items()):
@@ -352,8 +460,7 @@ class ECCPUScheduler:
mm_hash,
)
for mm_hash in r.retryable:
# No tombstone: dropping every trace of the attempt is what lets
# the next admit pass re-issue the read as if it were the first.
# Release this attempt's blocks, retaining the request wait budget.
self._in_flight.discard(mm_hash)
self._cache.discard(mm_hash)
logger.debug(
@@ -472,6 +579,8 @@ class ECCPUScheduler:
def request_finished(
self, request: "Request"
) -> tuple[bool, "dict[str, Any] | None"]:
for feature in request.mm_features:
self._deferred_since.pop((request.request_id, feature.identifier), None)
if not (self._nixl_enabled and self._is_producer):
return False, None
if not request.mm_features:
@@ -520,6 +629,7 @@ class ECCPUScheduler:
self._pending_saves.clear()
self._pending_loads.clear()
self._load_acks.clear()
self._deferred_since.clear()
self._is_producer = False
self._is_consumer = False
@@ -71,7 +71,7 @@ class XferState(enum.Enum):
WAITING_ACK = "waiting_ack" # XferReq sent, awaiting XferAck
READING = "reading" # NIXL READ in flight
DONE = "done" # READ completed; promote to local cache
ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + tombstone
ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + retry
READ_FAILED = "read_failed" # Unexpected NIXL state; free blocks + tombstone
QUARANTINED = "quarantined" # Read timed out, NIXL unabortable; keep blocks
SETTLED = "settled" # Quarantined DMA done; safe to free blocks
@@ -500,7 +500,18 @@ class ConsumerSession:
if state == XferState.DONE:
self._completed.add(mm_hash)
del self._xfers[mm_hash]
elif state in (XferState.ACK_TIMEOUT, XferState.READ_FAILED):
elif state == XferState.ACK_TIMEOUT:
# The scheduler bounds retries with a per-request deadline.
logger.warning(
"EC consumer: no XferAck for mm_hash=%s from %s:%d in time; "
"will re-request",
mm_hash,
self._addr[0],
self._addr[1],
)
self._retryable.add(mm_hash)
del self._xfers[mm_hash]
elif state == XferState.READ_FAILED:
self._tombstoned.add(mm_hash)
del self._xfers[mm_hash]
elif state == XferState.QUARANTINED: