[Bugfix][EC Connector] Fail the request, not the engine, when a remote encoding cannot arrive (#55290)
Signed-off-by: Tianyu Guo <guoty@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
050f29caf1
commit
f1aeff6007
@@ -104,6 +104,10 @@ class ECCPUConnector(ECConnectorBase):
|
||||
request, num_computed_tokens
|
||||
)
|
||||
|
||||
def take_unavailable_requests(self) -> set[str]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.take_unavailable_requests()
|
||||
|
||||
def update_state_after_alloc(self, request: "Request", index: int) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_state_after_alloc(request, index)
|
||||
|
||||
@@ -7,6 +7,7 @@ Owns the mmap region and the embedding cache, and handles the producer
|
||||
for the ECCPUConnector.
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Collection
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
@@ -29,12 +30,20 @@ from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ECConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Total remote wait budget per request item, including all retries.
|
||||
_ADMIT_DEFER_TIMEOUT_S = 60.0
|
||||
|
||||
|
||||
class _RemoteUnavailable(Exception):
|
||||
"""An item cannot be fetched within its request's remote wait budget."""
|
||||
|
||||
|
||||
class ECCPUScheduler:
|
||||
"""Scheduler delegate for the ECCPUConnector."""
|
||||
@@ -92,6 +101,11 @@ class ECCPUScheduler:
|
||||
self._in_flight: set[str] = set()
|
||||
self._tombstones: set[str] = set()
|
||||
self._step_completed: set[str] = set()
|
||||
# Per-request clocks survive retries and are cleared on completion.
|
||||
self._deferred_since: dict[tuple[str, str], float] = {}
|
||||
# Requests needing a remote encoding that will not arrive. Drained by
|
||||
# take_unavailable_requests(); the scheduler aborts them.
|
||||
self._unrecoverable: set[str] = set()
|
||||
self._peer_host: str | None = None
|
||||
self._peer_port: int | None = None
|
||||
# Model shape for size checks + compat hash; only set by
|
||||
@@ -117,6 +131,7 @@ class ECCPUScheduler:
|
||||
compute_ec_compatibility_hash,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
|
||||
PRODUCER_PIN_LEASE_S,
|
||||
ProducerSession,
|
||||
)
|
||||
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
|
||||
@@ -136,7 +151,9 @@ class ECCPUScheduler:
|
||||
# its scheduler step, so its reply latency scales with the encoder's
|
||||
# batch size: a deployment whose steps run longer must raise this.
|
||||
self._ack_timeout_s = float(
|
||||
self._ec_config.get_from_extra_config("consumer_ack_timeout_s", 2.0)
|
||||
self._ec_config.get_from_extra_config(
|
||||
"consumer_ack_timeout_s", PRODUCER_PIN_LEASE_S
|
||||
)
|
||||
)
|
||||
self._compat_hash = compute_ec_compatibility_hash(
|
||||
vllm_version=VLLM_VERSION,
|
||||
@@ -203,79 +220,172 @@ class ECCPUScheduler:
|
||||
def _nixl_consumer_admit(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
"""Admit a request only once all its remote encodings are cached.
|
||||
"""Admit a request once its required remote encodings are ready.
|
||||
|
||||
Returns True when every encoding the request needs is either already
|
||||
ready in the local cache or has no remote source. Any encoding whose
|
||||
NIXL READ is still in flight (or was just started this step) defers
|
||||
the request, which the scheduler re-presents on a later step.
|
||||
Each item has a per-request wait budget that survives transfer retries.
|
||||
Unavailable items fall back to local input; without it, the request is
|
||||
reported by take_unavailable_requests() for the scheduler to abort.
|
||||
|
||||
Returns:
|
||||
True if no item needs to wait for a remote encoding. False if any
|
||||
item is pending or the request must be aborted.
|
||||
"""
|
||||
params: dict[str, dict[str, Any]] = (
|
||||
getattr(request, "ec_transfer_params", None) or {}
|
||||
)
|
||||
if not params:
|
||||
if not request.ec_transfer_params:
|
||||
return True
|
||||
now = time.monotonic()
|
||||
pending = False
|
||||
for feature in request.mm_features:
|
||||
pos = feature.mm_position
|
||||
if pos.offset + pos.length <= num_computed_tokens:
|
||||
continue
|
||||
mm_hash = feature.identifier
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and entry.ready:
|
||||
# Local hit: upstream's update_state_after_alloc pins and
|
||||
# loads it through the same path as a natively cached entry.
|
||||
continue
|
||||
if mm_hash in self._in_flight:
|
||||
pending = True
|
||||
continue
|
||||
if mm_hash in self._step_completed:
|
||||
pending = True
|
||||
continue
|
||||
if mm_hash in self._tombstones:
|
||||
self._tombstones.discard(mm_hash)
|
||||
continue
|
||||
if entry is not None:
|
||||
# Present but not ready and not being fetched: its blocks are
|
||||
# held by a quarantined/settling DMA and cannot be reused.
|
||||
# Fall back to local compute this step; never re-alloc a
|
||||
# mm_hash already in the cache.
|
||||
continue
|
||||
info = params.get(mm_hash)
|
||||
# Absent entirely, or present with only placeholder metadata (no
|
||||
# cache entry existed on the producer side at request_finished
|
||||
# time): nothing to fetch, fall back to local compute.
|
||||
if not info or "peer_host" not in info:
|
||||
continue
|
||||
expected = pos.length * self._hidden_dim * self._element_size
|
||||
if int(info.get("size_bytes", -1)) != expected:
|
||||
logger.warning(
|
||||
"EC consumer: size mismatch mm_hash=%s; local encode", mm_hash
|
||||
)
|
||||
continue
|
||||
try:
|
||||
started = self._start_xfer(mm_hash, info, expected)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
|
||||
)
|
||||
continue
|
||||
if not started:
|
||||
continue
|
||||
self._in_flight.add(mm_hash)
|
||||
pending = True
|
||||
if not self._admit_item(request, feature, num_computed_tokens, now):
|
||||
pending = True
|
||||
except _RemoteUnavailable as error:
|
||||
if not self._fail_or_fallback(request, feature, str(error)):
|
||||
return False
|
||||
return not pending
|
||||
|
||||
def _admit_item(
|
||||
self,
|
||||
request: "Request",
|
||||
feature: "MultiModalFeatureSpec",
|
||||
num_computed_tokens: int,
|
||||
now: float,
|
||||
) -> bool:
|
||||
"""Check one encoding, starting a remote read if needed.
|
||||
|
||||
Returns:
|
||||
True if already computed, cached, or not remotely sourced. False
|
||||
while waiting for a read or cache space, including across retries.
|
||||
|
||||
Raises:
|
||||
_RemoteUnavailable: The remote source is invalid, the read failed,
|
||||
or the item's total wait budget expired.
|
||||
"""
|
||||
pos = feature.mm_position
|
||||
mm_hash = feature.identifier
|
||||
if pos.offset + pos.length <= num_computed_tokens:
|
||||
self._deferred_since.pop((request.request_id, mm_hash), None)
|
||||
return True
|
||||
announced = (request.ec_transfer_params or {}).get(mm_hash)
|
||||
# Without a producer address there is nothing to fetch: the request
|
||||
# carries what the model needs and the encoder runs locally.
|
||||
# `ec_transfer_params` reaches us from the request, so its shape is
|
||||
# checked rather than assumed.
|
||||
remote: dict[str, Any] | None = (
|
||||
announced
|
||||
if isinstance(announced, dict) and "peer_host" in announced
|
||||
else None
|
||||
)
|
||||
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and entry.ready:
|
||||
# Local hit: upstream's update_state_after_alloc pins and
|
||||
# loads it through the same path as a natively cached entry.
|
||||
self._deferred_since.pop((request.request_id, mm_hash), None)
|
||||
return True
|
||||
if remote is None:
|
||||
return True
|
||||
since = self._deferred_since.setdefault((request.request_id, mm_hash), now)
|
||||
if now - since > _ADMIT_DEFER_TIMEOUT_S:
|
||||
raise _RemoteUnavailable(f"the remote wait exceeded {now - since:.0f}s")
|
||||
if mm_hash in self._in_flight or mm_hash in self._step_completed:
|
||||
return False
|
||||
|
||||
if mm_hash in self._tombstones:
|
||||
self._tombstones.discard(mm_hash)
|
||||
raise _RemoteUnavailable("the remote read failed")
|
||||
|
||||
if entry is not None:
|
||||
# Present but not ready and not being fetched: its blocks are
|
||||
# held by a quarantined/settling DMA and cannot be reused.
|
||||
return False
|
||||
|
||||
expected = pos.length * self._hidden_dim * self._element_size
|
||||
try:
|
||||
size = int(remote["size_bytes"])
|
||||
except (KeyError, TypeError, ValueError) as error:
|
||||
raise _RemoteUnavailable("the announced size was unusable") from error
|
||||
if size != expected:
|
||||
logger.warning(
|
||||
"EC consumer: size mismatch mm_hash=%s announced=%d expected=%d",
|
||||
mm_hash,
|
||||
size,
|
||||
expected,
|
||||
)
|
||||
raise _RemoteUnavailable("the announced size was wrong")
|
||||
|
||||
try:
|
||||
started = self._start_xfer(mm_hash, remote, expected)
|
||||
except Exception as error:
|
||||
logger.exception(
|
||||
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
|
||||
)
|
||||
raise _RemoteUnavailable("the read could not be started") from error
|
||||
if started:
|
||||
self._in_flight.add(mm_hash)
|
||||
return False
|
||||
|
||||
def _fail_or_fallback(
|
||||
self, request: "Request", feature: "MultiModalFeatureSpec", why: str
|
||||
) -> bool:
|
||||
"""Resolve an unavailable remote encoding and clear its wait budget.
|
||||
|
||||
Returns:
|
||||
True if local media or embeddings are available, removing this
|
||||
item's remote announcement so subsequent steps do not retry it.
|
||||
False if only placeholder metadata (or no input) is available,
|
||||
recording the request for take_unavailable_requests() to drain.
|
||||
"""
|
||||
mm_hash = feature.identifier
|
||||
self._deferred_since.pop((request.request_id, mm_hash), None)
|
||||
data = feature.data
|
||||
metadata = (
|
||||
self._metadata_resolver.fields_for(feature.modality) if data else set()
|
||||
)
|
||||
if data and any(
|
||||
value.data is not None for key, value in data.items() if key not in metadata
|
||||
):
|
||||
# Passthrough embeddings bypass the processor cache. SHM address
|
||||
# items therefore refer to media that the worker can encode.
|
||||
request.ec_transfer_params = dict(request.ec_transfer_params or {})
|
||||
request.ec_transfer_params.pop(mm_hash, None)
|
||||
logger.warning(
|
||||
"EC consumer: request %s mm_hash=%s: %s; using local input",
|
||||
request.request_id,
|
||||
mm_hash,
|
||||
why,
|
||||
)
|
||||
return True
|
||||
self._unrecoverable.add(request.request_id)
|
||||
logger.error(
|
||||
"EC consumer: request %s needs remote encoding mm_hash=%s but %s. "
|
||||
"No local model input is available; failing the request.",
|
||||
request.request_id,
|
||||
mm_hash,
|
||||
why,
|
||||
)
|
||||
return False
|
||||
|
||||
def take_unavailable_requests(self) -> set[str]:
|
||||
"""Return and clear IDs of requests that cannot obtain their encodings.
|
||||
|
||||
The scheduler must abort these requests rather than leave them waiting.
|
||||
Requests that can fall back to local input are not included, and each
|
||||
recorded failure is returned only once.
|
||||
"""
|
||||
if not self._unrecoverable:
|
||||
return set()
|
||||
failed = self._unrecoverable
|
||||
self._unrecoverable = set()
|
||||
return failed
|
||||
|
||||
def _start_xfer(
|
||||
self, mm_hash: str, info: "dict[str, Any]", size_bytes: int
|
||||
) -> bool:
|
||||
"""Allocate a not-ready cache entry and start a NIXL READ into it.
|
||||
|
||||
Returns True if the transfer was started. Returns False when the
|
||||
cache cannot accommodate the encoding, in which case the request
|
||||
falls back to local recomputation.
|
||||
cache cannot accommodate the encoding; admission will retry it.
|
||||
"""
|
||||
import time
|
||||
from math import ceil
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
|
||||
@@ -286,7 +396,7 @@ class ECCPUScheduler:
|
||||
entry = self._cache.alloc(mm_hash, n_blocks)
|
||||
if entry is None:
|
||||
logger.debug(
|
||||
"EC consumer: cache full for mm_hash=%s (%d blocks); local encode",
|
||||
"EC consumer: cache full for mm_hash=%s (%d blocks); deferring",
|
||||
mm_hash,
|
||||
n_blocks,
|
||||
)
|
||||
@@ -319,8 +429,6 @@ class ECCPUScheduler:
|
||||
return True
|
||||
|
||||
def _poll_step(self) -> None:
|
||||
import time
|
||||
|
||||
now = time.monotonic()
|
||||
all_messages = self._transport.poll()
|
||||
for addr, session in list(self._sessions.items()):
|
||||
@@ -352,8 +460,7 @@ class ECCPUScheduler:
|
||||
mm_hash,
|
||||
)
|
||||
for mm_hash in r.retryable:
|
||||
# No tombstone: dropping every trace of the attempt is what lets
|
||||
# the next admit pass re-issue the read as if it were the first.
|
||||
# Release this attempt's blocks, retaining the request wait budget.
|
||||
self._in_flight.discard(mm_hash)
|
||||
self._cache.discard(mm_hash)
|
||||
logger.debug(
|
||||
@@ -472,6 +579,8 @@ class ECCPUScheduler:
|
||||
def request_finished(
|
||||
self, request: "Request"
|
||||
) -> tuple[bool, "dict[str, Any] | None"]:
|
||||
for feature in request.mm_features:
|
||||
self._deferred_since.pop((request.request_id, feature.identifier), None)
|
||||
if not (self._nixl_enabled and self._is_producer):
|
||||
return False, None
|
||||
if not request.mm_features:
|
||||
@@ -520,6 +629,7 @@ class ECCPUScheduler:
|
||||
self._pending_saves.clear()
|
||||
self._pending_loads.clear()
|
||||
self._load_acks.clear()
|
||||
self._deferred_since.clear()
|
||||
|
||||
self._is_producer = False
|
||||
self._is_consumer = False
|
||||
|
||||
@@ -71,7 +71,7 @@ class XferState(enum.Enum):
|
||||
WAITING_ACK = "waiting_ack" # XferReq sent, awaiting XferAck
|
||||
READING = "reading" # NIXL READ in flight
|
||||
DONE = "done" # READ completed; promote to local cache
|
||||
ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + tombstone
|
||||
ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + retry
|
||||
READ_FAILED = "read_failed" # Unexpected NIXL state; free blocks + tombstone
|
||||
QUARANTINED = "quarantined" # Read timed out, NIXL unabortable; keep blocks
|
||||
SETTLED = "settled" # Quarantined DMA done; safe to free blocks
|
||||
@@ -500,7 +500,18 @@ class ConsumerSession:
|
||||
if state == XferState.DONE:
|
||||
self._completed.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
elif state in (XferState.ACK_TIMEOUT, XferState.READ_FAILED):
|
||||
elif state == XferState.ACK_TIMEOUT:
|
||||
# The scheduler bounds retries with a per-request deadline.
|
||||
logger.warning(
|
||||
"EC consumer: no XferAck for mm_hash=%s from %s:%d in time; "
|
||||
"will re-request",
|
||||
mm_hash,
|
||||
self._addr[0],
|
||||
self._addr[1],
|
||||
)
|
||||
self._retryable.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
elif state == XferState.READ_FAILED:
|
||||
self._tombstoned.add(mm_hash)
|
||||
del self._xfers[mm_hash]
|
||||
elif state == XferState.QUARANTINED:
|
||||
|
||||
Reference in New Issue
Block a user