[Bugfix][EC Connector] Fail the request, not the engine, when a remote encoding cannot arrive (#55290)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Tianyu Guo
2026-09-08 20:13:33 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 050f29caf1
commit f1aeff6007
8 changed files with 536 additions and 104 deletions
@@ -58,6 +58,26 @@ EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integrati
## How It Works ## How It Works
### NIXL EC failure isolation (1E + 1PD)
```bash
PATH="$PWD/.venv/bin:$PATH" CUDA_VISIBLE_DEVICES=0,1 \
.venv/bin/python -m pytest \
tests/v1/ec_connector/integration/test_nixl_failure.py -v -s
```
Requires two CUDA GPUs and NIXL. `MODEL` overrides the default
`Qwen/Qwen2.5-VL-3B-Instruct`. The test starts real E/PD servers with CUDA
graphs enabled and uses the proxy's rewrite helper. Only the failing request's
control endpoint is replaced: a ZMQ peer returns `NACK_MISSING` and verifies
that the consumer actually requested the encoding.
Metadata-only requests must return a request-level error; requests retaining
the image must succeed through local fallback. After each case, a fresh
metadata-only image request must succeed through the real NIXL transfer path.
No server restart, proxy retry, cache hit, or timeout race can mask the result.
This is a correctness test, not a throughput benchmark.
### Mooncake EC (1E + 1PD) ### Mooncake EC (1E + 1PD)
```bash ```bash
@@ -0,0 +1,202 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NIXL request failure isolation with real E/PD servers and an injected NACK.
Requires two CUDA GPUs and NIXL. Set MODEL to override the small default model.
"""
import asyncio
import copy
import json
import os
import uuid
from pathlib import Path
import aiohttp
import msgspec
import pytest
import zmq
import zmq.asyncio
from PIL import Image
from examples.disaggregated.disaggregated_encoder.disagg_epd_proxy import (
rewrite_for_decode,
)
from tests.utils import RemoteOpenAIServer
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
XferAck,
XferReq,
XferStatus,
)
from vllm.multimodal.utils import encode_image_url
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
MODEL = os.environ.get("MODEL", "Qwen/Qwen2.5-VL-3B-Instruct")
@pytest.fixture(scope="module")
def servers():
pytest.importorskip("nixl")
if not current_platform.is_cuda() or current_platform.device_count() < 2:
pytest.skip("Requires two CUDA GPUs")
devices = os.environ.get("CUDA_VISIBLE_DEVICES", "0,1").split(",")
common = [
"--host",
"127.0.0.1",
"--dtype",
"bfloat16",
"--max-model-len",
"4096",
"--max-num-seqs",
"4",
"--gpu-memory-utilization",
"0.7",
"--no-enable-prefix-caching",
]
def config(role):
return json.dumps(
{
"ec_connector": "ECCPUConnector",
"ec_role": role,
"ec_connector_extra_config": {
"ec_enable_nixl": True,
"ec_cpu_bytes": 64 << 20,
},
}
)
with (
RemoteOpenAIServer(
MODEL,
common
+ [
"--mm-encoder-only",
"--disable-hybrid-kv-cache-manager",
"--ec-transfer-config",
config("ec_producer"),
],
env_dict={
"CUDA_VISIBLE_DEVICES": devices[0],
"VLLM_EC_SIDE_CHANNEL_HOST": "127.0.0.1",
"VLLM_EC_SIDE_CHANNEL_PORT": str(get_open_port()),
},
) as encoder,
RemoteOpenAIServer(
MODEL,
common
+ ["--enable-mm-embeds", "--ec-transfer-config", config("ec_consumer")],
env_dict={"CUDA_VISIBLE_DEVICES": devices[1]},
) as consumer,
):
yield encoder, consumer
async def _post(session, server, body):
async with session.post(server.url_for("v1/chat/completions"), json=body) as r:
return r.status, await r.json()
async def _encode(session, encoder):
image = Image.open(Path(__file__).with_name("hato.jpg")).convert("RGB")
image.thumbnail((224, 224))
item_uuid = uuid.uuid4().hex # Avoid both EC and processor cache hits.
body = {
"model": MODEL,
"messages": [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {"url": encode_image_url(image)},
"uuid": item_uuid,
},
{"type": "text", "text": "Describe the image briefly."},
],
}
],
"max_tokens": 16,
"temperature": 0,
"chat_template_kwargs": {"enable_thinking": False},
}
status, result = await _post(session, encoder, body)
assert status == 200, result
params = result["ec_transfer_params"]
assert len(params) == 1, params
((mm_hash, handle),) = params.items()
assert handle["metadata"] and handle["peer_host"] and handle["size_bytes"] > 0
rewritten = rewrite_for_decode(
body,
{
0: {
**handle["metadata"],
"mm_hash": item_uuid,
"ec_mm_hash": mm_hash,
}
},
)
assert rewritten["messages"][0]["content"][0]["type"] == "image_embeds"
for request in (body, rewritten):
request["ec_transfer_params"] = {item_uuid: copy.deepcopy(handle)}
return body, rewritten, item_uuid
@pytest.mark.asyncio
@pytest.mark.parametrize("local_image", [False, True], ids=["metadata-only", "pixels"])
async def test_missing_remote_encoding_is_request_scoped(servers, local_image):
"""A terminal NACK must preserve local fallback and subsequent serving."""
encoder, consumer = servers
timeout = aiohttp.ClientTimeout(total=120)
async with aiohttp.ClientSession(timeout=timeout) as session:
original, rewritten, mm_hash = await _encode(session, encoder)
request = original if local_image else rewritten
# Keep the real encoder's shape/metadata; replace only the control peer.
# The socket stays bound until completion, so no port or timing race is
# needed to make the consumer receive NACK_MISSING.
with zmq.asyncio.Context() as ctx, ctx.socket(zmq.ROUTER) as router:
router.setsockopt(zmq.LINGER, 0)
port = router.bind_to_random_port("tcp://127.0.0.1")
request["ec_transfer_params"][mm_hash].update(
peer_host="127.0.0.1", peer_port=port
)
async def reject_read():
identity, delimiter, payload = await router.recv_multipart()
req = msgspec.msgpack.decode(payload, type=XferReq)
assert req.mm_hash == mm_hash
ack = XferAck(
mm_hash=req.mm_hash,
status=XferStatus.NACK_MISSING,
session_id=req.session_id,
)
await router.send_multipart(
[identity, delimiter, msgspec.msgpack.encode(ack)]
)
rejection = asyncio.create_task(asyncio.wait_for(reject_read(), 30))
try:
status, result = await _post(session, consumer, request)
# A frontend rejection must not count as successful injection.
await rejection
finally:
rejection.cancel()
await asyncio.gather(rejection, return_exceptions=True)
if local_image:
assert status == 200, result
assert result["choices"][0]["message"]["content"], result
else:
assert status == 500, result
assert result["error"]["type"] == "InternalServerError", result
async with session.get(consumer.url_for("health")) as health:
assert health.status == 200, "The failed read killed EngineCore"
# A fresh metadata-only image must cross the real NIXL data path after
# the fault, not just pass a frontend health check or hit a local cache.
_, healthy, _ = await _encode(session, encoder)
status, result = await _post(session, consumer, healthy)
assert status == 200, result
assert result["choices"][0]["message"]["content"], result
@@ -1,13 +1,22 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
import uuid import uuid
import pytest
import torch
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import ( from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion, ECSharedRegion,
) )
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
from vllm.multimodal.inputs import (
MultiModalFieldElem,
MultiModalKwargsItem,
MultiModalSharedField,
)
_N, _BS, _HID, _ES = 16, 64, 32, 2 _N, _BS, _HID, _ES = 16, 64, 32, 2
@@ -18,10 +27,12 @@ class _Pos:
class _Feature: class _Feature:
def __init__(self, mm_hash, length=1): def __init__(self, mm_hash, length=1, data=None):
self.mm_hash = mm_hash self.mm_hash = mm_hash
self.identifier = mm_hash self.identifier = mm_hash
self.mm_position = _Pos(0, length) self.mm_position = _Pos(0, length)
self.data = data
self.modality = "image"
class _Request: class _Request:
@@ -94,6 +105,7 @@ def _consumer_sched(monkeypatch):
# directly since this helper builds gate-off then flips fields on. # directly since this helper builds gate-off then flips fields on.
s._hidden_dim = _HID s._hidden_dim = _HID
s._element_size = _ES s._element_size = _ES
s._metadata_resolver._cache["image"] = {"image_grid_thw"}
return s return s
@@ -150,13 +162,17 @@ def test_new_remote_read_defers_then_completes(monkeypatch):
s.shutdown() s.shutdown()
def test_no_params_is_ready(monkeypatch): @pytest.mark.parametrize("params", [None, {"x": 123}])
def test_no_remote_announcement_is_ready(monkeypatch, params):
s = _consumer_sched(monkeypatch) s = _consumer_sched(monkeypatch)
assert s.ensure_cache_available(_Request([_Feature("x")], params=None), 0) is True assert s.ensure_cache_available(_Request([_Feature("x")], params=params), 0)
assert s.take_unavailable_requests() == set()
s.shutdown() s.shutdown()
def test_tombstoned_read_discards_and_blocks_retry(monkeypatch): @pytest.mark.parametrize("payload", [None, "metadata", "pixels", "embeds", "shm"])
def test_failed_read_falls_back_only_with_local_input(monkeypatch, payload):
"""A remote miss is fatal only when local model input is unavailable."""
s = _consumer_sched(monkeypatch) s = _consumer_sched(monkeypatch)
fake = _FakeSession() fake = _FakeSession()
@@ -167,26 +183,71 @@ def test_tombstoned_read_discards_and_blocks_retry(monkeypatch):
return True return True
monkeypatch.setattr(s, "_start_xfer", _fake_start) monkeypatch.setattr(s, "_start_xfer", _fake_start)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1)) fields = {
"metadata": {"image_grid_thw": torch.tensor([1, 2, 2])},
"pixels": {"pixel_values": torch.ones(1, 3)},
"embeds": {"image_embeds": torch.ones(1, _HID)},
"shm": {"address": 123, "monotonic_id": 1},
}
data = (
MultiModalKwargsItem(
{
key: MultiModalFieldElem(value, MultiModalSharedField(batch_size=1))
for key, value in fields[payload].items()
}
)
if payload is not None
else None
)
req = _Request([_Feature("h1", 1, data)], params=_params("h1", 1))
# Step 1: start the read. # Step 1: start the read.
assert s.ensure_cache_available(req, 0) is False assert s.ensure_cache_available(req, 0) is False
s.build_connector_meta(scheduler_output=None) s.build_connector_meta(scheduler_output=None)
# Producer rejected the read -> tombstoned.
fake._results.tombstoned.add("h1") fake._results.tombstoned.add("h1")
s._sessions[("h", 1)] = fake s._sessions[("h", 1)] = fake
# Step 2: poll discards the entry and records a tombstone; the tombstone can_fallback = payload in ("pixels", "embeds", "shm")
# is consumed by admit so the request proceeds to local compute. assert s.ensure_cache_available(req, 0) is can_fallback
assert s.ensure_cache_available(req, 0) is True
assert s._cache.get("h1") is None assert s._cache.get("h1") is None
assert "h1" not in s._in_flight assert "h1" not in s._in_flight
s.build_connector_meta(scheduler_output=None) assert s.take_unavailable_requests() == (set() if can_fallback else {"r1"})
# Drained by the read, so the scheduler cannot abort it twice.
assert s.take_unavailable_requests() == set()
if can_fallback:
# A later scheduling step must not retry the failed remote source.
s.build_connector_meta(scheduler_output=None)
assert s.ensure_cache_available(req, 0)
assert fake.started == ["h1"]
s.shutdown()
# Step 3: tombstone was consumed, so a fresh transfer starts again.
assert s.ensure_cache_available(req, 0) is False @pytest.mark.parametrize("retry", [False, True], ids=["in-flight", "retryable"])
assert "h1" in s._in_flight def test_remote_wait_budget_survives_retries_and_long_steps(monkeypatch, retry):
"""Neither retries nor delayed scheduling may restart the request budget."""
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
s._sessions[("h", 1)] = fake
def start(mm_hash, info, size):
assert s._cache.alloc(mm_hash, 1) is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", start)
now = [1000.0]
monkeypatch.setattr(time, "monotonic", lambda: now[0])
req = _Request([_Feature("h1")], params=_params("h1", 1))
assert not s.ensure_cache_available(req, 0)
for elapsed in (20, 40, 121):
now[0] = 1000.0 + elapsed
if retry:
fake._results.retryable.add("h1")
s.build_connector_meta(scheduler_output=None)
assert not s.ensure_cache_available(req, 0)
assert s.take_unavailable_requests() == ({"r1"} if elapsed > 60 else set())
assert len(fake.started) == (3 if retry else 1)
s.shutdown() s.shutdown()
@@ -225,17 +286,50 @@ def test_retryable_read_re_requests_without_admitting(monkeypatch):
s.shutdown() s.shutdown()
def test_size_mismatch_skips_transfer(monkeypatch): @pytest.mark.parametrize(
"size_fields",
[{"size_bytes": 999}, {"size_bytes": None}, {"size_bytes": "big"}, {}],
ids=["mismatch", "null-size", "text-size", "no-size"],
)
def test_invalid_remote_size_fails_the_request(monkeypatch, size_fields):
"""Invalid remote sizes fail the request without raising in the scheduler."""
s = _consumer_sched(monkeypatch) s = _consumer_sched(monkeypatch)
# Advertised size disagrees with pos.length * hidden_dim * element_size. announced = {"peer_host": "h", "peer_port": 1, **size_fields}
bad = {"h1": {"peer_host": "h", "peer_port": 1, "size_bytes": 999}} req = _Request([_Feature("h1", 1)], params={"h1": announced})
req = _Request([_Feature("h1", 1)], params=bad) assert s.ensure_cache_available(req, 0) is False
assert s.ensure_cache_available(req, 0) is True assert s.take_unavailable_requests() == {"r1"}
assert "h1" not in s._in_flight assert "h1" not in s._in_flight
assert s._cache.get("h1") is None assert s._cache.get("h1") is None
s.shutdown() s.shutdown()
def test_deferral_budget_is_per_request(monkeypatch):
"""Shared hashes have independent wait budgets and cancellation cleanup."""
s = _consumer_sched(monkeypatch)
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
params = _params("h1", 1)
old = _Request([_Feature("h1", 1)], params=params, req_id="old")
new = _Request([_Feature("h1", 1)], params=params, req_id="new")
now = [1000.0]
monkeypatch.setattr(time, "monotonic", lambda: now[0])
assert s.ensure_cache_available(old, 0) is False
now[0] += sched_mod._ADMIT_DEFER_TIMEOUT_S - 1
assert s.ensure_cache_available(new, 0) is False
assert s.take_unavailable_requests() == set()
now[0] += 2
# `old` is now past its budget; `new` is not.
assert s.ensure_cache_available(new, 0) is False
assert s.take_unavailable_requests() == set()
assert s.request_finished(new) == (False, None)
assert set(s._deferred_since) == {("old", "h1")}
assert s.ensure_cache_available(old, 0) is False
assert s.take_unavailable_requests() == {"old"}
assert "h1" not in s._in_flight
s.shutdown()
def test_metadata_only_entry_admits_without_transfer(monkeypatch): def test_metadata_only_entry_admits_without_transfer(monkeypatch):
# A producer that reported placeholder metadata but had no cache entry # A producer that reported placeholder metadata but had no cache entry
# to transfer omits peer_host/peer_port/size_bytes entirely; the consumer # to transfer omits peer_host/peer_port/size_bytes entirely; the consumer
@@ -250,7 +344,7 @@ def test_metadata_only_entry_admits_without_transfer(monkeypatch):
s.shutdown() s.shutdown()
def test_orphan_not_ready_entry_falls_back_no_realloc(monkeypatch): def test_orphan_not_ready_entry_defers_no_realloc(monkeypatch):
# Post-quarantine, post-tombstone-consumed orphan: a not-ready cache # Post-quarantine, post-tombstone-consumed orphan: a not-ready cache
# entry exists for the mm_hash, but the hash is tracked in none of # entry exists for the mm_hash, but the hash is tracked in none of
# _in_flight / _step_completed / _tombstones. Its blocks are held by a # _in_flight / _step_completed / _tombstones. Its blocks are held by a
@@ -274,25 +368,13 @@ def test_orphan_not_ready_entry_falls_back_no_realloc(monkeypatch):
monkeypatch.setattr(s._cache, "alloc", _spy_alloc) monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1)) req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# Must not raise; request falls back to local compute (admitted True), # Must not raise; the request is deferred while the DMA settles, the
# the orphan entry is untouched, and alloc is never called again. # orphan entry is untouched, and alloc is never called again.
assert s.ensure_cache_available(req, 0) is True assert s.ensure_cache_available(req, 0) is False
assert calls == [] assert calls == []
assert s._cache.get("h1") is entry assert s._cache.get("h1") is entry
assert not entry.ready assert not entry.ready
s.shutdown() assert s.take_unavailable_requests() == set()
def test_alloc_failure_falls_back_to_local(monkeypatch):
s = _consumer_sched(monkeypatch)
# Force the cache to reject the allocation.
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# _start_xfer returns False -> request admitted for local recompute.
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
s.shutdown() s.shutdown()
@@ -63,6 +63,7 @@ def test_request_finished_producer_emits_params(monkeypatch):
delay, params = s.request_finished(_Request([_Feature("h1", length=2)])) delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False assert delay is False
assert entry.evictable
assert params == { assert params == {
"h1": { "h1": {
"metadata": {}, "metadata": {},
+5 -3
View File
@@ -451,14 +451,16 @@ def test_consumer_session_poll_not_ready_nack_goes_to_retryable():
assert not results.completed assert not results.completed
def test_consumer_session_take_results_clears_state(): def test_consumer_session_ack_timeout_reports_retry_once():
"""A delayed producer ACK is retryable, and its result is drained once."""
s = _make_consumer_session() s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() - 1) s.start_xfer("h1", [0, 1], deadline=time.monotonic() - 1)
s.poll([], time.monotonic()) s.poll([], time.monotonic())
r1 = s.take_results() r1 = s.take_results()
assert "h1" in r1.tombstoned assert "h1" in r1.retryable
assert not r1.tombstoned
r2 = s.take_results() r2 = s.take_results()
assert not r2.tombstoned # cleared after first take assert not r2.retryable # cleared after first take
def test_consumer_session_on_peer_down_cancels_waiting_ack(): def test_consumer_session_on_peer_down_cancels_waiting_ack():
@@ -104,6 +104,10 @@ class ECCPUConnector(ECConnectorBase):
request, num_computed_tokens request, num_computed_tokens
) )
def take_unavailable_requests(self) -> set[str]:
assert self.connector_scheduler is not None
return self.connector_scheduler.take_unavailable_requests()
def update_state_after_alloc(self, request: "Request", index: int) -> None: def update_state_after_alloc(self, request: "Request", index: int) -> None:
assert self.connector_scheduler is not None assert self.connector_scheduler is not None
self.connector_scheduler.update_state_after_alloc(request, index) self.connector_scheduler.update_state_after_alloc(request, index)
@@ -7,6 +7,7 @@ Owns the mmap region and the embedding cache, and handles the producer
for the ECCPUConnector. for the ECCPUConnector.
""" """
import time
from collections.abc import Collection from collections.abc import Collection
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
@@ -29,12 +30,20 @@ from vllm.logger import init_logger
if TYPE_CHECKING: if TYPE_CHECKING:
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ECConnectorOutput from vllm.v1.outputs import ECConnectorOutput
from vllm.v1.request import Request from vllm.v1.request import Request
logger = init_logger(__name__) logger = init_logger(__name__)
# Total remote wait budget per request item, including all retries.
_ADMIT_DEFER_TIMEOUT_S = 60.0
class _RemoteUnavailable(Exception):
"""An item cannot be fetched within its request's remote wait budget."""
class ECCPUScheduler: class ECCPUScheduler:
"""Scheduler delegate for the ECCPUConnector.""" """Scheduler delegate for the ECCPUConnector."""
@@ -92,6 +101,11 @@ class ECCPUScheduler:
self._in_flight: set[str] = set() self._in_flight: set[str] = set()
self._tombstones: set[str] = set() self._tombstones: set[str] = set()
self._step_completed: set[str] = set() self._step_completed: set[str] = set()
# Per-request clocks survive retries and are cleared on completion.
self._deferred_since: dict[tuple[str, str], float] = {}
# Requests needing a remote encoding that will not arrive. Drained by
# take_unavailable_requests(); the scheduler aborts them.
self._unrecoverable: set[str] = set()
self._peer_host: str | None = None self._peer_host: str | None = None
self._peer_port: int | None = None self._peer_port: int | None = None
# Model shape for size checks + compat hash; only set by # Model shape for size checks + compat hash; only set by
@@ -117,6 +131,7 @@ class ECCPUScheduler:
compute_ec_compatibility_hash, compute_ec_compatibility_hash,
) )
from vllm.distributed.ec_transfer.ec_connector.cpu.session import ( from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
PRODUCER_PIN_LEASE_S,
ProducerSession, ProducerSession,
) )
from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config from vllm.distributed.nixl_utils import NixlWrapper, nixl_agent_config
@@ -136,7 +151,9 @@ class ECCPUScheduler:
# its scheduler step, so its reply latency scales with the encoder's # its scheduler step, so its reply latency scales with the encoder's
# batch size: a deployment whose steps run longer must raise this. # batch size: a deployment whose steps run longer must raise this.
self._ack_timeout_s = float( self._ack_timeout_s = float(
self._ec_config.get_from_extra_config("consumer_ack_timeout_s", 2.0) self._ec_config.get_from_extra_config(
"consumer_ack_timeout_s", PRODUCER_PIN_LEASE_S
)
) )
self._compat_hash = compute_ec_compatibility_hash( self._compat_hash = compute_ec_compatibility_hash(
vllm_version=VLLM_VERSION, vllm_version=VLLM_VERSION,
@@ -203,79 +220,172 @@ class ECCPUScheduler:
def _nixl_consumer_admit( def _nixl_consumer_admit(
self, request: "Request", num_computed_tokens: int self, request: "Request", num_computed_tokens: int
) -> bool: ) -> bool:
"""Admit a request only once all its remote encodings are cached. """Admit a request once its required remote encodings are ready.
Returns True when every encoding the request needs is either already Each item has a per-request wait budget that survives transfer retries.
ready in the local cache or has no remote source. Any encoding whose Unavailable items fall back to local input; without it, the request is
NIXL READ is still in flight (or was just started this step) defers reported by take_unavailable_requests() for the scheduler to abort.
the request, which the scheduler re-presents on a later step.
Returns:
True if no item needs to wait for a remote encoding. False if any
item is pending or the request must be aborted.
""" """
params: dict[str, dict[str, Any]] = ( if not request.ec_transfer_params:
getattr(request, "ec_transfer_params", None) or {}
)
if not params:
return True return True
now = time.monotonic()
pending = False pending = False
for feature in request.mm_features: for feature in request.mm_features:
pos = feature.mm_position
if pos.offset + pos.length <= num_computed_tokens:
continue
mm_hash = feature.identifier
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
# Local hit: upstream's update_state_after_alloc pins and
# loads it through the same path as a natively cached entry.
continue
if mm_hash in self._in_flight:
pending = True
continue
if mm_hash in self._step_completed:
pending = True
continue
if mm_hash in self._tombstones:
self._tombstones.discard(mm_hash)
continue
if entry is not None:
# Present but not ready and not being fetched: its blocks are
# held by a quarantined/settling DMA and cannot be reused.
# Fall back to local compute this step; never re-alloc a
# mm_hash already in the cache.
continue
info = params.get(mm_hash)
# Absent entirely, or present with only placeholder metadata (no
# cache entry existed on the producer side at request_finished
# time): nothing to fetch, fall back to local compute.
if not info or "peer_host" not in info:
continue
expected = pos.length * self._hidden_dim * self._element_size
if int(info.get("size_bytes", -1)) != expected:
logger.warning(
"EC consumer: size mismatch mm_hash=%s; local encode", mm_hash
)
continue
try: try:
started = self._start_xfer(mm_hash, info, expected) if not self._admit_item(request, feature, num_computed_tokens, now):
except Exception: pending = True
logger.exception( except _RemoteUnavailable as error:
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash if not self._fail_or_fallback(request, feature, str(error)):
) return False
continue
if not started:
continue
self._in_flight.add(mm_hash)
pending = True
return not pending return not pending
def _admit_item(
self,
request: "Request",
feature: "MultiModalFeatureSpec",
num_computed_tokens: int,
now: float,
) -> bool:
"""Check one encoding, starting a remote read if needed.
Returns:
True if already computed, cached, or not remotely sourced. False
while waiting for a read or cache space, including across retries.
Raises:
_RemoteUnavailable: The remote source is invalid, the read failed,
or the item's total wait budget expired.
"""
pos = feature.mm_position
mm_hash = feature.identifier
if pos.offset + pos.length <= num_computed_tokens:
self._deferred_since.pop((request.request_id, mm_hash), None)
return True
announced = (request.ec_transfer_params or {}).get(mm_hash)
# Without a producer address there is nothing to fetch: the request
# carries what the model needs and the encoder runs locally.
# `ec_transfer_params` reaches us from the request, so its shape is
# checked rather than assumed.
remote: dict[str, Any] | None = (
announced
if isinstance(announced, dict) and "peer_host" in announced
else None
)
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
# Local hit: upstream's update_state_after_alloc pins and
# loads it through the same path as a natively cached entry.
self._deferred_since.pop((request.request_id, mm_hash), None)
return True
if remote is None:
return True
since = self._deferred_since.setdefault((request.request_id, mm_hash), now)
if now - since > _ADMIT_DEFER_TIMEOUT_S:
raise _RemoteUnavailable(f"the remote wait exceeded {now - since:.0f}s")
if mm_hash in self._in_flight or mm_hash in self._step_completed:
return False
if mm_hash in self._tombstones:
self._tombstones.discard(mm_hash)
raise _RemoteUnavailable("the remote read failed")
if entry is not None:
# Present but not ready and not being fetched: its blocks are
# held by a quarantined/settling DMA and cannot be reused.
return False
expected = pos.length * self._hidden_dim * self._element_size
try:
size = int(remote["size_bytes"])
except (KeyError, TypeError, ValueError) as error:
raise _RemoteUnavailable("the announced size was unusable") from error
if size != expected:
logger.warning(
"EC consumer: size mismatch mm_hash=%s announced=%d expected=%d",
mm_hash,
size,
expected,
)
raise _RemoteUnavailable("the announced size was wrong")
try:
started = self._start_xfer(mm_hash, remote, expected)
except Exception as error:
logger.exception(
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
)
raise _RemoteUnavailable("the read could not be started") from error
if started:
self._in_flight.add(mm_hash)
return False
def _fail_or_fallback(
self, request: "Request", feature: "MultiModalFeatureSpec", why: str
) -> bool:
"""Resolve an unavailable remote encoding and clear its wait budget.
Returns:
True if local media or embeddings are available, removing this
item's remote announcement so subsequent steps do not retry it.
False if only placeholder metadata (or no input) is available,
recording the request for take_unavailable_requests() to drain.
"""
mm_hash = feature.identifier
self._deferred_since.pop((request.request_id, mm_hash), None)
data = feature.data
metadata = (
self._metadata_resolver.fields_for(feature.modality) if data else set()
)
if data and any(
value.data is not None for key, value in data.items() if key not in metadata
):
# Passthrough embeddings bypass the processor cache. SHM address
# items therefore refer to media that the worker can encode.
request.ec_transfer_params = dict(request.ec_transfer_params or {})
request.ec_transfer_params.pop(mm_hash, None)
logger.warning(
"EC consumer: request %s mm_hash=%s: %s; using local input",
request.request_id,
mm_hash,
why,
)
return True
self._unrecoverable.add(request.request_id)
logger.error(
"EC consumer: request %s needs remote encoding mm_hash=%s but %s. "
"No local model input is available; failing the request.",
request.request_id,
mm_hash,
why,
)
return False
def take_unavailable_requests(self) -> set[str]:
"""Return and clear IDs of requests that cannot obtain their encodings.
The scheduler must abort these requests rather than leave them waiting.
Requests that can fall back to local input are not included, and each
recorded failure is returned only once.
"""
if not self._unrecoverable:
return set()
failed = self._unrecoverable
self._unrecoverable = set()
return failed
def _start_xfer( def _start_xfer(
self, mm_hash: str, info: "dict[str, Any]", size_bytes: int self, mm_hash: str, info: "dict[str, Any]", size_bytes: int
) -> bool: ) -> bool:
"""Allocate a not-ready cache entry and start a NIXL READ into it. """Allocate a not-ready cache entry and start a NIXL READ into it.
Returns True if the transfer was started. Returns False when the Returns True if the transfer was started. Returns False when the
cache cannot accommodate the encoding, in which case the request cache cannot accommodate the encoding; admission will retry it.
falls back to local recomputation.
""" """
import time
from math import ceil from math import ceil
from vllm.distributed.ec_transfer.ec_connector.cpu.session import ( from vllm.distributed.ec_transfer.ec_connector.cpu.session import (
@@ -286,7 +396,7 @@ class ECCPUScheduler:
entry = self._cache.alloc(mm_hash, n_blocks) entry = self._cache.alloc(mm_hash, n_blocks)
if entry is None: if entry is None:
logger.debug( logger.debug(
"EC consumer: cache full for mm_hash=%s (%d blocks); local encode", "EC consumer: cache full for mm_hash=%s (%d blocks); deferring",
mm_hash, mm_hash,
n_blocks, n_blocks,
) )
@@ -319,8 +429,6 @@ class ECCPUScheduler:
return True return True
def _poll_step(self) -> None: def _poll_step(self) -> None:
import time
now = time.monotonic() now = time.monotonic()
all_messages = self._transport.poll() all_messages = self._transport.poll()
for addr, session in list(self._sessions.items()): for addr, session in list(self._sessions.items()):
@@ -352,8 +460,7 @@ class ECCPUScheduler:
mm_hash, mm_hash,
) )
for mm_hash in r.retryable: for mm_hash in r.retryable:
# No tombstone: dropping every trace of the attempt is what lets # Release this attempt's blocks, retaining the request wait budget.
# the next admit pass re-issue the read as if it were the first.
self._in_flight.discard(mm_hash) self._in_flight.discard(mm_hash)
self._cache.discard(mm_hash) self._cache.discard(mm_hash)
logger.debug( logger.debug(
@@ -472,6 +579,8 @@ class ECCPUScheduler:
def request_finished( def request_finished(
self, request: "Request" self, request: "Request"
) -> tuple[bool, "dict[str, Any] | None"]: ) -> tuple[bool, "dict[str, Any] | None"]:
for feature in request.mm_features:
self._deferred_since.pop((request.request_id, feature.identifier), None)
if not (self._nixl_enabled and self._is_producer): if not (self._nixl_enabled and self._is_producer):
return False, None return False, None
if not request.mm_features: if not request.mm_features:
@@ -520,6 +629,7 @@ class ECCPUScheduler:
self._pending_saves.clear() self._pending_saves.clear()
self._pending_loads.clear() self._pending_loads.clear()
self._load_acks.clear() self._load_acks.clear()
self._deferred_since.clear()
self._is_producer = False self._is_producer = False
self._is_consumer = False self._is_consumer = False
@@ -71,7 +71,7 @@ class XferState(enum.Enum):
WAITING_ACK = "waiting_ack" # XferReq sent, awaiting XferAck WAITING_ACK = "waiting_ack" # XferReq sent, awaiting XferAck
READING = "reading" # NIXL READ in flight READING = "reading" # NIXL READ in flight
DONE = "done" # READ completed; promote to local cache DONE = "done" # READ completed; promote to local cache
ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + tombstone ACK_TIMEOUT = "ack_timeout" # XferAck never arrived; free blocks + retry
READ_FAILED = "read_failed" # Unexpected NIXL state; free blocks + tombstone READ_FAILED = "read_failed" # Unexpected NIXL state; free blocks + tombstone
QUARANTINED = "quarantined" # Read timed out, NIXL unabortable; keep blocks QUARANTINED = "quarantined" # Read timed out, NIXL unabortable; keep blocks
SETTLED = "settled" # Quarantined DMA done; safe to free blocks SETTLED = "settled" # Quarantined DMA done; safe to free blocks
@@ -500,7 +500,18 @@ class ConsumerSession:
if state == XferState.DONE: if state == XferState.DONE:
self._completed.add(mm_hash) self._completed.add(mm_hash)
del self._xfers[mm_hash] del self._xfers[mm_hash]
elif state in (XferState.ACK_TIMEOUT, XferState.READ_FAILED): elif state == XferState.ACK_TIMEOUT:
# The scheduler bounds retries with a per-request deadline.
logger.warning(
"EC consumer: no XferAck for mm_hash=%s from %s:%d in time; "
"will re-request",
mm_hash,
self._addr[0],
self._addr[1],
)
self._retryable.add(mm_hash)
del self._xfers[mm_hash]
elif state == XferState.READ_FAILED:
self._tombstoned.add(mm_hash) self._tombstoned.add(mm_hash)
del self._xfers[mm_hash] del self._xfers[mm_hash]
elif state == XferState.QUARANTINED: elif state == XferState.QUARANTINED: