[Bugfix][EC Connector] Fail the request, not the engine, when a remote encoding cannot arrive (#55290)
Signed-off-by: Tianyu Guo <guoty@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
050f29caf1
commit
f1aeff6007
@@ -58,6 +58,26 @@ EC_SHARED_STORAGE_PATH="/tmp/my_ec_cache" bash ./tests/v1/ec_connector/integrati
|
||||
|
||||
## How It Works
|
||||
|
||||
### NIXL EC failure isolation (1E + 1PD)
|
||||
|
||||
```bash
|
||||
PATH="$PWD/.venv/bin:$PATH" CUDA_VISIBLE_DEVICES=0,1 \
|
||||
.venv/bin/python -m pytest \
|
||||
tests/v1/ec_connector/integration/test_nixl_failure.py -v -s
|
||||
```
|
||||
|
||||
Requires two CUDA GPUs and NIXL. `MODEL` overrides the default
|
||||
`Qwen/Qwen2.5-VL-3B-Instruct`. The test starts real E/PD servers with CUDA
|
||||
graphs enabled and uses the proxy's rewrite helper. Only the failing request's
|
||||
control endpoint is replaced: a ZMQ peer returns `NACK_MISSING` and verifies
|
||||
that the consumer actually requested the encoding.
|
||||
|
||||
Metadata-only requests must return a request-level error; requests retaining
|
||||
the image must succeed through local fallback. After each case, a fresh
|
||||
metadata-only image request must succeed through the real NIXL transfer path.
|
||||
No server restart, proxy retry, cache hit, or timeout race can mask the result.
|
||||
This is a correctness test, not a throughput benchmark.
|
||||
|
||||
### Mooncake EC (1E + 1PD)
|
||||
|
||||
```bash
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NIXL request failure isolation with real E/PD servers and an injected NACK.
|
||||
|
||||
Requires two CUDA GPUs and NIXL. Set MODEL to override the small default model.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import aiohttp
|
||||
import msgspec
|
||||
import pytest
|
||||
import zmq
|
||||
import zmq.asyncio
|
||||
from PIL import Image
|
||||
|
||||
from examples.disaggregated.disaggregated_encoder.disagg_epd_proxy import (
|
||||
rewrite_for_decode,
|
||||
)
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.protocol import (
|
||||
XferAck,
|
||||
XferReq,
|
||||
XferStatus,
|
||||
)
|
||||
from vllm.multimodal.utils import encode_image_url
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
MODEL = os.environ.get("MODEL", "Qwen/Qwen2.5-VL-3B-Instruct")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def servers():
|
||||
pytest.importorskip("nixl")
|
||||
if not current_platform.is_cuda() or current_platform.device_count() < 2:
|
||||
pytest.skip("Requires two CUDA GPUs")
|
||||
devices = os.environ.get("CUDA_VISIBLE_DEVICES", "0,1").split(",")
|
||||
common = [
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"4096",
|
||||
"--max-num-seqs",
|
||||
"4",
|
||||
"--gpu-memory-utilization",
|
||||
"0.7",
|
||||
"--no-enable-prefix-caching",
|
||||
]
|
||||
|
||||
def config(role):
|
||||
return json.dumps(
|
||||
{
|
||||
"ec_connector": "ECCPUConnector",
|
||||
"ec_role": role,
|
||||
"ec_connector_extra_config": {
|
||||
"ec_enable_nixl": True,
|
||||
"ec_cpu_bytes": 64 << 20,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
with (
|
||||
RemoteOpenAIServer(
|
||||
MODEL,
|
||||
common
|
||||
+ [
|
||||
"--mm-encoder-only",
|
||||
"--disable-hybrid-kv-cache-manager",
|
||||
"--ec-transfer-config",
|
||||
config("ec_producer"),
|
||||
],
|
||||
env_dict={
|
||||
"CUDA_VISIBLE_DEVICES": devices[0],
|
||||
"VLLM_EC_SIDE_CHANNEL_HOST": "127.0.0.1",
|
||||
"VLLM_EC_SIDE_CHANNEL_PORT": str(get_open_port()),
|
||||
},
|
||||
) as encoder,
|
||||
RemoteOpenAIServer(
|
||||
MODEL,
|
||||
common
|
||||
+ ["--enable-mm-embeds", "--ec-transfer-config", config("ec_consumer")],
|
||||
env_dict={"CUDA_VISIBLE_DEVICES": devices[1]},
|
||||
) as consumer,
|
||||
):
|
||||
yield encoder, consumer
|
||||
|
||||
|
||||
async def _post(session, server, body):
|
||||
async with session.post(server.url_for("v1/chat/completions"), json=body) as r:
|
||||
return r.status, await r.json()
|
||||
|
||||
|
||||
async def _encode(session, encoder):
|
||||
image = Image.open(Path(__file__).with_name("hato.jpg")).convert("RGB")
|
||||
image.thumbnail((224, 224))
|
||||
item_uuid = uuid.uuid4().hex # Avoid both EC and processor cache hits.
|
||||
body = {
|
||||
"model": MODEL,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": encode_image_url(image)},
|
||||
"uuid": item_uuid,
|
||||
},
|
||||
{"type": "text", "text": "Describe the image briefly."},
|
||||
],
|
||||
}
|
||||
],
|
||||
"max_tokens": 16,
|
||||
"temperature": 0,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
status, result = await _post(session, encoder, body)
|
||||
assert status == 200, result
|
||||
params = result["ec_transfer_params"]
|
||||
assert len(params) == 1, params
|
||||
((mm_hash, handle),) = params.items()
|
||||
assert handle["metadata"] and handle["peer_host"] and handle["size_bytes"] > 0
|
||||
rewritten = rewrite_for_decode(
|
||||
body,
|
||||
{
|
||||
0: {
|
||||
**handle["metadata"],
|
||||
"mm_hash": item_uuid,
|
||||
"ec_mm_hash": mm_hash,
|
||||
}
|
||||
},
|
||||
)
|
||||
assert rewritten["messages"][0]["content"][0]["type"] == "image_embeds"
|
||||
for request in (body, rewritten):
|
||||
request["ec_transfer_params"] = {item_uuid: copy.deepcopy(handle)}
|
||||
return body, rewritten, item_uuid
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("local_image", [False, True], ids=["metadata-only", "pixels"])
|
||||
async def test_missing_remote_encoding_is_request_scoped(servers, local_image):
|
||||
"""A terminal NACK must preserve local fallback and subsequent serving."""
|
||||
encoder, consumer = servers
|
||||
timeout = aiohttp.ClientTimeout(total=120)
|
||||
async with aiohttp.ClientSession(timeout=timeout) as session:
|
||||
original, rewritten, mm_hash = await _encode(session, encoder)
|
||||
request = original if local_image else rewritten
|
||||
|
||||
# Keep the real encoder's shape/metadata; replace only the control peer.
|
||||
# The socket stays bound until completion, so no port or timing race is
|
||||
# needed to make the consumer receive NACK_MISSING.
|
||||
with zmq.asyncio.Context() as ctx, ctx.socket(zmq.ROUTER) as router:
|
||||
router.setsockopt(zmq.LINGER, 0)
|
||||
port = router.bind_to_random_port("tcp://127.0.0.1")
|
||||
request["ec_transfer_params"][mm_hash].update(
|
||||
peer_host="127.0.0.1", peer_port=port
|
||||
)
|
||||
|
||||
async def reject_read():
|
||||
identity, delimiter, payload = await router.recv_multipart()
|
||||
req = msgspec.msgpack.decode(payload, type=XferReq)
|
||||
assert req.mm_hash == mm_hash
|
||||
ack = XferAck(
|
||||
mm_hash=req.mm_hash,
|
||||
status=XferStatus.NACK_MISSING,
|
||||
session_id=req.session_id,
|
||||
)
|
||||
await router.send_multipart(
|
||||
[identity, delimiter, msgspec.msgpack.encode(ack)]
|
||||
)
|
||||
|
||||
rejection = asyncio.create_task(asyncio.wait_for(reject_read(), 30))
|
||||
try:
|
||||
status, result = await _post(session, consumer, request)
|
||||
# A frontend rejection must not count as successful injection.
|
||||
await rejection
|
||||
finally:
|
||||
rejection.cancel()
|
||||
await asyncio.gather(rejection, return_exceptions=True)
|
||||
|
||||
if local_image:
|
||||
assert status == 200, result
|
||||
assert result["choices"][0]["message"]["content"], result
|
||||
else:
|
||||
assert status == 500, result
|
||||
assert result["error"]["type"] == "InternalServerError", result
|
||||
|
||||
async with session.get(consumer.url_for("health")) as health:
|
||||
assert health.status == 200, "The failed read killed EngineCore"
|
||||
# A fresh metadata-only image must cross the real NIXL data path after
|
||||
# the fault, not just pass a frontend health check or hit a local cache.
|
||||
_, healthy, _ = await _encode(session, encoder)
|
||||
status, result = await _post(session, consumer, healthy)
|
||||
assert status == 200, result
|
||||
assert result["choices"][0]["message"]["content"], result
|
||||
@@ -1,13 +1,22 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalSharedField,
|
||||
)
|
||||
|
||||
_N, _BS, _HID, _ES = 16, 64, 32, 2
|
||||
|
||||
@@ -18,10 +27,12 @@ class _Pos:
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1):
|
||||
def __init__(self, mm_hash, length=1, data=None):
|
||||
self.mm_hash = mm_hash
|
||||
self.identifier = mm_hash
|
||||
self.mm_position = _Pos(0, length)
|
||||
self.data = data
|
||||
self.modality = "image"
|
||||
|
||||
|
||||
class _Request:
|
||||
@@ -94,6 +105,7 @@ def _consumer_sched(monkeypatch):
|
||||
# directly since this helper builds gate-off then flips fields on.
|
||||
s._hidden_dim = _HID
|
||||
s._element_size = _ES
|
||||
s._metadata_resolver._cache["image"] = {"image_grid_thw"}
|
||||
return s
|
||||
|
||||
|
||||
@@ -150,13 +162,17 @@ def test_new_remote_read_defers_then_completes(monkeypatch):
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_no_params_is_ready(monkeypatch):
|
||||
@pytest.mark.parametrize("params", [None, {"x": 123}])
|
||||
def test_no_remote_announcement_is_ready(monkeypatch, params):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
assert s.ensure_cache_available(_Request([_Feature("x")], params=None), 0) is True
|
||||
assert s.ensure_cache_available(_Request([_Feature("x")], params=params), 0)
|
||||
assert s.take_unavailable_requests() == set()
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_tombstoned_read_discards_and_blocks_retry(monkeypatch):
|
||||
@pytest.mark.parametrize("payload", [None, "metadata", "pixels", "embeds", "shm"])
|
||||
def test_failed_read_falls_back_only_with_local_input(monkeypatch, payload):
|
||||
"""A remote miss is fatal only when local model input is unavailable."""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
|
||||
@@ -167,26 +183,71 @@ def test_tombstoned_read_discards_and_blocks_retry(monkeypatch):
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", _fake_start)
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
fields = {
|
||||
"metadata": {"image_grid_thw": torch.tensor([1, 2, 2])},
|
||||
"pixels": {"pixel_values": torch.ones(1, 3)},
|
||||
"embeds": {"image_embeds": torch.ones(1, _HID)},
|
||||
"shm": {"address": 123, "monotonic_id": 1},
|
||||
}
|
||||
data = (
|
||||
MultiModalKwargsItem(
|
||||
{
|
||||
key: MultiModalFieldElem(value, MultiModalSharedField(batch_size=1))
|
||||
for key, value in fields[payload].items()
|
||||
}
|
||||
)
|
||||
if payload is not None
|
||||
else None
|
||||
)
|
||||
req = _Request([_Feature("h1", 1, data)], params=_params("h1", 1))
|
||||
|
||||
# Step 1: start the read.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# Producer rejected the read -> tombstoned.
|
||||
fake._results.tombstoned.add("h1")
|
||||
s._sessions[("h", 1)] = fake
|
||||
|
||||
# Step 2: poll discards the entry and records a tombstone; the tombstone
|
||||
# is consumed by admit so the request proceeds to local compute.
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
can_fallback = payload in ("pixels", "embeds", "shm")
|
||||
assert s.ensure_cache_available(req, 0) is can_fallback
|
||||
assert s._cache.get("h1") is None
|
||||
assert "h1" not in s._in_flight
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.take_unavailable_requests() == (set() if can_fallback else {"r1"})
|
||||
# Drained by the read, so the scheduler cannot abort it twice.
|
||||
assert s.take_unavailable_requests() == set()
|
||||
if can_fallback:
|
||||
# A later scheduling step must not retry the failed remote source.
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.ensure_cache_available(req, 0)
|
||||
assert fake.started == ["h1"]
|
||||
s.shutdown()
|
||||
|
||||
# Step 3: tombstone was consumed, so a fresh transfer starts again.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert "h1" in s._in_flight
|
||||
|
||||
@pytest.mark.parametrize("retry", [False, True], ids=["in-flight", "retryable"])
|
||||
def test_remote_wait_budget_survives_retries_and_long_steps(monkeypatch, retry):
|
||||
"""Neither retries nor delayed scheduling may restart the request budget."""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
fake = _FakeSession()
|
||||
s._sessions[("h", 1)] = fake
|
||||
|
||||
def start(mm_hash, info, size):
|
||||
assert s._cache.alloc(mm_hash, 1) is not None
|
||||
fake.started.append(mm_hash)
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(s, "_start_xfer", start)
|
||||
now = [1000.0]
|
||||
monkeypatch.setattr(time, "monotonic", lambda: now[0])
|
||||
req = _Request([_Feature("h1")], params=_params("h1", 1))
|
||||
assert not s.ensure_cache_available(req, 0)
|
||||
for elapsed in (20, 40, 121):
|
||||
now[0] = 1000.0 + elapsed
|
||||
if retry:
|
||||
fake._results.retryable.add("h1")
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert not s.ensure_cache_available(req, 0)
|
||||
assert s.take_unavailable_requests() == ({"r1"} if elapsed > 60 else set())
|
||||
assert len(fake.started) == (3 if retry else 1)
|
||||
s.shutdown()
|
||||
|
||||
|
||||
@@ -225,17 +286,50 @@ def test_retryable_read_re_requests_without_admitting(monkeypatch):
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_size_mismatch_skips_transfer(monkeypatch):
|
||||
@pytest.mark.parametrize(
|
||||
"size_fields",
|
||||
[{"size_bytes": 999}, {"size_bytes": None}, {"size_bytes": "big"}, {}],
|
||||
ids=["mismatch", "null-size", "text-size", "no-size"],
|
||||
)
|
||||
def test_invalid_remote_size_fails_the_request(monkeypatch, size_fields):
|
||||
"""Invalid remote sizes fail the request without raising in the scheduler."""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
# Advertised size disagrees with pos.length * hidden_dim * element_size.
|
||||
bad = {"h1": {"peer_host": "h", "peer_port": 1, "size_bytes": 999}}
|
||||
req = _Request([_Feature("h1", 1)], params=bad)
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
announced = {"peer_host": "h", "peer_port": 1, **size_fields}
|
||||
req = _Request([_Feature("h1", 1)], params={"h1": announced})
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert s.take_unavailable_requests() == {"r1"}
|
||||
assert "h1" not in s._in_flight
|
||||
assert s._cache.get("h1") is None
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_deferral_budget_is_per_request(monkeypatch):
|
||||
"""Shared hashes have independent wait budgets and cancellation cleanup."""
|
||||
s = _consumer_sched(monkeypatch)
|
||||
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
|
||||
params = _params("h1", 1)
|
||||
old = _Request([_Feature("h1", 1)], params=params, req_id="old")
|
||||
new = _Request([_Feature("h1", 1)], params=params, req_id="new")
|
||||
|
||||
now = [1000.0]
|
||||
monkeypatch.setattr(time, "monotonic", lambda: now[0])
|
||||
assert s.ensure_cache_available(old, 0) is False
|
||||
now[0] += sched_mod._ADMIT_DEFER_TIMEOUT_S - 1
|
||||
assert s.ensure_cache_available(new, 0) is False
|
||||
assert s.take_unavailable_requests() == set()
|
||||
|
||||
now[0] += 2
|
||||
# `old` is now past its budget; `new` is not.
|
||||
assert s.ensure_cache_available(new, 0) is False
|
||||
assert s.take_unavailable_requests() == set()
|
||||
assert s.request_finished(new) == (False, None)
|
||||
assert set(s._deferred_since) == {("old", "h1")}
|
||||
assert s.ensure_cache_available(old, 0) is False
|
||||
assert s.take_unavailable_requests() == {"old"}
|
||||
assert "h1" not in s._in_flight
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_metadata_only_entry_admits_without_transfer(monkeypatch):
|
||||
# A producer that reported placeholder metadata but had no cache entry
|
||||
# to transfer omits peer_host/peer_port/size_bytes entirely; the consumer
|
||||
@@ -250,7 +344,7 @@ def test_metadata_only_entry_admits_without_transfer(monkeypatch):
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_orphan_not_ready_entry_falls_back_no_realloc(monkeypatch):
|
||||
def test_orphan_not_ready_entry_defers_no_realloc(monkeypatch):
|
||||
# Post-quarantine, post-tombstone-consumed orphan: a not-ready cache
|
||||
# entry exists for the mm_hash, but the hash is tracked in none of
|
||||
# _in_flight / _step_completed / _tombstones. Its blocks are held by a
|
||||
@@ -274,25 +368,13 @@ def test_orphan_not_ready_entry_falls_back_no_realloc(monkeypatch):
|
||||
monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
# Must not raise; request falls back to local compute (admitted True),
|
||||
# the orphan entry is untouched, and alloc is never called again.
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
# Must not raise; the request is deferred while the DMA settles, the
|
||||
# orphan entry is untouched, and alloc is never called again.
|
||||
assert s.ensure_cache_available(req, 0) is False
|
||||
assert calls == []
|
||||
assert s._cache.get("h1") is entry
|
||||
assert not entry.ready
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_alloc_failure_falls_back_to_local(monkeypatch):
|
||||
s = _consumer_sched(monkeypatch)
|
||||
|
||||
# Force the cache to reject the allocation.
|
||||
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
|
||||
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
|
||||
|
||||
# _start_xfer returns False -> request admitted for local recompute.
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
assert "h1" not in s._in_flight
|
||||
assert s.take_unavailable_requests() == set()
|
||||
s.shutdown()
|
||||
|
||||
|
||||
|
||||
@@ -63,6 +63,7 @@ def test_request_finished_producer_emits_params(monkeypatch):
|
||||
|
||||
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
|
||||
assert delay is False
|
||||
assert entry.evictable
|
||||
assert params == {
|
||||
"h1": {
|
||||
"metadata": {},
|
||||
|
||||
@@ -451,14 +451,16 @@ def test_consumer_session_poll_not_ready_nack_goes_to_retryable():
|
||||
assert not results.completed
|
||||
|
||||
|
||||
def test_consumer_session_take_results_clears_state():
|
||||
def test_consumer_session_ack_timeout_reports_retry_once():
|
||||
"""A delayed producer ACK is retryable, and its result is drained once."""
|
||||
s = _make_consumer_session()
|
||||
s.start_xfer("h1", [0, 1], deadline=time.monotonic() - 1)
|
||||
s.poll([], time.monotonic())
|
||||
r1 = s.take_results()
|
||||
assert "h1" in r1.tombstoned
|
||||
assert "h1" in r1.retryable
|
||||
assert not r1.tombstoned
|
||||
r2 = s.take_results()
|
||||
assert not r2.tombstoned # cleared after first take
|
||||
assert not r2.retryable # cleared after first take
|
||||
|
||||
|
||||
def test_consumer_session_on_peer_down_cancels_waiting_ack():
|
||||
|
||||
Reference in New Issue
Block a user