[Bugfix][EC Connector] Fail the request, not the engine, when a remote encoding cannot arrive (#55290)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Tianyu Guo
2026-09-08 20:13:33 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 050f29caf1
commit f1aeff6007
8 changed files with 536 additions and 104 deletions
@@ -1,13 +1,22 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
import uuid
import pytest
import torch
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
from tests.v1.ec_connector.unit.utils import create_ec_vllm_config
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
ECSharedRegion,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
from vllm.multimodal.inputs import (
MultiModalFieldElem,
MultiModalKwargsItem,
MultiModalSharedField,
)
_N, _BS, _HID, _ES = 16, 64, 32, 2
@@ -18,10 +27,12 @@ class _Pos:
class _Feature:
def __init__(self, mm_hash, length=1):
def __init__(self, mm_hash, length=1, data=None):
self.mm_hash = mm_hash
self.identifier = mm_hash
self.mm_position = _Pos(0, length)
self.data = data
self.modality = "image"
class _Request:
@@ -94,6 +105,7 @@ def _consumer_sched(monkeypatch):
# directly since this helper builds gate-off then flips fields on.
s._hidden_dim = _HID
s._element_size = _ES
s._metadata_resolver._cache["image"] = {"image_grid_thw"}
return s
@@ -150,13 +162,17 @@ def test_new_remote_read_defers_then_completes(monkeypatch):
s.shutdown()
def test_no_params_is_ready(monkeypatch):
@pytest.mark.parametrize("params", [None, {"x": 123}])
def test_no_remote_announcement_is_ready(monkeypatch, params):
s = _consumer_sched(monkeypatch)
assert s.ensure_cache_available(_Request([_Feature("x")], params=None), 0) is True
assert s.ensure_cache_available(_Request([_Feature("x")], params=params), 0)
assert s.take_unavailable_requests() == set()
s.shutdown()
def test_tombstoned_read_discards_and_blocks_retry(monkeypatch):
@pytest.mark.parametrize("payload", [None, "metadata", "pixels", "embeds", "shm"])
def test_failed_read_falls_back_only_with_local_input(monkeypatch, payload):
"""A remote miss is fatal only when local model input is unavailable."""
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
@@ -167,26 +183,71 @@ def test_tombstoned_read_discards_and_blocks_retry(monkeypatch):
return True
monkeypatch.setattr(s, "_start_xfer", _fake_start)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
fields = {
"metadata": {"image_grid_thw": torch.tensor([1, 2, 2])},
"pixels": {"pixel_values": torch.ones(1, 3)},
"embeds": {"image_embeds": torch.ones(1, _HID)},
"shm": {"address": 123, "monotonic_id": 1},
}
data = (
MultiModalKwargsItem(
{
key: MultiModalFieldElem(value, MultiModalSharedField(batch_size=1))
for key, value in fields[payload].items()
}
)
if payload is not None
else None
)
req = _Request([_Feature("h1", 1, data)], params=_params("h1", 1))
# Step 1: start the read.
assert s.ensure_cache_available(req, 0) is False
s.build_connector_meta(scheduler_output=None)
# Producer rejected the read -> tombstoned.
fake._results.tombstoned.add("h1")
s._sessions[("h", 1)] = fake
# Step 2: poll discards the entry and records a tombstone; the tombstone
# is consumed by admit so the request proceeds to local compute.
assert s.ensure_cache_available(req, 0) is True
can_fallback = payload in ("pixels", "embeds", "shm")
assert s.ensure_cache_available(req, 0) is can_fallback
assert s._cache.get("h1") is None
assert "h1" not in s._in_flight
s.build_connector_meta(scheduler_output=None)
assert s.take_unavailable_requests() == (set() if can_fallback else {"r1"})
# Drained by the read, so the scheduler cannot abort it twice.
assert s.take_unavailable_requests() == set()
if can_fallback:
# A later scheduling step must not retry the failed remote source.
s.build_connector_meta(scheduler_output=None)
assert s.ensure_cache_available(req, 0)
assert fake.started == ["h1"]
s.shutdown()
# Step 3: tombstone was consumed, so a fresh transfer starts again.
assert s.ensure_cache_available(req, 0) is False
assert "h1" in s._in_flight
@pytest.mark.parametrize("retry", [False, True], ids=["in-flight", "retryable"])
def test_remote_wait_budget_survives_retries_and_long_steps(monkeypatch, retry):
"""Neither retries nor delayed scheduling may restart the request budget."""
s = _consumer_sched(monkeypatch)
fake = _FakeSession()
s._sessions[("h", 1)] = fake
def start(mm_hash, info, size):
assert s._cache.alloc(mm_hash, 1) is not None
fake.started.append(mm_hash)
return True
monkeypatch.setattr(s, "_start_xfer", start)
now = [1000.0]
monkeypatch.setattr(time, "monotonic", lambda: now[0])
req = _Request([_Feature("h1")], params=_params("h1", 1))
assert not s.ensure_cache_available(req, 0)
for elapsed in (20, 40, 121):
now[0] = 1000.0 + elapsed
if retry:
fake._results.retryable.add("h1")
s.build_connector_meta(scheduler_output=None)
assert not s.ensure_cache_available(req, 0)
assert s.take_unavailable_requests() == ({"r1"} if elapsed > 60 else set())
assert len(fake.started) == (3 if retry else 1)
s.shutdown()
@@ -225,17 +286,50 @@ def test_retryable_read_re_requests_without_admitting(monkeypatch):
s.shutdown()
def test_size_mismatch_skips_transfer(monkeypatch):
@pytest.mark.parametrize(
"size_fields",
[{"size_bytes": 999}, {"size_bytes": None}, {"size_bytes": "big"}, {}],
ids=["mismatch", "null-size", "text-size", "no-size"],
)
def test_invalid_remote_size_fails_the_request(monkeypatch, size_fields):
"""Invalid remote sizes fail the request without raising in the scheduler."""
s = _consumer_sched(monkeypatch)
# Advertised size disagrees with pos.length * hidden_dim * element_size.
bad = {"h1": {"peer_host": "h", "peer_port": 1, "size_bytes": 999}}
req = _Request([_Feature("h1", 1)], params=bad)
assert s.ensure_cache_available(req, 0) is True
announced = {"peer_host": "h", "peer_port": 1, **size_fields}
req = _Request([_Feature("h1", 1)], params={"h1": announced})
assert s.ensure_cache_available(req, 0) is False
assert s.take_unavailable_requests() == {"r1"}
assert "h1" not in s._in_flight
assert s._cache.get("h1") is None
s.shutdown()
def test_deferral_budget_is_per_request(monkeypatch):
"""Shared hashes have independent wait budgets and cancellation cleanup."""
s = _consumer_sched(monkeypatch)
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
params = _params("h1", 1)
old = _Request([_Feature("h1", 1)], params=params, req_id="old")
new = _Request([_Feature("h1", 1)], params=params, req_id="new")
now = [1000.0]
monkeypatch.setattr(time, "monotonic", lambda: now[0])
assert s.ensure_cache_available(old, 0) is False
now[0] += sched_mod._ADMIT_DEFER_TIMEOUT_S - 1
assert s.ensure_cache_available(new, 0) is False
assert s.take_unavailable_requests() == set()
now[0] += 2
# `old` is now past its budget; `new` is not.
assert s.ensure_cache_available(new, 0) is False
assert s.take_unavailable_requests() == set()
assert s.request_finished(new) == (False, None)
assert set(s._deferred_since) == {("old", "h1")}
assert s.ensure_cache_available(old, 0) is False
assert s.take_unavailable_requests() == {"old"}
assert "h1" not in s._in_flight
s.shutdown()
def test_metadata_only_entry_admits_without_transfer(monkeypatch):
# A producer that reported placeholder metadata but had no cache entry
# to transfer omits peer_host/peer_port/size_bytes entirely; the consumer
@@ -250,7 +344,7 @@ def test_metadata_only_entry_admits_without_transfer(monkeypatch):
s.shutdown()
def test_orphan_not_ready_entry_falls_back_no_realloc(monkeypatch):
def test_orphan_not_ready_entry_defers_no_realloc(monkeypatch):
# Post-quarantine, post-tombstone-consumed orphan: a not-ready cache
# entry exists for the mm_hash, but the hash is tracked in none of
# _in_flight / _step_completed / _tombstones. Its blocks are held by a
@@ -274,25 +368,13 @@ def test_orphan_not_ready_entry_falls_back_no_realloc(monkeypatch):
monkeypatch.setattr(s._cache, "alloc", _spy_alloc)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# Must not raise; request falls back to local compute (admitted True),
# the orphan entry is untouched, and alloc is never called again.
assert s.ensure_cache_available(req, 0) is True
# Must not raise; the request is deferred while the DMA settles, the
# orphan entry is untouched, and alloc is never called again.
assert s.ensure_cache_available(req, 0) is False
assert calls == []
assert s._cache.get("h1") is entry
assert not entry.ready
s.shutdown()
def test_alloc_failure_falls_back_to_local(monkeypatch):
s = _consumer_sched(monkeypatch)
# Force the cache to reject the allocation.
monkeypatch.setattr(s._cache, "alloc", lambda key, n: None)
req = _Request([_Feature("h1", 1)], params=_params("h1", 1))
# _start_xfer returns False -> request admitted for local recompute.
assert s.ensure_cache_available(req, 0) is True
assert "h1" not in s._in_flight
assert s.take_unavailable_requests() == set()
s.shutdown()
@@ -63,6 +63,7 @@ def test_request_finished_producer_emits_params(monkeypatch):
delay, params = s.request_finished(_Request([_Feature("h1", length=2)]))
assert delay is False
assert entry.evictable
assert params == {
"h1": {
"metadata": {},
+5 -3
View File
@@ -451,14 +451,16 @@ def test_consumer_session_poll_not_ready_nack_goes_to_retryable():
assert not results.completed
def test_consumer_session_take_results_clears_state():
def test_consumer_session_ack_timeout_reports_retry_once():
"""A delayed producer ACK is retryable, and its result is drained once."""
s = _make_consumer_session()
s.start_xfer("h1", [0, 1], deadline=time.monotonic() - 1)
s.poll([], time.monotonic())
r1 = s.take_results()
assert "h1" in r1.tombstoned
assert "h1" in r1.retryable
assert not r1.tombstoned
r2 = s.take_results()
assert not r2.tombstoned # cleared after first take
assert not r2.retryable # cleared after first take
def test_consumer_session_on_peer_down_cancels_waiting_ack():