[EC Connector] CPU Offloading EC Connector (#47423)
Signed-off-by: omerpaz95 <omerpaz95@gmail.com> Signed-off-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -0,0 +1,306 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for EmbeddingCache.
|
||||
|
||||
Pure integer bookkeeping — no mmap, torch, or CUDA — so these run anywhere.
|
||||
"""
|
||||
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
|
||||
|
||||
def _cache(num_blocks: int = 8) -> EmbeddingCache:
|
||||
return EmbeddingCache(num_blocks)
|
||||
|
||||
|
||||
# ── alloc ────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_alloc_returns_unique_block_ids():
|
||||
cache = _cache()
|
||||
entry = cache.alloc("a", 3)
|
||||
assert entry is not None
|
||||
assert len(entry.block_ids) == 3
|
||||
assert len(set(entry.block_ids)) == 3
|
||||
assert all(0 <= i < 8 for i in entry.block_ids)
|
||||
|
||||
|
||||
def test_alloc_returns_tuple():
|
||||
cache = _cache()
|
||||
entry = cache.alloc("a", 2)
|
||||
assert isinstance(entry.block_ids, tuple)
|
||||
|
||||
|
||||
def test_alloc_returns_none_when_full():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 4)
|
||||
assert cache.alloc("b", 1) is None
|
||||
|
||||
|
||||
def test_alloc_asserts_duplicate_key():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
with pytest.raises(AssertionError, match="duplicate alloc"):
|
||||
cache.alloc("a", 2)
|
||||
|
||||
|
||||
def test_alloc_asserts_n_blocks_exceeds_capacity():
|
||||
cache = _cache(num_blocks=4)
|
||||
with pytest.raises(AssertionError, match="capacity"):
|
||||
cache.alloc("a", 5)
|
||||
|
||||
|
||||
def test_alloc_zero_blocks():
|
||||
cache = _cache()
|
||||
entry = cache.alloc("a", 0)
|
||||
assert entry is not None
|
||||
assert entry.block_ids == ()
|
||||
|
||||
|
||||
def test_alloc_starts_not_ready():
|
||||
cache = _cache()
|
||||
entry = cache.alloc("a", 2)
|
||||
assert not entry.ready
|
||||
assert not entry.evictable
|
||||
|
||||
|
||||
# ── get ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_get_missing_returns_none():
|
||||
cache = _cache()
|
||||
assert cache.get("a") is None
|
||||
|
||||
|
||||
def test_get_returns_entry():
|
||||
cache = _cache()
|
||||
alloc_entry = cache.alloc("a", 2)
|
||||
got = cache.get("a")
|
||||
assert got is alloc_entry
|
||||
|
||||
|
||||
def test_get_returns_not_ready_entry():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
entry = cache.get("a")
|
||||
assert entry is not None
|
||||
assert not entry.ready
|
||||
|
||||
|
||||
# ── mark_ready ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_mark_ready_transitions():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
entry = cache.get("a")
|
||||
assert not entry.ready
|
||||
cache.mark_ready("a")
|
||||
assert entry.ready
|
||||
assert entry.evictable
|
||||
|
||||
|
||||
def test_mark_ready_asserts_already_ready():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
with pytest.raises(AssertionError, match="already-ready"):
|
||||
cache.mark_ready("a")
|
||||
|
||||
|
||||
def test_mark_ready_asserts_missing_key():
|
||||
cache = _cache()
|
||||
with pytest.raises(KeyError):
|
||||
cache.mark_ready("nonexistent")
|
||||
|
||||
|
||||
# ── pin / unpin ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_pin_asserts_not_ready():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
with pytest.raises(AssertionError, match="not-ready"):
|
||||
cache.pin("a")
|
||||
|
||||
|
||||
def test_pin_unpin_basic():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.pin("a")
|
||||
entry = cache.get("a")
|
||||
assert not entry.evictable
|
||||
cache.unpin("a")
|
||||
assert entry.evictable
|
||||
|
||||
|
||||
def test_nested_pin():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.pin("a")
|
||||
cache.pin("a")
|
||||
cache.unpin("a")
|
||||
entry = cache.get("a")
|
||||
assert not entry.evictable # still pinned (count=1)
|
||||
cache.unpin("a")
|
||||
assert entry.evictable
|
||||
|
||||
|
||||
def test_unpin_asserts_on_unpinned():
|
||||
cache = _cache()
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
with pytest.raises(AssertionError, match="unpinned"):
|
||||
cache.unpin("a")
|
||||
|
||||
|
||||
# ── eviction ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_eviction_frees_ready_unpinned_fifo():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.alloc("b", 2)
|
||||
cache.mark_ready("b")
|
||||
# Pool full. Alloc should evict "a" (oldest).
|
||||
entry_c = cache.alloc("c", 2)
|
||||
assert entry_c is not None
|
||||
assert cache.get("a") is None # evicted
|
||||
assert cache.get("b") is not None # still there
|
||||
|
||||
|
||||
def test_eviction_skips_pinned():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.pin("a")
|
||||
cache.alloc("b", 2)
|
||||
cache.mark_ready("b")
|
||||
# "a" is pinned, so "b" should be evicted instead.
|
||||
entry_c = cache.alloc("c", 2)
|
||||
assert entry_c is not None
|
||||
assert cache.get("a") is not None # pinned, survived
|
||||
assert cache.get("b") is None # evicted
|
||||
cache.unpin("a")
|
||||
|
||||
|
||||
def test_eviction_skips_not_ready():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 2) # not ready
|
||||
cache.alloc("b", 2)
|
||||
cache.mark_ready("b")
|
||||
# "a" is not ready (non-evictable), "b" is evictable.
|
||||
entry_c = cache.alloc("c", 2)
|
||||
assert entry_c is not None
|
||||
assert cache.get("a") is not None # not ready, survived
|
||||
assert cache.get("b") is None # evicted
|
||||
cache.mark_ready("a") # clean up
|
||||
|
||||
|
||||
def test_eviction_returns_none_if_all_non_evictable():
|
||||
cache = _cache(num_blocks=4)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.pin("a")
|
||||
cache.alloc("b", 2) # not ready
|
||||
# Both entries are non-evictable (a=pinned, b=not-ready).
|
||||
assert cache.alloc("c", 1) is None
|
||||
# Fail-fast: nothing was evicted.
|
||||
assert cache.get("a") is not None
|
||||
assert cache.get("b") is not None
|
||||
cache.unpin("a")
|
||||
|
||||
|
||||
def test_fail_fast_no_wasted_eviction():
|
||||
cache = _cache(num_blocks=6)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a") # evictable: 2 blocks
|
||||
cache.alloc("b", 2) # not ready
|
||||
cache.alloc("c", 2) # not ready
|
||||
# free=0, evictable=2, need 3 -> cannot succeed
|
||||
assert cache.alloc("d", 3) is None
|
||||
# "a" was NOT evicted despite being evictable
|
||||
assert cache.get("a") is not None
|
||||
|
||||
|
||||
def test_eviction_multiple_entries():
|
||||
cache = _cache(num_blocks=6)
|
||||
cache.alloc("a", 2)
|
||||
cache.mark_ready("a")
|
||||
cache.alloc("b", 2)
|
||||
cache.mark_ready("b")
|
||||
cache.alloc("c", 2)
|
||||
cache.mark_ready("c")
|
||||
# Need 4 blocks: must evict both "a" and "b".
|
||||
entry_d = cache.alloc("d", 4)
|
||||
assert entry_d is not None
|
||||
assert cache.get("a") is None
|
||||
assert cache.get("b") is None
|
||||
assert cache.get("c") is not None
|
||||
|
||||
|
||||
# ── thread safety ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_concurrent_pin_unpin():
|
||||
cache = _cache(num_blocks=8)
|
||||
cache.alloc("a", 4)
|
||||
cache.mark_ready("a")
|
||||
errors: list[Exception] = []
|
||||
barrier = threading.Barrier(8)
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
try:
|
||||
for _ in range(500):
|
||||
cache.pin("a")
|
||||
cache.unpin("a")
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(8)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert not errors, errors
|
||||
entry = cache.get("a")
|
||||
assert entry.evictable # all pins balanced
|
||||
|
||||
|
||||
def test_concurrent_alloc_evict():
|
||||
cache = _cache(num_blocks=4)
|
||||
errors: list[Exception] = []
|
||||
barrier = threading.Barrier(4)
|
||||
counter = {"i": 0}
|
||||
counter_lock = threading.Lock()
|
||||
|
||||
def worker():
|
||||
barrier.wait()
|
||||
try:
|
||||
for _ in range(200):
|
||||
with counter_lock:
|
||||
key = f"k{counter['i']}"
|
||||
counter["i"] += 1
|
||||
entry = cache.alloc(key, 1)
|
||||
if entry is not None:
|
||||
cache.mark_ready(key)
|
||||
except Exception as exc:
|
||||
errors.append(exc)
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(4)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
|
||||
assert not errors, errors
|
||||
@@ -0,0 +1,480 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import uuid
|
||||
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import ECCPUScheduler
|
||||
|
||||
_N = 16
|
||||
_BS = 64
|
||||
|
||||
|
||||
class _Pos:
|
||||
def __init__(self, offset, length):
|
||||
self.offset = offset
|
||||
self.length = length
|
||||
|
||||
|
||||
class _Feature:
|
||||
def __init__(self, mm_hash, length=1, identifier=None):
|
||||
self.mm_hash = mm_hash
|
||||
self.identifier = identifier if identifier is not None else mm_hash
|
||||
self.mm_position = _Pos(0, length)
|
||||
|
||||
|
||||
class _Request:
|
||||
_counter = 0
|
||||
|
||||
def __init__(self, features, request_id=None):
|
||||
self.mm_features = features
|
||||
if request_id is None:
|
||||
_Request._counter += 1
|
||||
self.request_id = f"req_{_Request._counter}"
|
||||
else:
|
||||
self.request_id = request_id
|
||||
|
||||
|
||||
def _make_scheduler(
|
||||
monkeypatch,
|
||||
num_blocks=_N,
|
||||
max_concurrent_batches=1,
|
||||
*,
|
||||
is_producer=True,
|
||||
is_consumer=True,
|
||||
) -> ECCPUScheduler:
|
||||
region = ECSharedRegion(
|
||||
engine_id=str(uuid.uuid4()),
|
||||
num_blocks=num_blocks,
|
||||
block_size_bytes=_BS,
|
||||
)
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
|
||||
|
||||
_is_prod = is_producer
|
||||
_is_cons = is_consumer
|
||||
_mcb = max_concurrent_batches
|
||||
|
||||
class _EC:
|
||||
is_ec_producer = _is_prod
|
||||
is_ec_consumer = _is_cons
|
||||
|
||||
class _Cfg:
|
||||
ec_transfer_config = _EC()
|
||||
|
||||
@property
|
||||
def max_concurrent_batches(self):
|
||||
return _mcb
|
||||
|
||||
return ECCPUScheduler(_Cfg())
|
||||
|
||||
|
||||
class _SchedulerOutput:
|
||||
def __init__(self, finished_req_ids=None):
|
||||
self.finished_req_ids = finished_req_ids or set()
|
||||
|
||||
|
||||
def _seed_cached(s: ECCPUScheduler, mm_hash: str, n_blocks: int):
|
||||
"""Pre-populate a ready cache entry backed by real blocks."""
|
||||
s._cache.alloc(mm_hash, n_blocks)
|
||||
s._cache.mark_ready(mm_hash)
|
||||
|
||||
|
||||
def test_offload_reuse_cycle(monkeypatch):
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
# Step A: first sight — allocate and save.
|
||||
assert s.has_cache_item("h1") is False
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta_a = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta_a.saves
|
||||
assert meta_a.loads == {}
|
||||
# Not ready yet (max_concurrent_batches=1 means ready next step).
|
||||
assert s.has_cache_item("h1") is False
|
||||
|
||||
# Step B: _step_readiness marks h1 ready (no new allocs).
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("h1") is True
|
||||
|
||||
# Step C: reload now works — entry is ready at update_state_after_alloc time.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta_c = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta_c.loads
|
||||
assert meta_c.loads["h1"] == meta_a.saves["h1"]
|
||||
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_has_cache_item_false_when_not_consumer(monkeypatch):
|
||||
s = _make_scheduler(monkeypatch, is_consumer=False)
|
||||
assert s.has_cache_item("anything") is False
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_connector_keys_on_identifier_not_mm_hash(monkeypatch):
|
||||
"""The connector must key the encoder cache on feature.identifier (what
|
||||
has_cache_item is called with), NOT feature.mm_hash."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("PROC_KEY", length=1, identifier="ENC_KEY")])
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "ENC_KEY" in meta.saves
|
||||
assert "PROC_KEY" not in meta.saves
|
||||
# Not ready yet (1 step delay).
|
||||
assert s.has_cache_item("ENC_KEY") is False
|
||||
# After another step it becomes ready — keyed on identifier.
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("ENC_KEY") is True
|
||||
assert s.has_cache_item("PROC_KEY") is False
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_cpu_region_fifo_eviction(monkeypatch):
|
||||
# Region holds exactly 2 one-block encodings.
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
|
||||
# Fill with A and B, marking each ready.
|
||||
for h in ("A", "B"):
|
||||
s.update_state_after_alloc(_Request([_Feature(h, length=1)]), 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
# After the step above, A's step was called in B's build. A is ready.
|
||||
# B was submitted in the last build, so one more step makes it ready.
|
||||
s.build_connector_meta(scheduler_output=None) # marks B ready
|
||||
assert s.has_cache_item("A") is True
|
||||
assert s.has_cache_item("B") is True
|
||||
|
||||
# Allocate C: must evict A (oldest) to make room.
|
||||
s.update_state_after_alloc(_Request([_Feature("C", length=1)]), 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("A") is False # evicted
|
||||
assert s.has_cache_item("B") is True
|
||||
|
||||
|
||||
def test_multiple_mm_items_per_request(monkeypatch):
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("h1", length=1), _Feature("h2", length=1)])
|
||||
s.update_state_after_alloc(req, 0)
|
||||
s.update_state_after_alloc(req, 1)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert set(meta.saves) == {"h1", "h2"}
|
||||
|
||||
# Step delay: mark both ready.
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("h1") is True
|
||||
assert s.has_cache_item("h2") is True
|
||||
|
||||
# Now reload both.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
s.update_state_after_alloc(req, 1)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert set(meta2.loads) == {"h1", "h2"}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_load_returns_correct_block_ids(monkeypatch):
|
||||
"""meta.loads must contain the same block IDs that meta.saves
|
||||
allocated — verified through the public API only."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("a", length=2)])
|
||||
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta_save = s.build_connector_meta(scheduler_output=None)
|
||||
assert "a" in meta_save.saves
|
||||
|
||||
# Readiness delay.
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# Reload.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta_load = s.build_connector_meta(scheduler_output=None)
|
||||
assert meta_load.loads["a"] == meta_save.saves["a"]
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_loads_only_serves_hashes_touched_this_step(monkeypatch):
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
_seed_cached(s, "b", n_blocks=2)
|
||||
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "a" in meta.loads
|
||||
assert "b" not in meta.loads
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_repeated_reload_same_step_loads_once(monkeypatch):
|
||||
"""The same mm_hash requested twice in one step must appear in
|
||||
meta.loads exactly once."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert list(meta.loads.keys()).count("a") == 1
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_load_not_emitted_for_uncached_entry(monkeypatch):
|
||||
"""A feature that was never saved must not appear in meta.loads."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
s.update_state_after_alloc(_Request([_Feature("missing")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert meta.loads == {}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_delayed_unpin_protects_blocks_during_worker_read(monkeypatch):
|
||||
"""Blocks being loaded must not be evictable until the delayed unpin
|
||||
fires. Observable: a save that needs eviction cannot reclaim a
|
||||
still-pinned loaded entry."""
|
||||
# 2 blocks, both occupied by "a" (ready).
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
# Load "a" — pins it.
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "a" in meta.loads
|
||||
|
||||
# Try to save "b" (needs 2 blocks, but "a" is pinned) — must fail.
|
||||
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" not in meta2.saves # can't evict pinned "a"
|
||||
|
||||
# After the delay, "a" unpins. Now "b" can evict it.
|
||||
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
|
||||
meta3 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" in meta3.saves
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_delayed_unpin_depth_2(monkeypatch):
|
||||
"""With max_concurrent_batches=2, loaded blocks stay pinned for 2 steps
|
||||
after build. Observable via eviction failure."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
# Load "a" → pinned.
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# Step 1 after load: still pinned — can't evict for "b".
|
||||
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
|
||||
meta1 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" not in meta1.saves
|
||||
|
||||
# Step 2 after load: still pinned.
|
||||
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" not in meta2.saves
|
||||
|
||||
# Step 3: unpin fires, now eviction works.
|
||||
s.update_state_after_alloc(_Request([_Feature("b", length=2)]), 0)
|
||||
meta3 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" in meta3.saves
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_shutdown_unpins_pending_loads(monkeypatch):
|
||||
"""Shutdown must unpin entries still in _pending_loads (not yet drained
|
||||
to the unpin deque). Observable: after shutdown, the entry is evictable
|
||||
(alloc can reclaim its blocks)."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
# Don't call build_connector_meta — "a" stays in _pending_loads.
|
||||
s.shutdown()
|
||||
|
||||
# After shutdown, scheduler reports no items (consumer role is off).
|
||||
assert s.has_cache_item("a") is False
|
||||
|
||||
|
||||
def test_shutdown_unpins_deferred_unpin_queue(monkeypatch):
|
||||
"""Shutdown must unpin entries queued in the deferred unpin deque
|
||||
(already drained from _pending_loads but not yet expired)."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=2)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
# Load "a" and build — moves to unpin deque.
|
||||
s.update_state_after_alloc(_Request([_Feature("a")]), 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# "a" is in the deferred unpin deque (depth=2, not yet expired).
|
||||
# Shutdown must unpin it.
|
||||
s.shutdown()
|
||||
assert s.has_cache_item("a") is False
|
||||
|
||||
|
||||
def test_eviction_skips_entry_pinned_by_pending_load(monkeypatch):
|
||||
"""A loaded entry still within the unpin delay window must survive
|
||||
eviction attempts from new saves."""
|
||||
# 2 blocks: A occupies both, ready.
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2)
|
||||
_seed_cached(s, "A", n_blocks=2)
|
||||
|
||||
# Load A → pinned.
|
||||
s.update_state_after_alloc(_Request([_Feature("A")]), 0)
|
||||
meta_load = s.build_connector_meta(scheduler_output=None)
|
||||
assert "A" in meta_load.loads
|
||||
|
||||
# Try to save C (needs 2 blocks) — can't evict pinned A.
|
||||
s.update_state_after_alloc(_Request([_Feature("C", length=2)]), 0)
|
||||
meta_save = s.build_connector_meta(scheduler_output=None)
|
||||
assert "C" not in meta_save.saves
|
||||
assert s.has_cache_item("A") is True # survived
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_region_full_skips_save_and_never_blocks(monkeypatch):
|
||||
"""When the region is fully occupied by pinned entries, new saves are
|
||||
silently skipped and ensure_cache_available never blocks."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=1)
|
||||
_seed_cached(s, "pinned", n_blocks=1)
|
||||
s._cache.pin("pinned")
|
||||
|
||||
req = _Request([_Feature("new", length=1)])
|
||||
assert s.ensure_cache_available(req, 0) is True
|
||||
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "new" not in meta.saves
|
||||
assert s.has_cache_item("new") is False
|
||||
|
||||
s._cache.unpin("pinned")
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_step_readiness_depth_2(monkeypatch):
|
||||
"""With max_concurrent_batches=2, entries need 2 steps to become ready."""
|
||||
s = _make_scheduler(monkeypatch, max_concurrent_batches=2)
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta_0 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta_0.saves
|
||||
assert s.has_cache_item("h1") is False
|
||||
|
||||
# One step later: still not ready (depth=2).
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("h1") is False
|
||||
|
||||
# Two steps later: now ready.
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("h1") is True
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_producer_only_never_emits_loads(monkeypatch):
|
||||
"""A producer-only scheduler must never populate meta.loads, even when
|
||||
entries are ready."""
|
||||
s = _make_scheduler(monkeypatch, is_consumer=False)
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta.saves
|
||||
assert meta.loads == {}
|
||||
|
||||
# Mark ready.
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
|
||||
# Even if we re-encounter the same feature, no load.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert meta2.loads == {}
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_consumer_only_never_emits_saves(monkeypatch):
|
||||
"""A consumer-only scheduler must never populate meta.saves."""
|
||||
s = _make_scheduler(monkeypatch, is_producer=False)
|
||||
_seed_cached(s, "a", n_blocks=1)
|
||||
|
||||
req = _Request([_Feature("a", length=1)])
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert meta.saves == {}
|
||||
assert "a" in meta.loads
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_consumer_only_has_cache_item(monkeypatch):
|
||||
"""Consumer-only: has_cache_item works normally for ready entries."""
|
||||
s = _make_scheduler(monkeypatch, is_producer=False)
|
||||
assert s.has_cache_item("a") is False
|
||||
_seed_cached(s, "a", n_blocks=1)
|
||||
assert s.has_cache_item("a") is True
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_save_not_emitted_for_already_cached_entry(monkeypatch):
|
||||
"""An entry already in the cache (from a prior save) must not trigger
|
||||
a second allocation or appear in meta.saves again."""
|
||||
s = _make_scheduler(monkeypatch)
|
||||
req = _Request([_Feature("h1", length=1)])
|
||||
|
||||
# First encounter → save.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta1 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" in meta1.saves
|
||||
|
||||
# Second encounter (same step or later) → no save.
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "h1" not in meta2.saves
|
||||
s.shutdown()
|
||||
|
||||
|
||||
# ── first-finish fast-path (idle engine) ────────────────────────────────────
|
||||
|
||||
|
||||
def test_first_finish_marks_ready_immediately(monkeypatch):
|
||||
"""When the originating request finishes, the save entry becomes ready
|
||||
without waiting for the step-count delay."""
|
||||
s = _make_scheduler(monkeypatch, max_concurrent_batches=5)
|
||||
req = _Request([_Feature("h1", length=1)], request_id="req_x")
|
||||
|
||||
s.update_state_after_alloc(req, 0)
|
||||
s.build_connector_meta(scheduler_output=None)
|
||||
assert s.has_cache_item("h1") is False
|
||||
|
||||
# Request finishes — readiness should fire immediately.
|
||||
s.build_connector_meta(_SchedulerOutput(finished_req_ids={"req_x"}))
|
||||
assert s.has_cache_item("h1") is True
|
||||
s.shutdown()
|
||||
|
||||
|
||||
def test_first_finish_unpins_immediately(monkeypatch):
|
||||
"""When the originating request finishes, the load entry is unpinned
|
||||
without waiting for the step-count delay."""
|
||||
s = _make_scheduler(monkeypatch, num_blocks=2, max_concurrent_batches=5)
|
||||
_seed_cached(s, "a", n_blocks=2)
|
||||
|
||||
req = _Request([_Feature("a")], request_id="req_y")
|
||||
s.update_state_after_alloc(req, 0)
|
||||
meta = s.build_connector_meta(scheduler_output=None)
|
||||
assert "a" in meta.loads
|
||||
|
||||
# "a" is pinned. Try to evict — must fail.
|
||||
s.update_state_after_alloc(
|
||||
_Request([_Feature("b", length=2)], request_id="req_z"), 0
|
||||
)
|
||||
meta2 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" not in meta2.saves
|
||||
|
||||
# req_y finishes — unpin fires in build_connector_meta (fast-path).
|
||||
s.build_connector_meta(_SchedulerOutput(finished_req_ids={"req_y"}))
|
||||
|
||||
# Next step: "a" is now evictable, so "b" can be allocated.
|
||||
s.update_state_after_alloc(
|
||||
_Request([_Feature("b", length=2)], request_id="req_w"), 0
|
||||
)
|
||||
meta3 = s.build_connector_meta(scheduler_output=None)
|
||||
assert "b" in meta3.saves
|
||||
s.shutdown()
|
||||
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for StepTracker."""
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import (
|
||||
StepTracker,
|
||||
)
|
||||
|
||||
|
||||
def test_basic_step_count_expiry_depth_1():
|
||||
"""With max_concurrent_batches=1, entry expires after 1 step."""
|
||||
t = StepTracker(max_concurrent_batches=1)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
# Step 1: entry was added, committed to slot. Deque fills to maxlen=1,
|
||||
# but no expiry yet (the entry is in the slot just committed).
|
||||
result = t.step(set())
|
||||
assert result == []
|
||||
|
||||
# Step 2: deque is full, oldest slot (containing h1) expires.
|
||||
result = t.step(set())
|
||||
assert result == ["h1"]
|
||||
|
||||
|
||||
def test_basic_step_count_expiry_depth_2():
|
||||
"""With max_concurrent_batches=2, entry expires after 2 steps."""
|
||||
t = StepTracker(max_concurrent_batches=2)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
assert t.step(set()) == [] # step 1: committed
|
||||
assert t.step(set()) == [] # step 2: deque fills but h1 in newer slot
|
||||
assert t.step(set()) == ["h1"] # step 3: h1's slot is oldest, expires
|
||||
|
||||
|
||||
def test_first_finish_fast_path():
|
||||
"""Entry is returned immediately when request_id appears in finished."""
|
||||
t = StepTracker(max_concurrent_batches=3)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
result = t.step({"req_a"})
|
||||
assert result == ["h1"]
|
||||
|
||||
|
||||
def test_no_double_processing_fast_path_then_expiry():
|
||||
"""Entry fast-pathed should NOT fire again on deque expiry."""
|
||||
t = StepTracker(max_concurrent_batches=1)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
# Fast-path fires h1.
|
||||
result = t.step({"req_a"})
|
||||
assert result == ["h1"]
|
||||
|
||||
# Normal expiry would fire now, but h1 is already processed.
|
||||
result = t.step(set())
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_no_double_processing_expiry_then_fast_path():
|
||||
"""Entry expired via step-count should NOT fire again via fast-path."""
|
||||
t = StepTracker(max_concurrent_batches=1)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
t.step(set()) # commit
|
||||
result = t.step(set()) # expiry fires h1
|
||||
assert result == ["h1"]
|
||||
|
||||
# req_a finishes later — must not re-fire.
|
||||
result = t.step({"req_a"})
|
||||
assert result == []
|
||||
|
||||
|
||||
def test_multiplicity_same_hash_different_requests():
|
||||
"""Two adds of the same mm_hash (different requests) → returned twice."""
|
||||
t = StepTracker(max_concurrent_batches=3)
|
||||
t.add("h1", "req_a")
|
||||
t.add("h1", "req_b")
|
||||
|
||||
# Both requests finish in the same step.
|
||||
result = t.step({"req_a", "req_b"})
|
||||
assert sorted(result) == ["h1", "h1"]
|
||||
|
||||
|
||||
def test_mixed_expiry_and_fast_path():
|
||||
"""Some entries expire, others fast-path, in the same step."""
|
||||
t = StepTracker(max_concurrent_batches=1)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
t.step(set()) # commit h1's slot
|
||||
|
||||
# Add h2 in the next step.
|
||||
t.add("h2", "req_b")
|
||||
|
||||
# This step: h1's slot expires AND req_b finishes.
|
||||
result = t.step({"req_b"})
|
||||
assert set(result) == {"h1", "h2"}
|
||||
|
||||
|
||||
def test_drain_all_returns_unprocessed():
|
||||
"""drain_all returns everything pending and clears state."""
|
||||
t = StepTracker(max_concurrent_batches=3)
|
||||
t.add("h1", "req_a")
|
||||
t.add("h2", "req_b")
|
||||
t.step(set()) # commit both
|
||||
t.add("h3", "req_c") # in _current, not yet committed
|
||||
|
||||
result = t.drain_all()
|
||||
assert set(result) == {"h1", "h2", "h3"}
|
||||
|
||||
# After drain, everything is empty.
|
||||
assert t.step(set()) == []
|
||||
assert t.drain_all() == []
|
||||
|
||||
|
||||
def test_drain_all_skips_already_processed():
|
||||
"""drain_all should not re-return entries already fast-pathed."""
|
||||
t = StepTracker(max_concurrent_batches=3)
|
||||
t.add("h1", "req_a")
|
||||
t.add("h2", "req_b")
|
||||
|
||||
# Fast-path h1.
|
||||
result = t.step({"req_a"})
|
||||
assert result == ["h1"]
|
||||
|
||||
# drain should only return h2.
|
||||
result = t.drain_all()
|
||||
assert result == ["h2"]
|
||||
|
||||
|
||||
def test_empty_step_returns_nothing():
|
||||
"""step() with nothing pending returns empty list."""
|
||||
t = StepTracker(max_concurrent_batches=2)
|
||||
assert t.step(set()) == []
|
||||
assert t.step({"req_x"}) == []
|
||||
|
||||
|
||||
def test_add_after_step_goes_to_next_slot():
|
||||
"""Entries added after a step go into the next slot, not the current."""
|
||||
t = StepTracker(max_concurrent_batches=1)
|
||||
|
||||
t.step(set()) # empty step
|
||||
t.add("h1", "req_a")
|
||||
|
||||
# h1 is in _current. Next step commits it.
|
||||
t.step(set())
|
||||
|
||||
# Now the deque is full (1 slot with h1). Next step expires it.
|
||||
result = t.step(set())
|
||||
assert result == ["h1"]
|
||||
|
||||
|
||||
def test_fast_path_on_current_entries():
|
||||
"""Entries still in _current (not yet committed) can be fast-pathed."""
|
||||
t = StepTracker(max_concurrent_batches=3)
|
||||
t.add("h1", "req_a")
|
||||
|
||||
# req_a finishes in the same step h1 was added.
|
||||
result = t.step({"req_a"})
|
||||
assert result == ["h1"]
|
||||
|
||||
# Entry is committed to slot but marked processed — won't re-fire.
|
||||
for _ in range(5):
|
||||
assert t.step(set()) == []
|
||||
@@ -0,0 +1,47 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorRole
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.connector import ECCPUConnector
|
||||
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
|
||||
|
||||
|
||||
def _cfg():
|
||||
ec = MagicMock()
|
||||
ec.is_ec_producer = True
|
||||
ec.is_ec_consumer = True
|
||||
cfg = MagicMock()
|
||||
cfg.ec_transfer_config = ec
|
||||
return cfg
|
||||
|
||||
|
||||
def test_scheduler_role_builds_only_scheduler(monkeypatch):
|
||||
fake_sched = MagicMock()
|
||||
monkeypatch.setattr(ECCPUConnector, "_make_scheduler", lambda self, cfg: fake_sched)
|
||||
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
|
||||
assert c.connector_scheduler is fake_sched
|
||||
assert c.connector_worker is None
|
||||
c.has_cache_item("x")
|
||||
fake_sched.has_cache_item.assert_called_once_with("x")
|
||||
|
||||
|
||||
def test_worker_role_builds_only_worker(monkeypatch):
|
||||
fake_worker = MagicMock()
|
||||
monkeypatch.setattr(ECCPUConnector, "_make_worker", lambda self, cfg: fake_worker)
|
||||
c = ECCPUConnector(_cfg(), ECConnectorRole.WORKER)
|
||||
assert c.connector_worker is fake_worker
|
||||
assert c.connector_scheduler is None
|
||||
|
||||
|
||||
def test_request_finished_inherited_noop(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
ECCPUConnector, "_make_scheduler", lambda self, cfg: MagicMock()
|
||||
)
|
||||
c = ECCPUConnector(_cfg(), ECConnectorRole.SCHEDULER)
|
||||
assert c.request_finished(MagicMock()) == (False, None)
|
||||
|
||||
|
||||
def test_factory_registered():
|
||||
cls = ECConnectorFactory._registry["ECCPUConnector"]()
|
||||
assert cls is ECCPUConnector
|
||||
@@ -0,0 +1,170 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ECSharedRegion (the mmap substrate)."""
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
_wait_for_file_size,
|
||||
)
|
||||
|
||||
|
||||
def _make_region(num_blocks: int = 8, block_size_bytes: int = 64) -> ECSharedRegion:
|
||||
return ECSharedRegion(
|
||||
engine_id=str(uuid.uuid4()),
|
||||
num_blocks=num_blocks,
|
||||
block_size_bytes=block_size_bytes,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def region() -> ECSharedRegion:
|
||||
r = _make_region()
|
||||
yield r
|
||||
r.cleanup()
|
||||
|
||||
|
||||
# ── mmap sharing between two instances ───────────────────────────────────────
|
||||
|
||||
|
||||
def test_second_instance_opens_existing_file_and_shares_memory():
|
||||
instance_id = str(uuid.uuid4())
|
||||
r1 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
num_blocks=4,
|
||||
block_size_bytes=64,
|
||||
)
|
||||
try:
|
||||
r2 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
num_blocks=4,
|
||||
block_size_bytes=64,
|
||||
)
|
||||
try:
|
||||
assert r1._is_creator
|
||||
assert not r2._is_creator
|
||||
# Both map the same physical pages; writes via r1 are visible via r2.
|
||||
r1.blocks[0, :4] = torch.tensor([10, 20, 30, 40], dtype=torch.int8)
|
||||
assert r2.blocks[0, :4].tolist() == [10, 20, 30, 40]
|
||||
finally:
|
||||
r2.cleanup()
|
||||
finally:
|
||||
r1.cleanup()
|
||||
|
||||
|
||||
def test_only_creator_unlinks_file_on_cleanup():
|
||||
"""Critical contract: if the non-creator unlinks, the creator's mmap path
|
||||
becomes a dangling backing file and a third opener would create a new one
|
||||
out from under the creator."""
|
||||
instance_id = str(uuid.uuid4())
|
||||
r1 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
num_blocks=4,
|
||||
block_size_bytes=64,
|
||||
)
|
||||
path = r1._mmap_path
|
||||
r2 = ECSharedRegion(
|
||||
engine_id=instance_id,
|
||||
num_blocks=4,
|
||||
block_size_bytes=64,
|
||||
)
|
||||
|
||||
# Non-creator goes away first — file must still be on disk for r1.
|
||||
r2.cleanup()
|
||||
assert os.path.exists(path), "non-creator cleanup must not unlink the file"
|
||||
|
||||
# Creator goes away — file is removed.
|
||||
r1.cleanup()
|
||||
assert not os.path.exists(path), "creator cleanup must unlink the file"
|
||||
|
||||
|
||||
# ── _wait_for_file_size ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_wait_for_file_size_returns_when_already_big_enough(tmp_path):
|
||||
"""The fast path: file already at expected size — return immediately."""
|
||||
p = tmp_path / "f.bin"
|
||||
p.write_bytes(b"\x00" * 128)
|
||||
fd = os.open(str(p), os.O_RDONLY)
|
||||
try:
|
||||
_wait_for_file_size(fd, expected_size=128, timeout=1.0) # must not raise
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def test_wait_for_file_size_times_out_when_file_stays_empty(tmp_path):
|
||||
p = tmp_path / "f.bin"
|
||||
p.write_bytes(b"")
|
||||
fd = os.open(str(p), os.O_RDONLY)
|
||||
try:
|
||||
with pytest.raises(TimeoutError):
|
||||
_wait_for_file_size(fd, expected_size=4096, timeout=0.05)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
# ── pin_memory ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_pin_memory_success_sets_flag(region):
|
||||
"""When cudaHostRegister returns 0, _is_pinned flips to True
|
||||
and cleanup will correspondingly call cudaHostUnregister."""
|
||||
fake_cudart = MagicMock()
|
||||
success = MagicMock()
|
||||
success.value = 0
|
||||
fake_cudart.cudaHostRegister.return_value = success
|
||||
fake_cudart.cudaHostUnregister.return_value = success
|
||||
|
||||
with (
|
||||
patch("torch.cuda.is_available", return_value=True),
|
||||
patch("torch.cuda.cudart", return_value=fake_cudart),
|
||||
):
|
||||
region.pin_memory()
|
||||
assert region._is_pinned is True
|
||||
# cleanup must pair with cudaHostUnregister exactly once.
|
||||
region.cleanup()
|
||||
fake_cudart.cudaHostUnregister.assert_called_once()
|
||||
|
||||
|
||||
def test_pin_memory_failure_leaves_flag_false():
|
||||
"""If cudaHostRegister fails (non-zero), don't pretend it succeeded —
|
||||
cleanup must NOT call cudaHostUnregister on memory we never registered."""
|
||||
r = _make_region()
|
||||
try:
|
||||
fake_cudart = MagicMock()
|
||||
fail = MagicMock()
|
||||
fail.value = 1 # non-zero == error
|
||||
fake_cudart.cudaHostRegister.return_value = fail
|
||||
|
||||
with (
|
||||
patch("torch.cuda.is_available", return_value=True),
|
||||
patch("torch.cuda.cudart", return_value=fake_cudart),
|
||||
):
|
||||
r.pin_memory()
|
||||
assert r._is_pinned is False
|
||||
# Now run cleanup and verify cudaHostUnregister was NOT called.
|
||||
r.cleanup()
|
||||
fake_cudart.cudaHostUnregister.assert_not_called()
|
||||
finally:
|
||||
r.cleanup()
|
||||
|
||||
|
||||
def test_pin_memory_noop_without_cuda(region):
|
||||
"""pin_memory is a no-op when CUDA is not available."""
|
||||
with patch("torch.cuda.is_available", return_value=False):
|
||||
region.pin_memory()
|
||||
assert region._is_pinned is False
|
||||
|
||||
|
||||
# ── cleanup idempotency ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_cleanup_is_idempotent(region):
|
||||
region.cleanup()
|
||||
region.cleanup() # fixture calls a third time — must not raise
|
||||
@@ -0,0 +1,598 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for ECCPUWorker.
|
||||
|
||||
The byte-level tests exercise real CUDA stream/event coordination
|
||||
against a real ``ECSharedRegion`` mmap and are skipped on hosts without
|
||||
CUDA. The lifecycle tests don't fire any ``torch.cuda.*`` primitives and
|
||||
run anywhere.
|
||||
|
||||
Mocking policy
|
||||
--------------
|
||||
- ``create_ec_shared_region`` is patched to inject a real ``ECSharedRegion``
|
||||
(mmap backed) with deterministic small dimensions, so byte assertions
|
||||
hit the actual mmap. The real responsibility of the worker — block
|
||||
index → mmap byte mapping, stream coordination, dtype/shape — is
|
||||
exercised end to end against a real GPU.
|
||||
- ``is_pin_memory_available`` is patched per-test so the rank-gating
|
||||
path can be probed deterministically without depending on host caps.
|
||||
- The shutdown test bypasses ``__init__`` via ``object.__new__`` and
|
||||
injects a ``MagicMock`` region.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import logging
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import ECCPUWorker
|
||||
|
||||
# ── shape constants ──────────────────────────────────────────────────────────
|
||||
|
||||
# hidden_dim=8, dtype=fp16 (2 bytes) → block_size_bytes = 16, so one row of
|
||||
# encoder output fits in exactly one block.
|
||||
_HIDDEN_DIM = 8
|
||||
_DTYPE = torch.float16
|
||||
_BLOCK_SIZE_BYTES = _HIDDEN_DIM * _DTYPE.itemsize
|
||||
_NUM_BLOCKS = 8
|
||||
|
||||
_requires_cuda = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="exercises real CUDA stream/event coordination in ECCPUWorker",
|
||||
)
|
||||
|
||||
|
||||
def _make_region() -> ECSharedRegion:
|
||||
"""Fresh region backed by a real per-test mmap file."""
|
||||
return ECSharedRegion(
|
||||
engine_id=str(uuid.uuid4()),
|
||||
num_blocks=_NUM_BLOCKS,
|
||||
block_size_bytes=_BLOCK_SIZE_BYTES,
|
||||
)
|
||||
|
||||
|
||||
def _vllm_config(rank: int = 0) -> Mock:
|
||||
cfg = Mock(spec=VllmConfig)
|
||||
cfg.parallel_config = Mock(spec=ParallelConfig)
|
||||
cfg.parallel_config.rank = rank
|
||||
cfg.model_config = Mock()
|
||||
cfg.model_config.dtype = _DTYPE
|
||||
return cfg
|
||||
|
||||
|
||||
def _meta(
|
||||
*, saves: dict | None = None, loads: dict | None = None
|
||||
) -> ECCPUConnectorMetadata:
|
||||
return ECCPUConnectorMetadata(saves=saves or {}, loads=loads or {})
|
||||
|
||||
|
||||
# ── fixtures ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def make_worker():
|
||||
"""Factory that builds an ``ECCPUWorker`` over a real ``ECSharedRegion``.
|
||||
|
||||
Tracks the workers and regions it creates and tears them down at the
|
||||
end of the test, so each test starts from a fresh /dev/shm file.
|
||||
"""
|
||||
workers: list[ECCPUWorker] = []
|
||||
regions: list[ECSharedRegion] = []
|
||||
|
||||
def factory(
|
||||
*,
|
||||
rank: int = 0,
|
||||
tp_rank: int = 0,
|
||||
pcp_rank: int = 0,
|
||||
pin_memory_available: bool = False,
|
||||
) -> ECCPUWorker:
|
||||
region = _make_region()
|
||||
regions.append(region)
|
||||
pcp_group = Mock()
|
||||
pcp_group.rank_in_group = pcp_rank
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.create_ec_shared_region",
|
||||
return_value=region,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.is_pin_memory_available",
|
||||
return_value=pin_memory_available,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_tensor_model_parallel_rank",
|
||||
return_value=tp_rank,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_pcp_group",
|
||||
return_value=pcp_group,
|
||||
),
|
||||
):
|
||||
worker = ECCPUWorker(_vllm_config(rank=rank))
|
||||
workers.append(worker)
|
||||
return worker
|
||||
|
||||
yield factory
|
||||
|
||||
for worker in workers:
|
||||
with contextlib.suppress(Exception):
|
||||
worker.shutdown()
|
||||
for region in regions:
|
||||
with contextlib.suppress(Exception):
|
||||
region.cleanup()
|
||||
|
||||
|
||||
# ── save_caches ──────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@pytest.mark.parametrize(
|
||||
"n_elements,n_blocks",
|
||||
[
|
||||
(3 * _HIDDEN_DIM, 3), # exact fit: 3 full blocks, no remainder
|
||||
(3 * _HIDDEN_DIM + 4, 4), # partial last block: 4 fp16 trail in slot 3
|
||||
],
|
||||
ids=["exact-fit", "partial-last-block"],
|
||||
)
|
||||
def test_save_caches_writes_to_assigned_blocks(make_worker, n_elements, n_blocks):
|
||||
"""``save_caches`` + ``flush_saves`` copies the source GPU tensor's bytes
|
||||
into the block IDs named by ``meta.saves``, in the order they appear,
|
||||
via a single batched GPU→CPU copy.
|
||||
|
||||
When ``total_bytes`` is not a multiple of ``block_size_bytes`` the last
|
||||
block is partially written; the unwritten tail must remain whatever
|
||||
was there before. Blocks not named in ``meta.saves`` must be untouched.
|
||||
"""
|
||||
worker = make_worker()
|
||||
sentinel = 0x5A
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
|
||||
src = torch.arange(n_elements, dtype=_DTYPE, device="cuda")
|
||||
expected_bytes = src.cpu().reshape(-1).view(torch.uint8)
|
||||
total_bytes = n_elements * _DTYPE.itemsize
|
||||
|
||||
block_ids = [7, 2, 5, 0][:n_blocks]
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
|
||||
worker.flush_saves()
|
||||
|
||||
for slot, block_idx in enumerate(block_ids):
|
||||
block_byte_start = slot * _BLOCK_SIZE_BYTES
|
||||
block_byte_end = min(block_byte_start + _BLOCK_SIZE_BYTES, total_bytes)
|
||||
n_written = block_byte_end - block_byte_start
|
||||
actual_bytes = worker._region.blocks[block_idx, :n_written].view(torch.uint8)
|
||||
assert torch.equal(
|
||||
actual_bytes, expected_bytes[block_byte_start:block_byte_end]
|
||||
), f"block {block_idx} (slot {slot}) bytes mismatch"
|
||||
if n_written < _BLOCK_SIZE_BYTES:
|
||||
tail = worker._region.blocks[block_idx, n_written:]
|
||||
assert torch.all(tail == sentinel), (
|
||||
f"block {block_idx} tail was overwritten"
|
||||
)
|
||||
|
||||
for idx in set(range(_NUM_BLOCKS)) - set(block_ids):
|
||||
assert torch.all(worker._region.blocks[idx] == sentinel), (
|
||||
f"block {idx} (unassigned) was overwritten"
|
||||
)
|
||||
|
||||
|
||||
def test_save_caches_noop_when_mm_hash_not_in_saves(make_worker):
|
||||
"""When the scheduler hasn't pre-allocated blocks for ``mm_hash``,
|
||||
``save_caches`` + ``flush_saves`` is a pure no-op."""
|
||||
worker = make_worker()
|
||||
sentinel = 0x42
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
|
||||
worker.save_caches({}, "h", _meta(saves={}))
|
||||
worker.flush_saves()
|
||||
|
||||
assert torch.all(worker._region.blocks == sentinel)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_rank,pcp_rank",
|
||||
[(1, 0), (0, 1), (1, 1)],
|
||||
ids=["tp1-pcp0", "tp0-pcp1", "tp1-pcp1"],
|
||||
)
|
||||
def test_save_caches_noop_for_non_save_rank(make_worker, tp_rank, pcp_rank):
|
||||
"""Only TP rank 0 + PCP rank 0 writes to mmap. All other combinations
|
||||
must skip the save entirely."""
|
||||
worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank)
|
||||
sentinel = 0x42
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
|
||||
worker.save_caches(
|
||||
{"h": torch.zeros(8, dtype=_DTYPE)}, "h", _meta(saves={"h": [0]})
|
||||
)
|
||||
worker.flush_saves()
|
||||
|
||||
assert torch.all(worker._region.blocks == sentinel)
|
||||
|
||||
|
||||
def test_save_caches_raises_when_allocated_blocks_too_small(make_worker):
|
||||
"""``save_caches`` must raise ``AssertionError`` when the encoder output
|
||||
is larger than the allocated block space."""
|
||||
worker = make_worker()
|
||||
src = torch.zeros(3 * _HIDDEN_DIM, dtype=_DTYPE)
|
||||
with pytest.raises(AssertionError, match="exceeds allocated blocks"):
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0, 1]}))
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_save_caches_batches_multiple_hashes(make_worker):
|
||||
"""Multiple save_caches calls are batched into a single flush."""
|
||||
worker = make_worker()
|
||||
sentinel = 0x5A
|
||||
worker._region.blocks.fill_(sentinel)
|
||||
|
||||
src_a = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda")
|
||||
src_b = torch.arange(_HIDDEN_DIM, 2 * _HIDDEN_DIM, dtype=_DTYPE, device="cuda")
|
||||
|
||||
cache = {"a": src_a, "b": src_b}
|
||||
worker.save_caches(cache, "a", _meta(saves={"a": [1], "b": [3]}))
|
||||
worker.save_caches(cache, "b", _meta(saves={"a": [1], "b": [3]}))
|
||||
worker.flush_saves()
|
||||
|
||||
expected_a = src_a.cpu().view(torch.uint8)
|
||||
expected_b = src_b.cpu().view(torch.uint8)
|
||||
actual_a = worker._region.blocks[1].view(torch.uint8)
|
||||
actual_b = worker._region.blocks[3].view(torch.uint8)
|
||||
assert torch.equal(actual_a, expected_a)
|
||||
assert torch.equal(actual_b, expected_b)
|
||||
|
||||
# Unassigned blocks untouched.
|
||||
for idx in (0, 2, 4, 5, 6, 7):
|
||||
assert torch.all(worker._region.blocks[idx] == sentinel)
|
||||
|
||||
|
||||
# ── start_load_caches ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_start_load_caches_copies_with_correct_shape_dtype_and_bytes(make_worker):
|
||||
"""Single batched load across all hashes with correct byte→dtype→shape."""
|
||||
worker = make_worker()
|
||||
n_blocks = 3
|
||||
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
|
||||
n_blocks, _HIDDEN_DIM
|
||||
)
|
||||
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
|
||||
|
||||
block_ids = [3, 1, 6]
|
||||
for i, idx in enumerate(block_ids):
|
||||
worker._region.blocks[idx].copy_(src_int8[i])
|
||||
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
|
||||
|
||||
out = encoder_cache["h"]
|
||||
assert out.is_cuda, "consumer worker must place the tensor on the GPU"
|
||||
assert out.shape == (n_blocks, _HIDDEN_DIM)
|
||||
assert out.dtype == _DTYPE
|
||||
assert torch.equal(out.cpu(), src_orig)
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_start_load_caches_preserves_existing_encoder_cache_entry(make_worker):
|
||||
"""If ``encoder_cache`` already holds the ``mm_hash``, the worker must
|
||||
not overwrite it."""
|
||||
worker = make_worker()
|
||||
worker._region.blocks[0].fill_(0x42)
|
||||
|
||||
sentinel = torch.full((_HIDDEN_DIM,), 7.0, dtype=_DTYPE, device="cuda")
|
||||
encoder_cache = {"h": sentinel}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"h": [0]}))
|
||||
|
||||
assert encoder_cache["h"] is sentinel, (
|
||||
"existing encoder_cache entry must not be replaced"
|
||||
)
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_start_load_caches_noop_when_loads_is_empty(make_worker):
|
||||
"""When ``meta.loads`` is empty the early-return must fire."""
|
||||
worker = make_worker()
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={}))
|
||||
|
||||
assert encoder_cache == {}
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_start_load_caches_skips_cached_and_loads_new_in_same_step(make_worker):
|
||||
"""Cached entries are preserved while new ones are loaded."""
|
||||
worker = make_worker()
|
||||
n_blocks = 2
|
||||
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
|
||||
n_blocks, _HIDDEN_DIM
|
||||
)
|
||||
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
|
||||
new_block_ids = [4, 2]
|
||||
for i, idx in enumerate(new_block_ids):
|
||||
worker._region.blocks[idx].copy_(src_int8[i])
|
||||
|
||||
cached_tensor = torch.full((1, _HIDDEN_DIM), 99.0, dtype=_DTYPE, device="cuda")
|
||||
encoder_cache: dict[str, torch.Tensor] = {"cached_h": cached_tensor}
|
||||
worker.start_load_caches(
|
||||
encoder_cache,
|
||||
_meta(loads={"cached_h": [0], "new_h": new_block_ids}),
|
||||
)
|
||||
|
||||
assert encoder_cache["cached_h"] is cached_tensor
|
||||
assert "new_h" in encoder_cache
|
||||
out = encoder_cache["new_h"]
|
||||
assert out.shape == (n_blocks, _HIDDEN_DIM)
|
||||
assert out.dtype == _DTYPE
|
||||
assert torch.equal(out.cpu(), src_orig)
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
@pytest.mark.parametrize(
|
||||
"tp_rank,pcp_rank",
|
||||
[(0, 0), (1, 0), (0, 1), (1, 1)],
|
||||
ids=["tp0-pcp0", "tp1-pcp0", "tp0-pcp1", "tp1-pcp1"],
|
||||
)
|
||||
def test_start_load_caches_works_on_all_ranks(make_worker, tp_rank, pcp_rank):
|
||||
"""All TP/PCP ranks must load from mmap — loads are NOT gated like saves."""
|
||||
worker = make_worker(tp_rank=tp_rank, pcp_rank=pcp_rank)
|
||||
n_blocks = 2
|
||||
src_orig = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE).reshape(
|
||||
n_blocks, _HIDDEN_DIM
|
||||
)
|
||||
src_int8 = src_orig.view(torch.int8).reshape(n_blocks, _BLOCK_SIZE_BYTES)
|
||||
block_ids = [1, 3]
|
||||
for i, idx in enumerate(block_ids):
|
||||
worker._region.blocks[idx].copy_(src_int8[i])
|
||||
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
|
||||
|
||||
out = encoder_cache["h"]
|
||||
assert out.is_cuda
|
||||
assert torch.equal(out.cpu(), src_orig)
|
||||
|
||||
|
||||
# ── round-trip ───────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_save_then_load_round_trips_bytes(make_worker):
|
||||
"""Full producer→mmap→consumer byte path in one shot."""
|
||||
worker = make_worker()
|
||||
n_blocks = 3
|
||||
block_ids = [5, 1, 6]
|
||||
|
||||
src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape(
|
||||
n_blocks, _HIDDEN_DIM
|
||||
)
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": block_ids}))
|
||||
worker.flush_saves()
|
||||
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"h": block_ids}))
|
||||
|
||||
out = encoder_cache["h"]
|
||||
assert out.shape == src.shape
|
||||
assert out.dtype == src.dtype
|
||||
assert torch.equal(out.cpu(), src.cpu())
|
||||
|
||||
|
||||
# ── buffer recycling ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_buffer_pool_is_reused_across_save_steps(make_worker):
|
||||
"""After flush_saves, descriptor buffers are returned to the pool and
|
||||
reused on the next flush — no reallocation."""
|
||||
worker = make_worker()
|
||||
src = torch.arange(_HIDDEN_DIM, dtype=_DTYPE, device="cuda")
|
||||
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": [0]}))
|
||||
worker.flush_saves()
|
||||
|
||||
assert len(worker._buf_pool._pool) == 1
|
||||
buf_id = id(worker._buf_pool._pool[0].src_ptrs)
|
||||
|
||||
# Second step reuses the same buffer.
|
||||
worker.save_caches({"h": src}, "h", _meta(saves={"h": [1]}))
|
||||
worker.flush_saves()
|
||||
|
||||
assert len(worker._buf_pool._pool) == 1
|
||||
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_buffer_pool_is_reused_across_load_steps(make_worker):
|
||||
"""After start_load_caches, descriptor buffers are returned to the pool
|
||||
and reused on the next call."""
|
||||
worker = make_worker()
|
||||
worker._region.blocks[0].fill_(0x01)
|
||||
worker._region.blocks[1].fill_(0x02)
|
||||
|
||||
encoder_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(encoder_cache, _meta(loads={"a": [0]}))
|
||||
|
||||
assert len(worker._buf_pool._pool) == 1
|
||||
buf_id = id(worker._buf_pool._pool[0].src_ptrs)
|
||||
|
||||
encoder_cache2: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(encoder_cache2, _meta(loads={"b": [1]}))
|
||||
|
||||
assert len(worker._buf_pool._pool) == 1
|
||||
assert id(worker._buf_pool._pool[0].src_ptrs) == buf_id
|
||||
|
||||
|
||||
# ── stream management ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_stream_initialized_at_construction(make_worker):
|
||||
"""``_load_stream`` must be a fully initialized CUDA stream."""
|
||||
worker = make_worker()
|
||||
assert isinstance(worker._load_stream, torch.cuda.Stream)
|
||||
|
||||
|
||||
# ── lifecycle ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pin_available,expected_pinned",
|
||||
[
|
||||
(True, True),
|
||||
(False, False),
|
||||
],
|
||||
ids=["available", "unavailable"],
|
||||
)
|
||||
def test_init_pins_memory_when_available(pin_available, expected_pinned):
|
||||
"""Every worker must call ``pin_memory()`` when the platform allows it."""
|
||||
region = _make_region()
|
||||
region.pin_memory = MagicMock() # type: ignore[method-assign]
|
||||
|
||||
pcp_group = Mock()
|
||||
pcp_group.rank_in_group = 0
|
||||
with (
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.create_ec_shared_region",
|
||||
return_value=region,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.is_pin_memory_available",
|
||||
return_value=pin_available,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_tensor_model_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch(
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.worker.get_pcp_group",
|
||||
return_value=pcp_group,
|
||||
),
|
||||
):
|
||||
ECCPUWorker(_vllm_config(rank=0))
|
||||
|
||||
try:
|
||||
if expected_pinned:
|
||||
region.pin_memory.assert_called_once()
|
||||
else:
|
||||
region.pin_memory.assert_not_called()
|
||||
finally:
|
||||
region.cleanup()
|
||||
|
||||
|
||||
def test_shutdown_calls_region_cleanup_and_swallows_errors(caplog_vllm):
|
||||
"""``shutdown`` must always call ``region.cleanup`` — and must never
|
||||
raise."""
|
||||
worker = object.__new__(ECCPUWorker)
|
||||
mock_region = Mock(spec=ECSharedRegion)
|
||||
worker._region = mock_region
|
||||
worker._load_stream = MagicMock()
|
||||
worker._save_bufs = None
|
||||
worker._save_count = 0
|
||||
|
||||
worker.shutdown()
|
||||
worker._load_stream.synchronize.assert_called_once()
|
||||
mock_region.cleanup.assert_called_once()
|
||||
|
||||
mock_region.cleanup.side_effect = RuntimeError("boom")
|
||||
with caplog_vllm.at_level(logging.DEBUG, logger="vllm"):
|
||||
worker.shutdown() # exception must be swallowed
|
||||
|
||||
assert mock_region.cleanup.call_count == 2
|
||||
assert any(
|
||||
"worker region cleanup failed" in r.message
|
||||
for r in caplog_vllm.records
|
||||
if r.levelno == logging.DEBUG
|
||||
)
|
||||
|
||||
|
||||
# ── e2e: scheduler + worker pipeline ────────────────────────────────────────
|
||||
|
||||
|
||||
@_requires_cuda
|
||||
def test_e2e_scheduler_worker_save_then_load(make_worker, monkeypatch):
|
||||
"""Full pipeline: scheduler allocates blocks, worker saves GPU tensor to
|
||||
mmap via flush_saves, scheduler marks ready after step delay, worker
|
||||
loads from mmap back to GPU, and the result matches the original.
|
||||
|
||||
Exercises the real scheduler + worker cooperation through a shared
|
||||
ECSharedRegion, with real CUDA transfers and stream coordination.
|
||||
"""
|
||||
import vllm.distributed.ec_transfer.ec_connector.cpu.scheduler as sched_mod
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
|
||||
ECCPUScheduler,
|
||||
)
|
||||
|
||||
# Build worker first (gets us a real region).
|
||||
worker = make_worker()
|
||||
region = worker._region
|
||||
|
||||
# Build scheduler sharing the same region.
|
||||
monkeypatch.setattr(sched_mod, "create_ec_shared_region", lambda cfg: region)
|
||||
|
||||
class _EC:
|
||||
is_ec_producer = True
|
||||
is_ec_consumer = True
|
||||
|
||||
class _Cfg:
|
||||
ec_transfer_config = _EC()
|
||||
max_concurrent_batches = 1
|
||||
|
||||
scheduler = ECCPUScheduler(_Cfg())
|
||||
|
||||
# -- Step 1: scheduler allocates, worker saves --
|
||||
n_blocks = 3
|
||||
src = torch.arange(n_blocks * _HIDDEN_DIM, dtype=_DTYPE, device="cuda").reshape(
|
||||
n_blocks, _HIDDEN_DIM
|
||||
)
|
||||
|
||||
class _Pos:
|
||||
offset = 0
|
||||
length = n_blocks
|
||||
|
||||
class _Feature:
|
||||
identifier = "img_001"
|
||||
mm_position = _Pos()
|
||||
|
||||
class _Request:
|
||||
request_id = "req_e2e"
|
||||
mm_features = [_Feature()]
|
||||
|
||||
scheduler.update_state_after_alloc(_Request(), 0)
|
||||
meta_save = scheduler.build_connector_meta(scheduler_output=None)
|
||||
assert "img_001" in meta_save.saves
|
||||
|
||||
encoder_cache = {"img_001": src}
|
||||
worker.save_caches(encoder_cache, "img_001", meta_save)
|
||||
worker.flush_saves()
|
||||
|
||||
# -- Step 2: readiness delay (max_concurrent_batches=1) --
|
||||
meta_step2 = scheduler.build_connector_meta(scheduler_output=None)
|
||||
assert scheduler.has_cache_item("img_001") is True
|
||||
assert meta_step2.loads == {} # no load requested yet
|
||||
|
||||
# -- Step 3: scheduler emits load, worker loads --
|
||||
scheduler.update_state_after_alloc(_Request(), 0)
|
||||
meta_load = scheduler.build_connector_meta(scheduler_output=None)
|
||||
assert "img_001" in meta_load.loads
|
||||
assert meta_load.loads["img_001"] == meta_save.saves["img_001"]
|
||||
|
||||
load_cache: dict[str, torch.Tensor] = {}
|
||||
worker.start_load_caches(load_cache, meta_load)
|
||||
|
||||
out = load_cache["img_001"]
|
||||
assert out.is_cuda
|
||||
assert out.shape == src.shape
|
||||
assert out.dtype == src.dtype
|
||||
assert torch.equal(out.cpu(), src.cpu())
|
||||
|
||||
scheduler.shutdown()
|
||||
@@ -0,0 +1,158 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E test for the ECCPUConnector.
|
||||
|
||||
Verifies:
|
||||
- Accuracy: outputs from EC CPU cache match fresh encoder computation.
|
||||
- Latency: loading from EC CPU cache is faster than a cold encoder run.
|
||||
|
||||
Requires a CUDA GPU and the Qwen2-VL-2B-Instruct model.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
import pytest
|
||||
from tqdm import tqdm
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.config import ECTransferConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MODEL = "Qwen/Qwen2-VL-2B-Instruct"
|
||||
EC_CPU_BYTES = 500 << 20 # 500 MiB
|
||||
|
||||
|
||||
def _build_image_prompt(image):
|
||||
"""Build a Qwen2-VL prompt with a single image."""
|
||||
return {
|
||||
"prompt": (
|
||||
"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
"<|vision_start|><|image_pad|><|vision_end|>"
|
||||
"Describe this image briefly.<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
),
|
||||
"multi_modal_data": {"image": image},
|
||||
}
|
||||
|
||||
|
||||
def _make_unique_image(base_image, iteration: int):
|
||||
"""Create a unique image by modifying a pixel to change the mm_hash."""
|
||||
img = base_image.copy()
|
||||
img.putpixel((0, 0), (iteration % 256, iteration // 256 % 256, 0))
|
||||
return img
|
||||
|
||||
|
||||
def _wait_for_ec_ready(llm: LLM) -> None:
|
||||
"""Send a dummy request to force a scheduler step.
|
||||
|
||||
After generate() returns, the EC entry's data is in the mmap but
|
||||
readiness may not have fired yet (first-finish fires in the NEXT
|
||||
scheduler step). This forces that step.
|
||||
"""
|
||||
llm.generate("hi", SamplingParams(max_tokens=1), use_tqdm=False)
|
||||
|
||||
|
||||
def _latency_test(llm: LLM) -> None:
|
||||
"""Verify EC CPU cache hit is faster than cold encoder computation."""
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("Latency test requires CUDA")
|
||||
|
||||
sampling_params = SamplingParams(max_tokens=1, temperature=0)
|
||||
base_image = ImageAsset("stop_sign").pil_image
|
||||
|
||||
num_times_ec_better_than_cold = 0
|
||||
num_tests = 10
|
||||
total_cold_time = 0.0
|
||||
total_ec_hit_time = 0.0
|
||||
|
||||
for i in tqdm(range(num_tests), desc="EC latency test"):
|
||||
# Use a unique image each iteration so the cold run is truly cold
|
||||
# (EC has never seen this mm_hash before).
|
||||
image = _make_unique_image(base_image, i)
|
||||
prompt = _build_image_prompt(image)
|
||||
|
||||
# Cold run: encoder computes from scratch, saves to EC CPU.
|
||||
llm.llm_engine.reset_encoder_cache()
|
||||
start_time = time.time()
|
||||
llm.generate([prompt], sampling_params, use_tqdm=False)
|
||||
cold_time = time.time() - start_time
|
||||
total_cold_time += cold_time
|
||||
|
||||
# Ensure EC entry is marked ready.
|
||||
_wait_for_ec_ready(llm)
|
||||
|
||||
# Clear GPU encoder cache but keep EC CPU cache intact.
|
||||
llm.llm_engine.reset_encoder_cache()
|
||||
|
||||
# EC hit: scheduler loads from CPU mmap instead of re-encoding.
|
||||
start_time = time.time()
|
||||
llm.generate([prompt], sampling_params, use_tqdm=False)
|
||||
ec_hit_time = time.time() - start_time
|
||||
total_ec_hit_time += ec_hit_time
|
||||
|
||||
if ec_hit_time < cold_time:
|
||||
num_times_ec_better_than_cold += 1
|
||||
|
||||
print("Average times:")
|
||||
print(f" Cold: {total_cold_time * 1000 / num_tests:.2f}ms")
|
||||
print(f" EC hit: {total_ec_hit_time * 1000 / num_tests:.2f}ms")
|
||||
|
||||
assert num_times_ec_better_than_cold >= 0.8 * num_tests, (
|
||||
f"EC hit was faster only {num_times_ec_better_than_cold}/{num_tests} "
|
||||
f"times (expected >= 80%)"
|
||||
)
|
||||
|
||||
|
||||
def _accuracy_test(llm: LLM) -> None:
|
||||
"""Verify EC CPU cache produces correct outputs."""
|
||||
sampling_params = SamplingParams(max_tokens=8, temperature=0)
|
||||
image = ImageAsset("stop_sign").pil_image
|
||||
prompt = _build_image_prompt(image)
|
||||
|
||||
# Seed: generate with fresh encoder to establish baseline output.
|
||||
llm.llm_engine.reset_encoder_cache()
|
||||
baseline = llm.generate([prompt], sampling_params, use_tqdm=False)
|
||||
baseline_text = baseline[0].outputs[0].text
|
||||
|
||||
# Ensure EC entry is marked ready.
|
||||
_wait_for_ec_ready(llm)
|
||||
|
||||
# Clear GPU encoder cache so subsequent runs go through EC CPU path.
|
||||
llm.llm_engine.reset_encoder_cache()
|
||||
|
||||
# Generate multiple times from EC cache and check correctness.
|
||||
test_count = 20
|
||||
results = llm.generate([prompt] * test_count, sampling_params, use_tqdm=False)
|
||||
|
||||
success_count = sum(1 for r in results if r.outputs[0].text == baseline_text)
|
||||
assert success_count >= 0.5 * test_count, (
|
||||
f"Only {success_count}/{test_count} outputs matched baseline "
|
||||
f"(expected >= 50%). Baseline: {baseline_text!r}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
def test_ec_cpu_offloading() -> None:
|
||||
"""Tests ECCPUConnector accuracy and latency with a VLM model."""
|
||||
ec_transfer_config = ECTransferConfig(
|
||||
ec_connector="ECCPUConnector",
|
||||
ec_role="ec_both",
|
||||
ec_connector_extra_config={"ec_cpu_bytes": EC_CPU_BYTES},
|
||||
)
|
||||
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
max_model_len=2048,
|
||||
gpu_memory_utilization=0.5,
|
||||
enforce_eager=True,
|
||||
ec_transfer_config=ec_transfer_config,
|
||||
)
|
||||
|
||||
try:
|
||||
_latency_test(llm)
|
||||
_accuracy_test(llm)
|
||||
finally:
|
||||
del llm
|
||||
@@ -0,0 +1,96 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for ECCPUConnectorMetadata."""
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
)
|
||||
|
||||
|
||||
def test_metadata_is_ec_connector_metadata():
|
||||
"""ECCPUConnectorMetadata must be a subclass of ECConnectorMetadata."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
assert isinstance(meta, ECConnectorMetadata)
|
||||
|
||||
|
||||
def test_metadata_initialization():
|
||||
"""ECCPUConnectorMetadata initializes with empty saves and loads dicts."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
assert hasattr(meta, "saves")
|
||||
assert hasattr(meta, "loads")
|
||||
assert meta.saves == {}
|
||||
assert meta.loads == {}
|
||||
|
||||
|
||||
def test_metadata_saves_dict_operations():
|
||||
"""The saves dict supports standard dict operations."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
# Add multiple entries
|
||||
meta.saves["mm_hash_1"] = [0, 1, 2]
|
||||
meta.saves["mm_hash_2"] = [10, 11]
|
||||
|
||||
assert "mm_hash_1" in meta.saves
|
||||
assert "mm_hash_2" in meta.saves
|
||||
assert len(meta.saves) == 2
|
||||
assert meta.saves["mm_hash_1"] == [0, 1, 2]
|
||||
assert meta.saves["mm_hash_2"] == [10, 11]
|
||||
|
||||
|
||||
def test_metadata_loads_dict_operations():
|
||||
"""The loads dict supports standard dict operations."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
# Add multiple entries
|
||||
meta.loads["mm_hash_1"] = [5, 6, 7]
|
||||
meta.loads["mm_hash_2"] = [100]
|
||||
|
||||
assert "mm_hash_1" in meta.loads
|
||||
assert "mm_hash_2" in meta.loads
|
||||
assert len(meta.loads) == 2
|
||||
assert meta.loads["mm_hash_1"] == [5, 6, 7]
|
||||
assert meta.loads["mm_hash_2"] == [100]
|
||||
|
||||
|
||||
def test_metadata_saves_and_loads_are_independent():
|
||||
"""saves and loads dicts are independent."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
meta.saves["key"] = [1, 2]
|
||||
meta.loads["key"] = [3, 4]
|
||||
|
||||
# Same key but different dicts should not interfere
|
||||
assert meta.saves["key"] == [1, 2]
|
||||
assert meta.loads["key"] == [3, 4]
|
||||
|
||||
# Modifying one should not affect the other
|
||||
meta.saves["other"] = [99]
|
||||
assert "other" not in meta.loads
|
||||
|
||||
|
||||
def test_metadata_block_indices_are_integers():
|
||||
"""Block indices in saves/loads lists should be integers."""
|
||||
meta = ECCPUConnectorMetadata()
|
||||
|
||||
meta.saves["hash"] = [0, 100, 999, 65535]
|
||||
meta.loads["hash"] = [1, 2, 3]
|
||||
|
||||
for idx in meta.saves["hash"]:
|
||||
assert isinstance(idx, int)
|
||||
for idx in meta.loads["hash"]:
|
||||
assert isinstance(idx, int)
|
||||
|
||||
|
||||
def test_metadata_multiple_instances_are_independent():
|
||||
"""Different ECCPUConnectorMetadata instances should not share state."""
|
||||
meta1 = ECCPUConnectorMetadata()
|
||||
meta2 = ECCPUConnectorMetadata()
|
||||
|
||||
meta1.saves["x"] = [1]
|
||||
meta2.saves["y"] = [2]
|
||||
|
||||
assert "x" in meta1.saves
|
||||
assert "x" not in meta2.saves
|
||||
assert "y" not in meta1.saves
|
||||
assert "y" in meta2.saves
|
||||
@@ -0,0 +1,89 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared types for the ECCPUConnector scheduler and worker delegates."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import ECConnectorMetadata
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.ec_shared_region import (
|
||||
ECSharedRegion,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class ECCPUConnectorMetadata(ECConnectorMetadata):
|
||||
"""Per-step scheduler → worker payload for the ECCPUConnector.
|
||||
|
||||
Populated by `ECCPUScheduler.build_connector_meta`; consumed by
|
||||
`ECCPUWorker` via the mixin's `bind_connector_metadata`.
|
||||
"""
|
||||
|
||||
# Producer role: mm_hashes the scheduler has just allocated CPU
|
||||
# blocks for this step; the worker's save_caches copies
|
||||
# encoder_cache[mm_hash] → mmap at these block IDs.
|
||||
saves: dict[str, list[int]] = field(default_factory=dict)
|
||||
|
||||
# Consumer role: mm_hashes whose bytes are available in the local mmap;
|
||||
# the worker's start_load_caches copies mmap[block_ids] → GPU
|
||||
# encoder_cache.
|
||||
loads: dict[str, list[int]] = field(default_factory=dict)
|
||||
|
||||
|
||||
def _get_encoder_cache_hidden_dim(vllm_config: "VllmConfig") -> int:
|
||||
"""Return the per-token hidden dimension for encoder cache entries.
|
||||
|
||||
For most models this equals the LLM's hidden size. Qwen3-VL (and any
|
||||
future model with deepstack visual encoding) is an exception: the ViT
|
||||
concatenates its own output with features from N decoder layers before
|
||||
storing in encoder_cache, producing a tensor of width
|
||||
``out_hidden_size * (1 + N)`` per visual token. Using the plain LLM
|
||||
hidden size would under-allocate EC blocks and silently truncate the
|
||||
transferred data, leading to a shape mismatch on the consumer.
|
||||
"""
|
||||
model_config = vllm_config.model_config
|
||||
hf_config = getattr(model_config, "hf_config", None)
|
||||
vision_config = (
|
||||
getattr(hf_config, "vision_config", None) if hf_config is not None else None
|
||||
)
|
||||
if vision_config is not None:
|
||||
out_hidden_size = getattr(vision_config, "out_hidden_size", None)
|
||||
deepstack_indexes = getattr(vision_config, "deepstack_visual_indexes", None)
|
||||
if out_hidden_size is not None and deepstack_indexes:
|
||||
return out_hidden_size * (1 + len(deepstack_indexes))
|
||||
return model_config.get_inputs_embeds_size()
|
||||
|
||||
|
||||
def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion:
|
||||
"""Build the EC mmap region from `vllm_config`.
|
||||
|
||||
Both `ECCPUScheduler` and `ECCPUWorker` call this to get the same
|
||||
shared region (same engine_id, same block_size_bytes).
|
||||
"""
|
||||
ec_config = vllm_config.ec_transfer_config
|
||||
assert ec_config is not None, "ec_transfer_config required to build region"
|
||||
|
||||
dp_rank = vllm_config.parallel_config.data_parallel_rank
|
||||
engine_id = f"{vllm_config.instance_id}_dp{dp_rank}"
|
||||
|
||||
dtype = vllm_config.model_config.dtype
|
||||
hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
|
||||
element_size = torch.empty(0, dtype=dtype).element_size()
|
||||
block_size_bytes = hidden_dim * element_size
|
||||
|
||||
cpu_bytes = ec_config.ec_connector_extra_config.get("ec_cpu_bytes")
|
||||
if not cpu_bytes:
|
||||
raise ValueError("ec_cpu_bytes must be specified in ec_connector_extra_config")
|
||||
cpu_bytes = int(cpu_bytes)
|
||||
num_blocks = cpu_bytes // block_size_bytes
|
||||
|
||||
return ECSharedRegion(
|
||||
engine_id=engine_id,
|
||||
num_blocks=num_blocks,
|
||||
block_size_bytes=block_size_bytes,
|
||||
)
|
||||
@@ -0,0 +1,121 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ECCPUConnector — CPU encoder-cache offloading.
|
||||
|
||||
A role-routed shell: one instance per process. The scheduler delegate owns the
|
||||
mmap region and offload bookkeeping; the worker delegate owns the mmap view and
|
||||
GPU copy plumbing. An ec_both instance reuses encoder outputs it has already
|
||||
offloaded to CPU instead of recomputing them.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorBase,
|
||||
ECConnectorRole,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ECCPUConnector(ECConnectorBase):
|
||||
"""EC connector that offloads encoder cache to a shared CPU mmap region."""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", role: ECConnectorRole) -> None:
|
||||
super().__init__(vllm_config=vllm_config, role=role)
|
||||
|
||||
self.connector_worker = None
|
||||
self.connector_scheduler = None
|
||||
|
||||
if role == ECConnectorRole.WORKER:
|
||||
self.connector_worker = self._make_worker(vllm_config)
|
||||
elif role == ECConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = self._make_scheduler(vllm_config)
|
||||
else:
|
||||
raise ValueError(f"Unknown ECConnectorRole: {role}")
|
||||
|
||||
# Construction seams.
|
||||
def _make_worker(self, vllm_config: "VllmConfig"):
|
||||
# Deferred import: the worker module touches torch/CUDA at import time
|
||||
# via the region, so keep that cost off the scheduler path.
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.worker import (
|
||||
ECCPUWorker,
|
||||
)
|
||||
|
||||
return ECCPUWorker(vllm_config)
|
||||
|
||||
def _make_scheduler(self, vllm_config: "VllmConfig"):
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler import (
|
||||
ECCPUScheduler,
|
||||
)
|
||||
|
||||
return ECCPUScheduler(vllm_config)
|
||||
|
||||
# Worker-side forwarders.
|
||||
def start_load_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], **kwargs
|
||||
) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, ECCPUConnectorMetadata)
|
||||
self.connector_worker.start_load_caches(
|
||||
encoder_cache, connector_metadata=metadata
|
||||
)
|
||||
|
||||
def save_caches(
|
||||
self, encoder_cache: dict[str, torch.Tensor], mm_hash: str, **kwargs
|
||||
) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, ECCPUConnectorMetadata)
|
||||
self.connector_worker.save_caches(
|
||||
encoder_cache, mm_hash, connector_metadata=metadata
|
||||
)
|
||||
|
||||
# Scheduler-side forwarders.
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.has_cache_item(identifier)
|
||||
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.ensure_cache_available(
|
||||
request, num_computed_tokens
|
||||
)
|
||||
|
||||
def update_state_after_alloc(self, request: "Request", index: int) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_state_after_alloc(request, index)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> ECCPUConnectorMetadata:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.flush_saves()
|
||||
return None, None
|
||||
|
||||
# Shared.
|
||||
def shutdown(self) -> None:
|
||||
if self.connector_scheduler is not None:
|
||||
self.connector_scheduler.shutdown()
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.shutdown()
|
||||
@@ -0,0 +1,159 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Lightweight mmap-backed shared memory region for encoder cache (EC) data.
|
||||
|
||||
Modeled after SharedOffloadRegion (vllm/v1/kv_offload/cpu/) but simplified
|
||||
for EC: flat shared layout, no multi-tensor cursor, no block_size_factor.
|
||||
"""
|
||||
|
||||
import mmap
|
||||
import os
|
||||
import time
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _wait_for_file_size(fd: int, expected_size: int, timeout: float = 30.0):
|
||||
"""Spin-wait until the file reaches expected_size (creator truncated it)."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
if os.fstat(fd).st_size >= expected_size:
|
||||
return
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for EC mmap file to reach {expected_size} bytes"
|
||||
)
|
||||
time.sleep(0.005)
|
||||
|
||||
|
||||
class ECSharedRegion:
|
||||
"""Flat mmap-backed memory region shared across TP workers for
|
||||
encoder cache blocks.
|
||||
|
||||
Layout: (num_blocks, block_size_bytes) — contiguous, no per-worker
|
||||
interleaving. All workers map the same file and see identical data.
|
||||
|
||||
File path: /dev/shm/vllm_ec_{engine_id}.mmap
|
||||
|
||||
This class owns only the shared memory substrate (mmap lifecycle, the
|
||||
`blocks` view, CUDA host registration). Block allocation and eviction
|
||||
are tracked by `EmbeddingCache` in the scheduler process.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_id: str,
|
||||
num_blocks: int,
|
||||
block_size_bytes: int,
|
||||
) -> None:
|
||||
self.num_blocks = num_blocks
|
||||
self.block_size_bytes = block_size_bytes
|
||||
|
||||
total_size_bytes = num_blocks * block_size_bytes
|
||||
# Path in /dev/shm (tmpfs); unique per engine instance.
|
||||
self._mmap_path = f"/dev/shm/vllm_ec_{engine_id}.mmap"
|
||||
# True for the process that created the file (responsible for unlink).
|
||||
self._is_creator = False
|
||||
# True after successful cudaHostRegister (cleanup must unregister).
|
||||
self._is_pinned = False
|
||||
|
||||
# File descriptor for the shared memory backing file.
|
||||
try:
|
||||
self._fd: int | None = os.open(
|
||||
self._mmap_path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o600
|
||||
)
|
||||
os.ftruncate(self._fd, total_size_bytes)
|
||||
self._is_creator = True
|
||||
logger.info(
|
||||
"Created EC mmap file %s (%.2f MiB)",
|
||||
self._mmap_path,
|
||||
total_size_bytes / (1 << 20),
|
||||
)
|
||||
except FileExistsError:
|
||||
self._fd = os.open(self._mmap_path, os.O_RDWR)
|
||||
try:
|
||||
_wait_for_file_size(self._fd, total_size_bytes)
|
||||
except Exception:
|
||||
os.close(self._fd)
|
||||
self._fd = None
|
||||
raise
|
||||
logger.info("Opened existing EC mmap file %s", self._mmap_path)
|
||||
|
||||
# MAP_SHARED mmap over _fd; all processes see the same pages.
|
||||
self._mmap_obj: mmap.mmap | None = mmap.mmap(
|
||||
self._fd,
|
||||
total_size_bytes,
|
||||
flags=mmap.MAP_SHARED,
|
||||
prot=mmap.PROT_READ | mmap.PROT_WRITE,
|
||||
)
|
||||
|
||||
if self._is_creator:
|
||||
_MADV_POPULATE_WRITE = getattr(mmap, "MADV_POPULATE_WRITE", 23)
|
||||
self._mmap_obj.madvise(_MADV_POPULATE_WRITE, 0, total_size_bytes)
|
||||
|
||||
# (num_blocks, block_size_bytes) int8 tensor over the mmap buffer.
|
||||
self.blocks: torch.Tensor = torch.frombuffer(
|
||||
memoryview(self._mmap_obj), dtype=torch.int8
|
||||
).view(num_blocks, block_size_bytes)
|
||||
# Cached for cudaHostRegister/Unregister and pointer math.
|
||||
self._blocks_ptr: int = self.blocks.data_ptr()
|
||||
self._blocks_nbytes: int = self.blocks.nbytes
|
||||
|
||||
def pin_memory(self) -> None:
|
||||
"""Register the entire mmap as CUDA pinned memory for fast DMA.
|
||||
|
||||
Each TP worker process owns its own virtual address for the same
|
||||
shared physical pages, so every process must register independently.
|
||||
No-op when CUDA is not available or already pinned.
|
||||
"""
|
||||
if self._is_pinned or not torch.cuda.is_available():
|
||||
return
|
||||
result = torch.cuda.cudart().cudaHostRegister(
|
||||
self._blocks_ptr, self._blocks_nbytes, 0
|
||||
)
|
||||
if result.value != 0:
|
||||
logger.warning(
|
||||
"cudaHostRegister failed (code=%d) — "
|
||||
"transfers will still work but may be slower (unpinned DMA)",
|
||||
result.value,
|
||||
)
|
||||
else:
|
||||
logger.debug("cudaHostRegister %.2f MB", self._blocks_nbytes / 1e6)
|
||||
self._is_pinned = True
|
||||
|
||||
def cleanup(self) -> None:
|
||||
"""Tear down the region. Lifecycle method; no concurrent access."""
|
||||
logger.info("Starting ECSharedRegion cleanup...")
|
||||
if self._is_creator:
|
||||
try:
|
||||
os.unlink(self._mmap_path)
|
||||
logger.info("Removed EC mmap file %s", self._mmap_path)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Failed to unlink path %s", self._mmap_path, exc_info=True
|
||||
)
|
||||
self._is_creator = False
|
||||
if self._is_pinned:
|
||||
result = torch.cuda.cudart().cudaHostUnregister(self._blocks_ptr)
|
||||
if result.value != 0:
|
||||
logger.warning("cudaHostUnregister failed (code=%d)", result)
|
||||
self._is_pinned = False
|
||||
if hasattr(self, "blocks"):
|
||||
del self.blocks
|
||||
if self._mmap_obj:
|
||||
try:
|
||||
self._mmap_obj.close()
|
||||
except Exception:
|
||||
logger.warning("Failed to close mmap_obj", exc_info=True)
|
||||
self._mmap_obj = None
|
||||
if self._fd is not None:
|
||||
try:
|
||||
os.close(self._fd)
|
||||
except Exception:
|
||||
logger.warning("Failed to close fd %s", self._fd, exc_info=True)
|
||||
self._fd = None
|
||||
@@ -0,0 +1,122 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ECCPUScheduler — CPU offload scheduler delegate.
|
||||
|
||||
Owns the mmap region and the embedding cache, and handles the producer
|
||||
(GPU->CPU offload) and consumer (CPU->GPU reload) scheduler-side logic
|
||||
for the ECCPUConnector.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
create_ec_shared_region,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.step_tracker import (
|
||||
StepTracker,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ECCPUScheduler:
|
||||
"""Scheduler delegate for the ECCPUConnector."""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig") -> None:
|
||||
ec_config = vllm_config.ec_transfer_config
|
||||
assert ec_config is not None
|
||||
self._is_producer: bool = ec_config.is_ec_producer
|
||||
self._is_consumer: bool = ec_config.is_ec_consumer
|
||||
|
||||
self._region = create_ec_shared_region(vllm_config)
|
||||
# Block allocator + LRU eviction policy for the shared region.
|
||||
self._cache = EmbeddingCache(self._region.num_blocks)
|
||||
|
||||
max_batches = vllm_config.max_concurrent_batches
|
||||
# Delays mark_ready until the GPU→mmap DMA is guaranteed complete.
|
||||
self._ready_tracker = StepTracker(max_batches)
|
||||
# Delays unpin until the mmap→GPU DMA is guaranteed complete.
|
||||
self._unpin_tracker = StepTracker(max_batches)
|
||||
|
||||
# mm_hash → block IDs allocated this step for GPU→mmap saves.
|
||||
self._pending_saves: dict[str, list[int]] = {}
|
||||
# mm_hash → block IDs to load from mmap→GPU this step.
|
||||
self._pending_loads: dict[str, list[int]] = {}
|
||||
|
||||
def has_cache_item(self, identifier: str) -> bool:
|
||||
if not self._is_consumer:
|
||||
return False
|
||||
entry = self._cache.get(identifier)
|
||||
return entry is not None and entry.ready
|
||||
|
||||
def ensure_cache_available(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> bool:
|
||||
return True
|
||||
|
||||
def update_state_after_alloc(self, request: "Request", index: int) -> None:
|
||||
feature = request.mm_features[index]
|
||||
mm_hash = feature.identifier
|
||||
|
||||
if self._is_producer and self._cache.get(mm_hash) is None:
|
||||
entry = self._cache.alloc(mm_hash, feature.mm_position.length)
|
||||
if entry is not None:
|
||||
self._pending_saves[mm_hash] = list(entry.block_ids)
|
||||
self._ready_tracker.add(mm_hash, request.request_id)
|
||||
|
||||
if self._is_consumer and mm_hash not in self._pending_loads:
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and entry.ready:
|
||||
self._cache.pin(mm_hash)
|
||||
self._pending_loads[mm_hash] = list(entry.block_ids)
|
||||
self._unpin_tracker.add(mm_hash, request.request_id)
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: "SchedulerOutput"
|
||||
) -> ECCPUConnectorMetadata:
|
||||
finished = (
|
||||
scheduler_output.finished_req_ids if scheduler_output is not None else set()
|
||||
)
|
||||
|
||||
for key in self._ready_tracker.step(finished):
|
||||
entry = self._cache.get(key)
|
||||
if entry is not None and not entry.ready:
|
||||
self._cache.mark_ready(key)
|
||||
|
||||
for key in self._unpin_tracker.step(finished):
|
||||
self._cache.unpin(key)
|
||||
|
||||
meta = ECCPUConnectorMetadata()
|
||||
if self._is_producer:
|
||||
meta.saves = self._pending_saves
|
||||
self._pending_saves = {}
|
||||
if self._is_consumer:
|
||||
meta.loads = self._pending_loads
|
||||
self._pending_loads = {}
|
||||
return meta
|
||||
|
||||
def shutdown(self) -> None:
|
||||
# drain_all() covers both entries still in _current (never
|
||||
# consumed by build_connector_meta) and entries in slots.
|
||||
self._pending_loads.clear()
|
||||
for mm_hash in self._unpin_tracker.drain_all():
|
||||
self._cache.unpin(mm_hash)
|
||||
self._ready_tracker.drain_all()
|
||||
|
||||
self._is_producer = False
|
||||
self._is_consumer = False
|
||||
|
||||
try:
|
||||
self._region.cleanup()
|
||||
except Exception:
|
||||
logger.debug("ec: region cleanup failed", exc_info=True)
|
||||
@@ -0,0 +1,131 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""EmbeddingCache — named-entry block cache with FIFO eviction.
|
||||
|
||||
Manages a fixed pool of block IDs keyed by content identity (mm_hash).
|
||||
Entries transition through: not-ready → ready (evictable) → pinned.
|
||||
Eviction targets ready + unpinned entries in FIFO order.
|
||||
|
||||
All public methods are thread-safe.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
class CacheEntry:
|
||||
"""A single cache entry. Read `.block_ids` and `.ready` freely;
|
||||
mutations only through EmbeddingCache methods."""
|
||||
|
||||
__slots__ = ("block_ids", "_pin_count")
|
||||
|
||||
def __init__(self, block_ids: tuple[int, ...]) -> None:
|
||||
self.block_ids = block_ids
|
||||
self._pin_count = -1 # not ready
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
return self._pin_count >= 0
|
||||
|
||||
@property
|
||||
def evictable(self) -> bool:
|
||||
return self._pin_count == 0
|
||||
|
||||
def mark_ready(self):
|
||||
self._pin_count = 0
|
||||
|
||||
def pin(self):
|
||||
self._pin_count += 1
|
||||
|
||||
|
||||
class EmbeddingCache:
|
||||
"""Fixed-size block cache with FIFO eviction.
|
||||
|
||||
Entries are keyed by content identity (e.g. mm_hash). Blocks are
|
||||
allocated from a free-set; when space is needed, ready + unpinned
|
||||
entries are evicted oldest-first.
|
||||
|
||||
The caller is responsible for deciding *when* to call ``mark_ready``
|
||||
(e.g. after enough engine steps have elapsed for the worker to have
|
||||
completed the write).
|
||||
"""
|
||||
|
||||
def __init__(self, num_blocks: int) -> None:
|
||||
self._num_blocks = num_blocks
|
||||
self._lock = threading.Lock()
|
||||
# Available block IDs, LIFO stack.
|
||||
self._free_blocks: list[int] = list(range(num_blocks))
|
||||
# Key → live entry (all states: not-ready, ready, pinned).
|
||||
self._entries: dict[str, CacheEntry] = {}
|
||||
# Ready + unpinned entries in insertion order (FIFO eviction).
|
||||
self._entries_free_list: OrderedDict[str, None] = OrderedDict()
|
||||
# Total blocks reclaimable via eviction (fail-fast check).
|
||||
self._evictable_block_count: int = 0
|
||||
|
||||
def get(self, key: str) -> CacheEntry | None:
|
||||
"""Return the entry for *key*, or None if not present."""
|
||||
with self._lock:
|
||||
return self._entries.get(key)
|
||||
|
||||
def alloc(self, key: str, n_blocks: int) -> CacheEntry | None:
|
||||
"""Allocate *n_blocks* for *key*, evicting as needed.
|
||||
|
||||
The entry starts not-ready. Returns None if there is not enough
|
||||
space even after evicting all evictable entries.
|
||||
"""
|
||||
with self._lock:
|
||||
assert key not in self._entries, (
|
||||
f"EmbeddingCache: duplicate alloc for {key!r}"
|
||||
)
|
||||
assert n_blocks <= self._num_blocks, (
|
||||
f"EmbeddingCache: {n_blocks} blocks requested but capacity "
|
||||
f"is {self._num_blocks}"
|
||||
)
|
||||
if len(self._free_blocks) + self._evictable_block_count < n_blocks:
|
||||
return None
|
||||
self._evict_until(n_blocks)
|
||||
block_ids = tuple(self._free_blocks.pop() for _ in range(n_blocks))
|
||||
entry = CacheEntry(block_ids)
|
||||
self._entries[key] = entry
|
||||
return entry
|
||||
|
||||
def mark_ready(self, key: str) -> None:
|
||||
"""Mark an entry as ready (data is CPU-visible)."""
|
||||
with self._lock:
|
||||
entry = self._entries[key]
|
||||
assert entry._pin_count == -1, (
|
||||
f"EmbeddingCache: mark_ready on already-ready entry {key!r}"
|
||||
)
|
||||
entry.mark_ready()
|
||||
self._entries_free_list[key] = None
|
||||
self._evictable_block_count += len(entry.block_ids)
|
||||
|
||||
def pin(self, key: str) -> None:
|
||||
"""Pin an entry (prevent eviction)."""
|
||||
with self._lock:
|
||||
entry = self._entries[key]
|
||||
assert entry.ready, f"EmbeddingCache: pin of not-ready entry {key!r}"
|
||||
if entry.evictable:
|
||||
del self._entries_free_list[key]
|
||||
self._evictable_block_count -= len(entry.block_ids)
|
||||
entry.pin()
|
||||
|
||||
def unpin(self, key: str) -> None:
|
||||
"""Unpin an entry. Asserts currently pinned."""
|
||||
with self._lock:
|
||||
entry = self._entries[key]
|
||||
assert entry._pin_count > 0, (
|
||||
f"EmbeddingCache: unpin of unpinned entry {key!r}"
|
||||
)
|
||||
entry._pin_count -= 1
|
||||
if entry._pin_count == 0:
|
||||
self._entries_free_list[key] = None
|
||||
self._evictable_block_count += len(entry.block_ids)
|
||||
|
||||
def _evict_until(self, n_blocks: int) -> None:
|
||||
"""Evict ready+unpinned entries FIFO until enough space. Lock held."""
|
||||
while len(self._free_blocks) < n_blocks:
|
||||
key, _ = self._entries_free_list.popitem(last=False)
|
||||
entry = self._entries.pop(key)
|
||||
self._evictable_block_count -= len(entry.block_ids)
|
||||
self._free_blocks.extend(entry.block_ids)
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""StepTracker — step-counting delay with first-finish fast-path.
|
||||
|
||||
Manages a collection of (mm_hash, request_id) entries that need to be
|
||||
processed after a configurable number of engine steps have elapsed.
|
||||
Additionally, if the originating request finishes before the step-count
|
||||
expires, the entry is processed immediately (the GPU transfer is
|
||||
guaranteed complete by request finish time).
|
||||
|
||||
Used by ECCPUScheduler: one instance for readiness marking, one for
|
||||
unpins.
|
||||
"""
|
||||
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _PendingEntry:
|
||||
mm_hash: str
|
||||
request_id: str
|
||||
processed: bool = False
|
||||
|
||||
|
||||
class StepTracker:
|
||||
"""Track pending operations with step-count delay and first-finish.
|
||||
|
||||
Each ``add(mm_hash, request_id)`` registers an operation that will be
|
||||
returned from ``step()`` either when:
|
||||
(a) ``max_concurrent_batches`` steps elapse (deque expiry), OR
|
||||
(b) ``request_id`` appears in ``finished_req_ids`` (first-finish).
|
||||
|
||||
Each add is processed exactly once — never both (a) and (b).
|
||||
"""
|
||||
|
||||
def __init__(self, max_concurrent_batches: int) -> None:
|
||||
# One slot per past step; oldest at the right. When the deque is
|
||||
# full the rightmost slot is popped and its entries expire.
|
||||
self._slots: deque[list[_PendingEntry]] = deque(maxlen=max_concurrent_batches)
|
||||
# Reverse index: request_id → entries across all slots + _current.
|
||||
# Enables O(1) lookup for the first-finish fast-path.
|
||||
self._req_index: dict[str, list[_PendingEntry]] = {}
|
||||
# Entries added this step (via add()), committed to a new slot at
|
||||
# the start of the next step() call.
|
||||
self._current: list[_PendingEntry] = []
|
||||
|
||||
def add(self, mm_hash: str, request_id: str) -> None:
|
||||
"""Register an mm_hash needing processing after a delay."""
|
||||
entry = _PendingEntry(mm_hash=mm_hash, request_id=request_id)
|
||||
self._current.append(entry)
|
||||
self._req_index.setdefault(request_id, []).append(entry)
|
||||
|
||||
def step(self, finished_req_ids: set[str]) -> list[str]:
|
||||
"""Advance one step. Return mm_hashes ready for processing.
|
||||
|
||||
Returns:
|
||||
List of mm_hashes to process. May contain duplicates if the
|
||||
same mm_hash was added multiple times (from different requests).
|
||||
"""
|
||||
result: list[str] = []
|
||||
|
||||
# Phase 1: first-finish fast-path.
|
||||
for req_id in finished_req_ids:
|
||||
entries = self._req_index.pop(req_id, None)
|
||||
if entries is None:
|
||||
continue
|
||||
for entry in entries:
|
||||
if not entry.processed:
|
||||
entry.processed = True
|
||||
result.append(entry.mm_hash)
|
||||
|
||||
# Phase 2: step-count expiry.
|
||||
if len(self._slots) == self._slots.maxlen:
|
||||
expired_slot = self._slots.pop()
|
||||
for entry in expired_slot:
|
||||
if not entry.processed:
|
||||
entry.processed = True
|
||||
result.append(entry.mm_hash)
|
||||
self._cleanup_expired(expired_slot)
|
||||
|
||||
# Phase 3: commit current entries to new slot.
|
||||
self._slots.appendleft(self._current)
|
||||
self._current = []
|
||||
|
||||
return result
|
||||
|
||||
def drain_all(self) -> list[str]:
|
||||
"""Return all unprocessed entries and clear internal state."""
|
||||
result: list[str] = []
|
||||
for entry in self._current:
|
||||
if not entry.processed:
|
||||
entry.processed = True
|
||||
result.append(entry.mm_hash)
|
||||
self._current = []
|
||||
|
||||
for slot in self._slots:
|
||||
for entry in slot:
|
||||
if not entry.processed:
|
||||
entry.processed = True
|
||||
result.append(entry.mm_hash)
|
||||
self._slots.clear()
|
||||
self._req_index.clear()
|
||||
return result
|
||||
|
||||
def _cleanup_expired(self, slot: list[_PendingEntry]) -> None:
|
||||
"""Prune reverse index for entries in an expired slot."""
|
||||
for entry in slot:
|
||||
entries_for_req = self._req_index.get(entry.request_id)
|
||||
if entries_for_req is None:
|
||||
# Already removed by fast-path (pop in step()).
|
||||
continue
|
||||
entries_for_req.remove(entry)
|
||||
if not entries_for_req:
|
||||
del self._req_index[entry.request_id]
|
||||
@@ -0,0 +1,202 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Worker-side of the ECCPUConnector.
|
||||
|
||||
Thin, stateless across steps: opens the shared mmap region and uses the
|
||||
per-step connector metadata (`ECCPUConnectorMetadata`) to decide which
|
||||
blocks to copy in each direction.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._custom_ops import swap_blocks_batch
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
create_ec_shared_region,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.worker.descriptor_buffers import (
|
||||
DescriptorBufferPool,
|
||||
DescriptorBuffers,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_pcp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ECCPUWorker:
|
||||
"""Worker-side delegate for the ECCPUConnector.
|
||||
|
||||
- Producer role: copies `encoder_cache[mm_hash]` → `mmap[block_ids]`
|
||||
for each entry in `metadata.saves`. Descriptor buffers are filled
|
||||
directly in `save_caches`; the actual DMA is issued as a single
|
||||
batched call in `flush_saves`.
|
||||
- Consumer role: copies `mmap[block_ids]` → `encoder_cache[mm_hash]`
|
||||
for all entries in `metadata.loads` via a single `swap_blocks_batch`
|
||||
call on the load stream.
|
||||
- On `ec_both` nodes both paths run back-to-back in a single step.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig") -> None:
|
||||
self._region = create_ec_shared_region(vllm_config)
|
||||
# Model dtype; used to reinterpret raw int8 blocks on load.
|
||||
self._dtype = vllm_config.model_config.dtype
|
||||
|
||||
if is_pin_memory_available():
|
||||
self._region.pin_memory()
|
||||
|
||||
# All TP/PCP ranks hold identical encoder output. Only one rank
|
||||
# per mmap needs to write — saves host memory bandwidth.
|
||||
# DCP is a subdivision of TP, so tp_rank==0 covers it.
|
||||
self._is_save_rank = (
|
||||
get_tensor_model_parallel_rank() == 0 and get_pcp_group().rank_in_group == 0
|
||||
)
|
||||
|
||||
# Dedicated stream for async mmap→GPU loads (overlaps with compute).
|
||||
self._load_stream = current_platform.Stream()
|
||||
|
||||
# Descriptor buffer pool (recycled across steps, shared by both paths).
|
||||
self._buf_pool = DescriptorBufferPool()
|
||||
|
||||
# Active save buffer being filled during save_caches calls this step.
|
||||
self._save_bufs: DescriptorBuffers | None = None
|
||||
self._save_count: int = 0
|
||||
|
||||
def save_caches(
|
||||
self,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
mm_hash: str,
|
||||
connector_metadata: ECCPUConnectorMetadata,
|
||||
) -> None:
|
||||
"""Fill descriptor buffers directly for batched flush."""
|
||||
if not self._is_save_rank:
|
||||
return
|
||||
block_ids = connector_metadata.saves.get(mm_hash)
|
||||
if block_ids is None:
|
||||
return
|
||||
|
||||
src = encoder_cache[mm_hash]
|
||||
total_bytes = src.numel() * src.element_size()
|
||||
block_size = self._region.block_size_bytes
|
||||
allocated_bytes = len(block_ids) * block_size
|
||||
assert total_bytes <= allocated_bytes, (
|
||||
f"EC: encoder output exceeds allocated blocks for mm_hash={mm_hash}: "
|
||||
f"{total_bytes} bytes but only {allocated_bytes} allocated "
|
||||
f"({len(block_ids)} blocks × {block_size}). "
|
||||
f"shape={list(src.shape)}"
|
||||
)
|
||||
|
||||
if self._save_bufs is None:
|
||||
total = sum(len(v) for v in connector_metadata.saves.values())
|
||||
self._save_bufs = self._buf_pool.acquire(total)
|
||||
|
||||
assert self._save_count + len(block_ids) <= self._save_bufs.src_ptrs.numel()
|
||||
|
||||
src_ptrs, dst_ptrs, sizes = self._save_bufs
|
||||
src_base = src.view(-1).view(torch.uint8).data_ptr()
|
||||
dst_base = self._region.blocks.data_ptr()
|
||||
idx = self._save_count
|
||||
|
||||
for i, block_idx in enumerate(block_ids):
|
||||
start = i * block_size
|
||||
src_ptrs[idx] = src_base + start
|
||||
dst_ptrs[idx] = dst_base + block_idx * block_size
|
||||
sizes[idx] = min(block_size, total_bytes - start)
|
||||
idx += 1
|
||||
|
||||
self._save_count = idx
|
||||
|
||||
def flush_saves(self) -> None:
|
||||
"""Flush all accumulated saves in a single swap_blocks_batch call."""
|
||||
if self._save_count == 0:
|
||||
return
|
||||
|
||||
bufs = self._save_bufs
|
||||
assert bufs is not None
|
||||
src_ptrs, dst_ptrs, sizes = bufs
|
||||
n = self._save_count
|
||||
swap_blocks_batch(src_ptrs[:n], dst_ptrs[:n], sizes[:n])
|
||||
|
||||
self._buf_pool.release(bufs)
|
||||
self._save_bufs = None
|
||||
self._save_count = 0
|
||||
|
||||
def start_load_caches(
|
||||
self,
|
||||
encoder_cache: dict[str, torch.Tensor],
|
||||
connector_metadata: ECCPUConnectorMetadata,
|
||||
) -> None:
|
||||
"""Consumer path: single batched copy of all loads from mmap→GPU."""
|
||||
if not connector_metadata.loads:
|
||||
return
|
||||
|
||||
block_size = self._region.block_size_bytes
|
||||
blocks = self._region.blocks
|
||||
dtype = self._dtype
|
||||
device_type = current_platform.device_type
|
||||
src_base = blocks.data_ptr()
|
||||
|
||||
# Pre-filter: only hashes not already in encoder_cache.
|
||||
load_items = {
|
||||
h: idxs
|
||||
for h, idxs in connector_metadata.loads.items()
|
||||
if h not in encoder_cache
|
||||
}
|
||||
if not load_items:
|
||||
return
|
||||
|
||||
total_blocks = sum(len(idxs) for idxs in load_items.values())
|
||||
|
||||
with current_platform.stream(self._load_stream):
|
||||
# Single contiguous destination buffer for all loads.
|
||||
dst_buf = torch.empty(
|
||||
total_blocks, block_size, dtype=torch.int8, device=device_type
|
||||
)
|
||||
dst_buf_base = dst_buf.data_ptr()
|
||||
|
||||
bufs = self._buf_pool.acquire(total_blocks)
|
||||
src_ptrs = bufs.src_ptrs[:total_blocks]
|
||||
dst_ptrs = bufs.dst_ptrs[:total_blocks]
|
||||
sizes = bufs.sizes[:total_blocks]
|
||||
sizes[:] = block_size
|
||||
|
||||
op_idx = 0
|
||||
for block_ids in load_items.values():
|
||||
for block_idx in block_ids:
|
||||
src_ptrs[op_idx] = src_base + block_idx * block_size
|
||||
dst_ptrs[op_idx] = dst_buf_base + op_idx * block_size
|
||||
op_idx += 1
|
||||
|
||||
swap_blocks_batch(src_ptrs, dst_ptrs, sizes, is_src_access_order_any=True)
|
||||
|
||||
self._buf_pool.release(bufs)
|
||||
|
||||
# Slice contiguous buffer into per-hash views.
|
||||
offset = 0
|
||||
for mm_hash, block_ids in load_items.items():
|
||||
n = len(block_ids)
|
||||
encoder_cache[mm_hash] = (
|
||||
dst_buf[offset : offset + n].view(dtype).reshape(n, -1)
|
||||
)
|
||||
offset += n
|
||||
|
||||
current_platform.current_stream().wait_stream(self._load_stream)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._load_stream.synchronize()
|
||||
self._save_bufs = None
|
||||
self._save_count = 0
|
||||
try:
|
||||
self._region.cleanup()
|
||||
except Exception:
|
||||
logger.debug("EC: worker region cleanup failed", exc_info=True)
|
||||
@@ -0,0 +1,46 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Reusable pool of (src_ptrs, dst_ptrs, sizes) int64 tensor triples.
|
||||
|
||||
Used by ECCPUWorker to batch swap_blocks_batch descriptors without
|
||||
per-step allocation overhead.
|
||||
"""
|
||||
|
||||
from typing import NamedTuple
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class DescriptorBuffers(NamedTuple):
|
||||
src_ptrs: torch.Tensor
|
||||
dst_ptrs: torch.Tensor
|
||||
sizes: torch.Tensor
|
||||
|
||||
|
||||
class DescriptorBufferPool:
|
||||
"""Pool of descriptor buffer triples for swap_blocks_batch.
|
||||
|
||||
Each buffer is a `DescriptorBuffers` namedtuple of three 1-D int64
|
||||
tensors of equal length. Buffers are recycled across steps; if a
|
||||
returned buffer is too small it is discarded and a fresh one allocated.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
# LIFO stack of idle buffer triples.
|
||||
self._pool: list[DescriptorBuffers] = []
|
||||
|
||||
def acquire(self, n: int) -> DescriptorBuffers:
|
||||
"""Get a buffer triple with capacity >= *n*."""
|
||||
if self._pool:
|
||||
bufs = self._pool.pop()
|
||||
if bufs.src_ptrs.numel() >= n:
|
||||
return bufs
|
||||
return DescriptorBuffers(
|
||||
torch.empty(n, dtype=torch.int64),
|
||||
torch.empty(n, dtype=torch.int64),
|
||||
torch.empty(n, dtype=torch.int64),
|
||||
)
|
||||
|
||||
def release(self, bufs: DescriptorBuffers) -> None:
|
||||
"""Return a buffer triple to the pool for reuse."""
|
||||
self._pool.append(bufs)
|
||||
@@ -83,3 +83,9 @@ ECConnectorFactory.register_connector(
|
||||
"vllm.distributed.ec_transfer.ec_connector.example_connector",
|
||||
"ECExampleConnector",
|
||||
)
|
||||
|
||||
ECConnectorFactory.register_connector(
|
||||
"ECCPUConnector",
|
||||
"vllm.distributed.ec_transfer.ec_connector.cpu.connector",
|
||||
"ECCPUConnector",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user