[Bugfix][EC Connector] Size transfers by measured encoder output widths (#57904)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-10-12 12:32:39 +00:00
committed by GitHub
parent aa1d44df16
commit 258b97ff6b
18 changed files with 416 additions and 121 deletions
@@ -33,12 +33,14 @@ class ECCPUConnectorMetadata(ECConnectorMetadata):
saves: dict[str, list[int]] = field(default_factory=dict)
# Consumer role: mm_hashes whose bytes are available in the local mmap,
# mapped to (transfer_id, block_ids); the worker's start_load_caches
# mapped to (transfer_id, block_ids, shape); the worker's start_load_caches
# copies mmap[block_ids] → GPU encoder_cache and reports the transfer_id
# once its copy lands. The id is minted per dispatch, so two loads of the
# same mm_hash in different steps are distinct and a late report from the
# earlier one cannot release the later one's pin.
loads: dict[str, tuple[int, list[int]]] = field(default_factory=dict)
loads: dict[str, tuple[int, list[int], tuple[int, int]]] = field(
default_factory=dict
)
@dataclass
@@ -66,30 +68,6 @@ class ECCPUWorkerMetadata(ECConnectorWorkerMetadata):
return self
def _get_encoder_cache_hidden_dim(vllm_config: "VllmConfig") -> int:
"""Return the per-token hidden dimension for encoder cache entries.
For most models this equals the LLM's hidden size. Qwen3-VL (and any
future model with deepstack visual encoding) is an exception: the ViT
concatenates its own output with features from N decoder layers before
storing in encoder_cache, producing a tensor of width
``out_hidden_size * (1 + N)`` per visual token. Using the plain LLM
hidden size would under-allocate EC blocks and silently truncate the
transferred data, leading to a shape mismatch on the consumer.
"""
model_config = vllm_config.model_config
hf_config = getattr(model_config, "hf_config", None)
vision_config = (
getattr(hf_config, "vision_config", None) if hf_config is not None else None
)
if vision_config is not None:
out_hidden_size = getattr(vision_config, "out_hidden_size", None)
deepstack_indexes = getattr(vision_config, "deepstack_visual_indexes", None)
if out_hidden_size is not None and deepstack_indexes:
return out_hidden_size * (1 + len(deepstack_indexes))
return model_config.get_inputs_embeds_size()
def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion:
"""Build the EC mmap region from `vllm_config`.
@@ -103,7 +81,7 @@ def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion:
engine_id = f"{vllm_config.instance_id}_dp{dp_index}"
dtype = vllm_config.model_config.dtype
hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
hidden_dim = vllm_config.model_config.get_inputs_embeds_size()
element_size = torch.empty(0, dtype=dtype).element_size()
block_size_bytes = hidden_dim * element_size
@@ -14,9 +14,9 @@ import json
import msgspec
# Bump when the on-wire shape (XferReq/XferAck) changes in a
# Bump when the wire format or tensor/block layout changes in a
# backward-incompatible way. Peers reject mismatched versions.
EC_CONNECTOR_VERSION: int = 1
EC_CONNECTOR_VERSION: int = 2
class XferStatus(enum.IntEnum):
@@ -16,12 +16,14 @@ import torch
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
ECCPUWorkerMetadata,
_get_encoder_cache_hidden_dim,
create_ec_shared_region,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
get_encoder_output_width,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
@@ -49,6 +51,7 @@ class ECCPUScheduler:
"""Scheduler delegate for the ECCPUConnector."""
def __init__(self, vllm_config: "VllmConfig") -> None:
self._vllm_config = vllm_config
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None
self._is_producer: bool = ec_config.is_ec_producer
@@ -61,8 +64,8 @@ class ECCPUScheduler:
# mm_hash → block IDs allocated this step for GPU→mmap saves.
self._pending_saves: dict[str, list[int]] = {}
# mm_hash → (transfer_id, block IDs) to load from mmap→GPU this step.
self._pending_loads: dict[str, tuple[int, list[int]]] = {}
# mm_hash → (transfer_id, block IDs, shape) to load from mmap→GPU.
self._pending_loads: dict[str, tuple[int, list[int], tuple[int, int]]] = {}
# Dispatched loads awaiting completion reports, keyed by transfer id:
# transfer_id → (mm_hash, reports still outstanding). The pin taken at
@@ -108,11 +111,9 @@ class ECCPUScheduler:
self._unrecoverable: set[str] = set()
self._peer_host: str | None = None
self._peer_port: int | None = None
# Model shape for size checks + compat hash; only set by
# _setup_nixl (the gate-off path never touches model_config).
self._dtype: torch.dtype | None = None
self._hidden_dim: int = 0
self._element_size: int = 0
dtype = vllm_config.model_config.dtype
assert isinstance(dtype, torch.dtype)
self._dtype = dtype
self._ack_timeout_s: float = 0.0
if self._nixl_enabled:
try:
@@ -148,9 +149,6 @@ class ECCPUScheduler:
)
engine_id = self._ec_config.engine_id
assert engine_id is not None
self._dtype = vllm_config.model_config.dtype
self._hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
self._element_size = torch.empty(0, dtype=self._dtype).element_size()
# How long a consumer waits for an XferAck. The producer answers from
# its scheduler step, so its reply latency scales with the encoder's
# batch size: a deployment whose steps run longer must raise this.
@@ -296,22 +294,30 @@ class ECCPUScheduler:
# held by a quarantined/settling DMA and cannot be reused.
return False
expected = pos.length * self._hidden_dim * self._element_size
try:
size = int(remote["size_bytes"])
shape = tuple(int(dim) for dim in remote["shape"])
except (KeyError, TypeError, ValueError) as error:
raise _RemoteUnavailable("the announced size was unusable") from error
if size != expected:
# The width is the producer's to report; the row count is not.
num_embeds = pos.get_num_embeds()
if (
len(shape) != 2
or shape[0] != num_embeds
or size != self._num_blocks(shape) * self._region.block_size_bytes
):
logger.warning(
"EC consumer: size mismatch mm_hash=%s announced=%d expected=%d",
"EC consumer: size mismatch mm_hash=%s announced=%d shape=%s "
"expected %d rows",
mm_hash,
size,
expected,
shape,
num_embeds,
)
raise _RemoteUnavailable("the announced size was wrong")
try:
started = self._start_xfer(mm_hash, remote, expected)
started = self._start_xfer(mm_hash, remote, size, (shape[0], shape[1]))
except Exception as error:
logger.exception(
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
@@ -377,7 +383,11 @@ class ECCPUScheduler:
return failed
def _start_xfer(
self, mm_hash: str, info: "dict[str, Any]", size_bytes: int
self,
mm_hash: str,
info: "dict[str, Any]",
size_bytes: int,
shape: tuple[int, int],
) -> bool:
"""Allocate a not-ready cache entry and start a NIXL READ into it.
@@ -391,7 +401,7 @@ class ECCPUScheduler:
)
n_blocks = max(1, ceil(size_bytes / self._region.block_size_bytes))
entry = self._cache.alloc(mm_hash, n_blocks)
entry = self._cache.alloc(mm_hash, n_blocks, shape)
if entry is None:
logger.debug(
"EC consumer: cache full for mm_hash=%s (%d blocks); deferring",
@@ -491,22 +501,36 @@ class ECCPUScheduler:
"""
self._step_completed.clear()
def _num_blocks(self, shape: tuple[int, ...]) -> int:
size_bytes = shape[0] * shape[1] * self._dtype.itemsize
block_size = self._region.block_size_bytes
return (size_bytes + block_size - 1) // block_size
def update_state_after_alloc(self, request: "Request", index: int) -> None:
feature = request.mm_features[index]
mm_hash = feature.identifier
if self._is_producer and self._cache.get(mm_hash) is None:
entry = self._cache.alloc(mm_hash, feature.mm_position.length)
shape = (
feature.mm_position.get_num_embeds(),
get_encoder_output_width(self._vllm_config, feature.modality),
)
entry = self._cache.alloc(mm_hash, self._num_blocks(shape), shape)
if entry is not None:
self._pending_saves[mm_hash] = list(entry.block_ids)
if self._is_consumer and mm_hash not in self._pending_loads:
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
assert entry.shape is not None
self._cache.pin(mm_hash)
transfer_id = self._next_transfer_id
self._next_transfer_id += 1
self._pending_loads[mm_hash] = (transfer_id, list(entry.block_ids))
self._pending_loads[mm_hash] = (
transfer_id,
list(entry.block_ids),
entry.shape,
)
self._load_acks[transfer_id] = (mm_hash, self._expected_load_reports)
def build_connector_meta(
@@ -608,13 +632,13 @@ class ECCPUScheduler:
# A read arriving before the save lands is NACKed NACK_NOT_READY,
# which the consumer retries on a later step rather than treating
# as a miss.
size_bytes = (
feature.mm_position.length * self._hidden_dim * self._element_size
)
assert entry.shape is not None
size_bytes = len(entry.block_ids) * self._region.block_size_bytes
items[mm_hash].update(
peer_host=self._peer_host,
peer_port=self._peer_port,
size_bytes=size_bytes,
shape=list(entry.shape),
)
logger.debug(
"EC producer: announcing NIXL-readable encodings req_id=%s items=%s",
@@ -14,13 +14,20 @@ from collections import OrderedDict
class CacheEntry:
"""A single cache entry. Read `.block_ids` and `.ready` freely;
mutations only through EmbeddingCache methods."""
"""A single cache entry. Read `.block_ids`, `.shape` and `.ready` freely;
mutations only through EmbeddingCache methods.
__slots__ = ("block_ids", "_pin_count")
`shape` is the encoder output the blocks hold, so a load can restore it
without knowing how wide this model's encoder output is.
"""
def __init__(self, block_ids: tuple[int, ...]) -> None:
__slots__ = ("block_ids", "shape", "_pin_count")
def __init__(
self, block_ids: tuple[int, ...], shape: tuple[int, int] | None = None
) -> None:
self.block_ids = block_ids
self.shape = shape
self._pin_count = -1 # not ready
@property
@@ -76,7 +83,9 @@ class EmbeddingCache:
with self._lock:
return len(self._entries) != len(self._entries_free_list)
def alloc(self, key: str, n_blocks: int) -> CacheEntry | None:
def alloc(
self, key: str, n_blocks: int, shape: tuple[int, int] | None = None
) -> CacheEntry | None:
"""Allocate *n_blocks* for *key*, evicting as needed.
The entry starts not-ready. Returns None if there is not enough
@@ -97,7 +106,7 @@ class EmbeddingCache:
# in the free list when it is evicted, and lets consumers detect
# which of an entry's blocks are adjacent.
block_ids = tuple(sorted(self._free_blocks.pop() for _ in range(n_blocks)))
entry = CacheEntry(block_ids)
entry = CacheEntry(block_ids, shape)
self._entries[key] = entry
return entry
@@ -267,8 +267,7 @@ class ECCPUWorker:
# so they are added up in uint64; signed arithmetic rejects them.
src_base = np.uint64(src.view(-1).view(torch.uint8).data_ptr())
dst_base = self._region.blocks.data_ptr()
# Entries are sized by placeholder count, which can exceed the number
# of embeddings, so blocks the output never reaches get no descriptor.
# Copy only the tensor bytes, excluding any block padding.
n_used = -(-total_bytes // block_size)
if n_used:
slots, first_blocks, num_blocks = _coalesce_runs(block_ids[:n_used])
@@ -367,7 +366,7 @@ class ECCPUWorker:
load_items = sorted(
connector_metadata.loads.items(), key=lambda kv: kv[1][1][0]
)
total_blocks = sum(len(block_ids) for _, (_, block_ids) in load_items)
total_blocks = sum(len(block_ids) for _, (_, block_ids, _) in load_items)
stream = self._acquire_stream()
compute_stream = current_platform.current_stream()
@@ -384,7 +383,7 @@ class ECCPUWorker:
bufs = self._buf_pool.acquire(total_blocks)
slots, first_blocks, num_blocks = _coalesce_runs(
list(chain.from_iterable(ids for _, (_, ids) in load_items))
list(chain.from_iterable(ids for _, (_, ids, _) in load_items))
)
bufs.add_copies(
0,
@@ -405,7 +404,7 @@ class ECCPUWorker:
Transfer(
start_event=start_event,
end_event=end_event,
completions=[tid for _, (tid, _) in load_items],
completions=[tid for _, (tid, _, _) in load_items],
bufs=bufs,
stream=stream,
num_bytes=total_blocks * block_size,
@@ -414,11 +413,10 @@ class ECCPUWorker:
# Slice contiguous buffer into per-hash views.
offset = 0
for mm_hash, (_, block_ids) in load_items:
for mm_hash, (_, block_ids, shape) in load_items:
n = len(block_ids)
encoder_cache[mm_hash] = (
dst_buf[offset : offset + n].view(dtype).reshape(n, -1)
)
flat = dst_buf[offset : offset + n].view(dtype).reshape(-1)
encoder_cache[mm_hash] = flat[: shape[0] * shape[1]].view(shape)
offset += n
current_platform.current_stream().wait_stream(stream)