[Bugfix][EC Connector] Size transfers by measured encoder output widths (#57904)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
Tianyu Guo
2026-10-12 12:32:39 +00:00
committed by GitHub
parent aa1d44df16
commit 258b97ff6b
18 changed files with 416 additions and 121 deletions
@@ -33,12 +33,14 @@ class ECCPUConnectorMetadata(ECConnectorMetadata):
saves: dict[str, list[int]] = field(default_factory=dict)
# Consumer role: mm_hashes whose bytes are available in the local mmap,
# mapped to (transfer_id, block_ids); the worker's start_load_caches
# mapped to (transfer_id, block_ids, shape); the worker's start_load_caches
# copies mmap[block_ids] → GPU encoder_cache and reports the transfer_id
# once its copy lands. The id is minted per dispatch, so two loads of the
# same mm_hash in different steps are distinct and a late report from the
# earlier one cannot release the later one's pin.
loads: dict[str, tuple[int, list[int]]] = field(default_factory=dict)
loads: dict[str, tuple[int, list[int], tuple[int, int]]] = field(
default_factory=dict
)
@dataclass
@@ -66,30 +68,6 @@ class ECCPUWorkerMetadata(ECConnectorWorkerMetadata):
return self
def _get_encoder_cache_hidden_dim(vllm_config: "VllmConfig") -> int:
"""Return the per-token hidden dimension for encoder cache entries.
For most models this equals the LLM's hidden size. Qwen3-VL (and any
future model with deepstack visual encoding) is an exception: the ViT
concatenates its own output with features from N decoder layers before
storing in encoder_cache, producing a tensor of width
``out_hidden_size * (1 + N)`` per visual token. Using the plain LLM
hidden size would under-allocate EC blocks and silently truncate the
transferred data, leading to a shape mismatch on the consumer.
"""
model_config = vllm_config.model_config
hf_config = getattr(model_config, "hf_config", None)
vision_config = (
getattr(hf_config, "vision_config", None) if hf_config is not None else None
)
if vision_config is not None:
out_hidden_size = getattr(vision_config, "out_hidden_size", None)
deepstack_indexes = getattr(vision_config, "deepstack_visual_indexes", None)
if out_hidden_size is not None and deepstack_indexes:
return out_hidden_size * (1 + len(deepstack_indexes))
return model_config.get_inputs_embeds_size()
def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion:
"""Build the EC mmap region from `vllm_config`.
@@ -103,7 +81,7 @@ def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion:
engine_id = f"{vllm_config.instance_id}_dp{dp_index}"
dtype = vllm_config.model_config.dtype
hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
hidden_dim = vllm_config.model_config.get_inputs_embeds_size()
element_size = torch.empty(0, dtype=dtype).element_size()
block_size_bytes = hidden_dim * element_size
@@ -14,9 +14,9 @@ import json
import msgspec
# Bump when the on-wire shape (XferReq/XferAck) changes in a
# Bump when the wire format or tensor/block layout changes in a
# backward-incompatible way. Peers reject mismatched versions.
EC_CONNECTOR_VERSION: int = 1
EC_CONNECTOR_VERSION: int = 2
class XferStatus(enum.IntEnum):
@@ -16,12 +16,14 @@ import torch
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
ECCPUConnectorMetadata,
ECCPUWorkerMetadata,
_get_encoder_cache_hidden_dim,
create_ec_shared_region,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
EmbeddingCache,
)
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
get_encoder_output_width,
)
from vllm.distributed.ec_transfer.ec_connector.utils import (
PlaceholderMetadataResolver,
collect_ec_item_metadata,
@@ -49,6 +51,7 @@ class ECCPUScheduler:
"""Scheduler delegate for the ECCPUConnector."""
def __init__(self, vllm_config: "VllmConfig") -> None:
self._vllm_config = vllm_config
ec_config = vllm_config.ec_transfer_config
assert ec_config is not None
self._is_producer: bool = ec_config.is_ec_producer
@@ -61,8 +64,8 @@ class ECCPUScheduler:
# mm_hash → block IDs allocated this step for GPU→mmap saves.
self._pending_saves: dict[str, list[int]] = {}
# mm_hash → (transfer_id, block IDs) to load from mmap→GPU this step.
self._pending_loads: dict[str, tuple[int, list[int]]] = {}
# mm_hash → (transfer_id, block IDs, shape) to load from mmap→GPU.
self._pending_loads: dict[str, tuple[int, list[int], tuple[int, int]]] = {}
# Dispatched loads awaiting completion reports, keyed by transfer id:
# transfer_id → (mm_hash, reports still outstanding). The pin taken at
@@ -108,11 +111,9 @@ class ECCPUScheduler:
self._unrecoverable: set[str] = set()
self._peer_host: str | None = None
self._peer_port: int | None = None
# Model shape for size checks + compat hash; only set by
# _setup_nixl (the gate-off path never touches model_config).
self._dtype: torch.dtype | None = None
self._hidden_dim: int = 0
self._element_size: int = 0
dtype = vllm_config.model_config.dtype
assert isinstance(dtype, torch.dtype)
self._dtype = dtype
self._ack_timeout_s: float = 0.0
if self._nixl_enabled:
try:
@@ -148,9 +149,6 @@ class ECCPUScheduler:
)
engine_id = self._ec_config.engine_id
assert engine_id is not None
self._dtype = vllm_config.model_config.dtype
self._hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
self._element_size = torch.empty(0, dtype=self._dtype).element_size()
# How long a consumer waits for an XferAck. The producer answers from
# its scheduler step, so its reply latency scales with the encoder's
# batch size: a deployment whose steps run longer must raise this.
@@ -296,22 +294,30 @@ class ECCPUScheduler:
# held by a quarantined/settling DMA and cannot be reused.
return False
expected = pos.length * self._hidden_dim * self._element_size
try:
size = int(remote["size_bytes"])
shape = tuple(int(dim) for dim in remote["shape"])
except (KeyError, TypeError, ValueError) as error:
raise _RemoteUnavailable("the announced size was unusable") from error
if size != expected:
# The width is the producer's to report; the row count is not.
num_embeds = pos.get_num_embeds()
if (
len(shape) != 2
or shape[0] != num_embeds
or size != self._num_blocks(shape) * self._region.block_size_bytes
):
logger.warning(
"EC consumer: size mismatch mm_hash=%s announced=%d expected=%d",
"EC consumer: size mismatch mm_hash=%s announced=%d shape=%s "
"expected %d rows",
mm_hash,
size,
expected,
shape,
num_embeds,
)
raise _RemoteUnavailable("the announced size was wrong")
try:
started = self._start_xfer(mm_hash, remote, expected)
started = self._start_xfer(mm_hash, remote, size, (shape[0], shape[1]))
except Exception as error:
logger.exception(
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
@@ -377,7 +383,11 @@ class ECCPUScheduler:
return failed
def _start_xfer(
self, mm_hash: str, info: "dict[str, Any]", size_bytes: int
self,
mm_hash: str,
info: "dict[str, Any]",
size_bytes: int,
shape: tuple[int, int],
) -> bool:
"""Allocate a not-ready cache entry and start a NIXL READ into it.
@@ -391,7 +401,7 @@ class ECCPUScheduler:
)
n_blocks = max(1, ceil(size_bytes / self._region.block_size_bytes))
entry = self._cache.alloc(mm_hash, n_blocks)
entry = self._cache.alloc(mm_hash, n_blocks, shape)
if entry is None:
logger.debug(
"EC consumer: cache full for mm_hash=%s (%d blocks); deferring",
@@ -491,22 +501,36 @@ class ECCPUScheduler:
"""
self._step_completed.clear()
def _num_blocks(self, shape: tuple[int, ...]) -> int:
size_bytes = shape[0] * shape[1] * self._dtype.itemsize
block_size = self._region.block_size_bytes
return (size_bytes + block_size - 1) // block_size
def update_state_after_alloc(self, request: "Request", index: int) -> None:
feature = request.mm_features[index]
mm_hash = feature.identifier
if self._is_producer and self._cache.get(mm_hash) is None:
entry = self._cache.alloc(mm_hash, feature.mm_position.length)
shape = (
feature.mm_position.get_num_embeds(),
get_encoder_output_width(self._vllm_config, feature.modality),
)
entry = self._cache.alloc(mm_hash, self._num_blocks(shape), shape)
if entry is not None:
self._pending_saves[mm_hash] = list(entry.block_ids)
if self._is_consumer and mm_hash not in self._pending_loads:
entry = self._cache.get(mm_hash)
if entry is not None and entry.ready:
assert entry.shape is not None
self._cache.pin(mm_hash)
transfer_id = self._next_transfer_id
self._next_transfer_id += 1
self._pending_loads[mm_hash] = (transfer_id, list(entry.block_ids))
self._pending_loads[mm_hash] = (
transfer_id,
list(entry.block_ids),
entry.shape,
)
self._load_acks[transfer_id] = (mm_hash, self._expected_load_reports)
def build_connector_meta(
@@ -608,13 +632,13 @@ class ECCPUScheduler:
# A read arriving before the save lands is NACKed NACK_NOT_READY,
# which the consumer retries on a later step rather than treating
# as a miss.
size_bytes = (
feature.mm_position.length * self._hidden_dim * self._element_size
)
assert entry.shape is not None
size_bytes = len(entry.block_ids) * self._region.block_size_bytes
items[mm_hash].update(
peer_host=self._peer_host,
peer_port=self._peer_port,
size_bytes=size_bytes,
shape=list(entry.shape),
)
logger.debug(
"EC producer: announcing NIXL-readable encodings req_id=%s items=%s",
@@ -14,13 +14,20 @@ from collections import OrderedDict
class CacheEntry:
"""A single cache entry. Read `.block_ids` and `.ready` freely;
mutations only through EmbeddingCache methods."""
"""A single cache entry. Read `.block_ids`, `.shape` and `.ready` freely;
mutations only through EmbeddingCache methods.
__slots__ = ("block_ids", "_pin_count")
`shape` is the encoder output the blocks hold, so a load can restore it
without knowing how wide this model's encoder output is.
"""
def __init__(self, block_ids: tuple[int, ...]) -> None:
__slots__ = ("block_ids", "shape", "_pin_count")
def __init__(
self, block_ids: tuple[int, ...], shape: tuple[int, int] | None = None
) -> None:
self.block_ids = block_ids
self.shape = shape
self._pin_count = -1 # not ready
@property
@@ -76,7 +83,9 @@ class EmbeddingCache:
with self._lock:
return len(self._entries) != len(self._entries_free_list)
def alloc(self, key: str, n_blocks: int) -> CacheEntry | None:
def alloc(
self, key: str, n_blocks: int, shape: tuple[int, int] | None = None
) -> CacheEntry | None:
"""Allocate *n_blocks* for *key*, evicting as needed.
The entry starts not-ready. Returns None if there is not enough
@@ -97,7 +106,7 @@ class EmbeddingCache:
# in the free list when it is evicted, and lets consumers detect
# which of an entry's blocks are adjacent.
block_ids = tuple(sorted(self._free_blocks.pop() for _ in range(n_blocks)))
entry = CacheEntry(block_ids)
entry = CacheEntry(block_ids, shape)
self._entries[key] = entry
return entry
@@ -267,8 +267,7 @@ class ECCPUWorker:
# so they are added up in uint64; signed arithmetic rejects them.
src_base = np.uint64(src.view(-1).view(torch.uint8).data_ptr())
dst_base = self._region.blocks.data_ptr()
# Entries are sized by placeholder count, which can exceed the number
# of embeddings, so blocks the output never reaches get no descriptor.
# Copy only the tensor bytes, excluding any block padding.
n_used = -(-total_bytes // block_size)
if n_used:
slots, first_blocks, num_blocks = _coalesce_runs(block_ids[:n_used])
@@ -367,7 +366,7 @@ class ECCPUWorker:
load_items = sorted(
connector_metadata.loads.items(), key=lambda kv: kv[1][1][0]
)
total_blocks = sum(len(block_ids) for _, (_, block_ids) in load_items)
total_blocks = sum(len(block_ids) for _, (_, block_ids, _) in load_items)
stream = self._acquire_stream()
compute_stream = current_platform.current_stream()
@@ -384,7 +383,7 @@ class ECCPUWorker:
bufs = self._buf_pool.acquire(total_blocks)
slots, first_blocks, num_blocks = _coalesce_runs(
list(chain.from_iterable(ids for _, (_, ids) in load_items))
list(chain.from_iterable(ids for _, (_, ids, _) in load_items))
)
bufs.add_copies(
0,
@@ -405,7 +404,7 @@ class ECCPUWorker:
Transfer(
start_event=start_event,
end_event=end_event,
completions=[tid for _, (tid, _) in load_items],
completions=[tid for _, (tid, _, _) in load_items],
bufs=bufs,
stream=stream,
num_bytes=total_blocks * block_size,
@@ -414,11 +413,10 @@ class ECCPUWorker:
# Slice contiguous buffer into per-hash views.
offset = 0
for mm_hash, (_, block_ids) in load_items:
for mm_hash, (_, block_ids, shape) in load_items:
n = len(block_ids)
encoder_cache[mm_hash] = (
dst_buf[offset : offset + n].view(dtype).reshape(n, -1)
)
flat = dst_buf[offset : offset + n].view(dtype).reshape(-1)
encoder_cache[mm_hash] = flat[: shape[0] * shape[1]].view(shape)
offset += n
current_platform.current_stream().wait_stream(stream)
@@ -0,0 +1,102 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Encoder output widths for sizing EC transfers.
An encoder can emit features wider than the LM embedding (DeepStack packs
several layers on the last dim) by an amount only the model knows. Producers
size transfers before the first real item is encoded, so the engine measures
the width once at startup by encoding a dummy item per modality.
"""
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
import torch
from vllm.logger import init_logger
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
@torch.inference_mode()
def _measure_on_worker(worker: Any) -> dict[str, int]:
"""Encode one dummy item per modality; runs on every worker.
Returns {} on ranks that hold no encoder.
"""
from vllm.distributed.parallel_state import get_pp_group
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.encoder_budget import MultiModalBudget
from vllm.multimodal.utils import group_and_batch_mm_kwargs
from vllm.utils.torch_utils import PIN_MEMORY
runner = worker.model_runner
if not runner.supports_mm_inputs or not get_pp_group().is_first_rank:
return {}
budget = MultiModalBudget(
worker.vllm_config, MULTIMODAL_REGISTRY, enable_cache=False
)
model = runner.get_model()
widths: dict[str, int] = {}
for modality in budget.mm_max_toks_per_item:
items = budget.get_dummy_encoder_profile_inputs(modality, 1)
if not items:
continue
_, _, mm_kwargs = next(
group_and_batch_mm_kwargs(
items, device=runner.device, pin_memory=PIN_MEMORY
)
)
outputs = model.embed_multimodal(**mm_kwargs)
widths[modality] = int(outputs[0].shape[-1])
del outputs
return widths
def measure_encoder_output_widths(
vllm_config: "VllmConfig", collective_rpc: Callable[..., list[Any]]
) -> None:
"""Record the encoder output widths on an EC producer's config.
Call after the model is loaded and before the KV cache is allocated: the
Scheduler's connector reads the widths, and once the KV cache holds the
memory budget there is no room left for a dummy encode.
Raises:
ValueError: No worker could encode a dummy item, or workers disagree.
"""
ec_config = vllm_config.ec_transfer_config
if ec_config is None or not ec_config.is_ec_producer:
return
reports = [widths for widths in collective_rpc(_measure_on_worker) if widths]
if not reports:
raise ValueError(
"EC producer could not measure the encoder output width: the model "
"encoded no dummy multimodal input."
)
if any(widths != reports[0] for widths in reports):
raise ValueError(f"Workers disagree on the encoder output widths: {reports}")
logger.info("EC producer encoder output widths: %s", reports[0])
ec_config.mm_encoder_output_widths = reports[0]
def get_encoder_output_width(vllm_config: "VllmConfig", modality: str) -> int:
"""Encoder output width for `modality`, as measured at producer startup.
Raises:
ValueError: No width was measured for `modality`, so a transfer
cannot be sized.
"""
assert vllm_config.ec_transfer_config is not None
widths = vllm_config.ec_transfer_config.mm_encoder_output_widths
if modality not in widths:
raise ValueError(
f"No encoder output width was measured for modality {modality!r}; "
f"measured: {sorted(widths)}"
)
return widths[modality]
@@ -20,8 +20,8 @@ import torch
from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorMetadata,
)
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
_get_encoder_cache_hidden_dim,
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
get_encoder_output_width,
)
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
_RESERVATION_TTL_SECONDS,
@@ -85,9 +85,7 @@ class ECMooncakeScheduler:
self._is_consumer = config.is_consumer
self._control_addr = config.control_addr
self._push_wait_timeout = config.push_wait_timeout_s
self._encoder_cache_hidden_dim = (
_get_encoder_cache_hidden_dim(vllm_config) if config.is_producer else None
)
self._vllm_config = vllm_config
self._model_config = vllm_config.model_config
self._control_client = ControlClient(config.control_timeout_ms)
self._control_executor = ThreadPoolExecutor(
@@ -447,8 +445,10 @@ class ECMooncakeScheduler:
def _encoder_output_spec(self, request: Any, index: int) -> TensorSpec:
dtype = self._model_config.dtype
assert isinstance(dtype, torch.dtype)
assert self._encoder_cache_hidden_dim is not None
shape = (request.get_num_encoder_embeds(index), self._encoder_cache_hidden_dim)
hidden_dim = get_encoder_output_width(
self._vllm_config, request.mm_features[index].modality
)
shape = (request.get_num_encoder_embeds(index), hidden_dim)
return TensorSpec(shape, str(dtype), math.prod(shape) * dtype.itemsize)
def update_state_after_alloc(self, request: Any, index: int) -> None: