[Bugfix][EC Connector] Size transfers by measured encoder output widths (#57904)
Signed-off-by: Tianyu Guo <guoty@inferact.ai>
This commit is contained in:
@@ -33,12 +33,14 @@ class ECCPUConnectorMetadata(ECConnectorMetadata):
|
||||
saves: dict[str, list[int]] = field(default_factory=dict)
|
||||
|
||||
# Consumer role: mm_hashes whose bytes are available in the local mmap,
|
||||
# mapped to (transfer_id, block_ids); the worker's start_load_caches
|
||||
# mapped to (transfer_id, block_ids, shape); the worker's start_load_caches
|
||||
# copies mmap[block_ids] → GPU encoder_cache and reports the transfer_id
|
||||
# once its copy lands. The id is minted per dispatch, so two loads of the
|
||||
# same mm_hash in different steps are distinct and a late report from the
|
||||
# earlier one cannot release the later one's pin.
|
||||
loads: dict[str, tuple[int, list[int]]] = field(default_factory=dict)
|
||||
loads: dict[str, tuple[int, list[int], tuple[int, int]]] = field(
|
||||
default_factory=dict
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -66,30 +68,6 @@ class ECCPUWorkerMetadata(ECConnectorWorkerMetadata):
|
||||
return self
|
||||
|
||||
|
||||
def _get_encoder_cache_hidden_dim(vllm_config: "VllmConfig") -> int:
|
||||
"""Return the per-token hidden dimension for encoder cache entries.
|
||||
|
||||
For most models this equals the LLM's hidden size. Qwen3-VL (and any
|
||||
future model with deepstack visual encoding) is an exception: the ViT
|
||||
concatenates its own output with features from N decoder layers before
|
||||
storing in encoder_cache, producing a tensor of width
|
||||
``out_hidden_size * (1 + N)`` per visual token. Using the plain LLM
|
||||
hidden size would under-allocate EC blocks and silently truncate the
|
||||
transferred data, leading to a shape mismatch on the consumer.
|
||||
"""
|
||||
model_config = vllm_config.model_config
|
||||
hf_config = getattr(model_config, "hf_config", None)
|
||||
vision_config = (
|
||||
getattr(hf_config, "vision_config", None) if hf_config is not None else None
|
||||
)
|
||||
if vision_config is not None:
|
||||
out_hidden_size = getattr(vision_config, "out_hidden_size", None)
|
||||
deepstack_indexes = getattr(vision_config, "deepstack_visual_indexes", None)
|
||||
if out_hidden_size is not None and deepstack_indexes:
|
||||
return out_hidden_size * (1 + len(deepstack_indexes))
|
||||
return model_config.get_inputs_embeds_size()
|
||||
|
||||
|
||||
def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion:
|
||||
"""Build the EC mmap region from `vllm_config`.
|
||||
|
||||
@@ -103,7 +81,7 @@ def create_ec_shared_region(vllm_config: "VllmConfig") -> ECSharedRegion:
|
||||
engine_id = f"{vllm_config.instance_id}_dp{dp_index}"
|
||||
|
||||
dtype = vllm_config.model_config.dtype
|
||||
hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
|
||||
hidden_dim = vllm_config.model_config.get_inputs_embeds_size()
|
||||
element_size = torch.empty(0, dtype=dtype).element_size()
|
||||
block_size_bytes = hidden_dim * element_size
|
||||
|
||||
|
||||
@@ -14,9 +14,9 @@ import json
|
||||
|
||||
import msgspec
|
||||
|
||||
# Bump when the on-wire shape (XferReq/XferAck) changes in a
|
||||
# Bump when the wire format or tensor/block layout changes in a
|
||||
# backward-incompatible way. Peers reject mismatched versions.
|
||||
EC_CONNECTOR_VERSION: int = 1
|
||||
EC_CONNECTOR_VERSION: int = 2
|
||||
|
||||
|
||||
class XferStatus(enum.IntEnum):
|
||||
|
||||
@@ -16,12 +16,14 @@ import torch
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
ECCPUConnectorMetadata,
|
||||
ECCPUWorkerMetadata,
|
||||
_get_encoder_cache_hidden_dim,
|
||||
create_ec_shared_region,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.scheduler.embedding_cache import (
|
||||
EmbeddingCache,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
|
||||
get_encoder_output_width,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.utils import (
|
||||
PlaceholderMetadataResolver,
|
||||
collect_ec_item_metadata,
|
||||
@@ -49,6 +51,7 @@ class ECCPUScheduler:
|
||||
"""Scheduler delegate for the ECCPUConnector."""
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig") -> None:
|
||||
self._vllm_config = vllm_config
|
||||
ec_config = vllm_config.ec_transfer_config
|
||||
assert ec_config is not None
|
||||
self._is_producer: bool = ec_config.is_ec_producer
|
||||
@@ -61,8 +64,8 @@ class ECCPUScheduler:
|
||||
|
||||
# mm_hash → block IDs allocated this step for GPU→mmap saves.
|
||||
self._pending_saves: dict[str, list[int]] = {}
|
||||
# mm_hash → (transfer_id, block IDs) to load from mmap→GPU this step.
|
||||
self._pending_loads: dict[str, tuple[int, list[int]]] = {}
|
||||
# mm_hash → (transfer_id, block IDs, shape) to load from mmap→GPU.
|
||||
self._pending_loads: dict[str, tuple[int, list[int], tuple[int, int]]] = {}
|
||||
|
||||
# Dispatched loads awaiting completion reports, keyed by transfer id:
|
||||
# transfer_id → (mm_hash, reports still outstanding). The pin taken at
|
||||
@@ -108,11 +111,9 @@ class ECCPUScheduler:
|
||||
self._unrecoverable: set[str] = set()
|
||||
self._peer_host: str | None = None
|
||||
self._peer_port: int | None = None
|
||||
# Model shape for size checks + compat hash; only set by
|
||||
# _setup_nixl (the gate-off path never touches model_config).
|
||||
self._dtype: torch.dtype | None = None
|
||||
self._hidden_dim: int = 0
|
||||
self._element_size: int = 0
|
||||
dtype = vllm_config.model_config.dtype
|
||||
assert isinstance(dtype, torch.dtype)
|
||||
self._dtype = dtype
|
||||
self._ack_timeout_s: float = 0.0
|
||||
if self._nixl_enabled:
|
||||
try:
|
||||
@@ -148,9 +149,6 @@ class ECCPUScheduler:
|
||||
)
|
||||
engine_id = self._ec_config.engine_id
|
||||
assert engine_id is not None
|
||||
self._dtype = vllm_config.model_config.dtype
|
||||
self._hidden_dim = _get_encoder_cache_hidden_dim(vllm_config)
|
||||
self._element_size = torch.empty(0, dtype=self._dtype).element_size()
|
||||
# How long a consumer waits for an XferAck. The producer answers from
|
||||
# its scheduler step, so its reply latency scales with the encoder's
|
||||
# batch size: a deployment whose steps run longer must raise this.
|
||||
@@ -296,22 +294,30 @@ class ECCPUScheduler:
|
||||
# held by a quarantined/settling DMA and cannot be reused.
|
||||
return False
|
||||
|
||||
expected = pos.length * self._hidden_dim * self._element_size
|
||||
try:
|
||||
size = int(remote["size_bytes"])
|
||||
shape = tuple(int(dim) for dim in remote["shape"])
|
||||
except (KeyError, TypeError, ValueError) as error:
|
||||
raise _RemoteUnavailable("the announced size was unusable") from error
|
||||
if size != expected:
|
||||
# The width is the producer's to report; the row count is not.
|
||||
num_embeds = pos.get_num_embeds()
|
||||
if (
|
||||
len(shape) != 2
|
||||
or shape[0] != num_embeds
|
||||
or size != self._num_blocks(shape) * self._region.block_size_bytes
|
||||
):
|
||||
logger.warning(
|
||||
"EC consumer: size mismatch mm_hash=%s announced=%d expected=%d",
|
||||
"EC consumer: size mismatch mm_hash=%s announced=%d shape=%s "
|
||||
"expected %d rows",
|
||||
mm_hash,
|
||||
size,
|
||||
expected,
|
||||
shape,
|
||||
num_embeds,
|
||||
)
|
||||
raise _RemoteUnavailable("the announced size was wrong")
|
||||
|
||||
try:
|
||||
started = self._start_xfer(mm_hash, remote, expected)
|
||||
started = self._start_xfer(mm_hash, remote, size, (shape[0], shape[1]))
|
||||
except Exception as error:
|
||||
logger.exception(
|
||||
"EC consumer: failed to start NIXL xfer mm_hash=%s", mm_hash
|
||||
@@ -377,7 +383,11 @@ class ECCPUScheduler:
|
||||
return failed
|
||||
|
||||
def _start_xfer(
|
||||
self, mm_hash: str, info: "dict[str, Any]", size_bytes: int
|
||||
self,
|
||||
mm_hash: str,
|
||||
info: "dict[str, Any]",
|
||||
size_bytes: int,
|
||||
shape: tuple[int, int],
|
||||
) -> bool:
|
||||
"""Allocate a not-ready cache entry and start a NIXL READ into it.
|
||||
|
||||
@@ -391,7 +401,7 @@ class ECCPUScheduler:
|
||||
)
|
||||
|
||||
n_blocks = max(1, ceil(size_bytes / self._region.block_size_bytes))
|
||||
entry = self._cache.alloc(mm_hash, n_blocks)
|
||||
entry = self._cache.alloc(mm_hash, n_blocks, shape)
|
||||
if entry is None:
|
||||
logger.debug(
|
||||
"EC consumer: cache full for mm_hash=%s (%d blocks); deferring",
|
||||
@@ -491,22 +501,36 @@ class ECCPUScheduler:
|
||||
"""
|
||||
self._step_completed.clear()
|
||||
|
||||
def _num_blocks(self, shape: tuple[int, ...]) -> int:
|
||||
size_bytes = shape[0] * shape[1] * self._dtype.itemsize
|
||||
block_size = self._region.block_size_bytes
|
||||
return (size_bytes + block_size - 1) // block_size
|
||||
|
||||
def update_state_after_alloc(self, request: "Request", index: int) -> None:
|
||||
feature = request.mm_features[index]
|
||||
mm_hash = feature.identifier
|
||||
|
||||
if self._is_producer and self._cache.get(mm_hash) is None:
|
||||
entry = self._cache.alloc(mm_hash, feature.mm_position.length)
|
||||
shape = (
|
||||
feature.mm_position.get_num_embeds(),
|
||||
get_encoder_output_width(self._vllm_config, feature.modality),
|
||||
)
|
||||
entry = self._cache.alloc(mm_hash, self._num_blocks(shape), shape)
|
||||
if entry is not None:
|
||||
self._pending_saves[mm_hash] = list(entry.block_ids)
|
||||
|
||||
if self._is_consumer and mm_hash not in self._pending_loads:
|
||||
entry = self._cache.get(mm_hash)
|
||||
if entry is not None and entry.ready:
|
||||
assert entry.shape is not None
|
||||
self._cache.pin(mm_hash)
|
||||
transfer_id = self._next_transfer_id
|
||||
self._next_transfer_id += 1
|
||||
self._pending_loads[mm_hash] = (transfer_id, list(entry.block_ids))
|
||||
self._pending_loads[mm_hash] = (
|
||||
transfer_id,
|
||||
list(entry.block_ids),
|
||||
entry.shape,
|
||||
)
|
||||
self._load_acks[transfer_id] = (mm_hash, self._expected_load_reports)
|
||||
|
||||
def build_connector_meta(
|
||||
@@ -608,13 +632,13 @@ class ECCPUScheduler:
|
||||
# A read arriving before the save lands is NACKed NACK_NOT_READY,
|
||||
# which the consumer retries on a later step rather than treating
|
||||
# as a miss.
|
||||
size_bytes = (
|
||||
feature.mm_position.length * self._hidden_dim * self._element_size
|
||||
)
|
||||
assert entry.shape is not None
|
||||
size_bytes = len(entry.block_ids) * self._region.block_size_bytes
|
||||
items[mm_hash].update(
|
||||
peer_host=self._peer_host,
|
||||
peer_port=self._peer_port,
|
||||
size_bytes=size_bytes,
|
||||
shape=list(entry.shape),
|
||||
)
|
||||
logger.debug(
|
||||
"EC producer: announcing NIXL-readable encodings req_id=%s items=%s",
|
||||
|
||||
@@ -14,13 +14,20 @@ from collections import OrderedDict
|
||||
|
||||
|
||||
class CacheEntry:
|
||||
"""A single cache entry. Read `.block_ids` and `.ready` freely;
|
||||
mutations only through EmbeddingCache methods."""
|
||||
"""A single cache entry. Read `.block_ids`, `.shape` and `.ready` freely;
|
||||
mutations only through EmbeddingCache methods.
|
||||
|
||||
__slots__ = ("block_ids", "_pin_count")
|
||||
`shape` is the encoder output the blocks hold, so a load can restore it
|
||||
without knowing how wide this model's encoder output is.
|
||||
"""
|
||||
|
||||
def __init__(self, block_ids: tuple[int, ...]) -> None:
|
||||
__slots__ = ("block_ids", "shape", "_pin_count")
|
||||
|
||||
def __init__(
|
||||
self, block_ids: tuple[int, ...], shape: tuple[int, int] | None = None
|
||||
) -> None:
|
||||
self.block_ids = block_ids
|
||||
self.shape = shape
|
||||
self._pin_count = -1 # not ready
|
||||
|
||||
@property
|
||||
@@ -76,7 +83,9 @@ class EmbeddingCache:
|
||||
with self._lock:
|
||||
return len(self._entries) != len(self._entries_free_list)
|
||||
|
||||
def alloc(self, key: str, n_blocks: int) -> CacheEntry | None:
|
||||
def alloc(
|
||||
self, key: str, n_blocks: int, shape: tuple[int, int] | None = None
|
||||
) -> CacheEntry | None:
|
||||
"""Allocate *n_blocks* for *key*, evicting as needed.
|
||||
|
||||
The entry starts not-ready. Returns None if there is not enough
|
||||
@@ -97,7 +106,7 @@ class EmbeddingCache:
|
||||
# in the free list when it is evicted, and lets consumers detect
|
||||
# which of an entry's blocks are adjacent.
|
||||
block_ids = tuple(sorted(self._free_blocks.pop() for _ in range(n_blocks)))
|
||||
entry = CacheEntry(block_ids)
|
||||
entry = CacheEntry(block_ids, shape)
|
||||
self._entries[key] = entry
|
||||
return entry
|
||||
|
||||
|
||||
@@ -267,8 +267,7 @@ class ECCPUWorker:
|
||||
# so they are added up in uint64; signed arithmetic rejects them.
|
||||
src_base = np.uint64(src.view(-1).view(torch.uint8).data_ptr())
|
||||
dst_base = self._region.blocks.data_ptr()
|
||||
# Entries are sized by placeholder count, which can exceed the number
|
||||
# of embeddings, so blocks the output never reaches get no descriptor.
|
||||
# Copy only the tensor bytes, excluding any block padding.
|
||||
n_used = -(-total_bytes // block_size)
|
||||
if n_used:
|
||||
slots, first_blocks, num_blocks = _coalesce_runs(block_ids[:n_used])
|
||||
@@ -367,7 +366,7 @@ class ECCPUWorker:
|
||||
load_items = sorted(
|
||||
connector_metadata.loads.items(), key=lambda kv: kv[1][1][0]
|
||||
)
|
||||
total_blocks = sum(len(block_ids) for _, (_, block_ids) in load_items)
|
||||
total_blocks = sum(len(block_ids) for _, (_, block_ids, _) in load_items)
|
||||
|
||||
stream = self._acquire_stream()
|
||||
compute_stream = current_platform.current_stream()
|
||||
@@ -384,7 +383,7 @@ class ECCPUWorker:
|
||||
|
||||
bufs = self._buf_pool.acquire(total_blocks)
|
||||
slots, first_blocks, num_blocks = _coalesce_runs(
|
||||
list(chain.from_iterable(ids for _, (_, ids) in load_items))
|
||||
list(chain.from_iterable(ids for _, (_, ids, _) in load_items))
|
||||
)
|
||||
bufs.add_copies(
|
||||
0,
|
||||
@@ -405,7 +404,7 @@ class ECCPUWorker:
|
||||
Transfer(
|
||||
start_event=start_event,
|
||||
end_event=end_event,
|
||||
completions=[tid for _, (tid, _) in load_items],
|
||||
completions=[tid for _, (tid, _, _) in load_items],
|
||||
bufs=bufs,
|
||||
stream=stream,
|
||||
num_bytes=total_blocks * block_size,
|
||||
@@ -414,11 +413,10 @@ class ECCPUWorker:
|
||||
|
||||
# Slice contiguous buffer into per-hash views.
|
||||
offset = 0
|
||||
for mm_hash, (_, block_ids) in load_items:
|
||||
for mm_hash, (_, block_ids, shape) in load_items:
|
||||
n = len(block_ids)
|
||||
encoder_cache[mm_hash] = (
|
||||
dst_buf[offset : offset + n].view(dtype).reshape(n, -1)
|
||||
)
|
||||
flat = dst_buf[offset : offset + n].view(dtype).reshape(-1)
|
||||
encoder_cache[mm_hash] = flat[: shape[0] * shape[1]].view(shape)
|
||||
offset += n
|
||||
|
||||
current_platform.current_stream().wait_stream(stream)
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Encoder output widths for sizing EC transfers.
|
||||
|
||||
An encoder can emit features wider than the LM embedding (DeepStack packs
|
||||
several layers on the last dim) by an amount only the model knows. Producers
|
||||
size transfers before the first real item is encoded, so the engine measures
|
||||
the width once at startup by encoding a dummy item per modality.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def _measure_on_worker(worker: Any) -> dict[str, int]:
|
||||
"""Encode one dummy item per modality; runs on every worker.
|
||||
|
||||
Returns {} on ranks that hold no encoder.
|
||||
"""
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.encoder_budget import MultiModalBudget
|
||||
from vllm.multimodal.utils import group_and_batch_mm_kwargs
|
||||
from vllm.utils.torch_utils import PIN_MEMORY
|
||||
|
||||
runner = worker.model_runner
|
||||
if not runner.supports_mm_inputs or not get_pp_group().is_first_rank:
|
||||
return {}
|
||||
budget = MultiModalBudget(
|
||||
worker.vllm_config, MULTIMODAL_REGISTRY, enable_cache=False
|
||||
)
|
||||
model = runner.get_model()
|
||||
widths: dict[str, int] = {}
|
||||
for modality in budget.mm_max_toks_per_item:
|
||||
items = budget.get_dummy_encoder_profile_inputs(modality, 1)
|
||||
if not items:
|
||||
continue
|
||||
_, _, mm_kwargs = next(
|
||||
group_and_batch_mm_kwargs(
|
||||
items, device=runner.device, pin_memory=PIN_MEMORY
|
||||
)
|
||||
)
|
||||
outputs = model.embed_multimodal(**mm_kwargs)
|
||||
widths[modality] = int(outputs[0].shape[-1])
|
||||
del outputs
|
||||
return widths
|
||||
|
||||
|
||||
def measure_encoder_output_widths(
|
||||
vllm_config: "VllmConfig", collective_rpc: Callable[..., list[Any]]
|
||||
) -> None:
|
||||
"""Record the encoder output widths on an EC producer's config.
|
||||
|
||||
Call after the model is loaded and before the KV cache is allocated: the
|
||||
Scheduler's connector reads the widths, and once the KV cache holds the
|
||||
memory budget there is no room left for a dummy encode.
|
||||
|
||||
Raises:
|
||||
ValueError: No worker could encode a dummy item, or workers disagree.
|
||||
|
||||
"""
|
||||
ec_config = vllm_config.ec_transfer_config
|
||||
if ec_config is None or not ec_config.is_ec_producer:
|
||||
return
|
||||
reports = [widths for widths in collective_rpc(_measure_on_worker) if widths]
|
||||
if not reports:
|
||||
raise ValueError(
|
||||
"EC producer could not measure the encoder output width: the model "
|
||||
"encoded no dummy multimodal input."
|
||||
)
|
||||
if any(widths != reports[0] for widths in reports):
|
||||
raise ValueError(f"Workers disagree on the encoder output widths: {reports}")
|
||||
logger.info("EC producer encoder output widths: %s", reports[0])
|
||||
ec_config.mm_encoder_output_widths = reports[0]
|
||||
|
||||
|
||||
def get_encoder_output_width(vllm_config: "VllmConfig", modality: str) -> int:
|
||||
"""Encoder output width for `modality`, as measured at producer startup.
|
||||
|
||||
Raises:
|
||||
ValueError: No width was measured for `modality`, so a transfer
|
||||
cannot be sized.
|
||||
|
||||
"""
|
||||
assert vllm_config.ec_transfer_config is not None
|
||||
widths = vllm_config.ec_transfer_config.mm_encoder_output_widths
|
||||
if modality not in widths:
|
||||
raise ValueError(
|
||||
f"No encoder output width was measured for modality {modality!r}; "
|
||||
f"measured: {sorted(widths)}"
|
||||
)
|
||||
return widths[modality]
|
||||
@@ -20,8 +20,8 @@ import torch
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.cpu.common import (
|
||||
_get_encoder_cache_hidden_dim,
|
||||
from vllm.distributed.ec_transfer.ec_connector.encoder_output_width import (
|
||||
get_encoder_output_width,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.mooncake.config import (
|
||||
_RESERVATION_TTL_SECONDS,
|
||||
@@ -85,9 +85,7 @@ class ECMooncakeScheduler:
|
||||
self._is_consumer = config.is_consumer
|
||||
self._control_addr = config.control_addr
|
||||
self._push_wait_timeout = config.push_wait_timeout_s
|
||||
self._encoder_cache_hidden_dim = (
|
||||
_get_encoder_cache_hidden_dim(vllm_config) if config.is_producer else None
|
||||
)
|
||||
self._vllm_config = vllm_config
|
||||
self._model_config = vllm_config.model_config
|
||||
self._control_client = ControlClient(config.control_timeout_ms)
|
||||
self._control_executor = ThreadPoolExecutor(
|
||||
@@ -447,8 +445,10 @@ class ECMooncakeScheduler:
|
||||
def _encoder_output_spec(self, request: Any, index: int) -> TensorSpec:
|
||||
dtype = self._model_config.dtype
|
||||
assert isinstance(dtype, torch.dtype)
|
||||
assert self._encoder_cache_hidden_dim is not None
|
||||
shape = (request.get_num_encoder_embeds(index), self._encoder_cache_hidden_dim)
|
||||
hidden_dim = get_encoder_output_width(
|
||||
self._vllm_config, request.mm_features[index].modality
|
||||
)
|
||||
shape = (request.get_num_encoder_embeds(index), hidden_dim)
|
||||
return TensorSpec(shape, str(dtype), math.prod(shape) * dtype.itemsize)
|
||||
|
||||
def update_state_after_alloc(self, request: Any, index: int) -> None:
|
||||
|
||||
Reference in New Issue
Block a user