[RFC] Replace shared-memory routed experts with ModelRunnerOutput transfer and HTTP support (#39568)

Signed-off-by: xhx1022 <1737006628@qq.com>
Signed-off-by: arlenxu <arlenxu@tencent.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: arlenxu <arlenxu@tencent.com>
Co-authored-by: Junjie Zhang <junj.jay.zhang@gmail.com>
This commit is contained in:
Hongxin Xu
2026-05-14 14:12:30 +00:00
committed by GitHub
co-authored by Claude Opus 4.6 arlenxu Junjie Zhang
parent 2317682f95
commit c7560af424
12 changed files with 605 additions and 369 deletions
@@ -5,22 +5,16 @@
from __future__ import annotations
import fcntl
import logging
import os
import tempfile
from collections.abc import Generator
from contextlib import contextmanager
from multiprocessing import shared_memory
from unittest.mock import patch
import numpy as np
import torch
from vllm.config import VllmConfig
from vllm.distributed import get_tensor_model_parallel_rank
from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.platforms import current_platform
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig
logger = logging.getLogger(__name__)
@@ -42,151 +36,104 @@ def _get_num_experts_per_tok(hf_config) -> int:
return val
# Constants
_TMP_DIR = tempfile.gettempdir()
_LOCK_FILE_PREFIX = os.path.join(_TMP_DIR, "vllm_routed_experts")
_BUFFER_PREFIX = "vllm_routed_experts_buffer"
def get_num_experts(hf_config) -> int:
"""Resolve ``num_experts`` across HuggingFace config naming conventions.
# Global singleton instances
_global_experts_capturer: RoutedExpertsCapturer | None = None
_global_experts_reader: RoutedExpertsReader | None = None
@contextmanager
def _file_lock(lock_file: str, mode: str = "wb+") -> Generator[None, None, None]:
"""Context manager for file-based locking."""
with open(lock_file, mode) as fp:
fcntl.flock(fp, fcntl.LOCK_EX)
try:
yield
finally:
fcntl.flock(fp, fcntl.LOCK_UN)
def _create_or_attach_shared_memory(
name: str, size: int, lock_file: str
) -> shared_memory.SharedMemory:
"""Create or attach to shared memory with proper locking."""
# Ensure lock file exists before acquiring lock
with open(lock_file, "wb"):
pass
with _file_lock(lock_file):
try:
shm = shared_memory.SharedMemory(name=name, create=True, size=size)
except FileExistsError:
shm = shared_memory.SharedMemory(name=name, create=False, size=size)
if shm.size != size:
logger.warning(
"Shared memory %s size mismatch; recreating",
name,
)
shm.close()
shm.unlink()
try:
shm = shared_memory.SharedMemory(name=name, create=True, size=size)
logger.info("Created shared memory %s", name)
except FileExistsError:
shm = shared_memory.SharedMemory(name=name, create=False, size=size)
logger.info("Linked to existing shared memory %s", name)
return shm
Different MoE model families expose this under different keys:
- ``num_experts``: Mixtral, Qwen2-MoE, Qwen3-MoE
- ``n_routed_experts``: DeepSeek-V2/V3
- ``num_local_experts``: Mixtral (older exports)
"""
for key in ("num_experts", "n_routed_experts", "num_local_experts"):
val = getattr(hf_config, key, None)
if val is not None:
return val
raise ValueError(
"Could not resolve num_experts from model config. "
"Expected one of 'num_experts', 'n_routed_experts', "
"or 'num_local_experts'."
)
class RoutedExpertsCapturer:
"""
Capturer for routed experts with device and optional shared memory buffer.
"""Worker-side capturer for routed experts, lives on GPU.
This class captures expert routing decisions during model forward passes
and optionally stores them in shared memory for cross-process access.
Layer-level hooks call :meth:`capture` from inside the forward pass
with the per-layer ``topk_ids`` tensor. The tensor is sliced to the
tokens owned by this DP rank and written into a preallocated device
buffer. At the end of the step, :class:`GPUModelRunner` reads the
device buffer, issues a D2H copy into a pinned CPU buffer, and hands
the result to the scheduler via :class:`RoutedExpertsLists`.
The device / pinned-CPU transit buffers use ``torch.int32`` (not a
narrow ``uint8``/``uint16`` sized by ``num_experts``). This keeps the
SP all-gather path free of dtype casts, matches the router's native
``topk_ids`` indices dtype more closely, and costs only a few MB per
worker (``max_num_batched_tokens * num_layers * top_k * 4`` bytes).
The scheduler-side slot buffer
(``RoutedExpertsManager.routed_experts_by_slot``) still uses the
narrow dtype -- numpy fancy-index assignment in ``store_batch``
narrows the data on the way in.
Invariants:
- One instance per worker; shape is fixed at init and covers the
worst-case step (``max_num_batched_tokens`` tokens).
- :meth:`clear_buffer` is called at the start of every step, so
unused slots stay zero.
- ``device_buffer.dtype`` is ``torch.int32``.
"""
_instance: RoutedExpertsCapturer | None = None
def __init__(self) -> None:
self._device_buffer: torch.Tensor | None = None
self._shm: shared_memory.SharedMemory | None = None
self._host_buffer_view: np.ndarray | None = None
self._lock_file: str | None = None
@classmethod
def create(cls) -> RoutedExpertsCapturer:
"""Create a global singleton instance."""
global _global_experts_capturer
if _global_experts_capturer is not None:
raise RuntimeError("Experts capturer already created.")
_global_experts_capturer = cls()
return _global_experts_capturer
@staticmethod
def get_instance() -> RoutedExpertsCapturer | None:
"""Get the global singleton instance."""
return _global_experts_capturer
def init_buffer(
def __init__(
self,
max_num_batched_tokens: int,
max_num_kv_tokens: int,
vllm_config: VllmConfig,
) -> None:
"""
Initialize the device buffer and optionally shared memory buffer.
Args:
max_num_batched_tokens: Maximum number of tokens in a batch.
max_num_kv_tokens: Maximum number of KV tokens for shared memory.
vllm_config: vllm configuration containing layer and expert info.
"""
if self._device_buffer is not None:
raise RuntimeError("Device buffer has already been initialized")
hf_config = vllm_config.model_config.hf_text_config
num_layers = hf_config.num_hidden_layers
num_experts_per_tok = _get_num_experts_per_tok(hf_config)
# Initialize device buffer
self._device_buffer = torch.zeros(
(max_num_batched_tokens, num_layers, num_experts_per_tok),
self.device_buffer = torch.zeros(
(
max_num_batched_tokens,
hf_config.num_hidden_layers,
num_experts_per_tok,
),
# Use int32 for the device / host transit buffers: it
# matches the router's native topk_ids dtype, is universally
# supported by NCCL (uint8/uint16 are version-dependent),
# and the extra bytes are small (few MB per worker). The
# big scheduler-side slot buffer stays narrow.
dtype=torch.int32,
device=current_platform.device_type,
)
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
if get_tensor_model_parallel_rank() != 0:
return
# Initialize shared memory
shape = (max_num_kv_tokens, num_layers, num_experts_per_tok)
buffer_size = int(np.prod(shape)) * np.dtype(np.int32).itemsize
instance_id = vllm_config.instance_id
self._lock_file = f"{_LOCK_FILE_PREFIX}_{instance_id}_{self.dp_rank}.lock"
shm_name = f"{_BUFFER_PREFIX}_{instance_id}_{self.dp_rank}"
self._shm = _create_or_attach_shared_memory(
shm_name, buffer_size, self._lock_file
)
self._host_buffer_view = np.ndarray(shape, dtype=np.int32, buffer=self._shm.buf)
self._host_buffer_view.fill(0)
logger.debug(
"Created shared memory buffer '%s' with shape %s",
shm_name,
shape,
)
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
def capture(self, layer_id: int, topk_ids: torch.Tensor) -> None:
"""
Capture expert routing decisions for a specific layer.
"""Capture expert routing decisions for a specific layer.
Under data parallelism, ``topk_ids`` may have three different batch
layouts depending on where the DP combine happens and whether
Sequence Parallelism (SP) is active for the MoE layer:
- ``n == total`` (naive dispatch): all DP ranks' tokens are
concatenated before routing; we slice out this rank's span
using the cumulative per-rank counts.
- ``n == token_num_per_dp`` (modular-kernel path): DP combine
happens inside ``quant_method.apply``; ``select_experts`` only
ever sees this rank's tokens, so we take the whole tensor.
- ``n == ceil(token_num_per_dp / tp_size)`` (SP + modular-kernel
path): tokens were split along dim=0 across the TP group by
``_sequence_parallel_context``
(``moe_runner_base.py:_sequence_parallel_context``), so each
TP rank only sees its shard. We all-gather along dim=0 to
reconstruct this DP rank's full routing tensor. SP pads with
ceil-div (see ``_compute_sp_num_tokens`` in
``forward_context.py``), so the gathered tensor may contain a
few trailing padding rows which are trimmed by the downstream
``[:token_num_per_dp]`` slice.
Args:
layer_id: The layer index.
topk_ids: Tensor of shape (batch_size, num_routed_experts).
"""
if self._device_buffer is None:
raise RuntimeError("Buffer not initialized. Call init_buffer() first.")
ctx = get_forward_context()
if ctx.dp_metadata is None: # single dp
@@ -200,172 +147,203 @@ class RoutedExpertsCapturer:
n = topk_ids.shape[0]
if n == total:
# Naive dispatch: all DP ranks' tokens concatenated before routing.
# Naive dispatch: all DP ranks' tokens concatenated
# before routing. This rank owns tokens
# [end_loc - token_num_per_dp, end_loc).
cumsum = torch.cumsum(num_tokens_dp, dim=0)
end_loc = int(cumsum[self.dp_rank].item())
start_loc = end_loc - token_num_per_dp
elif n == token_num_per_dp:
# Modular-kernel path: DP combine happens inside quant_method.apply;
# select_experts only sees this rank's tokens.
# Modular-kernel path: DP combine happens inside
# quant_method.apply; select_experts only sees this
# rank's tokens, take the whole tensor.
start_loc = 0
end_loc = token_num_per_dp
elif (
self.tp_size > 1
and n != token_num_per_dp
and n == (token_num_per_dp + self.tp_size - 1) // self.tp_size
):
# SP + modular-kernel path. All-gather across the TP
# group along dim=0 to reconstruct the full per-DP-rank
# tensor; keep only the first ``token_num_per_dp`` rows
# (trailing rows are SP ceil-div padding). The TP group
# is always initialized on real rollout workers, and
# every rank in the group reaches this branch in
# lockstep (bind is per-FusedMoE layer, SP is a global
# condition), so a bare all_gather here will not
# deadlock -- let it raise if the precondition is
# violated rather than skip silently.
#
# ``topk_ids`` is already whatever the router produced
# (typically int32/int64, both supported by NCCL); the
# downstream ``device_buffer[...] = topk_ids[...]``
# setitem narrows into int32 automatically.
topk_ids = get_tp_group().all_gather(topk_ids, dim=0)
start_loc = 0
end_loc = token_num_per_dp
else:
sp_expected = (
(token_num_per_dp + self.tp_size - 1) // self.tp_size
if self.tp_size > 0
else -1
)
raise AssertionError(
"RoutedExpertsCapturer: unexpected topk_ids batch dim "
f"{n} (expected {total} or {token_num_per_dp} "
f"for dp_rank={self.dp_rank})"
"RoutedExpertsCapturer: unexpected topk_ids batch "
f"dim {n} (expected {total}, {token_num_per_dp}, "
f"or {sp_expected} for dp_rank={self.dp_rank}, "
f"tp_size={self.tp_size})"
)
if layer_id >= self._device_buffer.shape[1]:
# Defensive: model may expose more layers than the capture buffer
# was sized for (unusual, but guards against miss-config).
if layer_id >= self.device_buffer.shape[1]:
return
self._device_buffer[:token_num_per_dp, layer_id, :] = topk_ids[
self.device_buffer[:token_num_per_dp, layer_id, :] = topk_ids[
start_loc:end_loc, :
]
def clear_buffer(self) -> None:
"""Clear the device buffer."""
if self._device_buffer is not None:
self._device_buffer.zero_()
def save_captured_experts(self, indices: np.ndarray) -> None:
"""Zero the device buffer. Called at the start of every step so
slots belonging to finished / preempted tokens don't leak into
the next step.
"""
Save captured experts from device buffer to shared memory.
self.device_buffer.zero_()
Args:
indices: Array of indices indicating where to store the data.
def get_device_buffer(self) -> torch.Tensor:
"""Return the underlying device buffer so the model runner can
issue the D2H copy. The tensor is shared; callers must either
clone or fully drain it before the next forward pass runs
:meth:`clear_buffer`.
"""
if get_tensor_model_parallel_rank() != 0:
return
if self._lock_file is None:
raise RuntimeError("Shared memory not initialized.")
if self._host_buffer_view is None:
return
if self._device_buffer is None:
raise RuntimeError("Device buffer not initialized.")
num_tokens = len(indices)
data = self._device_buffer[:num_tokens, :, :].cpu().numpy()
with _file_lock(self._lock_file):
self._host_buffer_view[indices, :, :] = data
def cleanup(self) -> None:
"""Explicitly clean up shared memory resources."""
if self._shm is not None:
try:
self._shm.close()
self._shm.unlink()
except Exception:
logger.debug("Exception during cleanup for capturer", exc_info=True)
finally:
self._shm = None
def __del__(self) -> None:
"""Clean up shared memory on destruction."""
self.cleanup()
return self.device_buffer
class RoutedExpertsReader:
"""
Reader for routed experts from shared memory.
class RoutedExpertsManager:
"""Scheduler-side slot-indexed buffer for routed experts.
This class attaches to shared memory created by RoutedExpertsCapturer
and reads expert routing decisions.
Lives on CPU in the scheduler process. Each slot corresponds to
``block_id * block_size + offset_in_block`` where ``block_id`` is
drawn from the physical KV-cache block pool, so routing data is
tied to physical blocks and naturally survives preemption for
prefix-cached blocks (prefix hits re-expose the same slots).
Data flow per step:
1. Worker D2Hs its device capture buffer into
:class:`RoutedExpertsLists` and returns it via
:class:`ModelRunnerOutput`.
2. Scheduler calls :meth:`store_batch` with that step's
``(routing_data, slot_mapping)`` — a single CPU->CPU
fancy-index assign, ~few MB per step.
3. On request completion / abort / preemption, the scheduler
calls :meth:`get` with the request's block IDs to recover
the full per-token routing.
Memory: ``routed_experts_by_slot`` is sized for the whole block
pool (``num_blocks * block_size`` slots). For large block pools
this can reach multiple GB; see the init log for the exact size.
"""
_instance: RoutedExpertsReader | None = None
def __init__(self) -> None:
self._shm: shared_memory.SharedMemory | None = None
self._host_buffer_view: np.ndarray | None = None
self._lock_file: str | None = None
@classmethod
def create(cls) -> RoutedExpertsReader:
"""Create a global singleton instance."""
global _global_experts_reader
if _global_experts_reader is not None:
raise RuntimeError("Experts reader already created.")
_global_experts_reader = cls()
return _global_experts_reader
@staticmethod
def get_instance() -> RoutedExpertsReader | None:
"""Get the global singleton instance."""
if _global_experts_reader is None:
logger.info("Experts reader not initialized.")
return _global_experts_reader
def attach_buffer(
def __init__(
self,
max_num_kv_tokens: int,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
) -> None:
"""
Attach to an existing shared memory buffer.
Args:
max_num_kv_tokens: Maximum number of KV tokens.
vllm_config: vllm configuration.
"""
if self._shm is not None:
logger.warning("Already attached to shared memory buffer.")
return # Already attached
# Pick the attention group for block/slot mapping. We require
# a FullAttentionSpec group rather than any AttentionSpec to
# stay consistent with the worker-side lookup in
# ``GPUModelRunner._get_attention_kv_cache_gid``; hybrid models
# (Mamba / linear attention) also have other AttentionSpec
# groups whose slot layout differs.
self.attn_gid = next(
gid
for gid, g in enumerate(kv_cache_config.kv_cache_groups)
if isinstance(g.kv_cache_spec, FullAttentionSpec)
)
attn_group = kv_cache_config.kv_cache_groups[self.attn_gid]
self.block_size = attn_group.kv_cache_spec.block_size
# All kv_cache_groups share the same physical block pool, so
# block IDs span [0, num_blocks) regardless of how many groups
# exist. Sizing to the full pool avoids index-out-of-range
# when different groups happen to land on the same block.
hf_config = vllm_config.model_config.hf_text_config
shape = (
max_num_kv_tokens,
num_experts = get_num_experts(hf_config)
num_experts_per_tok = _get_num_experts_per_tok(hf_config)
max_num_slots = kv_cache_config.num_blocks * self.block_size
# Expert IDs are 0..num_experts-1; uint8 fits 256 distinct
# values so the boundary is ``<= 256`` (NOT ``< 256``). Keeping
# this narrow matters because the slot buffer is sized for the
# whole block pool and can reach multiple GB.
expert_id_dtype = np.uint8 if num_experts <= 256 else np.uint16
self.routed_experts_by_slot = np.zeros(
(
max_num_slots,
hf_config.num_hidden_layers,
num_experts_per_tok,
),
dtype=expert_id_dtype,
)
logger.info(
"RoutedExpertsManager CPU buffer: %.2f GB "
"(slots=%d, layers=%d, top_k=%d, dtype=%s)",
self.routed_experts_by_slot.nbytes / 1e9,
max_num_slots,
hf_config.num_hidden_layers,
_get_num_experts_per_tok(hf_config),
hf_config.num_experts_per_tok,
self.routed_experts_by_slot.dtype.name,
)
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
instance_id = vllm_config.instance_id
self._lock_file = f"{_LOCK_FILE_PREFIX}_{instance_id}_{self.dp_rank}.lock"
shm_name = f"{_BUFFER_PREFIX}_{instance_id}_{self.dp_rank}"
def store_batch(self, data: np.ndarray, slot_mapping: np.ndarray) -> None:
"""Persist one step's routed experts into the slot buffer.
with _file_lock(self._lock_file, mode="rb+"):
# Avoid resource_tracker registering the shared memory
with patch(
"multiprocessing.resource_tracker.register",
lambda *args, **kwargs: None,
):
self._shm = shared_memory.SharedMemory(name=shm_name)
self._host_buffer_view = np.ndarray(
shape, dtype=np.int32, buffer=self._shm.buf
)
def get_routed_experts(self, indices: np.ndarray) -> np.ndarray:
Equivalent to ``slot_buffer[slot_mapping] = data``; numpy fancy
indexing handles repeated / out-of-order indices. Called once
per scheduler step in ``update_from_output``.
"""
Read routed expert data from shared memory.
self.routed_experts_by_slot[slot_mapping] = data
def get(
self,
block_ids: list[int],
num_tokens: int,
token_start: int = 0,
) -> np.ndarray:
"""Read routed experts data for a completed / preempted request.
Reconstructs a per-token slot_mapping from the request's block
IDs and returns the routing slice. Because numpy fancy indexing
returns a **copy** (not a view), the returned ndarray is safe
to hold across subsequent :meth:`store_batch` calls — do not
replace the fancy index with a slice without re-verifying.
Args:
indices: Array of indices to read.
block_ids: Block IDs from the attention KV-cache group.
num_tokens: Number of tokens that have gone through a forward
pass and therefore have routing data written to their
slots (typically ``request.num_tokens - 1``; the last
sampled token has not been forwarded yet). Slots beyond
``request.num_computed_tokens`` are zero-initialized.
token_start: Skip the first ``token_start`` tokens from the
result. The slot_mapping is sliced before the fancy-index
read, so only the requested slots are fetched — no large
intermediate array is allocated. Clamped to
``[0, num_tokens]`` automatically.
Returns:
Copy of the expert routing data for the given indices.
Array of shape (num_tokens - token_start, num_layers,
num_experts_per_tok).
"""
if self._host_buffer_view is None:
raise RuntimeError("Buffer not attached. Call attach_buffer() first.")
if self._lock_file is None:
raise RuntimeError("Lock file not initialized.")
with _file_lock(self._lock_file, mode="rb+"):
return self._host_buffer_view[indices, :, :].copy()
def cleanup(self) -> None:
"""Explicitly clean up resources (close without unlink)."""
if self._shm is not None:
try:
self._shm.close()
except Exception:
logger.debug("Exception during cleanup for reader", exc_info=True)
finally:
self._shm = None
def __del__(self) -> None:
"""Close shared memory on destruction (do not unlink)."""
self.cleanup()
bs = self.block_size
block_ids_array = np.array(block_ids, dtype=np.int32)
block_offsets = np.arange(bs)
# slot = block_id * block_size + offset_in_block; flatten the
# (num_blocks, block_size) grid and trim to num_tokens, then
# skip the first token_start entries so only the requested
# range is fetched in a single fancy-index read.
slot_mapping = (
block_ids_array.reshape(-1, 1) * bs + block_offsets.reshape(1, -1)
).flatten()[:num_tokens]
slot_mapping = slot_mapping[token_start:]
return self.routed_experts_by_slot[slot_mapping]