[MRV2] Enable routed-experts capture (#50721)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
aoshen02
2026-08-03 21:00:13 +00:00
committed by GitHub
co-authored by Nick Hill
parent e279f71583
commit 42ab184ea7
10 changed files with 335 additions and 227 deletions
-4
View File
@@ -2228,10 +2228,6 @@ class VllmConfig:
if self.parallel_config.enable_elastic_ep:
unsupported.append("elastic expert parallelism")
if model_config is not None and model_config.enable_return_routed_experts:
# Will be added by https://github.com/vllm-project/vllm/pull/38163
unsupported.append("routed experts capture")
has_logitsproc_plugins = False
if model_config is not None:
from importlib.metadata import entry_points
@@ -15,6 +15,7 @@ from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.platforms import current_platform
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig
from vllm.v1.outputs import RoutedExpertsTensors
logger = logging.getLogger(__name__)
@@ -78,8 +79,7 @@ class RoutedExpertsCapturer:
Invariants:
- One instance per worker; shape is fixed at init and covers the
worst-case step (``max_num_batched_tokens`` tokens).
- :meth:`clear_buffer` is called at the start of every step, so
unused slots stay zero.
- Every routed layer overwrites the current step's token rows.
- ``device_buffer.dtype`` is ``torch.int32``.
"""
@@ -87,6 +87,7 @@ class RoutedExpertsCapturer:
self,
max_num_batched_tokens: int,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
) -> None:
hf_config = vllm_config.model_config.hf_text_config
num_experts_per_tok = _get_num_experts_per_tok(hf_config)
@@ -116,6 +117,8 @@ class RoutedExpertsCapturer:
)
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
# KV cache group whose slot layout the routing data is keyed by.
self.attn_gid = get_routed_experts_attn_gid(kv_cache_config)
def capture(self, layer_id: int, topk_ids: torch.Tensor) -> None:
"""Capture expert routing decisions for a specific layer.
@@ -205,30 +208,102 @@ class RoutedExpertsCapturer:
f"tp_size={self.tp_size})"
)
# Defensive: model may expose more layers than the capture buffer
# was sized for (unusual, but guards against miss-config).
if layer_id >= self.device_buffer.shape[1]:
return
raise IndexError(
f"routed-experts layer {layer_id} exceeds capture buffer "
f"layer count {self.device_buffer.shape[1]}"
)
self.device_buffer[:token_num_per_dp, layer_id, :] = topk_ids[
start_loc:end_loc, :
]
def clear_buffer(self) -> None:
"""Zero the device buffer. Called at the start of every step so
slots belonging to finished / preempted tokens don't leak into
the next step.
"""
self.device_buffer.zero_()
def get_device_buffer(self) -> torch.Tensor:
"""Return the underlying device buffer so the model runner can
issue the D2H copy. The tensor is shared; callers must either
clone or fully drain it before the next forward pass runs
:meth:`clear_buffer`.
clone or fully drain it before the next forward pass overwrites it.
"""
return self.device_buffer
def get_routed_experts(
self, slot_mappings: torch.Tensor, num_tokens: int
) -> RoutedExpertsTensors:
"""Snapshot this step's routing data and its attention slot mapping.
Both tensors are cloned since the capture buffer and the shared
``slot_mappings`` are overwritten by the next step, which may race
with an in-flight D2H copy.
Args:
slot_mappings: Per-KV-cache-group slot mappings for this step,
shape ``(num_kv_cache_groups, max_num_batched_tokens)``.
num_tokens: Total number of tokens scheduled in this step.
"""
return RoutedExpertsTensors(
routing_data=self.device_buffer[:num_tokens].clone(),
slot_mapping=slot_mappings[self.attn_gid, :num_tokens].clone(),
)
def bind_routed_experts_capturer(
model: torch.nn.Module,
capturer: RoutedExpertsCapturer,
) -> None:
"""Attach capture callbacks to the target model's MoE routers."""
from vllm.model_executor.layers.fused_moe.layer import MoERunner
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
num_bound = 0
for module in model.modules():
if not isinstance(module, MoERunner):
continue
layer_id = module.layer_id
def capture_fn(
topk_ids: torch.Tensor,
layer_id: int = layer_id,
capturer: RoutedExpertsCapturer = capturer,
) -> None:
capturer.capture(layer_id, topk_ids)
quant_method = module._quant_method
moe_kernel = getattr(quant_method, "moe_kernel", None)
impl = getattr(moe_kernel, "impl", None)
fused_experts = getattr(impl, "fused_experts", None)
if quant_method.is_monolithic:
if not (
isinstance(fused_experts, FusedMoEExpertsMonolithic)
and fused_experts.supports_routing_replay_capture()
):
raise ValueError(
"Routed-experts capture is not supported with monolithic "
f"MoE kernel {type(fused_experts).__name__}."
)
fused_experts.set_capture_fn(capture_fn)
num_bound += 1
elif isinstance(module.router, BaseRouter):
module.router.set_capture_fn(capture_fn)
num_bound += 1
else:
raise ValueError(
"Routed-experts capture is not supported with router "
f"{type(module.router).__name__}."
)
if num_bound == 0:
raise ValueError("No supported MoE router found for routed-experts capture.")
def get_routed_experts_attn_gid(kv_cache_config: KVCacheConfig) -> int:
"""Return the full-attention KV cache group used for routed experts."""
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
if isinstance(group.kv_cache_spec, FullAttentionSpec):
return gid
raise ValueError("Routed-experts capture requires a full-attention KV cache group.")
class RoutedExpertsManager:
"""Scheduler-side slot-indexed buffer for routed experts.
@@ -260,17 +335,8 @@ class RoutedExpertsManager:
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
) -> None:
# Pick the attention group for block/slot mapping. We require
# a FullAttentionSpec group rather than any AttentionSpec to
# stay consistent with the worker-side lookup in
# ``GPUModelRunner._get_attention_kv_cache_gid``; hybrid models
# (Mamba / linear attention) also have other AttentionSpec
# groups whose slot layout differs.
self.attn_gid = next(
gid
for gid, g in enumerate(kv_cache_config.kv_cache_groups)
if isinstance(g.kv_cache_spec, FullAttentionSpec)
)
# Hybrid models also have KV groups whose slot layout differs.
self.attn_gid = get_routed_experts_attn_gid(kv_cache_config)
attn_group = kv_cache_config.kv_cache_groups[self.attn_gid]
self.block_size = attn_group.kv_cache_spec.block_size
+28 -18
View File
@@ -202,10 +202,10 @@ def detach_zero_copy_from_model_runner_output(output: "ModelRunnerOutput") -> No
backed by Ray's shared-memory object store. Ray's channel docs explicitly
warn that subsequent reads may block if such an object is still in scope.
vLLM can return numpy-backed logprobs in `ModelRunnerOutput.logprobs`. If
those arrays are backed by Ray SHM (commonly read-only), retaining them in
scope across scheduler iterations can stall the channel and eventually hit
`RAY_CGRAPH_get_timeout`.
vLLM can return numpy-backed logprobs and routed experts in
`ModelRunnerOutput`. If those arrays are backed by Ray SHM (commonly
read-only), retaining them in scope across scheduler iterations can stall
the channel and eventually hit `RAY_CGRAPH_get_timeout`.
Copy read-only numpy arrays so the returned output no longer retains
references to Ray's shared-memory buffers.
@@ -214,27 +214,37 @@ def detach_zero_copy_from_model_runner_output(output: "ModelRunnerOutput") -> No
`LogprobsTensors` backed by PyTorch-owned CPU tensors (`to_cpu_nonblocking`
or `empty_cpu`), not NumPy views decoded from Ray channels.
"""
if output.logprobs is None:
return
token_ids, logprobs, ranks, cu_num_generated_tokens = output.logprobs
def _copy_if_readonly(arr):
if isinstance(arr, np.ndarray) and not arr.flags.writeable:
return arr.copy()
return arr
# `cu_num_generated_tokens` is already a plain Python list (or None), so it
# never aliases Ray SHM buffers and can be reused as-is.
token_ids_c = _copy_if_readonly(token_ids)
logprobs_c = _copy_if_readonly(logprobs)
ranks_c = _copy_if_readonly(ranks)
if token_ids_c is token_ids and logprobs_c is logprobs and ranks_c is ranks:
return
if output.logprobs is not None:
token_ids, logprobs, ranks, cu_num_generated_tokens = output.logprobs
output.logprobs = type(output.logprobs)(
token_ids_c, logprobs_c, ranks_c, cu_num_generated_tokens
)
# `cu_num_generated_tokens` is already a plain Python list (or None),
# so it never aliases Ray SHM buffers and can be reused as-is.
token_ids_c = _copy_if_readonly(token_ids)
logprobs_c = _copy_if_readonly(logprobs)
ranks_c = _copy_if_readonly(ranks)
if (
token_ids_c is not token_ids
or logprobs_c is not logprobs
or ranks_c is not ranks
):
output.logprobs = type(output.logprobs)(
token_ids_c, logprobs_c, ranks_c, cu_num_generated_tokens
)
if output.routed_experts is not None:
routing_data, slot_mapping = output.routed_experts
routing_data_c = _copy_if_readonly(routing_data)
slot_mapping_c = _copy_if_readonly(slot_mapping)
if routing_data_c is not routing_data or slot_mapping_c is not slot_mapping:
output.routed_experts = type(output.routed_experts)(
routing_data_c, slot_mapping_c
)
class FutureWrapper(Future):
+8
View File
@@ -11,6 +11,7 @@ from vllm.v1.outputs import (
LogprobsTensors,
ModelRunnerOutput,
PoolerOutput,
RoutedExpertsTensors,
)
from vllm.v1.worker.gpu.sample.output import SamplerOutput
@@ -24,6 +25,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
main_stream: torch.cuda.Stream,
copy_stream: torch.cuda.Stream,
check_ep_fault: bool = False,
routed_experts: RoutedExpertsTensors | None = None,
):
# NOTE(woosuk): We must retain references to the GPU tensors,
# as the copy operations are performed on a different CUDA stream than
@@ -31,6 +33,7 @@ class AsyncOutput(AsyncModelRunnerOutput):
self.model_runner_output = model_runner_output
self.sampler_output = sampler_output
self.num_sampled_tokens = num_sampled_tokens
self.routed_experts = routed_experts
# Blocking (sleep) event to avoid busy-polling the CUDA driver lock.
self.copy_event = torch.cuda.Event(blocking=True)
self._has_fault: torch.Tensor | None = None
@@ -48,6 +51,9 @@ class AsyncOutput(AsyncModelRunnerOutput):
if sampler_output.num_nans is not None:
self.num_nans = async_copy_to_np(sampler_output.num_nans)
self.num_sampled_tokens_np = async_copy_to_np(num_sampled_tokens)
self.routed_experts_cpu: RoutedExpertsTensors | None = None
if routed_experts is not None:
self.routed_experts_cpu = routed_experts.to_cpu_nonblocking()
self.prompt_logprobs_dict = {
k: v.to_cpu_nonblocking() if v is not None else None
for k, v in self.model_runner_output.prompt_logprobs_dict.items()
@@ -78,6 +84,8 @@ class AsyncOutput(AsyncModelRunnerOutput):
if self.logprobs_tensors is not None:
self.model_runner_output.logprobs = self.logprobs_tensors.tolists()
self.model_runner_output.prompt_logprobs_dict = self.prompt_logprobs_dict
if self.routed_experts_cpu is not None:
self.model_runner_output.routed_experts = self.routed_experts_cpu.tolists()
if self._has_fault is not None and self._has_fault.item():
mask = get_ep_all2all_manager().query_active_mask()
+28 -1
View File
@@ -39,6 +39,10 @@ from vllm.distributed.parallel_state import (
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
bind_routed_experts_capturer,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
initialize_mamba_ssu_backend,
)
@@ -55,7 +59,11 @@ from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput
from vllm.v1.outputs import (
DraftTokenIds,
ModelRunnerOutput,
RoutedExpertsTensors,
)
from vllm.v1.worker.block_table import get_block_table_width
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
from vllm.v1.worker.gpu import pcp_manager as pcp
@@ -279,11 +287,21 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Expert parallelism load balancer.
self.eplb = EPLBController(self.parallel_config, self.device)
self.routed_experts_capturer: RoutedExpertsCapturer | None = None
def update_max_model_len(self, max_model_len: int) -> None:
self.max_model_len = max_model_len
self.req_states.max_model_len = max_model_len
def init_routed_experts_capturer(self) -> None:
"""Initialize target-model capture on every participating worker."""
self.routed_experts_capturer = RoutedExpertsCapturer(
max_num_batched_tokens=self.max_num_tokens,
vllm_config=self.vllm_config,
kv_cache_config=self.kv_cache_config,
)
bind_routed_experts_capturer(self.model, self.routed_experts_capturer)
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
tasks: list[SupportedTask] = []
if self.model_config.runner_type == "generate":
@@ -1437,6 +1455,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
aux_hidden_states = None
output_intermediate_tensors = model_output
routed_experts = None
if not dummy_run and (capturer := self.routed_experts_capturer) is not None:
assert slot_mappings is not None
routed_experts = capturer.get_routed_experts(slot_mappings, num_toks)
finished_req_ids = scheduler_output.finished_req_ids
self.execute_model_state = ExecuteModelState(
input_batch=input_batch,
@@ -1445,6 +1468,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
hidden_states=hidden_states,
aux_hidden_states=aux_hidden_states,
finished_req_ids=finished_req_ids,
routed_experts=routed_experts,
)
if not self.is_last_pp_rank:
@@ -1467,6 +1491,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
hidden_states = self.execute_model_state.hidden_states
aux_hidden_states = self.execute_model_state.aux_hidden_states
finished_req_ids = self.execute_model_state.finished_req_ids
routed_experts = self.execute_model_state.routed_experts
self.execute_model_state = None
if not self.is_last_pp_rank:
@@ -1532,6 +1557,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
main_stream=self.main_stream,
copy_stream=self.output_copy_stream,
check_ep_fault=self.check_ep_fault,
routed_experts=routed_experts,
)
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
@@ -1712,6 +1738,7 @@ class ExecuteModelState(NamedTuple):
hidden_states: torch.Tensor | None
aux_hidden_states: list[torch.Tensor] | None
finished_req_ids: set[str]
routed_experts: RoutedExpertsTensors | None
def sort_batch_req_ids(
+19 -66
View File
@@ -63,6 +63,7 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
bind_routed_experts_capturer,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
initialize_mamba_ssu_backend,
@@ -2350,8 +2351,7 @@ class GPUModelRunner(
# by the next ``_prepare_inputs``; we need a stable snapshot
# because the async D2H may still be in flight on the copy
# stream when the next step runs.
attn_gid = self.routed_experts_attn_gid
slot_mapping_attn = slot_mappings[attn_gid]
slot_mapping_attn = slot_mappings[self.routed_experts_capturer.attn_gid]
self.routed_experts_slot_mapping_device[:num_tokens].copy_(
slot_mapping_attn[:num_tokens]
)
@@ -4173,9 +4173,6 @@ class GPUModelRunner(
"after execute_model() returns None."
)
if self.routed_experts_initialized:
self.routed_experts_capturer.clear_buffer()
# If ngram_gpu is used, we need to copy the scheduler_output to avoid
# the modification has influence on the scheduler_output in engine core process.
# The replace is much faster than deepcopy.
@@ -4789,24 +4786,16 @@ class GPUModelRunner(
# copy stream can D2H later. Both tensors must be private
# clones because:
# - ``routing_data`` source is the shared capturer buffer,
# which is ``clear_buffer()``-ed at the start of the
# next step on the default stream.
# which the next forward overwrites on the default stream.
# - ``slot_mapping`` source is our own
# ``routed_experts_slot_mapping_device``, which the
# next ``_prepare_inputs`` overwrites on the default
# stream while the D2H is still pending on the copy
# stream.
# Without clones, the copy stream would read torn data.
routed_experts_snapshot = None
if self.routed_experts_initialized:
buf = self.routed_experts_capturer.get_device_buffer()
total = scheduler_output.total_num_scheduled_tokens
routed_experts_snapshot = RoutedExpertsTensors(
routing_data=buf[:total].clone(),
slot_mapping=self.routed_experts_slot_mapping_device[
:total
].clone(),
)
routed_experts_snapshot = self.get_routed_experts(
scheduler_output.total_num_scheduled_tokens
)
async_output = AsyncGPUModelRunnerOutput(
model_runner_output=output,
@@ -7663,19 +7652,18 @@ class GPUModelRunner(
kv_transfer_group.register_kv_caches(kv_caches)
kv_transfer_group.set_host_xfer_buffer_ops(copy_kv_blocks)
def _get_attention_kv_cache_gid(self) -> int:
"""Find the KV cache group index for attention layers.
def get_routed_experts(
self,
num_tokens: int,
) -> RoutedExpertsTensors | None:
if not self.routed_experts_initialized:
return None
Must match :attr:`RoutedExpertsManager.attn_gid` in the scheduler:
both pick the first ``FullAttentionSpec`` group so hybrid models
(Mamba / linear-attention layers that use other AttentionSpec
subclasses) end up indexing the same slot layout on both sides.
Falls back to 0 only for legacy single-group configs.
"""
for gid, group in enumerate(self.kv_cache_config.kv_cache_groups):
if isinstance(group.kv_cache_spec, FullAttentionSpec):
return gid
return 0
device_buffer = self.routed_experts_capturer.get_device_buffer()
return RoutedExpertsTensors(
routing_data=device_buffer[:num_tokens].clone(),
slot_mapping=self.routed_experts_slot_mapping_device[:num_tokens].clone(),
)
def init_routed_experts_capturer(self):
logger.info(
@@ -7685,9 +7673,9 @@ class GPUModelRunner(
self.routed_experts_capturer = RoutedExpertsCapturer(
max_num_batched_tokens=self.scheduler_config.max_num_batched_tokens,
vllm_config=self.vllm_config,
kv_cache_config=self.kv_cache_config,
)
self.routed_experts_attn_gid = self._get_attention_kv_cache_gid()
self._bind_routed_experts_capturer(self.routed_experts_capturer)
bind_routed_experts_capturer(self.model, self.routed_experts_capturer)
# Pinned CPU buffer for non-blocking D2H of ``routing_data`` on
# the sync scheduling path. Shape / dtype mirror the device
@@ -7719,41 +7707,6 @@ class GPUModelRunner(
)
self.routed_experts_initialized = True
def _bind_routed_experts_capturer(self, capturer: RoutedExpertsCapturer) -> None:
from vllm.model_executor.layers.fused_moe.layer import MoERunner
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.router.base_router import (
BaseRouter,
)
for module in self.model.modules():
if not isinstance(module, MoERunner):
continue
layer_id = module.layer_id
def _capture_fn(topk_ids, _layer_id=layer_id, _capturer=capturer):
_capturer.capture(_layer_id, topk_ids)
quant_method = module._quant_method
moe_kernel = getattr(quant_method, "moe_kernel", None)
impl = getattr(moe_kernel, "impl", None)
fused_experts = getattr(impl, "fused_experts", None)
if quant_method.is_monolithic:
if not (
isinstance(fused_experts, FusedMoEExpertsMonolithic)
and fused_experts.supports_routing_replay_capture()
):
raise ValueError(
"--enable-return-routed-experts is not supported with "
f"monolithic MoE kernel {type(fused_experts).__name__}; "
"routed expert IDs would be silently all-zero."
)
fused_experts.set_capture_fn(_capture_fn)
elif isinstance(module.router, BaseRouter):
module.router.set_capture_fn(_capture_fn)
def may_add_encoder_only_layers_to_kv_cache_config(self) -> None:
"""
Add encoder-only layers to the KV cache config.