[MRV2] Enable routed-experts capture (#50721)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
aoshen02
2026-08-03 21:00:13 +00:00
committed by GitHub
co-authored by Nick Hill
parent e279f71583
commit 42ab184ea7
10 changed files with 335 additions and 227 deletions
@@ -15,6 +15,7 @@ from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.platforms import current_platform
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig
from vllm.v1.outputs import RoutedExpertsTensors
logger = logging.getLogger(__name__)
@@ -78,8 +79,7 @@ class RoutedExpertsCapturer:
Invariants:
- One instance per worker; shape is fixed at init and covers the
worst-case step (``max_num_batched_tokens`` tokens).
- :meth:`clear_buffer` is called at the start of every step, so
unused slots stay zero.
- Every routed layer overwrites the current step's token rows.
- ``device_buffer.dtype`` is ``torch.int32``.
"""
@@ -87,6 +87,7 @@ class RoutedExpertsCapturer:
self,
max_num_batched_tokens: int,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
) -> None:
hf_config = vllm_config.model_config.hf_text_config
num_experts_per_tok = _get_num_experts_per_tok(hf_config)
@@ -116,6 +117,8 @@ class RoutedExpertsCapturer:
)
self.dp_rank = vllm_config.parallel_config.data_parallel_rank
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
# KV cache group whose slot layout the routing data is keyed by.
self.attn_gid = get_routed_experts_attn_gid(kv_cache_config)
def capture(self, layer_id: int, topk_ids: torch.Tensor) -> None:
"""Capture expert routing decisions for a specific layer.
@@ -205,30 +208,102 @@ class RoutedExpertsCapturer:
f"tp_size={self.tp_size})"
)
# Defensive: model may expose more layers than the capture buffer
# was sized for (unusual, but guards against miss-config).
if layer_id >= self.device_buffer.shape[1]:
return
raise IndexError(
f"routed-experts layer {layer_id} exceeds capture buffer "
f"layer count {self.device_buffer.shape[1]}"
)
self.device_buffer[:token_num_per_dp, layer_id, :] = topk_ids[
start_loc:end_loc, :
]
def clear_buffer(self) -> None:
"""Zero the device buffer. Called at the start of every step so
slots belonging to finished / preempted tokens don't leak into
the next step.
"""
self.device_buffer.zero_()
def get_device_buffer(self) -> torch.Tensor:
"""Return the underlying device buffer so the model runner can
issue the D2H copy. The tensor is shared; callers must either
clone or fully drain it before the next forward pass runs
:meth:`clear_buffer`.
clone or fully drain it before the next forward pass overwrites it.
"""
return self.device_buffer
def get_routed_experts(
self, slot_mappings: torch.Tensor, num_tokens: int
) -> RoutedExpertsTensors:
"""Snapshot this step's routing data and its attention slot mapping.
Both tensors are cloned since the capture buffer and the shared
``slot_mappings`` are overwritten by the next step, which may race
with an in-flight D2H copy.
Args:
slot_mappings: Per-KV-cache-group slot mappings for this step,
shape ``(num_kv_cache_groups, max_num_batched_tokens)``.
num_tokens: Total number of tokens scheduled in this step.
"""
return RoutedExpertsTensors(
routing_data=self.device_buffer[:num_tokens].clone(),
slot_mapping=slot_mappings[self.attn_gid, :num_tokens].clone(),
)
def bind_routed_experts_capturer(
model: torch.nn.Module,
capturer: RoutedExpertsCapturer,
) -> None:
"""Attach capture callbacks to the target model's MoE routers."""
from vllm.model_executor.layers.fused_moe.layer import MoERunner
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
num_bound = 0
for module in model.modules():
if not isinstance(module, MoERunner):
continue
layer_id = module.layer_id
def capture_fn(
topk_ids: torch.Tensor,
layer_id: int = layer_id,
capturer: RoutedExpertsCapturer = capturer,
) -> None:
capturer.capture(layer_id, topk_ids)
quant_method = module._quant_method
moe_kernel = getattr(quant_method, "moe_kernel", None)
impl = getattr(moe_kernel, "impl", None)
fused_experts = getattr(impl, "fused_experts", None)
if quant_method.is_monolithic:
if not (
isinstance(fused_experts, FusedMoEExpertsMonolithic)
and fused_experts.supports_routing_replay_capture()
):
raise ValueError(
"Routed-experts capture is not supported with monolithic "
f"MoE kernel {type(fused_experts).__name__}."
)
fused_experts.set_capture_fn(capture_fn)
num_bound += 1
elif isinstance(module.router, BaseRouter):
module.router.set_capture_fn(capture_fn)
num_bound += 1
else:
raise ValueError(
"Routed-experts capture is not supported with router "
f"{type(module.router).__name__}."
)
if num_bound == 0:
raise ValueError("No supported MoE router found for routed-experts capture.")
def get_routed_experts_attn_gid(kv_cache_config: KVCacheConfig) -> int:
"""Return the full-attention KV cache group used for routed experts."""
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
if isinstance(group.kv_cache_spec, FullAttentionSpec):
return gid
raise ValueError("Routed-experts capture requires a full-attention KV cache group.")
class RoutedExpertsManager:
"""Scheduler-side slot-indexed buffer for routed experts.
@@ -260,17 +335,8 @@ class RoutedExpertsManager:
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
) -> None:
# Pick the attention group for block/slot mapping. We require
# a FullAttentionSpec group rather than any AttentionSpec to
# stay consistent with the worker-side lookup in
# ``GPUModelRunner._get_attention_kv_cache_gid``; hybrid models
# (Mamba / linear attention) also have other AttentionSpec
# groups whose slot layout differs.
self.attn_gid = next(
gid
for gid, g in enumerate(kv_cache_config.kv_cache_groups)
if isinstance(g.kv_cache_spec, FullAttentionSpec)
)
# Hybrid models also have KV groups whose slot layout differs.
self.attn_gid = get_routed_experts_attn_gid(kv_cache_config)
attn_group = kv_cache_config.kv_cache_groups[self.attn_gid]
self.block_size = attn_group.kv_cache_spec.block_size