[MRV2] Enable routed-experts capture (#50721)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
aoshen02
2026-08-03 21:00:13 +00:00
committed by GitHub
co-authored by Nick Hill
parent e279f71583
commit 42ab184ea7
10 changed files with 335 additions and 227 deletions
+19 -66
View File
@@ -63,6 +63,7 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
bind_routed_experts_capturer,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
initialize_mamba_ssu_backend,
@@ -2350,8 +2351,7 @@ class GPUModelRunner(
# by the next ``_prepare_inputs``; we need a stable snapshot
# because the async D2H may still be in flight on the copy
# stream when the next step runs.
attn_gid = self.routed_experts_attn_gid
slot_mapping_attn = slot_mappings[attn_gid]
slot_mapping_attn = slot_mappings[self.routed_experts_capturer.attn_gid]
self.routed_experts_slot_mapping_device[:num_tokens].copy_(
slot_mapping_attn[:num_tokens]
)
@@ -4173,9 +4173,6 @@ class GPUModelRunner(
"after execute_model() returns None."
)
if self.routed_experts_initialized:
self.routed_experts_capturer.clear_buffer()
# If ngram_gpu is used, we need to copy the scheduler_output to avoid
# the modification has influence on the scheduler_output in engine core process.
# The replace is much faster than deepcopy.
@@ -4789,24 +4786,16 @@ class GPUModelRunner(
# copy stream can D2H later. Both tensors must be private
# clones because:
# - ``routing_data`` source is the shared capturer buffer,
# which is ``clear_buffer()``-ed at the start of the
# next step on the default stream.
# which the next forward overwrites on the default stream.
# - ``slot_mapping`` source is our own
# ``routed_experts_slot_mapping_device``, which the
# next ``_prepare_inputs`` overwrites on the default
# stream while the D2H is still pending on the copy
# stream.
# Without clones, the copy stream would read torn data.
routed_experts_snapshot = None
if self.routed_experts_initialized:
buf = self.routed_experts_capturer.get_device_buffer()
total = scheduler_output.total_num_scheduled_tokens
routed_experts_snapshot = RoutedExpertsTensors(
routing_data=buf[:total].clone(),
slot_mapping=self.routed_experts_slot_mapping_device[
:total
].clone(),
)
routed_experts_snapshot = self.get_routed_experts(
scheduler_output.total_num_scheduled_tokens
)
async_output = AsyncGPUModelRunnerOutput(
model_runner_output=output,
@@ -7663,19 +7652,18 @@ class GPUModelRunner(
kv_transfer_group.register_kv_caches(kv_caches)
kv_transfer_group.set_host_xfer_buffer_ops(copy_kv_blocks)
def _get_attention_kv_cache_gid(self) -> int:
"""Find the KV cache group index for attention layers.
def get_routed_experts(
self,
num_tokens: int,
) -> RoutedExpertsTensors | None:
if not self.routed_experts_initialized:
return None
Must match :attr:`RoutedExpertsManager.attn_gid` in the scheduler:
both pick the first ``FullAttentionSpec`` group so hybrid models
(Mamba / linear-attention layers that use other AttentionSpec
subclasses) end up indexing the same slot layout on both sides.
Falls back to 0 only for legacy single-group configs.
"""
for gid, group in enumerate(self.kv_cache_config.kv_cache_groups):
if isinstance(group.kv_cache_spec, FullAttentionSpec):
return gid
return 0
device_buffer = self.routed_experts_capturer.get_device_buffer()
return RoutedExpertsTensors(
routing_data=device_buffer[:num_tokens].clone(),
slot_mapping=self.routed_experts_slot_mapping_device[:num_tokens].clone(),
)
def init_routed_experts_capturer(self):
logger.info(
@@ -7685,9 +7673,9 @@ class GPUModelRunner(
self.routed_experts_capturer = RoutedExpertsCapturer(
max_num_batched_tokens=self.scheduler_config.max_num_batched_tokens,
vllm_config=self.vllm_config,
kv_cache_config=self.kv_cache_config,
)
self.routed_experts_attn_gid = self._get_attention_kv_cache_gid()
self._bind_routed_experts_capturer(self.routed_experts_capturer)
bind_routed_experts_capturer(self.model, self.routed_experts_capturer)
# Pinned CPU buffer for non-blocking D2H of ``routing_data`` on
# the sync scheduling path. Shape / dtype mirror the device
@@ -7719,41 +7707,6 @@ class GPUModelRunner(
)
self.routed_experts_initialized = True
def _bind_routed_experts_capturer(self, capturer: RoutedExpertsCapturer) -> None:
from vllm.model_executor.layers.fused_moe.layer import MoERunner
from vllm.model_executor.layers.fused_moe.modular_kernel import (
FusedMoEExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.router.base_router import (
BaseRouter,
)
for module in self.model.modules():
if not isinstance(module, MoERunner):
continue
layer_id = module.layer_id
def _capture_fn(topk_ids, _layer_id=layer_id, _capturer=capturer):
_capturer.capture(_layer_id, topk_ids)
quant_method = module._quant_method
moe_kernel = getattr(quant_method, "moe_kernel", None)
impl = getattr(moe_kernel, "impl", None)
fused_experts = getattr(impl, "fused_experts", None)
if quant_method.is_monolithic:
if not (
isinstance(fused_experts, FusedMoEExpertsMonolithic)
and fused_experts.supports_routing_replay_capture()
):
raise ValueError(
"--enable-return-routed-experts is not supported with "
f"monolithic MoE kernel {type(fused_experts).__name__}; "
"routed expert IDs would be silently all-zero."
)
fused_experts.set_capture_fn(_capture_fn)
elif isinstance(module.router, BaseRouter):
module.router.set_capture_fn(_capture_fn)
def may_add_encoder_only_layers_to_kv_cache_config(self) -> None:
"""
Add encoder-only layers to the KV cache config.