[MRV2] Enable routed-experts capture (#50721)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
aoshen02
2026-08-03 21:00:13 +00:00
committed by GitHub
co-authored by Nick Hill
parent e279f71583
commit 42ab184ea7
10 changed files with 335 additions and 227 deletions
+28 -1
View File
@@ -39,6 +39,10 @@ from vllm.distributed.parallel_state import (
from vllm.forward_context import BatchDescriptor, set_forward_context
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
bind_routed_experts_capturer,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
initialize_mamba_ssu_backend,
)
@@ -55,7 +59,11 @@ from vllm.utils.mem_utils import DeviceMemoryProfiler, format_gib
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig, MambaSpec
from vllm.v1.outputs import DraftTokenIds, ModelRunnerOutput
from vllm.v1.outputs import (
DraftTokenIds,
ModelRunnerOutput,
RoutedExpertsTensors,
)
from vllm.v1.worker.block_table import get_block_table_width
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
from vllm.v1.worker.gpu import pcp_manager as pcp
@@ -279,11 +287,21 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# Expert parallelism load balancer.
self.eplb = EPLBController(self.parallel_config, self.device)
self.routed_experts_capturer: RoutedExpertsCapturer | None = None
def update_max_model_len(self, max_model_len: int) -> None:
self.max_model_len = max_model_len
self.req_states.max_model_len = max_model_len
def init_routed_experts_capturer(self) -> None:
"""Initialize target-model capture on every participating worker."""
self.routed_experts_capturer = RoutedExpertsCapturer(
max_num_batched_tokens=self.max_num_tokens,
vllm_config=self.vllm_config,
kv_cache_config=self.kv_cache_config,
)
bind_routed_experts_capturer(self.model, self.routed_experts_capturer)
def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
tasks: list[SupportedTask] = []
if self.model_config.runner_type == "generate":
@@ -1437,6 +1455,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
aux_hidden_states = None
output_intermediate_tensors = model_output
routed_experts = None
if not dummy_run and (capturer := self.routed_experts_capturer) is not None:
assert slot_mappings is not None
routed_experts = capturer.get_routed_experts(slot_mappings, num_toks)
finished_req_ids = scheduler_output.finished_req_ids
self.execute_model_state = ExecuteModelState(
input_batch=input_batch,
@@ -1445,6 +1468,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
hidden_states=hidden_states,
aux_hidden_states=aux_hidden_states,
finished_req_ids=finished_req_ids,
routed_experts=routed_experts,
)
if not self.is_last_pp_rank:
@@ -1467,6 +1491,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
hidden_states = self.execute_model_state.hidden_states
aux_hidden_states = self.execute_model_state.aux_hidden_states
finished_req_ids = self.execute_model_state.finished_req_ids
routed_experts = self.execute_model_state.routed_experts
self.execute_model_state = None
if not self.is_last_pp_rank:
@@ -1532,6 +1557,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
main_stream=self.main_stream,
copy_stream=self.output_copy_stream,
check_ep_fault=self.check_ep_fault,
routed_experts=routed_experts,
)
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
@@ -1712,6 +1738,7 @@ class ExecuteModelState(NamedTuple):
hidden_states: torch.Tensor | None
aux_hidden_states: list[torch.Tensor] | None
finished_req_ids: set[str]
routed_experts: RoutedExpertsTensors | None
def sort_batch_req_ids(