[Core] Replace routing replay with device cache and async D2H pipeline (#39917)

Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
TomerBN-Nvidia
2026-05-07 11:24:56 -07:00
committed by GitHub
parent 8eb401134e
commit 8189a15914
15 changed files with 1398 additions and 642 deletions
+51
View File
@@ -1157,6 +1157,12 @@ class VllmConfig:
if envs.VLLM_USE_V2_MODEL_RUNNER:
self._validate_v2_model_runner()
if (
self.model_config is not None
and self.model_config.enable_return_routed_experts
):
self._validate_return_routed_experts()
# Re-compute compile ranges after platform-specific config updates
# (e.g., XPU may lower max_num_batched_tokens when MLA is enabled)
self._set_compile_ranges()
@@ -1846,6 +1852,51 @@ class VllmConfig:
+ ", ".join(unsupported)
)
def _validate_return_routed_experts(self) -> None:
"""Reject parallelism configurations not yet validated with
--enable-return-routed-experts.
Validated scope (PR #39917): TP, EP, DP, single-node and multi-node,
prefix caching, and speculative decoding (MTP validated end-to-end;
Eagle/Eagle3/Ngram/Medusa supported by construction since the
routing buffer is bound only to the target model and verified-token
routing lands at the correct positions during the main forward).
Out-of-scope (block until validated): PP > 1, prefill context
parallelism (PCP) > 1, decode context parallelism (DCP) > 1,
async scheduling.
"""
unsupported: list[str] = []
if self.parallel_config.pipeline_parallel_size > 1:
unsupported.append(
"pipeline parallelism "
f"(pipeline_parallel_size="
f"{self.parallel_config.pipeline_parallel_size})"
)
if self.parallel_config.prefill_context_parallel_size > 1:
unsupported.append(
"prefill context parallelism "
f"(prefill_context_parallel_size="
f"{self.parallel_config.prefill_context_parallel_size})"
)
if self.parallel_config.decode_context_parallel_size > 1:
unsupported.append(
"decode context parallelism "
f"(decode_context_parallel_size="
f"{self.parallel_config.decode_context_parallel_size})"
)
if self.scheduler_config.async_scheduling:
unsupported.append("async scheduling")
if unsupported:
raise ValueError(
"--enable-return-routed-experts is not yet validated with: "
+ ", ".join(unsupported)
+ ". Disable these features or omit "
"--enable-return-routed-experts."
)
def validate_block_size(self) -> None:
"""Validate block_size against DCP and mamba constraints.
@@ -92,12 +92,16 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
# not part of the OpenAI spec but is useful for tracing the tokens
# in agent scenarios
token_ids: list[int] | None = None
routed_experts: list[list[list[int]]] | None = None # [gen_len, num_layers, top_k]
class ChatCompletionResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
object: Literal["chat.completion"] = "chat.completion"
created: int = Field(default_factory=lambda: int(time.time()))
prompt_routed_experts: list[list[list[int]]] | None = (
None # [prompt_len, num_layers, top_k]
)
model: str
choices: list[ChatCompletionResponseChoice]
service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
@@ -1088,6 +1088,11 @@ class OpenAIServingChat(OpenAIServing):
token_ids=(
as_list(output.token_ids) if request.return_token_ids else None
),
routed_experts=(
output.routed_experts.tolist()
if output.routed_experts is not None
else None
),
)
choices.append(choice_data)
continue
@@ -1309,6 +1314,11 @@ class OpenAIServingChat(OpenAIServing):
token_ids=(
as_list(output.token_ids) if request.return_token_ids else None
),
routed_experts=(
output.routed_experts.tolist()
if output.routed_experts is not None
else None
),
)
choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
@@ -1348,6 +1358,10 @@ class OpenAIServingChat(OpenAIServing):
request_metadata.final_usage_info = usage
prompt_routed_experts = None
if final_res.prompt_routed_experts is not None:
prompt_routed_experts = final_res.prompt_routed_experts.tolist()
response = ChatCompletionResponse(
id=request_id,
created=created_time,
@@ -1360,6 +1374,7 @@ class OpenAIServingChat(OpenAIServing):
final_res.prompt_token_ids if request.return_token_ids else None
),
kv_transfer_params=final_res.kv_transfer_params,
prompt_routed_experts=prompt_routed_experts,
)
# Log complete response if output logging is enabled
@@ -468,12 +468,16 @@ class CompletionResponseChoice(OpenAIBaseModel):
token_ids: list[int] | None = None # For response
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
prompt_token_ids: list[int] | None = None # For prompt
routed_experts: list[list[list[int]]] | None = None # [gen_len, num_layers, top_k]
class CompletionResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
object: Literal["text_completion"] = "text_completion"
created: int = Field(default_factory=lambda: int(time.time()))
prompt_routed_experts: list[list[list[int]]] | None = (
None # [prompt_len, num_layers, top_k]
)
model: str
choices: list[CompletionResponseChoice]
service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
@@ -542,6 +542,11 @@ class OpenAIServingCompletion(OpenAIServing):
token_ids=(
as_list(output.token_ids) if request.return_token_ids else None
),
routed_experts=(
output.routed_experts.tolist()
if output.routed_experts is not None
else None
),
)
choices.append(choice_data)
@@ -565,8 +570,13 @@ class OpenAIServingCompletion(OpenAIServing):
)
request_metadata.final_usage_info = usage
prompt_routed_experts = None
if final_res_batch:
kv_transfer_params = final_res_batch[0].kv_transfer_params
pre = final_res_batch[0].prompt_routed_experts
if pre is not None:
prompt_routed_experts = pre.tolist()
return CompletionResponse(
id=request_id,
created=created_time,
@@ -575,6 +585,7 @@ class OpenAIServingCompletion(OpenAIServing):
usage=usage,
system_fingerprint=self.system_fingerprint,
kv_transfer_params=kv_transfer_params,
prompt_routed_experts=prompt_routed_experts,
)
def _create_completion_logprobs(
@@ -246,6 +246,9 @@ class FusedMoE(PluggableLayer):
not supported by the router (or the experts).
"""
# Auto-incrementing layer ID for routing replay buffer binding.
_next_moe_layer_id: int = 0
# --8<-- [end:fused_moe]
def __init__(
@@ -290,6 +293,10 @@ class FusedMoE(PluggableLayer):
):
super().__init__()
# Assign unique layer ID for routing replay buffer binding.
self.moe_layer_id = FusedMoE._next_moe_layer_id
FusedMoE._next_moe_layer_id += 1
if params_dtype is None:
params_dtype = torch.get_default_dtype()
self.params_dtype = params_dtype
File diff suppressed because it is too large Load Diff
@@ -451,6 +451,10 @@ class MoERunner(MoERunnerInterface):
shared_experts_input, SharedExpertsOrder.NO_OVERLAP
)
# Get routing replay buffer from persistent layer attribute
# (set by bind_routing_capture_to_model during capturer init)
routing_replay_out = getattr(layer, "_routing_replay_out", None)
if self._quant_method.is_monolithic:
fused_out = self._quant_method.apply_monolithic(
layer=layer,
@@ -465,6 +469,10 @@ class MoERunner(MoERunnerInterface):
input_ids=input_ids,
)
# Write routing data for non-monolithic path (Triton, etc.)
if routing_replay_out is not None:
routing_replay_out[: topk_ids.shape[0]].copy_(topk_ids.to(torch.int16))
# Passing shared_experts_input in case SharedExpertsOrder is
# MK_INTERNAL_OVERLAPPED.
fused_out = self._quant_method.apply(
+4
View File
@@ -121,6 +121,7 @@ class RequestOutput:
num_cached_tokens: int | None = None,
*,
kv_transfer_params: dict[str, Any] | None = None,
prompt_routed_experts: np.ndarray | None = None,
# Forward compatibility, code that uses args added in new release can
# still run with older versions of vLLM without breaking.
**kwargs: Any,
@@ -141,12 +142,15 @@ class RequestOutput:
self.encoder_prompt_token_ids = encoder_prompt_token_ids
self.num_cached_tokens = num_cached_tokens
self.kv_transfer_params = kv_transfer_params
self.prompt_routed_experts = prompt_routed_experts
def add(self, next_output: "RequestOutput", aggregate: bool) -> None:
"""Merge subsequent RequestOutput into this one"""
self.finished |= next_output.finished
self.kv_transfer_params = next_output.kv_transfer_params
if next_output.prompt_routed_experts is not None:
self.prompt_routed_experts = next_output.prompt_routed_experts
for next_completion in next_output.outputs:
for i, completion in enumerate(self.outputs):
+7 -70
View File
@@ -7,8 +7,6 @@ from collections.abc import Iterable
from dataclasses import replace
from typing import Any
import numpy as np
from vllm import envs
from vllm.compilation.cuda_graph import CUDAGraphStat
from vllm.config import VllmConfig
@@ -27,9 +25,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1 import (
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsReader,
)
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
from vllm.multimodal.encoder_budget import MultiModalBudget
from vllm.v1.core.encoder_cache_manager import (
@@ -52,7 +47,7 @@ from vllm.v1.core.sched.request_queue import (
)
from vllm.v1.core.sched.utils import check_stop, remove_all
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheConfig
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.metrics.perf import ModelMetrics, PerfStats
from vllm.v1.metrics.stats import PrefixCacheStats, SchedulerStats
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
@@ -260,43 +255,6 @@ class Scheduler(SchedulerInterface):
if self.log_stats and vllm_config.observability_config.enable_mfu_metrics:
self.perf_metrics = ModelMetrics(vllm_config)
if self.vllm_config.model_config.enable_return_routed_experts:
assert self.dcp_world_size == 1 and self.pcp_world_size == 1, (
"enable_return_routed_experts does not support context parallelism "
"(dcp_world_size > 1 or pcp_world_size > 1)"
)
self.routed_experts_reader = RoutedExpertsReader.create()
assert len(kv_cache_config.kv_cache_groups) > 0, (
"enable_return_routed_experts requires at least one kv cache group"
)
# Find the attention group for routed experts indexing.
self.routed_experts_attn_gid = 0
for gid, group in enumerate(kv_cache_config.kv_cache_groups):
if isinstance(group.kv_cache_spec, AttentionSpec):
self.routed_experts_attn_gid = gid
break
min_block_size = min(
[
group.kv_cache_spec.block_size
for group in kv_cache_config.kv_cache_groups
]
)
num_groups = len(kv_cache_config.kv_cache_groups)
self.max_num_kv_tokens = (
kv_cache_config.num_blocks // num_groups
) * min_block_size
dcp_size = self.vllm_config.parallel_config.decode_context_parallel_size
pcp_size = self.vllm_config.parallel_config.prefill_context_parallel_size
if pcp_size * dcp_size > 1:
self.max_num_kv_tokens *= pcp_size * dcp_size
self.routed_experts_reader.attach_buffer(
max_num_kv_tokens=self.max_num_kv_tokens,
vllm_config=self.vllm_config,
)
self._pause_state: PauseState = PauseState.UNPAUSED
def _mamba_block_aligned_split(
@@ -1415,11 +1373,15 @@ class Scheduler(SchedulerInterface):
request.resumable = False
stopped = True
# Get routing data from ModelRunnerOutput (via worker D2H pipeline)
routed_experts = None
if (
model_runner_output.routed_experts_dict is not None
and req_id in model_runner_output.routed_experts_dict
):
routed_experts = model_runner_output.routed_experts_dict[req_id]
finish_reason = None
if stopped:
routed_experts = self._get_routed_experts(request)
# Capture finish_reason BEFORE _handle_stopped_request, which may
# reset the status to WAITING for streaming requests that continue.
finish_reason = request.get_finished_reason()
@@ -1594,31 +1556,6 @@ class Scheduler(SchedulerInterface):
self._enqueue_waiting_request(request)
return False
def _get_routed_experts(self, request: Request) -> np.ndarray | None:
if not self.vllm_config.model_config.enable_return_routed_experts:
return None
kv_blocks = self.kv_cache_manager.get_blocks(request.request_id)
block_ids = kv_blocks.get_block_ids()[self.routed_experts_attn_gid]
num_tokens = request.num_tokens - 1
# compute slot mapping using attention group's block_size
block_ids_array = np.array(block_ids, dtype=np.int32)
num_blocks = len(block_ids)
attn_group = self.kv_cache_config.kv_cache_groups[self.routed_experts_attn_gid]
block_size = attn_group.kv_cache_spec.block_size
# generate block offsets
block_offsets = np.arange(0, block_size)
# compute slot mapping: slot = block_id * block_size + offset
slot_mapping = (
block_offsets.reshape((1, block_size))
+ block_ids_array.reshape((num_blocks, 1)) * block_size
).flatten()[:num_tokens]
return self.routed_experts_reader.get_routed_experts(indices=slot_mapping)
def _update_request_with_output(
self, request: Request, new_token_ids: list[int]
) -> tuple[list[int], bool]:
+27 -2
View File
@@ -11,6 +11,9 @@ import numpy as np
import torch
from vllm.lora.request import LoRARequest
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
split_routed_experts,
)
from vllm.outputs import (
STREAM_FINISHED,
CompletionOutput,
@@ -314,8 +317,24 @@ class RequestState:
finished,
)
# Split routing data into prompt and generation portions.
# Prompt routing lives on RequestOutput (shared across n>1
# completions); generation routing lives on each CompletionOutput.
prompt_routed_experts = None
gen_routed_experts = None
if routed_experts is not None:
prompt_len = len(self.prompt_token_ids) if self.prompt_token_ids else 0
num_gen = (
self.detokenizer.num_output_tokens()
if self.detokenizer is not None
else None
)
prompt_routed_experts, gen_routed_experts = split_routed_experts(
routed_experts, prompt_len, num_gen
)
output = self._new_completion_output(
new_token_ids, finish_reason, stop_reason, routed_experts
new_token_ids, finish_reason, stop_reason, gen_routed_experts
)
if self.parent_req is None:
@@ -327,7 +346,11 @@ class RequestState:
external_req_id = self.parent_req.external_req_id
return self._new_request_output(
external_req_id, outputs, finished, kv_transfer_params
external_req_id,
outputs,
finished,
kv_transfer_params,
prompt_routed_experts,
)
def _new_request_output(
@@ -336,6 +359,7 @@ class RequestState:
outputs: list[CompletionOutput] | list[PoolingOutput],
finished: bool,
kv_transfer_params: dict[str, Any] | None = None,
prompt_routed_experts: np.ndarray | None = None,
) -> RequestOutput | PoolingRequestOutput:
# If prompt embeds were used, put placeholder prompt token ids
prompt_token_ids = self.prompt_token_ids
@@ -371,6 +395,7 @@ class RequestState:
kv_transfer_params=kv_transfer_params,
num_cached_tokens=self.num_cached_tokens,
metrics=self.stats,
prompt_routed_experts=prompt_routed_experts,
)
def _new_completion_output(
+3
View File
@@ -198,6 +198,9 @@ class ModelRunnerOutput:
# req_id -> num_nans_in_logits
num_nans_in_logits: dict[str, int] | None = None
# req_id -> routed experts ndarray of shape (seq_len, num_moe_layers, top_k)
routed_experts_dict: dict[str, np.ndarray] | None = None
# information related to cudagraph execution
cudagraph_stats: CUDAGraphStat | None = None
+64 -45
View File
@@ -54,7 +54,11 @@ from vllm.lora.layers import LoRAMapping, LoRAMappingType
from vllm.model_executor.layers.attention import Attention, MLAAttention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
extract_routed_experts_for_current_batch,
free_routing_buffers,
get_global_experts_capturer,
init_routed_experts_capturer_with_shared_cache,
issue_routing_d2h_copy,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
initialize_mamba_ssu_backend,
@@ -1092,6 +1096,12 @@ class GPUModelRunner(
for req_id in scheduler_output.finished_req_ids:
self.input_batch.remove_request(req_id)
if self.routed_experts_initialized:
free_routing_buffers(
scheduler_output.finished_req_ids,
scheduler_output.preempted_req_ids,
)
# Zero GPU memory for freshly allocated cache blocks to prevent
# stale NaN/data from corrupting attention or SSM computation.
if scheduler_output.new_block_ids_to_zero:
@@ -2175,10 +2185,6 @@ class GPUModelRunner(
block_table_gid_0 = _get_block_table(0)
slot_mapping_gid_0 = slot_mappings[0]
if self.routed_experts_initialized:
attn_gid = self.routed_experts_attn_gid
slot_mapping_attn = slot_mappings[attn_gid]
self.slot_mapping = slot_mapping_attn[:num_tokens].cpu().numpy()
num_computed_tokens_cpu = self.input_batch.num_computed_tokens_cpu_tensor[
:num_reqs_padded
]
@@ -3859,11 +3865,9 @@ class GPUModelRunner(
)
if self.routed_experts_initialized:
capturer = RoutedExpertsCapturer.get_instance()
capturer = get_global_experts_capturer()
if capturer is not None:
capturer.clear_buffer() # noqa
else:
logger.error("RoutedExpertsCapturer not initialized.")
capturer.finalize_pending_copy()
# If ngram_gpu is used, we need to copy the scheduler_output to avoid
# the modification has influence on the scheduler_output in engine core process.
@@ -4373,6 +4377,14 @@ class GPUModelRunner(
scheduler_output.total_num_scheduled_tokens,
)
if self.routed_experts_initialized:
issue_routing_d2h_copy(
input_batch_req_ids=self.input_batch.req_ids,
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
positions=self.positions,
positions_cpu=self._positions_cpu,
)
if propose_drafts_after_bookkeeping:
# ngram and other speculative decoding methods use the sampled
# tokens on the CPU, so they are run after bookkeeping.
@@ -4392,12 +4404,15 @@ class GPUModelRunner(
self.kv_connector_output = None
with record_function_or_nullcontext("gpu_model_runner: ModelRunnerOutput"):
routed_experts_dict = None
if self.routed_experts_initialized:
capturer = RoutedExpertsCapturer.get_instance()
if capturer is not None:
capturer.save_captured_experts(indices=self.slot_mapping) # noqa
else:
logger.error("RoutedExpertsCapturer not initialized.")
routed_experts_dict = extract_routed_experts_for_current_batch(
req_ids=req_ids_output_copy,
requests=self.requests,
req_id_to_index=self.input_batch.req_id_to_index,
num_tokens_no_spec=self.input_batch.num_tokens_no_spec,
max_model_len=self.max_model_len,
)
output = ModelRunnerOutput(
req_ids=req_ids_output_copy,
@@ -4411,6 +4426,7 @@ class GPUModelRunner(
else None,
num_nans_in_logits=num_nans_in_logits,
cudagraph_stats=cudagraph_stats,
routed_experts_dict=routed_experts_dict,
)
if not self.use_async_scheduling:
@@ -6138,6 +6154,7 @@ class GPUModelRunner(
"Skipping CUDA graph capture. To turn on CUDA graph capture, "
"ensure `cudagraph_mode` was not manually set to `NONE`"
)
self.init_routed_experts_capturer()
return 0
# Initialize encoder CUDA graph manager if enabled.
@@ -6171,6 +6188,13 @@ class GPUModelRunner(
start_time = time.perf_counter()
# Initialize the routed experts capturer once before any CUDA graph
# capture. Must happen before graphs are captured so the buffer
# address is baked into the graph. Do NOT call this inside
# _capture_cudagraphs() -- creating the capturer twice replaces
# the device buffer, causing graphs to write to a dead buffer.
self.init_routed_experts_capturer()
# Trigger CUDA graph capture for specific shapes.
# Capture the large shapes first so that the smaller shapes
# can reuse the memory pool allocated for the large shapes.
@@ -6965,45 +6989,40 @@ class GPUModelRunner(
"Initializing routed experts capturer, enable_return_routed_experts: %s",
self.model_config.enable_return_routed_experts,
)
routed_experts_capturer = RoutedExpertsCapturer.create()
self.routed_experts_attn_gid = self._get_attention_kv_cache_gid()
min_block_size = min(
[
group.kv_cache_spec.block_size
for group in self.kv_cache_config.kv_cache_groups
]
)
num_groups = len(self.kv_cache_config.kv_cache_groups)
self.max_num_kv_tokens = (
self.kv_cache_config.num_blocks // num_groups
) * min_block_size
dcp_size = self.vllm_config.parallel_config.decode_context_parallel_size
pcp_size = self.vllm_config.parallel_config.prefill_context_parallel_size
if pcp_size * dcp_size > 1:
self.max_num_kv_tokens *= pcp_size * dcp_size
from vllm.distributed import get_tp_group
routed_experts_capturer.init_buffer(
if hasattr(self.model_config.hf_text_config, "n_shared_experts"):
num_fused_shared_experts = 1
else:
num_fused_shared_experts = 0
tp_group = get_tp_group()
init_routed_experts_capturer_with_shared_cache(
enable=self.model_config.enable_return_routed_experts,
model_config=self.model_config,
num_fused_shared_experts=num_fused_shared_experts,
max_num_batched_tokens=self.scheduler_config.max_num_batched_tokens,
max_num_kv_tokens=self.max_num_kv_tokens,
vllm_config=self.vllm_config,
max_model_len=self.max_model_len,
device=self.device,
rank=tp_group.rank_in_group,
world_size=tp_group.world_size,
)
self._bind_routed_experts_capturer(routed_experts_capturer)
self._bind_routed_experts_capturer()
self.routed_experts_initialized = True
def _bind_routed_experts_capturer(self, capturer: RoutedExpertsCapturer) -> None:
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.fused_moe.router.base_router import (
BaseRouter,
# Pinned CPU buffer for async positions D2H (avoids sync .cpu() call)
self._positions_cpu = torch.empty(
self.scheduler_config.max_num_batched_tokens,
dtype=torch.long,
pin_memory=True,
)
for module in self.compilation_config.static_forward_context.values():
if isinstance(module, FusedMoE) and isinstance(module.router, BaseRouter):
layer_id = module.layer_id
def _bind_routed_experts_capturer(self) -> None:
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
bind_routing_capture_to_model,
)
def _capture_fn(topk_ids, _layer_id=layer_id, _capturer=capturer):
_capturer.capture(_layer_id, topk_ids)
module.router.set_capture_fn(_capture_fn)
bind_routing_capture_to_model(self.model)
def may_add_encoder_only_layers_to_kv_cache_config(self) -> None:
"""