[Core] Replace routing replay with device cache and async D2H pipeline (#39917)

Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
TomerBN-Nvidia
2026-05-07 11:24:56 -07:00
committed by GitHub
parent 8eb401134e
commit 8189a15914
15 changed files with 1398 additions and 642 deletions
+64 -45
View File
@@ -54,7 +54,11 @@ from vllm.lora.layers import LoRAMapping, LoRAMappingType
from vllm.model_executor.layers.attention import Attention, MLAAttention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsCapturer,
extract_routed_experts_for_current_batch,
free_routing_buffers,
get_global_experts_capturer,
init_routed_experts_capturer_with_shared_cache,
issue_routing_d2h_copy,
)
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
initialize_mamba_ssu_backend,
@@ -1092,6 +1096,12 @@ class GPUModelRunner(
for req_id in scheduler_output.finished_req_ids:
self.input_batch.remove_request(req_id)
if self.routed_experts_initialized:
free_routing_buffers(
scheduler_output.finished_req_ids,
scheduler_output.preempted_req_ids,
)
# Zero GPU memory for freshly allocated cache blocks to prevent
# stale NaN/data from corrupting attention or SSM computation.
if scheduler_output.new_block_ids_to_zero:
@@ -2175,10 +2185,6 @@ class GPUModelRunner(
block_table_gid_0 = _get_block_table(0)
slot_mapping_gid_0 = slot_mappings[0]
if self.routed_experts_initialized:
attn_gid = self.routed_experts_attn_gid
slot_mapping_attn = slot_mappings[attn_gid]
self.slot_mapping = slot_mapping_attn[:num_tokens].cpu().numpy()
num_computed_tokens_cpu = self.input_batch.num_computed_tokens_cpu_tensor[
:num_reqs_padded
]
@@ -3859,11 +3865,9 @@ class GPUModelRunner(
)
if self.routed_experts_initialized:
capturer = RoutedExpertsCapturer.get_instance()
capturer = get_global_experts_capturer()
if capturer is not None:
capturer.clear_buffer() # noqa
else:
logger.error("RoutedExpertsCapturer not initialized.")
capturer.finalize_pending_copy()
# If ngram_gpu is used, we need to copy the scheduler_output to avoid
# the modification has influence on the scheduler_output in engine core process.
@@ -4373,6 +4377,14 @@ class GPUModelRunner(
scheduler_output.total_num_scheduled_tokens,
)
if self.routed_experts_initialized:
issue_routing_d2h_copy(
input_batch_req_ids=self.input_batch.req_ids,
num_scheduled_tokens=scheduler_output.num_scheduled_tokens,
positions=self.positions,
positions_cpu=self._positions_cpu,
)
if propose_drafts_after_bookkeeping:
# ngram and other speculative decoding methods use the sampled
# tokens on the CPU, so they are run after bookkeeping.
@@ -4392,12 +4404,15 @@ class GPUModelRunner(
self.kv_connector_output = None
with record_function_or_nullcontext("gpu_model_runner: ModelRunnerOutput"):
routed_experts_dict = None
if self.routed_experts_initialized:
capturer = RoutedExpertsCapturer.get_instance()
if capturer is not None:
capturer.save_captured_experts(indices=self.slot_mapping) # noqa
else:
logger.error("RoutedExpertsCapturer not initialized.")
routed_experts_dict = extract_routed_experts_for_current_batch(
req_ids=req_ids_output_copy,
requests=self.requests,
req_id_to_index=self.input_batch.req_id_to_index,
num_tokens_no_spec=self.input_batch.num_tokens_no_spec,
max_model_len=self.max_model_len,
)
output = ModelRunnerOutput(
req_ids=req_ids_output_copy,
@@ -4411,6 +4426,7 @@ class GPUModelRunner(
else None,
num_nans_in_logits=num_nans_in_logits,
cudagraph_stats=cudagraph_stats,
routed_experts_dict=routed_experts_dict,
)
if not self.use_async_scheduling:
@@ -6138,6 +6154,7 @@ class GPUModelRunner(
"Skipping CUDA graph capture. To turn on CUDA graph capture, "
"ensure `cudagraph_mode` was not manually set to `NONE`"
)
self.init_routed_experts_capturer()
return 0
# Initialize encoder CUDA graph manager if enabled.
@@ -6171,6 +6188,13 @@ class GPUModelRunner(
start_time = time.perf_counter()
# Initialize the routed experts capturer once before any CUDA graph
# capture. Must happen before graphs are captured so the buffer
# address is baked into the graph. Do NOT call this inside
# _capture_cudagraphs() -- creating the capturer twice replaces
# the device buffer, causing graphs to write to a dead buffer.
self.init_routed_experts_capturer()
# Trigger CUDA graph capture for specific shapes.
# Capture the large shapes first so that the smaller shapes
# can reuse the memory pool allocated for the large shapes.
@@ -6965,45 +6989,40 @@ class GPUModelRunner(
"Initializing routed experts capturer, enable_return_routed_experts: %s",
self.model_config.enable_return_routed_experts,
)
routed_experts_capturer = RoutedExpertsCapturer.create()
self.routed_experts_attn_gid = self._get_attention_kv_cache_gid()
min_block_size = min(
[
group.kv_cache_spec.block_size
for group in self.kv_cache_config.kv_cache_groups
]
)
num_groups = len(self.kv_cache_config.kv_cache_groups)
self.max_num_kv_tokens = (
self.kv_cache_config.num_blocks // num_groups
) * min_block_size
dcp_size = self.vllm_config.parallel_config.decode_context_parallel_size
pcp_size = self.vllm_config.parallel_config.prefill_context_parallel_size
if pcp_size * dcp_size > 1:
self.max_num_kv_tokens *= pcp_size * dcp_size
from vllm.distributed import get_tp_group
routed_experts_capturer.init_buffer(
if hasattr(self.model_config.hf_text_config, "n_shared_experts"):
num_fused_shared_experts = 1
else:
num_fused_shared_experts = 0
tp_group = get_tp_group()
init_routed_experts_capturer_with_shared_cache(
enable=self.model_config.enable_return_routed_experts,
model_config=self.model_config,
num_fused_shared_experts=num_fused_shared_experts,
max_num_batched_tokens=self.scheduler_config.max_num_batched_tokens,
max_num_kv_tokens=self.max_num_kv_tokens,
vllm_config=self.vllm_config,
max_model_len=self.max_model_len,
device=self.device,
rank=tp_group.rank_in_group,
world_size=tp_group.world_size,
)
self._bind_routed_experts_capturer(routed_experts_capturer)
self._bind_routed_experts_capturer()
self.routed_experts_initialized = True
def _bind_routed_experts_capturer(self, capturer: RoutedExpertsCapturer) -> None:
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
from vllm.model_executor.layers.fused_moe.router.base_router import (
BaseRouter,
# Pinned CPU buffer for async positions D2H (avoids sync .cpu() call)
self._positions_cpu = torch.empty(
self.scheduler_config.max_num_batched_tokens,
dtype=torch.long,
pin_memory=True,
)
for module in self.compilation_config.static_forward_context.values():
if isinstance(module, FusedMoE) and isinstance(module.router, BaseRouter):
layer_id = module.layer_id
def _bind_routed_experts_capturer(self) -> None:
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
bind_routing_capture_to_model,
)
def _capture_fn(topk_ids, _layer_id=layer_id, _capturer=capturer):
_capturer.capture(_layer_id, topk_ids)
module.router.set_capture_fn(_capture_fn)
bind_routing_capture_to_model(self.model)
def may_add_encoder_only_layers_to_kv_cache_config(self) -> None:
"""