[Core] Replace routing replay with device cache and async D2H pipeline (#39917)

Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
TomerBN-Nvidia
2026-05-07 11:24:56 -07:00
committed by GitHub
parent 8eb401134e
commit 8189a15914
15 changed files with 1398 additions and 642 deletions
+27 -2
View File
@@ -11,6 +11,9 @@ import numpy as np
import torch
from vllm.lora.request import LoRARequest
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
split_routed_experts,
)
from vllm.outputs import (
STREAM_FINISHED,
CompletionOutput,
@@ -314,8 +317,24 @@ class RequestState:
finished,
)
# Split routing data into prompt and generation portions.
# Prompt routing lives on RequestOutput (shared across n>1
# completions); generation routing lives on each CompletionOutput.
prompt_routed_experts = None
gen_routed_experts = None
if routed_experts is not None:
prompt_len = len(self.prompt_token_ids) if self.prompt_token_ids else 0
num_gen = (
self.detokenizer.num_output_tokens()
if self.detokenizer is not None
else None
)
prompt_routed_experts, gen_routed_experts = split_routed_experts(
routed_experts, prompt_len, num_gen
)
output = self._new_completion_output(
new_token_ids, finish_reason, stop_reason, routed_experts
new_token_ids, finish_reason, stop_reason, gen_routed_experts
)
if self.parent_req is None:
@@ -327,7 +346,11 @@ class RequestState:
external_req_id = self.parent_req.external_req_id
return self._new_request_output(
external_req_id, outputs, finished, kv_transfer_params
external_req_id,
outputs,
finished,
kv_transfer_params,
prompt_routed_experts,
)
def _new_request_output(
@@ -336,6 +359,7 @@ class RequestState:
outputs: list[CompletionOutput] | list[PoolingOutput],
finished: bool,
kv_transfer_params: dict[str, Any] | None = None,
prompt_routed_experts: np.ndarray | None = None,
) -> RequestOutput | PoolingRequestOutput:
# If prompt embeds were used, put placeholder prompt token ids
prompt_token_ids = self.prompt_token_ids
@@ -371,6 +395,7 @@ class RequestState:
kv_transfer_params=kv_transfer_params,
num_cached_tokens=self.num_cached_tokens,
metrics=self.stats,
prompt_routed_experts=prompt_routed_experts,
)
def _new_completion_output(