[Core] Replace routing replay with device cache and async D2H pipeline (#39917)
Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
@@ -11,6 +11,9 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
split_routed_experts,
|
||||
)
|
||||
from vllm.outputs import (
|
||||
STREAM_FINISHED,
|
||||
CompletionOutput,
|
||||
@@ -314,8 +317,24 @@ class RequestState:
|
||||
finished,
|
||||
)
|
||||
|
||||
# Split routing data into prompt and generation portions.
|
||||
# Prompt routing lives on RequestOutput (shared across n>1
|
||||
# completions); generation routing lives on each CompletionOutput.
|
||||
prompt_routed_experts = None
|
||||
gen_routed_experts = None
|
||||
if routed_experts is not None:
|
||||
prompt_len = len(self.prompt_token_ids) if self.prompt_token_ids else 0
|
||||
num_gen = (
|
||||
self.detokenizer.num_output_tokens()
|
||||
if self.detokenizer is not None
|
||||
else None
|
||||
)
|
||||
prompt_routed_experts, gen_routed_experts = split_routed_experts(
|
||||
routed_experts, prompt_len, num_gen
|
||||
)
|
||||
|
||||
output = self._new_completion_output(
|
||||
new_token_ids, finish_reason, stop_reason, routed_experts
|
||||
new_token_ids, finish_reason, stop_reason, gen_routed_experts
|
||||
)
|
||||
|
||||
if self.parent_req is None:
|
||||
@@ -327,7 +346,11 @@ class RequestState:
|
||||
external_req_id = self.parent_req.external_req_id
|
||||
|
||||
return self._new_request_output(
|
||||
external_req_id, outputs, finished, kv_transfer_params
|
||||
external_req_id,
|
||||
outputs,
|
||||
finished,
|
||||
kv_transfer_params,
|
||||
prompt_routed_experts,
|
||||
)
|
||||
|
||||
def _new_request_output(
|
||||
@@ -336,6 +359,7 @@ class RequestState:
|
||||
outputs: list[CompletionOutput] | list[PoolingOutput],
|
||||
finished: bool,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
prompt_routed_experts: np.ndarray | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput:
|
||||
# If prompt embeds were used, put placeholder prompt token ids
|
||||
prompt_token_ids = self.prompt_token_ids
|
||||
@@ -371,6 +395,7 @@ class RequestState:
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
num_cached_tokens=self.num_cached_tokens,
|
||||
metrics=self.stats,
|
||||
prompt_routed_experts=prompt_routed_experts,
|
||||
)
|
||||
|
||||
def _new_completion_output(
|
||||
|
||||
Reference in New Issue
Block a user