Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -11,9 +11,6 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
split_routed_experts,
|
||||
)
|
||||
from vllm.outputs import (
|
||||
STREAM_FINISHED,
|
||||
CompletionOutput,
|
||||
@@ -317,24 +314,8 @@ class RequestState:
|
||||
finished,
|
||||
)
|
||||
|
||||
# Split routing data into prompt and generation portions.
|
||||
# Prompt routing lives on RequestOutput (shared across n>1
|
||||
# completions); generation routing lives on each CompletionOutput.
|
||||
prompt_routed_experts = None
|
||||
gen_routed_experts = None
|
||||
if routed_experts is not None:
|
||||
prompt_len = len(self.prompt_token_ids) if self.prompt_token_ids else 0
|
||||
num_gen = (
|
||||
self.detokenizer.num_output_tokens()
|
||||
if self.detokenizer is not None
|
||||
else None
|
||||
)
|
||||
prompt_routed_experts, gen_routed_experts = split_routed_experts(
|
||||
routed_experts, prompt_len, num_gen
|
||||
)
|
||||
|
||||
output = self._new_completion_output(
|
||||
new_token_ids, finish_reason, stop_reason, gen_routed_experts
|
||||
new_token_ids, finish_reason, stop_reason, routed_experts
|
||||
)
|
||||
|
||||
if self.parent_req is None:
|
||||
@@ -346,11 +327,7 @@ class RequestState:
|
||||
external_req_id = self.parent_req.external_req_id
|
||||
|
||||
return self._new_request_output(
|
||||
external_req_id,
|
||||
outputs,
|
||||
finished,
|
||||
kv_transfer_params,
|
||||
prompt_routed_experts,
|
||||
external_req_id, outputs, finished, kv_transfer_params
|
||||
)
|
||||
|
||||
def _new_request_output(
|
||||
@@ -359,7 +336,6 @@ class RequestState:
|
||||
outputs: list[CompletionOutput] | list[PoolingOutput],
|
||||
finished: bool,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
prompt_routed_experts: np.ndarray | None = None,
|
||||
) -> RequestOutput | PoolingRequestOutput:
|
||||
# If prompt embeds were used, put placeholder prompt token ids
|
||||
prompt_token_ids = self.prompt_token_ids
|
||||
@@ -395,7 +371,6 @@ class RequestState:
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
num_cached_tokens=self.num_cached_tokens,
|
||||
metrics=self.stats,
|
||||
prompt_routed_experts=prompt_routed_experts,
|
||||
)
|
||||
|
||||
def _new_completion_output(
|
||||
|
||||
Reference in New Issue
Block a user