[Core] Replace routing replay with device cache and async D2H pipeline (#39917)
Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
@@ -542,6 +542,11 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
token_ids=(
|
||||
as_list(output.token_ids) if request.return_token_ids else None
|
||||
),
|
||||
routed_experts=(
|
||||
output.routed_experts.tolist()
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
choices.append(choice_data)
|
||||
|
||||
@@ -565,8 +570,13 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
)
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
prompt_routed_experts = None
|
||||
if final_res_batch:
|
||||
kv_transfer_params = final_res_batch[0].kv_transfer_params
|
||||
pre = final_res_batch[0].prompt_routed_experts
|
||||
if pre is not None:
|
||||
prompt_routed_experts = pre.tolist()
|
||||
|
||||
return CompletionResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
@@ -575,6 +585,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
usage=usage,
|
||||
system_fingerprint=self.system_fingerprint,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
prompt_routed_experts=prompt_routed_experts,
|
||||
)
|
||||
|
||||
def _create_completion_logprobs(
|
||||
|
||||
Reference in New Issue
Block a user