[Core] Replace routing replay with device cache and async D2H pipeline (#39917)

Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
TomerBN-Nvidia
2026-05-07 11:24:56 -07:00
committed by GitHub
parent 8eb401134e
commit 8189a15914
15 changed files with 1398 additions and 642 deletions
@@ -92,12 +92,16 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
# not part of the OpenAI spec but is useful for tracing the tokens
# in agent scenarios
token_ids: list[int] | None = None
routed_experts: list[list[list[int]]] | None = None # [gen_len, num_layers, top_k]
class ChatCompletionResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
object: Literal["chat.completion"] = "chat.completion"
created: int = Field(default_factory=lambda: int(time.time()))
prompt_routed_experts: list[list[list[int]]] | None = (
None # [prompt_len, num_layers, top_k]
)
model: str
choices: list[ChatCompletionResponseChoice]
service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
@@ -1088,6 +1088,11 @@ class OpenAIServingChat(OpenAIServing):
token_ids=(
as_list(output.token_ids) if request.return_token_ids else None
),
routed_experts=(
output.routed_experts.tolist()
if output.routed_experts is not None
else None
),
)
choices.append(choice_data)
continue
@@ -1309,6 +1314,11 @@ class OpenAIServingChat(OpenAIServing):
token_ids=(
as_list(output.token_ids) if request.return_token_ids else None
),
routed_experts=(
output.routed_experts.tolist()
if output.routed_experts is not None
else None
),
)
choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
@@ -1348,6 +1358,10 @@ class OpenAIServingChat(OpenAIServing):
request_metadata.final_usage_info = usage
prompt_routed_experts = None
if final_res.prompt_routed_experts is not None:
prompt_routed_experts = final_res.prompt_routed_experts.tolist()
response = ChatCompletionResponse(
id=request_id,
created=created_time,
@@ -1360,6 +1374,7 @@ class OpenAIServingChat(OpenAIServing):
final_res.prompt_token_ids if request.return_token_ids else None
),
kv_transfer_params=final_res.kv_transfer_params,
prompt_routed_experts=prompt_routed_experts,
)
# Log complete response if output logging is enabled
@@ -468,12 +468,16 @@ class CompletionResponseChoice(OpenAIBaseModel):
token_ids: list[int] | None = None # For response
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
prompt_token_ids: list[int] | None = None # For prompt
routed_experts: list[list[list[int]]] | None = None # [gen_len, num_layers, top_k]
class CompletionResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
object: Literal["text_completion"] = "text_completion"
created: int = Field(default_factory=lambda: int(time.time()))
prompt_routed_experts: list[list[list[int]]] | None = (
None # [prompt_len, num_layers, top_k]
)
model: str
choices: list[CompletionResponseChoice]
service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
@@ -542,6 +542,11 @@ class OpenAIServingCompletion(OpenAIServing):
token_ids=(
as_list(output.token_ids) if request.return_token_ids else None
),
routed_experts=(
output.routed_experts.tolist()
if output.routed_experts is not None
else None
),
)
choices.append(choice_data)
@@ -565,8 +570,13 @@ class OpenAIServingCompletion(OpenAIServing):
)
request_metadata.final_usage_info = usage
prompt_routed_experts = None
if final_res_batch:
kv_transfer_params = final_res_batch[0].kv_transfer_params
pre = final_res_batch[0].prompt_routed_experts
if pre is not None:
prompt_routed_experts = pre.tolist()
return CompletionResponse(
id=request_id,
created=created_time,
@@ -575,6 +585,7 @@ class OpenAIServingCompletion(OpenAIServing):
usage=usage,
system_fingerprint=self.system_fingerprint,
kv_transfer_params=kv_transfer_params,
prompt_routed_experts=prompt_routed_experts,
)
def _create_completion_logprobs(