Revert "[Core] Replace routing replay with device cache and async D2H pipeline" (#39917) (#42434)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
aoshen02
2026-05-13 23:49:01 -07:00
committed by GitHub
parent 0d2732dd91
commit 8c79ad6580
15 changed files with 646 additions and 1409 deletions
@@ -92,16 +92,12 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
# not part of the OpenAI spec but is useful for tracing the tokens
# in agent scenarios
token_ids: list[int] | None = None
routed_experts: list[list[list[int]]] | None = None # [gen_len, num_layers, top_k]
class ChatCompletionResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
object: Literal["chat.completion"] = "chat.completion"
created: int = Field(default_factory=lambda: int(time.time()))
prompt_routed_experts: list[list[list[int]]] | None = (
None # [prompt_len, num_layers, top_k]
)
model: str
choices: list[ChatCompletionResponseChoice]
service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None