[Core] Replace routing replay with device cache and async D2H pipeline (#39917)
Signed-off-by: Tomer Barnatan <tbarnatan@nvidia.com>
This commit is contained in:
@@ -468,12 +468,16 @@ class CompletionResponseChoice(OpenAIBaseModel):
|
||||
token_ids: list[int] | None = None # For response
|
||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||
prompt_token_ids: list[int] | None = None # For prompt
|
||||
routed_experts: list[list[list[int]]] | None = None # [gen_len, num_layers, top_k]
|
||||
|
||||
|
||||
class CompletionResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
|
||||
object: Literal["text_completion"] = "text_completion"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
prompt_routed_experts: list[list[list[int]]] | None = (
|
||||
None # [prompt_len, num_layers, top_k]
|
||||
)
|
||||
model: str
|
||||
choices: list[CompletionResponseChoice]
|
||||
service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
|
||||
|
||||
Reference in New Issue
Block a user