Signed-off-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
@@ -92,16 +92,12 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||
# not part of the OpenAI spec but is useful for tracing the tokens
|
||||
# in agent scenarios
|
||||
token_ids: list[int] | None = None
|
||||
routed_experts: list[list[list[int]]] | None = None # [gen_len, num_layers, top_k]
|
||||
|
||||
|
||||
class ChatCompletionResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"chatcmpl-{random_uuid()}")
|
||||
object: Literal["chat.completion"] = "chat.completion"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
prompt_routed_experts: list[list[list[int]]] | None = (
|
||||
None # [prompt_len, num_layers, top_k]
|
||||
)
|
||||
model: str
|
||||
choices: list[ChatCompletionResponseChoice]
|
||||
service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
|
||||
|
||||
@@ -1101,11 +1101,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
token_ids=(
|
||||
as_list(output.token_ids) if request.return_token_ids else None
|
||||
),
|
||||
routed_experts=(
|
||||
output.routed_experts.tolist()
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
choices.append(choice_data)
|
||||
continue
|
||||
@@ -1327,11 +1322,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
token_ids=(
|
||||
as_list(output.token_ids) if request.return_token_ids else None
|
||||
),
|
||||
routed_experts=(
|
||||
output.routed_experts.tolist()
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
|
||||
|
||||
@@ -1371,10 +1361,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
|
||||
prompt_routed_experts = None
|
||||
if final_res.prompt_routed_experts is not None:
|
||||
prompt_routed_experts = final_res.prompt_routed_experts.tolist()
|
||||
|
||||
# ``final_res.prompt`` is the rendered chat-templated prompt text
|
||||
prompt_text = final_res.prompt if request.return_prompt_text else None
|
||||
|
||||
@@ -1391,7 +1377,6 @@ class OpenAIServingChat(OpenAIServing):
|
||||
),
|
||||
prompt_text=prompt_text,
|
||||
kv_transfer_params=final_res.kv_transfer_params,
|
||||
prompt_routed_experts=prompt_routed_experts,
|
||||
)
|
||||
|
||||
# Log complete response if output logging is enabled
|
||||
|
||||
@@ -477,16 +477,12 @@ class CompletionResponseChoice(OpenAIBaseModel):
|
||||
token_ids: list[int] | None = None # For response
|
||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||
prompt_token_ids: list[int] | None = None # For prompt
|
||||
routed_experts: list[list[list[int]]] | None = None # [gen_len, num_layers, top_k]
|
||||
|
||||
|
||||
class CompletionResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
|
||||
object: Literal["text_completion"] = "text_completion"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
prompt_routed_experts: list[list[list[int]]] | None = (
|
||||
None # [prompt_len, num_layers, top_k]
|
||||
)
|
||||
model: str
|
||||
choices: list[CompletionResponseChoice]
|
||||
service_tier: Literal["auto", "default", "flex", "scale", "priority"] | None = None
|
||||
|
||||
@@ -551,11 +551,6 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
token_ids=(
|
||||
as_list(output.token_ids) if request.return_token_ids else None
|
||||
),
|
||||
routed_experts=(
|
||||
output.routed_experts.tolist()
|
||||
if output.routed_experts is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
choices.append(choice_data)
|
||||
|
||||
@@ -579,13 +574,8 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
)
|
||||
|
||||
request_metadata.final_usage_info = usage
|
||||
prompt_routed_experts = None
|
||||
if final_res_batch:
|
||||
kv_transfer_params = final_res_batch[0].kv_transfer_params
|
||||
pre = final_res_batch[0].prompt_routed_experts
|
||||
if pre is not None:
|
||||
prompt_routed_experts = pre.tolist()
|
||||
|
||||
return CompletionResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
@@ -594,7 +584,6 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
usage=usage,
|
||||
system_fingerprint=self.system_fingerprint,
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
prompt_routed_experts=prompt_routed_experts,
|
||||
)
|
||||
|
||||
def _create_completion_logprobs(
|
||||
|
||||
Reference in New Issue
Block a user