[Frontend] Add routed-experts prompt offset (#51906)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f3c1638927
commit
7553aac77b
@@ -114,7 +114,7 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||
token_ids: list[int] | None = None
|
||||
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
|
||||
# (numpy serialization). Shape after decode:
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
|
||||
# ``num_tokens - 1`` because the last sampled token has not been
|
||||
# forwarded yet and therefore has no routing data.
|
||||
# Decode:
|
||||
@@ -422,6 +422,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"need to map generated text back to input tokens."
|
||||
),
|
||||
)
|
||||
routed_experts_prompt_start: int = Field(
|
||||
default=0,
|
||||
ge=0,
|
||||
description="Skip the first N prompt tokens from returned routed-expert data.",
|
||||
)
|
||||
return_token_offsets: bool | None = Field(
|
||||
default=False,
|
||||
description=(
|
||||
@@ -741,6 +746,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
extra_args=extra_args or None,
|
||||
skip_clone=True, # Created fresh per request, safe to skip clone
|
||||
repetition_detection=self.repetition_detection,
|
||||
routed_experts_prompt_start=self.routed_experts_prompt_start,
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
|
||||
Reference in New Issue
Block a user