[Frontend] Add routed-experts prompt offset (#51906)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-08-13 08:09:00 -07:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f3c1638927
commit 7553aac77b
13 changed files with 114 additions and 61 deletions
@@ -114,7 +114,7 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
token_ids: list[int] | None = None
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
# (numpy serialization). Shape after decode:
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
# ``num_tokens - 1`` because the last sampled token has not been
# forwarded yet and therefore has no routing data.
# Decode:
@@ -422,6 +422,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
"need to map generated text back to input tokens."
),
)
routed_experts_prompt_start: int = Field(
default=0,
ge=0,
description="Skip the first N prompt tokens from returned routed-expert data.",
)
return_token_offsets: bool | None = Field(
default=False,
description=(
@@ -741,6 +746,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
extra_args=extra_args or None,
skip_clone=True, # Created fresh per request, safe to skip clone
repetition_detection=self.repetition_detection,
routed_experts_prompt_start=self.routed_experts_prompt_start,
)
@model_validator(mode="before")