[Frontend] Add routed-experts prompt offset (#51906)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f3c1638927
commit
7553aac77b
@@ -175,6 +175,11 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
"need to map generated text back to input tokens."
|
||||
),
|
||||
)
|
||||
routed_experts_prompt_start: int = Field(
|
||||
default=0,
|
||||
ge=0,
|
||||
description="Skip the first N prompt tokens from returned routed-expert data.",
|
||||
)
|
||||
return_token_offsets: bool | None = Field(
|
||||
default=False,
|
||||
description=(
|
||||
@@ -394,6 +399,7 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
skip_clone=True, # Created fresh per request, safe to skip clone
|
||||
repetition_detection=self.repetition_detection,
|
||||
thinking_token_budget=self.thinking_token_budget,
|
||||
routed_experts_prompt_start=self.routed_experts_prompt_start,
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@@ -611,7 +617,7 @@ class CompletionResponseChoice(OpenAIBaseModel):
|
||||
prompt_token_ids: list[int] | None = None # For prompt
|
||||
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
|
||||
# (numpy serialization). Shape after decode:
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
|
||||
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
|
||||
# ``num_tokens - 1`` because the last sampled token has not been
|
||||
# forwarded yet and therefore has no routing data.
|
||||
# Decode:
|
||||
|
||||
Reference in New Issue
Block a user