[Frontend] Add routed-experts prompt offset (#51906)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-08-13 08:09:00 -07:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f3c1638927
commit 7553aac77b
13 changed files with 114 additions and 61 deletions
@@ -114,7 +114,7 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
token_ids: list[int] | None = None
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
# (numpy serialization). Shape after decode:
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
# ``num_tokens - 1`` because the last sampled token has not been
# forwarded yet and therefore has no routing data.
# Decode:
@@ -422,6 +422,11 @@ class ChatCompletionRequest(OpenAIBaseModel):
"need to map generated text back to input tokens."
),
)
routed_experts_prompt_start: int = Field(
default=0,
ge=0,
description="Skip the first N prompt tokens from returned routed-expert data.",
)
return_token_offsets: bool | None = Field(
default=False,
description=(
@@ -741,6 +746,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
extra_args=extra_args or None,
skip_clone=True, # Created fresh per request, safe to skip clone
repetition_detection=self.repetition_detection,
routed_experts_prompt_start=self.routed_experts_prompt_start,
)
@model_validator(mode="before")
@@ -2,15 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import io
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from http import HTTPStatus
from typing import Any, Final, cast
import numpy as np
import pybase64 as base64
from fastapi import Request
from vllm.engine.protocol import EngineClient
@@ -64,6 +61,7 @@ from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.utils.collection_utils import as_list
from vllm.utils.serial_utils import numpy2base64
logger = init_logger(__name__)
@@ -1007,15 +1005,11 @@ class OpenAIServingChat(GenerateBaseServing):
and output.finish_reason == "stop"
)
# Encode routed_experts for transport. JSON can't carry raw
# bytes, so we write the ndarray as a ``.npy`` byte stream
# and base64-encode it. ``pybase64`` is ~3x faster than the
# stdlib ``base64`` on large payloads thanks to SIMD.
routed_experts_b64 = None
if output.routed_experts is not None:
buf = io.BytesIO()
np.save(buf, output.routed_experts)
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode("ascii")
routed_experts_b64 = (
numpy2base64(output.routed_experts)
if output.routed_experts is not None
else None
)
choice_data = ChatCompletionResponseChoice(
index=output.index,
@@ -175,6 +175,11 @@ class CompletionRequest(OpenAIBaseModel):
"need to map generated text back to input tokens."
),
)
routed_experts_prompt_start: int = Field(
default=0,
ge=0,
description="Skip the first N prompt tokens from returned routed-expert data.",
)
return_token_offsets: bool | None = Field(
default=False,
description=(
@@ -394,6 +399,7 @@ class CompletionRequest(OpenAIBaseModel):
skip_clone=True, # Created fresh per request, safe to skip clone
repetition_detection=self.repetition_detection,
thinking_token_budget=self.thinking_token_budget,
routed_experts_prompt_start=self.routed_experts_prompt_start,
)
@model_validator(mode="before")
@@ -611,7 +617,7 @@ class CompletionResponseChoice(OpenAIBaseModel):
prompt_token_ids: list[int] | None = None # For prompt
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
# (numpy serialization). Shape after decode:
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
# ``num_tokens - 1`` because the last sampled token has not been
# forwarded yet and therefore has no routing data.
# Decode:
+6 -14
View File
@@ -2,14 +2,11 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import io
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import cast
import numpy as np
import pybase64 as base64
from fastapi import Request
from vllm.engine.protocol import EngineClient
@@ -48,6 +45,7 @@ from vllm.sampling_params import BeamSearchParams, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.utils.async_utils import merge_async_iterators
from vllm.utils.collection_utils import as_list
from vllm.utils.serial_utils import numpy2base64
logger = init_logger(__name__)
@@ -570,17 +568,11 @@ class OpenAIServingCompletion(GenerateBaseServing):
else:
logprobs = None
# Encode routed_experts for transport. JSON can't carry raw
# bytes, so we write the ndarray as a ``.npy`` byte stream
# and base64-encode it. ``pybase64`` is ~3x faster than the
# stdlib ``base64`` on large payloads thanks to SIMD.
routed_experts_b64 = None
if output.routed_experts is not None:
buf = io.BytesIO()
np.save(buf, output.routed_experts)
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
"ascii"
)
routed_experts_b64 = (
numpy2base64(output.routed_experts)
if output.routed_experts is not None
else None
)
choice_data = CompletionResponseChoice(
index=len(choices),
@@ -202,7 +202,7 @@ class GenerateResponseChoice(BaseModel):
token_ids: list[int] | None = None
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
# (numpy serialization). Shape after decode:
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16/int32
# ``num_tokens - 1`` because the last sampled token has not been
# forwarded yet and therefore has no routing data.
# Decode:
@@ -3,14 +3,11 @@
import asyncio
import io
import time
from collections.abc import AsyncGenerator
from collections.abc import Sequence as GenericSequence
import msgspec
import numpy as np
import pybase64 as base64
from fastapi import Request
from vllm.engine.protocol import EngineClient
@@ -46,6 +43,7 @@ from vllm.outputs import RequestOutput
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.utils.collection_utils import as_list
from vllm.utils.serial_utils import numpy2base64
from .mm_serde import decode_mm_kwargs_item
from .protocol import (
@@ -303,17 +301,11 @@ class ServingTokens(GenerateBaseServing):
else:
logprobs = None
# Encode routed_experts for transport. JSON can't carry raw
# bytes, so we write the ndarray as a ``.npy`` byte stream
# and base64-encode it. ``pybase64`` is ~3x faster than the
# stdlib ``base64`` on large payloads thanks to SIMD.
# This is the only base64 hop in the pipeline -- the
# engine<->API-server link is binary msgpack + zmq.
routed_experts_b64 = None
if output.routed_experts is not None:
buf = io.BytesIO()
np.save(buf, output.routed_experts)
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode("ascii")
routed_experts_b64 = (
numpy2base64(output.routed_experts)
if output.routed_experts is not None
else None
)
sampling_mask = None
if output.sampling_mask is not None:
@@ -435,13 +427,11 @@ class ServingTokens(GenerateBaseServing):
else:
logprobs = None
routed_experts_b64 = None
if output.routed_experts is not None:
buf = io.BytesIO()
np.save(buf, output.routed_experts)
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
"ascii"
)
routed_experts_b64 = (
numpy2base64(output.routed_experts)
if output.routed_experts is not None
else None
)
chunk = GenerateStreamResponse(
request_id=request_id,