[R3] Add routed experts to openai entrypoint (#38939)

Signed-off-by: ahao-anyscale <ahao@anyscale.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Aaron Hao
2026-05-20 09:07:59 -07:00
committed by GitHub
co-authored by Isotr0py
parent 9c78c99995
commit 2d6b3489b9
6 changed files with 208 additions and 0 deletions
@@ -0,0 +1,64 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import numpy as np
import pybase64 as base64
import pytest
from ...utils import RemoteOpenAIServer
MODEL_NAME = "TitanML/tiny-mixtral"
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
# The published config has sliding_window=4096, which produces
# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a
# FullAttentionSpec group, so we override sliding_window=null below.
NUM_LOCAL_EXPERTS = 8
NUM_EXPERTS_PER_TOK = 2
NUM_HIDDEN_LAYERS = 2
@pytest.fixture(scope="module")
def server():
args = [
"--max-model-len",
"256",
"--max-num-seqs",
"32",
"--enforce-eager",
"--enable-return-routed-experts",
"--hf-overrides",
'{"sliding_window": null}',
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest.mark.asyncio
async def test_routed_experts(server):
"""Test that /v1/completions returns routed_experts when enabled."""
async with server.get_async_client() as client:
result = await client.completions.create(
model=MODEL_NAME,
prompt="Hello, world",
max_tokens=10,
temperature=0,
extra_body={"return_token_ids": True},
)
choice = result.model_dump()["choices"][0]
assert choice["routed_experts"] is not None
assert choice["token_ids"] is not None
# routed_experts is base64-encoded .npy bytes; decode to ndarray.
routed_experts = np.load(io.BytesIO(base64.b64decode(choice["routed_experts"])))
assert routed_experts.ndim == 3
num_tokens, num_layers, topk = routed_experts.shape
assert num_tokens > 0
assert num_layers == NUM_HIDDEN_LAYERS
assert topk == NUM_EXPERTS_PER_TOK
assert (routed_experts >= 0).all()
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import httpx
import numpy as np
import pybase64 as base64
import pytest
import pytest_asyncio
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "TitanML/tiny-mixtral"
GEN_ENDPOINT = "/inference/v1/generate"
# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers.
# The published config has sliding_window=4096, which produces
# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a
# FullAttentionSpec group, so we override sliding_window=null below.
NUM_LOCAL_EXPERTS = 8
NUM_EXPERTS_PER_TOK = 2
NUM_HIDDEN_LAYERS = 2
@pytest.fixture(scope="module")
def server():
args = [
"--max-model-len",
"256",
"--max-num-seqs",
"32",
"--enforce-eager",
"--enable-return-routed-experts",
"--hf-overrides",
'{"sliding_window": null}',
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@pytest_asyncio.fixture
async def client(server: RemoteOpenAIServer):
transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None
headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"}
async with httpx.AsyncClient(
transport=transport,
base_url=server.url_root,
timeout=600,
headers=headers,
) as c:
yield c
@pytest.mark.asyncio
async def test_generate_routed_experts(client):
"""Test that /inference/v1/generate returns routed_experts when enabled."""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {"max_tokens": 10, "temperature": 0.0},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
choice = data["choices"][0]
assert choice["routed_experts"] is not None
assert choice["token_ids"] is not None
# routed_experts is base64-encoded .npy bytes; decode to ndarray.
routed_experts = np.load(io.BytesIO(base64.b64decode(choice["routed_experts"])))
assert routed_experts.ndim == 3
num_tokens, num_layers, topk = routed_experts.shape
assert num_tokens > 0
assert num_layers == NUM_HIDDEN_LAYERS
assert topk == NUM_EXPERTS_PER_TOK
assert (routed_experts >= 0).all()
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
@@ -92,6 +92,16 @@ class ChatCompletionResponseChoice(OpenAIBaseModel):
# not part of the OpenAI spec but is useful for tracing the tokens
# in agent scenarios
token_ids: list[int] | None = None
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
# (numpy serialization). Shape after decode:
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
# ``num_tokens - 1`` because the last sampled token has not been
# forwarded yet and therefore has no routing data.
# Decode:
# np.load(io.BytesIO(base64.b64decode(s)))
# ``None`` if (a) the request was aborted before any forward pass,
# or (b) ``enable_return_routed_experts`` is off server-side.
routed_experts: str | None = None
class ChatCompletionResponse(OpenAIBaseModel):
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import io
import json
import time
from collections.abc import AsyncGenerator, AsyncIterator
@@ -9,6 +10,8 @@ from collections.abc import Sequence as GenericSequence
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, Final
import numpy as np
import pybase64 as base64
from fastapi import Request
from vllm.engine.protocol import EngineClient
@@ -1086,6 +1089,18 @@ class OpenAIServingChat(OpenAIServing):
content=content,
)
# Encode routed_experts for transport. JSON can't carry raw
# bytes, so we write the ndarray as a ``.npy`` byte stream
# and base64-encode it. ``pybase64`` is ~3x faster than the
# stdlib ``base64`` on large payloads thanks to SIMD.
routed_experts_b64 = None
if output.routed_experts is not None:
buf = io.BytesIO()
np.save(buf, output.routed_experts)
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
"ascii"
)
choice_data = ChatCompletionResponseChoice(
index=output.index,
message=message,
@@ -1101,6 +1116,7 @@ class OpenAIServingChat(OpenAIServing):
token_ids=(
as_list(output.token_ids) if request.return_token_ids else None
),
routed_experts=routed_experts_b64,
)
choices.append(choice_data)
continue
@@ -1309,6 +1325,16 @@ class OpenAIServingChat(OpenAIServing):
and output.finish_reason == "stop"
)
# Encode routed_experts for transport. JSON can't carry raw
# bytes, so we write the ndarray as a ``.npy`` byte stream
# and base64-encode it. ``pybase64`` is ~3x faster than the
# stdlib ``base64`` on large payloads thanks to SIMD.
routed_experts_b64 = None
if output.routed_experts is not None:
buf = io.BytesIO()
np.save(buf, output.routed_experts)
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode("ascii")
choice_data = ChatCompletionResponseChoice(
index=output.index,
message=message,
@@ -1322,6 +1348,7 @@ class OpenAIServingChat(OpenAIServing):
token_ids=(
as_list(output.token_ids) if request.return_token_ids else None
),
routed_experts=routed_experts_b64,
)
choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
@@ -477,6 +477,16 @@ class CompletionResponseChoice(OpenAIBaseModel):
token_ids: list[int] | None = None # For response
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
prompt_token_ids: list[int] | None = None # For prompt
# Per-token expert routing decisions, base64-encoded ``.npy`` bytes
# (numpy serialization). Shape after decode:
# (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16
# ``num_tokens - 1`` because the last sampled token has not been
# forwarded yet and therefore has no routing data.
# Decode:
# np.load(io.BytesIO(base64.b64decode(s)))
# ``None`` if (a) the request was aborted before any forward pass,
# or (b) ``enable_return_routed_experts`` is off server-side.
routed_experts: str | None = None
class CompletionResponse(OpenAIBaseModel):
@@ -2,11 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import io
import time
from collections.abc import AsyncGenerator, AsyncIterator
from collections.abc import Sequence as GenericSequence
from typing import TYPE_CHECKING, cast
import numpy as np
import pybase64 as base64
from fastapi import Request
from vllm.engine.protocol import EngineClient
@@ -538,6 +541,18 @@ class OpenAIServingCompletion(OpenAIServing):
else:
logprobs = None
# Encode routed_experts for transport. JSON can't carry raw
# bytes, so we write the ndarray as a ``.npy`` byte stream
# and base64-encode it. ``pybase64`` is ~3x faster than the
# stdlib ``base64`` on large payloads thanks to SIMD.
routed_experts_b64 = None
if output.routed_experts is not None:
buf = io.BytesIO()
np.save(buf, output.routed_experts)
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
"ascii"
)
choice_data = CompletionResponseChoice(
index=len(choices),
text=output_text,
@@ -551,6 +566,7 @@ class OpenAIServingCompletion(OpenAIServing):
token_ids=(
as_list(output.token_ids) if request.return_token_ids else None
),
routed_experts=routed_experts_b64,
)
choices.append(choice_data)