diff --git a/tests/entrypoints/openai/test_return_routed_experts.py b/tests/entrypoints/openai/test_return_routed_experts.py new file mode 100644 index 0000000000..b69c4c4915 --- /dev/null +++ b/tests/entrypoints/openai/test_return_routed_experts.py @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import io + +import numpy as np +import pybase64 as base64 +import pytest + +from ...utils import RemoteOpenAIServer + +MODEL_NAME = "TitanML/tiny-mixtral" + +# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers. +# The published config has sliding_window=4096, which produces +# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a +# FullAttentionSpec group, so we override sliding_window=null below. +NUM_LOCAL_EXPERTS = 8 +NUM_EXPERTS_PER_TOK = 2 +NUM_HIDDEN_LAYERS = 2 + + +@pytest.fixture(scope="module") +def server(): + args = [ + "--max-model-len", + "256", + "--max-num-seqs", + "32", + "--enforce-eager", + "--enable-return-routed-experts", + "--hf-overrides", + '{"sliding_window": null}', + ] + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: + yield remote_server + + +@pytest.mark.asyncio +async def test_routed_experts(server): + """Test that /v1/completions returns routed_experts when enabled.""" + async with server.get_async_client() as client: + result = await client.completions.create( + model=MODEL_NAME, + prompt="Hello, world", + max_tokens=10, + temperature=0, + extra_body={"return_token_ids": True}, + ) + + choice = result.model_dump()["choices"][0] + + assert choice["routed_experts"] is not None + assert choice["token_ids"] is not None + + # routed_experts is base64-encoded .npy bytes; decode to ndarray. + routed_experts = np.load(io.BytesIO(base64.b64decode(choice["routed_experts"]))) + assert routed_experts.ndim == 3 + num_tokens, num_layers, topk = routed_experts.shape + assert num_tokens > 0 + assert num_layers == NUM_HIDDEN_LAYERS + assert topk == NUM_EXPERTS_PER_TOK + assert (routed_experts >= 0).all() + assert (routed_experts < NUM_LOCAL_EXPERTS).all() diff --git a/tests/entrypoints/serve/disagg/test_return_routed_experts.py b/tests/entrypoints/serve/disagg/test_return_routed_experts.py new file mode 100644 index 0000000000..f7fd179516 --- /dev/null +++ b/tests/entrypoints/serve/disagg/test_return_routed_experts.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import io + +import httpx +import numpy as np +import pybase64 as base64 +import pytest +import pytest_asyncio + +from tests.utils import RemoteOpenAIServer + +MODEL_NAME = "TitanML/tiny-mixtral" +GEN_ENDPOINT = "/inference/v1/generate" + +# tiny-mixtral config: 8 local experts, top-2 routing, 2 hidden layers. +# The published config has sliding_window=4096, which produces +# SlidingWindowSpec kv-cache groups; RoutedExpertsManager requires a +# FullAttentionSpec group, so we override sliding_window=null below. +NUM_LOCAL_EXPERTS = 8 +NUM_EXPERTS_PER_TOK = 2 +NUM_HIDDEN_LAYERS = 2 + + +@pytest.fixture(scope="module") +def server(): + args = [ + "--max-model-len", + "256", + "--max-num-seqs", + "32", + "--enforce-eager", + "--enable-return-routed-experts", + "--hf-overrides", + '{"sliding_window": null}', + ] + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: + yield remote_server + + +@pytest_asyncio.fixture +async def client(server: RemoteOpenAIServer): + transport = httpx.AsyncHTTPTransport(uds=server.uds) if server.uds else None + headers = {"Authorization": f"Bearer {server.DUMMY_API_KEY}"} + async with httpx.AsyncClient( + transport=transport, + base_url=server.url_root, + timeout=600, + headers=headers, + ) as c: + yield c + + +@pytest.mark.asyncio +async def test_generate_routed_experts(client): + """Test that /inference/v1/generate returns routed_experts when enabled.""" + payload = { + "model": MODEL_NAME, + "token_ids": [1, 2, 3], + "sampling_params": {"max_tokens": 10, "temperature": 0.0}, + "stream": False, + } + resp = await client.post(GEN_ENDPOINT, json=payload) + resp.raise_for_status() + data = resp.json() + + choice = data["choices"][0] + + assert choice["routed_experts"] is not None + assert choice["token_ids"] is not None + + # routed_experts is base64-encoded .npy bytes; decode to ndarray. + routed_experts = np.load(io.BytesIO(base64.b64decode(choice["routed_experts"]))) + assert routed_experts.ndim == 3 + num_tokens, num_layers, topk = routed_experts.shape + assert num_tokens > 0 + assert num_layers == NUM_HIDDEN_LAYERS + assert topk == NUM_EXPERTS_PER_TOK + assert (routed_experts >= 0).all() + assert (routed_experts < NUM_LOCAL_EXPERTS).all() diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index cf843f6f1d..23ca42a3ce 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -92,6 +92,16 @@ class ChatCompletionResponseChoice(OpenAIBaseModel): # not part of the OpenAI spec but is useful for tracing the tokens # in agent scenarios token_ids: list[int] | None = None + # Per-token expert routing decisions, base64-encoded ``.npy`` bytes + # (numpy serialization). Shape after decode: + # (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16 + # ``num_tokens - 1`` because the last sampled token has not been + # forwarded yet and therefore has no routing data. + # Decode: + # np.load(io.BytesIO(base64.b64decode(s))) + # ``None`` if (a) the request was aborted before any forward pass, + # or (b) ``enable_return_routed_experts`` is off server-side. + routed_experts: str | None = None class ChatCompletionResponse(OpenAIBaseModel): diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 29a0249f69..8f6d76d785 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio +import io import json import time from collections.abc import AsyncGenerator, AsyncIterator @@ -9,6 +10,8 @@ from collections.abc import Sequence as GenericSequence from http import HTTPStatus from typing import TYPE_CHECKING, Any, Final +import numpy as np +import pybase64 as base64 from fastapi import Request from vllm.engine.protocol import EngineClient @@ -1086,6 +1089,18 @@ class OpenAIServingChat(OpenAIServing): content=content, ) + # Encode routed_experts for transport. JSON can't carry raw + # bytes, so we write the ndarray as a ``.npy`` byte stream + # and base64-encode it. ``pybase64`` is ~3x faster than the + # stdlib ``base64`` on large payloads thanks to SIMD. + routed_experts_b64 = None + if output.routed_experts is not None: + buf = io.BytesIO() + np.save(buf, output.routed_experts) + routed_experts_b64 = base64.b64encode(buf.getvalue()).decode( + "ascii" + ) + choice_data = ChatCompletionResponseChoice( index=output.index, message=message, @@ -1101,6 +1116,7 @@ class OpenAIServingChat(OpenAIServing): token_ids=( as_list(output.token_ids) if request.return_token_ids else None ), + routed_experts=routed_experts_b64, ) choices.append(choice_data) continue @@ -1309,6 +1325,16 @@ class OpenAIServingChat(OpenAIServing): and output.finish_reason == "stop" ) + # Encode routed_experts for transport. JSON can't carry raw + # bytes, so we write the ndarray as a ``.npy`` byte stream + # and base64-encode it. ``pybase64`` is ~3x faster than the + # stdlib ``base64`` on large payloads thanks to SIMD. + routed_experts_b64 = None + if output.routed_experts is not None: + buf = io.BytesIO() + np.save(buf, output.routed_experts) + routed_experts_b64 = base64.b64encode(buf.getvalue()).decode("ascii") + choice_data = ChatCompletionResponseChoice( index=output.index, message=message, @@ -1322,6 +1348,7 @@ class OpenAIServingChat(OpenAIServing): token_ids=( as_list(output.token_ids) if request.return_token_ids else None ), + routed_experts=routed_experts_b64, ) choice_data = maybe_filter_parallel_tool_calls(choice_data, request) diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index 831e5d9100..f05cebc9c7 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -477,6 +477,16 @@ class CompletionResponseChoice(OpenAIBaseModel): token_ids: list[int] | None = None # For response prompt_logprobs: list[dict[int, Logprob] | None] | None = None prompt_token_ids: list[int] | None = None # For prompt + # Per-token expert routing decisions, base64-encoded ``.npy`` bytes + # (numpy serialization). Shape after decode: + # (num_tokens - 1, num_layers, num_experts_per_tok) dtype uint8/uint16 + # ``num_tokens - 1`` because the last sampled token has not been + # forwarded yet and therefore has no routing data. + # Decode: + # np.load(io.BytesIO(base64.b64decode(s))) + # ``None`` if (a) the request was aborted before any forward pass, + # or (b) ``enable_return_routed_experts`` is off server-side. + routed_experts: str | None = None class CompletionResponse(OpenAIBaseModel): diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index bfe9fb2267..f393954e2a 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -2,11 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio +import io import time from collections.abc import AsyncGenerator, AsyncIterator from collections.abc import Sequence as GenericSequence from typing import TYPE_CHECKING, cast +import numpy as np +import pybase64 as base64 from fastapi import Request from vllm.engine.protocol import EngineClient @@ -538,6 +541,18 @@ class OpenAIServingCompletion(OpenAIServing): else: logprobs = None + # Encode routed_experts for transport. JSON can't carry raw + # bytes, so we write the ndarray as a ``.npy`` byte stream + # and base64-encode it. ``pybase64`` is ~3x faster than the + # stdlib ``base64`` on large payloads thanks to SIMD. + routed_experts_b64 = None + if output.routed_experts is not None: + buf = io.BytesIO() + np.save(buf, output.routed_experts) + routed_experts_b64 = base64.b64encode(buf.getvalue()).decode( + "ascii" + ) + choice_data = CompletionResponseChoice( index=len(choices), text=output_text, @@ -551,6 +566,7 @@ class OpenAIServingCompletion(OpenAIServing): token_ids=( as_list(output.token_ids) if request.return_token_ids else None ), + routed_experts=routed_experts_b64, ) choices.append(choice_data)