[Frontend] Add routed-experts prompt offset (#51906)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-08-13 08:09:00 -07:00
committed by GitHub
co-authored by OpenAI Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f3c1638927
commit 7553aac77b
13 changed files with 114 additions and 61 deletions
@@ -822,3 +822,19 @@ def test_completion_request_bad_words_default_empty():
default_sampling_params={},
)
assert sampling_params.bad_words == []
def test_completion_request_forwards_routed_experts_prompt_start():
request = CompletionRequest(
model="test-model",
prompt="Hello",
max_tokens=10,
routed_experts_prompt_start=3,
)
sampling_params = request.to_sampling_params(
max_tokens=10,
default_sampling_params={},
)
assert sampling_params.routed_experts_prompt_start == 3
@@ -20,6 +20,18 @@ NUM_EXPERTS_PER_TOK = 2
NUM_HIDDEN_LAYERS = 2
def assert_valid_routed_experts(encoded: str | None) -> None:
assert encoded is not None
routed_experts = np.load(io.BytesIO(base64.b64decode(encoded)))
assert routed_experts.ndim == 3
num_tokens, num_layers, topk = routed_experts.shape
assert num_tokens > 0
assert num_layers == NUM_HIDDEN_LAYERS
assert topk == NUM_EXPERTS_PER_TOK
assert (routed_experts >= 0).all()
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
@pytest.fixture(scope="module")
def server():
args = [
@@ -50,15 +62,5 @@ async def test_routed_experts(server):
choice = result.model_dump()["choices"][0]
assert choice["routed_experts"] is not None
assert choice["token_ids"] is not None
# routed_experts is base64-encoded .npy bytes; decode to ndarray.
routed_experts = np.load(io.BytesIO(base64.b64decode(choice["routed_experts"])))
assert routed_experts.ndim == 3
num_tokens, num_layers, topk = routed_experts.shape
assert num_tokens > 0
assert num_layers == NUM_HIDDEN_LAYERS
assert topk == NUM_EXPERTS_PER_TOK
assert (routed_experts >= 0).all()
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
assert_valid_routed_experts(choice["routed_experts"])
@@ -87,6 +87,20 @@ class TestChatCompletionStopTokenIds:
assert set(sampling_params.stop_token_ids) == {42, 43}
def test_routed_experts_prompt_start_is_forwarded(self):
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hello"}],
routed_experts_prompt_start=3,
)
sampling_params = request.to_sampling_params(
max_tokens=100,
default_sampling_params={},
)
assert sampling_params.routed_experts_prompt_start == 3
def test_duplicate_stop_token_ids_deduplicated(self):
"""Overlapping stop_token_ids between client and server are deduplicated."""
request = ChatCompletionRequest(
+16
View File
@@ -1,5 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import io
import numpy as np
import pybase64
import pytest
import torch
@@ -12,6 +16,7 @@ from vllm.utils.serial_utils import (
Endianness,
MmMetadataDType,
binary2tensor,
numpy2base64,
tensor2binary,
)
@@ -19,6 +24,17 @@ FLOAT_EMBED_DTYPES = tuple(EMBED_DTYPES.keys())
INTEGER_EMBED_DTYPES = tuple(MM_METADATA_DTYPES.keys())
def test_numpy2base64_round_trip():
array = np.arange(24, dtype=np.uint8).reshape(2, 3, 4)
decoded = np.load(
io.BytesIO(pybase64.b64decode(numpy2base64(array))),
allow_pickle=False,
)
np.testing.assert_array_equal(decoded, array)
def _build_integer_tensor(
embed_dtype: MmMetadataDType, shape: tuple[int, ...]
) -> torch.Tensor: