[Frontend] Add routed-experts prompt offset (#51906)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
f3c1638927
commit
7553aac77b
@@ -822,3 +822,19 @@ def test_completion_request_bad_words_default_empty():
|
||||
default_sampling_params={},
|
||||
)
|
||||
assert sampling_params.bad_words == []
|
||||
|
||||
|
||||
def test_completion_request_forwards_routed_experts_prompt_start():
|
||||
request = CompletionRequest(
|
||||
model="test-model",
|
||||
prompt="Hello",
|
||||
max_tokens=10,
|
||||
routed_experts_prompt_start=3,
|
||||
)
|
||||
|
||||
sampling_params = request.to_sampling_params(
|
||||
max_tokens=10,
|
||||
default_sampling_params={},
|
||||
)
|
||||
|
||||
assert sampling_params.routed_experts_prompt_start == 3
|
||||
|
||||
@@ -20,6 +20,18 @@ NUM_EXPERTS_PER_TOK = 2
|
||||
NUM_HIDDEN_LAYERS = 2
|
||||
|
||||
|
||||
def assert_valid_routed_experts(encoded: str | None) -> None:
|
||||
assert encoded is not None
|
||||
routed_experts = np.load(io.BytesIO(base64.b64decode(encoded)))
|
||||
assert routed_experts.ndim == 3
|
||||
num_tokens, num_layers, topk = routed_experts.shape
|
||||
assert num_tokens > 0
|
||||
assert num_layers == NUM_HIDDEN_LAYERS
|
||||
assert topk == NUM_EXPERTS_PER_TOK
|
||||
assert (routed_experts >= 0).all()
|
||||
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
@@ -50,15 +62,5 @@ async def test_routed_experts(server):
|
||||
|
||||
choice = result.model_dump()["choices"][0]
|
||||
|
||||
assert choice["routed_experts"] is not None
|
||||
assert choice["token_ids"] is not None
|
||||
|
||||
# routed_experts is base64-encoded .npy bytes; decode to ndarray.
|
||||
routed_experts = np.load(io.BytesIO(base64.b64decode(choice["routed_experts"])))
|
||||
assert routed_experts.ndim == 3
|
||||
num_tokens, num_layers, topk = routed_experts.shape
|
||||
assert num_tokens > 0
|
||||
assert num_layers == NUM_HIDDEN_LAYERS
|
||||
assert topk == NUM_EXPERTS_PER_TOK
|
||||
assert (routed_experts >= 0).all()
|
||||
assert (routed_experts < NUM_LOCAL_EXPERTS).all()
|
||||
assert_valid_routed_experts(choice["routed_experts"])
|
||||
|
||||
@@ -87,6 +87,20 @@ class TestChatCompletionStopTokenIds:
|
||||
|
||||
assert set(sampling_params.stop_token_ids) == {42, 43}
|
||||
|
||||
def test_routed_experts_prompt_start_is_forwarded(self):
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
routed_experts_prompt_start=3,
|
||||
)
|
||||
|
||||
sampling_params = request.to_sampling_params(
|
||||
max_tokens=100,
|
||||
default_sampling_params={},
|
||||
)
|
||||
|
||||
assert sampling_params.routed_experts_prompt_start == 3
|
||||
|
||||
def test_duplicate_stop_token_ids_deduplicated(self):
|
||||
"""Overlapping stop_token_ids between client and server are deduplicated."""
|
||||
request = ChatCompletionRequest(
|
||||
|
||||
Reference in New Issue
Block a user