[AuxOutput] Add block-keyed storage for routed-expert outputs (#45635)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Hongxin Xu
2026-09-21 10:25:05 +00:00
committed by GitHub
co-authored by aoshen02 OpenAI Codex Roger Wang Nick Hill Codex
parent 0aee727ff6
commit 0b7f11a1ee
32 changed files with 3604 additions and 877 deletions
+47 -1
View File
@@ -3,8 +3,9 @@
import math
import time
from unittest.mock import MagicMock
from unittest.mock import MagicMock, Mock
import numpy as np
import pytest
from tests.v1.engine.utils import (
@@ -1307,6 +1308,7 @@ def test_lora_request_tracking(log_stats: bool, dummy_test_vectors):
async def test_request_output_collector():
NUM_REQS = 3
TEXT = "a"
routed_experts = np.arange(12, dtype=np.uint8).reshape(2, 3, 2)
def make_outputs() -> list[RequestOutput]:
return [
@@ -1322,6 +1324,9 @@ async def test_request_output_collector():
token_ids=[idx],
cumulative_logprob=(idx + 1 * 1.0),
logprobs=[{"a": idx, "b": idx}],
routed_experts=(
routed_experts if idx == NUM_REQS - 1 else None
),
finish_reason="length" if (idx == NUM_REQS - 1) else None,
)
],
@@ -1374,6 +1379,7 @@ async def test_request_output_collector():
assert output.finished
assert output.outputs[0].finish_reason == "length"
np.testing.assert_array_equal(output.outputs[0].routed_experts, routed_experts)
# Text, token_ids, and logprobs should get merged.
assert output.outputs[0].text == TEXT * num_to_put
for tok_0, tok_1 in zip(output.outputs[0].token_ids, list(range(num_to_put))):
@@ -1385,6 +1391,46 @@ async def test_request_output_collector():
assert output.outputs[0].cumulative_logprob == cumulative_logprob_expected
def test_routed_experts_are_accumulated_until_finish():
state = RequestState(
request_id="request-int",
external_req_id="request",
parent_req=None,
request_index=0,
lora_request=None,
output_kind=RequestOutputKind.DELTA,
prompt="prompt",
prompt_token_ids=[1, 2, 3, 4],
prompt_embeds=None,
logprobs_processor=Mock(
logprobs=None,
cumulative_logprob=0.0,
pop_prompt_logprobs=Mock(return_value=None),
),
detokenizer=Mock(get_next_output_text=Mock(return_value="")),
max_tokens_param=2,
arrival_time=0.0,
queue=None,
log_stats=False,
stream_interval=1,
)
prompt_chunk = np.arange(24, dtype=np.uint8).reshape(4, 3, 2)
decode_chunk = np.arange(12, dtype=np.uint8).reshape(2, 3, 2) + 24
state.routed_experts_chunks.append(prompt_chunk)
partial = state.make_request_output([5], None, None, None)
assert partial is not None
assert partial.outputs[0].routed_experts is None
state.routed_experts_chunks.append(decode_chunk)
finished = state.make_request_output([6], None, FinishReason.LENGTH, None)
assert finished is not None
np.testing.assert_array_equal(
finished.outputs[0].routed_experts,
np.concatenate((prompt_chunk, decode_chunk)),
)
@pytest.mark.asyncio
async def test_cumulative_output_collector_n():
"""Test collector correctly handles multiple outputs by index."""