[AuxOutput] Add block-keyed storage for routed-expert outputs (#45635)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Hongxin Xu
2026-09-21 10:25:05 +00:00
committed by GitHub
co-authored by aoshen02 OpenAI Codex Roger Wang Nick Hill Codex
parent 0aee727ff6
commit 0b7f11a1ee
32 changed files with 3604 additions and 877 deletions
+12 -17
View File
@@ -3,12 +3,14 @@
import numpy as np
from vllm.distributed.aux_output_connector.connector import (
AuxRequestOutput,
)
from vllm.v1.executor.ray_utils import detach_zero_copy_from_model_runner_output
from vllm.v1.outputs import (
LogprobsLists,
LogprobsTensors,
ModelRunnerOutput,
RoutedExpertsLists,
)
@@ -59,26 +61,19 @@ def test_detach_zero_copy_from_model_runner_output_copies_only_numpy_views():
assert output.prompt_logprobs_dict["req-0"] is prompt_logprobs
def test_detach_zero_copy_routed_experts_without_logprobs():
def test_detach_zero_copy_aux_output_without_logprobs():
rows = _make_readonly(np.arange(12, dtype=np.uint8).reshape(2, 3, 2))
output = ModelRunnerOutput(
req_ids=["req-0"],
req_id_to_index={"req-0": 0},
routed_experts=RoutedExpertsLists(
routing_data=_make_readonly(np.arange(12, dtype=np.int32).reshape(2, 3, 2)),
slot_mapping=_make_readonly(np.array([7, 8], dtype=np.int64)),
),
aux_output_connector_output={"req-0": AuxRequestOutput(0, rows)},
)
original = output.routed_experts
assert output.logprobs is None
detach_zero_copy_from_model_runner_output(output)
detached = output.routed_experts
assert detached is not None
assert detached is not original
assert detached.routing_data is not original.routing_data
assert detached.slot_mapping is not original.slot_mapping
assert detached.routing_data.flags.writeable
assert detached.slot_mapping.flags.writeable
np.testing.assert_array_equal(detached.routing_data, original.routing_data)
np.testing.assert_array_equal(detached.slot_mapping, original.slot_mapping)
aux_output = output.aux_output_connector_output
assert aux_output is not None
detached = aux_output["req-0"].rows
assert detached is not rows
assert detached.flags.writeable
np.testing.assert_array_equal(detached, rows)