[AuxOutput] Add block-keyed storage for routed-expert outputs (#45635)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
Roger Wang
Nick Hill
Codex
parent
0aee727ff6
commit
0b7f11a1ee
@@ -3,12 +3,14 @@
|
||||
|
||||
import numpy as np
|
||||
|
||||
from vllm.distributed.aux_output_connector.connector import (
|
||||
AuxRequestOutput,
|
||||
)
|
||||
from vllm.v1.executor.ray_utils import detach_zero_copy_from_model_runner_output
|
||||
from vllm.v1.outputs import (
|
||||
LogprobsLists,
|
||||
LogprobsTensors,
|
||||
ModelRunnerOutput,
|
||||
RoutedExpertsLists,
|
||||
)
|
||||
|
||||
|
||||
@@ -59,26 +61,19 @@ def test_detach_zero_copy_from_model_runner_output_copies_only_numpy_views():
|
||||
assert output.prompt_logprobs_dict["req-0"] is prompt_logprobs
|
||||
|
||||
|
||||
def test_detach_zero_copy_routed_experts_without_logprobs():
|
||||
def test_detach_zero_copy_aux_output_without_logprobs():
|
||||
rows = _make_readonly(np.arange(12, dtype=np.uint8).reshape(2, 3, 2))
|
||||
output = ModelRunnerOutput(
|
||||
req_ids=["req-0"],
|
||||
req_id_to_index={"req-0": 0},
|
||||
routed_experts=RoutedExpertsLists(
|
||||
routing_data=_make_readonly(np.arange(12, dtype=np.int32).reshape(2, 3, 2)),
|
||||
slot_mapping=_make_readonly(np.array([7, 8], dtype=np.int64)),
|
||||
),
|
||||
aux_output_connector_output={"req-0": AuxRequestOutput(0, rows)},
|
||||
)
|
||||
original = output.routed_experts
|
||||
assert output.logprobs is None
|
||||
|
||||
detach_zero_copy_from_model_runner_output(output)
|
||||
|
||||
detached = output.routed_experts
|
||||
assert detached is not None
|
||||
assert detached is not original
|
||||
assert detached.routing_data is not original.routing_data
|
||||
assert detached.slot_mapping is not original.slot_mapping
|
||||
assert detached.routing_data.flags.writeable
|
||||
assert detached.slot_mapping.flags.writeable
|
||||
np.testing.assert_array_equal(detached.routing_data, original.routing_data)
|
||||
np.testing.assert_array_equal(detached.slot_mapping, original.slot_mapping)
|
||||
aux_output = output.aux_output_connector_output
|
||||
assert aux_output is not None
|
||||
detached = aux_output["req-0"].rows
|
||||
assert detached is not rows
|
||||
assert detached.flags.writeable
|
||||
np.testing.assert_array_equal(detached, rows)
|
||||
|
||||
Reference in New Issue
Block a user