[MRV2] Enable routed-experts capture (#50721)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -4,7 +4,12 @@
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.executor.ray_utils import detach_zero_copy_from_model_runner_output
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, ModelRunnerOutput
|
||||
from vllm.v1.outputs import (
|
||||
LogprobsLists,
|
||||
LogprobsTensors,
|
||||
ModelRunnerOutput,
|
||||
RoutedExpertsLists,
|
||||
)
|
||||
|
||||
|
||||
def _make_readonly(arr: np.ndarray) -> np.ndarray:
|
||||
@@ -52,3 +57,28 @@ def test_detach_zero_copy_from_model_runner_output_copies_only_numpy_views():
|
||||
assert detached_logprobs.sampled_token_ranks.flags.writeable
|
||||
assert detached_logprobs.cu_num_generated_tokens is cu_num_generated_tokens
|
||||
assert output.prompt_logprobs_dict["req-0"] is prompt_logprobs
|
||||
|
||||
|
||||
def test_detach_zero_copy_routed_experts_without_logprobs():
|
||||
output = ModelRunnerOutput(
|
||||
req_ids=["req-0"],
|
||||
req_id_to_index={"req-0": 0},
|
||||
routed_experts=RoutedExpertsLists(
|
||||
routing_data=_make_readonly(np.arange(12, dtype=np.int32).reshape(2, 3, 2)),
|
||||
slot_mapping=_make_readonly(np.array([7, 8], dtype=np.int64)),
|
||||
),
|
||||
)
|
||||
original = output.routed_experts
|
||||
assert output.logprobs is None
|
||||
|
||||
detach_zero_copy_from_model_runner_output(output)
|
||||
|
||||
detached = output.routed_experts
|
||||
assert detached is not None
|
||||
assert detached is not original
|
||||
assert detached.routing_data is not original.routing_data
|
||||
assert detached.slot_mapping is not original.slot_mapping
|
||||
assert detached.routing_data.flags.writeable
|
||||
assert detached.slot_mapping.flags.writeable
|
||||
np.testing.assert_array_equal(detached.routing_data, original.routing_data)
|
||||
np.testing.assert_array_equal(detached.slot_mapping, original.slot_mapping)
|
||||
|
||||
Reference in New Issue
Block a user