[Bugfix] Validate routed-expert prompt offsets before engine submission (#56844)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Bhushan Asati <bhushanasati25@gmail.com> Co-authored-by: Megha Agarwal <19240983+meghaagr13@users.noreply.github.com>
This commit is contained in:
co-authored by
Bhushan Asati
Megha Agarwal
parent
ef5f7cd119
commit
84e5f3edac
@@ -51,6 +51,7 @@ from vllm.v1.outputs import (
|
||||
ECConnectorOutput,
|
||||
KVConnectorOutput,
|
||||
ModelRunnerOutput,
|
||||
RoutedExpertsLists,
|
||||
SamplingMaskLists,
|
||||
make_empty_encoder_model_runner_output,
|
||||
)
|
||||
@@ -131,6 +132,44 @@ def test_add_requests():
|
||||
assert len(scheduler.waiting) == i + 1
|
||||
|
||||
|
||||
def test_routed_experts_prompt_start_at_prompt_end():
|
||||
"""A prompt-end offset should return empty routing data, not crash."""
|
||||
scheduler = create_scheduler()
|
||||
(request,) = create_requests(num_requests=1)
|
||||
request.sampling_params.routed_experts_prompt_start = request.num_prompt_tokens
|
||||
scheduler.add_request(request)
|
||||
scheduler_output = scheduler.schedule()
|
||||
num_tokens = scheduler_output.num_scheduled_tokens[request.request_id]
|
||||
|
||||
manager = Mock()
|
||||
manager.routed_experts_by_slot = np.empty((0, 1, 1), dtype=np.uint8)
|
||||
manager.get.return_value = np.empty((0, 1, 1), dtype=np.uint8)
|
||||
scheduler.enable_return_routed_experts = True
|
||||
scheduler.routed_experts_mgr = manager
|
||||
scheduler._re_block_ids = {request.request_id: []}
|
||||
|
||||
outputs = scheduler.update_from_output(
|
||||
scheduler_output,
|
||||
ModelRunnerOutput(
|
||||
req_ids=[request.request_id],
|
||||
req_id_to_index={request.request_id: 0},
|
||||
sampled_token_ids=[[0]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
routed_experts=RoutedExpertsLists(
|
||||
routing_data=np.zeros((num_tokens, 1, 1), dtype=np.uint8),
|
||||
slot_mapping=np.arange(num_tokens),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
manager.get.assert_called_once_with(
|
||||
[], request.num_prompt_tokens, token_start=request.num_prompt_tokens
|
||||
)
|
||||
assert outputs[request.client_index].outputs[0].routed_experts.shape == (0, 1, 1)
|
||||
|
||||
|
||||
def test_finish_request():
|
||||
scheduler = create_scheduler()
|
||||
requests = create_requests(num_requests=10)
|
||||
|
||||
Reference in New Issue
Block a user