[Bugfix] Validate routed-expert prompt offsets before engine submission (#56844)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Bhushan Asati <bhushanasati25@gmail.com>
Co-authored-by: Megha Agarwal <19240983+meghaagr13@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-09-15 09:17:41 +08:00
committed by GitHub
co-authored by Bhushan Asati Megha Agarwal
parent ef5f7cd119
commit 84e5f3edac
4 changed files with 83 additions and 10 deletions
+30
View File
@@ -12,6 +12,7 @@ import subprocess
import sys
from collections.abc import Callable
from typing import Protocol
from unittest.mock import Mock
import pytest
from pydantic import ValidationError
@@ -27,10 +28,39 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import BeamSearchParams
from vllm.tokenizers import deepseek_v4_encoding, deepseek_v32_encoding
from vllm.v1.engine.input_processor import InputProcessor
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
@pytest.mark.parametrize("offset", [-1, 0, 2, 3, 4, 2**127])
def test_routed_experts_offset_validated_before_engine_submission(offset):
"""Reject invalid offsets even when routed-expert output is disabled."""
processor = Mock(spec=InputProcessor)
processor.tokenizer = None
processor.generation_config_fields = {}
processor.renderer = Mock()
processor.renderer.get_eos_token_id.return_value = None
processor.vllm_config = Mock()
params = SamplingParams(routed_experts_prompt_start=offset)
prompt = {"type": "token", "prompt_token_ids": [1, 2, 3]}
if not 0 <= offset <= 3:
with pytest.raises(VLLMValidationError, match="routed_experts_prompt_start"):
InputProcessor.process_inputs(
processor, "invalid", prompt, params, ("generate",)
)
params.routed_experts_prompt_start = 0
request = InputProcessor.process_inputs(
processor, "valid", prompt, params, ("generate",)
)
assert (
request.sampling_params.routed_experts_prompt_start
== params.routed_experts_prompt_start
)
# --- Stop strings: public requests cap the number of stop strings ---------
+39
View File
@@ -51,6 +51,7 @@ from vllm.v1.outputs import (
ECConnectorOutput,
KVConnectorOutput,
ModelRunnerOutput,
RoutedExpertsLists,
SamplingMaskLists,
make_empty_encoder_model_runner_output,
)
@@ -131,6 +132,44 @@ def test_add_requests():
assert len(scheduler.waiting) == i + 1
def test_routed_experts_prompt_start_at_prompt_end():
"""A prompt-end offset should return empty routing data, not crash."""
scheduler = create_scheduler()
(request,) = create_requests(num_requests=1)
request.sampling_params.routed_experts_prompt_start = request.num_prompt_tokens
scheduler.add_request(request)
scheduler_output = scheduler.schedule()
num_tokens = scheduler_output.num_scheduled_tokens[request.request_id]
manager = Mock()
manager.routed_experts_by_slot = np.empty((0, 1, 1), dtype=np.uint8)
manager.get.return_value = np.empty((0, 1, 1), dtype=np.uint8)
scheduler.enable_return_routed_experts = True
scheduler.routed_experts_mgr = manager
scheduler._re_block_ids = {request.request_id: []}
outputs = scheduler.update_from_output(
scheduler_output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[0]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
routed_experts=RoutedExpertsLists(
routing_data=np.zeros((num_tokens, 1, 1), dtype=np.uint8),
slot_mapping=np.arange(num_tokens),
),
),
)
manager.get.assert_called_once_with(
[], request.num_prompt_tokens, token_start=request.num_prompt_tokens
)
assert outputs[request.client_index].outputs[0].routed_experts.shape == (0, 1, 1)
def test_finish_request():
scheduler = create_scheduler()
requests = create_requests(num_requests=10)
+1 -1
View File
@@ -2101,7 +2101,7 @@ class Scheduler(SchedulerInterface):
prompt_start = (
request.sampling_params.routed_experts_prompt_start
)
assert prompt_start < request.num_prompt_tokens
assert prompt_start <= request.num_prompt_tokens
else:
prompt_start = 0
routed_experts = self.routed_experts_mgr.get(
+13 -9
View File
@@ -354,12 +354,21 @@ class InputProcessor:
if isinstance(params, SamplingParams):
# TODO: can we avoid cloning here in multiproc case?
sampling_params = params.clone()
prompt_len = length_from_prompt_token_ids_or_embeds(
prompt_token_ids, prompt_embeds
)
if not 0 <= sampling_params.routed_experts_prompt_start <= prompt_len:
raise VLLMValidationError(
f"routed_experts_prompt_start must be between 0 and "
f"the prompt length ({prompt_len}), inclusive.",
parameter="routed_experts_prompt_start",
value=sampling_params.routed_experts_prompt_start,
)
# If unset max tokens, then generate up to the max_model_len.
if sampling_params.max_tokens is None:
seq_len = length_from_prompt_token_ids_or_embeds(
prompt_token_ids, prompt_embeds
sampling_params.max_tokens = (
self.model_config.max_model_len - prompt_len
)
sampling_params.max_tokens = self.model_config.max_model_len - seq_len
sampling_params.update_from_generation_config(
self.generation_config_fields,
@@ -368,12 +377,7 @@ class InputProcessor:
if self.tokenizer is not None:
sampling_params.update_from_tokenizer(self.tokenizer)
if sampling_params.trace_decode_token_ids:
self._normalize_trace_replay_params(
sampling_params,
length_from_prompt_token_ids_or_embeds(
prompt_token_ids, prompt_embeds
),
)
self._normalize_trace_replay_params(sampling_params, prompt_len)
else:
pooling_params = params.clone()