diff --git a/tests/test_request_input_bounds.py b/tests/test_request_input_bounds.py index 44f536e7fb..990f10cfeb 100644 --- a/tests/test_request_input_bounds.py +++ b/tests/test_request_input_bounds.py @@ -12,6 +12,7 @@ import subprocess import sys from collections.abc import Callable from typing import Protocol +from unittest.mock import Mock import pytest from pydantic import ValidationError @@ -27,10 +28,39 @@ from vllm.entrypoints.openai.responses.protocol import ResponsesRequest from vllm.exceptions import VLLMValidationError from vllm.sampling_params import BeamSearchParams from vllm.tokenizers import deepseek_v4_encoding, deepseek_v32_encoding +from vllm.v1.engine.input_processor import InputProcessor pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup] +@pytest.mark.parametrize("offset", [-1, 0, 2, 3, 4, 2**127]) +def test_routed_experts_offset_validated_before_engine_submission(offset): + """Reject invalid offsets even when routed-expert output is disabled.""" + processor = Mock(spec=InputProcessor) + processor.tokenizer = None + processor.generation_config_fields = {} + processor.renderer = Mock() + processor.renderer.get_eos_token_id.return_value = None + processor.vllm_config = Mock() + params = SamplingParams(routed_experts_prompt_start=offset) + prompt = {"type": "token", "prompt_token_ids": [1, 2, 3]} + + if not 0 <= offset <= 3: + with pytest.raises(VLLMValidationError, match="routed_experts_prompt_start"): + InputProcessor.process_inputs( + processor, "invalid", prompt, params, ("generate",) + ) + params.routed_experts_prompt_start = 0 + + request = InputProcessor.process_inputs( + processor, "valid", prompt, params, ("generate",) + ) + assert ( + request.sampling_params.routed_experts_prompt_start + == params.routed_experts_prompt_start + ) + + # --- Stop strings: public requests cap the number of stop strings --------- diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 3a972e682e..6ca6a8cf87 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -51,6 +51,7 @@ from vllm.v1.outputs import ( ECConnectorOutput, KVConnectorOutput, ModelRunnerOutput, + RoutedExpertsLists, SamplingMaskLists, make_empty_encoder_model_runner_output, ) @@ -131,6 +132,44 @@ def test_add_requests(): assert len(scheduler.waiting) == i + 1 +def test_routed_experts_prompt_start_at_prompt_end(): + """A prompt-end offset should return empty routing data, not crash.""" + scheduler = create_scheduler() + (request,) = create_requests(num_requests=1) + request.sampling_params.routed_experts_prompt_start = request.num_prompt_tokens + scheduler.add_request(request) + scheduler_output = scheduler.schedule() + num_tokens = scheduler_output.num_scheduled_tokens[request.request_id] + + manager = Mock() + manager.routed_experts_by_slot = np.empty((0, 1, 1), dtype=np.uint8) + manager.get.return_value = np.empty((0, 1, 1), dtype=np.uint8) + scheduler.enable_return_routed_experts = True + scheduler.routed_experts_mgr = manager + scheduler._re_block_ids = {request.request_id: []} + + outputs = scheduler.update_from_output( + scheduler_output, + ModelRunnerOutput( + req_ids=[request.request_id], + req_id_to_index={request.request_id: 0}, + sampled_token_ids=[[0]], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + routed_experts=RoutedExpertsLists( + routing_data=np.zeros((num_tokens, 1, 1), dtype=np.uint8), + slot_mapping=np.arange(num_tokens), + ), + ), + ) + + manager.get.assert_called_once_with( + [], request.num_prompt_tokens, token_start=request.num_prompt_tokens + ) + assert outputs[request.client_index].outputs[0].routed_experts.shape == (0, 1, 1) + + def test_finish_request(): scheduler = create_scheduler() requests = create_requests(num_requests=10) diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 1b17eb3b15..3af527124d 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -2101,7 +2101,7 @@ class Scheduler(SchedulerInterface): prompt_start = ( request.sampling_params.routed_experts_prompt_start ) - assert prompt_start < request.num_prompt_tokens + assert prompt_start <= request.num_prompt_tokens else: prompt_start = 0 routed_experts = self.routed_experts_mgr.get( diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 4cc433cbfe..3f86f3c8b2 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -354,12 +354,21 @@ class InputProcessor: if isinstance(params, SamplingParams): # TODO: can we avoid cloning here in multiproc case? sampling_params = params.clone() + prompt_len = length_from_prompt_token_ids_or_embeds( + prompt_token_ids, prompt_embeds + ) + if not 0 <= sampling_params.routed_experts_prompt_start <= prompt_len: + raise VLLMValidationError( + f"routed_experts_prompt_start must be between 0 and " + f"the prompt length ({prompt_len}), inclusive.", + parameter="routed_experts_prompt_start", + value=sampling_params.routed_experts_prompt_start, + ) # If unset max tokens, then generate up to the max_model_len. if sampling_params.max_tokens is None: - seq_len = length_from_prompt_token_ids_or_embeds( - prompt_token_ids, prompt_embeds + sampling_params.max_tokens = ( + self.model_config.max_model_len - prompt_len ) - sampling_params.max_tokens = self.model_config.max_model_len - seq_len sampling_params.update_from_generation_config( self.generation_config_fields, @@ -368,12 +377,7 @@ class InputProcessor: if self.tokenizer is not None: sampling_params.update_from_tokenizer(self.tokenizer) if sampling_params.trace_decode_token_ids: - self._normalize_trace_replay_params( - sampling_params, - length_from_prompt_token_ids_or_embeds( - prompt_token_ids, prompt_embeds - ), - ) + self._normalize_trace_replay_params(sampling_params, prompt_len) else: pooling_params = params.clone()