From b64a2708b06b6329420702bffce994543c1ec6d2 Mon Sep 17 00:00:00 2001 From: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Date: Tue, 11 Aug 2026 10:03:50 -0400 Subject: [PATCH] Bound generation inputs before expensive work (#51447) Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- .../server/src/routes/openai/utils/types.rs | 49 +++- tests/test_request_input_bounds.py | 273 ++++++++++++++++++ .../openai/chat_completion/protocol.py | 5 +- .../entrypoints/openai/completion/protocol.py | 3 +- vllm/entrypoints/openai/engine/protocol.py | 7 +- vllm/entrypoints/openai/responses/protocol.py | 4 +- vllm/envs.py | 14 + vllm/sampling_params.py | 46 ++- vllm/tokenizers/deepseek_v32_encoding.py | 16 +- vllm/tokenizers/deepseek_v4_encoding.py | 17 +- vllm/v1/worker/gpu/sample/bad_words.py | 24 +- 11 files changed, 421 insertions(+), 37 deletions(-) create mode 100644 tests/test_request_input_bounds.py diff --git a/rust/src/server/src/routes/openai/utils/types.rs b/rust/src/server/src/routes/openai/utils/types.rs index ffd665dbd4..d87dcd676d 100644 --- a/rust/src/server/src/routes/openai/utils/types.rs +++ b/rust/src/server/src/routes/openai/utils/types.rs @@ -54,9 +54,23 @@ impl StringOrArray { } } -/// Validates stop sequences (non-empty strings) +fn max_stop_strings() -> usize { + std::env::var("VLLM_MAX_STOP_STRINGS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(4) +} + +/// Validates stop sequences (at most the configured number of non-empty strings). pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> { - if stop.as_slice().iter().any(|s| s.is_empty()) { + let stop = stop.as_slice(); + let max_stop_strings = max_stop_strings(); + if stop.len() > max_stop_strings { + let mut error = validator::ValidationError::new("too_many_stop_strings"); + error.code = format!("stop strings must contain at most {max_stop_strings} items").into(); + return Err(error); + } + if stop.iter().any(|s| s.is_empty()) { return Err(validator::ValidationError::new( "stop strings cannot be empty", )); @@ -64,6 +78,37 @@ pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationEr Ok(()) } +#[cfg(test)] +mod tests { + use super::{StringOrArray, validate_stop}; + + #[test] + fn validate_stop_accepts_four_stop_strings() { + let stop = StringOrArray::Array( + ["one", "two", "three", "four"].into_iter().map(str::to_string).collect(), + ); + + validate_stop(&stop).expect("four stop strings should be accepted"); + } + + #[test] + fn validate_stop_rejects_more_than_four_stop_strings() { + let stop = StringOrArray::Array( + ["one", "two", "three", "four", "five"] + .into_iter() + .map(str::to_string) + .collect(), + ); + + let error = validate_stop(&stop).expect_err("five stop strings should be rejected"); + + assert_eq!( + error.code.as_ref(), + "stop strings must contain at most 4 items" + ); + } +} + // ============================================================================ // Validation helpers // ============================================================================ diff --git a/tests/test_request_input_bounds.py b/tests/test_request_input_bounds.py new file mode 100644 index 0000000000..ecab38c922 --- /dev/null +++ b/tests/test_request_input_bounds.py @@ -0,0 +1,273 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Bounds on request-controlled inputs that would otherwise amplify work. + +Consolidates the regression tests for the request-input amplification fixes: +stop-string caps, bad-words dedup/tokenization limit, stop-token-id dedup, +beam-width/sequence caps, and the DeepSeek history-scan bound. +""" + +import os +import subprocess +import sys +from collections.abc import Callable +from typing import Protocol + +import pytest +from pydantic import ValidationError + +import vllm.envs as envs +from vllm import SamplingParams +from vllm.entrypoints.openai.chat_completion.protocol import ( + BatchChatCompletionRequest, + ChatCompletionRequest, +) +from vllm.entrypoints.openai.completion.protocol import CompletionRequest +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest +from vllm.exceptions import VLLMValidationError +from vllm.sampling_params import BeamSearchParams +from vllm.tokenizers import deepseek_v4_encoding, deepseek_v32_encoding + +pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup] + + +# --- Stop strings: public requests cap the number of stop strings --------- + + +class _StopRequest(Protocol): + stop: str | list[str] | None + + +def _completion_request(stop: list[str]) -> _StopRequest: + return CompletionRequest(model="test-model", prompt="hello", stop=stop) + + +def _chat_request(stop: list[str]) -> _StopRequest: + return ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "hello"}], + stop=stop, + ) + + +def _batch_chat_request(stop: list[str]) -> _StopRequest: + return BatchChatCompletionRequest( + model="test-model", + messages=[[{"role": "user", "content": "hello"}]], + stop=stop, + ) + + +def _responses_request(stop: list[str]) -> _StopRequest: + return ResponsesRequest(model="test-model", input="hello", stop=stop) + + +REQUEST_BUILDERS: list[Callable[[list[str]], _StopRequest]] = [ + _completion_request, + _chat_request, + _batch_chat_request, + _responses_request, +] + + +@pytest.mark.parametrize("build_request", REQUEST_BUILDERS) +def test_public_requests_accept_four_stop_strings( + build_request: Callable[[list[str]], _StopRequest], +): + stop = ["one", "two", "three", "four"] + + request = build_request(stop) + + assert request.stop == stop + + +@pytest.mark.parametrize("build_request", REQUEST_BUILDERS) +def test_public_requests_reject_more_than_four_stop_strings( + build_request: Callable[[list[str]], _StopRequest], +): + with pytest.raises(ValidationError, match="at most 4"): + build_request(["one", "two", "three", "four", "five"]) + + +def test_stop_string_limit_can_be_overridden(): + env = os.environ.copy() + env["VLLM_MAX_STOP_STRINGS"] = "1" + code = """ +from pydantic import ValidationError +from vllm.entrypoints.openai.completion.protocol import CompletionRequest + +try: + CompletionRequest( + model="test-model", + prompt="hello", + stop=["one", "two"], + ) +except ValidationError as error: + assert "at most 1" in str(error) +else: + raise AssertionError("configured stop-string limit was not enforced") +""" + + subprocess.run([sys.executable, "-c", code], check=True, env=env) + + +# --- Stop token ids: duplicates are deduplicated in order ------------------ + + +def test_duplicate_stop_token_ids_are_deduplicated_in_order(): + params = SamplingParams(stop_token_ids=[42, 7, 42, 9, 7]) + + assert params.stop_token_ids == [42, 7, 9] + assert params.all_stop_token_ids == {7, 9, 42} + + +# --- Bad words: dedup, and the tokenization pass is bounded ---------------- + + +class MockTokenizer: + max_token_id = 1024 + + def __init__(self): + self.calls = 0 + + def encode(self, text: str, add_special_tokens: bool = False) -> list[int]: + self.calls += 1 + return [2] if text.startswith(" ") else [1] + + +def test_duplicate_bad_words_are_deduplicated_in_order(): + params = SamplingParams(bad_words=["bad", "worse", "bad", "worst"]) + + assert params.bad_words == ["bad", "worse", "worst"] + + +def test_bad_word_tokenization_stops_at_worker_limit(): + params = SamplingParams(bad_words=[f"word-{i}" for i in range(65)]) + tokenizer = MockTokenizer() + + with pytest.raises(VLLMValidationError, match="Too many bad words"): + params.update_from_tokenizer(tokenizer) + + assert tokenizer.calls == 129 + + +def test_bad_word_tokenization_limit_can_be_overridden(monkeypatch): + monkeypatch.setenv("VLLM_MAX_NUM_BAD_WORDS", "2") + params = SamplingParams(bad_words=["bad", "worse"]) + tokenizer = MockTokenizer() + + with pytest.raises(VLLMValidationError, match="The max number is 2"): + params.update_from_tokenizer(tokenizer) + + assert tokenizer.calls == 3 + + +# --- Beam search: beam width / n honor the sequence cap -------------------- + + +def _set_max_n(monkeypatch: pytest.MonkeyPatch, value: int) -> None: + monkeypatch.setenv("VLLM_MAX_N_SEQUENCES", str(value)) + if hasattr(envs.__getattr__, "cache_clear"): + envs.__getattr__.cache_clear() + + +def test_direct_beam_width_rejects_values_over_sequence_cap( + monkeypatch: pytest.MonkeyPatch, +): + _set_max_n(monkeypatch, 4) + + with pytest.raises(VLLMValidationError, match="beam_width must be at most 4"): + BeamSearchParams(beam_width=5, max_tokens=1) + + +def test_chat_beam_conversion_rejects_n_before_stream_state_allocation( + monkeypatch: pytest.MonkeyPatch, +): + _set_max_n(monkeypatch, 4) + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "test"}], + n=5, + stream=True, + use_beam_search=True, + max_tokens=1, + ) + + with pytest.raises(VLLMValidationError, match="beam_width must be at most 4"): + request.to_beam_search_params(max_tokens=1, default_sampling_params={}) + + +def test_chat_beam_conversion_accepts_n_at_sequence_cap( + monkeypatch: pytest.MonkeyPatch, +): + _set_max_n(monkeypatch, 4) + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "test"}], + n=4, + stream=True, + use_beam_search=True, + max_tokens=1, + ) + + params = request.to_beam_search_params(max_tokens=1, default_sampling_params={}) + + assert params.beam_width == 4 + + +# --- DeepSeek encoders: the last-user scan runs once per conversation ------ + + +ENCODING_MODULES = [deepseek_v32_encoding, deepseek_v4_encoding] + + +@pytest.mark.parametrize( + "encoding_module", + ENCODING_MODULES, + ids=["deepseek_v32", "deepseek_v4"], +) +def test_encode_messages_scans_last_user_once_per_conversation( + monkeypatch: pytest.MonkeyPatch, + encoding_module, +): + calls = 0 + original_find_last_user_index = encoding_module.find_last_user_index + + def counted_find_last_user_index(messages): + nonlocal calls + calls += 1 + return original_find_last_user_index(messages) + + monkeypatch.setattr( + encoding_module, + "find_last_user_index", + counted_find_last_user_index, + ) + + messages = [{"role": "user", "content": "Hello"}] + messages.extend({"role": "assistant", "content": "Hi"} for _ in range(8)) + + encoding_module.encode_messages(messages, thinking_mode="chat") + + assert calls == 1 + + +@pytest.mark.parametrize( + "encoding_module", + ENCODING_MODULES, + ids=["deepseek_v32", "deepseek_v4"], +) +def test_encode_messages_preserves_small_chat_prompt(encoding_module): + prompt = encoding_module.encode_messages( + [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi"}, + {"role": "assistant", "content": "Again"}, + ], + thinking_mode="chat", + ) + + assert prompt == ( + "<|begin▁of▁sentence|><|User|>Hello<|Assistant|>" + "Hi<|end▁of▁sentence|>Again<|end▁of▁sentence|>" + ) diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index eda0f5a3ba..13b8133172 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -30,6 +30,7 @@ from vllm.entrypoints.openai.engine.protocol import ( FunctionDefinition, OpenAIBaseModel, PerRequestTimingMetrics, + StopParam, StreamOptions, ToolCall, UsageInfo, @@ -228,7 +229,7 @@ class ChatCompletionRequest(OpenAIBaseModel): presence_penalty: float | None = 0.0 response_format: AnyResponseFormat | None = None seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX) - stop: str | list[str] | None = [] + stop: StopParam = [] stream: bool | None = False stream_options: StreamOptions | None = None temperature: float | None = None @@ -1077,7 +1078,7 @@ class BatchChatCompletionRequest(OpenAIBaseModel): presence_penalty: float | None = 0.0 response_format: Any | None = None seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX) - stop: str | list[str] | None = Field(default_factory=list) + stop: StopParam = Field(default_factory=list) temperature: float | None = None top_p: float | None = None user: str | None = None diff --git a/vllm/entrypoints/openai/completion/protocol.py b/vllm/entrypoints/openai/completion/protocol.py index 516a2c371f..54de327c64 100644 --- a/vllm/entrypoints/openai/completion/protocol.py +++ b/vllm/entrypoints/openai/completion/protocol.py @@ -14,6 +14,7 @@ from vllm.entrypoints.openai.engine.protocol import ( AnyResponseFormat, OpenAIBaseModel, PerRequestTimingMetrics, + StopParam, StreamOptions, UsageInfo, structured_outputs_from_response_format, @@ -61,7 +62,7 @@ class CompletionRequest(OpenAIBaseModel): n: int = 1 presence_penalty: float | None = 0.0 seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX) - stop: str | list[str] | None = [] + stop: StopParam = [] stream: bool | None = False stream_options: StreamOptions | None = None suffix: str | None = None diff --git a/vllm/entrypoints/openai/engine/protocol.py b/vllm/entrypoints/openai/engine/protocol.py index 805c639d7d..4d07c29408 100644 --- a/vllm/entrypoints/openai/engine/protocol.py +++ b/vllm/entrypoints/openai/engine/protocol.py @@ -6,7 +6,7 @@ import json import time from http import HTTPStatus -from typing import Any, ClassVar, Literal, TypeAlias +from typing import Annotated, Any, ClassVar, Literal, TypeAlias import regex as re from pydantic import ( @@ -17,6 +17,7 @@ from pydantic import ( model_validator, ) +import vllm.envs as envs from vllm.config.utils import replace from vllm.entrypoints.chat_utils import make_tool_call_id from vllm.exceptions import VLLMServerError, VLLMValidationError @@ -27,6 +28,10 @@ from vllm.utils.import_utils import resolve_obj_by_qualname logger = init_logger(__name__) +StopParam: TypeAlias = ( + str | Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None +) + class OpenAIBaseModel(BaseModel): # OpenAI API does allow extra fields diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index ec8d25d10a..7ad9f57116 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -60,7 +60,7 @@ from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, ) -from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel +from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, StopParam from vllm.exceptions import VLLMValidationError from vllm.logger import init_logger from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs @@ -281,7 +281,7 @@ class ResponsesRequest(OpenAIBaseModel): repetition_penalty: float | None = None seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX) - stop: str | list[str] | None = [] + stop: StopParam = [] ignore_eos: bool = False vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field( default=None, diff --git a/vllm/envs.py b/vllm/envs.py index 1b941db970..819c6d30cd 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -108,6 +108,9 @@ if TYPE_CHECKING: VLLM_HTTP_TIMEOUT_KEEP_ALIVE: int = 5 # seconds VLLM_MAX_N_SEQUENCES: int = 16384 VLLM_MAX_COMPLETION_PROMPTS: int = 1024 + VLLM_MAX_STOP_STRINGS: int = 4 + VLLM_MAX_NUM_BAD_WORDS: int = 128 + VLLM_MAX_BAD_WORDS_TOTAL_TOKENS: int = 1024 VLLM_PLUGINS: list[str] | None = None VLLM_LORA_RESOLVER_CACHE_DIR: str | None = None VLLM_LORA_RESOLVER_HF_REPO_LIST: str | None = None @@ -1114,6 +1117,17 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_MAX_COMPLETION_PROMPTS": lambda: int( os.environ.get("VLLM_MAX_COMPLETION_PROMPTS", "1024") ), + # Maximum number of stop strings allowed in a single request. + "VLLM_MAX_STOP_STRINGS": lambda: int(os.environ.get("VLLM_MAX_STOP_STRINGS", "4")), + # Maximum number of bad-word token sequences generated per request. + "VLLM_MAX_NUM_BAD_WORDS": lambda: int( + os.environ.get("VLLM_MAX_NUM_BAD_WORDS", "128") + ), + # Maximum total number of bad-word tokens (summed across all bad words) + # allowed per request. Bounds the per-request GPU buffer width. + "VLLM_MAX_BAD_WORDS_TOTAL_TOKENS": lambda: int( + os.environ.get("VLLM_MAX_BAD_WORDS_TOTAL_TOKENS", "1024") + ), # a list of plugin names to load, separated by commas. # if this is not set, it means all plugins will be loaded # if this is set to an empty string, no plugins will be loaded diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 5dedbde372..8964f4047e 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -32,6 +32,22 @@ MAX_LOGPROB_TOKEN_IDS = 128 the per-request row width allocated by the sampler's `LogprobTokenIdsState`.""" +def _verify_num_sequences(value: int, parameter_name: str) -> None: + if not isinstance(value, int): + raise VLLMValidationError( + f"{parameter_name} must be an int, but is of type {type(value)}" + ) + if value < 1: + raise VLLMValidationError(f"{parameter_name} must be at least 1, got {value}.") + max_n = envs.VLLM_MAX_N_SEQUENCES + if value > max_n: + raise VLLMValidationError( + f"{parameter_name} must be at most {max_n}, got {value}. " + "To increase this limit, set the VLLM_MAX_N_SEQUENCES " + "environment variable." + ) + + def validate_thinking_token_budget(value: int | float | bool | None) -> int | None: """Validate ``thinking_token_budget``; return ``None`` if unset.""" if value is None: @@ -479,9 +495,13 @@ class SamplingParams( if self.stop_token_ids is None: self.stop_token_ids = [] + else: + self.stop_token_ids = list(dict.fromkeys(self.stop_token_ids)) if self.bad_words is None: self.bad_words = [] + else: + self.bad_words = list(dict.fromkeys(self.bad_words)) if self.logprobs is True: self.logprobs = 1 @@ -513,19 +533,7 @@ class SamplingParams( self.skip_reading_prefix_cache = self.prompt_logprobs is not None def _verify_args(self) -> None: - if not isinstance(self.n, int): - raise VLLMValidationError( - f"n must be an int, but is of type {type(self.n)}" - ) - if self.n < 1: - raise VLLMValidationError(f"n must be at least 1, got {self.n}.") - max_n = envs.VLLM_MAX_N_SEQUENCES - if self.n > max_n: - raise VLLMValidationError( - f"n must be at most {max_n}, got {self.n}. " - "To increase this limit, set the VLLM_MAX_N_SEQUENCES " - "environment variable." - ) + _verify_num_sequences(self.n, "n") if not -2.0 <= self.presence_penalty <= 2.0: raise VLLMValidationError( f"presence_penalty must be in [-2, 2], got {self.presence_penalty}." @@ -677,6 +685,7 @@ class SamplingParams( if not self.bad_words: return self._bad_words_token_ids = [] + max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS for bad_word in self.bad_words: # To prohibit words both at the beginning # and in the middle of text @@ -696,6 +705,14 @@ class SamplingParams( and len(prompt_token_ids) == len(self._bad_words_token_ids[-1]) ): self._bad_words_token_ids.append(prompt_token_ids) + if len(self._bad_words_token_ids) > max_num_bad_words: + raise VLLMValidationError( + f"Too many bad words after tokenization: " + f"{len(self._bad_words_token_ids)}. " + f"The max number is {max_num_bad_words}.", + parameter="bad_words", + value=self.bad_words, + ) invalid_token_ids = [ token_id @@ -1147,3 +1164,6 @@ class BeamSearchParams( length_penalty: float = 1.0 include_stop_str_in_output: bool = False structured_outputs: StructuredOutputsParams | None = None + + def __post_init__(self) -> None: + _verify_num_sequences(self.beam_width, "beam_width") diff --git a/vllm/tokenizers/deepseek_v32_encoding.py b/vllm/tokenizers/deepseek_v32_encoding.py index b02449020e..dfe4e478f0 100644 --- a/vllm/tokenizers/deepseek_v32_encoding.py +++ b/vllm/tokenizers/deepseek_v32_encoding.py @@ -119,7 +119,10 @@ def find_last_user_index(messages: list[dict[str, Any]]) -> int: def render_message( - index: int, messages: list[dict[str, Any]], thinking_mode: str + index: int, + messages: list[dict[str, Any]], + thinking_mode: str, + last_user_idx: int | None = None, ) -> str: if not (0 <= index < len(messages)): raise ValueError( @@ -130,7 +133,9 @@ def render_message( prompt = "" msg = messages[index] - last_user_idx = find_last_user_index(messages) + last_user_idx = ( + find_last_user_index(messages) if last_user_idx is None else last_user_idx + ) role = msg.get("role") content = msg.get("content") @@ -294,9 +299,14 @@ def encode_messages( if thinking_mode == "thinking" and drop_thinking: full_messages = drop_thinking_messages(full_messages) + last_user_idx = find_last_user_index(full_messages) + for idx in range(len(messages)): prompt += render_message( - idx + len(context), full_messages, thinking_mode=thinking_mode + idx + len(context), + full_messages, + thinking_mode=thinking_mode, + last_user_idx=last_user_idx, ) return prompt diff --git a/vllm/tokenizers/deepseek_v4_encoding.py b/vllm/tokenizers/deepseek_v4_encoding.py index c7c74a396f..461f7f391d 100644 --- a/vllm/tokenizers/deepseek_v4_encoding.py +++ b/vllm/tokenizers/deepseek_v4_encoding.py @@ -199,7 +199,14 @@ def find_last_user_index(messages: List[Dict[str, Any]]) -> int: # Message Rendering # ============================================================ -def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: str, drop_thinking: bool = True, reasoning_effort: Optional[str] = None) -> str: +def render_message( + index: int, + messages: List[Dict[str, Any]], + thinking_mode: str, + drop_thinking: bool = True, + reasoning_effort: Optional[str] = None, + last_user_idx: Optional[int] = None, +) -> str: """ Render a single message at the given index into its encoded string form. @@ -213,6 +220,7 @@ def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: st drop_thinking: Whether to drop reasoning content from earlier turns. reasoning_effort: Reasoning effort level, one of "low", "high", "max". None is treated as "low". + last_user_idx: Cached index of the last user/developer message. Returns: Encoded string for this message. @@ -222,7 +230,9 @@ def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: st prompt = "" msg = messages[index] - last_user_idx = find_last_user_index(messages) + last_user_idx = ( + find_last_user_index(messages) if last_user_idx is None else last_user_idx + ) role = msg.get("role") content = msg.get("content") @@ -544,6 +554,8 @@ def encode_messages( num_to_render = len(messages) context_len = len(context) + last_user_idx = find_last_user_index(full_messages) + for idx in range(num_to_render): prompt += render_message( idx + context_len, @@ -551,6 +563,7 @@ def encode_messages( thinking_mode=thinking_mode, drop_thinking=effective_drop_thinking, reasoning_effort=reasoning_effort, + last_user_idx=last_user_idx, ) return prompt diff --git a/vllm/v1/worker/gpu/sample/bad_words.py b/vllm/v1/worker/gpu/sample/bad_words.py index b5517dee1b..768ff30a0f 100644 --- a/vllm/v1/worker/gpu/sample/bad_words.py +++ b/vllm/v1/worker/gpu/sample/bad_words.py @@ -3,14 +3,12 @@ import numpy as np import torch +import vllm.envs as envs from vllm.sampling_params import SamplingParams from vllm.triton_utils import tl, triton from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor from vllm.v1.worker.gpu.states import RequestState -MAX_BAD_WORDS_TOTAL_TOKENS = 1024 # Max total tokens for all bad words per request -MAX_NUM_BAD_WORDS = 128 # Max number of bad words per request - class BadWordsState: def __init__(self, req_states: RequestState): @@ -18,15 +16,17 @@ class BadWordsState: self.max_num_reqs = req_states.max_num_reqs self.device = req_states.device - # flattened bad word tokens: [max_num_reqs, MAX_BAD_WORDS_TOTAL_TOKENS] + max_total_tokens = envs.VLLM_MAX_BAD_WORDS_TOTAL_TOKENS + max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS + # flattened bad word tokens: [max_num_reqs, VLLM_MAX_BAD_WORDS_TOTAL_TOKENS] self.bad_word_token_ids = StagedWriteTensor( - (self.max_num_reqs, MAX_BAD_WORDS_TOTAL_TOKENS), + (self.max_num_reqs, max_total_tokens), dtype=torch.int32, device=self.device, ) - # cumulative offsets of bad words: [max_num_reqs, MAX_NUM_BAD_WORDS + 1] + # cumulative offsets of bad words: [max_num_reqs, VLLM_MAX_NUM_BAD_WORDS + 1] self.bad_word_offsets = StagedWriteTensor( - (self.max_num_reqs, MAX_NUM_BAD_WORDS + 1), + (self.max_num_reqs, max_num_bad_words + 1), dtype=torch.int32, device=self.device, ) @@ -40,10 +40,11 @@ class BadWordsState: return num_bad_words = len(bad_words_token_ids) - if num_bad_words > MAX_NUM_BAD_WORDS: + max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS + if num_bad_words > max_num_bad_words: raise ValueError( f"Too many bad words: {num_bad_words}. " - f"The max number is {MAX_NUM_BAD_WORDS}." + f"The max number is {max_num_bad_words}." ) # Flatten bad words and compute offsets @@ -53,10 +54,11 @@ class BadWordsState: flattened_tokens.extend(bad_word) offsets.append(len(flattened_tokens)) - if len(flattened_tokens) > MAX_BAD_WORDS_TOTAL_TOKENS: + max_total_tokens = envs.VLLM_MAX_BAD_WORDS_TOTAL_TOKENS + if len(flattened_tokens) > max_total_tokens: raise ValueError( f"Too many total bad word tokens: {len(flattened_tokens)}. " - f"The max is {MAX_BAD_WORDS_TOTAL_TOKENS}." + f"The max is {max_total_tokens}." ) # Stage writes