Bound generation inputs before expensive work (#51447)
Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com> Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Bourtoule
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
5b184f775a
commit
b64a2708b0
@@ -54,9 +54,23 @@ impl StringOrArray {
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates stop sequences (non-empty strings)
|
||||
fn max_stop_strings() -> usize {
|
||||
std::env::var("VLLM_MAX_STOP_STRINGS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse().ok())
|
||||
.unwrap_or(4)
|
||||
}
|
||||
|
||||
/// Validates stop sequences (at most the configured number of non-empty strings).
|
||||
pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> {
|
||||
if stop.as_slice().iter().any(|s| s.is_empty()) {
|
||||
let stop = stop.as_slice();
|
||||
let max_stop_strings = max_stop_strings();
|
||||
if stop.len() > max_stop_strings {
|
||||
let mut error = validator::ValidationError::new("too_many_stop_strings");
|
||||
error.code = format!("stop strings must contain at most {max_stop_strings} items").into();
|
||||
return Err(error);
|
||||
}
|
||||
if stop.iter().any(|s| s.is_empty()) {
|
||||
return Err(validator::ValidationError::new(
|
||||
"stop strings cannot be empty",
|
||||
));
|
||||
@@ -64,6 +78,37 @@ pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationEr
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{StringOrArray, validate_stop};
|
||||
|
||||
#[test]
|
||||
fn validate_stop_accepts_four_stop_strings() {
|
||||
let stop = StringOrArray::Array(
|
||||
["one", "two", "three", "four"].into_iter().map(str::to_string).collect(),
|
||||
);
|
||||
|
||||
validate_stop(&stop).expect("four stop strings should be accepted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_stop_rejects_more_than_four_stop_strings() {
|
||||
let stop = StringOrArray::Array(
|
||||
["one", "two", "three", "four", "five"]
|
||||
.into_iter()
|
||||
.map(str::to_string)
|
||||
.collect(),
|
||||
);
|
||||
|
||||
let error = validate_stop(&stop).expect_err("five stop strings should be rejected");
|
||||
|
||||
assert_eq!(
|
||||
error.code.as_ref(),
|
||||
"stop strings must contain at most 4 items"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Validation helpers
|
||||
// ============================================================================
|
||||
|
||||
@@ -0,0 +1,273 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Bounds on request-controlled inputs that would otherwise amplify work.
|
||||
|
||||
Consolidates the regression tests for the request-input amplification fixes:
|
||||
stop-string caps, bad-words dedup/tokenization limit, stop-token-id dedup,
|
||||
beam-width/sequence caps, and the DeepSeek history-scan bound.
|
||||
"""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
from typing import Protocol
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import SamplingParams
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
BatchChatCompletionRequest,
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.sampling_params import BeamSearchParams
|
||||
from vllm.tokenizers import deepseek_v4_encoding, deepseek_v32_encoding
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
|
||||
# --- Stop strings: public requests cap the number of stop strings ---------
|
||||
|
||||
|
||||
class _StopRequest(Protocol):
|
||||
stop: str | list[str] | None
|
||||
|
||||
|
||||
def _completion_request(stop: list[str]) -> _StopRequest:
|
||||
return CompletionRequest(model="test-model", prompt="hello", stop=stop)
|
||||
|
||||
|
||||
def _chat_request(stop: list[str]) -> _StopRequest:
|
||||
return ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
stop=stop,
|
||||
)
|
||||
|
||||
|
||||
def _batch_chat_request(stop: list[str]) -> _StopRequest:
|
||||
return BatchChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[[{"role": "user", "content": "hello"}]],
|
||||
stop=stop,
|
||||
)
|
||||
|
||||
|
||||
def _responses_request(stop: list[str]) -> _StopRequest:
|
||||
return ResponsesRequest(model="test-model", input="hello", stop=stop)
|
||||
|
||||
|
||||
REQUEST_BUILDERS: list[Callable[[list[str]], _StopRequest]] = [
|
||||
_completion_request,
|
||||
_chat_request,
|
||||
_batch_chat_request,
|
||||
_responses_request,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("build_request", REQUEST_BUILDERS)
|
||||
def test_public_requests_accept_four_stop_strings(
|
||||
build_request: Callable[[list[str]], _StopRequest],
|
||||
):
|
||||
stop = ["one", "two", "three", "four"]
|
||||
|
||||
request = build_request(stop)
|
||||
|
||||
assert request.stop == stop
|
||||
|
||||
|
||||
@pytest.mark.parametrize("build_request", REQUEST_BUILDERS)
|
||||
def test_public_requests_reject_more_than_four_stop_strings(
|
||||
build_request: Callable[[list[str]], _StopRequest],
|
||||
):
|
||||
with pytest.raises(ValidationError, match="at most 4"):
|
||||
build_request(["one", "two", "three", "four", "five"])
|
||||
|
||||
|
||||
def test_stop_string_limit_can_be_overridden():
|
||||
env = os.environ.copy()
|
||||
env["VLLM_MAX_STOP_STRINGS"] = "1"
|
||||
code = """
|
||||
from pydantic import ValidationError
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
|
||||
try:
|
||||
CompletionRequest(
|
||||
model="test-model",
|
||||
prompt="hello",
|
||||
stop=["one", "two"],
|
||||
)
|
||||
except ValidationError as error:
|
||||
assert "at most 1" in str(error)
|
||||
else:
|
||||
raise AssertionError("configured stop-string limit was not enforced")
|
||||
"""
|
||||
|
||||
subprocess.run([sys.executable, "-c", code], check=True, env=env)
|
||||
|
||||
|
||||
# --- Stop token ids: duplicates are deduplicated in order ------------------
|
||||
|
||||
|
||||
def test_duplicate_stop_token_ids_are_deduplicated_in_order():
|
||||
params = SamplingParams(stop_token_ids=[42, 7, 42, 9, 7])
|
||||
|
||||
assert params.stop_token_ids == [42, 7, 9]
|
||||
assert params.all_stop_token_ids == {7, 9, 42}
|
||||
|
||||
|
||||
# --- Bad words: dedup, and the tokenization pass is bounded ----------------
|
||||
|
||||
|
||||
class MockTokenizer:
|
||||
max_token_id = 1024
|
||||
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
|
||||
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
|
||||
self.calls += 1
|
||||
return [2] if text.startswith(" ") else [1]
|
||||
|
||||
|
||||
def test_duplicate_bad_words_are_deduplicated_in_order():
|
||||
params = SamplingParams(bad_words=["bad", "worse", "bad", "worst"])
|
||||
|
||||
assert params.bad_words == ["bad", "worse", "worst"]
|
||||
|
||||
|
||||
def test_bad_word_tokenization_stops_at_worker_limit():
|
||||
params = SamplingParams(bad_words=[f"word-{i}" for i in range(65)])
|
||||
tokenizer = MockTokenizer()
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="Too many bad words"):
|
||||
params.update_from_tokenizer(tokenizer)
|
||||
|
||||
assert tokenizer.calls == 129
|
||||
|
||||
|
||||
def test_bad_word_tokenization_limit_can_be_overridden(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_MAX_NUM_BAD_WORDS", "2")
|
||||
params = SamplingParams(bad_words=["bad", "worse"])
|
||||
tokenizer = MockTokenizer()
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="The max number is 2"):
|
||||
params.update_from_tokenizer(tokenizer)
|
||||
|
||||
assert tokenizer.calls == 3
|
||||
|
||||
|
||||
# --- Beam search: beam width / n honor the sequence cap --------------------
|
||||
|
||||
|
||||
def _set_max_n(monkeypatch: pytest.MonkeyPatch, value: int) -> None:
|
||||
monkeypatch.setenv("VLLM_MAX_N_SEQUENCES", str(value))
|
||||
if hasattr(envs.__getattr__, "cache_clear"):
|
||||
envs.__getattr__.cache_clear()
|
||||
|
||||
|
||||
def test_direct_beam_width_rejects_values_over_sequence_cap(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_set_max_n(monkeypatch, 4)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="beam_width must be at most 4"):
|
||||
BeamSearchParams(beam_width=5, max_tokens=1)
|
||||
|
||||
|
||||
def test_chat_beam_conversion_rejects_n_before_stream_state_allocation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_set_max_n(monkeypatch, 4)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
n=5,
|
||||
stream=True,
|
||||
use_beam_search=True,
|
||||
max_tokens=1,
|
||||
)
|
||||
|
||||
with pytest.raises(VLLMValidationError, match="beam_width must be at most 4"):
|
||||
request.to_beam_search_params(max_tokens=1, default_sampling_params={})
|
||||
|
||||
|
||||
def test_chat_beam_conversion_accepts_n_at_sequence_cap(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_set_max_n(monkeypatch, 4)
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
n=4,
|
||||
stream=True,
|
||||
use_beam_search=True,
|
||||
max_tokens=1,
|
||||
)
|
||||
|
||||
params = request.to_beam_search_params(max_tokens=1, default_sampling_params={})
|
||||
|
||||
assert params.beam_width == 4
|
||||
|
||||
|
||||
# --- DeepSeek encoders: the last-user scan runs once per conversation ------
|
||||
|
||||
|
||||
ENCODING_MODULES = [deepseek_v32_encoding, deepseek_v4_encoding]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"encoding_module",
|
||||
ENCODING_MODULES,
|
||||
ids=["deepseek_v32", "deepseek_v4"],
|
||||
)
|
||||
def test_encode_messages_scans_last_user_once_per_conversation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
encoding_module,
|
||||
):
|
||||
calls = 0
|
||||
original_find_last_user_index = encoding_module.find_last_user_index
|
||||
|
||||
def counted_find_last_user_index(messages):
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
return original_find_last_user_index(messages)
|
||||
|
||||
monkeypatch.setattr(
|
||||
encoding_module,
|
||||
"find_last_user_index",
|
||||
counted_find_last_user_index,
|
||||
)
|
||||
|
||||
messages = [{"role": "user", "content": "Hello"}]
|
||||
messages.extend({"role": "assistant", "content": "Hi"} for _ in range(8))
|
||||
|
||||
encoding_module.encode_messages(messages, thinking_mode="chat")
|
||||
|
||||
assert calls == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"encoding_module",
|
||||
ENCODING_MODULES,
|
||||
ids=["deepseek_v32", "deepseek_v4"],
|
||||
)
|
||||
def test_encode_messages_preserves_small_chat_prompt(encoding_module):
|
||||
prompt = encoding_module.encode_messages(
|
||||
[
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi"},
|
||||
{"role": "assistant", "content": "Again"},
|
||||
],
|
||||
thinking_mode="chat",
|
||||
)
|
||||
|
||||
assert prompt == (
|
||||
"<|begin▁of▁sentence|><|User|>Hello<|Assistant|></think>"
|
||||
"Hi<|end▁of▁sentence|>Again<|end▁of▁sentence|>"
|
||||
)
|
||||
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
FunctionDefinition,
|
||||
OpenAIBaseModel,
|
||||
PerRequestTimingMetrics,
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
ToolCall,
|
||||
UsageInfo,
|
||||
@@ -228,7 +229,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
presence_penalty: float | None = 0.0
|
||||
response_format: AnyResponseFormat | None = None
|
||||
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
|
||||
stop: str | list[str] | None = []
|
||||
stop: StopParam = []
|
||||
stream: bool | None = False
|
||||
stream_options: StreamOptions | None = None
|
||||
temperature: float | None = None
|
||||
@@ -1077,7 +1078,7 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
|
||||
presence_penalty: float | None = 0.0
|
||||
response_format: Any | None = None
|
||||
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
|
||||
stop: str | list[str] | None = Field(default_factory=list)
|
||||
stop: StopParam = Field(default_factory=list)
|
||||
temperature: float | None = None
|
||||
top_p: float | None = None
|
||||
user: str | None = None
|
||||
|
||||
@@ -14,6 +14,7 @@ from vllm.entrypoints.openai.engine.protocol import (
|
||||
AnyResponseFormat,
|
||||
OpenAIBaseModel,
|
||||
PerRequestTimingMetrics,
|
||||
StopParam,
|
||||
StreamOptions,
|
||||
UsageInfo,
|
||||
structured_outputs_from_response_format,
|
||||
@@ -61,7 +62,7 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
n: int = 1
|
||||
presence_penalty: float | None = 0.0
|
||||
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
|
||||
stop: str | list[str] | None = []
|
||||
stop: StopParam = []
|
||||
stream: bool | None = False
|
||||
stream_options: StreamOptions | None = None
|
||||
suffix: str | None = None
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
import json
|
||||
import time
|
||||
from http import HTTPStatus
|
||||
from typing import Any, ClassVar, Literal, TypeAlias
|
||||
from typing import Annotated, Any, ClassVar, Literal, TypeAlias
|
||||
|
||||
import regex as re
|
||||
from pydantic import (
|
||||
@@ -17,6 +17,7 @@ from pydantic import (
|
||||
model_validator,
|
||||
)
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.utils import replace
|
||||
from vllm.entrypoints.chat_utils import make_tool_call_id
|
||||
from vllm.exceptions import VLLMServerError, VLLMValidationError
|
||||
@@ -27,6 +28,10 @@ from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
StopParam: TypeAlias = (
|
||||
str | Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
|
||||
)
|
||||
|
||||
|
||||
class OpenAIBaseModel(BaseModel):
|
||||
# OpenAI API does allow extra fields
|
||||
|
||||
@@ -60,7 +60,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel
|
||||
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, StopParam
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
@@ -281,7 +281,7 @@ class ResponsesRequest(OpenAIBaseModel):
|
||||
|
||||
repetition_penalty: float | None = None
|
||||
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
|
||||
stop: str | list[str] | None = []
|
||||
stop: StopParam = []
|
||||
ignore_eos: bool = False
|
||||
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
|
||||
default=None,
|
||||
|
||||
@@ -108,6 +108,9 @@ if TYPE_CHECKING:
|
||||
VLLM_HTTP_TIMEOUT_KEEP_ALIVE: int = 5 # seconds
|
||||
VLLM_MAX_N_SEQUENCES: int = 16384
|
||||
VLLM_MAX_COMPLETION_PROMPTS: int = 1024
|
||||
VLLM_MAX_STOP_STRINGS: int = 4
|
||||
VLLM_MAX_NUM_BAD_WORDS: int = 128
|
||||
VLLM_MAX_BAD_WORDS_TOTAL_TOKENS: int = 1024
|
||||
VLLM_PLUGINS: list[str] | None = None
|
||||
VLLM_LORA_RESOLVER_CACHE_DIR: str | None = None
|
||||
VLLM_LORA_RESOLVER_HF_REPO_LIST: str | None = None
|
||||
@@ -1114,6 +1117,17 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_MAX_COMPLETION_PROMPTS": lambda: int(
|
||||
os.environ.get("VLLM_MAX_COMPLETION_PROMPTS", "1024")
|
||||
),
|
||||
# Maximum number of stop strings allowed in a single request.
|
||||
"VLLM_MAX_STOP_STRINGS": lambda: int(os.environ.get("VLLM_MAX_STOP_STRINGS", "4")),
|
||||
# Maximum number of bad-word token sequences generated per request.
|
||||
"VLLM_MAX_NUM_BAD_WORDS": lambda: int(
|
||||
os.environ.get("VLLM_MAX_NUM_BAD_WORDS", "128")
|
||||
),
|
||||
# Maximum total number of bad-word tokens (summed across all bad words)
|
||||
# allowed per request. Bounds the per-request GPU buffer width.
|
||||
"VLLM_MAX_BAD_WORDS_TOTAL_TOKENS": lambda: int(
|
||||
os.environ.get("VLLM_MAX_BAD_WORDS_TOTAL_TOKENS", "1024")
|
||||
),
|
||||
# a list of plugin names to load, separated by commas.
|
||||
# if this is not set, it means all plugins will be loaded
|
||||
# if this is set to an empty string, no plugins will be loaded
|
||||
|
||||
+33
-13
@@ -32,6 +32,22 @@ MAX_LOGPROB_TOKEN_IDS = 128
|
||||
the per-request row width allocated by the sampler's `LogprobTokenIdsState`."""
|
||||
|
||||
|
||||
def _verify_num_sequences(value: int, parameter_name: str) -> None:
|
||||
if not isinstance(value, int):
|
||||
raise VLLMValidationError(
|
||||
f"{parameter_name} must be an int, but is of type {type(value)}"
|
||||
)
|
||||
if value < 1:
|
||||
raise VLLMValidationError(f"{parameter_name} must be at least 1, got {value}.")
|
||||
max_n = envs.VLLM_MAX_N_SEQUENCES
|
||||
if value > max_n:
|
||||
raise VLLMValidationError(
|
||||
f"{parameter_name} must be at most {max_n}, got {value}. "
|
||||
"To increase this limit, set the VLLM_MAX_N_SEQUENCES "
|
||||
"environment variable."
|
||||
)
|
||||
|
||||
|
||||
def validate_thinking_token_budget(value: int | float | bool | None) -> int | None:
|
||||
"""Validate ``thinking_token_budget``; return ``None`` if unset."""
|
||||
if value is None:
|
||||
@@ -479,9 +495,13 @@ class SamplingParams(
|
||||
|
||||
if self.stop_token_ids is None:
|
||||
self.stop_token_ids = []
|
||||
else:
|
||||
self.stop_token_ids = list(dict.fromkeys(self.stop_token_ids))
|
||||
|
||||
if self.bad_words is None:
|
||||
self.bad_words = []
|
||||
else:
|
||||
self.bad_words = list(dict.fromkeys(self.bad_words))
|
||||
|
||||
if self.logprobs is True:
|
||||
self.logprobs = 1
|
||||
@@ -513,19 +533,7 @@ class SamplingParams(
|
||||
self.skip_reading_prefix_cache = self.prompt_logprobs is not None
|
||||
|
||||
def _verify_args(self) -> None:
|
||||
if not isinstance(self.n, int):
|
||||
raise VLLMValidationError(
|
||||
f"n must be an int, but is of type {type(self.n)}"
|
||||
)
|
||||
if self.n < 1:
|
||||
raise VLLMValidationError(f"n must be at least 1, got {self.n}.")
|
||||
max_n = envs.VLLM_MAX_N_SEQUENCES
|
||||
if self.n > max_n:
|
||||
raise VLLMValidationError(
|
||||
f"n must be at most {max_n}, got {self.n}. "
|
||||
"To increase this limit, set the VLLM_MAX_N_SEQUENCES "
|
||||
"environment variable."
|
||||
)
|
||||
_verify_num_sequences(self.n, "n")
|
||||
if not -2.0 <= self.presence_penalty <= 2.0:
|
||||
raise VLLMValidationError(
|
||||
f"presence_penalty must be in [-2, 2], got {self.presence_penalty}."
|
||||
@@ -677,6 +685,7 @@ class SamplingParams(
|
||||
if not self.bad_words:
|
||||
return
|
||||
self._bad_words_token_ids = []
|
||||
max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS
|
||||
for bad_word in self.bad_words:
|
||||
# To prohibit words both at the beginning
|
||||
# and in the middle of text
|
||||
@@ -696,6 +705,14 @@ class SamplingParams(
|
||||
and len(prompt_token_ids) == len(self._bad_words_token_ids[-1])
|
||||
):
|
||||
self._bad_words_token_ids.append(prompt_token_ids)
|
||||
if len(self._bad_words_token_ids) > max_num_bad_words:
|
||||
raise VLLMValidationError(
|
||||
f"Too many bad words after tokenization: "
|
||||
f"{len(self._bad_words_token_ids)}. "
|
||||
f"The max number is {max_num_bad_words}.",
|
||||
parameter="bad_words",
|
||||
value=self.bad_words,
|
||||
)
|
||||
|
||||
invalid_token_ids = [
|
||||
token_id
|
||||
@@ -1147,3 +1164,6 @@ class BeamSearchParams(
|
||||
length_penalty: float = 1.0
|
||||
include_stop_str_in_output: bool = False
|
||||
structured_outputs: StructuredOutputsParams | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
_verify_num_sequences(self.beam_width, "beam_width")
|
||||
|
||||
@@ -119,7 +119,10 @@ def find_last_user_index(messages: list[dict[str, Any]]) -> int:
|
||||
|
||||
|
||||
def render_message(
|
||||
index: int, messages: list[dict[str, Any]], thinking_mode: str
|
||||
index: int,
|
||||
messages: list[dict[str, Any]],
|
||||
thinking_mode: str,
|
||||
last_user_idx: int | None = None,
|
||||
) -> str:
|
||||
if not (0 <= index < len(messages)):
|
||||
raise ValueError(
|
||||
@@ -130,7 +133,9 @@ def render_message(
|
||||
|
||||
prompt = ""
|
||||
msg = messages[index]
|
||||
last_user_idx = find_last_user_index(messages)
|
||||
last_user_idx = (
|
||||
find_last_user_index(messages) if last_user_idx is None else last_user_idx
|
||||
)
|
||||
|
||||
role = msg.get("role")
|
||||
content = msg.get("content")
|
||||
@@ -294,9 +299,14 @@ def encode_messages(
|
||||
if thinking_mode == "thinking" and drop_thinking:
|
||||
full_messages = drop_thinking_messages(full_messages)
|
||||
|
||||
last_user_idx = find_last_user_index(full_messages)
|
||||
|
||||
for idx in range(len(messages)):
|
||||
prompt += render_message(
|
||||
idx + len(context), full_messages, thinking_mode=thinking_mode
|
||||
idx + len(context),
|
||||
full_messages,
|
||||
thinking_mode=thinking_mode,
|
||||
last_user_idx=last_user_idx,
|
||||
)
|
||||
|
||||
return prompt
|
||||
|
||||
@@ -199,7 +199,14 @@ def find_last_user_index(messages: List[Dict[str, Any]]) -> int:
|
||||
# Message Rendering
|
||||
# ============================================================
|
||||
|
||||
def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: str, drop_thinking: bool = True, reasoning_effort: Optional[str] = None) -> str:
|
||||
def render_message(
|
||||
index: int,
|
||||
messages: List[Dict[str, Any]],
|
||||
thinking_mode: str,
|
||||
drop_thinking: bool = True,
|
||||
reasoning_effort: Optional[str] = None,
|
||||
last_user_idx: Optional[int] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Render a single message at the given index into its encoded string form.
|
||||
|
||||
@@ -213,6 +220,7 @@ def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: st
|
||||
drop_thinking: Whether to drop reasoning content from earlier turns.
|
||||
reasoning_effort: Reasoning effort level, one of "low", "high", "max".
|
||||
None is treated as "low".
|
||||
last_user_idx: Cached index of the last user/developer message.
|
||||
|
||||
Returns:
|
||||
Encoded string for this message.
|
||||
@@ -222,7 +230,9 @@ def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: st
|
||||
|
||||
prompt = ""
|
||||
msg = messages[index]
|
||||
last_user_idx = find_last_user_index(messages)
|
||||
last_user_idx = (
|
||||
find_last_user_index(messages) if last_user_idx is None else last_user_idx
|
||||
)
|
||||
|
||||
role = msg.get("role")
|
||||
content = msg.get("content")
|
||||
@@ -544,6 +554,8 @@ def encode_messages(
|
||||
num_to_render = len(messages)
|
||||
context_len = len(context)
|
||||
|
||||
last_user_idx = find_last_user_index(full_messages)
|
||||
|
||||
for idx in range(num_to_render):
|
||||
prompt += render_message(
|
||||
idx + context_len,
|
||||
@@ -551,6 +563,7 @@ def encode_messages(
|
||||
thinking_mode=thinking_mode,
|
||||
drop_thinking=effective_drop_thinking,
|
||||
reasoning_effort=reasoning_effort,
|
||||
last_user_idx=last_user_idx,
|
||||
)
|
||||
|
||||
return prompt
|
||||
|
||||
@@ -3,14 +3,12 @@
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
|
||||
from vllm.v1.worker.gpu.states import RequestState
|
||||
|
||||
MAX_BAD_WORDS_TOTAL_TOKENS = 1024 # Max total tokens for all bad words per request
|
||||
MAX_NUM_BAD_WORDS = 128 # Max number of bad words per request
|
||||
|
||||
|
||||
class BadWordsState:
|
||||
def __init__(self, req_states: RequestState):
|
||||
@@ -18,15 +16,17 @@ class BadWordsState:
|
||||
self.max_num_reqs = req_states.max_num_reqs
|
||||
self.device = req_states.device
|
||||
|
||||
# flattened bad word tokens: [max_num_reqs, MAX_BAD_WORDS_TOTAL_TOKENS]
|
||||
max_total_tokens = envs.VLLM_MAX_BAD_WORDS_TOTAL_TOKENS
|
||||
max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS
|
||||
# flattened bad word tokens: [max_num_reqs, VLLM_MAX_BAD_WORDS_TOTAL_TOKENS]
|
||||
self.bad_word_token_ids = StagedWriteTensor(
|
||||
(self.max_num_reqs, MAX_BAD_WORDS_TOTAL_TOKENS),
|
||||
(self.max_num_reqs, max_total_tokens),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
# cumulative offsets of bad words: [max_num_reqs, MAX_NUM_BAD_WORDS + 1]
|
||||
# cumulative offsets of bad words: [max_num_reqs, VLLM_MAX_NUM_BAD_WORDS + 1]
|
||||
self.bad_word_offsets = StagedWriteTensor(
|
||||
(self.max_num_reqs, MAX_NUM_BAD_WORDS + 1),
|
||||
(self.max_num_reqs, max_num_bad_words + 1),
|
||||
dtype=torch.int32,
|
||||
device=self.device,
|
||||
)
|
||||
@@ -40,10 +40,11 @@ class BadWordsState:
|
||||
return
|
||||
|
||||
num_bad_words = len(bad_words_token_ids)
|
||||
if num_bad_words > MAX_NUM_BAD_WORDS:
|
||||
max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS
|
||||
if num_bad_words > max_num_bad_words:
|
||||
raise ValueError(
|
||||
f"Too many bad words: {num_bad_words}. "
|
||||
f"The max number is {MAX_NUM_BAD_WORDS}."
|
||||
f"The max number is {max_num_bad_words}."
|
||||
)
|
||||
|
||||
# Flatten bad words and compute offsets
|
||||
@@ -53,10 +54,11 @@ class BadWordsState:
|
||||
flattened_tokens.extend(bad_word)
|
||||
offsets.append(len(flattened_tokens))
|
||||
|
||||
if len(flattened_tokens) > MAX_BAD_WORDS_TOTAL_TOKENS:
|
||||
max_total_tokens = envs.VLLM_MAX_BAD_WORDS_TOTAL_TOKENS
|
||||
if len(flattened_tokens) > max_total_tokens:
|
||||
raise ValueError(
|
||||
f"Too many total bad word tokens: {len(flattened_tokens)}. "
|
||||
f"The max is {MAX_BAD_WORDS_TOTAL_TOKENS}."
|
||||
f"The max is {max_total_tokens}."
|
||||
)
|
||||
|
||||
# Stage writes
|
||||
|
||||
Reference in New Issue
Block a user