Bound generation inputs before expensive work (#51447)

Signed-off-by: Clinton Thomas <1033162+KernelClint@users.noreply.github.com>
Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Clinton Thomas
2026-08-11 22:03:50 +08:00
committed by GitHub
co-authored by Lucas Bourtoule mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5b184f775a
commit b64a2708b0
11 changed files with 421 additions and 37 deletions
@@ -54,9 +54,23 @@ impl StringOrArray {
}
}
/// Validates stop sequences (non-empty strings)
fn max_stop_strings() -> usize {
std::env::var("VLLM_MAX_STOP_STRINGS")
.ok()
.and_then(|value| value.parse().ok())
.unwrap_or(4)
}
/// Validates stop sequences (at most the configured number of non-empty strings).
pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> {
if stop.as_slice().iter().any(|s| s.is_empty()) {
let stop = stop.as_slice();
let max_stop_strings = max_stop_strings();
if stop.len() > max_stop_strings {
let mut error = validator::ValidationError::new("too_many_stop_strings");
error.code = format!("stop strings must contain at most {max_stop_strings} items").into();
return Err(error);
}
if stop.iter().any(|s| s.is_empty()) {
return Err(validator::ValidationError::new(
"stop strings cannot be empty",
));
@@ -64,6 +78,37 @@ pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationEr
Ok(())
}
#[cfg(test)]
mod tests {
use super::{StringOrArray, validate_stop};
#[test]
fn validate_stop_accepts_four_stop_strings() {
let stop = StringOrArray::Array(
["one", "two", "three", "four"].into_iter().map(str::to_string).collect(),
);
validate_stop(&stop).expect("four stop strings should be accepted");
}
#[test]
fn validate_stop_rejects_more_than_four_stop_strings() {
let stop = StringOrArray::Array(
["one", "two", "three", "four", "five"]
.into_iter()
.map(str::to_string)
.collect(),
);
let error = validate_stop(&stop).expect_err("five stop strings should be rejected");
assert_eq!(
error.code.as_ref(),
"stop strings must contain at most 4 items"
);
}
}
// ============================================================================
// Validation helpers
// ============================================================================
+273
View File
@@ -0,0 +1,273 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Bounds on request-controlled inputs that would otherwise amplify work.
Consolidates the regression tests for the request-input amplification fixes:
stop-string caps, bad-words dedup/tokenization limit, stop-token-id dedup,
beam-width/sequence caps, and the DeepSeek history-scan bound.
"""
import os
import subprocess
import sys
from collections.abc import Callable
from typing import Protocol
import pytest
from pydantic import ValidationError
import vllm.envs as envs
from vllm import SamplingParams
from vllm.entrypoints.openai.chat_completion.protocol import (
BatchChatCompletionRequest,
ChatCompletionRequest,
)
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import BeamSearchParams
from vllm.tokenizers import deepseek_v4_encoding, deepseek_v32_encoding
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
# --- Stop strings: public requests cap the number of stop strings ---------
class _StopRequest(Protocol):
stop: str | list[str] | None
def _completion_request(stop: list[str]) -> _StopRequest:
return CompletionRequest(model="test-model", prompt="hello", stop=stop)
def _chat_request(stop: list[str]) -> _StopRequest:
return ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "hello"}],
stop=stop,
)
def _batch_chat_request(stop: list[str]) -> _StopRequest:
return BatchChatCompletionRequest(
model="test-model",
messages=[[{"role": "user", "content": "hello"}]],
stop=stop,
)
def _responses_request(stop: list[str]) -> _StopRequest:
return ResponsesRequest(model="test-model", input="hello", stop=stop)
REQUEST_BUILDERS: list[Callable[[list[str]], _StopRequest]] = [
_completion_request,
_chat_request,
_batch_chat_request,
_responses_request,
]
@pytest.mark.parametrize("build_request", REQUEST_BUILDERS)
def test_public_requests_accept_four_stop_strings(
build_request: Callable[[list[str]], _StopRequest],
):
stop = ["one", "two", "three", "four"]
request = build_request(stop)
assert request.stop == stop
@pytest.mark.parametrize("build_request", REQUEST_BUILDERS)
def test_public_requests_reject_more_than_four_stop_strings(
build_request: Callable[[list[str]], _StopRequest],
):
with pytest.raises(ValidationError, match="at most 4"):
build_request(["one", "two", "three", "four", "five"])
def test_stop_string_limit_can_be_overridden():
env = os.environ.copy()
env["VLLM_MAX_STOP_STRINGS"] = "1"
code = """
from pydantic import ValidationError
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
try:
CompletionRequest(
model="test-model",
prompt="hello",
stop=["one", "two"],
)
except ValidationError as error:
assert "at most 1" in str(error)
else:
raise AssertionError("configured stop-string limit was not enforced")
"""
subprocess.run([sys.executable, "-c", code], check=True, env=env)
# --- Stop token ids: duplicates are deduplicated in order ------------------
def test_duplicate_stop_token_ids_are_deduplicated_in_order():
params = SamplingParams(stop_token_ids=[42, 7, 42, 9, 7])
assert params.stop_token_ids == [42, 7, 9]
assert params.all_stop_token_ids == {7, 9, 42}
# --- Bad words: dedup, and the tokenization pass is bounded ----------------
class MockTokenizer:
max_token_id = 1024
def __init__(self):
self.calls = 0
def encode(self, text: str, add_special_tokens: bool = False) -> list[int]:
self.calls += 1
return [2] if text.startswith(" ") else [1]
def test_duplicate_bad_words_are_deduplicated_in_order():
params = SamplingParams(bad_words=["bad", "worse", "bad", "worst"])
assert params.bad_words == ["bad", "worse", "worst"]
def test_bad_word_tokenization_stops_at_worker_limit():
params = SamplingParams(bad_words=[f"word-{i}" for i in range(65)])
tokenizer = MockTokenizer()
with pytest.raises(VLLMValidationError, match="Too many bad words"):
params.update_from_tokenizer(tokenizer)
assert tokenizer.calls == 129
def test_bad_word_tokenization_limit_can_be_overridden(monkeypatch):
monkeypatch.setenv("VLLM_MAX_NUM_BAD_WORDS", "2")
params = SamplingParams(bad_words=["bad", "worse"])
tokenizer = MockTokenizer()
with pytest.raises(VLLMValidationError, match="The max number is 2"):
params.update_from_tokenizer(tokenizer)
assert tokenizer.calls == 3
# --- Beam search: beam width / n honor the sequence cap --------------------
def _set_max_n(monkeypatch: pytest.MonkeyPatch, value: int) -> None:
monkeypatch.setenv("VLLM_MAX_N_SEQUENCES", str(value))
if hasattr(envs.__getattr__, "cache_clear"):
envs.__getattr__.cache_clear()
def test_direct_beam_width_rejects_values_over_sequence_cap(
monkeypatch: pytest.MonkeyPatch,
):
_set_max_n(monkeypatch, 4)
with pytest.raises(VLLMValidationError, match="beam_width must be at most 4"):
BeamSearchParams(beam_width=5, max_tokens=1)
def test_chat_beam_conversion_rejects_n_before_stream_state_allocation(
monkeypatch: pytest.MonkeyPatch,
):
_set_max_n(monkeypatch, 4)
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "test"}],
n=5,
stream=True,
use_beam_search=True,
max_tokens=1,
)
with pytest.raises(VLLMValidationError, match="beam_width must be at most 4"):
request.to_beam_search_params(max_tokens=1, default_sampling_params={})
def test_chat_beam_conversion_accepts_n_at_sequence_cap(
monkeypatch: pytest.MonkeyPatch,
):
_set_max_n(monkeypatch, 4)
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "test"}],
n=4,
stream=True,
use_beam_search=True,
max_tokens=1,
)
params = request.to_beam_search_params(max_tokens=1, default_sampling_params={})
assert params.beam_width == 4
# --- DeepSeek encoders: the last-user scan runs once per conversation ------
ENCODING_MODULES = [deepseek_v32_encoding, deepseek_v4_encoding]
@pytest.mark.parametrize(
"encoding_module",
ENCODING_MODULES,
ids=["deepseek_v32", "deepseek_v4"],
)
def test_encode_messages_scans_last_user_once_per_conversation(
monkeypatch: pytest.MonkeyPatch,
encoding_module,
):
calls = 0
original_find_last_user_index = encoding_module.find_last_user_index
def counted_find_last_user_index(messages):
nonlocal calls
calls += 1
return original_find_last_user_index(messages)
monkeypatch.setattr(
encoding_module,
"find_last_user_index",
counted_find_last_user_index,
)
messages = [{"role": "user", "content": "Hello"}]
messages.extend({"role": "assistant", "content": "Hi"} for _ in range(8))
encoding_module.encode_messages(messages, thinking_mode="chat")
assert calls == 1
@pytest.mark.parametrize(
"encoding_module",
ENCODING_MODULES,
ids=["deepseek_v32", "deepseek_v4"],
)
def test_encode_messages_preserves_small_chat_prompt(encoding_module):
prompt = encoding_module.encode_messages(
[
{"role": "user", "content": "Hello"},
{"role": "assistant", "content": "Hi"},
{"role": "assistant", "content": "Again"},
],
thinking_mode="chat",
)
assert prompt == (
"<|begin▁of▁sentence|><|User|>Hello<|Assistant|></think>"
"Hi<|end▁of▁sentence|>Again<|end▁of▁sentence|>"
)
@@ -30,6 +30,7 @@ from vllm.entrypoints.openai.engine.protocol import (
FunctionDefinition,
OpenAIBaseModel,
PerRequestTimingMetrics,
StopParam,
StreamOptions,
ToolCall,
UsageInfo,
@@ -228,7 +229,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
presence_penalty: float | None = 0.0
response_format: AnyResponseFormat | None = None
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
stop: str | list[str] | None = []
stop: StopParam = []
stream: bool | None = False
stream_options: StreamOptions | None = None
temperature: float | None = None
@@ -1077,7 +1078,7 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
presence_penalty: float | None = 0.0
response_format: Any | None = None
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
stop: str | list[str] | None = Field(default_factory=list)
stop: StopParam = Field(default_factory=list)
temperature: float | None = None
top_p: float | None = None
user: str | None = None
@@ -14,6 +14,7 @@ from vllm.entrypoints.openai.engine.protocol import (
AnyResponseFormat,
OpenAIBaseModel,
PerRequestTimingMetrics,
StopParam,
StreamOptions,
UsageInfo,
structured_outputs_from_response_format,
@@ -61,7 +62,7 @@ class CompletionRequest(OpenAIBaseModel):
n: int = 1
presence_penalty: float | None = 0.0
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
stop: str | list[str] | None = []
stop: StopParam = []
stream: bool | None = False
stream_options: StreamOptions | None = None
suffix: str | None = None
+6 -1
View File
@@ -6,7 +6,7 @@
import json
import time
from http import HTTPStatus
from typing import Any, ClassVar, Literal, TypeAlias
from typing import Annotated, Any, ClassVar, Literal, TypeAlias
import regex as re
from pydantic import (
@@ -17,6 +17,7 @@ from pydantic import (
model_validator,
)
import vllm.envs as envs
from vllm.config.utils import replace
from vllm.entrypoints.chat_utils import make_tool_call_id
from vllm.exceptions import VLLMServerError, VLLMValidationError
@@ -27,6 +28,10 @@ from vllm.utils.import_utils import resolve_obj_by_qualname
logger = init_logger(__name__)
StopParam: TypeAlias = (
str | Annotated[list[str], Field(max_length=envs.VLLM_MAX_STOP_STRINGS)] | None
)
class OpenAIBaseModel(BaseModel):
# OpenAI API does allow extra fields
@@ -60,7 +60,7 @@ from vllm.entrypoints.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption,
)
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, StopParam
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
@@ -281,7 +281,7 @@ class ResponsesRequest(OpenAIBaseModel):
repetition_penalty: float | None = None
seed: int | None = Field(None, ge=_INT64_MIN, le=_INT64_MAX)
stop: str | list[str] | None = []
stop: StopParam = []
ignore_eos: bool = False
vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
default=None,
+14
View File
@@ -108,6 +108,9 @@ if TYPE_CHECKING:
VLLM_HTTP_TIMEOUT_KEEP_ALIVE: int = 5 # seconds
VLLM_MAX_N_SEQUENCES: int = 16384
VLLM_MAX_COMPLETION_PROMPTS: int = 1024
VLLM_MAX_STOP_STRINGS: int = 4
VLLM_MAX_NUM_BAD_WORDS: int = 128
VLLM_MAX_BAD_WORDS_TOTAL_TOKENS: int = 1024
VLLM_PLUGINS: list[str] | None = None
VLLM_LORA_RESOLVER_CACHE_DIR: str | None = None
VLLM_LORA_RESOLVER_HF_REPO_LIST: str | None = None
@@ -1114,6 +1117,17 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_MAX_COMPLETION_PROMPTS": lambda: int(
os.environ.get("VLLM_MAX_COMPLETION_PROMPTS", "1024")
),
# Maximum number of stop strings allowed in a single request.
"VLLM_MAX_STOP_STRINGS": lambda: int(os.environ.get("VLLM_MAX_STOP_STRINGS", "4")),
# Maximum number of bad-word token sequences generated per request.
"VLLM_MAX_NUM_BAD_WORDS": lambda: int(
os.environ.get("VLLM_MAX_NUM_BAD_WORDS", "128")
),
# Maximum total number of bad-word tokens (summed across all bad words)
# allowed per request. Bounds the per-request GPU buffer width.
"VLLM_MAX_BAD_WORDS_TOTAL_TOKENS": lambda: int(
os.environ.get("VLLM_MAX_BAD_WORDS_TOTAL_TOKENS", "1024")
),
# a list of plugin names to load, separated by commas.
# if this is not set, it means all plugins will be loaded
# if this is set to an empty string, no plugins will be loaded
+33 -13
View File
@@ -32,6 +32,22 @@ MAX_LOGPROB_TOKEN_IDS = 128
the per-request row width allocated by the sampler's `LogprobTokenIdsState`."""
def _verify_num_sequences(value: int, parameter_name: str) -> None:
if not isinstance(value, int):
raise VLLMValidationError(
f"{parameter_name} must be an int, but is of type {type(value)}"
)
if value < 1:
raise VLLMValidationError(f"{parameter_name} must be at least 1, got {value}.")
max_n = envs.VLLM_MAX_N_SEQUENCES
if value > max_n:
raise VLLMValidationError(
f"{parameter_name} must be at most {max_n}, got {value}. "
"To increase this limit, set the VLLM_MAX_N_SEQUENCES "
"environment variable."
)
def validate_thinking_token_budget(value: int | float | bool | None) -> int | None:
"""Validate ``thinking_token_budget``; return ``None`` if unset."""
if value is None:
@@ -479,9 +495,13 @@ class SamplingParams(
if self.stop_token_ids is None:
self.stop_token_ids = []
else:
self.stop_token_ids = list(dict.fromkeys(self.stop_token_ids))
if self.bad_words is None:
self.bad_words = []
else:
self.bad_words = list(dict.fromkeys(self.bad_words))
if self.logprobs is True:
self.logprobs = 1
@@ -513,19 +533,7 @@ class SamplingParams(
self.skip_reading_prefix_cache = self.prompt_logprobs is not None
def _verify_args(self) -> None:
if not isinstance(self.n, int):
raise VLLMValidationError(
f"n must be an int, but is of type {type(self.n)}"
)
if self.n < 1:
raise VLLMValidationError(f"n must be at least 1, got {self.n}.")
max_n = envs.VLLM_MAX_N_SEQUENCES
if self.n > max_n:
raise VLLMValidationError(
f"n must be at most {max_n}, got {self.n}. "
"To increase this limit, set the VLLM_MAX_N_SEQUENCES "
"environment variable."
)
_verify_num_sequences(self.n, "n")
if not -2.0 <= self.presence_penalty <= 2.0:
raise VLLMValidationError(
f"presence_penalty must be in [-2, 2], got {self.presence_penalty}."
@@ -677,6 +685,7 @@ class SamplingParams(
if not self.bad_words:
return
self._bad_words_token_ids = []
max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS
for bad_word in self.bad_words:
# To prohibit words both at the beginning
# and in the middle of text
@@ -696,6 +705,14 @@ class SamplingParams(
and len(prompt_token_ids) == len(self._bad_words_token_ids[-1])
):
self._bad_words_token_ids.append(prompt_token_ids)
if len(self._bad_words_token_ids) > max_num_bad_words:
raise VLLMValidationError(
f"Too many bad words after tokenization: "
f"{len(self._bad_words_token_ids)}. "
f"The max number is {max_num_bad_words}.",
parameter="bad_words",
value=self.bad_words,
)
invalid_token_ids = [
token_id
@@ -1147,3 +1164,6 @@ class BeamSearchParams(
length_penalty: float = 1.0
include_stop_str_in_output: bool = False
structured_outputs: StructuredOutputsParams | None = None
def __post_init__(self) -> None:
_verify_num_sequences(self.beam_width, "beam_width")
+13 -3
View File
@@ -119,7 +119,10 @@ def find_last_user_index(messages: list[dict[str, Any]]) -> int:
def render_message(
index: int, messages: list[dict[str, Any]], thinking_mode: str
index: int,
messages: list[dict[str, Any]],
thinking_mode: str,
last_user_idx: int | None = None,
) -> str:
if not (0 <= index < len(messages)):
raise ValueError(
@@ -130,7 +133,9 @@ def render_message(
prompt = ""
msg = messages[index]
last_user_idx = find_last_user_index(messages)
last_user_idx = (
find_last_user_index(messages) if last_user_idx is None else last_user_idx
)
role = msg.get("role")
content = msg.get("content")
@@ -294,9 +299,14 @@ def encode_messages(
if thinking_mode == "thinking" and drop_thinking:
full_messages = drop_thinking_messages(full_messages)
last_user_idx = find_last_user_index(full_messages)
for idx in range(len(messages)):
prompt += render_message(
idx + len(context), full_messages, thinking_mode=thinking_mode
idx + len(context),
full_messages,
thinking_mode=thinking_mode,
last_user_idx=last_user_idx,
)
return prompt
+15 -2
View File
@@ -199,7 +199,14 @@ def find_last_user_index(messages: List[Dict[str, Any]]) -> int:
# Message Rendering
# ============================================================
def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: str, drop_thinking: bool = True, reasoning_effort: Optional[str] = None) -> str:
def render_message(
index: int,
messages: List[Dict[str, Any]],
thinking_mode: str,
drop_thinking: bool = True,
reasoning_effort: Optional[str] = None,
last_user_idx: Optional[int] = None,
) -> str:
"""
Render a single message at the given index into its encoded string form.
@@ -213,6 +220,7 @@ def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: st
drop_thinking: Whether to drop reasoning content from earlier turns.
reasoning_effort: Reasoning effort level, one of "low", "high", "max".
None is treated as "low".
last_user_idx: Cached index of the last user/developer message.
Returns:
Encoded string for this message.
@@ -222,7 +230,9 @@ def render_message(index: int, messages: List[Dict[str, Any]], thinking_mode: st
prompt = ""
msg = messages[index]
last_user_idx = find_last_user_index(messages)
last_user_idx = (
find_last_user_index(messages) if last_user_idx is None else last_user_idx
)
role = msg.get("role")
content = msg.get("content")
@@ -544,6 +554,8 @@ def encode_messages(
num_to_render = len(messages)
context_len = len(context)
last_user_idx = find_last_user_index(full_messages)
for idx in range(num_to_render):
prompt += render_message(
idx + context_len,
@@ -551,6 +563,7 @@ def encode_messages(
thinking_mode=thinking_mode,
drop_thinking=effective_drop_thinking,
reasoning_effort=reasoning_effort,
last_user_idx=last_user_idx,
)
return prompt
+13 -11
View File
@@ -3,14 +3,12 @@
import numpy as np
import torch
import vllm.envs as envs
from vllm.sampling_params import SamplingParams
from vllm.triton_utils import tl, triton
from vllm.v1.worker.gpu.buffer_utils import StagedWriteTensor, UvaBackedTensor
from vllm.v1.worker.gpu.states import RequestState
MAX_BAD_WORDS_TOTAL_TOKENS = 1024 # Max total tokens for all bad words per request
MAX_NUM_BAD_WORDS = 128 # Max number of bad words per request
class BadWordsState:
def __init__(self, req_states: RequestState):
@@ -18,15 +16,17 @@ class BadWordsState:
self.max_num_reqs = req_states.max_num_reqs
self.device = req_states.device
# flattened bad word tokens: [max_num_reqs, MAX_BAD_WORDS_TOTAL_TOKENS]
max_total_tokens = envs.VLLM_MAX_BAD_WORDS_TOTAL_TOKENS
max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS
# flattened bad word tokens: [max_num_reqs, VLLM_MAX_BAD_WORDS_TOTAL_TOKENS]
self.bad_word_token_ids = StagedWriteTensor(
(self.max_num_reqs, MAX_BAD_WORDS_TOTAL_TOKENS),
(self.max_num_reqs, max_total_tokens),
dtype=torch.int32,
device=self.device,
)
# cumulative offsets of bad words: [max_num_reqs, MAX_NUM_BAD_WORDS + 1]
# cumulative offsets of bad words: [max_num_reqs, VLLM_MAX_NUM_BAD_WORDS + 1]
self.bad_word_offsets = StagedWriteTensor(
(self.max_num_reqs, MAX_NUM_BAD_WORDS + 1),
(self.max_num_reqs, max_num_bad_words + 1),
dtype=torch.int32,
device=self.device,
)
@@ -40,10 +40,11 @@ class BadWordsState:
return
num_bad_words = len(bad_words_token_ids)
if num_bad_words > MAX_NUM_BAD_WORDS:
max_num_bad_words = envs.VLLM_MAX_NUM_BAD_WORDS
if num_bad_words > max_num_bad_words:
raise ValueError(
f"Too many bad words: {num_bad_words}. "
f"The max number is {MAX_NUM_BAD_WORDS}."
f"The max number is {max_num_bad_words}."
)
# Flatten bad words and compute offsets
@@ -53,10 +54,11 @@ class BadWordsState:
flattened_tokens.extend(bad_word)
offsets.append(len(flattened_tokens))
if len(flattened_tokens) > MAX_BAD_WORDS_TOTAL_TOKENS:
max_total_tokens = envs.VLLM_MAX_BAD_WORDS_TOTAL_TOKENS
if len(flattened_tokens) > max_total_tokens:
raise ValueError(
f"Too many total bad word tokens: {len(flattened_tokens)}. "
f"The max is {MAX_BAD_WORDS_TOTAL_TOKENS}."
f"The max is {max_total_tokens}."
)
# Stage writes