[Bugfix] Treat max_tokens as output upper bound, not input-space reservation (#57035)
Signed-off-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com> Signed-off-by: YuvrajSinghBhadoria2 <yuvrajsinghbhado2030@gmail.com> Signed-off-by: Yuvraj Singh Bhadoria <yuvrajsinghbhado2030@gmail.com> Co-authored-by: YuvrajSinghBhadoria2 <245782664+YuvrajSinghBhadoria2@users.noreply.github.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
YuvrajSinghBhadoria2
Cyrus Leung
Claude Opus 4.8
parent
89ac8d971b
commit
1a3a7a8f01
@@ -183,15 +183,15 @@ class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
if pooler_config.enable_chunked_processing:
|
||||
max_total_tokens = pooler_config.max_embed_len
|
||||
else:
|
||||
max_embed_len = pooler_config.max_embed_len or default_max_total_tokens
|
||||
max_output_tokens = default_max_total_tokens - max_embed_len
|
||||
max_total_tokens = (
|
||||
pooler_config.max_embed_len or default_max_total_tokens
|
||||
)
|
||||
|
||||
tok_params = self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
max_output_tokens_param="max_model_len - max_embed_len",
|
||||
)
|
||||
|
||||
if (
|
||||
|
||||
+49
-22
@@ -207,11 +207,38 @@ class TokenizeParams:
|
||||
|
||||
@property
|
||||
def max_input_tokens(self) -> int | None:
|
||||
"""Maximum allowed number of input tokens."""
|
||||
"""Maximum allowed number of input tokens.
|
||||
|
||||
``max_output_tokens`` is *not* subtracted here: the value represents the
|
||||
actual model context length, not a reservation of output space.
|
||||
Downstream clamping of the sampling ``max_tokens`` is handled by
|
||||
``get_max_tokens()`` in the entrypoint layer.
|
||||
"""
|
||||
if self.max_total_tokens is None:
|
||||
return None
|
||||
|
||||
return self.max_total_tokens - self.max_output_tokens
|
||||
return self.max_total_tokens
|
||||
|
||||
@property
|
||||
def max_truncation_tokens(self) -> int | None:
|
||||
"""Maximum prompt length that truncation may reduce a prompt to.
|
||||
|
||||
For generative requests this is `max_input_tokens` less one, holding a
|
||||
token of context back for output. Truncating to the full
|
||||
`max_total_tokens` would leave `get_max_tokens()` clamping the sampling
|
||||
budget to 0, turning an over-long prompt into a silent zero-token
|
||||
generation.
|
||||
|
||||
Pooling requests set `max_output_tokens = 0` and emit an embedding
|
||||
rather than generated tokens, so they keep the full input budget.
|
||||
"""
|
||||
max_input_tokens = self.max_input_tokens
|
||||
if max_input_tokens is None:
|
||||
return None
|
||||
if self.max_output_tokens == 0:
|
||||
return max_input_tokens
|
||||
|
||||
return max(0, max_input_tokens - 1)
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
max_total_tokens = self.max_total_tokens
|
||||
@@ -247,14 +274,15 @@ class TokenizeParams:
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
f"{self.truncate_prompt_tokens_param}={truncate_prompt_tokens} "
|
||||
f"cannot be greater than {self.max_total_tokens_param} - "
|
||||
f"{self.max_output_tokens_param} = {max_input_tokens}. "
|
||||
f"cannot be greater than {self.max_total_tokens_param} "
|
||||
f"= {max_input_tokens}. "
|
||||
f"Please request a smaller truncation size.",
|
||||
parameter=self.truncate_prompt_tokens_param,
|
||||
value=truncate_prompt_tokens,
|
||||
)
|
||||
|
||||
def with_kwargs(self, **tokenization_kwargs: Any):
|
||||
has_max_length = "max_length" in tokenization_kwargs
|
||||
max_length = tokenization_kwargs.pop("max_length", self.max_input_tokens)
|
||||
pad_prompt_tokens = tokenization_kwargs.pop(
|
||||
"pad_prompt_tokens", self.pad_prompt_tokens
|
||||
@@ -301,13 +329,17 @@ class TokenizeParams:
|
||||
|
||||
max_total_tokens = self.max_total_tokens
|
||||
|
||||
if has_max_length and max_total_tokens is not None and max_length is not None:
|
||||
max_output_tokens = max_total_tokens - max_length
|
||||
else:
|
||||
# No explicit ``max_length`` override: preserve the original output
|
||||
# budget instead of recomputing it from ``max_input_tokens`` (which
|
||||
# is now the full context length).
|
||||
max_output_tokens = self.max_output_tokens
|
||||
|
||||
return TokenizeParams(
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=(
|
||||
0
|
||||
if max_total_tokens is None or max_length is None
|
||||
else max_total_tokens - max_length
|
||||
),
|
||||
max_output_tokens=max_output_tokens,
|
||||
pad_prompt_tokens=pad_prompt_tokens,
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
truncation_side=truncation_side,
|
||||
@@ -320,7 +352,7 @@ class TokenizeParams:
|
||||
"""The arguments to pass to `tokenizer.encode`."""
|
||||
max_length = self.truncate_prompt_tokens
|
||||
if max_length is not None and max_length < 0:
|
||||
max_length = self.max_input_tokens
|
||||
max_length = self.max_truncation_tokens
|
||||
elif max_length is None and self.max_input_tokens is not None:
|
||||
# This prevents tokenization from taking up more resources than necessary
|
||||
# while still failing `self._token_len_check` as expected by users
|
||||
@@ -355,13 +387,11 @@ class TokenizeParams:
|
||||
if len(text) > max_input_chars:
|
||||
raise VLLMValidationError(
|
||||
f"This model's maximum context length is "
|
||||
f"{self.max_total_tokens} tokens. However, you requested "
|
||||
f"{self.max_output_tokens} output tokens and your prompt "
|
||||
f"{self.max_total_tokens} tokens. However, your prompt "
|
||||
f"contains {len(text)} characters (more than "
|
||||
f"{max_input_chars} characters, which is the upper bound "
|
||||
f"for {max_input_tokens} input tokens). "
|
||||
f"Please reduce the length of the input prompt or the "
|
||||
f"number of requested output tokens.",
|
||||
f"Please reduce the length of the input prompt.",
|
||||
parameter="input_text",
|
||||
value=len(text),
|
||||
)
|
||||
@@ -459,7 +489,7 @@ class TokenizeParams:
|
||||
"""
|
||||
max_length = self.truncate_prompt_tokens
|
||||
if max_length is not None and max_length < 0:
|
||||
max_length = self.max_input_tokens
|
||||
max_length = self.max_truncation_tokens
|
||||
|
||||
if max_length is None or max_length >= length:
|
||||
return None
|
||||
@@ -494,15 +524,12 @@ class TokenizeParams:
|
||||
# max_input_tokens + 1 (see get_encode_kwargs), so the
|
||||
# actual prompt length could be larger.
|
||||
qualifier = "at least " if token_count == max_input_tokens + 1 else ""
|
||||
total = token_count + self.max_output_tokens
|
||||
raise VLLMValidationError(
|
||||
f"This model's maximum context length is "
|
||||
f"{self.max_total_tokens} tokens. However, you requested "
|
||||
f"{self.max_output_tokens} output tokens and your prompt "
|
||||
f"contains {qualifier}{token_count} input tokens, "
|
||||
f"for a total of {qualifier}{total} tokens. "
|
||||
f"Please reduce the length of the input prompt or the "
|
||||
f"number of requested output tokens.",
|
||||
f"{self.max_total_tokens} tokens. However, your prompt "
|
||||
f"contains {qualifier}{token_count} input tokens, which "
|
||||
f"exceeds this limit even without any output tokens. "
|
||||
f"Please reduce the length of the input prompt.",
|
||||
parameter="input_tokens",
|
||||
value=token_count,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user