[Structured Output] Let terminal grammars stop under min_tokens (II) (#54218)
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
@@ -840,11 +840,10 @@ def test_stop_via_update_from_output():
|
||||
|
||||
|
||||
def test_check_stop_min_tokens():
|
||||
"""Test that requests don't stop when min_tokens requirement isn't met."""
|
||||
"""Test that sampled stop tokens still finish under min_tokens."""
|
||||
from vllm.v1.core.sched.utils import check_stop
|
||||
|
||||
# Test case 1: num_output_tokens < min_tokens
|
||||
# Should return False (don't stop)
|
||||
# Test case 1: regular tokens below min_tokens should keep running.
|
||||
sampling_params = SamplingParams(
|
||||
ignore_eos=False,
|
||||
max_tokens=20,
|
||||
@@ -857,30 +856,41 @@ def test_check_stop_min_tokens():
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
)
|
||||
# Simulate having generated 3 output tokens (less than min_tokens=5)
|
||||
request.append_output_token_ids([10, 11, EOS_TOKEN_ID]) # EOS token present
|
||||
request.append_output_token_ids([10, 11, 12])
|
||||
|
||||
result = check_stop(request, max_model_len=100)
|
||||
assert result is False, "Should not stop when num_output_tokens<min_tokens"
|
||||
|
||||
# Test case 2: num_output_tokens >= min_tokens
|
||||
# Should follow normal stopping logic (stop on EOS)
|
||||
request.append_output_token_ids(
|
||||
[
|
||||
10,
|
||||
11,
|
||||
12,
|
||||
13,
|
||||
14,
|
||||
EOS_TOKEN_ID,
|
||||
]
|
||||
) # 6 tokens > min_tokens
|
||||
# Test case 2: if EOS is sampled, it should stop even below min_tokens.
|
||||
request_eos = Request(
|
||||
request_id="eos",
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
)
|
||||
request_eos.append_output_token_ids([10, 11, EOS_TOKEN_ID])
|
||||
result = check_stop(request_eos, max_model_len=100)
|
||||
assert result is True, "Sampled EOS should stop even below min_tokens"
|
||||
assert request_eos.status == RequestStatus.FINISHED_STOPPED
|
||||
|
||||
result = check_stop(request, max_model_len=100)
|
||||
assert result is True, "Should stop on EOS when min_tokens met"
|
||||
assert request.status == RequestStatus.FINISHED_STOPPED
|
||||
# Test case 3: ignore_eos should still prevent EOS-based stopping.
|
||||
sampling_params_ignore_eos = SamplingParams(
|
||||
ignore_eos=True,
|
||||
max_tokens=20,
|
||||
min_tokens=5,
|
||||
)
|
||||
sampling_params_ignore_eos.update_from_generation_config({}, EOS_TOKEN_ID)
|
||||
request_ignore_eos = Request(
|
||||
request_id="ignore-eos",
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=sampling_params_ignore_eos,
|
||||
pooling_params=None,
|
||||
)
|
||||
request_ignore_eos.append_output_token_ids([EOS_TOKEN_ID])
|
||||
result = check_stop(request_ignore_eos, max_model_len=100)
|
||||
assert result is False, "ignore_eos should still suppress EOS stopping"
|
||||
|
||||
# Test case 3: min_tokens = 0, should follow normal stopping logic
|
||||
# Test case 4: min_tokens = 0 should follow normal stopping logic.
|
||||
sampling_params_no_min = SamplingParams(
|
||||
ignore_eos=False,
|
||||
max_tokens=20,
|
||||
@@ -899,7 +909,7 @@ def test_check_stop_min_tokens():
|
||||
assert result is True, "Should stop on EOS when min_tokens=0"
|
||||
assert request_no_min.status == RequestStatus.FINISHED_STOPPED
|
||||
|
||||
# Test case 4: min_tokens > 0 with stop token (not EOS)
|
||||
# Test case 5: if a stop token is sampled, it should stop immediately.
|
||||
sampling_params_stop = SamplingParams(
|
||||
ignore_eos=False,
|
||||
max_tokens=20,
|
||||
@@ -913,22 +923,67 @@ def test_check_stop_min_tokens():
|
||||
sampling_params=sampling_params_stop,
|
||||
pooling_params=None,
|
||||
)
|
||||
# Only 3 output tokens, less than min_tokens=5, but has stop token
|
||||
request_stop.append_output_token_ids([10, 11, 42])
|
||||
result = check_stop(request_stop, max_model_len=100)
|
||||
assert result is False, "Should not stop when num_output_tokens<min_tokens"
|
||||
|
||||
# Test case 5: min_tokens met, should stop on stop token
|
||||
request_stop.append_output_token_ids(
|
||||
[10, 11, 12, 13, 14, 42]
|
||||
) # 6 tokens >= min_tokens=5
|
||||
|
||||
result = check_stop(request_stop, max_model_len=100)
|
||||
assert result is True, "Should stop on stop token when min_tokens met"
|
||||
assert result is True, "Sampled stop tokens should stop even below min_tokens"
|
||||
assert request_stop.status == RequestStatus.FINISHED_STOPPED
|
||||
assert request_stop.stop_reason == 42
|
||||
|
||||
|
||||
def test_check_stop_length_cap_below_min_tokens():
|
||||
"""Length caps must finish a request even below min_tokens.
|
||||
|
||||
Guards against requests hanging forever when min_tokens cannot be
|
||||
satisfied: min_tokens only defers EOS/stop tokens, so a request that
|
||||
reaches max_model_len (or a max_tokens clamped below min_tokens) must
|
||||
still finish as FINISHED_LENGTH_CAPPED instead of spinning in the
|
||||
scheduler with zero new tokens to schedule.
|
||||
"""
|
||||
from vllm.v1.core.sched.utils import check_stop
|
||||
|
||||
# Reaching max_model_len below min_tokens must stop the request.
|
||||
sampling_params = SamplingParams(
|
||||
ignore_eos=True,
|
||||
max_tokens=500,
|
||||
min_tokens=500,
|
||||
)
|
||||
sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)
|
||||
request = Request(
|
||||
request_id="0",
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
)
|
||||
# 7 output tokens: num_tokens == 10 == max_model_len, min_tokens unmet.
|
||||
request.append_output_token_ids([10, 11, 12, 13, 14, 15, 16])
|
||||
|
||||
result = check_stop(request, max_model_len=10)
|
||||
assert result is True, "Should stop at max_model_len even below min_tokens"
|
||||
assert request.status == RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
|
||||
# Reaching max_tokens below min_tokens must stop the request. This can
|
||||
# happen when max_tokens is clamped to the context window after
|
||||
# SamplingParams validation (e.g. max_tokens=None on a long prompt).
|
||||
sampling_params_capped = SamplingParams(
|
||||
ignore_eos=True,
|
||||
max_tokens=500,
|
||||
min_tokens=500,
|
||||
)
|
||||
sampling_params_capped.max_tokens = 3
|
||||
sampling_params_capped.update_from_generation_config({}, EOS_TOKEN_ID)
|
||||
request_capped = Request(
|
||||
request_id="1",
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=sampling_params_capped,
|
||||
pooling_params=None,
|
||||
)
|
||||
request_capped.append_output_token_ids([10, 11, 12])
|
||||
|
||||
result = check_stop(request_capped, max_model_len=100)
|
||||
assert result is True, "Should stop at max_tokens even below min_tokens"
|
||||
assert request_capped.status == RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enable_prefix_caching, prompt_logprobs",
|
||||
[
|
||||
|
||||
@@ -13,11 +13,14 @@ Covers:
|
||||
5) Multiple stop conditions
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
|
||||
# Test configuration
|
||||
TEST_MODEL = "facebook/opt-125m" # Small model for fast CI execution
|
||||
@@ -168,6 +171,27 @@ def llm_v1():
|
||||
return llm
|
||||
|
||||
|
||||
def test_min_tokens_with_terminal_structured_output(llm_v1: LLM):
|
||||
"""A grammar whose only legal continuation is EOS must stop under
|
||||
min_tokens instead of sampling outside the grammar."""
|
||||
params = SamplingParams(
|
||||
temperature=GREEDY,
|
||||
min_tokens=20,
|
||||
max_tokens=40,
|
||||
structured_outputs=StructuredOutputsParams(
|
||||
json=json.dumps({"type": "boolean"})
|
||||
),
|
||||
)
|
||||
|
||||
output = llm_v1.generate("Answer:", params)[0].outputs[0]
|
||||
eos_token_id = llm_v1.get_tokenizer().eos_token_id
|
||||
|
||||
assert output.finish_reason == "stop"
|
||||
assert isinstance(json.loads(output.text), bool)
|
||||
assert output.token_ids[-1] == eos_token_id
|
||||
assert len(output.token_ids) < params.min_tokens
|
||||
|
||||
|
||||
def get_token_count(output: RequestOutput) -> int:
|
||||
"""Extract token count from LLM output"""
|
||||
if not output.outputs:
|
||||
|
||||
@@ -20,7 +20,11 @@ from tests.v1.sample.utils import (
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import SamplingParams, validate_thinking_token_budget
|
||||
from vllm.sampling_params import (
|
||||
SamplingParams,
|
||||
StructuredOutputsParams,
|
||||
validate_thinking_token_budget,
|
||||
)
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.v1.sample.logits_processor import (
|
||||
BatchUpdate,
|
||||
@@ -371,7 +375,7 @@ def _min_tokens_validate(
|
||||
|
||||
# Validate min-token logits processor state
|
||||
if min_tok:
|
||||
(_, out_tok, all_stop_token_ids) = min_tok
|
||||
(_, out_tok, all_stop_token_ids, _) = min_tok
|
||||
num_out_tokens = len(out_tok)
|
||||
if num_out_tokens != ref_num_out_tokens:
|
||||
_raise_error_invalid(
|
||||
@@ -448,6 +452,113 @@ def _min_tokens_validate(
|
||||
)
|
||||
|
||||
|
||||
def _make_min_tokens_processor() -> MinTokensLogitsProcessor:
|
||||
return MinTokensLogitsProcessor(VllmConfig(), torch.device("cpu"), False)
|
||||
|
||||
|
||||
# Constructing the processor allocates pinned host memory, which creates a
|
||||
# CUDA driver context even though the tensors stay on CPU. Run these in their
|
||||
# own process so the pytest process can still fork for `test_logitsprocs`.
|
||||
@create_new_process_for_each_test()
|
||||
def test_min_tokens_keeps_all_masked_behavior_without_structured_output():
|
||||
processor = _make_min_tokens_processor()
|
||||
processor.update_state(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
0,
|
||||
SamplingParams(min_tokens=2, stop_token_ids=[0]),
|
||||
None,
|
||||
[],
|
||||
)
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
|
||||
logits = torch.full((1, 3), -float("inf"))
|
||||
logits[0, 0] = 1.0
|
||||
|
||||
processor.apply(logits)
|
||||
|
||||
assert torch.isneginf(logits).all()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_min_tokens_restores_all_masked_structured_output_stop_token():
|
||||
processor = _make_min_tokens_processor()
|
||||
processor.update_state(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
0,
|
||||
SamplingParams(
|
||||
min_tokens=2,
|
||||
stop_token_ids=[0],
|
||||
structured_outputs=StructuredOutputsParams(
|
||||
json={"type": "boolean"}
|
||||
),
|
||||
),
|
||||
None,
|
||||
[],
|
||||
)
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
|
||||
logits = torch.full((1, 3), -float("inf"))
|
||||
logits[0, 0] = 1.0
|
||||
|
||||
processor.apply(logits)
|
||||
|
||||
assert logits[0, 0] == 1.0
|
||||
assert torch.isneginf(logits[0, 1:]).all()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_min_tokens_restores_all_masked_structured_output_stop_token_spec_decode():
|
||||
processor = _make_min_tokens_processor()
|
||||
processor.update_state(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
0,
|
||||
SamplingParams(
|
||||
min_tokens=2,
|
||||
stop_token_ids=[0, 1],
|
||||
structured_outputs=StructuredOutputsParams(
|
||||
json={"type": "boolean"}
|
||||
),
|
||||
),
|
||||
None,
|
||||
[],
|
||||
)
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
|
||||
logits = torch.full((2, 4), -float("inf"))
|
||||
logits[0, 0] = 1.0
|
||||
logits[0, 1] = 2.0
|
||||
logits[1, 0] = 3.0
|
||||
|
||||
processor.apply_with_spec_decode(logits, [2])
|
||||
|
||||
assert logits[0, 0] == 1.0
|
||||
assert logits[0, 1] == 2.0
|
||||
assert logits[1, 0] == 3.0
|
||||
assert torch.isneginf(logits[1, 1:]).all()
|
||||
assert torch.isneginf(logits[:, 2:]).all()
|
||||
|
||||
|
||||
def _thinking_budget_params(kwargs: dict) -> None:
|
||||
"""Set SamplingParams kwargs for thinking token budget tests"""
|
||||
kwargs["thinking_token_budget"] = THINKING_TOKEN_BUDGET
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
pytest.importorskip("triton")
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip(
|
||||
"CUDA required for Model Runner V2 logit bias tests",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.sampling_params import SamplingParams, StructuredOutputsParams
|
||||
from vllm.v1.worker.gpu.sample.logit_bias import LogitBiasState
|
||||
|
||||
DEVICE = torch.device("cuda")
|
||||
VOCAB_SIZE = 128
|
||||
STOP_TOKEN = 3
|
||||
OTHER_TOKEN = 4
|
||||
PROMPT_LEN = 2
|
||||
MIN_TOKENS = 10
|
||||
# min_len is PROMPT_LEN + MIN_TOKENS, so this position is below the minimum
|
||||
# and the kernel masks stop tokens.
|
||||
POS = 5
|
||||
|
||||
|
||||
def _params(structured: bool) -> SamplingParams:
|
||||
return SamplingParams(
|
||||
min_tokens=MIN_TOKENS,
|
||||
stop_token_ids=[STOP_TOKEN],
|
||||
structured_outputs=(
|
||||
StructuredOutputsParams(json={"type": "boolean"}) if structured else None
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _only_stop_token_left(num_rows: int) -> torch.Tensor:
|
||||
"""Logits as a terminal grammar leaves them: every token masked but one."""
|
||||
logits = torch.full((num_rows, VOCAB_SIZE), -float("inf"), device=DEVICE)
|
||||
logits[:, STOP_TOKEN] = 1.0
|
||||
return logits
|
||||
|
||||
|
||||
def _apply(logits: torch.Tensor, structured: list[bool]) -> torch.Tensor:
|
||||
"""Run the fused bias kernel with one logits row per request."""
|
||||
state = LogitBiasState(max_num_reqs=4, device=DEVICE)
|
||||
for req_idx, is_structured in enumerate(structured):
|
||||
state.add_request(req_idx, PROMPT_LEN, _params(is_structured))
|
||||
state.apply_staged_writes()
|
||||
|
||||
num_reqs = len(structured)
|
||||
state.apply_logit_bias(
|
||||
logits,
|
||||
torch.arange(num_reqs, dtype=torch.int32, device=DEVICE),
|
||||
np.arange(num_reqs, dtype=np.intp),
|
||||
torch.full((num_reqs,), POS, dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
return logits.cpu()
|
||||
|
||||
|
||||
def test_v2_min_tokens_restores_stop_token_when_row_would_be_empty():
|
||||
"""Structured output plus min_tokens must not leave an all -inf row."""
|
||||
out = _apply(_only_stop_token_left(1), structured=[True])
|
||||
assert out[0, STOP_TOKEN] == 1.0
|
||||
|
||||
|
||||
def test_v2_min_tokens_keeps_empty_row_without_structured_output():
|
||||
"""Plain min_tokens requests keep the pre-existing masking behavior."""
|
||||
out = _apply(_only_stop_token_left(1), structured=[False])
|
||||
assert torch.isneginf(out).all()
|
||||
|
||||
|
||||
def test_v2_min_tokens_keeps_masking_when_other_candidates_remain():
|
||||
"""The fallback must not weaken min_tokens while a non-stop token is legal."""
|
||||
logits = _only_stop_token_left(1)
|
||||
logits[0, OTHER_TOKEN] = 2.0
|
||||
|
||||
out = _apply(logits, structured=[True])
|
||||
|
||||
assert torch.isneginf(out[0, STOP_TOKEN])
|
||||
assert out[0, OTHER_TOKEN] == 2.0
|
||||
|
||||
|
||||
def test_v2_min_tokens_mixed_batch_gates_restore_per_request():
|
||||
"""CHECK_ALL_MASKED_ROWS is a batch-wide constexpr, so a mixed batch must
|
||||
still consult the per-request flag."""
|
||||
out = _apply(_only_stop_token_left(2), structured=[True, False])
|
||||
|
||||
assert out[0, STOP_TOKEN] == 1.0
|
||||
assert torch.isneginf(out[1]).all()
|
||||
Reference in New Issue
Block a user