[Structured Output] Let terminal grammars stop under min_tokens (II) (#54218)

Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
Artem Perevedentsev
2026-08-31 20:39:27 +04:00
committed by GitHub
parent e9dd6d4834
commit 39e276eaeb
7 changed files with 501 additions and 60 deletions
+87 -32
View File
@@ -840,11 +840,10 @@ def test_stop_via_update_from_output():
def test_check_stop_min_tokens():
"""Test that requests don't stop when min_tokens requirement isn't met."""
"""Test that sampled stop tokens still finish under min_tokens."""
from vllm.v1.core.sched.utils import check_stop
# Test case 1: num_output_tokens < min_tokens
# Should return False (don't stop)
# Test case 1: regular tokens below min_tokens should keep running.
sampling_params = SamplingParams(
ignore_eos=False,
max_tokens=20,
@@ -857,30 +856,41 @@ def test_check_stop_min_tokens():
sampling_params=sampling_params,
pooling_params=None,
)
# Simulate having generated 3 output tokens (less than min_tokens=5)
request.append_output_token_ids([10, 11, EOS_TOKEN_ID]) # EOS token present
request.append_output_token_ids([10, 11, 12])
result = check_stop(request, max_model_len=100)
assert result is False, "Should not stop when num_output_tokens<min_tokens"
# Test case 2: num_output_tokens >= min_tokens
# Should follow normal stopping logic (stop on EOS)
request.append_output_token_ids(
[
10,
11,
12,
13,
14,
EOS_TOKEN_ID,
]
) # 6 tokens > min_tokens
# Test case 2: if EOS is sampled, it should stop even below min_tokens.
request_eos = Request(
request_id="eos",
prompt_token_ids=[0, 1, 2],
sampling_params=sampling_params,
pooling_params=None,
)
request_eos.append_output_token_ids([10, 11, EOS_TOKEN_ID])
result = check_stop(request_eos, max_model_len=100)
assert result is True, "Sampled EOS should stop even below min_tokens"
assert request_eos.status == RequestStatus.FINISHED_STOPPED
result = check_stop(request, max_model_len=100)
assert result is True, "Should stop on EOS when min_tokens met"
assert request.status == RequestStatus.FINISHED_STOPPED
# Test case 3: ignore_eos should still prevent EOS-based stopping.
sampling_params_ignore_eos = SamplingParams(
ignore_eos=True,
max_tokens=20,
min_tokens=5,
)
sampling_params_ignore_eos.update_from_generation_config({}, EOS_TOKEN_ID)
request_ignore_eos = Request(
request_id="ignore-eos",
prompt_token_ids=[0, 1, 2],
sampling_params=sampling_params_ignore_eos,
pooling_params=None,
)
request_ignore_eos.append_output_token_ids([EOS_TOKEN_ID])
result = check_stop(request_ignore_eos, max_model_len=100)
assert result is False, "ignore_eos should still suppress EOS stopping"
# Test case 3: min_tokens = 0, should follow normal stopping logic
# Test case 4: min_tokens = 0 should follow normal stopping logic.
sampling_params_no_min = SamplingParams(
ignore_eos=False,
max_tokens=20,
@@ -899,7 +909,7 @@ def test_check_stop_min_tokens():
assert result is True, "Should stop on EOS when min_tokens=0"
assert request_no_min.status == RequestStatus.FINISHED_STOPPED
# Test case 4: min_tokens > 0 with stop token (not EOS)
# Test case 5: if a stop token is sampled, it should stop immediately.
sampling_params_stop = SamplingParams(
ignore_eos=False,
max_tokens=20,
@@ -913,22 +923,67 @@ def test_check_stop_min_tokens():
sampling_params=sampling_params_stop,
pooling_params=None,
)
# Only 3 output tokens, less than min_tokens=5, but has stop token
request_stop.append_output_token_ids([10, 11, 42])
result = check_stop(request_stop, max_model_len=100)
assert result is False, "Should not stop when num_output_tokens<min_tokens"
# Test case 5: min_tokens met, should stop on stop token
request_stop.append_output_token_ids(
[10, 11, 12, 13, 14, 42]
) # 6 tokens >= min_tokens=5
result = check_stop(request_stop, max_model_len=100)
assert result is True, "Should stop on stop token when min_tokens met"
assert result is True, "Sampled stop tokens should stop even below min_tokens"
assert request_stop.status == RequestStatus.FINISHED_STOPPED
assert request_stop.stop_reason == 42
def test_check_stop_length_cap_below_min_tokens():
"""Length caps must finish a request even below min_tokens.
Guards against requests hanging forever when min_tokens cannot be
satisfied: min_tokens only defers EOS/stop tokens, so a request that
reaches max_model_len (or a max_tokens clamped below min_tokens) must
still finish as FINISHED_LENGTH_CAPPED instead of spinning in the
scheduler with zero new tokens to schedule.
"""
from vllm.v1.core.sched.utils import check_stop
# Reaching max_model_len below min_tokens must stop the request.
sampling_params = SamplingParams(
ignore_eos=True,
max_tokens=500,
min_tokens=500,
)
sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)
request = Request(
request_id="0",
prompt_token_ids=[0, 1, 2],
sampling_params=sampling_params,
pooling_params=None,
)
# 7 output tokens: num_tokens == 10 == max_model_len, min_tokens unmet.
request.append_output_token_ids([10, 11, 12, 13, 14, 15, 16])
result = check_stop(request, max_model_len=10)
assert result is True, "Should stop at max_model_len even below min_tokens"
assert request.status == RequestStatus.FINISHED_LENGTH_CAPPED
# Reaching max_tokens below min_tokens must stop the request. This can
# happen when max_tokens is clamped to the context window after
# SamplingParams validation (e.g. max_tokens=None on a long prompt).
sampling_params_capped = SamplingParams(
ignore_eos=True,
max_tokens=500,
min_tokens=500,
)
sampling_params_capped.max_tokens = 3
sampling_params_capped.update_from_generation_config({}, EOS_TOKEN_ID)
request_capped = Request(
request_id="1",
prompt_token_ids=[0, 1, 2],
sampling_params=sampling_params_capped,
pooling_params=None,
)
request_capped.append_output_token_ids([10, 11, 12])
result = check_stop(request_capped, max_model_len=100)
assert result is True, "Should stop at max_tokens even below min_tokens"
assert request_capped.status == RequestStatus.FINISHED_LENGTH_CAPPED
@pytest.mark.parametrize(
"enable_prefix_caching, prompt_logprobs",
[