[Structured Output] Let terminal grammars stop under min_tokens (II) (#54218)
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
@@ -840,11 +840,10 @@ def test_stop_via_update_from_output():
|
||||
|
||||
|
||||
def test_check_stop_min_tokens():
|
||||
"""Test that requests don't stop when min_tokens requirement isn't met."""
|
||||
"""Test that sampled stop tokens still finish under min_tokens."""
|
||||
from vllm.v1.core.sched.utils import check_stop
|
||||
|
||||
# Test case 1: num_output_tokens < min_tokens
|
||||
# Should return False (don't stop)
|
||||
# Test case 1: regular tokens below min_tokens should keep running.
|
||||
sampling_params = SamplingParams(
|
||||
ignore_eos=False,
|
||||
max_tokens=20,
|
||||
@@ -857,30 +856,41 @@ def test_check_stop_min_tokens():
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
)
|
||||
# Simulate having generated 3 output tokens (less than min_tokens=5)
|
||||
request.append_output_token_ids([10, 11, EOS_TOKEN_ID]) # EOS token present
|
||||
request.append_output_token_ids([10, 11, 12])
|
||||
|
||||
result = check_stop(request, max_model_len=100)
|
||||
assert result is False, "Should not stop when num_output_tokens<min_tokens"
|
||||
|
||||
# Test case 2: num_output_tokens >= min_tokens
|
||||
# Should follow normal stopping logic (stop on EOS)
|
||||
request.append_output_token_ids(
|
||||
[
|
||||
10,
|
||||
11,
|
||||
12,
|
||||
13,
|
||||
14,
|
||||
EOS_TOKEN_ID,
|
||||
]
|
||||
) # 6 tokens > min_tokens
|
||||
# Test case 2: if EOS is sampled, it should stop even below min_tokens.
|
||||
request_eos = Request(
|
||||
request_id="eos",
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
)
|
||||
request_eos.append_output_token_ids([10, 11, EOS_TOKEN_ID])
|
||||
result = check_stop(request_eos, max_model_len=100)
|
||||
assert result is True, "Sampled EOS should stop even below min_tokens"
|
||||
assert request_eos.status == RequestStatus.FINISHED_STOPPED
|
||||
|
||||
result = check_stop(request, max_model_len=100)
|
||||
assert result is True, "Should stop on EOS when min_tokens met"
|
||||
assert request.status == RequestStatus.FINISHED_STOPPED
|
||||
# Test case 3: ignore_eos should still prevent EOS-based stopping.
|
||||
sampling_params_ignore_eos = SamplingParams(
|
||||
ignore_eos=True,
|
||||
max_tokens=20,
|
||||
min_tokens=5,
|
||||
)
|
||||
sampling_params_ignore_eos.update_from_generation_config({}, EOS_TOKEN_ID)
|
||||
request_ignore_eos = Request(
|
||||
request_id="ignore-eos",
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=sampling_params_ignore_eos,
|
||||
pooling_params=None,
|
||||
)
|
||||
request_ignore_eos.append_output_token_ids([EOS_TOKEN_ID])
|
||||
result = check_stop(request_ignore_eos, max_model_len=100)
|
||||
assert result is False, "ignore_eos should still suppress EOS stopping"
|
||||
|
||||
# Test case 3: min_tokens = 0, should follow normal stopping logic
|
||||
# Test case 4: min_tokens = 0 should follow normal stopping logic.
|
||||
sampling_params_no_min = SamplingParams(
|
||||
ignore_eos=False,
|
||||
max_tokens=20,
|
||||
@@ -899,7 +909,7 @@ def test_check_stop_min_tokens():
|
||||
assert result is True, "Should stop on EOS when min_tokens=0"
|
||||
assert request_no_min.status == RequestStatus.FINISHED_STOPPED
|
||||
|
||||
# Test case 4: min_tokens > 0 with stop token (not EOS)
|
||||
# Test case 5: if a stop token is sampled, it should stop immediately.
|
||||
sampling_params_stop = SamplingParams(
|
||||
ignore_eos=False,
|
||||
max_tokens=20,
|
||||
@@ -913,22 +923,67 @@ def test_check_stop_min_tokens():
|
||||
sampling_params=sampling_params_stop,
|
||||
pooling_params=None,
|
||||
)
|
||||
# Only 3 output tokens, less than min_tokens=5, but has stop token
|
||||
request_stop.append_output_token_ids([10, 11, 42])
|
||||
result = check_stop(request_stop, max_model_len=100)
|
||||
assert result is False, "Should not stop when num_output_tokens<min_tokens"
|
||||
|
||||
# Test case 5: min_tokens met, should stop on stop token
|
||||
request_stop.append_output_token_ids(
|
||||
[10, 11, 12, 13, 14, 42]
|
||||
) # 6 tokens >= min_tokens=5
|
||||
|
||||
result = check_stop(request_stop, max_model_len=100)
|
||||
assert result is True, "Should stop on stop token when min_tokens met"
|
||||
assert result is True, "Sampled stop tokens should stop even below min_tokens"
|
||||
assert request_stop.status == RequestStatus.FINISHED_STOPPED
|
||||
assert request_stop.stop_reason == 42
|
||||
|
||||
|
||||
def test_check_stop_length_cap_below_min_tokens():
|
||||
"""Length caps must finish a request even below min_tokens.
|
||||
|
||||
Guards against requests hanging forever when min_tokens cannot be
|
||||
satisfied: min_tokens only defers EOS/stop tokens, so a request that
|
||||
reaches max_model_len (or a max_tokens clamped below min_tokens) must
|
||||
still finish as FINISHED_LENGTH_CAPPED instead of spinning in the
|
||||
scheduler with zero new tokens to schedule.
|
||||
"""
|
||||
from vllm.v1.core.sched.utils import check_stop
|
||||
|
||||
# Reaching max_model_len below min_tokens must stop the request.
|
||||
sampling_params = SamplingParams(
|
||||
ignore_eos=True,
|
||||
max_tokens=500,
|
||||
min_tokens=500,
|
||||
)
|
||||
sampling_params.update_from_generation_config({}, EOS_TOKEN_ID)
|
||||
request = Request(
|
||||
request_id="0",
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=sampling_params,
|
||||
pooling_params=None,
|
||||
)
|
||||
# 7 output tokens: num_tokens == 10 == max_model_len, min_tokens unmet.
|
||||
request.append_output_token_ids([10, 11, 12, 13, 14, 15, 16])
|
||||
|
||||
result = check_stop(request, max_model_len=10)
|
||||
assert result is True, "Should stop at max_model_len even below min_tokens"
|
||||
assert request.status == RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
|
||||
# Reaching max_tokens below min_tokens must stop the request. This can
|
||||
# happen when max_tokens is clamped to the context window after
|
||||
# SamplingParams validation (e.g. max_tokens=None on a long prompt).
|
||||
sampling_params_capped = SamplingParams(
|
||||
ignore_eos=True,
|
||||
max_tokens=500,
|
||||
min_tokens=500,
|
||||
)
|
||||
sampling_params_capped.max_tokens = 3
|
||||
sampling_params_capped.update_from_generation_config({}, EOS_TOKEN_ID)
|
||||
request_capped = Request(
|
||||
request_id="1",
|
||||
prompt_token_ids=[0, 1, 2],
|
||||
sampling_params=sampling_params_capped,
|
||||
pooling_params=None,
|
||||
)
|
||||
request_capped.append_output_token_ids([10, 11, 12])
|
||||
|
||||
result = check_stop(request_capped, max_model_len=100)
|
||||
assert result is True, "Should stop at max_tokens even below min_tokens"
|
||||
assert request_capped.status == RequestStatus.FINISHED_LENGTH_CAPPED
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"enable_prefix_caching, prompt_logprobs",
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user