[Structured Output] Let terminal grammars stop under min_tokens (II) (#54218)

Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
Artem Perevedentsev
2026-08-31 20:39:27 +04:00
committed by GitHub
parent e9dd6d4834
commit 39e276eaeb
7 changed files with 501 additions and 60 deletions
+113 -2
View File
@@ -20,7 +20,11 @@ from tests.v1.sample.utils import (
)
from vllm.config import VllmConfig
from vllm.platforms import current_platform
from vllm.sampling_params import SamplingParams, validate_thinking_token_budget
from vllm.sampling_params import (
SamplingParams,
StructuredOutputsParams,
validate_thinking_token_budget,
)
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.v1.sample.logits_processor import (
BatchUpdate,
@@ -371,7 +375,7 @@ def _min_tokens_validate(
# Validate min-token logits processor state
if min_tok:
(_, out_tok, all_stop_token_ids) = min_tok
(_, out_tok, all_stop_token_ids, _) = min_tok
num_out_tokens = len(out_tok)
if num_out_tokens != ref_num_out_tokens:
_raise_error_invalid(
@@ -448,6 +452,113 @@ def _min_tokens_validate(
)
def _make_min_tokens_processor() -> MinTokensLogitsProcessor:
return MinTokensLogitsProcessor(VllmConfig(), torch.device("cpu"), False)
# Constructing the processor allocates pinned host memory, which creates a
# CUDA driver context even though the tensors stay on CPU. Run these in their
# own process so the pytest process can still fork for `test_logitsprocs`.
@create_new_process_for_each_test()
def test_min_tokens_keeps_all_masked_behavior_without_structured_output():
processor = _make_min_tokens_processor()
processor.update_state(
BatchUpdate(
batch_size=1,
removed=(),
added=[
(
0,
SamplingParams(min_tokens=2, stop_token_ids=[0]),
None,
[],
)
],
moved=(),
)
)
logits = torch.full((1, 3), -float("inf"))
logits[0, 0] = 1.0
processor.apply(logits)
assert torch.isneginf(logits).all()
@create_new_process_for_each_test()
def test_min_tokens_restores_all_masked_structured_output_stop_token():
processor = _make_min_tokens_processor()
processor.update_state(
BatchUpdate(
batch_size=1,
removed=(),
added=[
(
0,
SamplingParams(
min_tokens=2,
stop_token_ids=[0],
structured_outputs=StructuredOutputsParams(
json={"type": "boolean"}
),
),
None,
[],
)
],
moved=(),
)
)
logits = torch.full((1, 3), -float("inf"))
logits[0, 0] = 1.0
processor.apply(logits)
assert logits[0, 0] == 1.0
assert torch.isneginf(logits[0, 1:]).all()
@create_new_process_for_each_test()
def test_min_tokens_restores_all_masked_structured_output_stop_token_spec_decode():
processor = _make_min_tokens_processor()
processor.update_state(
BatchUpdate(
batch_size=1,
removed=(),
added=[
(
0,
SamplingParams(
min_tokens=2,
stop_token_ids=[0, 1],
structured_outputs=StructuredOutputsParams(
json={"type": "boolean"}
),
),
None,
[],
)
],
moved=(),
)
)
logits = torch.full((2, 4), -float("inf"))
logits[0, 0] = 1.0
logits[0, 1] = 2.0
logits[1, 0] = 3.0
processor.apply_with_spec_decode(logits, [2])
assert logits[0, 0] == 1.0
assert logits[0, 1] == 2.0
assert logits[1, 0] == 3.0
assert torch.isneginf(logits[1, 1:]).all()
assert torch.isneginf(logits[:, 2:]).all()
def _thinking_budget_params(kwargs: dict) -> None:
"""Set SamplingParams kwargs for thinking token budget tests"""
kwargs["thinking_token_budget"] = THINKING_TOKEN_BUDGET