[Structured Output] Let terminal grammars stop under min_tokens (II) (#54218)
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
@@ -20,7 +20,11 @@ from tests.v1.sample.utils import (
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sampling_params import SamplingParams, validate_thinking_token_budget
|
||||
from vllm.sampling_params import (
|
||||
SamplingParams,
|
||||
StructuredOutputsParams,
|
||||
validate_thinking_token_budget,
|
||||
)
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.v1.sample.logits_processor import (
|
||||
BatchUpdate,
|
||||
@@ -371,7 +375,7 @@ def _min_tokens_validate(
|
||||
|
||||
# Validate min-token logits processor state
|
||||
if min_tok:
|
||||
(_, out_tok, all_stop_token_ids) = min_tok
|
||||
(_, out_tok, all_stop_token_ids, _) = min_tok
|
||||
num_out_tokens = len(out_tok)
|
||||
if num_out_tokens != ref_num_out_tokens:
|
||||
_raise_error_invalid(
|
||||
@@ -448,6 +452,113 @@ def _min_tokens_validate(
|
||||
)
|
||||
|
||||
|
||||
def _make_min_tokens_processor() -> MinTokensLogitsProcessor:
|
||||
return MinTokensLogitsProcessor(VllmConfig(), torch.device("cpu"), False)
|
||||
|
||||
|
||||
# Constructing the processor allocates pinned host memory, which creates a
|
||||
# CUDA driver context even though the tensors stay on CPU. Run these in their
|
||||
# own process so the pytest process can still fork for `test_logitsprocs`.
|
||||
@create_new_process_for_each_test()
|
||||
def test_min_tokens_keeps_all_masked_behavior_without_structured_output():
|
||||
processor = _make_min_tokens_processor()
|
||||
processor.update_state(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
0,
|
||||
SamplingParams(min_tokens=2, stop_token_ids=[0]),
|
||||
None,
|
||||
[],
|
||||
)
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
|
||||
logits = torch.full((1, 3), -float("inf"))
|
||||
logits[0, 0] = 1.0
|
||||
|
||||
processor.apply(logits)
|
||||
|
||||
assert torch.isneginf(logits).all()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_min_tokens_restores_all_masked_structured_output_stop_token():
|
||||
processor = _make_min_tokens_processor()
|
||||
processor.update_state(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
0,
|
||||
SamplingParams(
|
||||
min_tokens=2,
|
||||
stop_token_ids=[0],
|
||||
structured_outputs=StructuredOutputsParams(
|
||||
json={"type": "boolean"}
|
||||
),
|
||||
),
|
||||
None,
|
||||
[],
|
||||
)
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
|
||||
logits = torch.full((1, 3), -float("inf"))
|
||||
logits[0, 0] = 1.0
|
||||
|
||||
processor.apply(logits)
|
||||
|
||||
assert logits[0, 0] == 1.0
|
||||
assert torch.isneginf(logits[0, 1:]).all()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_min_tokens_restores_all_masked_structured_output_stop_token_spec_decode():
|
||||
processor = _make_min_tokens_processor()
|
||||
processor.update_state(
|
||||
BatchUpdate(
|
||||
batch_size=1,
|
||||
removed=(),
|
||||
added=[
|
||||
(
|
||||
0,
|
||||
SamplingParams(
|
||||
min_tokens=2,
|
||||
stop_token_ids=[0, 1],
|
||||
structured_outputs=StructuredOutputsParams(
|
||||
json={"type": "boolean"}
|
||||
),
|
||||
),
|
||||
None,
|
||||
[],
|
||||
)
|
||||
],
|
||||
moved=(),
|
||||
)
|
||||
)
|
||||
|
||||
logits = torch.full((2, 4), -float("inf"))
|
||||
logits[0, 0] = 1.0
|
||||
logits[0, 1] = 2.0
|
||||
logits[1, 0] = 3.0
|
||||
|
||||
processor.apply_with_spec_decode(logits, [2])
|
||||
|
||||
assert logits[0, 0] == 1.0
|
||||
assert logits[0, 1] == 2.0
|
||||
assert logits[1, 0] == 3.0
|
||||
assert torch.isneginf(logits[1, 1:]).all()
|
||||
assert torch.isneginf(logits[:, 2:]).all()
|
||||
|
||||
|
||||
def _thinking_budget_params(kwargs: dict) -> None:
|
||||
"""Set SamplingParams kwargs for thinking token budget tests"""
|
||||
kwargs["thinking_token_budget"] = THINKING_TOKEN_BUDGET
|
||||
|
||||
Reference in New Issue
Block a user