[Structured Output] Let terminal grammars stop under min_tokens (II) (#54218)
Signed-off-by: Artem Perevedentsev <aperevedents@nvidia.com>
This commit is contained in:
@@ -13,11 +13,14 @@ Covers:
|
||||
5) Multiple stop conditions
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sampling_params import StructuredOutputsParams
|
||||
|
||||
# Test configuration
|
||||
TEST_MODEL = "facebook/opt-125m" # Small model for fast CI execution
|
||||
@@ -168,6 +171,27 @@ def llm_v1():
|
||||
return llm
|
||||
|
||||
|
||||
def test_min_tokens_with_terminal_structured_output(llm_v1: LLM):
|
||||
"""A grammar whose only legal continuation is EOS must stop under
|
||||
min_tokens instead of sampling outside the grammar."""
|
||||
params = SamplingParams(
|
||||
temperature=GREEDY,
|
||||
min_tokens=20,
|
||||
max_tokens=40,
|
||||
structured_outputs=StructuredOutputsParams(
|
||||
json=json.dumps({"type": "boolean"})
|
||||
),
|
||||
)
|
||||
|
||||
output = llm_v1.generate("Answer:", params)[0].outputs[0]
|
||||
eos_token_id = llm_v1.get_tokenizer().eos_token_id
|
||||
|
||||
assert output.finish_reason == "stop"
|
||||
assert isinstance(json.loads(output.text), bool)
|
||||
assert output.token_ids[-1] == eos_token_id
|
||||
assert len(output.token_ids) < params.min_tokens
|
||||
|
||||
|
||||
def get_token_count(output: RequestOutput) -> int:
|
||||
"""Extract token count from LLM output"""
|
||||
if not output.outputs:
|
||||
|
||||
Reference in New Issue
Block a user