[Structured Output] Keep invalid structured-output requests from stopping the engine (#51450)

Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com>
Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
This commit is contained in:
Clinton Thomas
2026-09-09 15:32:01 +00:00
committed by GitHub
co-authored by Lucas Bourtoule Vadim Gimpelson
parent 7e95735eb9
commit c55e15a44e
6 changed files with 193 additions and 0 deletions
+90
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
from concurrent.futures import Future
from types import SimpleNamespace
from unittest.mock import Mock
import numpy as np
@@ -6425,3 +6426,92 @@ def test_encoder_input_skipped_when_connector_already_has_the_item(ec_role: str)
assert output.num_scheduled_tokens[req_id] > 0
assert not output.scheduled_encoder_inputs.get(req_id)
class _RecordingGrammar:
"""Records what reaches ``validate_tokens`` and raises on a negative id.
llguidance raises ``OverflowError`` on a negative token id, so this mirrors
the real failure: the tests below fail on the unpatched scheduler (the -1
padding reaches ``validate_tokens``) and pass once it is stripped first.
"""
def __init__(self) -> None:
self.seen: list[list[int]] = []
def validate_tokens(self, tokens: list[int]) -> list[int]:
if any(t < 0 for t in tokens):
raise OverflowError("llguidance rejects negative token ids")
self.seen.append(list(tokens))
return tokens
def _decode_ready_request(scheduler):
"""Add one request and advance it out of prefill into decode."""
request = create_requests(num_requests=1, num_tokens=1)[0]
scheduler.add_request(request)
output = scheduler.schedule()
scheduler.update_from_output(
output,
ModelRunnerOutput(
req_ids=[request.request_id],
req_id_to_index={request.request_id: 0},
sampled_token_ids=[[0]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
return request
def test_update_draft_token_ids_strips_ngram_padding(monkeypatch):
"""ngram_gpu pads unfilled draft slots with -1; update_draft_token_ids must
strip them before grammar.validate_tokens (which otherwise raises)."""
scheduler = create_scheduler(num_speculative_tokens=4)
request = _decode_ready_request(scheduler)
grammar = _RecordingGrammar()
request.structured_output_request = SimpleNamespace(grammar=grammar)
monkeypatch.setattr(
scheduler.structured_output_manager, "should_advance", lambda req: True
)
scheduler.update_draft_token_ids(
DraftTokenIds([request.request_id], [[10, 11, -1, -1]])
)
assert grammar.seen == [[10, 11]]
assert request.spec_token_ids == [10, 11]
def test_update_draft_token_ids_in_output_strips_padding(monkeypatch):
"""Same guard on the output path; the -1 pad-back for the rejected count
is preserved (only the input to validate_tokens is stripped)."""
scheduler = create_scheduler(num_speculative_tokens=4)
request = _decode_ready_request(scheduler)
grammar = _RecordingGrammar()
request.structured_output_request = SimpleNamespace(grammar=grammar)
monkeypatch.setattr(
scheduler.structured_output_manager, "should_advance", lambda req: True
)
scheduler_output = SimpleNamespace(
scheduled_spec_decode_tokens={request.request_id: [0, 0, 0, 0]}
)
scheduler.update_draft_token_ids_in_output(
DraftTokenIds([request.request_id], [[10, 11, -1, -1]]),
scheduler_output,
)
# The grammar only saw the stripped prefix, never a -1.
assert grammar.seen == [[10, 11]]
# Two drafts were rejected (4 scheduled - 2 valid), padded back with -1.
assert scheduler_output.scheduled_spec_decode_tokens[request.request_id] == [
10,
11,
-1,
-1,
]
assert scheduler_output.num_invalid_spec_tokens == {request.request_id: 2}