[Structured Output] Keep invalid structured-output requests from stopping the engine (#51450)
Co-authored-by: Lucas Bourtoule <35483370+dhalf@users.noreply.github.com> Co-authored-by: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com>
This commit is contained in:
co-authored by
Lucas Bourtoule
Vadim Gimpelson
parent
7e95735eb9
commit
c55e15a44e
@@ -21,6 +21,7 @@ from vllm.v1.structured_output.backend_types import (
|
||||
StructuredOutputOptions,
|
||||
)
|
||||
from vllm.v1.structured_output.request import get_structured_output_key
|
||||
from vllm.v1.structured_output.utils import strip_speculative_padding
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import llguidance
|
||||
@@ -198,6 +199,10 @@ class GuidanceGrammar(StructuredOutputGrammar):
|
||||
if self.ll_matcher.is_stopped():
|
||||
return []
|
||||
|
||||
tokens = strip_speculative_padding(tokens)
|
||||
if len(tokens) == 0:
|
||||
return []
|
||||
|
||||
num_tokens = self.ll_matcher.validate_tokens(tokens)
|
||||
|
||||
self.check_error()
|
||||
|
||||
@@ -46,6 +46,20 @@ _T = TypeVar("_T")
|
||||
CACHE = None
|
||||
|
||||
|
||||
def strip_speculative_padding(token_ids: list[int]) -> list[int]:
|
||||
"""Drop speculative-decoding padding from a token block.
|
||||
|
||||
ngram and other speculative backends pad rejected draft positions with a
|
||||
-1 sentinel. Structured-output grammars treat every entry as a real token
|
||||
id, so the sentinels (and everything after the first one) are removed here,
|
||||
before tokens reach any backend, rather than inside a single backend.
|
||||
"""
|
||||
for i, token_id in enumerate(token_ids):
|
||||
if token_id < 0:
|
||||
return token_ids[:i]
|
||||
return token_ids
|
||||
|
||||
|
||||
def compile_regex_with_timeout(fn: Callable[[str], _T], pattern: str) -> _T:
|
||||
"""Run a regex compilation callable with a timeout.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user