Files
2026-10-09 21:59:02 +03:00

387 lines
13 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import json
import os
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any
import torch
from transformers import TokenizersBackend
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.sampling_params import SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.utils.import_utils import LazyLoader
from vllm.utils.mistral import is_mistral_tokenizer
from vllm.v1.structured_output.backend_types import (
StructuredOutputBackend,
StructuredOutputGrammar,
StructuredOutputOptions,
)
from vllm.v1.structured_output.request import get_structured_output_key
from vllm.v1.structured_output.utils import strip_speculative_padding
if TYPE_CHECKING:
import llguidance
import llguidance.hf as llguidance_hf
import llguidance.torch as llguidance_torch
else:
llguidance = LazyLoader("llguidance", globals(), "llguidance")
llguidance_hf = LazyLoader("llguidance.hf", globals(), "llguidance.hf")
llguidance_torch = LazyLoader("llguidance.torch", globals(), "llguidance.torch")
logger = init_logger(__name__)
_SCHEMA_MAP_KEYWORDS = (
"properties",
"patternProperties",
"$defs",
"definitions",
"dependentSchemas",
"dependencies",
)
_SUBSCHEMA_KEYWORDS = (
"additionalProperties",
"unevaluatedProperties",
"propertyNames",
"contains",
"additionalItems",
"unevaluatedItems",
"not",
"if",
"then",
"else",
"contentSchema",
"items",
"prefixItems",
"allOf",
"anyOf",
"oneOf",
)
def _walk_json_for_additional_properties(data: object):
if isinstance(data, dict):
for key in _SCHEMA_MAP_KEYWORDS:
value = data.get(key)
if isinstance(value, dict):
for subschema in value.values():
if isinstance(subschema, dict):
_walk_json_for_additional_properties(subschema)
for key in _SUBSCHEMA_KEYWORDS:
value = data.get(key)
if isinstance(value, (dict, list)):
_walk_json_for_additional_properties(value)
if "additionalProperties" not in data and (
isinstance(data.get("properties"), dict)
or isinstance(data.get("patternProperties"), dict)
):
data["additionalProperties"] = False
elif isinstance(data, list):
for item in data:
if isinstance(item, dict):
_walk_json_for_additional_properties(item)
def has_guidance_unsupported_json_features(schema: dict[str, Any]) -> bool:
"""Check if JSON schema contains features unsupported by guidance/llguidance."""
def check_object(obj: dict[str, Any]) -> bool:
if not isinstance(obj, dict):
return False
# patternProperties is not supported by llguidance
if "patternProperties" in obj:
return True
# Recursively check all nested objects and arrays
for value in obj.values():
if isinstance(value, dict):
if check_object(value):
return True
elif isinstance(value, list):
for item in value:
if isinstance(item, dict) and check_object(item):
return True
return False
return check_object(schema)
def process_for_additional_properties(
guide_json: str | dict[str, Any],
) -> dict[str, Any]:
if isinstance(guide_json, str):
guide_json_obj = json.loads(guide_json)
else:
# copy for modifications
guide_json_obj = copy.deepcopy(guide_json)
_walk_json_for_additional_properties(guide_json_obj)
return guide_json_obj
@dataclass
class GuidanceBackend(StructuredOutputBackend):
@staticmethod
def is_tokenizer_supported(tokenizer: TokenizerLike) -> bool:
"""Check if tokenizer is supported by guidance backend.
TODO(arpera):
guidance backend does NOT support some types of tokenizers
so, we MUST verify tokenizer before using guidance
The problem is that guidance API does not provide
a way to validate tokenizer before building LLTokenizer
So, we have to do it manually here in vLLM.
This is ofcourse very bad, we duplicate complex logic
that can change arbitrary in future.
Our check here is based on guidance source code
on function `from_tokenizer` implementation:
https://github.com/guidance-ai/llguidance/blob/main/python/llguidance/hf.py#L29-L47
If you want to contribute to vLLM, please, add to llguidance
API function to implement this check there.
In that case we will remove this check from vLLM.
That kind of work is really appreciated!
"""
if is_mistral_tokenizer(tokenizer):
return tokenizer.is_tekken
# PreTrainedTokenizerFast is the same as TokenizersBackend
return isinstance(tokenizer, TokenizersBackend)
@staticmethod
def _build_ll_tokenizer(tokenizer: TokenizerLike, vocab_size: int) -> Any:
if is_mistral_tokenizer(tokenizer):
return tokenizer.llg_tokenizer
return llguidance_hf.from_tokenizer(tokenizer, max(vocab_size, len(tokenizer)))
def __post_init__(self):
self.disable_any_whitespace = (
self.vllm_config.structured_outputs_config.disable_any_whitespace
)
self.disable_additional_properties = (
self.vllm_config.structured_outputs_config.disable_additional_properties
)
if not self.is_tokenizer_supported(self.tokenizer):
raise ValueError(
"Only fast tokenizers are supported."
"This error should NOT happen,"
"if you see this, please report an issue."
)
self.ll_tokenizer = self._build_ll_tokenizer(self.tokenizer, self.vocab_size)
def compile_grammar(
self,
request_type: StructuredOutputOptions,
grammar_spec: str,
stop_token_ids: set[int] | None = None,
) -> StructuredOutputGrammar:
self.serialized_grammar = serialize_guidance_grammar(
request_type,
grammar_spec,
self.disable_any_whitespace,
self.disable_additional_properties,
)
ll_matcher = llguidance.LLMatcher(
self.ll_tokenizer,
self.serialized_grammar,
log_level=int(os.environ.get("LLGUIDANCE_LOG_LEVEL", "1")),
)
r = GuidanceGrammar(
ll_matcher=ll_matcher,
ll_tokenizer=self.ll_tokenizer,
vocab_size=self.vocab_size,
)
r.check_error()
return r
def allocate_token_bitmask(self, max_num_seqs: int):
return llguidance_torch.allocate_token_bitmask(
max_num_seqs, self.ll_tokenizer.vocab_size
)
def destroy(self):
pass
@dataclass
class GuidanceGrammar(StructuredOutputGrammar):
ll_matcher: llguidance.LLMatcher
ll_tokenizer: llguidance.LLTokenizer
vocab_size: int
printed_error: bool = False
terminated: bool = False
rollback_lag: int = 0
def check_error(self):
if not self.printed_error:
err = self.ll_matcher.get_error()
if err:
self.printed_error = True
logger.warning("LLMatcher error: %s", err)
def accept_tokens(self, request_id: str, tokens: list[int]) -> bool:
"""Accepts a list of tokens and advances the parser.
Returns True if the parser was advanced successfully.
Returns False if the parser failed to advance.
"""
if self.ll_tokenizer.eos_token in tokens:
if self.ll_matcher.is_stopped() and not self.terminated:
self.rollback_lag = 1
self.terminated = True
if self.ll_matcher.is_stopped():
return True
# TODO - Add jump decoding support in the future:
# self.ll_matcher.compute_ff_bytes() - this should always work
# self.ll_matcher.compute_ff_tokens() - this only works for
# "canonical" tokenizers
# For conversion between the two, see
# https://github.com/guidance-ai/llguidance/blob/main/docs/fast_forward.md
r = self.ll_matcher.consume_tokens(tokens)
self.check_error()
return r
def validate_tokens(self, tokens: list[int]) -> list[int]:
"""Checks if the list of tokens are accepted by the parser in sequence.
Will not advance the parser.
Returns the prefix list of tokens that are accepted by the parser.
"""
if len(tokens) == 0:
return []
if self.ll_matcher.is_stopped():
return []
tokens = strip_speculative_padding(tokens)
if len(tokens) == 0:
return []
num_tokens = self.ll_matcher.validate_tokens(tokens)
self.check_error()
return tokens[:num_tokens]
def rollback(self, num_tokens: int) -> None:
if num_tokens > 0:
self.ll_matcher.rollback(num_tokens - self.rollback_lag)
self.terminated = False
self.rollback_lag = 0
self.check_error()
def fill_bitmask(self, bitmask: torch.Tensor, idx: int) -> None:
# this will automatically return [EOS] mask if the matcher is stopped
# or otherwise in an error state
llguidance_torch.fill_next_token_bitmask(self.ll_matcher, bitmask, idx)
self.check_error()
def is_terminated(self) -> bool:
return self.terminated
def reset(self):
# This method may be not needed anymore? TODO
self.ll_matcher.reset()
def serialize_guidance_grammar(
request_type: StructuredOutputOptions,
grammar_spec: str | dict[str, Any],
disable_any_whitespace: bool = False,
disable_additional_properties: bool = False,
) -> str:
def _process_schema(
grammar_spec: str | dict[str, Any],
) -> str:
if disable_additional_properties:
grammar_spec = process_for_additional_properties(grammar_spec)
return llguidance.LLMatcher.grammar_from_json_schema(
grammar_spec,
defaults={
"whitespace_flexible": not disable_any_whitespace,
},
)
if request_type == StructuredOutputOptions.JSON:
return _process_schema(grammar_spec)
elif request_type == StructuredOutputOptions.JSON_OBJECT:
return llguidance.LLMatcher.grammar_from_json_schema(
'{"type": "object"}',
defaults={
"whitespace_flexible": not disable_any_whitespace,
},
)
else:
if request_type == StructuredOutputOptions.REGEX:
tp = "regex"
elif request_type == StructuredOutputOptions.GRAMMAR:
tp = "grammar"
elif request_type == StructuredOutputOptions.CHOICE:
tp = "choice"
elif request_type == StructuredOutputOptions.STRUCTURAL_TAG:
if isinstance(grammar_spec, str):
s_tag = json.loads(grammar_spec)
else:
s_tag = grammar_spec
triggers: list[str] = s_tag["triggers"]
tags: list[llguidance.StructTag] = []
for s in s_tag["structures"]:
begin: str = s["begin"]
trig = next((t for t in triggers if begin.startswith(t)), None)
if trig is None:
raise VLLMValidationError(
f"Trigger {begin} not found in triggers {triggers}"
)
tags.append(
llguidance.StructTag(
trigger=trig,
begin=s["begin"],
grammar=_process_schema(s["schema"]),
end=s["end"],
)
)
if not tags:
raise VLLMValidationError(
"No structural tags found in the grammar spec."
)
return llguidance.StructTag.to_grammar(tags)
else:
logger.error(
"Validation should have already occurred. Please file an issue."
)
raise ValueError(
f"grammar is not of valid supported types. ({request_type!s})"
)
return llguidance.grammar_from(tp, grammar_spec)
def validate_guidance_grammar(
sampling_params: SamplingParams, tokenizer: llguidance.LLTokenizer | None = None
) -> None:
# if structured output is not enabled, there is nothing to validate
if sampling_params.structured_outputs is None:
return
tp, grm = get_structured_output_key(sampling_params.structured_outputs)
try:
guidance_grm = serialize_guidance_grammar(tp, grm)
except (ValueError, KeyError, TypeError) as e:
raise VLLMValidationError(f"Invalid grammar specification: {e}") from e
err = llguidance.LLMatcher.validate_grammar(guidance_grm, tokenizer)
if err:
raise VLLMValidationError(f"Grammar error: {err}")