# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Copyright 2026 The HuggingFace Team. All rights reserved. # ruff: noqa # fmt: off # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import annotations import copy import json from typing import Any from .content_parsers import STREAMABLE_PARSERS, process_field from .response_templates import ( ResponseTemplate, ResponseTemplateField, load_response_template, ) def _schema_types(schema: Any) -> tuple[str, ...]: if not isinstance(schema, dict): return () declared = schema.get("type") types = [declared] if isinstance(declared, str) else [] if isinstance(declared, list): types.extend(t for t in declared if isinstance(t, str)) for union_name in ("anyOf", "oneOf"): for choice in schema.get(union_name) or []: types.extend(_schema_types(choice)) if schema.get("nullable") and "null" not in types: # `nullable` is how get_json_schema marks Optionals types.append("null") return tuple(types) def _coerce(raw: str, types: tuple[str, ...]) -> Any: for type_name in types: try: if type_name == "integer": try: return int(raw) except ValueError: # JSON Schema integers include whole numbers written as floats, e.g. "3.0" number = float(raw) if not number.is_integer(): continue return int(number) if type_name == "number": number = float(raw) if number != number or number in (float("inf"), float("-inf")): continue # NaN / inf are not valid JSON numbers # Preserve ints when the source text had no fractional part. return int(number) if number.is_integer() and "." not in raw else number if type_name == "boolean" and raw.strip().lower() in ("true", "1", "false", "0"): return raw.strip().lower() in ("true", "1") if type_name == "null" and raw.strip() in ("null", "None"): return None if type_name in ("object", "array"): decoded = json.loads(raw) if isinstance(decoded, dict if type_name == "object" else list): return decoded except ValueError: continue return raw # `string` params, unknown types and failed casts all keep the original text def parse_response( text: str, response_template: dict | ResponseTemplate, *, prefix: str | None = None, tools: list[dict] | None = None, ) -> dict: """The main function for response parsing when you don't want streaming. Takes generated output and the prompt prefix and parses them without streaming any events, then returns the parsed message. Pass OpenAI-style `tools` dictionaries to cast tool-call arguments using the calling tool's JSON schema. """ response_template = load_response_template(response_template) stream = ResponseParser(response_template, prefix=prefix, tools=tools) events = stream.feed(text) message, final_events = stream.finalize() for event in stream.initial_events + events + final_events: if event["type"] == "region_malformed": raise event["error"] return message class ResponseParser: """This class implements a streaming parser with a `response_template`. If you don't need streaming and just want to parse a complete message, use the `parse_response` function above. Streaming parsing emits events indicating when regions (message fields) are opened and closed, with the model writing to the region that is currently open. Usage: parser = ResponseParser(response_template, prefix=chat_prompt) for event in parser.initial_events: handle(event) for chunk in model_text_stream: for event in parser.feed(chunk): handle(event) message, final_events = parser.finalize() for event in final_events: handle(event) Pass OpenAI-style `tools=` dictionaries to cast tool-call arguments using the calling tool's JSON schema as each region closes. Events can be "region_open", "region_chunk", "region_close", or "region_malformed". Open, close and malformed events carry `start` and `end`, the span of `input_text` the event consumed: its opening or closing delimiter, empty for implicit boundaries. Explicit opens also carry the opener's named `captures`. A region whose value fails to parse ends with "region_malformed" instead of "region_close", carrying the `error`, and parsing continues after it. ResponseParser requires the chat `prefix` (i.e. the chat history, the prefill before the current generation). This is because chat templates or assistant prefills can sometimes write part of the message, and if we only see the model output, and not the template, then we can't reliably parse the message in those cases. Any events produced while consuming the prefix are exposed as `initial_events`, so renderers can show prefill regions before the model writes anything; closed prefill regions also land in the output dict. """ def __init__( self, response_template: dict | ResponseTemplate, prefix: str | None = None, *, tools: list[dict] | None = None, ): self._spec = load_response_template(response_template) if prefix is None: raise ValueError( "`ResponseParser`/`parse_response` requires `prefix` (the chat prompt sent to the model before " "generation), because chat templates often pre-write part of the assistant message (e.g. an " "opening `` tag) that the parser must see to parse the output correctly. If the generation " 'already contains the complete message, pass `prefix=""` to opt out explicitly.' ) # Maps tool name -> schema `properties`, used to cast parsed tool-call arguments self._tool_params: dict[str, dict] = {} for tool in tools or []: fn = tool.get("function", tool) if isinstance(tool, dict) else None if isinstance(fn, dict) and isinstance(fn.get("name"), str): parameters = fn.get("parameters") properties = parameters.get("properties") if isinstance(parameters, dict) else None self._tool_params[fn["name"]] = properties if isinstance(properties, dict) else {} self._buffer: str = "" self._pos: int = 0 self._output: dict[str, Any] = copy.deepcopy(self._spec.defaults) self._implicit_name: str | None = self._spec.implicit # Unified current-region state: starts in the implicit region (or a # null sink if none was declared), and returns there after every close. # For explicit regions `_opened` flips to True eagerly on the open # match; for the implicit region it flips lazily on the first byte. self._current: str | None = self._implicit_name self._captures: dict[str, str] = {} self._body: str = "" self._opened: bool = False self._finalized: bool = False self._prefix_len: int = 0 self._malformed_fields: set[str] = set() self.initial_events: list[dict] = [] if prefix: self._consume_prefix(prefix) @property def input_text(self) -> str: """Raw parser input after start-anchor truncation. Before the first `feed()`, this is exactly the truncated prefix, so its length marks where generated text begins.""" return self._buffer def _consume_prefix(self, prefix: str) -> None: """Loads the prefix (the chat prefill sent to the model), right-truncates it to the start of the assistant message (as determined by start_anchor) and then runs the remainder through the parser. Events produced while processing the prefix are stashed on `initial_events` so callers can replay them into a renderer before feeding model output. Think of this as the "get the parser up to speed on the story so far" method. """ truncated = self._spec.truncate_past_last_anchor(prefix) if not truncated: return self._buffer = truncated self._process(self.initial_events, eos=False) self._prefix_len = len(truncated) def feed(self, text: str) -> list[dict]: """Feeds more text/tokens from the model output into the tokenizer, and returns any events that result (regions entered or left). This is the method you want to call after each generation step.""" if self._finalized: raise RuntimeError("ResponseParser already finalized") if text: self._buffer += text events: list[dict] = [] self._process(events, eos=False) return events def finalize(self) -> tuple[dict, list[dict]]: """Close the stream and return the final message dict together with any finalization events. This is necessary because some regions may only end at the end of the sequence, so you won't see the event telling you they're ready until the sequence is finalized.""" def _is_empty(v: Any) -> bool: return v is None or (isinstance(v, (list, dict, str)) and not v) if self._finalized: raise RuntimeError("ResponseParser already finalized") events: list[dict] = [] self._process(events, eos=True) missing = [ n for n, f in self._spec.fields.items() if not f.optional and n not in self._output and n not in self._malformed_fields ] if missing: raise ValueError(f"Required response_template fields missing from parsed output: {missing}") defaults = self._spec.defaults self._output = {k: v for k, v in self._output.items() if k in defaults or not _is_empty(v)} self._finalized = True return self._output, events def _process(self, events: list[dict], eos: bool) -> None: while True: watch = self._watchlist() best, hold_start = self._scan(watch, eos) if best is not None: kind, field, m = best if m.start() > self._pos: self._accumulate(events, self._buffer[self._pos : m.start()]) self._pos = m.end() if kind == "open": self._close_current(events, m.start(), m.start()) self._open_explicit(events, field, m) else: # "close" (always the implicit region's close here, # since explicit regions only expose their own close) had_content = self._opened self._close_current(events, m.start(), m.end()) # Zero-width close on an already-empty region would just # re-fire next iteration -- bail out to make progress. if not had_content and m.start() == m.end(): break continue # No committable match in the current buffer. if eos: if self._pos < len(self._buffer): self._accumulate(events, self._buffer[self._pos :]) self._pos = len(self._buffer) self._close_current(events, self._pos, self._pos) break # Stream everything up to the earliest still-pending delimiter. When # nothing is pending `hold_start == len(self._buffer)`, so this flushes # the whole buffer; otherwise we hold the (possibly partial) delimiter # bytes back until more input resolves them. if hold_start > self._pos: self._accumulate(events, self._buffer[self._pos : hold_start]) self._pos = hold_start break def _watchlist(self) -> list[tuple[str, ResponseTemplateField]]: """Patterns we care about right now: the close of the currently-open explicit region, or -- if we're in the implicit/null region -- every explicit open plus the implicit's own close (if any).""" if self._current is not None and self._current != self._implicit_name: field = self._spec.fields[self._current] return [("close", field)] if field.close_re is not None else [] watch: list[tuple[str, ResponseTemplateField]] = [] for field in self._spec.fields.values(): if field.open_re is not None: watch.append(("open", field)) if self._implicit_name is not None: impl = self._spec.fields[self._implicit_name] if impl.close_re is not None: watch.append(("close", impl)) return watch def _scan( self, watch: list[tuple[str, ResponseTemplateField]], eos: bool ) -> tuple[tuple[str, ResponseTemplateField, Any] | None, int]: """Single pass over the watched delimiters, using the `regex` module's partial matching to decide -- per delimiter -- whether it can be committed now or must be held. Returns `(best, hold_start)`: * `best` is the earliest-starting delimiter we can safely commit *now* (longest on ties, opens before closes), or `None`. * `hold_start` is the leftmost buffer position occupied by a still-pending match: a partial (incomplete) delimiter, or a complete one ending at the buffer edge that more input could still grow. Bytes before it are safe to emit; bytes from it onward must be held. It stays `len(self._buffer)` when nothing is pending, letting the caller flush the whole buffer. A complete match is committable only if it starts strictly before `hold_start` -- otherwise an earlier (or co-located) pending delimiter could turn out to be the real one. At EOS nothing can grow, so partial matching is skipped and every complete match is committable. (The `regex` module always reports the empty string as a live prefix, so a partial search with no real match returns a zero-width match at the buffer end; that lands in the pending branch with `start == len(self._buffer)`, a no-op for `hold_start`.) """ best_key: tuple | None = None best: tuple[str, ResponseTemplateField, Any] | None = None hold_start = len(self._buffer) for kind, field in watch: # The watchlist only includes fields whose delimiter regex is set, so pattern is never None here. pattern = field.open_re if kind == "open" else field.close_re if eos: m = pattern.search(self._buffer, self._pos) else: m = pattern.search(self._buffer, self._pos, partial=True) if m is None: continue if not eos and (m.partial or self._can_grow(kind, field, m)): # Pending: can't commit, and blocks emitting from its start onward. hold_start = min(hold_start, m.start()) continue key = (m.start(), -(m.end() - m.start()), 0 if kind == "open" else 1, field.name) if best_key is None or key < best_key: best_key, best = key, (kind, field, m) # A committable match co-located with or after a pending one must wait too: # the pending delimiter starts no later and might be the one that fires. if best is not None and best[2].start() >= hold_start: best = None return best, hold_start def _can_grow(self, kind: str, field: ResponseTemplateField, m: Any) -> bool: r"""Whether a *complete* match ending at the current buffer edge could still change as more input arrives -- in which case we defer rather than commit. A match ending before the edge has already seen its terminating byte and is final. At the edge: zero-width matches (`$` / `\Z`) are only real at true EOS; a fully-present literal that no other literal in its set extends cannot grow (the fast path that keeps literal delimiters zero-latency); anything else (regex delimiters, prefix-overlapping literal lists) might.""" if m.end() != len(self._buffer): return False if m.start() == m.end(): return True literals, can_extend = ( (field.open_literals, field.open_literal_can_extend) if kind == "open" else (field.close_literals, field.close_literal_can_extend) ) return literals is None or can_extend def _accumulate(self, events: list[dict], text: str) -> None: """Route `text` into the currently active region. When the current region is the null sink (no implicit declared, no explicit open), we silently discard. Every routed chunk emits a `region_chunk` event so consumers can render live; `dirty=True` flags chunks from structured parsers (json, xml-inline, kv-lines) whose raw bytes will only be parsed into the final value on close.""" if not text or self._current is None: return field = self._spec.fields[self._current] if not self._opened: events.append({"type": "region_open", "field": self._current, "start": self._pos, "end": self._pos}) self._opened = True self._body += text # Prefix bytes held back as a possible delimiter belong to the prompt, not the chunk. text = text[max(0, self._prefix_len - self._pos) :] if text: dirty = field.content not in STREAMABLE_PARSERS events.append({"type": "region_chunk", "field": self._current, "text": text, "dirty": dirty}) def _open_explicit(self, events: list[dict], field: ResponseTemplateField, m: Any) -> None: self._current = field.name self._captures = {k: v for k, v in m.groupdict().items() if v is not None} self._body = "" self._opened = True events.append( {"type": "region_open", "field": field.name, "start": m.start(), "end": m.end(), "captures": self._captures} ) def _close_current(self, events: list[dict], close_start: int, close_end: int) -> None: """Close the current region and reset to the implicit/null region. Skipped (aside from the reset) when the current region never opened -- avoids vacuous open/close pairs at every explicit boundary.""" if self._current is None or not self._opened: self._reset_to_implicit() return field = self._spec.fields[self._current] try: value = process_field(self._body, field, self._captures) if self._tool_params: value = self._coerce_tool_calls(value) if field.join is not None and not isinstance(value, str): raise ValueError( f"Field '{field.name}': 'join' requires each match to parse to a string, " f"got {type(value).__name__}." ) except (KeyError, TypeError, ValueError) as error: self._malformed_fields.add(self._current) events.append( {"type": "region_malformed", "field": self._current, "start": close_start, "end": close_end, "error": error} ) self._reset_to_implicit() return if field.join is not None: previous = self._output.get(self._current) self._output[self._current] = value if previous is None else previous + field.join + value elif field.repeats: self._output.setdefault(self._current, []).append(value) else: self._output[self._current] = value events.append({"type": "region_close", "field": self._current, "start": close_start, "end": close_end, "value": value}) self._reset_to_implicit() def _coerce_tool_calls(self, value: Any) -> Any: if isinstance(value, list): return [self._coerce_tool_calls(item) for item in value] fn = value.get("function") if isinstance(value, dict) else None if not isinstance(fn, dict): return value name, arguments = fn.get("name"), fn.get("arguments") if not isinstance(name, str) or not isinstance(arguments, dict): return value if properties := self._tool_params.get(name): for key, argument in arguments.items(): if key not in properties or not (types := _schema_types(properties[key])): continue if isinstance(argument, str): arguments[key] = _coerce(argument, types) elif isinstance(argument, list) and "array" not in types: # duplicate keys collected by `merge_duplicates` arguments[key] = [_coerce(item, types) if isinstance(item, str) else item for item in argument] return value def _reset_to_implicit(self) -> None: self._current = self._implicit_name self._captures = {} self._body = "" self._opened = False # fmt: on