[V1] Structured Outputs + Thinking compatibility (#16577)

Signed-off-by: Aaron Pham <contact@aarnphm.xyz>
Co-authored-by: Russell Bryant <rbryant@redhat.com>
This commit is contained in:
Aaron Pham
2025-05-14 15:45:24 -07:00
committed by GitHub
co-authored by Russell Bryant
parent d93c976a0d
commit 2fc9075b82
10 changed files with 233 additions and 75 deletions
+4 -4
View File
@@ -758,7 +758,8 @@ class Scheduler(SchedulerInterface):
# the outer lists can be of length > 1.
new_logprobs = logprobs.slice(req_index, req_index + 1)
if new_token_ids and request.use_structured_output:
if new_token_ids and self.structured_output_manager.should_advance(
request):
# NOTE: structured_output_request
# should not be None if use_structured_output, we have
# check above, so safe to ignore type warning
@@ -767,11 +768,10 @@ class Scheduler(SchedulerInterface):
# Add newly generated spec token ids to the request.
if spec_token_ids is not None:
if request.use_structured_output:
if self.structured_output_manager.should_advance(request):
metadata = request.structured_output_request
assert metadata is not None and metadata.grammar is not None
# Needs to happen after new_token_ids are accepted.
request.spec_token_ids = metadata.grammar.validate_tokens(
request.spec_token_ids = metadata.grammar.validate_tokens( # type: ignore[union-attr]
spec_token_ids[req_index])
else:
request.spec_token_ids = spec_token_ids[req_index]