[Refactor] StructuredOutputManager x Speculative Decoding Refactor (#48200)

This commit is contained in:
yzong-rh
2026-09-15 09:48:49 -04:00
committed by GitHub
parent 64856080b0
commit e6960af33b
12 changed files with 596 additions and 1192 deletions
+17 -36
View File
@@ -70,8 +70,7 @@ from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
from vllm.v1.spec_decode.metrics import SpecDecodingStats
from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager
from vllm.v1.structured_output.utils import strip_speculative_padding
from vllm.v1.structured_output import StructuredOutputManager
from vllm.v1.utils import record_function_or_nullcontext
logger = init_logger(__name__)
@@ -2052,33 +2051,18 @@ class Scheduler(SchedulerInterface):
request.status = RequestStatus.FINISHED_STOPPED
stopped = True
if new_token_ids and self.structured_output_manager.should_advance(
request, new_token_ids=new_token_ids
if new_token_ids and not self.structured_output_manager.accept_tokens(
request, new_token_ids
):
struct_output_request = request.structured_output_request
assert struct_output_request is not None
grammar = struct_output_request.grammar
assert isinstance(grammar, StructuredOutputGrammar)
# new_token_ids can be a mixed block of reasoning content, then
# the reasoning end marker, then the start of the grammar content.
# Trim the reasoning content so the grammar only sees grammar content.
advance_token_ids = (
self.structured_output_manager.trim_reasoning_for_advance(
request, new_token_ids
)
logger.error(
"Unexpected: grammar rejected tokens %s for request %s. "
"Terminating request.",
new_token_ids,
req_id,
)
if advance_token_ids and not grammar.accept_tokens(
req_id, advance_token_ids
):
logger.error(
"Unexpected: grammar rejected tokens %s for request %s. "
"Terminating request.",
advance_token_ids,
req_id,
)
request.status = RequestStatus.FINISHED_ERROR
request.resumable = False
stopped = True
request.status = RequestStatus.FINISHED_ERROR
request.resumable = False
stopped = True
routed_experts = None
if (
@@ -2434,11 +2418,9 @@ class Scheduler(SchedulerInterface):
continue
# Add newly generated spec token ids to the request.
if self.structured_output_manager.should_advance(request):
metadata = request.structured_output_request
spec_token_ids = strip_speculative_padding(spec_token_ids)
spec_token_ids = metadata.grammar.validate_tokens(spec_token_ids) # type: ignore[union-attr]
request.spec_token_ids = spec_token_ids
request.spec_token_ids = self.structured_output_manager.validate_tokens(
request, spec_token_ids
)
def update_draft_token_ids_in_output(
self, draft_token_ids: DraftTokenIds, scheduler_output: SchedulerOutput
@@ -2464,10 +2446,9 @@ class Scheduler(SchedulerInterface):
# (needed for chunked prefill case for example).
del spec_token_ids[orig_num_spec_tokens:]
# Filter out spec tokens which do not adhere to the grammar.
if self.structured_output_manager.should_advance(request):
metadata = request.structured_output_request
spec_token_ids = strip_speculative_padding(spec_token_ids)
spec_token_ids = metadata.grammar.validate_tokens(spec_token_ids) # type: ignore[union-attr]
spec_token_ids = self.structured_output_manager.validate_tokens(
request, spec_token_ids
)
# Pad to original number of spec tokens.
num_invalid_tokens = orig_num_spec_tokens - len(spec_token_ids)
if num_invalid_tokens: