[Refactor] StructuredOutputManager x Speculative Decoding Refactor (#48200)
This commit is contained in:
@@ -70,8 +70,7 @@ from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import Request, RequestStatus, StreamingUpdate
|
||||
from vllm.v1.spec_decode.dynamic.utils import build_dynamic_sd_schedule_lookup
|
||||
from vllm.v1.spec_decode.metrics import SpecDecodingStats
|
||||
from vllm.v1.structured_output import StructuredOutputGrammar, StructuredOutputManager
|
||||
from vllm.v1.structured_output.utils import strip_speculative_padding
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
from vllm.v1.utils import record_function_or_nullcontext
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -2052,33 +2051,18 @@ class Scheduler(SchedulerInterface):
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
stopped = True
|
||||
|
||||
if new_token_ids and self.structured_output_manager.should_advance(
|
||||
request, new_token_ids=new_token_ids
|
||||
if new_token_ids and not self.structured_output_manager.accept_tokens(
|
||||
request, new_token_ids
|
||||
):
|
||||
struct_output_request = request.structured_output_request
|
||||
assert struct_output_request is not None
|
||||
grammar = struct_output_request.grammar
|
||||
assert isinstance(grammar, StructuredOutputGrammar)
|
||||
# new_token_ids can be a mixed block of reasoning content, then
|
||||
# the reasoning end marker, then the start of the grammar content.
|
||||
# Trim the reasoning content so the grammar only sees grammar content.
|
||||
advance_token_ids = (
|
||||
self.structured_output_manager.trim_reasoning_for_advance(
|
||||
request, new_token_ids
|
||||
)
|
||||
logger.error(
|
||||
"Unexpected: grammar rejected tokens %s for request %s. "
|
||||
"Terminating request.",
|
||||
new_token_ids,
|
||||
req_id,
|
||||
)
|
||||
if advance_token_ids and not grammar.accept_tokens(
|
||||
req_id, advance_token_ids
|
||||
):
|
||||
logger.error(
|
||||
"Unexpected: grammar rejected tokens %s for request %s. "
|
||||
"Terminating request.",
|
||||
advance_token_ids,
|
||||
req_id,
|
||||
)
|
||||
request.status = RequestStatus.FINISHED_ERROR
|
||||
request.resumable = False
|
||||
stopped = True
|
||||
request.status = RequestStatus.FINISHED_ERROR
|
||||
request.resumable = False
|
||||
stopped = True
|
||||
|
||||
routed_experts = None
|
||||
if (
|
||||
@@ -2434,11 +2418,9 @@ class Scheduler(SchedulerInterface):
|
||||
continue
|
||||
|
||||
# Add newly generated spec token ids to the request.
|
||||
if self.structured_output_manager.should_advance(request):
|
||||
metadata = request.structured_output_request
|
||||
spec_token_ids = strip_speculative_padding(spec_token_ids)
|
||||
spec_token_ids = metadata.grammar.validate_tokens(spec_token_ids) # type: ignore[union-attr]
|
||||
request.spec_token_ids = spec_token_ids
|
||||
request.spec_token_ids = self.structured_output_manager.validate_tokens(
|
||||
request, spec_token_ids
|
||||
)
|
||||
|
||||
def update_draft_token_ids_in_output(
|
||||
self, draft_token_ids: DraftTokenIds, scheduler_output: SchedulerOutput
|
||||
@@ -2464,10 +2446,9 @@ class Scheduler(SchedulerInterface):
|
||||
# (needed for chunked prefill case for example).
|
||||
del spec_token_ids[orig_num_spec_tokens:]
|
||||
# Filter out spec tokens which do not adhere to the grammar.
|
||||
if self.structured_output_manager.should_advance(request):
|
||||
metadata = request.structured_output_request
|
||||
spec_token_ids = strip_speculative_padding(spec_token_ids)
|
||||
spec_token_ids = metadata.grammar.validate_tokens(spec_token_ids) # type: ignore[union-attr]
|
||||
spec_token_ids = self.structured_output_manager.validate_tokens(
|
||||
request, spec_token_ids
|
||||
)
|
||||
# Pad to original number of spec tokens.
|
||||
num_invalid_tokens = orig_num_spec_tokens - len(spec_token_ids)
|
||||
if num_invalid_tokens:
|
||||
|
||||
Reference in New Issue
Block a user