[Core] Don't schedule spec tokens with prefill chunks (#33652)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -17,33 +17,22 @@ class AsyncScheduler(Scheduler):
|
||||
|
||||
def _update_after_schedule(self, scheduler_output: SchedulerOutput) -> None:
|
||||
super()._update_after_schedule(scheduler_output)
|
||||
has_structured_output_requests = False
|
||||
pending_structured_output_tokens = False
|
||||
spec_decode_tokens = scheduler_output.scheduled_spec_decode_tokens
|
||||
for req_id in scheduler_output.num_scheduled_tokens:
|
||||
request = self.requests[req_id]
|
||||
has_structured_output_requests |= request.use_structured_output
|
||||
pending_structured_output_tokens |= (
|
||||
if request.is_prefill_chunk:
|
||||
continue
|
||||
|
||||
scheduler_output.pending_structured_output_tokens |= (
|
||||
request.use_structured_output and request.num_output_placeholders > 0
|
||||
)
|
||||
# The request will generate a new token plus num_spec_tokens
|
||||
# in this scheduling step.
|
||||
cur_num_spec_tokens = len(spec_decode_tokens.get(req_id, ()))
|
||||
if (
|
||||
request.num_computed_tokens
|
||||
== request.num_tokens
|
||||
+ request.num_output_placeholders
|
||||
+ cur_num_spec_tokens
|
||||
):
|
||||
# The request will generate a new token plus num_spec_tokens
|
||||
# in this scheduling step.
|
||||
request.num_output_placeholders += 1 + cur_num_spec_tokens
|
||||
# Add placeholders for the new draft/spec tokens.
|
||||
# We will update the actual spec token ids in the worker process.
|
||||
request.spec_token_ids = self._spec_token_placeholders
|
||||
|
||||
scheduler_output.has_structured_output_requests = has_structured_output_requests
|
||||
scheduler_output.pending_structured_output_tokens = (
|
||||
pending_structured_output_tokens
|
||||
)
|
||||
request.num_output_placeholders += 1 + cur_num_spec_tokens
|
||||
# Add placeholders for the new draft/spec tokens.
|
||||
# We will update the actual spec token ids in the worker process.
|
||||
request.spec_token_ids = self._spec_token_placeholders
|
||||
|
||||
def _update_request_with_output(
|
||||
self, request: Request, new_token_ids: list[int]
|
||||
|
||||
@@ -912,6 +912,12 @@ class Scheduler(SchedulerInterface):
|
||||
for req_id, num_scheduled_token in num_scheduled_tokens.items():
|
||||
request = self.requests[req_id]
|
||||
request.num_computed_tokens += num_scheduled_token
|
||||
request.is_prefill_chunk = request.num_computed_tokens < (
|
||||
request.num_tokens + request.num_output_placeholders
|
||||
)
|
||||
scheduler_output.has_structured_output_requests |= (
|
||||
request.use_structured_output
|
||||
)
|
||||
|
||||
# NOTE: _free_encoder_inputs relies on num_computed_tokens, which
|
||||
# may be updated again in _update_from_output for speculative
|
||||
@@ -1562,6 +1568,12 @@ class Scheduler(SchedulerInterface):
|
||||
# The request may have been finished. Skip.
|
||||
continue
|
||||
|
||||
if request.is_prefill_chunk:
|
||||
# Ignore draft tokens for prefill chunks.
|
||||
if request.spec_token_ids:
|
||||
request.spec_token_ids = []
|
||||
continue
|
||||
|
||||
# Add newly generated spec token ids to the request.
|
||||
if self.structured_output_manager.should_advance(request):
|
||||
metadata = request.structured_output_request
|
||||
|
||||
@@ -147,6 +147,9 @@ class Request:
|
||||
# The number of tokens with prefix cache hits.
|
||||
self.num_cached_tokens = -1
|
||||
|
||||
# True if this request is scheduled as a non-final prefill chunk.
|
||||
self.is_prefill_chunk = False
|
||||
|
||||
# The number of NaNs in logits. A value greater than 0
|
||||
# indicates that the output is corrupted
|
||||
self.num_nans_in_logits = 0
|
||||
|
||||
@@ -16,21 +16,21 @@ class DraftTokensHandler:
|
||||
|
||||
self.req_ids: list[str] = []
|
||||
self.draft_tokens_np: np.ndarray | None = None
|
||||
self.num_draft_tokens: int = 0
|
||||
|
||||
def set_draft_tokens(
|
||||
self, input_batch: InputBatch, draft_tokens: torch.Tensor
|
||||
) -> None:
|
||||
self.req_ids = input_batch.req_ids
|
||||
self.num_draft_tokens = draft_tokens.shape[1]
|
||||
if not input_batch.has_structured_output_reqs:
|
||||
# No draft token validation needs to be performed by
|
||||
# the scheduler for this batch.
|
||||
if self.req_ids:
|
||||
self.req_ids = []
|
||||
self.draft_tokens_np = None
|
||||
return
|
||||
|
||||
# For spec decoding + structured outputs, we must transfer the
|
||||
# draft tokens back to the scheduler for grammar validation.
|
||||
self.req_ids = input_batch.req_ids
|
||||
current_stream = torch.cuda.current_stream(self.device)
|
||||
self.copy_stream.wait_stream(current_stream)
|
||||
with torch.cuda.stream(self.copy_stream):
|
||||
@@ -38,8 +38,10 @@ class DraftTokensHandler:
|
||||
self.copy_event.record()
|
||||
|
||||
def get_draft_tokens(self) -> DraftTokenIds | None:
|
||||
if self.draft_tokens_np is None:
|
||||
return None
|
||||
|
||||
self.copy_event.synchronize()
|
||||
return DraftTokenIds(self.req_ids, self.draft_tokens_np.tolist())
|
||||
if self.draft_tokens_np is not None:
|
||||
self.copy_event.synchronize()
|
||||
draft_token_ids = self.draft_tokens_np.tolist()
|
||||
else:
|
||||
# This case only happens when async scheduling is disabled.
|
||||
draft_token_ids = [[-1] * self.num_draft_tokens for _ in self.req_ids]
|
||||
return DraftTokenIds(self.req_ids, draft_token_ids)
|
||||
|
||||
Reference in New Issue
Block a user