[BugFix] Fix spec decoding max_tokens scheduling perf issue (#29542)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -234,11 +234,15 @@ class Scheduler(SchedulerInterface):
|
||||
num_new_tokens = self.scheduler_config.long_prefill_token_threshold
|
||||
num_new_tokens = min(num_new_tokens, token_budget)
|
||||
|
||||
# Make sure the input position does not exceed the max model len or
|
||||
# request's max_tokens.
|
||||
# This is necessary when using spec decoding and/or async scheduling.
|
||||
num_spec_placeholders = max(0, request.num_output_placeholders - 1)
|
||||
max_total_tokens = min(
|
||||
request.num_prompt_tokens + request.max_tokens, self.max_model_len
|
||||
# Avoid scheduling tokens that we're sure won't will be needed based on
|
||||
# request.max_tokens. For this calculation we assume placeholder
|
||||
# speculated output tokens are rejected.
|
||||
request.num_prompt_tokens + request.max_tokens + num_spec_placeholders,
|
||||
# Make sure the input position does not exceed the max model len.
|
||||
# This is necessary when using spec decoding.
|
||||
self.max_model_len,
|
||||
)
|
||||
num_new_tokens = min(
|
||||
num_new_tokens, max_total_tokens - 1 - request.num_computed_tokens
|
||||
@@ -1089,7 +1093,7 @@ class Scheduler(SchedulerInterface):
|
||||
and request.sampling_params.logprobs is not None
|
||||
and logprobs
|
||||
):
|
||||
new_logprobs = logprobs.slice(req_index, req_index + 1)
|
||||
new_logprobs = logprobs.slice_request(req_index, len(new_token_ids))
|
||||
|
||||
if new_token_ids and self.structured_output_manager.should_advance(request):
|
||||
struct_output_request = request.structured_output_request
|
||||
|
||||
+8
-20
@@ -29,27 +29,15 @@ class LogprobsLists(NamedTuple):
|
||||
# different for each request.
|
||||
cu_num_generated_tokens: list[int] | None = None
|
||||
|
||||
def slice(self, start_req_idx: int, end_req_idx: int):
|
||||
if self.cu_num_generated_tokens:
|
||||
start = self.cu_num_generated_tokens[start_req_idx]
|
||||
end = self.cu_num_generated_tokens[end_req_idx]
|
||||
# Recompute cumulative array starting from 0
|
||||
cu_num_offset = self.cu_num_generated_tokens[start_req_idx]
|
||||
sliced_cu_num_generated_tokens = [
|
||||
cu_num - cu_num_offset
|
||||
for cu_num in self.cu_num_generated_tokens[
|
||||
start_req_idx : end_req_idx + 1
|
||||
]
|
||||
]
|
||||
else:
|
||||
start = start_req_idx
|
||||
end = end_req_idx
|
||||
sliced_cu_num_generated_tokens = None
|
||||
def slice_request(self, req_idx: int, num_positions: int):
|
||||
if self.cu_num_generated_tokens is not None:
|
||||
req_idx = self.cu_num_generated_tokens[req_idx]
|
||||
end_idx = req_idx + num_positions
|
||||
return LogprobsLists(
|
||||
self.logprob_token_ids[start:end],
|
||||
self.logprobs[start:end],
|
||||
self.sampled_token_ranks[start:end],
|
||||
sliced_cu_num_generated_tokens,
|
||||
self.logprob_token_ids[req_idx:end_idx],
|
||||
self.logprobs[req_idx:end_idx],
|
||||
self.sampled_token_ranks[req_idx:end_idx],
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user