[Spec Decode] Make propose_draft_token_ids non-blocking for lower TTFT (#23041)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
+9
-4
@@ -94,9 +94,6 @@ class ModelRunnerOutput:
|
||||
# each request due to speculative/jump decoding.
|
||||
sampled_token_ids: list[list[int]]
|
||||
|
||||
# num_reqs x num_spec_tokens
|
||||
spec_token_ids: Optional[list[list[int]]]
|
||||
|
||||
# [num_reqs, max_num_logprobs + 1]
|
||||
# [num_reqs, max_num_logprobs + 1]
|
||||
# [num_reqs]
|
||||
@@ -117,10 +114,18 @@ class ModelRunnerOutput:
|
||||
num_nans_in_logits: Optional[dict[str, int]] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class DraftTokenIds:
|
||||
|
||||
# [num_reqs]
|
||||
req_ids: list[str]
|
||||
# num_reqs x num_draft_tokens
|
||||
draft_token_ids: list[list[int]]
|
||||
|
||||
|
||||
EMPTY_MODEL_RUNNER_OUTPUT = ModelRunnerOutput(req_ids=[],
|
||||
req_id_to_index={},
|
||||
sampled_token_ids=[],
|
||||
spec_token_ids=None,
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
|
||||
Reference in New Issue
Block a user