[Core] Bound draft-token RPC waits by the execute-model timeout (#58779)
Signed-off-by: VS Chandra Mourya <219748331+vschandramourya@users.noreply.github.com> Co-authored-by: VS Chandra Mourya <219748331+vschandramourya@users.noreply.github.com>
This commit is contained in:
co-authored by
VS Chandra Mourya
parent
fdcce47e9b
commit
8a2364605c
@@ -369,7 +369,9 @@ class MultiprocExecutor(Executor):
|
||||
def take_draft_token_ids(self) -> DraftTokenIds | None:
|
||||
# OPTIMIZATION: Get output only from a single worker (output_rank)
|
||||
return self.collective_rpc(
|
||||
"take_draft_token_ids", unique_reply_rank=self.output_rank
|
||||
"take_draft_token_ids",
|
||||
unique_reply_rank=self.output_rank,
|
||||
timeout=envs.VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS,
|
||||
)
|
||||
|
||||
def collective_rpc( # type: ignore[override]
|
||||
|
||||
Reference in New Issue
Block a user