[Async][Spec Decoding] Zero-bubble async scheduling + spec decoding (#32951)
Signed-off-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: zhuhaoran <zhuhaoran.zhr@alibaba-inc.com> Co-authored-by: zhrrr <43847754+izhuhaoran@users.noreply.github.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
zhuhaoran
zhrrr
Lucas Wilkinson
Benjamin Chislett
parent
ffb5b32b5f
commit
fafe76b4af
@@ -3,6 +3,7 @@
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, replace
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
CommonAttentionMetadata,
|
||||
@@ -463,3 +464,36 @@ def copy_and_expand_eagle_inputs_kernel(
|
||||
out_idx,
|
||||
mask=is_new_token_region & in_bounds,
|
||||
)
|
||||
|
||||
|
||||
@torch.compile(dynamic=True, backend=current_platform.simple_compile_backend)
|
||||
def update_num_computed_tokens_for_batch_change(
|
||||
num_computed_tokens: torch.Tensor,
|
||||
num_accepted_tokens: torch.Tensor,
|
||||
prev_positions: torch.Tensor,
|
||||
valid_sampled_token_count: torch.Tensor,
|
||||
prev_num_draft_tokens: torch.Tensor,
|
||||
cpu_num_computed_tokens: torch.Tensor,
|
||||
) -> None:
|
||||
"""Correct num_computed_tokens for async spec decode drift.
|
||||
|
||||
Requests that had drafts: corrected = prev_gpu + valid_count.
|
||||
New requests or non-draft (e.g. prefills): use CPU value directly.
|
||||
"""
|
||||
# Clamp because prev_positions can be -1 for new requests
|
||||
gather_indices = prev_positions.clamp(min=0)
|
||||
|
||||
valid_counts = valid_sampled_token_count[gather_indices]
|
||||
prev_computed = num_computed_tokens[gather_indices]
|
||||
prev_drafts = prev_num_draft_tokens[gather_indices]
|
||||
|
||||
participating = (prev_positions >= 0) & (prev_drafts > 0)
|
||||
corrected = prev_computed + valid_counts.int()
|
||||
|
||||
n = prev_positions.shape[0]
|
||||
num_computed_tokens[:n].copy_(
|
||||
torch.where(participating, corrected, cpu_num_computed_tokens)
|
||||
)
|
||||
num_accepted_tokens.copy_(
|
||||
torch.where(participating, valid_counts, num_accepted_tokens)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user