[Spec Decode] DSpark confidence-scheduled verification (#47808)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
Claude Opus 5
Benjamin Chislett
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Nick Hill
OpenAI Codex
parent
fe889ac925
commit
7f7a32cfec
@@ -21,6 +21,7 @@ def sync_cudagraph_and_dp_padding(
|
||||
uniform_token_count: int | None,
|
||||
dp_size: int,
|
||||
dp_rank: int,
|
||||
max_query_len: int | None = None,
|
||||
num_active_loras: int = 0,
|
||||
) -> tuple[BatchExecutionDescriptor, torch.Tensor | None]:
|
||||
"""
|
||||
@@ -30,15 +31,17 @@ def sync_cudagraph_and_dp_padding(
|
||||
"""
|
||||
assert dp_size > 1, "DP size must be greater than 1"
|
||||
group = get_dp_group().cpu_group
|
||||
tensor = torch.zeros(3, dp_size, dtype=torch.int32, device="cpu")
|
||||
tensor = torch.zeros(4, dp_size, dtype=torch.int32, device="cpu")
|
||||
tensor[0][dp_rank] = num_tokens
|
||||
tensor[1][dp_rank] = desired_batch_desc.cg_mode.value
|
||||
tensor[2][dp_rank] = uniform_token_count or 0 # (0 means None)
|
||||
tensor[3][dp_rank] = max_query_len or -1 # (-1 means None)
|
||||
dist.all_reduce(tensor, group=group)
|
||||
|
||||
num_tokens_across_dp = tensor[0]
|
||||
cg_mode_across_dp = tensor[1]
|
||||
uniform_token_counts_across_dp = tensor[2]
|
||||
max_query_lens_across_dp = tensor[3]
|
||||
|
||||
if torch.all(num_tokens_across_dp == 0).item():
|
||||
synced_desc = BatchExecutionDescriptor(
|
||||
@@ -69,6 +72,12 @@ def sync_cudagraph_and_dp_padding(
|
||||
):
|
||||
synced_uniform_token_count = None
|
||||
|
||||
# Varlen decode graphs are selected by the query-length bound, so ranks must agree
|
||||
# on it or they pad to different token counts below.
|
||||
synced_max_query_len: int | None = None
|
||||
if bool(torch.all(max_query_lens_across_dp != -1).item()):
|
||||
synced_max_query_len = int(max_query_lens_across_dp.max().item())
|
||||
|
||||
# Dispatch for the final synced values, use num_reqs instead of synced_num_reqs
|
||||
# so we don't perform request padding for PIECEWISE graphs.
|
||||
# num_active_loras is per-rank and doesn't need cross-rank agreement.
|
||||
@@ -77,6 +86,7 @@ def sync_cudagraph_and_dp_padding(
|
||||
synced_num_tokens,
|
||||
synced_uniform_token_count,
|
||||
num_active_loras=num_active_loras,
|
||||
max_query_len=synced_max_query_len,
|
||||
)
|
||||
|
||||
# Update num_tokens_across_dp to reflect padded size.
|
||||
@@ -92,6 +102,7 @@ def dispatch_cg_and_sync_dp(
|
||||
uniform_token_count: int | None,
|
||||
dp_size: int,
|
||||
dp_rank: int,
|
||||
max_query_len: int | None = None,
|
||||
need_eager: bool = False,
|
||||
num_active_loras: int = 0,
|
||||
) -> tuple[BatchExecutionDescriptor, torch.Tensor | None]:
|
||||
@@ -112,6 +123,7 @@ def dispatch_cg_and_sync_dp(
|
||||
num_tokens,
|
||||
uniform_token_count,
|
||||
num_active_loras=num_active_loras,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
|
||||
if dp_size == 1:
|
||||
@@ -125,5 +137,6 @@ def dispatch_cg_and_sync_dp(
|
||||
uniform_token_count,
|
||||
dp_size,
|
||||
dp_rank,
|
||||
max_query_len=max_query_len,
|
||||
num_active_loras=num_active_loras,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user