[Spec Decode] DSpark confidence-scheduled verification (#47808)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com>
Signed-off-by: Lucas Wilkinson <wilkinson.lucas@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-08-12 12:27:48 -07:00
committed by GitHub
co-authored by OpenAI Codex Claude Opus 5 Benjamin Chislett mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Nick Hill OpenAI Codex
parent fe889ac925
commit 7f7a32cfec
40 changed files with 1643 additions and 111 deletions
+14 -1
View File
@@ -21,6 +21,7 @@ def sync_cudagraph_and_dp_padding(
uniform_token_count: int | None,
dp_size: int,
dp_rank: int,
max_query_len: int | None = None,
num_active_loras: int = 0,
) -> tuple[BatchExecutionDescriptor, torch.Tensor | None]:
"""
@@ -30,15 +31,17 @@ def sync_cudagraph_and_dp_padding(
"""
assert dp_size > 1, "DP size must be greater than 1"
group = get_dp_group().cpu_group
tensor = torch.zeros(3, dp_size, dtype=torch.int32, device="cpu")
tensor = torch.zeros(4, dp_size, dtype=torch.int32, device="cpu")
tensor[0][dp_rank] = num_tokens
tensor[1][dp_rank] = desired_batch_desc.cg_mode.value
tensor[2][dp_rank] = uniform_token_count or 0 # (0 means None)
tensor[3][dp_rank] = max_query_len or -1 # (-1 means None)
dist.all_reduce(tensor, group=group)
num_tokens_across_dp = tensor[0]
cg_mode_across_dp = tensor[1]
uniform_token_counts_across_dp = tensor[2]
max_query_lens_across_dp = tensor[3]
if torch.all(num_tokens_across_dp == 0).item():
synced_desc = BatchExecutionDescriptor(
@@ -69,6 +72,12 @@ def sync_cudagraph_and_dp_padding(
):
synced_uniform_token_count = None
# Varlen decode graphs are selected by the query-length bound, so ranks must agree
# on it or they pad to different token counts below.
synced_max_query_len: int | None = None
if bool(torch.all(max_query_lens_across_dp != -1).item()):
synced_max_query_len = int(max_query_lens_across_dp.max().item())
# Dispatch for the final synced values, use num_reqs instead of synced_num_reqs
# so we don't perform request padding for PIECEWISE graphs.
# num_active_loras is per-rank and doesn't need cross-rank agreement.
@@ -77,6 +86,7 @@ def sync_cudagraph_and_dp_padding(
synced_num_tokens,
synced_uniform_token_count,
num_active_loras=num_active_loras,
max_query_len=synced_max_query_len,
)
# Update num_tokens_across_dp to reflect padded size.
@@ -92,6 +102,7 @@ def dispatch_cg_and_sync_dp(
uniform_token_count: int | None,
dp_size: int,
dp_rank: int,
max_query_len: int | None = None,
need_eager: bool = False,
num_active_loras: int = 0,
) -> tuple[BatchExecutionDescriptor, torch.Tensor | None]:
@@ -112,6 +123,7 @@ def dispatch_cg_and_sync_dp(
num_tokens,
uniform_token_count,
num_active_loras=num_active_loras,
max_query_len=max_query_len,
)
if dp_size == 1:
@@ -125,5 +137,6 @@ def dispatch_cg_and_sync_dp(
uniform_token_count,
dp_size,
dp_rank,
max_query_len=max_query_len,
num_active_loras=num_active_loras,
)