[Model Runner V2][Spec Decode] Skip DP sync for all speculator uniform decodes (#54856)

This commit is contained in:
Giancarlo Delfin
2026-09-03 10:32:33 +00:00
committed by GitHub
parent 0d3ede3e3b
commit facd9a74a1
4 changed files with 48 additions and 7 deletions
@@ -367,9 +367,7 @@ class DFlashSpeculator(DraftModelSpeculator):
num_query_tokens,
attn_metadata=None,
slot_mappings=None,
num_tokens_across_dp=(
dp_sync.num_tokens_across_dp if dp_sync is not None else None
),
num_tokens_across_dp=None,
cudagraph_runtime_mode=CUDAGraphMode.NONE,
)
return self.draft_tokens[:num_reqs]
@@ -429,17 +427,22 @@ class DFlashSpeculator(DraftModelSpeculator):
context_slots,
)
batch_sync, num_batch_tokens = (
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, self.num_query_per_req)
if dp_sync is not None
else (None, num_query_tokens)
)
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
self.query_cudagraph_manager,
num_reqs,
num_query_tokens,
num_batch_tokens,
uniform_token_count=self.num_query_per_req,
dp_size=self.dp_size,
dp_rank=self.dp_rank,
need_eager=is_profile,
dp_sync=batch_sync,
)
num_reqs_padded = batch_desc.num_reqs or num_reqs
num_tokens_padded = batch_desc.num_tokens
num_tokens_across_dp = (