[Model Runner V2][Spec Decode] Skip DP sync for all speculator uniform decodes (#54856)
This commit is contained in:
@@ -367,9 +367,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
num_query_tokens,
|
||||
attn_metadata=None,
|
||||
slot_mappings=None,
|
||||
num_tokens_across_dp=(
|
||||
dp_sync.num_tokens_across_dp if dp_sync is not None else None
|
||||
),
|
||||
num_tokens_across_dp=None,
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
return self.draft_tokens[:num_reqs]
|
||||
@@ -429,17 +427,22 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
context_slots,
|
||||
)
|
||||
|
||||
batch_sync, num_batch_tokens = (
|
||||
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, self.num_query_per_req)
|
||||
if dp_sync is not None
|
||||
else (None, num_query_tokens)
|
||||
)
|
||||
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
|
||||
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
|
||||
self.query_cudagraph_manager,
|
||||
num_reqs,
|
||||
num_query_tokens,
|
||||
num_batch_tokens,
|
||||
uniform_token_count=self.num_query_per_req,
|
||||
dp_size=self.dp_size,
|
||||
dp_rank=self.dp_rank,
|
||||
need_eager=is_profile,
|
||||
dp_sync=batch_sync,
|
||||
)
|
||||
|
||||
num_reqs_padded = batch_desc.num_reqs or num_reqs
|
||||
num_tokens_padded = batch_desc.num_tokens
|
||||
num_tokens_across_dp = (
|
||||
|
||||
Reference in New Issue
Block a user