[Model Runner V2][Spec Decode] Skip DP sync for all speculator uniform decodes (#54856)
This commit is contained in:
@@ -33,6 +33,9 @@ class DPSyncState:
|
|||||||
# Whether the ranks agreed to run eager. A dispatch reusing this must run
|
# Whether the ranks agreed to run eager. A dispatch reusing this must run
|
||||||
# eager too; no shape was agreed, so picking a graph per rank would diverge.
|
# eager too; no shape was agreed, so picking a graph per rank would diverge.
|
||||||
eager: bool
|
eager: bool
|
||||||
|
# Agreed upper bound on any rank's request count. Holds the padded count when
|
||||||
|
# a FULL descriptor imposed one, else the most any rank scheduled.
|
||||||
|
num_reqs: int
|
||||||
|
|
||||||
|
|
||||||
def sync_cudagraph_and_dp_padding(
|
def sync_cudagraph_and_dp_padding(
|
||||||
@@ -53,17 +56,19 @@ def sync_cudagraph_and_dp_padding(
|
|||||||
"""
|
"""
|
||||||
assert dp_size > 1, "DP size must be greater than 1"
|
assert dp_size > 1, "DP size must be greater than 1"
|
||||||
group = get_dp_group().cpu_group
|
group = get_dp_group().cpu_group
|
||||||
tensor = torch.zeros(4, dp_size, dtype=torch.int32, device="cpu")
|
tensor = torch.zeros(5, dp_size, dtype=torch.int32, device="cpu")
|
||||||
tensor[0][dp_rank] = num_tokens
|
tensor[0][dp_rank] = num_tokens
|
||||||
tensor[1][dp_rank] = desired_batch_desc.cg_mode.value
|
tensor[1][dp_rank] = desired_batch_desc.cg_mode.value
|
||||||
tensor[2][dp_rank] = uniform_token_count or 0 # (0 means None)
|
tensor[2][dp_rank] = uniform_token_count or 0 # (0 means None)
|
||||||
tensor[3][dp_rank] = max_query_len or -1 # (-1 means None)
|
tensor[3][dp_rank] = max_query_len or -1 # (-1 means None)
|
||||||
|
tensor[4][dp_rank] = num_reqs
|
||||||
dist.all_reduce(tensor, group=group)
|
dist.all_reduce(tensor, group=group)
|
||||||
|
|
||||||
num_tokens_across_dp = tensor[0]
|
num_tokens_across_dp = tensor[0]
|
||||||
cg_mode_across_dp = tensor[1]
|
cg_mode_across_dp = tensor[1]
|
||||||
uniform_token_counts_across_dp = tensor[2]
|
uniform_token_counts_across_dp = tensor[2]
|
||||||
max_query_lens_across_dp = tensor[3]
|
max_query_lens_across_dp = tensor[3]
|
||||||
|
num_reqs_across_dp = tensor[4]
|
||||||
|
|
||||||
# If ranks disagree on the uniform token count, or its 0 (means None) set to None
|
# If ranks disagree on the uniform token count, or its 0 (means None) set to None
|
||||||
synced_uniform_token_count: int | None = int(uniform_token_counts_across_dp[0])
|
synced_uniform_token_count: int | None = int(uniform_token_counts_across_dp[0])
|
||||||
@@ -93,6 +98,7 @@ def sync_cudagraph_and_dp_padding(
|
|||||||
num_tokens_across_dp=num_tokens_across_dp,
|
num_tokens_across_dp=num_tokens_across_dp,
|
||||||
uniform_token_count=synced_uniform_token_count,
|
uniform_token_count=synced_uniform_token_count,
|
||||||
eager=True,
|
eager=True,
|
||||||
|
num_reqs=int(num_reqs_across_dp.max().item()),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -126,6 +132,12 @@ def sync_cudagraph_and_dp_padding(
|
|||||||
num_tokens_across_dp=num_tokens_across_dp,
|
num_tokens_across_dp=num_tokens_across_dp,
|
||||||
uniform_token_count=synced_uniform_token_count,
|
uniform_token_count=synced_uniform_token_count,
|
||||||
eager=False,
|
eager=False,
|
||||||
|
num_reqs=(
|
||||||
|
synced_desc.num_reqs
|
||||||
|
if synced_desc.cg_mode == CUDAGraphMode.FULL
|
||||||
|
and synced_desc.num_reqs is not None
|
||||||
|
else int(num_reqs_across_dp.max().item())
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -336,16 +336,22 @@ class AutoRegressiveSpeculator(DraftModelSpeculator):
|
|||||||
advance_draft_positions=self.advance_draft_positions,
|
advance_draft_positions=self.advance_draft_positions,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
decode_batch_sync, num_batch_tokens = (
|
||||||
|
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, num_query_per_req=1)
|
||||||
|
if dp_sync is not None
|
||||||
|
else (None, num_reqs)
|
||||||
|
)
|
||||||
# Each request produces exactly 1 token per draft generation step,
|
# Each request produces exactly 1 token per draft generation step,
|
||||||
# enabling FULL graph replay.
|
# enabling FULL graph replay.
|
||||||
decode_batch_desc, decode_batch_sync = dispatch_cg_and_sync_dp(
|
decode_batch_desc, decode_batch_sync = dispatch_cg_and_sync_dp(
|
||||||
self.decode_cudagraph_manager,
|
self.decode_cudagraph_manager,
|
||||||
num_reqs,
|
num_reqs,
|
||||||
num_reqs,
|
num_batch_tokens,
|
||||||
uniform_token_count=1,
|
uniform_token_count=1,
|
||||||
dp_size=self.dp_size,
|
dp_size=self.dp_size,
|
||||||
dp_rank=self.dp_rank,
|
dp_rank=self.dp_rank,
|
||||||
need_eager=is_profile,
|
need_eager=is_profile,
|
||||||
|
dp_sync=decode_batch_sync,
|
||||||
)
|
)
|
||||||
num_tokens_across_dp = (
|
num_tokens_across_dp = (
|
||||||
decode_batch_sync.num_tokens_across_dp
|
decode_batch_sync.num_tokens_across_dp
|
||||||
|
|||||||
@@ -367,9 +367,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
|||||||
num_query_tokens,
|
num_query_tokens,
|
||||||
attn_metadata=None,
|
attn_metadata=None,
|
||||||
slot_mappings=None,
|
slot_mappings=None,
|
||||||
num_tokens_across_dp=(
|
num_tokens_across_dp=None,
|
||||||
dp_sync.num_tokens_across_dp if dp_sync is not None else None
|
|
||||||
),
|
|
||||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||||
)
|
)
|
||||||
return self.draft_tokens[:num_reqs]
|
return self.draft_tokens[:num_reqs]
|
||||||
@@ -429,17 +427,22 @@ class DFlashSpeculator(DraftModelSpeculator):
|
|||||||
context_slots,
|
context_slots,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
batch_sync, num_batch_tokens = (
|
||||||
|
self._build_uniform_batch_dp_sync(dp_sync, num_reqs, self.num_query_per_req)
|
||||||
|
if dp_sync is not None
|
||||||
|
else (None, num_query_tokens)
|
||||||
|
)
|
||||||
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
|
# Every DFlash step has exactly num_query_per_req tokens, so we can use FULL CGs
|
||||||
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
|
batch_desc, batch_sync = dispatch_cg_and_sync_dp(
|
||||||
self.query_cudagraph_manager,
|
self.query_cudagraph_manager,
|
||||||
num_reqs,
|
num_reqs,
|
||||||
num_query_tokens,
|
num_batch_tokens,
|
||||||
uniform_token_count=self.num_query_per_req,
|
uniform_token_count=self.num_query_per_req,
|
||||||
dp_size=self.dp_size,
|
dp_size=self.dp_size,
|
||||||
dp_rank=self.dp_rank,
|
dp_rank=self.dp_rank,
|
||||||
need_eager=is_profile,
|
need_eager=is_profile,
|
||||||
|
dp_sync=batch_sync,
|
||||||
)
|
)
|
||||||
|
|
||||||
num_reqs_padded = batch_desc.num_reqs or num_reqs
|
num_reqs_padded = batch_desc.num_reqs or num_reqs
|
||||||
num_tokens_padded = batch_desc.num_tokens
|
num_tokens_padded = batch_desc.num_tokens
|
||||||
num_tokens_across_dp = (
|
num_tokens_across_dp = (
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import replace
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -409,3 +410,22 @@ class DraftModelSpeculator(BaseSpeculator):
|
|||||||
# idx_mapping for CG padded requests points to -1, which is ignored
|
# idx_mapping for CG padded requests points to -1, which is ignored
|
||||||
# during sampling to prevent writing stale values to draft logits.
|
# during sampling to prevent writing stale values to draft logits.
|
||||||
self.idx_mapping[num_reqs:].fill_(-1)
|
self.idx_mapping[num_reqs:].fill_(-1)
|
||||||
|
|
||||||
|
def _build_uniform_batch_dp_sync(
|
||||||
|
self,
|
||||||
|
target_dp_sync: DPSyncState,
|
||||||
|
num_reqs: int,
|
||||||
|
num_query_per_req: int = 1,
|
||||||
|
) -> tuple[DPSyncState, int]:
|
||||||
|
num_batch_tokens = target_dp_sync.num_reqs * num_query_per_req
|
||||||
|
assert num_reqs * num_query_per_req <= num_batch_tokens, (
|
||||||
|
"reusing a DP sync that does not cover this batch's requests"
|
||||||
|
)
|
||||||
|
return replace(
|
||||||
|
target_dp_sync,
|
||||||
|
num_tokens_across_dp=torch.full_like(
|
||||||
|
target_dp_sync.num_tokens_across_dp, num_batch_tokens
|
||||||
|
),
|
||||||
|
uniform_token_count=num_query_per_req,
|
||||||
|
eager=False,
|
||||||
|
), num_batch_tokens
|
||||||
|
|||||||
Reference in New Issue
Block a user