speculative decoding under tensor parallelism (TP>1) , workspace creation select max hidden dim of target and draft model (#52193)

Signed-off-by: khushali9 <khushali.desai9@gmail.com>
This commit is contained in:
Khushali Desai
2026-08-24 21:33:36 +00:00
committed by GitHub
parent f59bb0bd8c
commit 23ab0cfdbc
2 changed files with 66 additions and 3 deletions
@@ -989,6 +989,23 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
)
def _fused_ar_workspace_hidden_dim(config: VllmConfig) -> int:
"""Widest hidden size across the target and (optional) draft models.
The FlashInfer allreduce+RMSNorm workspace is a process-global singleton
created eagerly at pass construction and reused by every model. Under
speculative decoding the draft shares it, so it must fit the larger of the
two hidden sizes; a draft wider than the target otherwise overflows the
target-sized buffer (vLLM #52023).
"""
hidden_dim = config.model_config.get_hidden_size()
spec = config.speculative_config
draft_model_config = getattr(spec, "draft_model_config", None) if spec else None
if draft_model_config is not None:
hidden_dim = max(hidden_dim, draft_model_config.get_hidden_size())
return hidden_dim
class AllReduceFusionPass(VllmPatternMatcherPass):
def __init__(self, config: VllmConfig) -> None:
super().__init__(config)
@@ -1005,7 +1022,9 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
"AllReduce fusion pass is disabled for missing model_config."
)
return
self.hidden_dim = config.model_config.get_hidden_size()
# Size the shared workspace for the widest model that reuses it (the
# draft under speculative decoding may be wider than the target).
self.workspace_hidden_dim = _fused_ar_workspace_hidden_dim(config)
self.group = get_tp_group().cpu_group
rank = get_tensor_model_parallel_rank()
if flashinfer_comm is None:
@@ -1026,7 +1045,7 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
)
return
element_size = torch.tensor([], dtype=self.model_dtype).element_size()
self.max_token_num = max_size // (self.hidden_dim * element_size)
self.max_token_num = max_size // (self.workspace_hidden_dim * element_size)
# take the min to save workspace size and we'll never use more
# than max_num_batched_tokens anyways
self.max_token_num = min(
@@ -1043,7 +1062,7 @@ class AllReduceFusionPass(VllmPatternMatcherPass):
world_size=self.tp_size,
rank=rank,
max_token_num=self.max_token_num,
hidden_dim=self.hidden_dim,
hidden_dim=self.workspace_hidden_dim,
dtype=self.model_dtype,
group=self.group,
)