[Perf][DSV4.1] Fuse TP all-reduce with mHC input preparation (#57643)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Woosuk Kwon
2026-09-20 20:28:06 -07:00
committed by GitHub
co-authored by Codex
parent eb87980585
commit 9a70c233cd
9 changed files with 549 additions and 105 deletions
+19 -2
View File
@@ -17,6 +17,7 @@ from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_reduce,
)
from vllm.forward_context import (
get_forward_context,
@@ -88,10 +89,11 @@ from vllm.v1.worker.ubatching import dbo_current_ubatch_id
from ..common.engram import EngramLayout, NgramHashState
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
from .engram import Engram, gather_engram_hashes
from .ops.mega_mhc import mhc_shifted_post_pre
from .ops.mhc import (
MHC_OVERLAP_MAX_TOKENS,
mhc_pre_delayed_overlap,
mhc_shifted_post_pre,
supports_mhc_all_reduce,
supports_mhc_overlap,
)
@@ -107,6 +109,7 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
vllm_config: VllmConfig,
prefix: str = "",
use_sequence_parallel: bool = False,
reduce_results: bool = True,
):
config = vllm_config.model_config.hf_config
n_routed_experts = config.n_routed_experts
@@ -125,6 +128,7 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
n_routed_experts=n_routed_experts,
n_activated_experts=n_activated_experts,
num_hash_layers=0,
reduce_results=reduce_results,
image_sentinel_lo=IMAGE_SENTINEL_BASE_ID,
)
@@ -199,12 +203,14 @@ class DeepseekV4DecoderLayer(nn.Module):
candidate_block_buffer: torch.Tensor | None = None,
engram_layout: EngramLayout | None = None,
mhc_stream: torch.cuda.Stream | None = None,
fuse_mhc_all_reduce: bool = False,
):
super().__init__()
config = vllm_config.model_config.hf_config
self.hidden_size = config.hidden_size
self.mhc_stream = mhc_stream
self.fuse_mhc_all_reduce = fuse_mhc_all_reduce
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
self.engram: Engram | None = None
@@ -228,12 +234,13 @@ class DeepseekV4DecoderLayer(nn.Module):
aux_stream_list=aux_stream_list,
candidate_block_buffer=candidate_block_buffer,
)
if self.use_sequence_parallel:
if self.use_sequence_parallel or fuse_mhc_all_reduce:
self.attn.wo_b.reduce_results = False
self.ffn = DeepseekV4MoE(
vllm_config,
prefix=f"{prefix}.ffn",
use_sequence_parallel=self.use_sequence_parallel,
reduce_results=not fuse_mhc_all_reduce,
)
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
@@ -430,6 +437,8 @@ class DeepseekV4DecoderLayer(nn.Module):
# and this block's pre, on the full hc stream, so the mix
# coefficients see the injected stream. The injection also keeps
# the post out of the pre-norm GEMM's fused prologue.
if self.fuse_mhc_all_reduce:
x = tensor_model_parallel_all_reduce(x)
previous_post = mhc_post_tilelang(x, residual, post_mix, res_mix)
if capture_previous_aux:
previous_aux = previous_post.mean(dim=1)
@@ -473,6 +482,7 @@ class DeepseekV4DecoderLayer(nn.Module):
norm_eps=self.attn_norm.variance_epsilon,
capture_aux=capture_previous_aux,
stream=mhc_stream,
reduce_results=self.fuse_mhc_all_reduce,
)
if capture_previous_aux:
previous_aux = aux
@@ -503,6 +513,7 @@ class DeepseekV4DecoderLayer(nn.Module):
norm_weight=self.ffn_norm.weight,
norm_eps=self.ffn_norm.variance_epsilon,
stream=mhc_stream,
reduce_results=self.fuse_mhc_all_reduce,
)
x = self.ffn(x, input_ids, mega_gate_metadata)
if mhc_stream is not None:
@@ -540,6 +551,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
# Keep mHC independent of the streams used inside attention.
mhc_stream = torch.cuda.Stream() if supports_mhc_overlap(vllm_config) else None
self.fuse_mhc_all_reduce = mhc_stream is not None and supports_mhc_all_reduce(
vllm_config
)
# Reserved topk indices buffer for all Indexer layers to reuse.
self.topk_indices_buffer = torch.empty(
@@ -585,6 +599,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
candidate_block_buffer=self.candidate_block_buffer,
engram_layout=self.engram_layout,
mhc_stream=mhc_stream,
fuse_mhc_all_reduce=self.fuse_mhc_all_reduce,
),
prefix=f"{prefix}.layers",
)
@@ -794,6 +809,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
aux_hidden_by_layer[idx] = previous_aux
if layer is not None:
# The last layer has no successor to fold its post into.
if self.fuse_mhc_all_reduce:
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
hidden_states = mhc_post_tilelang(
hidden_states, residual, post_mix, res_mix
)