[Perf][DSV4.1] Fuse TP all-reduce with mHC input preparation (#57643)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Woosuk Kwon
2026-09-20 20:28:06 -07:00
committed by GitHub
co-authored by Codex
parent eb87980585
commit 9a70c233cd
9 changed files with 549 additions and 105 deletions
+3
View File
@@ -793,6 +793,7 @@ class DeepseekV4MoE(nn.Module):
use_sequence_parallel: bool = False,
*,
num_hash_layers: int,
reduce_results: bool = True,
n_routed_experts: int | None = None,
n_activated_experts: int | None = None,
image_sentinel_lo: int = IMAGE_SENTINEL_BASE_ID,
@@ -804,6 +805,7 @@ class DeepseekV4MoE(nn.Module):
quant_config = vllm_config.quant_config
self.prefix = prefix
self.use_sequence_parallel = use_sequence_parallel
self.reduce_results = reduce_results
moe_backend = vllm_config.kernel_config.moe_backend
validate_fi_moe_ep_config(vllm_config)
self.use_mega_moe = moe_backend in MEGA_MOE_BACKENDS
@@ -1010,6 +1012,7 @@ class DeepseekV4MoE(nn.Module):
self.physical_expert_end = self.experts_end_idx
self.experts = FusedMoEFactory(
reduce_results=self.reduce_results,
shared_experts=self.shared_experts,
gate=self.gate,
num_experts=self.n_routed_experts,
+19 -2
View File
@@ -17,6 +17,7 @@ from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_reduce,
)
from vllm.forward_context import (
get_forward_context,
@@ -88,10 +89,11 @@ from vllm.v1.worker.ubatching import dbo_current_ubatch_id
from ..common.engram import EngramLayout, NgramHashState
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
from .engram import Engram, gather_engram_hashes
from .ops.mega_mhc import mhc_shifted_post_pre
from .ops.mhc import (
MHC_OVERLAP_MAX_TOKENS,
mhc_pre_delayed_overlap,
mhc_shifted_post_pre,
supports_mhc_all_reduce,
supports_mhc_overlap,
)
@@ -107,6 +109,7 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
vllm_config: VllmConfig,
prefix: str = "",
use_sequence_parallel: bool = False,
reduce_results: bool = True,
):
config = vllm_config.model_config.hf_config
n_routed_experts = config.n_routed_experts
@@ -125,6 +128,7 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
n_routed_experts=n_routed_experts,
n_activated_experts=n_activated_experts,
num_hash_layers=0,
reduce_results=reduce_results,
image_sentinel_lo=IMAGE_SENTINEL_BASE_ID,
)
@@ -199,12 +203,14 @@ class DeepseekV4DecoderLayer(nn.Module):
candidate_block_buffer: torch.Tensor | None = None,
engram_layout: EngramLayout | None = None,
mhc_stream: torch.cuda.Stream | None = None,
fuse_mhc_all_reduce: bool = False,
):
super().__init__()
config = vllm_config.model_config.hf_config
self.hidden_size = config.hidden_size
self.mhc_stream = mhc_stream
self.fuse_mhc_all_reduce = fuse_mhc_all_reduce
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
self.engram: Engram | None = None
@@ -228,12 +234,13 @@ class DeepseekV4DecoderLayer(nn.Module):
aux_stream_list=aux_stream_list,
candidate_block_buffer=candidate_block_buffer,
)
if self.use_sequence_parallel:
if self.use_sequence_parallel or fuse_mhc_all_reduce:
self.attn.wo_b.reduce_results = False
self.ffn = DeepseekV4MoE(
vllm_config,
prefix=f"{prefix}.ffn",
use_sequence_parallel=self.use_sequence_parallel,
reduce_results=not fuse_mhc_all_reduce,
)
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
@@ -430,6 +437,8 @@ class DeepseekV4DecoderLayer(nn.Module):
# and this block's pre, on the full hc stream, so the mix
# coefficients see the injected stream. The injection also keeps
# the post out of the pre-norm GEMM's fused prologue.
if self.fuse_mhc_all_reduce:
x = tensor_model_parallel_all_reduce(x)
previous_post = mhc_post_tilelang(x, residual, post_mix, res_mix)
if capture_previous_aux:
previous_aux = previous_post.mean(dim=1)
@@ -473,6 +482,7 @@ class DeepseekV4DecoderLayer(nn.Module):
norm_eps=self.attn_norm.variance_epsilon,
capture_aux=capture_previous_aux,
stream=mhc_stream,
reduce_results=self.fuse_mhc_all_reduce,
)
if capture_previous_aux:
previous_aux = aux
@@ -503,6 +513,7 @@ class DeepseekV4DecoderLayer(nn.Module):
norm_weight=self.ffn_norm.weight,
norm_eps=self.ffn_norm.variance_epsilon,
stream=mhc_stream,
reduce_results=self.fuse_mhc_all_reduce,
)
x = self.ffn(x, input_ids, mega_gate_metadata)
if mhc_stream is not None:
@@ -540,6 +551,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
# Keep mHC independent of the streams used inside attention.
mhc_stream = torch.cuda.Stream() if supports_mhc_overlap(vllm_config) else None
self.fuse_mhc_all_reduce = mhc_stream is not None and supports_mhc_all_reduce(
vllm_config
)
# Reserved topk indices buffer for all Indexer layers to reuse.
self.topk_indices_buffer = torch.empty(
@@ -585,6 +599,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
candidate_block_buffer=self.candidate_block_buffer,
engram_layout=self.engram_layout,
mhc_stream=mhc_stream,
fuse_mhc_all_reduce=self.fuse_mhc_all_reduce,
),
prefix=f"{prefix}.layers",
)
@@ -794,6 +809,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
aux_hidden_by_layer[idx] = previous_aux
if layer is not None:
# The last layer has no successor to fold its post into.
if self.fuse_mhc_all_reduce:
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
hidden_states = mhc_post_tilelang(
hidden_states, residual, post_mix, res_mix
)
@@ -5,10 +5,6 @@ import functools
import torch
from vllm.model_executor.kernels.mhc.tilelang import (
mhc_fused_post_pre_delayed_tilelang,
mhc_post_tilelang,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
_import_deep_gemm,
@@ -88,97 +84,3 @@ def mhc_shifted_post_pre_deep_gemm(
y_bf16,
new_prev_mix.squeeze(-1),
)
def mhc_shifted_post_pre(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
capture_aux: bool = False,
*,
stream: torch.cuda.Stream | None = None,
) -> tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
]:
"""Dispatch shifted post/pre to overlap, Mega-mHC, or fused TileLang.
When stream is supplied, join it before consuming the returned coefficients.
"""
if stream is not None:
from .mhc import mhc_pre_delayed_overlap
residual = mhc_post_tilelang(x, residual, post_layer_mix, comb_res_mix)
aux = residual.mean(dim=1) if capture_aux else x.new_empty(0, x.shape[1])
pre_outputs = mhc_pre_delayed_overlap(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix=pre_mix,
norm_weight=norm_weight,
norm_eps=norm_eps,
stream=stream,
)
return residual, *pre_outputs, aux
if (
pre_mix is not None
and norm_weight is not None
and not capture_aux
and x.shape[0] <= 1 << 20
and is_mega_mhc_supported(x.shape[1], residual.shape[1])
):
outputs = mhc_shifted_post_pre_deep_gemm(
x,
residual,
pre_mix,
post_layer_mix,
comb_res_mix,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_post_mult_value,
hc_sinkhorn_eps,
sinkhorn_repeat,
norm_weight,
norm_eps,
)
return *outputs, x.new_empty(0, x.shape[1])
return mhc_fused_post_pre_delayed_tilelang(
x,
residual,
post_layer_mix,
comb_res_mix,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix=pre_mix,
norm_weight=norm_weight,
norm_eps=norm_eps,
capture_aux=capture_aux,
)
+156 -4
View File
@@ -1,15 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Split shifted mHC pre so coefficient generation can overlap the sublayer."""
"""Dispatch DSV4.1 mHC operations and overlap coefficient generation."""
from functools import partial
from typing import TYPE_CHECKING, cast
import torch
from vllm.config import VllmConfig
from vllm.distributed import get_tp_group
from vllm.model_executor.kernels.mhc.tilelang import (
mhc_fused_post_pre_delayed_tilelang,
mhc_post_tilelang,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
from .mega_mhc import is_mega_mhc_supported, mhc_shifted_post_pre_deep_gemm
if TYPE_CHECKING:
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
# GB200 TP4/FlashInfer improves through 16 tokens; larger screens tie or regress.
MHC_OVERLAP_MAX_TOKENS = 16
@@ -29,6 +40,20 @@ def supports_mhc_overlap(vllm_config: VllmConfig) -> bool:
)
def supports_mhc_all_reduce(vllm_config: VllmConfig) -> bool:
parallel = vllm_config.parallel_config
config = vllm_config.model_config.hf_config
if (
parallel.tensor_parallel_size != 4
or parallel.enable_expert_parallel
or config.hidden_size != 5120
or config.hc_mult != 4
):
return False
comm = cast("CudaCommunicator", get_tp_group().device_communicator).ca_comm
return comm is not None and bool(comm.mnnvl_lamport_ag_multicast_ptr)
def mhc_pre_delayed_overlap(
residual: torch.Tensor,
fn: torch.Tensor,
@@ -45,6 +70,7 @@ def mhc_pre_delayed_overlap(
norm_eps: float = 1e-6,
*,
stream: torch.cuda.Stream,
layer_input: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Prepare the input on the caller stream and coefficients on another stream.
@@ -67,7 +93,11 @@ def mhc_pre_delayed_overlap(
post = torch.empty((n, hc), device=residual.device, dtype=torch.float32)
comb = torch.empty((n, hc * hc), device=residual.device, dtype=torch.float32)
next_pre = torch.empty_like(post)
layer_input = torch.empty((n, hidden), device=residual.device, dtype=torch.bfloat16)
compute_input = layer_input is None
if layer_input is None:
layer_input = torch.empty(
(n, hidden), device=residual.device, dtype=torch.bfloat16
)
outputs = post.unsqueeze(-1), comb.view(n, hc, hc), layer_input, next_pre
if n == 0:
return outputs
@@ -104,7 +134,7 @@ def mhc_pre_delayed_overlap(
)
main = torch.cuda.current_stream()
# Prioritize input readiness for small batches before releasing statistics.
if n <= 8:
if compute_input and n <= 8:
epilogue(split_mode="input")
stream.wait_stream(main)
with torch.cuda.stream(stream):
@@ -112,6 +142,128 @@ def mhc_pre_delayed_overlap(
epilogue(split_mode="stats")
for tensor in (x, mix, sqr):
tensor.record_stream(stream)
if n > 8:
if compute_input and n > 8:
epilogue(split_mode="input")
return outputs
def mhc_shifted_post_pre(
x: torch.Tensor,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
fn: torch.Tensor,
hc_scale: torch.Tensor,
hc_base: torch.Tensor,
rms_eps: float,
hc_pre_eps: float,
hc_sinkhorn_eps: float,
hc_post_mult_value: float,
sinkhorn_repeat: int,
pre_mix: torch.Tensor | None = None,
norm_weight: torch.Tensor | None = None,
norm_eps: float = 1e-6,
capture_aux: bool = False,
*,
stream: torch.cuda.Stream | None = None,
reduce_results: bool = False,
) -> tuple[
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
]:
"""Dispatch shifted post/pre to overlap, Mega-mHC, or fused TileLang.
When stream is supplied, join it before consuming the returned coefficients.
"""
layer_input = None
if reduce_results:
tp = get_tp_group()
if stream is not None and 0 < x.shape[0] <= MHC_OVERLAP_MAX_TOKENS:
comm = cast("CudaCommunicator", tp.device_communicator).ca_comm
assert comm is not None and comm.mnnvl_lamport_epochs is not None
output = torch.empty_like(residual)
layer_input = torch.empty_like(x)
torch.ops._C_custom_ar.all_reduce_mhc(
x,
residual,
post_layer_mix,
comb_res_mix,
pre_mix,
norm_weight,
output,
layer_input,
comm.mnnvl_lamport_ag_local_ptr,
comm.mnnvl_lamport_ag_multicast_ptr,
comm.mnnvl_lamport_epochs[0],
comm.rank,
comm.mnnvl_buffer_size,
norm_eps,
)
residual = output
else:
x = tp.all_reduce(x)
if stream is not None:
if layer_input is None:
residual = mhc_post_tilelang(x, residual, post_layer_mix, comb_res_mix)
aux = residual.mean(dim=1) if capture_aux else x.new_empty(0, x.shape[1])
pre_outputs = mhc_pre_delayed_overlap(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix=pre_mix,
norm_weight=norm_weight,
norm_eps=norm_eps,
stream=stream,
layer_input=layer_input,
)
return residual, *pre_outputs, aux
if (
pre_mix is not None
and norm_weight is not None
and not capture_aux
and x.shape[0] <= 1 << 20
and is_mega_mhc_supported(x.shape[1], residual.shape[1])
):
outputs = mhc_shifted_post_pre_deep_gemm(
x,
residual,
pre_mix,
post_layer_mix,
comb_res_mix,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_post_mult_value,
hc_sinkhorn_eps,
sinkhorn_repeat,
norm_weight,
norm_eps,
)
return *outputs, x.new_empty(0, x.shape[1])
return mhc_fused_post_pre_delayed_tilelang(
x,
residual,
post_layer_mix,
comb_res_mix,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_mult_value,
sinkhorn_repeat,
pre_mix=pre_mix,
norm_weight=norm_weight,
norm_eps=norm_eps,
capture_aux=capture_aux,
)