[Kernel][DSV4.1] Fuse MoE finalize into the TP all-reduce + mHC boundary (#58586)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yongye Zhu
2026-09-27 03:08:36 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 379e9a1ea8
commit 77871126f9
8 changed files with 1804 additions and 306 deletions
+76 -4
View File
@@ -33,6 +33,14 @@ from vllm.model_executor.kernels.mhc.triton import hc_collapse_triton
from vllm.model_executor.layers.fused_moe import (
fused_moe_make_expert_params_mapping,
)
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
TrtLlmMxfp4ExpertsModular,
)
from vllm.model_executor.layers.fused_moe.moe_output import (
MoEOutput,
UnfinalizedMoEOutput,
)
from vllm.model_executor.layers.fused_moe.runner.moe_runner import _unpack
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -94,6 +102,7 @@ from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
from .engram import Engram, can_share_engram_tables, gather_engram_hashes
from .ops.mhc import (
MHC_OVERLAP_MAX_TOKENS,
init_mhc_all_reduce,
mhc_pre_delayed_overlap,
mhc_shifted_post_pre,
supports_mhc_all_reduce,
@@ -135,6 +144,48 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
image_sentinel_lo=IMAGE_SENTINEL_BASE_ID,
)
def defer_finalize(self) -> None:
"""Leave the routed top-k reduction to the next fused all-reduce + mHC.
Only the modular TRTLLM MXFP4 experts can stop after GEMM2; the fused
kernel takes as many tokens as the mHC overlap path.
"""
moe_config = self.experts.moe_config
experts_cls = getattr(
self.experts.routed_experts.quant_method, "experts_cls", None
)
if (
experts_cls is TrtLlmMxfp4ExpertsModular
and self.experts.routed_scaling_factor == 1.0
):
moe_config.defer_moe_finalize(MHC_OVERLAP_MAX_TOKENS)
if moe_config.use_deferred_moe_finalize:
logger.info_once(
"DSV4.1 mHC: MoE top-k finalize fused into the all-reduce for "
"up to %d tokens.",
MHC_OVERLAP_MAX_TOKENS,
)
def defers_finalize(self, num_tokens: int) -> bool:
return (
not self.use_mega_moe
and self.experts.moe_config.should_defer_moe_finalize(num_tokens)
)
def forward_unfinalized(
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None
) -> MoEOutput:
"""``forward`` with the routed top-k reduction and all-reduce left open."""
# The runner's custom op returns tensors only, so run its body directly.
shared_output, routed = _unpack(
self.experts._forward_impl(
hidden_states, hidden_states, hidden_states, input_ids
)
)
assert shared_output is not None
assert isinstance(routed, UnfinalizedMoEOutput)
return MoEOutput(routed=routed, shared_output=shared_output)
def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
"""Pick the CUDA sparse-MLA attention class for the configured backend.
@@ -394,7 +445,7 @@ class DeepseekV4DecoderLayer(nn.Module):
def forward(
self,
x: torch.Tensor,
x: torch.Tensor | MoEOutput,
positions: torch.Tensor,
input_ids: torch.Tensor | None,
pre_mix: torch.Tensor | None = None,
@@ -407,7 +458,7 @@ class DeepseekV4DecoderLayer(nn.Module):
capture_previous_aux: bool = False,
mega_gate_metadata: MegaGateRoutingMetadata | None = None,
) -> tuple[
torch.Tensor,
torch.Tensor | MoEOutput,
torch.Tensor,
torch.Tensor,
torch.Tensor,
@@ -432,6 +483,7 @@ class DeepseekV4DecoderLayer(nn.Module):
# sublayer's pre-mix: attention uses the pre-mix carried in (identity
# for the first layer), the FFN uses this layer's attention pre-mix.
if residual is None:
assert isinstance(x, torch.Tensor)
if x.dim() == 2:
# First layer: the stream is the embedding broadcast to hc
# copies and the identity pre-mix selects copy 0.
@@ -471,7 +523,9 @@ class DeepseekV4DecoderLayer(nn.Module):
# Engram injection happens between the previous sublayer's post
# and this block's pre, on the full hc stream, so the mix
# coefficients see the injected stream. The injection also keeps
# the post out of the pre-norm GEMM's fused prologue.
# the post out of the pre-norm GEMM's fused prologue. The layer
# before an engram layer finalizes its own MoE.
assert isinstance(x, torch.Tensor)
if self.fuse_mhc_all_reduce:
x = tensor_model_parallel_all_reduce(x)
previous_post = mhc_post_tilelang(x, residual, post_mix, res_mix)
@@ -552,7 +606,11 @@ class DeepseekV4DecoderLayer(nn.Module):
stream=mhc_stream,
reduce_results=self.fuse_mhc_all_reduce,
)
x = self.ffn(x, input_ids, mega_gate_metadata)
if self.fuse_mhc_all_reduce and self.ffn.defers_finalize(x.shape[0]):
# The next layer's first boundary finalizes it with the all-reduce.
x = self.ffn.forward_unfinalized(x, input_ids)
else:
x = self.ffn(x, input_ids, mega_gate_metadata)
if mhc_stream is not None:
torch.cuda.current_stream().wait_stream(mhc_stream)
return x, residual, post_mix, res_mix, ffn_pre, previous_aux
@@ -591,6 +649,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
self.fuse_mhc_all_reduce = mhc_stream is not None and supports_mhc_all_reduce(
vllm_config
)
if self.fuse_mhc_all_reduce:
init_mhc_all_reduce(vllm_config)
# Reserved topk indices buffer for all Indexer layers to reuse.
self.topk_indices_buffer = torch.empty(
@@ -670,6 +730,16 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
),
prefix=f"{prefix}.layers",
)
if self.fuse_mhc_all_reduce:
# A MoE's top-k finalize folds into the next layer's first mHC
# boundary, which the last local layer lacks and an engram layer
# replaces with its own all-reduce.
local_layers = list(islice(self.layers, self.start_layer, self.end_layer))
for layer, successor in zip(local_layers, local_layers[1:]):
assert isinstance(layer, DeepseekV4DecoderLayer)
assert isinstance(successor, DeepseekV4DecoderLayer)
if successor.engram is None:
layer.ffn.defer_finalize()
# The n-gram hash needs a slot-keyed rolling store of compressed ids
# (chunked prefill / decode lookback); key it off the first local
@@ -874,6 +944,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
if self.use_sequence_parallel:
previous_aux = sp_all_gather(previous_aux)[:full_num_tokens]
aux_hidden_by_layer[idx] = previous_aux
# Without a successor boundary, the last layer finalized its own MoE.
assert isinstance(hidden_states, torch.Tensor)
if layer is not None:
# The last layer has no successor to fold its post into.
if self.fuse_mhc_all_reduce:
@@ -0,0 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CuTe DSL kernels for DSV4.1 mHC."""
from .all_reduce_mhc import AllReduceMHC
__all__ = ["AllReduceMHC"]
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,396 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from FlashInfer's flashinfer/comm/mnnvl_cutedsl/cute_dsl_primitives.py
# and runtime.py (flashinfer-ai/flashinfer@139af6f6).
"""PTX building blocks and launch helpers for the DSV4.1 LL all-reduce."""
import cuda.bindings.driver as cuda
import cutlass
import cutlass.cute as cute
import torch
from cutlass import Float32, Int32, Int64, Uint16, Uint32
from cutlass._mlir import ir
from cutlass._mlir.dialects import llvm, vector
from cutlass.cute.runtime import from_dlpack, make_fake_compact_tensor
from cutlass.cutlass_dsl import T, dsl_user_op
WARP_SIZE = 32
VEC_BF16 = 8
QUAD_BF16 = 4
NEGATIVE_ZERO_BF16_BITS = 0x8000
NEGATIVE_ZERO_BF16_PAIR = 0x80008000
def _asm(result, operands, text, constraints, *, side_effects, loc, ip):
return llvm.inline_asm(
result,
operands,
text,
constraints,
has_side_effects=side_effects,
is_align_stack=False,
asm_dialect=llvm.AsmDialect.AD_ATT,
loc=loc,
ip=ip,
)
@dsl_user_op
def load_global_u32x4(
pointer: cute.Pointer,
*,
volatile: cutlass.Constexpr[bool] = False,
loc=None,
ip=None,
):
address = pointer.toint(loc=loc, ip=ip)
opcode = "ld.volatile.global.v4.u32" if volatile else "ld.global.v4.u32"
loaded = _asm(
llvm.StructType.get_literal([T.i32()] * 4),
[address.ir_value(loc=loc, ip=ip)],
f"{opcode} {{$0, $1, $2, $3}}, [$4];",
"=r,=r,=r,=r,l",
side_effects=volatile,
loc=loc,
ip=ip,
)
packed = vector.from_elements(
ir.VectorType.get([4], T.i32(), loc=loc),
[
llvm.extractvalue(T.i32(), loaded, [index], loc=loc, ip=ip)
for index in range(4)
],
loc=loc,
ip=ip,
)
return cute.TensorSSA(packed, 4, Uint32)
@dsl_user_op
def load_global_u32x2(pointer: cute.Pointer, *, loc=None, ip=None):
address = pointer.toint(loc=loc, ip=ip)
loaded = _asm(
llvm.StructType.get_literal([T.i32()] * 2),
[address.ir_value(loc=loc, ip=ip)],
"ld.global.v2.u32 {$0, $1}, [$2];",
"=r,=r,l",
side_effects=False,
loc=loc,
ip=ip,
)
packed = vector.from_elements(
ir.VectorType.get([2], T.i32(), loc=loc),
[
llvm.extractvalue(T.i32(), loaded, [index], loc=loc, ip=ip)
for index in range(2)
],
loc=loc,
ip=ip,
)
return cute.TensorSSA(packed, 2, Uint32)
@dsl_user_op
def load_global_bf16_as_f32(address: Int64, *, loc=None, ip=None) -> Float32:
return Float32(
_asm(
T.f32(),
[address.ir_value(loc=loc, ip=ip)],
"{\n\t.reg .b16 bits;\n\tld.global.b16 bits, [$1];\n\t"
"cvt.f32.bf16 $0, bits;\n\t}",
"=f,l",
side_effects=False,
loc=loc,
ip=ip,
)
)
@dsl_user_op
def store_global_u32x4(address: Int64, packed, *, loc=None, ip=None) -> None:
words = [packed[index].ir_value(loc=loc, ip=ip) for index in range(4)]
_asm(
None,
[address.ir_value(loc=loc, ip=ip), *words],
"st.global.v4.u32 [$0], {$1, $2, $3, $4};",
"l,r,r,r,r",
side_effects=True,
loc=loc,
ip=ip,
)
@dsl_user_op
def store_lamport_sentinel_u32x4(address: Int64, *, loc=None, ip=None) -> None:
sentinel = Uint32(NEGATIVE_ZERO_BF16_PAIR).ir_value(loc=loc, ip=ip)
_asm(
None,
[address.ir_value(loc=loc, ip=ip), sentinel, sentinel, sentinel, sentinel],
"st.global.v4.u32 [$0], {$1, $2, $3, $4};",
"l,r,r,r,r",
side_effects=True,
loc=loc,
ip=ip,
)
@dsl_user_op
def load_volatile_u32(pointer: cute.Pointer, *, loc=None, ip=None) -> Uint32:
address = pointer.toint(loc=loc, ip=ip)
return Uint32(
_asm(
T.i32(),
[address.ir_value(loc=loc, ip=ip)],
"ld.volatile.global.u32 $0, [$1];",
"=r,l",
side_effects=True,
loc=loc,
ip=ip,
)
)
@dsl_user_op
def store_global_u32(
pointer: cute.Pointer, value: Uint32, *, loc=None, ip=None
) -> None:
address = pointer.toint(loc=loc, ip=ip)
_asm(
None,
[address.ir_value(loc=loc, ip=ip), value.ir_value(loc=loc, ip=ip)],
"st.global.u32 [$0], $1;",
"l,r",
side_effects=True,
loc=loc,
ip=ip,
)
@dsl_user_op
def packed_u32x4_to_bf16x8(packed, *, loc=None, ip=None):
values = llvm.bitcast(
ir.VectorType.get([VEC_BF16], cutlass.BFloat16.mlir_type, loc=loc),
packed.ir_value(loc=loc, ip=ip),
loc=loc,
ip=ip,
)
return cute.TensorSSA(values, VEC_BF16, cutlass.BFloat16)
@dsl_user_op
def packed_u32x2_to_bf16x4(packed, *, loc=None, ip=None):
values = llvm.bitcast(
ir.VectorType.get([QUAD_BF16], cutlass.BFloat16.mlir_type, loc=loc),
packed.ir_value(loc=loc, ip=ip),
loc=loc,
ip=ip,
)
return cute.TensorSSA(values, QUAD_BF16, cutlass.BFloat16)
@dsl_user_op
def bf16x4_to_packed_u32x2(values, *, loc=None, ip=None):
packed = llvm.bitcast(
ir.VectorType.get([2], T.i32(), loc=loc),
values.ir_value(loc=loc, ip=ip),
loc=loc,
ip=ip,
)
return cute.TensorSSA(packed, 2, Uint32)
@dsl_user_op
def bf16x8_to_packed_u32x4(values, *, loc=None, ip=None):
packed = llvm.bitcast(
ir.VectorType.get([4], T.i32(), loc=loc),
values.ir_value(loc=loc, ip=ip),
loc=loc,
ip=ip,
)
return cute.TensorSSA(packed, 4, Uint32)
@cute.jit
def sanitize_negative_zero_u32(word: Uint32) -> Uint32:
low = Uint16(word & Uint32(0xFFFF))
high = Uint16(word >> Uint32(16))
if low == Uint16(NEGATIVE_ZERO_BF16_BITS):
word = word & Uint32(0xFFFF0000)
if high == Uint16(NEGATIVE_ZERO_BF16_BITS):
word = word & Uint32(0x0000FFFF)
return word
@cute.jit
def sanitize_negative_zero_u32x4(packed):
sanitized = cute.make_rmem_tensor(cute.make_layout((4,)), Uint32)
for index in cutlass.range_constexpr(4):
sanitized[index] = sanitize_negative_zero_u32(packed[index])
return sanitized.load()
@cute.jit
def sanitize_negative_zero_u32x2(packed):
sanitized = cute.make_rmem_tensor(cute.make_layout((2,)), Uint32)
for index in cutlass.range_constexpr(2):
sanitized[index] = sanitize_negative_zero_u32(packed[index])
return sanitized.load()
@cute.jit
def fragment_has_negative_zero(packed):
dirty = False
for index in cutlass.range_constexpr(4):
word = packed[index]
dirty = (
dirty
| (Uint16(word & Uint32(0xFFFF)) == Uint16(NEGATIVE_ZERO_BF16_BITS))
| (Uint16(word >> Uint32(16)) == Uint16(NEGATIVE_ZERO_BF16_BITS))
)
return dirty
@dsl_user_op
def map_shared_to_peer(
smem_pointer: cute.Pointer, peer_rank: Int32, *, loc=None, ip=None
) -> Int32:
address = smem_pointer.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip)
return Int32(
_asm(
T.i32(),
[address, peer_rank.ir_value(loc=loc, ip=ip)],
"mapa.shared::cluster.u32 $0, $1, $2;",
"=r,r,r",
side_effects=False,
loc=loc,
ip=ip,
)
)
@dsl_user_op
def store_shared_cluster_f32(
remote_address: Int32, value: Float32, *, loc=None, ip=None
) -> None:
_asm(
None,
[remote_address.ir_value(loc=loc, ip=ip), value.ir_value(loc=loc, ip=ip)],
"st.shared::cluster.f32 [$0], $1;",
"r,f",
side_effects=True,
loc=loc,
ip=ip,
)
@dsl_user_op
def f32_to_bf16_bits(value: Float32, *, loc=None, ip=None) -> Uint32:
return Uint32(
_asm(
T.i32(),
[value.ir_value(loc=loc, ip=ip)],
"{\n\t.reg .b16 bits;\n\tcvt.rn.bf16.f32 bits, $1;\n\t"
"cvt.u32.u16 $0, bits;\n\t}",
"=r,f",
side_effects=False,
loc=loc,
ip=ip,
)
)
@dsl_user_op
def shuffle_sync_idx_u32(
value: Uint32, source_lane: Int32, *, loc=None, ip=None
) -> Uint32:
return Uint32(
_asm(
T.i32(),
[value.ir_value(loc=loc, ip=ip), source_lane.ir_value(loc=loc, ip=ip)],
"shfl.sync.idx.b32 $0, $1, $2, 0x1f, 0xffffffff;",
"=r,r,r",
# Preserve full-warp execution across later divergent consumers.
side_effects=True,
loc=loc,
ip=ip,
)
)
@dsl_user_op
def stmc_bf16x2(address: Int64, packed: Uint32, *, loc=None, ip=None) -> None:
_asm(
None,
[address.ir_value(loc=loc, ip=ip), packed.ir_value(loc=loc, ip=ip)],
"multimem.st.relaxed.sys.global.bf16x2 [$0], $1;",
"l,r",
side_effects=True,
loc=loc,
ip=ip,
)
@dsl_user_op
def stmc_bf16x4(address: Int64, values, *, loc=None, ip=None) -> None:
words = [values[index].ir_value(loc=loc, ip=ip) for index in range(2)]
_asm(
None,
[address.ir_value(loc=loc, ip=ip), *words],
"multimem.st.relaxed.sys.global.v2.bf16x2 [$0], {$1, $2};",
"l,r,r",
side_effects=True,
loc=loc,
ip=ip,
)
@dsl_user_op
def stmc_bf16x8(address: Int64, values, *, loc=None, ip=None) -> None:
words = [values[index].ir_value(loc=loc, ip=ip) for index in range(4)]
_asm(
None,
[address.ir_value(loc=loc, ip=ip), *words],
"multimem.st.relaxed.sys.global.v4.bf16x2 [$0], {$1, $2, $3, $4};",
"l,r,r,r,r",
side_effects=True,
loc=loc,
ip=ip,
)
class _GraphSafeDLPack:
__slots__ = ("tensor",)
def __init__(self, tensor: torch.Tensor) -> None:
self.tensor = tensor
def __dlpack__(self, stream=None):
# stream=-1 skips producer sync; CuTe launches on the current stream,
# which is the capturing one under CUDA graphs.
return self.tensor.__dlpack__(stream=-1)
def __dlpack_device__(self):
return self.tensor.__dlpack_device__()
def to_cute(tensor: torch.Tensor, alignment: int) -> cute.Tensor:
return from_dlpack(_GraphSafeDLPack(tensor.detach()), assumed_align=alignment)
def to_cute_dynamic(
tensor: torch.Tensor, alignment: int, *, divisibility: int
) -> cute.Tensor:
return to_cute(tensor, alignment).mark_compact_shape_dynamic(
mode=0, divisibility=divisibility
)
def make_fake_dynamic_compact_tensor(dtype, *, alignment: int, divisibility: int):
return make_fake_compact_tensor(
dtype, (cute.sym_int32(divisibility=divisibility),), assumed_align=alignment
)
def current_cu_stream() -> cuda.CUstream:
return cuda.CUstream(torch.cuda.current_stream().cuda_stream)
+73 -18
View File
@@ -9,10 +9,15 @@ import torch
from vllm.config import VllmConfig
from vllm.distributed import get_tp_group
from vllm.logger import init_logger
from vllm.model_executor.kernels.mhc.tilelang import (
mhc_fused_post_pre_delayed_tilelang,
mhc_post_tilelang,
)
from vllm.model_executor.layers.fused_moe.moe_output import (
MoEOutput,
UnfinalizedMoEOutput,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
@@ -24,9 +29,17 @@ from .mega_mhc import (
if TYPE_CHECKING:
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
from .cute_dsl import AllReduceMHC
logger = init_logger(__name__)
# GB200 TP4/FlashInfer improves through 16 tokens; larger screens tie or regress.
MHC_OVERLAP_MAX_TOKENS = 16
# The fused all-reduce + mHC kernel of the overlap path, built once by
# init_mhc_all_reduce.
_all_reduce_mhc: "AllReduceMHC | None" = None
def supports_mhc_overlap(vllm_config: VllmConfig) -> bool:
"""Check kernel requirements and safety of sharing the coefficient stream."""
@@ -53,10 +66,35 @@ def supports_mhc_all_reduce(vllm_config: VllmConfig) -> bool:
or config.hc_mult != 4
):
return False
# The custom all-reduce's MNNVL buffers succeed exactly when NVLink
# multicast does, which the fused kernel's own mailbox needs too.
comm = cast("CudaCommunicator", get_tp_group().device_communicator).ca_comm
return comm is not None and bool(comm.mnnvl_lamport_ag_multicast_ptr)
def init_mhc_all_reduce(vllm_config: VllmConfig) -> None:
"""Build the fused all-reduce + mHC kernel the overlap path reduces with.
Collective over the TP group, so every rank must call it, and only when
``supports_mhc_all_reduce`` holds.
"""
global _all_reduce_mhc
from .cute_dsl import AllReduceMHC
config = vllm_config.model_config.hf_config
_all_reduce_mhc = AllReduceMHC(
hidden_size=config.hidden_size,
hc_mult=config.hc_mult,
max_num_tokens=MHC_OVERLAP_MAX_TOKENS,
top_k=config.num_experts_per_tok,
device=current_platform.current_device(),
)
logger.info_once(
"DSV4.1 mHC: CuTe DSL Lamport all-reduce fused with mHC for up to %d tokens.",
MHC_OVERLAP_MAX_TOKENS,
)
def mhc_pre_delayed_overlap(
residual: torch.Tensor,
fn: torch.Tensor,
@@ -151,7 +189,7 @@ def mhc_pre_delayed_overlap(
def mhc_shifted_post_pre(
x: torch.Tensor,
x: torch.Tensor | MoEOutput,
residual: torch.Tensor,
post_layer_mix: torch.Tensor,
comb_res_mix: torch.Tensor,
@@ -175,39 +213,55 @@ def mhc_shifted_post_pre(
]:
"""Dispatch shifted post/pre to overlap, Mega-mHC, or fused TileLang.
A MoE output left unfinalized is finalized inside the fused all-reduce.
When stream is supplied, join it before consuming the returned coefficients.
"""
layer_input = None
if reduce_results:
tp = get_tp_group()
if isinstance(x, MoEOutput):
assert _all_reduce_mhc is not None, "init_mhc_all_reduce was not called"
assert pre_mix is not None and norm_weight is not None
routed = x.routed
assert isinstance(routed, UnfinalizedMoEOutput)
assert x.shared_output is not None
residual, layer_input = _all_reduce_mhc.finalize(
routed.gemm2_permuted,
routed.expert_weights,
routed.expanded_idx_to_permuted_idx,
x.shared_output,
residual,
post_layer_mix,
comb_res_mix,
pre_mix,
norm_weight,
norm_eps,
)
if stream is None:
# Outside FULL graphs, compute the next mixes inline.
stream = torch.cuda.current_stream()
elif reduce_results:
if stream is not None and 0 < x.shape[0] <= MHC_OVERLAP_MAX_TOKENS:
comm = cast("CudaCommunicator", tp.device_communicator).ca_comm
assert comm is not None and comm.mnnvl_lamport_epochs is not None
output = torch.empty_like(residual)
layer_input = torch.empty_like(x)
torch.ops._C_custom_ar.all_reduce_mhc(
assert _all_reduce_mhc is not None, "init_mhc_all_reduce was not called"
assert pre_mix is not None and norm_weight is not None
residual, layer_input = _all_reduce_mhc(
x,
residual,
post_layer_mix,
comb_res_mix,
pre_mix,
norm_weight,
output,
layer_input,
comm.mnnvl_lamport_ag_local_ptr,
comm.mnnvl_lamport_ag_multicast_ptr,
comm.mnnvl_lamport_epochs[0],
comm.rank,
comm.mnnvl_buffer_size,
norm_eps,
)
residual = output
else:
x = tp.all_reduce(x)
x = get_tp_group().all_reduce(x)
if stream is not None:
if layer_input is None:
assert isinstance(x, torch.Tensor)
residual = mhc_post_tilelang(x, residual, post_layer_mix, comb_res_mix)
aux = residual.mean(dim=1) if capture_aux else x.new_empty(0, x.shape[1])
aux = (
residual.mean(dim=1)
if capture_aux
else residual.new_empty(0, residual.shape[-1])
)
pre_outputs = mhc_pre_delayed_overlap(
residual,
fn,
@@ -226,6 +280,7 @@ def mhc_shifted_post_pre(
)
return residual, *pre_outputs, aux
assert isinstance(x, torch.Tensor)
if can_use_mega_mhc(x, residual, pre_mix, norm_weight, capture_aux):
assert pre_mix is not None and norm_weight is not None
outputs = mhc_shifted_post_pre_deep_gemm(