[Kernel][DSV4.1] Fuse MoE finalize into the TP all-reduce + mHC boundary (#58586)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
379e9a1ea8
commit
77871126f9
@@ -33,6 +33,14 @@ from vllm.model_executor.kernels.mhc.triton import hc_collapse_triton
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe import (
|
||||
TrtLlmMxfp4ExpertsModular,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import (
|
||||
MoEOutput,
|
||||
UnfinalizedMoEOutput,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.runner.moe_runner import _unpack
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -94,6 +102,7 @@ from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
|
||||
from .engram import Engram, can_share_engram_tables, gather_engram_hashes
|
||||
from .ops.mhc import (
|
||||
MHC_OVERLAP_MAX_TOKENS,
|
||||
init_mhc_all_reduce,
|
||||
mhc_pre_delayed_overlap,
|
||||
mhc_shifted_post_pre,
|
||||
supports_mhc_all_reduce,
|
||||
@@ -135,6 +144,48 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
|
||||
image_sentinel_lo=IMAGE_SENTINEL_BASE_ID,
|
||||
)
|
||||
|
||||
def defer_finalize(self) -> None:
|
||||
"""Leave the routed top-k reduction to the next fused all-reduce + mHC.
|
||||
|
||||
Only the modular TRTLLM MXFP4 experts can stop after GEMM2; the fused
|
||||
kernel takes as many tokens as the mHC overlap path.
|
||||
"""
|
||||
moe_config = self.experts.moe_config
|
||||
experts_cls = getattr(
|
||||
self.experts.routed_experts.quant_method, "experts_cls", None
|
||||
)
|
||||
if (
|
||||
experts_cls is TrtLlmMxfp4ExpertsModular
|
||||
and self.experts.routed_scaling_factor == 1.0
|
||||
):
|
||||
moe_config.defer_moe_finalize(MHC_OVERLAP_MAX_TOKENS)
|
||||
if moe_config.use_deferred_moe_finalize:
|
||||
logger.info_once(
|
||||
"DSV4.1 mHC: MoE top-k finalize fused into the all-reduce for "
|
||||
"up to %d tokens.",
|
||||
MHC_OVERLAP_MAX_TOKENS,
|
||||
)
|
||||
|
||||
def defers_finalize(self, num_tokens: int) -> bool:
|
||||
return (
|
||||
not self.use_mega_moe
|
||||
and self.experts.moe_config.should_defer_moe_finalize(num_tokens)
|
||||
)
|
||||
|
||||
def forward_unfinalized(
|
||||
self, hidden_states: torch.Tensor, input_ids: torch.Tensor | None
|
||||
) -> MoEOutput:
|
||||
"""``forward`` with the routed top-k reduction and all-reduce left open."""
|
||||
# The runner's custom op returns tensors only, so run its body directly.
|
||||
shared_output, routed = _unpack(
|
||||
self.experts._forward_impl(
|
||||
hidden_states, hidden_states, hidden_states, input_ids
|
||||
)
|
||||
)
|
||||
assert shared_output is not None
|
||||
assert isinstance(routed, UnfinalizedMoEOutput)
|
||||
return MoEOutput(routed=routed, shared_output=shared_output)
|
||||
|
||||
|
||||
def _select_dsv4_attn_cls(vllm_config: VllmConfig) -> type[DeepseekV4Attention]:
|
||||
"""Pick the CUDA sparse-MLA attention class for the configured backend.
|
||||
@@ -394,7 +445,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
x: torch.Tensor | MoEOutput,
|
||||
positions: torch.Tensor,
|
||||
input_ids: torch.Tensor | None,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
@@ -407,7 +458,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
capture_previous_aux: bool = False,
|
||||
mega_gate_metadata: MegaGateRoutingMetadata | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor | MoEOutput,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
@@ -432,6 +483,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
# sublayer's pre-mix: attention uses the pre-mix carried in (identity
|
||||
# for the first layer), the FFN uses this layer's attention pre-mix.
|
||||
if residual is None:
|
||||
assert isinstance(x, torch.Tensor)
|
||||
if x.dim() == 2:
|
||||
# First layer: the stream is the embedding broadcast to hc
|
||||
# copies and the identity pre-mix selects copy 0.
|
||||
@@ -471,7 +523,9 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
# Engram injection happens between the previous sublayer's post
|
||||
# and this block's pre, on the full hc stream, so the mix
|
||||
# coefficients see the injected stream. The injection also keeps
|
||||
# the post out of the pre-norm GEMM's fused prologue.
|
||||
# the post out of the pre-norm GEMM's fused prologue. The layer
|
||||
# before an engram layer finalizes its own MoE.
|
||||
assert isinstance(x, torch.Tensor)
|
||||
if self.fuse_mhc_all_reduce:
|
||||
x = tensor_model_parallel_all_reduce(x)
|
||||
previous_post = mhc_post_tilelang(x, residual, post_mix, res_mix)
|
||||
@@ -552,7 +606,11 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
stream=mhc_stream,
|
||||
reduce_results=self.fuse_mhc_all_reduce,
|
||||
)
|
||||
x = self.ffn(x, input_ids, mega_gate_metadata)
|
||||
if self.fuse_mhc_all_reduce and self.ffn.defers_finalize(x.shape[0]):
|
||||
# The next layer's first boundary finalizes it with the all-reduce.
|
||||
x = self.ffn.forward_unfinalized(x, input_ids)
|
||||
else:
|
||||
x = self.ffn(x, input_ids, mega_gate_metadata)
|
||||
if mhc_stream is not None:
|
||||
torch.cuda.current_stream().wait_stream(mhc_stream)
|
||||
return x, residual, post_mix, res_mix, ffn_pre, previous_aux
|
||||
@@ -591,6 +649,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
self.fuse_mhc_all_reduce = mhc_stream is not None and supports_mhc_all_reduce(
|
||||
vllm_config
|
||||
)
|
||||
if self.fuse_mhc_all_reduce:
|
||||
init_mhc_all_reduce(vllm_config)
|
||||
|
||||
# Reserved topk indices buffer for all Indexer layers to reuse.
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
@@ -670,6 +730,16 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
if self.fuse_mhc_all_reduce:
|
||||
# A MoE's top-k finalize folds into the next layer's first mHC
|
||||
# boundary, which the last local layer lacks and an engram layer
|
||||
# replaces with its own all-reduce.
|
||||
local_layers = list(islice(self.layers, self.start_layer, self.end_layer))
|
||||
for layer, successor in zip(local_layers, local_layers[1:]):
|
||||
assert isinstance(layer, DeepseekV4DecoderLayer)
|
||||
assert isinstance(successor, DeepseekV4DecoderLayer)
|
||||
if successor.engram is None:
|
||||
layer.ffn.defer_finalize()
|
||||
|
||||
# The n-gram hash needs a slot-keyed rolling store of compressed ids
|
||||
# (chunked prefill / decode lookback); key it off the first local
|
||||
@@ -874,6 +944,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
if self.use_sequence_parallel:
|
||||
previous_aux = sp_all_gather(previous_aux)[:full_num_tokens]
|
||||
aux_hidden_by_layer[idx] = previous_aux
|
||||
# Without a successor boundary, the last layer finalized its own MoE.
|
||||
assert isinstance(hidden_states, torch.Tensor)
|
||||
if layer is not None:
|
||||
# The last layer has no successor to fold its post into.
|
||||
if self.fuse_mhc_all_reduce:
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""CuTe DSL kernels for DSV4.1 mHC."""
|
||||
|
||||
from .all_reduce_mhc import AllReduceMHC
|
||||
|
||||
__all__ = ["AllReduceMHC"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,396 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adapted from FlashInfer's flashinfer/comm/mnnvl_cutedsl/cute_dsl_primitives.py
|
||||
# and runtime.py (flashinfer-ai/flashinfer@139af6f6).
|
||||
|
||||
"""PTX building blocks and launch helpers for the DSV4.1 LL all-reduce."""
|
||||
|
||||
import cuda.bindings.driver as cuda
|
||||
import cutlass
|
||||
import cutlass.cute as cute
|
||||
import torch
|
||||
from cutlass import Float32, Int32, Int64, Uint16, Uint32
|
||||
from cutlass._mlir import ir
|
||||
from cutlass._mlir.dialects import llvm, vector
|
||||
from cutlass.cute.runtime import from_dlpack, make_fake_compact_tensor
|
||||
from cutlass.cutlass_dsl import T, dsl_user_op
|
||||
|
||||
WARP_SIZE = 32
|
||||
VEC_BF16 = 8
|
||||
QUAD_BF16 = 4
|
||||
NEGATIVE_ZERO_BF16_BITS = 0x8000
|
||||
NEGATIVE_ZERO_BF16_PAIR = 0x80008000
|
||||
|
||||
|
||||
def _asm(result, operands, text, constraints, *, side_effects, loc, ip):
|
||||
return llvm.inline_asm(
|
||||
result,
|
||||
operands,
|
||||
text,
|
||||
constraints,
|
||||
has_side_effects=side_effects,
|
||||
is_align_stack=False,
|
||||
asm_dialect=llvm.AsmDialect.AD_ATT,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def load_global_u32x4(
|
||||
pointer: cute.Pointer,
|
||||
*,
|
||||
volatile: cutlass.Constexpr[bool] = False,
|
||||
loc=None,
|
||||
ip=None,
|
||||
):
|
||||
address = pointer.toint(loc=loc, ip=ip)
|
||||
opcode = "ld.volatile.global.v4.u32" if volatile else "ld.global.v4.u32"
|
||||
loaded = _asm(
|
||||
llvm.StructType.get_literal([T.i32()] * 4),
|
||||
[address.ir_value(loc=loc, ip=ip)],
|
||||
f"{opcode} {{$0, $1, $2, $3}}, [$4];",
|
||||
"=r,=r,=r,=r,l",
|
||||
side_effects=volatile,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
packed = vector.from_elements(
|
||||
ir.VectorType.get([4], T.i32(), loc=loc),
|
||||
[
|
||||
llvm.extractvalue(T.i32(), loaded, [index], loc=loc, ip=ip)
|
||||
for index in range(4)
|
||||
],
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(packed, 4, Uint32)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def load_global_u32x2(pointer: cute.Pointer, *, loc=None, ip=None):
|
||||
address = pointer.toint(loc=loc, ip=ip)
|
||||
loaded = _asm(
|
||||
llvm.StructType.get_literal([T.i32()] * 2),
|
||||
[address.ir_value(loc=loc, ip=ip)],
|
||||
"ld.global.v2.u32 {$0, $1}, [$2];",
|
||||
"=r,=r,l",
|
||||
side_effects=False,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
packed = vector.from_elements(
|
||||
ir.VectorType.get([2], T.i32(), loc=loc),
|
||||
[
|
||||
llvm.extractvalue(T.i32(), loaded, [index], loc=loc, ip=ip)
|
||||
for index in range(2)
|
||||
],
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(packed, 2, Uint32)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def load_global_bf16_as_f32(address: Int64, *, loc=None, ip=None) -> Float32:
|
||||
return Float32(
|
||||
_asm(
|
||||
T.f32(),
|
||||
[address.ir_value(loc=loc, ip=ip)],
|
||||
"{\n\t.reg .b16 bits;\n\tld.global.b16 bits, [$1];\n\t"
|
||||
"cvt.f32.bf16 $0, bits;\n\t}",
|
||||
"=f,l",
|
||||
side_effects=False,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def store_global_u32x4(address: Int64, packed, *, loc=None, ip=None) -> None:
|
||||
words = [packed[index].ir_value(loc=loc, ip=ip) for index in range(4)]
|
||||
_asm(
|
||||
None,
|
||||
[address.ir_value(loc=loc, ip=ip), *words],
|
||||
"st.global.v4.u32 [$0], {$1, $2, $3, $4};",
|
||||
"l,r,r,r,r",
|
||||
side_effects=True,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def store_lamport_sentinel_u32x4(address: Int64, *, loc=None, ip=None) -> None:
|
||||
sentinel = Uint32(NEGATIVE_ZERO_BF16_PAIR).ir_value(loc=loc, ip=ip)
|
||||
_asm(
|
||||
None,
|
||||
[address.ir_value(loc=loc, ip=ip), sentinel, sentinel, sentinel, sentinel],
|
||||
"st.global.v4.u32 [$0], {$1, $2, $3, $4};",
|
||||
"l,r,r,r,r",
|
||||
side_effects=True,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def load_volatile_u32(pointer: cute.Pointer, *, loc=None, ip=None) -> Uint32:
|
||||
address = pointer.toint(loc=loc, ip=ip)
|
||||
return Uint32(
|
||||
_asm(
|
||||
T.i32(),
|
||||
[address.ir_value(loc=loc, ip=ip)],
|
||||
"ld.volatile.global.u32 $0, [$1];",
|
||||
"=r,l",
|
||||
side_effects=True,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def store_global_u32(
|
||||
pointer: cute.Pointer, value: Uint32, *, loc=None, ip=None
|
||||
) -> None:
|
||||
address = pointer.toint(loc=loc, ip=ip)
|
||||
_asm(
|
||||
None,
|
||||
[address.ir_value(loc=loc, ip=ip), value.ir_value(loc=loc, ip=ip)],
|
||||
"st.global.u32 [$0], $1;",
|
||||
"l,r",
|
||||
side_effects=True,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def packed_u32x4_to_bf16x8(packed, *, loc=None, ip=None):
|
||||
values = llvm.bitcast(
|
||||
ir.VectorType.get([VEC_BF16], cutlass.BFloat16.mlir_type, loc=loc),
|
||||
packed.ir_value(loc=loc, ip=ip),
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(values, VEC_BF16, cutlass.BFloat16)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def packed_u32x2_to_bf16x4(packed, *, loc=None, ip=None):
|
||||
values = llvm.bitcast(
|
||||
ir.VectorType.get([QUAD_BF16], cutlass.BFloat16.mlir_type, loc=loc),
|
||||
packed.ir_value(loc=loc, ip=ip),
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(values, QUAD_BF16, cutlass.BFloat16)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def bf16x4_to_packed_u32x2(values, *, loc=None, ip=None):
|
||||
packed = llvm.bitcast(
|
||||
ir.VectorType.get([2], T.i32(), loc=loc),
|
||||
values.ir_value(loc=loc, ip=ip),
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(packed, 2, Uint32)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def bf16x8_to_packed_u32x4(values, *, loc=None, ip=None):
|
||||
packed = llvm.bitcast(
|
||||
ir.VectorType.get([4], T.i32(), loc=loc),
|
||||
values.ir_value(loc=loc, ip=ip),
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
return cute.TensorSSA(packed, 4, Uint32)
|
||||
|
||||
|
||||
@cute.jit
|
||||
def sanitize_negative_zero_u32(word: Uint32) -> Uint32:
|
||||
low = Uint16(word & Uint32(0xFFFF))
|
||||
high = Uint16(word >> Uint32(16))
|
||||
if low == Uint16(NEGATIVE_ZERO_BF16_BITS):
|
||||
word = word & Uint32(0xFFFF0000)
|
||||
if high == Uint16(NEGATIVE_ZERO_BF16_BITS):
|
||||
word = word & Uint32(0x0000FFFF)
|
||||
return word
|
||||
|
||||
|
||||
@cute.jit
|
||||
def sanitize_negative_zero_u32x4(packed):
|
||||
sanitized = cute.make_rmem_tensor(cute.make_layout((4,)), Uint32)
|
||||
for index in cutlass.range_constexpr(4):
|
||||
sanitized[index] = sanitize_negative_zero_u32(packed[index])
|
||||
return sanitized.load()
|
||||
|
||||
|
||||
@cute.jit
|
||||
def sanitize_negative_zero_u32x2(packed):
|
||||
sanitized = cute.make_rmem_tensor(cute.make_layout((2,)), Uint32)
|
||||
for index in cutlass.range_constexpr(2):
|
||||
sanitized[index] = sanitize_negative_zero_u32(packed[index])
|
||||
return sanitized.load()
|
||||
|
||||
|
||||
@cute.jit
|
||||
def fragment_has_negative_zero(packed):
|
||||
dirty = False
|
||||
for index in cutlass.range_constexpr(4):
|
||||
word = packed[index]
|
||||
dirty = (
|
||||
dirty
|
||||
| (Uint16(word & Uint32(0xFFFF)) == Uint16(NEGATIVE_ZERO_BF16_BITS))
|
||||
| (Uint16(word >> Uint32(16)) == Uint16(NEGATIVE_ZERO_BF16_BITS))
|
||||
)
|
||||
return dirty
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def map_shared_to_peer(
|
||||
smem_pointer: cute.Pointer, peer_rank: Int32, *, loc=None, ip=None
|
||||
) -> Int32:
|
||||
address = smem_pointer.toint(loc=loc, ip=ip).ir_value(loc=loc, ip=ip)
|
||||
return Int32(
|
||||
_asm(
|
||||
T.i32(),
|
||||
[address, peer_rank.ir_value(loc=loc, ip=ip)],
|
||||
"mapa.shared::cluster.u32 $0, $1, $2;",
|
||||
"=r,r,r",
|
||||
side_effects=False,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def store_shared_cluster_f32(
|
||||
remote_address: Int32, value: Float32, *, loc=None, ip=None
|
||||
) -> None:
|
||||
_asm(
|
||||
None,
|
||||
[remote_address.ir_value(loc=loc, ip=ip), value.ir_value(loc=loc, ip=ip)],
|
||||
"st.shared::cluster.f32 [$0], $1;",
|
||||
"r,f",
|
||||
side_effects=True,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def f32_to_bf16_bits(value: Float32, *, loc=None, ip=None) -> Uint32:
|
||||
return Uint32(
|
||||
_asm(
|
||||
T.i32(),
|
||||
[value.ir_value(loc=loc, ip=ip)],
|
||||
"{\n\t.reg .b16 bits;\n\tcvt.rn.bf16.f32 bits, $1;\n\t"
|
||||
"cvt.u32.u16 $0, bits;\n\t}",
|
||||
"=r,f",
|
||||
side_effects=False,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def shuffle_sync_idx_u32(
|
||||
value: Uint32, source_lane: Int32, *, loc=None, ip=None
|
||||
) -> Uint32:
|
||||
return Uint32(
|
||||
_asm(
|
||||
T.i32(),
|
||||
[value.ir_value(loc=loc, ip=ip), source_lane.ir_value(loc=loc, ip=ip)],
|
||||
"shfl.sync.idx.b32 $0, $1, $2, 0x1f, 0xffffffff;",
|
||||
"=r,r,r",
|
||||
# Preserve full-warp execution across later divergent consumers.
|
||||
side_effects=True,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def stmc_bf16x2(address: Int64, packed: Uint32, *, loc=None, ip=None) -> None:
|
||||
_asm(
|
||||
None,
|
||||
[address.ir_value(loc=loc, ip=ip), packed.ir_value(loc=loc, ip=ip)],
|
||||
"multimem.st.relaxed.sys.global.bf16x2 [$0], $1;",
|
||||
"l,r",
|
||||
side_effects=True,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def stmc_bf16x4(address: Int64, values, *, loc=None, ip=None) -> None:
|
||||
words = [values[index].ir_value(loc=loc, ip=ip) for index in range(2)]
|
||||
_asm(
|
||||
None,
|
||||
[address.ir_value(loc=loc, ip=ip), *words],
|
||||
"multimem.st.relaxed.sys.global.v2.bf16x2 [$0], {$1, $2};",
|
||||
"l,r,r",
|
||||
side_effects=True,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
@dsl_user_op
|
||||
def stmc_bf16x8(address: Int64, values, *, loc=None, ip=None) -> None:
|
||||
words = [values[index].ir_value(loc=loc, ip=ip) for index in range(4)]
|
||||
_asm(
|
||||
None,
|
||||
[address.ir_value(loc=loc, ip=ip), *words],
|
||||
"multimem.st.relaxed.sys.global.v4.bf16x2 [$0], {$1, $2, $3, $4};",
|
||||
"l,r,r,r,r",
|
||||
side_effects=True,
|
||||
loc=loc,
|
||||
ip=ip,
|
||||
)
|
||||
|
||||
|
||||
class _GraphSafeDLPack:
|
||||
__slots__ = ("tensor",)
|
||||
|
||||
def __init__(self, tensor: torch.Tensor) -> None:
|
||||
self.tensor = tensor
|
||||
|
||||
def __dlpack__(self, stream=None):
|
||||
# stream=-1 skips producer sync; CuTe launches on the current stream,
|
||||
# which is the capturing one under CUDA graphs.
|
||||
return self.tensor.__dlpack__(stream=-1)
|
||||
|
||||
def __dlpack_device__(self):
|
||||
return self.tensor.__dlpack_device__()
|
||||
|
||||
|
||||
def to_cute(tensor: torch.Tensor, alignment: int) -> cute.Tensor:
|
||||
return from_dlpack(_GraphSafeDLPack(tensor.detach()), assumed_align=alignment)
|
||||
|
||||
|
||||
def to_cute_dynamic(
|
||||
tensor: torch.Tensor, alignment: int, *, divisibility: int
|
||||
) -> cute.Tensor:
|
||||
return to_cute(tensor, alignment).mark_compact_shape_dynamic(
|
||||
mode=0, divisibility=divisibility
|
||||
)
|
||||
|
||||
|
||||
def make_fake_dynamic_compact_tensor(dtype, *, alignment: int, divisibility: int):
|
||||
return make_fake_compact_tensor(
|
||||
dtype, (cute.sym_int32(divisibility=divisibility),), assumed_align=alignment
|
||||
)
|
||||
|
||||
|
||||
def current_cu_stream() -> cuda.CUstream:
|
||||
return cuda.CUstream(torch.cuda.current_stream().cuda_stream)
|
||||
@@ -9,10 +9,15 @@ import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
mhc_fused_post_pre_delayed_tilelang,
|
||||
mhc_post_tilelang,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import (
|
||||
MoEOutput,
|
||||
UnfinalizedMoEOutput,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
@@ -24,9 +29,17 @@ from .mega_mhc import (
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
|
||||
|
||||
from .cute_dsl import AllReduceMHC
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# GB200 TP4/FlashInfer improves through 16 tokens; larger screens tie or regress.
|
||||
MHC_OVERLAP_MAX_TOKENS = 16
|
||||
|
||||
# The fused all-reduce + mHC kernel of the overlap path, built once by
|
||||
# init_mhc_all_reduce.
|
||||
_all_reduce_mhc: "AllReduceMHC | None" = None
|
||||
|
||||
|
||||
def supports_mhc_overlap(vllm_config: VllmConfig) -> bool:
|
||||
"""Check kernel requirements and safety of sharing the coefficient stream."""
|
||||
@@ -53,10 +66,35 @@ def supports_mhc_all_reduce(vllm_config: VllmConfig) -> bool:
|
||||
or config.hc_mult != 4
|
||||
):
|
||||
return False
|
||||
# The custom all-reduce's MNNVL buffers succeed exactly when NVLink
|
||||
# multicast does, which the fused kernel's own mailbox needs too.
|
||||
comm = cast("CudaCommunicator", get_tp_group().device_communicator).ca_comm
|
||||
return comm is not None and bool(comm.mnnvl_lamport_ag_multicast_ptr)
|
||||
|
||||
|
||||
def init_mhc_all_reduce(vllm_config: VllmConfig) -> None:
|
||||
"""Build the fused all-reduce + mHC kernel the overlap path reduces with.
|
||||
|
||||
Collective over the TP group, so every rank must call it, and only when
|
||||
``supports_mhc_all_reduce`` holds.
|
||||
"""
|
||||
global _all_reduce_mhc
|
||||
from .cute_dsl import AllReduceMHC
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
_all_reduce_mhc = AllReduceMHC(
|
||||
hidden_size=config.hidden_size,
|
||||
hc_mult=config.hc_mult,
|
||||
max_num_tokens=MHC_OVERLAP_MAX_TOKENS,
|
||||
top_k=config.num_experts_per_tok,
|
||||
device=current_platform.current_device(),
|
||||
)
|
||||
logger.info_once(
|
||||
"DSV4.1 mHC: CuTe DSL Lamport all-reduce fused with mHC for up to %d tokens.",
|
||||
MHC_OVERLAP_MAX_TOKENS,
|
||||
)
|
||||
|
||||
|
||||
def mhc_pre_delayed_overlap(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
@@ -151,7 +189,7 @@ def mhc_pre_delayed_overlap(
|
||||
|
||||
|
||||
def mhc_shifted_post_pre(
|
||||
x: torch.Tensor,
|
||||
x: torch.Tensor | MoEOutput,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
@@ -175,39 +213,55 @@ def mhc_shifted_post_pre(
|
||||
]:
|
||||
"""Dispatch shifted post/pre to overlap, Mega-mHC, or fused TileLang.
|
||||
|
||||
A MoE output left unfinalized is finalized inside the fused all-reduce.
|
||||
When stream is supplied, join it before consuming the returned coefficients.
|
||||
"""
|
||||
layer_input = None
|
||||
if reduce_results:
|
||||
tp = get_tp_group()
|
||||
if isinstance(x, MoEOutput):
|
||||
assert _all_reduce_mhc is not None, "init_mhc_all_reduce was not called"
|
||||
assert pre_mix is not None and norm_weight is not None
|
||||
routed = x.routed
|
||||
assert isinstance(routed, UnfinalizedMoEOutput)
|
||||
assert x.shared_output is not None
|
||||
residual, layer_input = _all_reduce_mhc.finalize(
|
||||
routed.gemm2_permuted,
|
||||
routed.expert_weights,
|
||||
routed.expanded_idx_to_permuted_idx,
|
||||
x.shared_output,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
pre_mix,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
if stream is None:
|
||||
# Outside FULL graphs, compute the next mixes inline.
|
||||
stream = torch.cuda.current_stream()
|
||||
elif reduce_results:
|
||||
if stream is not None and 0 < x.shape[0] <= MHC_OVERLAP_MAX_TOKENS:
|
||||
comm = cast("CudaCommunicator", tp.device_communicator).ca_comm
|
||||
assert comm is not None and comm.mnnvl_lamport_epochs is not None
|
||||
output = torch.empty_like(residual)
|
||||
layer_input = torch.empty_like(x)
|
||||
torch.ops._C_custom_ar.all_reduce_mhc(
|
||||
assert _all_reduce_mhc is not None, "init_mhc_all_reduce was not called"
|
||||
assert pre_mix is not None and norm_weight is not None
|
||||
residual, layer_input = _all_reduce_mhc(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
pre_mix,
|
||||
norm_weight,
|
||||
output,
|
||||
layer_input,
|
||||
comm.mnnvl_lamport_ag_local_ptr,
|
||||
comm.mnnvl_lamport_ag_multicast_ptr,
|
||||
comm.mnnvl_lamport_epochs[0],
|
||||
comm.rank,
|
||||
comm.mnnvl_buffer_size,
|
||||
norm_eps,
|
||||
)
|
||||
residual = output
|
||||
else:
|
||||
x = tp.all_reduce(x)
|
||||
x = get_tp_group().all_reduce(x)
|
||||
if stream is not None:
|
||||
if layer_input is None:
|
||||
assert isinstance(x, torch.Tensor)
|
||||
residual = mhc_post_tilelang(x, residual, post_layer_mix, comb_res_mix)
|
||||
aux = residual.mean(dim=1) if capture_aux else x.new_empty(0, x.shape[1])
|
||||
aux = (
|
||||
residual.mean(dim=1)
|
||||
if capture_aux
|
||||
else residual.new_empty(0, residual.shape[-1])
|
||||
)
|
||||
pre_outputs = mhc_pre_delayed_overlap(
|
||||
residual,
|
||||
fn,
|
||||
@@ -226,6 +280,7 @@ def mhc_shifted_post_pre(
|
||||
)
|
||||
return residual, *pre_outputs, aux
|
||||
|
||||
assert isinstance(x, torch.Tensor)
|
||||
if can_use_mega_mhc(x, residual, pre_mix, norm_weight, capture_aux):
|
||||
assert pre_mix is not None and norm_weight is not None
|
||||
outputs = mhc_shifted_post_pre_deep_gemm(
|
||||
|
||||
Reference in New Issue
Block a user