[Perf][DSV4.1] Fuse TP all-reduce with mHC input preparation (#57643)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
@@ -793,6 +793,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
use_sequence_parallel: bool = False,
|
||||
*,
|
||||
num_hash_layers: int,
|
||||
reduce_results: bool = True,
|
||||
n_routed_experts: int | None = None,
|
||||
n_activated_experts: int | None = None,
|
||||
image_sentinel_lo: int = IMAGE_SENTINEL_BASE_ID,
|
||||
@@ -804,6 +805,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
quant_config = vllm_config.quant_config
|
||||
self.prefix = prefix
|
||||
self.use_sequence_parallel = use_sequence_parallel
|
||||
self.reduce_results = reduce_results
|
||||
moe_backend = vllm_config.kernel_config.moe_backend
|
||||
validate_fi_moe_ep_config(vllm_config)
|
||||
self.use_mega_moe = moe_backend in MEGA_MOE_BACKENDS
|
||||
@@ -1010,6 +1012,7 @@ class DeepseekV4MoE(nn.Module):
|
||||
self.physical_expert_end = self.experts_end_idx
|
||||
|
||||
self.experts = FusedMoEFactory(
|
||||
reduce_results=self.reduce_results,
|
||||
shared_experts=self.shared_experts,
|
||||
gate=self.gate,
|
||||
num_experts=self.n_routed_experts,
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
from vllm.forward_context import (
|
||||
get_forward_context,
|
||||
@@ -88,10 +89,11 @@ from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
from ..common.engram import EngramLayout, NgramHashState
|
||||
from ..common.mm_preprocess import IMAGE_SENTINEL_BASE_ID, image_sentinel_mask
|
||||
from .engram import Engram, gather_engram_hashes
|
||||
from .ops.mega_mhc import mhc_shifted_post_pre
|
||||
from .ops.mhc import (
|
||||
MHC_OVERLAP_MAX_TOKENS,
|
||||
mhc_pre_delayed_overlap,
|
||||
mhc_shifted_post_pre,
|
||||
supports_mhc_all_reduce,
|
||||
supports_mhc_overlap,
|
||||
)
|
||||
|
||||
@@ -107,6 +109,7 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
use_sequence_parallel: bool = False,
|
||||
reduce_results: bool = True,
|
||||
):
|
||||
config = vllm_config.model_config.hf_config
|
||||
n_routed_experts = config.n_routed_experts
|
||||
@@ -125,6 +128,7 @@ class DeepseekV4MoE(DeepseekV4MoEBase):
|
||||
n_routed_experts=n_routed_experts,
|
||||
n_activated_experts=n_activated_experts,
|
||||
num_hash_layers=0,
|
||||
reduce_results=reduce_results,
|
||||
image_sentinel_lo=IMAGE_SENTINEL_BASE_ID,
|
||||
)
|
||||
|
||||
@@ -199,12 +203,14 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
candidate_block_buffer: torch.Tensor | None = None,
|
||||
engram_layout: EngramLayout | None = None,
|
||||
mhc_stream: torch.cuda.Stream | None = None,
|
||||
fuse_mhc_all_reduce: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.mhc_stream = mhc_stream
|
||||
self.fuse_mhc_all_reduce = fuse_mhc_all_reduce
|
||||
self.use_sequence_parallel = _use_sequence_parallel(vllm_config)
|
||||
|
||||
self.engram: Engram | None = None
|
||||
@@ -228,12 +234,13 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
aux_stream_list=aux_stream_list,
|
||||
candidate_block_buffer=candidate_block_buffer,
|
||||
)
|
||||
if self.use_sequence_parallel:
|
||||
if self.use_sequence_parallel or fuse_mhc_all_reduce:
|
||||
self.attn.wo_b.reduce_results = False
|
||||
self.ffn = DeepseekV4MoE(
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.ffn",
|
||||
use_sequence_parallel=self.use_sequence_parallel,
|
||||
reduce_results=not fuse_mhc_all_reduce,
|
||||
)
|
||||
|
||||
self.attn_norm = RMSNorm(self.hidden_size, self.rms_norm_eps)
|
||||
@@ -430,6 +437,8 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
# and this block's pre, on the full hc stream, so the mix
|
||||
# coefficients see the injected stream. The injection also keeps
|
||||
# the post out of the pre-norm GEMM's fused prologue.
|
||||
if self.fuse_mhc_all_reduce:
|
||||
x = tensor_model_parallel_all_reduce(x)
|
||||
previous_post = mhc_post_tilelang(x, residual, post_mix, res_mix)
|
||||
if capture_previous_aux:
|
||||
previous_aux = previous_post.mean(dim=1)
|
||||
@@ -473,6 +482,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
norm_eps=self.attn_norm.variance_epsilon,
|
||||
capture_aux=capture_previous_aux,
|
||||
stream=mhc_stream,
|
||||
reduce_results=self.fuse_mhc_all_reduce,
|
||||
)
|
||||
if capture_previous_aux:
|
||||
previous_aux = aux
|
||||
@@ -503,6 +513,7 @@ class DeepseekV4DecoderLayer(nn.Module):
|
||||
norm_weight=self.ffn_norm.weight,
|
||||
norm_eps=self.ffn_norm.variance_epsilon,
|
||||
stream=mhc_stream,
|
||||
reduce_results=self.fuse_mhc_all_reduce,
|
||||
)
|
||||
x = self.ffn(x, input_ids, mega_gate_metadata)
|
||||
if mhc_stream is not None:
|
||||
@@ -540,6 +551,9 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
aux_stream_list = [torch.cuda.Stream() for _ in range(3)]
|
||||
# Keep mHC independent of the streams used inside attention.
|
||||
mhc_stream = torch.cuda.Stream() if supports_mhc_overlap(vllm_config) else None
|
||||
self.fuse_mhc_all_reduce = mhc_stream is not None and supports_mhc_all_reduce(
|
||||
vllm_config
|
||||
)
|
||||
|
||||
# Reserved topk indices buffer for all Indexer layers to reuse.
|
||||
self.topk_indices_buffer = torch.empty(
|
||||
@@ -585,6 +599,7 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
candidate_block_buffer=self.candidate_block_buffer,
|
||||
engram_layout=self.engram_layout,
|
||||
mhc_stream=mhc_stream,
|
||||
fuse_mhc_all_reduce=self.fuse_mhc_all_reduce,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
@@ -794,6 +809,8 @@ class DeepseekV4Model(nn.Module, EagleModelMixin):
|
||||
aux_hidden_by_layer[idx] = previous_aux
|
||||
if layer is not None:
|
||||
# The last layer has no successor to fold its post into.
|
||||
if self.fuse_mhc_all_reduce:
|
||||
hidden_states = tensor_model_parallel_all_reduce(hidden_states)
|
||||
hidden_states = mhc_post_tilelang(
|
||||
hidden_states, residual, post_mix, res_mix
|
||||
)
|
||||
|
||||
@@ -5,10 +5,6 @@ import functools
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
mhc_fused_post_pre_delayed_tilelang,
|
||||
mhc_post_tilelang,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
_import_deep_gemm,
|
||||
@@ -88,97 +84,3 @@ def mhc_shifted_post_pre_deep_gemm(
|
||||
y_bf16,
|
||||
new_prev_mix.squeeze(-1),
|
||||
)
|
||||
|
||||
|
||||
def mhc_shifted_post_pre(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
capture_aux: bool = False,
|
||||
*,
|
||||
stream: torch.cuda.Stream | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
|
||||
]:
|
||||
"""Dispatch shifted post/pre to overlap, Mega-mHC, or fused TileLang.
|
||||
|
||||
When stream is supplied, join it before consuming the returned coefficients.
|
||||
"""
|
||||
if stream is not None:
|
||||
from .mhc import mhc_pre_delayed_overlap
|
||||
|
||||
residual = mhc_post_tilelang(x, residual, post_layer_mix, comb_res_mix)
|
||||
aux = residual.mean(dim=1) if capture_aux else x.new_empty(0, x.shape[1])
|
||||
pre_outputs = mhc_pre_delayed_overlap(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix=pre_mix,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
stream=stream,
|
||||
)
|
||||
return residual, *pre_outputs, aux
|
||||
|
||||
if (
|
||||
pre_mix is not None
|
||||
and norm_weight is not None
|
||||
and not capture_aux
|
||||
and x.shape[0] <= 1 << 20
|
||||
and is_mega_mhc_supported(x.shape[1], residual.shape[1])
|
||||
):
|
||||
outputs = mhc_shifted_post_pre_deep_gemm(
|
||||
x,
|
||||
residual,
|
||||
pre_mix,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_post_mult_value,
|
||||
hc_sinkhorn_eps,
|
||||
sinkhorn_repeat,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
return *outputs, x.new_empty(0, x.shape[1])
|
||||
|
||||
return mhc_fused_post_pre_delayed_tilelang(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix=pre_mix,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
capture_aux=capture_aux,
|
||||
)
|
||||
|
||||
@@ -1,15 +1,26 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Split shifted mHC pre so coefficient generation can overlap the sublayer."""
|
||||
"""Dispatch DSV4.1 mHC operations and overlap coefficient generation."""
|
||||
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_tp_group
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
mhc_fused_post_pre_delayed_tilelang,
|
||||
mhc_post_tilelang,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
from .mega_mhc import is_mega_mhc_supported, mhc_shifted_post_pre_deep_gemm
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
|
||||
|
||||
# GB200 TP4/FlashInfer improves through 16 tokens; larger screens tie or regress.
|
||||
MHC_OVERLAP_MAX_TOKENS = 16
|
||||
|
||||
@@ -29,6 +40,20 @@ def supports_mhc_overlap(vllm_config: VllmConfig) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def supports_mhc_all_reduce(vllm_config: VllmConfig) -> bool:
|
||||
parallel = vllm_config.parallel_config
|
||||
config = vllm_config.model_config.hf_config
|
||||
if (
|
||||
parallel.tensor_parallel_size != 4
|
||||
or parallel.enable_expert_parallel
|
||||
or config.hidden_size != 5120
|
||||
or config.hc_mult != 4
|
||||
):
|
||||
return False
|
||||
comm = cast("CudaCommunicator", get_tp_group().device_communicator).ca_comm
|
||||
return comm is not None and bool(comm.mnnvl_lamport_ag_multicast_ptr)
|
||||
|
||||
|
||||
def mhc_pre_delayed_overlap(
|
||||
residual: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
@@ -45,6 +70,7 @@ def mhc_pre_delayed_overlap(
|
||||
norm_eps: float = 1e-6,
|
||||
*,
|
||||
stream: torch.cuda.Stream,
|
||||
layer_input: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Prepare the input on the caller stream and coefficients on another stream.
|
||||
|
||||
@@ -67,7 +93,11 @@ def mhc_pre_delayed_overlap(
|
||||
post = torch.empty((n, hc), device=residual.device, dtype=torch.float32)
|
||||
comb = torch.empty((n, hc * hc), device=residual.device, dtype=torch.float32)
|
||||
next_pre = torch.empty_like(post)
|
||||
layer_input = torch.empty((n, hidden), device=residual.device, dtype=torch.bfloat16)
|
||||
compute_input = layer_input is None
|
||||
if layer_input is None:
|
||||
layer_input = torch.empty(
|
||||
(n, hidden), device=residual.device, dtype=torch.bfloat16
|
||||
)
|
||||
outputs = post.unsqueeze(-1), comb.view(n, hc, hc), layer_input, next_pre
|
||||
if n == 0:
|
||||
return outputs
|
||||
@@ -104,7 +134,7 @@ def mhc_pre_delayed_overlap(
|
||||
)
|
||||
main = torch.cuda.current_stream()
|
||||
# Prioritize input readiness for small batches before releasing statistics.
|
||||
if n <= 8:
|
||||
if compute_input and n <= 8:
|
||||
epilogue(split_mode="input")
|
||||
stream.wait_stream(main)
|
||||
with torch.cuda.stream(stream):
|
||||
@@ -112,6 +142,128 @@ def mhc_pre_delayed_overlap(
|
||||
epilogue(split_mode="stats")
|
||||
for tensor in (x, mix, sqr):
|
||||
tensor.record_stream(stream)
|
||||
if n > 8:
|
||||
if compute_input and n > 8:
|
||||
epilogue(split_mode="input")
|
||||
return outputs
|
||||
|
||||
|
||||
def mhc_shifted_post_pre(
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
post_layer_mix: torch.Tensor,
|
||||
comb_res_mix: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
hc_scale: torch.Tensor,
|
||||
hc_base: torch.Tensor,
|
||||
rms_eps: float,
|
||||
hc_pre_eps: float,
|
||||
hc_sinkhorn_eps: float,
|
||||
hc_post_mult_value: float,
|
||||
sinkhorn_repeat: int,
|
||||
pre_mix: torch.Tensor | None = None,
|
||||
norm_weight: torch.Tensor | None = None,
|
||||
norm_eps: float = 1e-6,
|
||||
capture_aux: bool = False,
|
||||
*,
|
||||
stream: torch.cuda.Stream | None = None,
|
||||
reduce_results: bool = False,
|
||||
) -> tuple[
|
||||
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
|
||||
]:
|
||||
"""Dispatch shifted post/pre to overlap, Mega-mHC, or fused TileLang.
|
||||
|
||||
When stream is supplied, join it before consuming the returned coefficients.
|
||||
"""
|
||||
layer_input = None
|
||||
if reduce_results:
|
||||
tp = get_tp_group()
|
||||
if stream is not None and 0 < x.shape[0] <= MHC_OVERLAP_MAX_TOKENS:
|
||||
comm = cast("CudaCommunicator", tp.device_communicator).ca_comm
|
||||
assert comm is not None and comm.mnnvl_lamport_epochs is not None
|
||||
output = torch.empty_like(residual)
|
||||
layer_input = torch.empty_like(x)
|
||||
torch.ops._C_custom_ar.all_reduce_mhc(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
pre_mix,
|
||||
norm_weight,
|
||||
output,
|
||||
layer_input,
|
||||
comm.mnnvl_lamport_ag_local_ptr,
|
||||
comm.mnnvl_lamport_ag_multicast_ptr,
|
||||
comm.mnnvl_lamport_epochs[0],
|
||||
comm.rank,
|
||||
comm.mnnvl_buffer_size,
|
||||
norm_eps,
|
||||
)
|
||||
residual = output
|
||||
else:
|
||||
x = tp.all_reduce(x)
|
||||
if stream is not None:
|
||||
if layer_input is None:
|
||||
residual = mhc_post_tilelang(x, residual, post_layer_mix, comb_res_mix)
|
||||
aux = residual.mean(dim=1) if capture_aux else x.new_empty(0, x.shape[1])
|
||||
pre_outputs = mhc_pre_delayed_overlap(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix=pre_mix,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
stream=stream,
|
||||
layer_input=layer_input,
|
||||
)
|
||||
return residual, *pre_outputs, aux
|
||||
|
||||
if (
|
||||
pre_mix is not None
|
||||
and norm_weight is not None
|
||||
and not capture_aux
|
||||
and x.shape[0] <= 1 << 20
|
||||
and is_mega_mhc_supported(x.shape[1], residual.shape[1])
|
||||
):
|
||||
outputs = mhc_shifted_post_pre_deep_gemm(
|
||||
x,
|
||||
residual,
|
||||
pre_mix,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_post_mult_value,
|
||||
hc_sinkhorn_eps,
|
||||
sinkhorn_repeat,
|
||||
norm_weight,
|
||||
norm_eps,
|
||||
)
|
||||
return *outputs, x.new_empty(0, x.shape[1])
|
||||
|
||||
return mhc_fused_post_pre_delayed_tilelang(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_mult_value,
|
||||
sinkhorn_repeat,
|
||||
pre_mix=pre_mix,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
capture_aux=capture_aux,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user