Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
323 lines
10 KiB
Python
323 lines
10 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Dispatch DSV4.1 mHC operations and overlap coefficient generation."""
|
|
|
|
from functools import partial
|
|
from typing import TYPE_CHECKING, cast
|
|
|
|
import torch
|
|
|
|
from vllm.config import VllmConfig
|
|
from vllm.distributed import get_tp_group
|
|
from vllm.logger import init_logger
|
|
from vllm.model_executor.kernels.mhc.tilelang import (
|
|
mhc_fused_post_pre_delayed_tilelang,
|
|
mhc_post_tilelang,
|
|
)
|
|
from vllm.model_executor.layers.fused_moe.moe_output import (
|
|
MoEOutput,
|
|
UnfinalizedMoEOutput,
|
|
)
|
|
from vllm.platforms import current_platform
|
|
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
|
|
|
from .mega_mhc import (
|
|
can_use_mega_mhc,
|
|
mhc_shifted_post_pre_deep_gemm,
|
|
)
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.distributed.device_communicators.cuda_communicator import CudaCommunicator
|
|
|
|
from .cute_dsl import AllReduceMHC
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
# GB200 TP4/FlashInfer improves through 16 tokens; larger screens tie or regress.
|
|
MHC_OVERLAP_MAX_TOKENS = 16
|
|
|
|
# The fused all-reduce + mHC kernel of the overlap path, built once by
|
|
# init_mhc_all_reduce.
|
|
_all_reduce_mhc: "AllReduceMHC | None" = None
|
|
|
|
|
|
def supports_mhc_overlap(vllm_config: VllmConfig) -> bool:
|
|
"""Check kernel requirements and safety of sharing the coefficient stream."""
|
|
config = vllm_config.model_config.hf_config
|
|
# DeepGEMM's prenorm kernel requires K % 64 == 0, N % 8 == 0, and N <= 32.
|
|
mix_size = config.hc_mult * (config.hc_mult + 2)
|
|
return (
|
|
current_platform.is_device_capability_family(100)
|
|
and is_deep_gemm_supported()
|
|
and config.hidden_size % 64 == 0
|
|
and 0 < mix_size <= 32
|
|
and mix_size % 8 == 0
|
|
and not vllm_config.parallel_config.use_ubatching
|
|
)
|
|
|
|
|
|
def supports_mhc_all_reduce(vllm_config: VllmConfig) -> bool:
|
|
parallel = vllm_config.parallel_config
|
|
config = vllm_config.model_config.hf_config
|
|
if (
|
|
parallel.tensor_parallel_size != 4
|
|
or parallel.enable_expert_parallel
|
|
or config.hidden_size != 5120
|
|
or config.hc_mult != 4
|
|
):
|
|
return False
|
|
# The custom all-reduce's MNNVL buffers succeed exactly when NVLink
|
|
# multicast does, which the fused kernel's own mailbox needs too.
|
|
comm = cast("CudaCommunicator", get_tp_group().device_communicator).ca_comm
|
|
return comm is not None and bool(comm.mnnvl_lamport_ag_multicast_ptr)
|
|
|
|
|
|
def init_mhc_all_reduce(vllm_config: VllmConfig) -> None:
|
|
"""Build the fused all-reduce + mHC kernel the overlap path reduces with.
|
|
|
|
Collective over the TP group, so every rank must call it, and only when
|
|
``supports_mhc_all_reduce`` holds.
|
|
"""
|
|
global _all_reduce_mhc
|
|
from .cute_dsl import AllReduceMHC
|
|
|
|
config = vllm_config.model_config.hf_config
|
|
_all_reduce_mhc = AllReduceMHC(
|
|
hidden_size=config.hidden_size,
|
|
hc_mult=config.hc_mult,
|
|
max_num_tokens=MHC_OVERLAP_MAX_TOKENS,
|
|
top_k=config.num_experts_per_tok,
|
|
device=current_platform.current_device(),
|
|
)
|
|
logger.info_once(
|
|
"DSV4.1 mHC: CuTe DSL Lamport all-reduce fused with mHC for up to %d tokens.",
|
|
MHC_OVERLAP_MAX_TOKENS,
|
|
)
|
|
|
|
|
|
def mhc_pre_delayed_overlap(
|
|
residual: torch.Tensor,
|
|
fn: torch.Tensor,
|
|
hc_scale: torch.Tensor,
|
|
hc_base: torch.Tensor,
|
|
rms_eps: float,
|
|
hc_pre_eps: float,
|
|
hc_sinkhorn_eps: float,
|
|
hc_post_mult_value: float,
|
|
sinkhorn_repeat: int,
|
|
pre_mix: torch.Tensor | None = None,
|
|
x: torch.Tensor | None = None,
|
|
norm_weight: torch.Tensor | None = None,
|
|
norm_eps: float = 1e-6,
|
|
*,
|
|
stream: torch.cuda.Stream,
|
|
layer_input: torch.Tensor | None = None,
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Prepare the input on the caller stream and coefficients on another stream.
|
|
|
|
Returns post mix, residual mix, normalized input, and next pre-mix. Only the
|
|
input is ready on the caller stream; join stream before using coefficients.
|
|
The caller must retain the weights and coefficient outputs until the join.
|
|
"""
|
|
from vllm.model_executor.kernels.mhc.warmup import (
|
|
MHC_PRE_NORM_KERNEL,
|
|
compute_mhc_pre_num_splits,
|
|
)
|
|
from vllm.utils.deep_gemm import tf32_hc_prenorm_gemm
|
|
|
|
n, hc, hidden = residual.shape
|
|
assert residual.is_contiguous() and residual.dtype == torch.bfloat16
|
|
assert norm_weight is not None
|
|
if x is None:
|
|
x = residual.view(n, hc * hidden)
|
|
assert x.is_contiguous()
|
|
post = torch.empty((n, hc), device=residual.device, dtype=torch.float32)
|
|
comb = torch.empty((n, hc * hc), device=residual.device, dtype=torch.float32)
|
|
next_pre = torch.empty_like(post)
|
|
compute_input = layer_input is None
|
|
if layer_input is None:
|
|
layer_input = torch.empty(
|
|
(n, hidden), device=residual.device, dtype=torch.bfloat16
|
|
)
|
|
outputs = post.unsqueeze(-1), comb.view(n, hc, hc), layer_input, next_pre
|
|
if n == 0:
|
|
return outputs
|
|
splits = compute_mhc_pre_num_splits(x.shape[1], n)
|
|
mix = torch.empty(
|
|
(splits, n, hc * (hc + 2)), device=residual.device, dtype=torch.float32
|
|
)
|
|
sqr = torch.empty((splits, n), device=residual.device, dtype=torch.float32)
|
|
epilogue = partial(
|
|
MHC_PRE_NORM_KERNEL,
|
|
mix,
|
|
sqr,
|
|
hc_scale,
|
|
hc_base,
|
|
residual,
|
|
post,
|
|
comb,
|
|
layer_input,
|
|
norm_weight,
|
|
pre_mix if pre_mix is not None else post,
|
|
next_pre,
|
|
layer_input, # Unused aux output in split modes.
|
|
hidden_size=hidden,
|
|
rms_eps=rms_eps,
|
|
hc_pre_eps=hc_pre_eps,
|
|
hc_sinkhorn_eps=hc_sinkhorn_eps,
|
|
hc_post_mult_value=hc_post_mult_value,
|
|
sinkhorn_repeat=sinkhorn_repeat,
|
|
norm_eps=norm_eps,
|
|
hc_mult=hc,
|
|
use_pre_mix_in=pre_mix is not None,
|
|
save_pre_mix=True,
|
|
rms_numel=x.shape[1],
|
|
)
|
|
main = torch.cuda.current_stream()
|
|
# Prioritize input readiness for small batches before releasing statistics.
|
|
if compute_input and n <= 8:
|
|
epilogue(split_mode="input")
|
|
stream.wait_stream(main)
|
|
with torch.cuda.stream(stream):
|
|
tf32_hc_prenorm_gemm(x, fn, mix, sqr, splits)
|
|
epilogue(split_mode="stats")
|
|
for tensor in (x, mix, sqr):
|
|
tensor.record_stream(stream)
|
|
if compute_input and n > 8:
|
|
epilogue(split_mode="input")
|
|
return outputs
|
|
|
|
|
|
def mhc_shifted_post_pre(
|
|
x: torch.Tensor | MoEOutput,
|
|
residual: torch.Tensor,
|
|
post_layer_mix: torch.Tensor,
|
|
comb_res_mix: torch.Tensor,
|
|
fn: torch.Tensor,
|
|
hc_scale: torch.Tensor,
|
|
hc_base: torch.Tensor,
|
|
rms_eps: float,
|
|
hc_pre_eps: float,
|
|
hc_sinkhorn_eps: float,
|
|
hc_post_mult_value: float,
|
|
sinkhorn_repeat: int,
|
|
pre_mix: torch.Tensor | None = None,
|
|
norm_weight: torch.Tensor | None = None,
|
|
norm_eps: float = 1e-6,
|
|
capture_aux: bool = False,
|
|
*,
|
|
stream: torch.cuda.Stream | None = None,
|
|
reduce_results: bool = False,
|
|
) -> tuple[
|
|
torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor
|
|
]:
|
|
"""Dispatch shifted post/pre to overlap, Mega-mHC, or fused TileLang.
|
|
|
|
A MoE output left unfinalized is finalized inside the fused all-reduce.
|
|
When stream is supplied, join it before consuming the returned coefficients.
|
|
"""
|
|
layer_input = None
|
|
if isinstance(x, MoEOutput):
|
|
assert _all_reduce_mhc is not None, "init_mhc_all_reduce was not called"
|
|
assert pre_mix is not None and norm_weight is not None
|
|
routed = x.routed
|
|
assert isinstance(routed, UnfinalizedMoEOutput)
|
|
assert x.shared_output is not None
|
|
residual, layer_input = _all_reduce_mhc.finalize(
|
|
routed.gemm2_permuted,
|
|
routed.expert_weights,
|
|
routed.expanded_idx_to_permuted_idx,
|
|
x.shared_output,
|
|
residual,
|
|
post_layer_mix,
|
|
comb_res_mix,
|
|
pre_mix,
|
|
norm_weight,
|
|
norm_eps,
|
|
)
|
|
if stream is None:
|
|
# Outside FULL graphs, compute the next mixes inline.
|
|
stream = torch.cuda.current_stream()
|
|
elif reduce_results:
|
|
if stream is not None and 0 < x.shape[0] <= MHC_OVERLAP_MAX_TOKENS:
|
|
assert _all_reduce_mhc is not None, "init_mhc_all_reduce was not called"
|
|
assert pre_mix is not None and norm_weight is not None
|
|
residual, layer_input = _all_reduce_mhc(
|
|
x,
|
|
residual,
|
|
post_layer_mix,
|
|
comb_res_mix,
|
|
pre_mix,
|
|
norm_weight,
|
|
norm_eps,
|
|
)
|
|
else:
|
|
x = get_tp_group().all_reduce(x)
|
|
if stream is not None:
|
|
if layer_input is None:
|
|
assert isinstance(x, torch.Tensor)
|
|
residual = mhc_post_tilelang(x, residual, post_layer_mix, comb_res_mix)
|
|
aux = (
|
|
residual.mean(dim=1)
|
|
if capture_aux
|
|
else residual.new_empty(0, residual.shape[-1])
|
|
)
|
|
pre_outputs = mhc_pre_delayed_overlap(
|
|
residual,
|
|
fn,
|
|
hc_scale,
|
|
hc_base,
|
|
rms_eps,
|
|
hc_pre_eps,
|
|
hc_sinkhorn_eps,
|
|
hc_post_mult_value,
|
|
sinkhorn_repeat,
|
|
pre_mix=pre_mix,
|
|
norm_weight=norm_weight,
|
|
norm_eps=norm_eps,
|
|
stream=stream,
|
|
layer_input=layer_input,
|
|
)
|
|
return residual, *pre_outputs, aux
|
|
|
|
assert isinstance(x, torch.Tensor)
|
|
if can_use_mega_mhc(x, residual, pre_mix, norm_weight, capture_aux):
|
|
assert pre_mix is not None and norm_weight is not None
|
|
outputs = mhc_shifted_post_pre_deep_gemm(
|
|
x,
|
|
residual,
|
|
pre_mix,
|
|
post_layer_mix,
|
|
comb_res_mix,
|
|
fn,
|
|
hc_scale,
|
|
hc_base,
|
|
rms_eps,
|
|
hc_pre_eps,
|
|
hc_post_mult_value,
|
|
hc_sinkhorn_eps,
|
|
sinkhorn_repeat,
|
|
norm_weight,
|
|
norm_eps,
|
|
)
|
|
return *outputs, x.new_empty(0, x.shape[1])
|
|
|
|
return mhc_fused_post_pre_delayed_tilelang(
|
|
x,
|
|
residual,
|
|
post_layer_mix,
|
|
comb_res_mix,
|
|
fn,
|
|
hc_scale,
|
|
hc_base,
|
|
rms_eps,
|
|
hc_pre_eps,
|
|
hc_sinkhorn_eps,
|
|
hc_post_mult_value,
|
|
sinkhorn_repeat,
|
|
pre_mix=pre_mix,
|
|
norm_weight=norm_weight,
|
|
norm_eps=norm_eps,
|
|
capture_aux=capture_aux,
|
|
)
|