diff --git a/docs/design/moe_kernel_features.md b/docs/design/moe_kernel_features.md index 279ab2d0d6..d49790e833 100644 --- a/docs/design/moe_kernel_features.md +++ b/docs/design/moe_kernel_features.md @@ -89,6 +89,7 @@ To be used with a particular `FusedMoEPrepareAndFinalizeModular` subclass, MoE k | gpt oss triton | standard | N/A | N/A | 5 | Y | Y | [`triton_kernel_fused_experts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.triton_kernel_fused_experts],
[`OAITritonExperts`][vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe.OAITritonExperts] | | marlin | standard,
batched | 3 / N/A | 3 / N/A | silu,
swigluoai | Y | Y | [`fused_marlin_moe`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.fused_marlin_moe],
[`MarlinExperts`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.MarlinExperts],
[`BatchedMarlinExperts`][vllm.model_executor.layers.fused_moe.experts.marlin_moe.BatchedMarlinExperts] | | trtllm | standard | mxfp4,
nvfp4 | G(16),G(32) | 5 | N | Y | [`TrtLlmMxfp4ExpertsMonolithic`][vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe.TrtLlmMxfp4ExpertsMonolithic],
[`TrtLlmMxfp4ExpertsModular`][vllm.model_executor.layers.fused_moe.experts.trtllm_mxfp4_moe.TrtLlmMxfp4ExpertsModular],
[`TrtLlmNvFp4ExpertsMonolithic`][vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe.TrtLlmNvFp4ExpertsMonolithic],
[`TrtLlmNvfp4ExpertsModular`][vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe.TrtLlmNvFp4ExpertsModular] | +| hpc | standard | fp8 | G(128),T | silu | Y | Y | [`HPCExperts`][vllm.model_executor.layers.fused_moe.experts.hpc.HPCExperts] | | rocm aiter moe | standard | mxfp4,
fp8 | G(32),G(128),A,T | silu, gelu,
swigluoai | Y | N | `rocm_aiter_fused_experts`,
`AiterExperts` | | cpu_fused_moe | standard | N/A | N/A | silu | N | N | [`CPUFusedMOE`][vllm.model_executor.layers.fused_moe.cpu_fused_moe.CPUFusedMOE] | | naive batched4 | batched | int8,
fp8 | G,A,T | silu, gelu | 6 | Y | [`NaiveBatchedExperts`][vllm.model_executor.layers.fused_moe.experts.fused_batched_moe.NaiveBatchedExperts] | diff --git a/vllm/config/kernel.py b/vllm/config/kernel.py index e9f41c538a..770daad1ce 100644 --- a/vllm/config/kernel.py +++ b/vllm/config/kernel.py @@ -134,6 +134,7 @@ MoEBackend = Literal[ "triton_unfused", "aiter", "flydsl", + "hpc", "emulation", ] @@ -189,6 +190,7 @@ class KernelConfig: - "triton_unfused": Use Triton unfused MoE kernels - "aiter": Use AMD AITer kernels (ROCm only) - "flydsl": Use AMD FlyDSL kernels (ROCm only) + - "hpc": Use HPC kernels (FP8 and Hopper only) - "emulation": use BF16/FP16 GEMM, dequantizing weights and running QDQ on activations. """ diff --git a/vllm/model_executor/layers/fused_moe/hpc_moe.py b/vllm/model_executor/layers/fused_moe/hpc_moe.py new file mode 100644 index 0000000000..6c7e20fa3c --- /dev/null +++ b/vllm/model_executor/layers/fused_moe/hpc_moe.py @@ -0,0 +1,211 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +import vllm.model_executor.layers.fused_moe.modular_kernel as mk +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.activation import MoEActivation +from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEParallelConfig, + FusedMoEQuantConfig, +) +from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import ( + TopKWeightAndReduceNoOP, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kFp8Dynamic128Sym, + kFp8Static128BlockSym, + kFp8StaticTensorSym, +) +from vllm.platforms import current_platform +from vllm.utils.hpc import has_hpc, hpc_fuse_moe, hpc_fuse_moe_blockwise + +logger = init_logger(__name__) + + +class HPCExperts(mk.FusedMoEExpertsModular): + """MoE implementation powered by [HPC](https://github.com/Tencent/hpc-ops). + + Only supported on NVIDIA Hopper GPUs (e.g. H20, H200), and currently limited to + FP8 models such as Hy3-FP8, Qwen3-235B-A22B-FP8, etc. + """ + + def __init__( + self, + moe_config: mk.FusedMoEConfig, + quant_config: FusedMoEQuantConfig, + ): + super().__init__(moe_config, quant_config) + + assert quant_config.weight_quant_dtype in (torch.float8_e4m3fn,), ( + "Only fp8 quantization is currently supported." + ) + + self.device = moe_config.device + self.num_experts = moe_config.num_local_experts + self.ep_rank = moe_config.moe_parallel_config.ep_rank + self.ep_size = moe_config.moe_parallel_config.ep_size + self.tp_rank = moe_config.moe_parallel_config.tp_rank + self.tp_size = moe_config.moe_parallel_config.tp_size + self.out_dtype = moe_config.in_dtype + + @property + def expects_unquantized_inputs(self) -> bool: + return False + + @staticmethod + def activation_format() -> mk.FusedMoEActivationFormat: + return mk.FusedMoEActivationFormat.Standard + + @staticmethod + def _supports_current_device() -> bool: + p = current_platform + return ( + p.is_cuda() + and (p.is_device_capability(90) or p.is_device_capability_family(100)) + and has_hpc() + ) + + @staticmethod + def _supports_no_act_and_mul() -> bool: + return False + + @staticmethod + def _supports_quant_scheme( + weight_key: QuantKey | None, + activation_key: QuantKey | None, + ) -> bool: + scheme = (weight_key, activation_key) + # The following are supported by HPCExperts: + return scheme in [ + # fp8 static per-tensor on 9.0+ + (kFp8StaticTensorSym, kFp8StaticTensorSym), + (kFp8Static128BlockSym, kFp8Dynamic128Sym), + ] + + @staticmethod + def _supports_activation(activation: MoEActivation) -> bool: + return activation in [ + MoEActivation.SILU, + ] + + @staticmethod + def _supports_shape(hidden_dim: int) -> bool: + # HPC fused MoE kernels process hidden_size in blocks of 128: + # block-wise fp8 requires hidden_size % 128 == 0 (per-128 quant), and + # the group GEMM tiles N by 128. Require 128-alignment to cover all + # code paths. + return hidden_dim % 128 == 0 + + @staticmethod + def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: + return True + + def supports_expert_map(self) -> bool: + return False + + def supports_chunking(self) -> bool: + # This refers to TP chunking; DP chunking is handled separately. + return True + + def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: + return TopKWeightAndReduceNoOP() + + def workspace_shapes( + self, + M: int, + N: int, + K: int, + topk: int, + global_num_experts: int, + local_num_experts: int, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + activation: MoEActivation, + ) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]: + # We use global_num_experts due to how moe_align_block_size handles + # expert_maps. + """ + Compute the shapes for the temporary and final outputs of the two gemms + and activation in the fused expert function. Since the gemms are + independent, the workspace for the first gemm can be shared with the + workspace for the last gemm. + + Returns a tuple of: + - workspace13 shape tuple: must be large enough to hold the + result of either expert gemm. + - workspace2 shape tuple: must be large enough to hold the + result of the activation function. + - output shape tuple: must be exact size of the final gemm output. + - Workspace type: The dtype to use for the workspace tensors. + - Note: in order for activation chunking to work, the first dimension + of each tuple must be the number of tokens. + """ + workspace1 = (M, K) + workspace2 = (0,) + output_shape = (M, K) + # The workspace is determined by `aq`, since it comes after any + # potential communication op and is involved in the expert computation. + return (workspace1, workspace2, output_shape) + + def apply( + self, + output: torch.Tensor, + hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, + topk_weights: torch.Tensor, + topk_ids: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor | None, + workspace2: torch.Tensor | None, + expert_tokens_meta: mk.ExpertTokensMetadata | None, + apply_router_weight_on_input: bool | None, + ): + assert self._supports_activation(activation), f"{activation=} not supported" + assert self.quant_config.w1_scale is not None, ( + "w13_weight_scale must be provided" + ) + assert self.quant_config.w2_scale is not None, ( + "w2_weight_scale must be provided" + ) + + if self.quant_config.is_block_quantized: + hpc_fuse_moe_blockwise( + x=hidden_states, + x_scale=a1q_scale, + gate_up_weight=w1, + gate_up_weight_scale=self.quant_config.w1_scale, + down_weight=w2, + down_weight_scale=self.quant_config.w2_scale, + topk_ids=topk_ids, + topk_scale=topk_weights, + rank_ep=self.ep_rank, + num_expert_total=global_num_experts, + output=output, + ) + else: + assert self.quant_config.a1_scale is not None, ( + "w13_input_scale must be provided" + ) + assert self.quant_config.a2_scale is not None, ( + "w2_input_scale must be provided" + ) + hpc_fuse_moe( + x=hidden_states, + gate_up_weight=w1, + down_weight=w2, + gate_up_scale=self.quant_config.g1_alphas, + down_scale=self.quant_config.g2_alphas, + act_and_mul_scale=self.quant_config.a2_gscale, + topk_ids=topk_ids, + topk_scale=topk_weights, + rank_ep=self.ep_rank, + num_expert_total=global_num_experts, + output=output, + ) diff --git a/vllm/model_executor/layers/fused_moe/oracle/fp8.py b/vllm/model_executor/layers/fused_moe/oracle/fp8.py index 1b5030b190..9f930f1d58 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/fp8.py +++ b/vllm/model_executor/layers/fused_moe/oracle/fp8.py @@ -52,6 +52,7 @@ class Fp8MoeBackend(Enum): BATCHED_VLLM_CUTLASS = "BATCHED_VLLM_CUTLASS" XPU = "XPU" CPU = "CPU" + HPC = "HPC" # Dequantize-to-BF16 emulation for MXFP8 on devices without a native # MXFP8 MoE kernel (e.g. ROCm). Weights pass through unchanged here. EMULATION = "EMULATION" @@ -85,6 +86,7 @@ def _get_priority_backends( Fp8MoeBackend.BATCHED_TRITON, Fp8MoeBackend.XPU, Fp8MoeBackend.CPU, + Fp8MoeBackend.HPC, ] def _move_to_front(backends: list[Fp8MoeBackend], backend: Fp8MoeBackend) -> None: @@ -216,6 +218,13 @@ def backend_to_kernel_cls( return [CPUExpertsFp8] + elif backend == Fp8MoeBackend.HPC: + from vllm.model_executor.layers.fused_moe.hpc_moe import ( + HPCExperts, + ) + + return [HPCExperts] + else: raise ValueError(f"Unknown FP8 MoE backend: {backend.value}") @@ -230,6 +239,7 @@ def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend: "flashinfer_cutlass": Fp8MoeBackend.FLASHINFER_CUTLASS, "marlin": Fp8MoeBackend.MARLIN, "aiter": Fp8MoeBackend.AITER, + "hpc": Fp8MoeBackend.HPC, } if backend := mapping.get(runner_backend): return backend @@ -470,6 +480,7 @@ def convert_to_fp8_moe_kernel_format( Fp8MoeBackend.VLLM_CUTLASS, Fp8MoeBackend.BATCHED_VLLM_CUTLASS, Fp8MoeBackend.XPU, + Fp8MoeBackend.HPC, # EMULATION dequantizes weights at runtime; NATIVE_MXFP8 consumes # the MXFP8 weights as-is — neither needs a load-time layout change. Fp8MoeBackend.EMULATION, @@ -521,9 +532,12 @@ def make_fp8_moe_quant_config( gemm1_clamp_limit=swiglu_limit, ) - # Flashinfer CUTLASS per-tensor uses single dq scale + # Flashinfer CUTLASS or HPC per-tensor uses single dq scale # (alpha = w_scale * a_scale) and inverse a2 scale. - if fp8_backend == Fp8MoeBackend.FLASHINFER_CUTLASS and block_shape is None: + if ( + fp8_backend in [Fp8MoeBackend.FLASHINFER_CUTLASS, Fp8MoeBackend.HPC] + and block_shape is None + ): assert a1_scale is not None and a2_scale is not None return fp8_w8a8_moe_quant_config( w1_scale=w1_scale, diff --git a/vllm/utils/hpc.py b/vllm/utils/hpc.py new file mode 100644 index 0000000000..abe546c916 --- /dev/null +++ b/vllm/utils/hpc.py @@ -0,0 +1,223 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Compatibility wrapper for HPC API changes. + +Users of vLLM should always import **only** these wrappers. +""" + +import functools +import importlib +import importlib.util + +import torch + +from vllm.logger import init_logger + +logger = init_logger(__name__) + + +@functools.cache +def has_hpc() -> bool: + """Return `True` if hpc package is available.""" + # Use find_spec to check if the module exists without importing it + # This avoids potential CUDA initialization side effects + if importlib.util.find_spec("hpc") is None: + logger.warning_once( + "HPC attention requires the hpc module to be installed. " + "Please install it from https://github.com/Tencent/hpc-ops" + ) + return False + return True + + +# Remove 'torch._library.custom_ops': +# The output of this custom operator (1) must not also be an input to +# this custom operator and (2) may not alias any inputs to this custom +# operator or other returns. The most common way to trigger this error +# is if we have y = custom_op(x) and y and x are the same Tensor. +# Please instead return a clone of the offending output tensor(s) (e.g. +# return x.clone()) or refactor the custom operator to not return y. +# @torch.library.custom_op( +# "vllm::fuse_moe_impl", +# mutates_args=[], +# device_types="cuda", +# ) +def fuse_moe_impl( + x: torch.Tensor, + gate_up_weight: torch.Tensor, + down_weight: torch.Tensor, + gate_up_scale: torch.Tensor, + down_scale: torch.Tensor, + act_and_mul_scale: torch.Tensor, + topk_ids: torch.Tensor, + topk_scale: torch.Tensor, + rank_ep: int, + num_expert_total: int, + use_bf16_mul: bool = True, + shared_output: torch.Tensor = None, + output: torch.Tensor = None, +) -> torch.Tensor: + from hpc import fuse_moe as fuse_moe_ + + return fuse_moe_( + x, + gate_up_weight, + down_weight, + gate_up_scale, + down_scale, + act_and_mul_scale, + topk_ids, + topk_scale, + rank_ep, + num_expert_total, + use_bf16_mul, + shared_output, + output=output, + ) + + +# @torch.library.register_fake( +# "vllm::fuse_moe_impl", +# ) +def fuse_moe_impl_fake( + x: torch.Tensor, + gate_up_weight: torch.Tensor, + down_weight: torch.Tensor, + gate_up_scale: torch.Tensor, + down_scale: torch.Tensor, + act_and_mul_scale: torch.Tensor, + topk_ids: torch.Tensor, + topk_scale: torch.Tensor, + rank_ep: int, + num_expert_total: int, + use_bf16_mul: bool = True, + shared_output: torch.Tensor = None, + output: torch.Tensor = None, +) -> torch.Tensor: + return torch.empty_like(x) + + +def hpc_fuse_moe( + x: torch.Tensor, + gate_up_weight: torch.Tensor, + down_weight: torch.Tensor, + gate_up_scale: torch.Tensor, + down_scale: torch.Tensor, + act_and_mul_scale: torch.Tensor, + topk_ids: torch.Tensor, + topk_scale: torch.Tensor, + rank_ep: int, + num_expert_total: int, + use_bf16_mul: bool = True, + shared_output: torch.Tensor = None, + output: torch.Tensor = None, +) -> torch.Tensor: + return fuse_moe_impl( + x, + gate_up_weight, + down_weight, + gate_up_scale, + down_scale, + act_and_mul_scale, + topk_ids, + topk_scale, + rank_ep, + num_expert_total, + use_bf16_mul, + shared_output, + output=output, + ) + + +# @torch.library.custom_op( +# "vllm::fuse_moe_blockwise_impl", +# mutates_args=[], +# device_types="cuda", +# ) +def fuse_moe_blockwise_impl( + x: torch.Tensor, + x_scale: torch.Tensor, + gate_up_weight: torch.Tensor, + gate_up_weight_scale: torch.Tensor, + down_weight: torch.Tensor, + down_weight_scale: torch.Tensor, + topk_ids: torch.Tensor, + topk_scale: torch.Tensor, + rank_ep: int, + num_expert_total: int, + shared_output: torch.Tensor = None, + output: torch.Tensor = None, +) -> torch.Tensor: + from hpc import fuse_moe_blockwise as fuse_moe_blockwise_ + + return fuse_moe_blockwise_( + x, + x_scale, + gate_up_weight, + gate_up_weight_scale, + down_weight, + down_weight_scale, + topk_ids, + topk_scale, + rank_ep, + num_expert_total, + shared_output, + output=output, + ) + + +# @torch.library.register_fake( +# "vllm::fuse_moe_blockwise_impl", +# ) +def fuse_moe_blockwise_impl_fake( + x: torch.Tensor, + x_scale: torch.Tensor, + gate_up_weight: torch.Tensor, + gate_up_weight_scale: torch.Tensor, + down_weight: torch.Tensor, + down_weight_scale: torch.Tensor, + topk_ids: torch.Tensor, + topk_scale: torch.Tensor, + rank_ep: int, + num_expert_total: int, + shared_output: torch.Tensor = None, + output: torch.Tensor = None, +) -> torch.Tensor: + return torch.empty_like(x) + + +def hpc_fuse_moe_blockwise( + x: torch.Tensor, + x_scale: torch.Tensor, + gate_up_weight: torch.Tensor, + gate_up_weight_scale: torch.Tensor, + down_weight: torch.Tensor, + down_weight_scale: torch.Tensor, + topk_ids: torch.Tensor, + topk_scale: torch.Tensor, + rank_ep: int, + num_expert_total: int, + shared_output: torch.Tensor = None, + output: torch.Tensor = None, +) -> torch.Tensor: + return fuse_moe_blockwise_impl( + x, + x_scale, + gate_up_weight, + gate_up_weight_scale, + down_weight, + down_weight_scale, + topk_ids, + topk_scale, + rank_ep, + num_expert_total, + shared_output, + output=output, + ) + + +__all__ = [ + "has_hpc", + "hpc_fuse_moe", + "hpc_fuse_moe_blockwise", +]