[Mini-VLLM] Remove orphan subsystems: snapshot, CPU workers, dead decoders

- Delete the snapshot package, CLI command, and its launch tests
- Delete CPU workers, cpu_communicator, and the hw_agnostic layer package
- Delete the EC CPU connector, keeping the EC base glue intact
- Delete dead composite/triton_flashinfer backends and third_party/flashmla
- Delete dead video decoders, video-bound tests, and orphan utils
- Remove the deep_gemm_mega_moe option and dead allreduce fusion pair

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 09:27:06 +07:00
co-authored by Hinata
parent 2b544b7ba0
commit bd70a61c20
88 changed files with 4 additions and 20247 deletions
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,314 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import inspect
import torch
import torch.nn as nn
from vllm.config import get_cached_compilation_config
from vllm.logger import init_logger
from vllm.model_executor.utils import maybe_disable_graph_partition
from vllm.platforms import current_platform
logger = init_logger(__name__)
# Dictionary of all custom ops (classes, indexed by registered name).
# To check if an op with a name is enabled, call .enabled() on the class.
# Examples:
# - MyOp.enabled()
# - op_registry["my_op"].enabled()
op_registry: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {}
op_registry_oot: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {}
def maybe_get_oot_by_class(class_type: type) -> type:
class_name = class_type.__name__
if class_name in op_registry_oot:
return op_registry_oot[class_name]
return class_type
class PluggableLayer(nn.Module):
"""Base class for pluggable layers.
A PluggableLayer is a *module-composing* abstraction: it may instantiate other
``torch.nn.Module`` objects as sub-layers, and its functionality depends on
these sub-layers following a generalized invocation sequence. Also, it is stateful
and may hold parameters or buffers.
Unlike :class:`CustomOp`, PluggableLayer does NOT provide per-platform
``forward_*`` dispatch. Instead, it supports out-of-tree (OOT) replacement
of the entire layer class at instantiation time, allowing customized
initialization and submodule composition.
"""
def __new__(cls, *args, **kwargs):
try:
layer_class_name = cls.__name__
except AttributeError:
raise TypeError(
f"Cannot instantiate '{cls.__name__}': its 'name' attribute "
f"was not set, possibly because it was not decorated with "
f"@PluggableLayer.register, or it's the PluggableLayer itself."
) from None
if layer_class_name not in op_registry_oot:
layer_cls_to_instantiate = cls
else:
layer_cls_to_instantiate = op_registry_oot[layer_class_name]
logger.debug(
"Instantiating pluggable layer: %s using %s",
layer_class_name,
str(layer_cls_to_instantiate),
)
return super().__new__(layer_cls_to_instantiate)
# Decorator to register pluggable layers.
@classmethod
def register(cls, name: str):
def decorator(op_cls):
assert name not in op_registry, f"Duplicate op name: {name}"
op_cls.name = name
op_registry[name] = op_cls
return op_cls
return decorator
# Decorator to register out-of-tree(oot) pluggable layers.
# For OOT pluggable layers:
# if in-tree layer class is registered with an oot_custom_layer,
# the oot_custom_layer will be used instead.
@classmethod
def register_oot(cls, _decorated_layer_cls=None, name: str | None = None):
def decorator(layer_cls):
reg_name = name if name is not None else cls.__name__
assert reg_name not in op_registry_oot, f"Duplicate layer name: {reg_name}"
layer_cls.name = reg_name
op_registry_oot[reg_name] = layer_cls
return layer_cls
if _decorated_layer_cls is None:
# Called with parentheses: @PluggableLayer.register_oot()
# or @PluggableLayer.register_oot(name="...")
return decorator
elif isinstance(_decorated_layer_cls, type): # Check if it's a class
# Called without parentheses: @PluggableLayer.register_oot
return decorator(_decorated_layer_cls)
else:
raise TypeError("Decorator can only be applied to classes.")
class CustomOp(nn.Module):
"""Base class for custom ops.
Dispatches the forward method to the appropriate backend.
"""
def __new__(cls, *args, **kwargs):
try:
op_name = cls.__name__
except AttributeError:
raise TypeError(
f"Cannot instantiate '{cls.__name__}': its 'name' attribute "
f"was not set, possibly because it was not decorated with "
f"@CustomOp.register, or it's the CustomOp base class itself."
) from None
if op_name not in op_registry_oot:
op_cls_to_instantiate = cls
else:
op_cls_to_instantiate = op_registry_oot[op_name]
logger.debug(
"Instantiating custom op: %s using %s",
op_name,
str(op_cls_to_instantiate),
)
return super().__new__(op_cls_to_instantiate)
def __init__(self, *, enforce_enable: bool = False, compile_native: bool = False):
super().__init__()
self._enforce_enable = enforce_enable
self._forward_method = self.dispatch_forward(compile_native=compile_native)
def forward(self, *args, **kwargs):
return self._forward_method(*args, **kwargs)
def forward_native(self, *args, **kwargs):
"""PyTorch-native implementation; OOT plugins override via
:meth:`forward_oot`."""
raise NotImplementedError
def forward_oot(self, *args, **kwargs):
# By default, OOT ops fall back to the PyTorch-native implementation.
return self.forward_native(*args, **kwargs)
def dispatch_forward(self, compile_native: bool):
# The hw-agnostic CustomOp dispatches only between forward_native
# (in-tree, used on every platform) and forward_oot (overridden by
# an OOT plugin via register_oot).
compilation_config = get_cached_compilation_config()
# NOTE(shen-shanshan): CustomOp object can be enforce enabled, e.g.,
# enable device-specific kernels in ViT models when enabling graph
# mode. By default, it will follow the compilation_config to determine
# whether enable itself.
# This enforce_enable mechanism will be removed after we adding a
# separate compilation_config for multi-modal part.
enabled = self._enforce_enable or self.enabled()
if enabled:
compilation_config.enabled_custom_ops.update([self.__class__.name])
else:
compilation_config.disabled_custom_ops.update([self.__class__.name])
if not enabled:
# Compile forward_native to avoid eager torch ops if inside
# opaque torch custom op (e.g. fused_moe, unified_attention, etc.)
return self.maybe_compile(self.forward_native, enable=compile_native)
if current_platform.is_out_of_tree():
return self.forward_oot
return self.forward_native
def maybe_compile(self, fn, *, enable: bool = True):
"""Compile fn if compilation enabled.
Useful for CustomOp instances called from within a torch custom op,
meaning the forward call is hidden from the model-level torch.compile.
NOTE: this does not enable fusion across ops, so opaque custom ops
should still be unwrapped wherever possible.
"""
from vllm.config.compilation import CompilationMode
# Do not compile if compilation disabled
if not enable:
return fn
# Do not compile if global compilation disabled
compilation_config = get_cached_compilation_config()
if compilation_config.mode == CompilationMode.NONE:
return fn
# If eager backend is used, do not compile either
if compilation_config.backend == "eager":
return fn
compile_options = maybe_disable_graph_partition(
current_platform.simple_compile_backend
)
backend = current_platform.simple_compile_backend
dynamic_arg_dims = getattr(self.__class__, "_dynamic_arg_dims", None)
if dynamic_arg_dims is not None:
compiled_fn = torch.compile(
fn,
dynamic=False,
backend=backend,
options=compile_options,
)
sig = inspect.signature(fn)
@functools.wraps(fn)
def wrapper(*args, **kwargs):
bound = sig.bind(*args, **kwargs)
bound.apply_defaults()
for name, dims in dynamic_arg_dims.items():
arg = bound.arguments.get(name)
if arg is not None and isinstance(arg, torch.Tensor):
dims_list = [dims] if isinstance(dims, int) else dims
for d in dims_list:
real_d = arg.ndim + d if d < 0 else d
torch._dynamo.mark_dynamic(arg, real_d)
return compiled_fn(*args, **kwargs)
return wrapper
# dynamic=True to avoid recompilations
return torch.compile(
fn,
dynamic=True,
backend=backend,
options=compile_options,
)
@classmethod
def enabled(cls) -> bool:
# if no name, then it was not registered
compilation_config = get_cached_compilation_config()
custom_ops = compilation_config.custom_ops
if not hasattr(cls, "name"):
logger.warning_once(
"Custom op %s was not registered, which means it won't appear "
"in the op registry. It will be enabled/disabled based on the "
"global settings.",
cls.__name__,
)
return CustomOp.default_on()
enabled = f"+{cls.name}" in custom_ops
disabled = f"-{cls.name}" in custom_ops
assert not (enabled and disabled), f"Cannot enable and disable {cls.name}"
return (CustomOp.default_on() or enabled) and not disabled
@staticmethod
def default_on() -> bool:
"""Behavior controlled by `CompilationConfig.custom_ops`: On by default if
'all', off by default if 'none'.
When PyTorch Inductor is used, 'none' is the default value,
otherwise 'all'.
"""
compilation_config = get_cached_compilation_config()
count_none = compilation_config.custom_ops.count("none")
count_all = compilation_config.custom_ops.count("all")
assert count_none + count_all == 1
return not count_none > 0 or count_all > 0
# Decorator to register custom ops.
@classmethod
def register(
cls,
name: str,
dynamic_arg_dims: dict[str, int | list[int]] | None = None,
):
def decorator(op_cls):
assert name not in op_registry, f"Duplicate op name: {name}"
op_cls.name = name
op_cls._dynamic_arg_dims = dynamic_arg_dims
op_registry[name] = op_cls
return op_cls
return decorator
# Decorator to register out-of-tree(oot) custom ops.
# For OOT custom ops:
# if in-tree layer class is registered with an oot_custom_op layer,
# the oot_custom_op layer will be used instead.
# Example:
# - @UnquantizedFusedMoEMethod.register_oot
# class HPUUnquantizedFusedMoEMethod(UnquantizedFusedMoEMethod)
# or
# - @CustomOP.register_oot(name="UnquantizedFusedMoEMethod")
@classmethod
def register_oot(cls, _decorated_op_cls=None, name: str | None = None):
def decorator(op_cls):
reg_name = name if name is not None else cls.__name__
assert reg_name not in op_registry_oot, f"Duplicate op name: {reg_name}"
op_cls.name = reg_name
op_registry_oot[reg_name] = op_cls
return op_cls
if _decorated_op_cls is None:
# Called with parentheses: @CustomOP.register_oot()
# or @CustomOP.register_oot(name="...")
# So, _decorated_op_cls is None.
# We return the actual decorator function.
return decorator
elif isinstance(_decorated_op_cls, type): # Check if it's a class
# Called without parentheses: @CustomOP.register_oot
# The first argument is the class itself.
# We call the 'decorator' function immediately with the class.
return decorator(_decorated_op_cls)
else:
# Handle other unexpected cases if necessary
raise TypeError("Decorator can only be applied to classes.")
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,27 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn.functional as F
from vllm.model_executor.hw_agnostic.custom_op import CustomOp
@CustomOp.register("silu_and_mul")
class SiluAndMul(CustomOp):
"""SwiGLU: ``x -> silu(x[:d]) * x[d:]`` where ``d = x.shape[-1] // 2``."""
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
d = x.shape[-1] // 2
return F.silu(x[..., :d]) * x[..., d:]
# Activation-and-mul ops keyed by HF activation name.
_ACTIVATION_AND_MUL_REGISTRY = {
"silu": SiluAndMul,
"swish": SiluAndMul,
}
def get_act_and_mul_fn(act_fn_name: str) -> CustomOp:
"""Build the hw-agnostic activation-and-mul op named `act_fn_name`."""
return _ACTIVATION_AND_MUL_REGISTRY[act_fn_name.lower()]()
@@ -1,90 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn as nn
from torch import Tensor
from vllm.model_executor.hw_agnostic.custom_op import CustomOp
@CustomOp.register("rms_norm")
class RMSNorm(CustomOp):
"""``x -> w * x / sqrt(E[x^2] + eps)``. With ``residual``, fuses
``residual += x`` then RMSNorm and returns ``(normalized, residual)``."""
def __init__(
self,
hidden_size: int,
eps: float = 1e-6,
var_hidden_size: int | None = None,
has_weight: bool = True,
dtype: torch.dtype | None = None,
) -> None:
super().__init__()
self.hidden_size = hidden_size
self.variance_epsilon = eps
self.variance_size_override = (
None if var_hidden_size == hidden_size else var_hidden_size
)
weight_dtype = dtype or torch.get_default_dtype()
self.has_weight = has_weight
weight = torch.ones(hidden_size, dtype=weight_dtype)
if has_weight:
self.weight = nn.Parameter(weight)
else:
self.register_buffer("weight", weight, persistent=False)
def _rms_norm(
self,
x: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> Tensor:
"""Weighted root-mean-square layer normalization"""
orig_dtype = x.dtype
x = x.to(torch.float32)
x_var = x if variance_size is None else x[..., :variance_size]
variance = x_var.pow(2).mean(dim=-1, keepdim=True)
x = x * torch.rsqrt(variance + epsilon)
if weight is not None:
x = x.to(weight.dtype) * weight
return x.to(orig_dtype)
def _fused_add_rms_norm(
self,
x: Tensor,
x_residual: Tensor,
weight: Tensor | None,
epsilon: float,
variance_size: int | None = None,
) -> tuple[Tensor, Tensor]:
"""Fused add and weighted root-mean-square layer normalization"""
orig_dtype = x.dtype
x = x.to(torch.float32)
x = x + x_residual.to(torch.float32)
x_residual = x.to(orig_dtype)
x_var = x if variance_size is None else x[..., :variance_size]
variance = x_var.pow(2).mean(dim=-1, keepdim=True)
x = x * torch.rsqrt(variance + epsilon)
if weight is not None:
x = x.to(weight.dtype) * weight
return x.to(orig_dtype), x_residual
def forward_native(
self,
x: torch.Tensor,
residual: torch.Tensor | None = None,
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
weight = self.weight if self.has_weight else None
epsilon = self.variance_epsilon
variance_size = self.variance_size_override
if residual is None:
return self._rms_norm(x, weight, epsilon, variance_size)
else:
return self._fused_add_rms_norm(x, residual, weight, epsilon, variance_size)
def extra_repr(self) -> str:
return f"hidden_size={self.hidden_size}, eps={self.variance_epsilon}"
@@ -1,190 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Manual fusion of tensor-parallel all-reduce with the following GemmaRMSNorm.
Under tensor parallelism a ``RowParallelLinear`` (e.g. attention ``o_proj``)
produces a per-rank partial sum that is all-reduced, and the result is then fed
into a ``GemmaRMSNorm`` that adds the residual and normalizes. flashinfer ships a
kernel that fuses all-reduce + residual-add + RMSNorm into a single launch; this
helper drives it directly (no torch.compile pass) for models that run eager.
Scope: attention output only, no quantization. When the flashinfer fast path is
not applicable (TP==1, flashinfer/NVSwitch unavailable, unsupported dtype, or an
oversize batch) it falls back to ``all_reduce`` + ``GemmaRMSNorm``, which is
numerically identical to the unfused model path.
"""
import torch
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
get_tp_group,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
from vllm.platforms import current_platform
MiB = 1024 * 1024
# flashinfer fused all-reduce + RMSNorm is wired as a registered custom op in
# allreduce_rms_fusion; both that op and the workspace helpers only exist when
# flashinfer.comm.allreduce_fusion is importable.
try:
from vllm.compilation.passes.fusion.allreduce_rms_fusion import (
flashinfer_trtllm_fused_allreduce_norm,
)
from vllm.distributed.device_communicators.flashinfer_all_reduce import (
flashinfer_comm,
get_fi_ar_workspace,
)
_AR_RESIDUAL_RMS_NORM = (
flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNorm
if flashinfer_comm is not None
else None
)
except (ImportError, AttributeError):
flashinfer_trtllm_fused_allreduce_norm = None # type: ignore[assignment]
get_fi_ar_workspace = None # type: ignore[assignment]
_AR_RESIDUAL_RMS_NORM = None
_FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
"""Workspace token budget for flashinfer fused all-reduce, or None if the
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
from vllm.config.compilation import PassConfig
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
if not max_size_mb:
return None
element_size = torch.tensor([], dtype=dtype).element_size()
return int(max_size_mb * MiB) // (hidden_size * element_size)
def _can_use_flashinfer(hidden_states: torch.Tensor, tp_size: int) -> tuple[bool, int]:
"""Whether the flashinfer fused path applies; returns (ok, max_token_num)."""
if (
flashinfer_trtllm_fused_allreduce_norm is None
or get_fi_ar_workspace is None
or _AR_RESIDUAL_RMS_NORM is None
):
return False, 0
if (
not hidden_states.is_cuda
or hidden_states.dim() != 2
or not hidden_states.is_contiguous()
or hidden_states.dtype not in _FI_SUPPORTED_DTYPES
):
return False, 0
num_tokens, hidden_size = hidden_states.shape
max_token_num = _max_token_num(tp_size, hidden_size, hidden_states.dtype)
if max_token_num is None or num_tokens > max_token_num:
return False, 0
# Lazily create / fetch the (globally cached) workspace; returns None on
# GPUs without NVSwitch, in which case we fall back gracefully.
workspace = get_fi_ar_workspace(
world_size=tp_size,
rank=get_tensor_model_parallel_rank(),
max_token_num=max_token_num,
hidden_dim=hidden_size,
dtype=hidden_states.dtype,
group=get_tp_group().cpu_group,
)
if workspace is None:
return False, 0
# The token-count bound above uses the whole workspace budget, but a backend
# may use only a fraction of it per call (mnnvl rotates through three Lamport
# buffers). Ask the workspace so we don't admit tensors that the kernel will
# reject with "The buffer size in the given workspace is insufficient".
if not workspace.is_buffer_size_sufficient(
tp_size=tp_size,
num_tokens=num_tokens,
hidden_dim=hidden_size,
dtype=hidden_states.dtype,
):
return False, 0
return True, max_token_num
def _can_use_aiter_fused_ar_rms(hidden_states: torch.Tensor) -> bool:
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
if not rocm_aiter_ops.is_custom_all_reduce_enabled():
return False
if (
hidden_states.dim() != 2
or not hidden_states.is_contiguous()
or hidden_states.dtype not in _FI_SUPPORTED_DTYPES
):
return False
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
if aiter_ar is None or aiter_ar.disabled:
return False
total_bytes = hidden_states.numel() * hidden_states.element_size()
if total_bytes > aiter_ar.effective_max_size():
return False
if not aiter_ar.should_custom_ar(hidden_states):
return False
return aiter_ar.use_1stage_fused_ar_rms(hidden_states)
def fused_allreduce_gemma_rms_norm(
hidden_states: torch.Tensor,
residual: torch.Tensor,
norm: GemmaRMSNorm,
) -> tuple[torch.Tensor, torch.Tensor]:
"""All-reduce ``hidden_states`` + add ``residual`` + GemmaRMSNorm, fused.
``hidden_states`` is the per-rank *partial* (un-reduced) output of a
row-parallel linear; ``norm`` is the GemmaRMSNorm applied right after.
Returns ``(normed_output, new_residual)``, equivalent to
``norm(all_reduce(hidden_states), residual)``.
"""
tp_size = get_tensor_model_parallel_world_size()
if tp_size == 1:
# No all-reduce needed; identical to the unfused path.
return norm(hidden_states, residual)
ok, max_token_num = _can_use_flashinfer(hidden_states, tp_size)
if ok:
norm_out = torch.empty_like(hidden_states)
# With norm_out provided, the kernel writes the new residual
# (all_reduce(hidden_states) + residual) into the hidden_states buffer
# and the normalized result into norm_out, leaving `residual` untouched.
flashinfer_trtllm_fused_allreduce_norm(
allreduce_in=hidden_states,
residual=residual,
rms_gamma=norm.weight,
rms_eps=norm.variance_epsilon,
world_size=tp_size,
weight_bias=1.0, # GemmaRMSNorm-style
launch_with_pdl=True,
fp32_acc=True,
max_token_num=max_token_num,
pattern_code=_AR_RESIDUAL_RMS_NORM,
norm_out=norm_out,
)
return norm_out, hidden_states
if _can_use_aiter_fused_ar_rms(hidden_states):
from vllm._aiter_ops import rocm_aiter_ops
return rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()(
input_=hidden_states,
residual=residual,
weight=norm.weight,
epsilon=norm.variance_epsilon,
gemma_norm=True,
)
# Fallback: explicit all-reduce + GemmaRMSNorm (matches the unfused model).
reduced = tensor_model_parallel_all_reduce(hidden_states)
return norm(reduced, residual)
@@ -1,857 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepGEMM MegaMoE experts: fused EP dispatch, expert GEMMs and combine.
Each ``MEGA_MOE_*`` format is one row of the SM100 feature matrix. MX weights
and activations carry one UE8M0 scale per 1x32 block. NVFP4 weights carry one
E4M3 scale per 1x16 block plus a per-expert FP32 global scale; NVFP4 dispatch
scales each token by its own FP32 scale.
"""
import math
from collections.abc import Callable
from dataclasses import dataclass
from inspect import signature
from typing import Any, ClassVar, Literal
import torch
import torch.nn as nn
import vllm.envs as envs
from vllm.config import VllmConfig
from vllm.distributed import get_ep_group
from vllm.distributed.eplb.eplb_state import EplbLayerState
from vllm.forward_context import get_forward_context, is_forward_context_available
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe.prepare_megamoe import (
prepare_megamoe_inputs,
prepare_megamoe_nvfp4_inputs,
)
from vllm.model_executor.layers.fused_moe.router.base_router import (
eplb_map_to_physical_and_record,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp8Dynamic,
kNvfp4DynamicToken,
)
from vllm.model_executor.models.utils import extract_layer_index
from vllm.model_executor.utils import set_weight_attrs
from vllm.platforms import current_platform
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
logger = init_logger(__name__)
@dataclass(frozen=True)
class MegaMoEActivation:
"""Gated activation fused into the MegaMoE L1 epilogue.
``swiglu`` computes ``gate * sigmoid(alpha * gate) * (up + beta)`` after
clamping gate from above and up symmetrically to ``clamp``; this covers
plain, DeepSeek-V4 (``clamp``) and GPT-OSS style (``alpha``/``beta``)
experts. ``situ`` is Kimi K3's SiTU with gate tanh scale ``alpha`` and
linear tanh scale ``beta``; ``beta=0`` leaves the linear branch unscaled.
"""
name: Literal["swiglu", "situ"] = "swiglu"
clamp: float | None = None
alpha: float = 1.0
beta: float = 0.0
def __post_init__(self) -> None:
if self.name not in ("swiglu", "situ"):
raise ValueError(f"Unsupported MegaMoE activation {self.name!r}.")
if not (math.isfinite(self.alpha) and math.isfinite(self.beta)):
raise ValueError(f"MegaMoE activation scales must be finite: {self}.")
if self.clamp is not None and not self.clamp >= 0:
raise ValueError(f"MegaMoE activation clamp must be >= 0: {self}.")
if self.name == "situ" and (
self.clamp is not None or self.alpha <= 0 or self.beta < 0
):
raise ValueError(
"SiTU takes no clamp, a positive alpha and a non-negative beta; "
f"got {self}."
)
def kernel_kwargs(self) -> dict[str, Any]:
# Defaults stay implicit so DeepGEMM builds without alpha/beta still work.
kwargs: dict[str, Any] = {"activation_clamp": self.clamp}
if self.name != "swiglu":
kwargs["activation"] = self.name
if self.alpha != 1.0:
kwargs["activation_alpha"] = self.alpha
if self.beta != 0.0:
kwargs["activation_beta"] = self.beta
return kwargs
@dataclass(frozen=True)
class MegaMoEWeightSpec:
"""Storage of one MegaMoE weight in the kernel layout."""
dtype: torch.dtype
# Weight values packed per storage element along K (2 for FP4).
values_per_element: int = 1
# K values per block scale; None for unscaled weights.
scale_group_k: int | None = None
# Block scale dtype: UE8M0 (MX) or E4M3 (NVFP4).
scale_dtype: torch.dtype | None = None
@property
def has_global_scale(self) -> bool:
"""E4M3 block scales pair with an FP32 per-tensor scale."""
return self.scale_dtype == torch.float8_e4m3fn
_MXFP4_WEIGHT = MegaMoEWeightSpec(
torch.int8,
values_per_element=2,
scale_group_k=32,
scale_dtype=torch.float8_e8m0fnu,
)
_MXFP8_WEIGHT = MegaMoEWeightSpec(
torch.float8_e4m3fn, scale_group_k=32, scale_dtype=torch.float8_e8m0fnu
)
_NVFP4_WEIGHT = MegaMoEWeightSpec(
torch.int8,
values_per_element=2,
scale_group_k=16,
scale_dtype=torch.float8_e4m3fn,
)
_BF16_WEIGHT = MegaMoEWeightSpec(torch.bfloat16)
@dataclass(frozen=True)
class MegaMoEWeightFormat:
"""One routed-expert precision of the MegaMoE feature matrix."""
name: str
# DeepGEMM MMA tag; it also keys the symmetric-buffer layout.
mma_type: str
kernel: str
routed: MegaMoEWeightSpec
# Precisions a shared expert may have to run inside the same launch.
shared: tuple[MegaMoEWeightSpec, ...]
# Quantization of the dispatched hidden states; None dispatches BF16.
hidden_quant: QuantKey | None
activations: frozenset[str]
@property
def uses_nvfp4_api(self) -> bool:
"""``nvfp4_mega_moe`` has its own symmetric buffer, BLOCK_M and scales."""
return self.kernel == "nvfp4_mega_moe"
MEGA_MOE_MXFP4 = MegaMoEWeightFormat(
name="mxfp4",
mma_type="fp8xfp4",
kernel="fp8_fp4_mega_moe",
routed=_MXFP4_WEIGHT,
shared=(_MXFP8_WEIGHT,),
hidden_quant=kMxfp8Dynamic,
activations=frozenset({"swiglu", "situ"}),
)
MEGA_MOE_MXFP8 = MegaMoEWeightFormat(
name="mxfp8",
mma_type="fp8xfp8",
kernel="fp8_fp4_mega_moe",
routed=_MXFP8_WEIGHT,
shared=(_MXFP8_WEIGHT,),
hidden_quant=kMxfp8Dynamic,
activations=frozenset({"swiglu", "situ"}),
)
MEGA_MOE_NVFP4 = MegaMoEWeightFormat(
name="nvfp4",
mma_type="nvfp4",
kernel="nvfp4_mega_moe",
routed=_NVFP4_WEIGHT,
shared=(_MXFP8_WEIGHT, _BF16_WEIGHT),
hidden_quant=kNvfp4DynamicToken,
activations=frozenset({"swiglu"}),
)
MEGA_MOE_BF16 = MegaMoEWeightFormat(
name="bf16",
mma_type="bf16xbf16",
kernel="bf16_mega_moe",
routed=_BF16_WEIGHT,
shared=(_BF16_WEIGHT,),
hidden_quant=None,
activations=frozenset({"swiglu"}),
)
MEGA_MOE_WEIGHT_FORMATS = {
fmt.name: fmt
for fmt in (MEGA_MOE_MXFP4, MEGA_MOE_MXFP8, MEGA_MOE_NVFP4, MEGA_MOE_BF16)
}
def check_deep_gemm_mega_moe_supported(
device: torch.device, hidden_size: int, intermediate_size: int
) -> None:
"""Raise unless DeepGEMM MegaMoE can run on ``device`` with these shapes.
Raises:
NotImplementedError: If the device architecture is unsupported.
ValueError: If the shapes are unsupported.
"""
if not current_platform.is_device_capability_family(
100, device_id=device.index or 0
):
raise NotImplementedError("DeepGEMM MegaMoE requires SM100 GPUs.")
if hidden_size % 128 != 0 or intermediate_size % 128 != 0:
raise ValueError(
"DeepGEMM MegaMoE requires hidden and intermediate sizes "
"to be multiples of 128."
)
def _linear_weight_scale(linear: nn.Module) -> torch.Tensor | None:
scale = getattr(linear, "weight_scale", None)
return getattr(linear, "weight_scale_inv", None) if scale is None else scale
def _matches_weight_spec(linear: nn.Module, spec: MegaMoEWeightSpec) -> bool:
"""Whether a checkpoint linear can be written into a ``spec`` weight."""
weight, scale = linear.weight, _linear_weight_scale(linear)
if weight.dtype != spec.dtype or weight.dim() != 2:
return False
if spec.scale_group_k is None:
return scale is None
if scale is None or scale.dim() != 2:
return False
(rows, cols), (scale_rows, scale_cols) = weight.shape, scale.shape
scale_dtypes = {spec.scale_dtype}
if spec.scale_dtype == torch.float8_e8m0fnu:
scale_dtypes.add(torch.uint8)
return (
scale.dtype in scale_dtypes
and rows % scale_rows == 0
and cols % scale_cols == 0
and (cols // scale_cols) % spec.scale_group_k == 0
)
class DeepGemmMegaMoEExperts(nn.Module):
"""Routed experts, and optionally the shared expert, as one MegaMoE launch.
Parameters are registered in the kernel layout and every checkpoint shard is
written straight into it, so loading (and reloading) needs no post-load
transform. Whether the shared expert runs inside the launch is decided here,
before any weight loads; a fused layer owns the shared-expert weights and
the caller drops its shared MLP.
NVFP4 experts also take per-expert ``weight_scale_2`` (``w1`` and ``w3``
must match: one L1 scale per expert) and ``w2`` ``input_scale``; the kernel
quantizes every L2 input with the max of the latter over all loaded
experts. Dispatch uses per-token scales, so ``w1``/``w3`` ``input_scale``
are not loaded.
Args:
num_experts: Physical experts across EP ranks, redundant included.
activation: Activation fused into the L1 epilogue.
weight_format: Routed-expert precision, a row of the feature matrix.
shared_experts: Shared MLP to fuse if it is unsharded and in one of
the ``weight_format.shared`` precisions; ``None`` never fuses.
"""
_symm_buffers: ClassVar[dict[tuple, Any]] = {}
_synchronized_ep_groups: ClassVar[set[tuple[int, int]]] = set()
def __init__(
self,
vllm_config: VllmConfig,
*,
num_experts: int,
num_local_experts: int,
experts_start_idx: int,
top_k: int,
hidden_size: int,
intermediate_size: int,
activation: MegaMoEActivation,
weight_format: MegaMoEWeightFormat,
shared_experts: nn.Module | None = None,
num_logical_experts: int | None = None,
fast_math: bool = True,
prefix: str = "",
):
super().__init__()
if activation.name not in weight_format.activations:
raise NotImplementedError(
f"DeepGEMM MegaMoE {weight_format.name} experts support "
f"{sorted(weight_format.activations)}, not {activation.name!r}."
)
device = torch.get_default_device()
if device.type != current_platform.device_type: # meta under ipc_cache
device = torch.device(
current_platform.device_type, torch.accelerator.current_device_index()
)
check_deep_gemm_mega_moe_supported(device, hidden_size, intermediate_size)
self.prefix = prefix
self.capture_fn: Callable[[torch.Tensor], None] | None = None
self.num_experts = num_experts
self.num_local_experts = num_local_experts
self.experts_start_idx = experts_start_idx
self.experts_end_idx = experts_start_idx + num_local_experts
self.num_logical_experts = (
num_logical_experts if num_logical_experts is not None else num_experts
)
self.top_k = top_k
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
self.weight_format = weight_format
self.activation = activation
self.fast_math = fast_math
self._kernel_kwargs = activation.kernel_kwargs()
self.eplb_state = EplbLayerState()
# L2 input scale of every logical expert; the NVFP4 kernel takes the max.
self._w2_input_scales = torch.zeros(self.num_logical_experts, device="cpu")
self.l2_activation_scale = 1.0
# The first of each expert's w1/w3 weight_scale_2 until the other loads.
self._pending_l1_scales: dict[int, tuple[str, float]] = {}
self.num_shared_experts, self.shared_spec = self._fusable_shared_experts(
shared_experts
)
self._create_weights()
for name, param in self.named_parameters(recurse=False):
shared = name.startswith("shared_")
loader = self.shared_weight_loader if shared else self.weight_loader
set_weight_attrs(param, {"weight_loader": loader})
if not shared and name.endswith("weight_scale"):
param.quant_method = "block"
# Register in the static forward context so the custom-op wrapper
# can look up this module by name from within a torch.compile graph.
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
compilation_config.static_forward_context[prefix] = self
@property
def has_fused_shared_experts(self) -> bool:
return self.num_shared_experts > 0
def _fusable_shared_experts(
self, shared_experts: nn.Module | None
) -> tuple[int, MegaMoEWeightSpec | None]:
"""How many shared experts the launch fuses, and in which precision."""
if shared_experts is None:
return 0, None
gate_up, down = shared_experts.gate_up_proj, shared_experts.down_proj
hidden, inter = self.hidden_size, self.intermediate_size
shared_size = down.weight.shape[-1]
spec = next(
(
spec
for spec in self.weight_format.shared
if all(_matches_weight_spec(lin, spec) for lin in (gate_up, down))
),
None,
)
if (
tuple(gate_up.weight.shape) != (2 * shared_size, hidden)
or tuple(down.weight.shape) != (hidden, shared_size)
or shared_size % inter
):
reason = (
f"needs an unsharded shared MLP over hidden size {hidden} with a "
f"multiple of {inter} intermediate; got {tuple(gate_up.weight.shape)}"
f" and {tuple(down.weight.shape)}"
)
elif spec is None:
scale = _linear_weight_scale(gate_up)
supported = ", ".join(
f"{s.dtype}{' with 1x32 UE8M0 scales' if s.scale_group_k else ''}"
for s in self.weight_format.shared
)
reason = (
f"{self.weight_format.name} experts fuse {supported} shared "
f"experts; got weight {gate_up.weight.dtype} and scale "
f"{None if scale is None else (tuple(scale.shape), scale.dtype)}"
)
elif not self._kernel_fuses_shared_experts(spec):
reason = (
"the installed DeepGEMM Python API is older than the vLLM source. "
"Rebuild the vendored _deep_gemm_C extension to enable it"
)
else:
return shared_size // inter, spec
logger.warning_once("Disabling MegaMoE shared-expert fusion: %s.", reason)
return 0, None
def _kernel_fuses_shared_experts(self, spec: MegaMoEWeightSpec) -> bool:
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
fmt = self.weight_format
buffer_api = "get_symm_buffer_for_mega_moe"
block_m_api = "get_block_m_for_mega_moe"
if fmt.uses_nvfp4_api:
buffer_api = "NVFP4SymmBuffer"
block_m_api = "get_block_m_for_nvfp4_mega_moe"
try:
api = {
*signature(getattr(deep_gemm, buffer_api)).parameters,
*signature(getattr(deep_gemm, fmt.kernel)).parameters,
}
except (AttributeError, TypeError, ValueError):
return False
# Scaled shared inputs are laid out by the runtime BLOCK_M.
return {
"num_shared_experts",
"shared_l1_weights",
"shared_l2_weights",
} <= api and (spec.scale_group_k is None or hasattr(deep_gemm, block_m_api))
def _create_weights(self) -> None:
"""Register the routed and fused shared-expert parameters.
Routed parameters are indexed by local expert on dim 0, which EPLB moves.
"""
e, h, i = self.num_local_experts, self.hidden_size, self.intermediate_size
fmt = self.weight_format
self._register_weight("w13", (e, 2 * i, h), fmt.routed)
self._register_weight("w2", (e, h, i), fmt.routed)
if fmt.routed.has_global_scale:
# w13_weight_scale_2 is the kernel's L1 alpha (per-token dispatch).
for name in ("w13_weight_scale_2", "w2_weight_scale_2", "w2_input_scale"):
scale = torch.zeros(e, dtype=torch.float32)
self.register_parameter(name, nn.Parameter(scale, requires_grad=False))
# w2 global weight scale times the L2 activation scale, per launch.
l2_alpha = torch.zeros(e, dtype=torch.float32)
self.register_buffer("_l2_alpha", l2_alpha, persistent=False)
if self.shared_spec is not None:
# The shared expert has the layout of one routed expert slice.
s = i * self.num_shared_experts
self._register_weight("shared_l1", (2 * s, h), self.shared_spec)
self._register_weight("shared_l2", (h, s), self.shared_spec)
def _register_weight(
self, name: str, shape: tuple[int, ...], spec: MegaMoEWeightSpec
) -> None:
def register(name: str, data: torch.Tensor) -> None:
self.register_parameter(name, nn.Parameter(data, requires_grad=False))
*outer, n, k = shape
register(
f"{name}_weight",
torch.zeros(*outer, n, k // spec.values_per_element, dtype=spec.dtype),
)
if spec.scale_group_k is not None:
# Block scales, MN-major, four per int32 along K.
scale_k = k // (4 * spec.scale_group_k)
scale = torch.zeros(*outer, scale_k, n, dtype=torch.int32)
register(f"{name}_weight_scale", scale.transpose(-1, -2))
def _kernel_weights(
self, name: str
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
weight = getattr(self, f"{name}_weight")
scale = getattr(self, f"{name}_weight_scale", None)
return weight if scale is None else (weight, scale)
@property
def l1_weights(self) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
return self._kernel_weights("w13")
@property
def l2_weights(self) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
return self._kernel_weights("w2")
def _map_global_expert_id(self, expert_id: int) -> list[int]:
"""Local slots that logical expert ``expert_id`` occupies on this rank."""
physical_ids: list[int] = []
for p in range(self.experts_start_idx, self.experts_end_idx):
if p % self.num_logical_experts == expert_id:
physical_ids.append(p - self.experts_start_idx)
return physical_ids
def weight_loader(
self,
param: nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
return_success: bool = False,
) -> bool | None:
if not loaded_weight.is_meta:
self._record_global_scale(weight_name, loaded_weight, shard_id, expert_id)
local_expert_ids = self._map_global_expert_id(expert_id)
param_prefix = "w2_" if shard_id == "w2" else "w13_"
if not local_expert_ids or param_prefix not in weight_name:
return False if return_success else None
is_global_scale = weight_name.endswith(("weight_scale_2", "input_scale"))
is_scale = weight_name.endswith("weight_scale")
for local_expert_id in local_expert_ids:
if is_global_scale:
param.data[local_expert_id].copy_(loaded_weight.reshape(()))
else:
self._load_weight(
param.data[local_expert_id], loaded_weight, shard_id, is_scale
)
return True if return_success else None
def _record_global_scale(
self, weight_name: str, scale: torch.Tensor, shard_id: str, expert_id: int
) -> None:
"""Track NVFP4 global scales the kernel takes one of per launch or expert.
Every rank sees the scales of all experts, so ranks agree and EPLB can
move experts.
"""
if weight_name.endswith("w2_input_scale"):
self._w2_input_scales[expert_id] = scale.max()
self.l2_activation_scale = float(self._w2_input_scales.max()) or 1.0
elif weight_name.endswith("w13_weight_scale_2"):
# Compare w1 and w3 of the same load, not the live (maybe stale) slot.
prev = self._pending_l1_scales.pop(expert_id, None)
value = float(scale)
if prev is None or prev[0] == shard_id:
self._pending_l1_scales[expert_id] = (shard_id, value)
elif not math.isclose(value, prev[1], rel_tol=1e-5):
raise ValueError(
f"MegaMoE NVFP4 expert {expert_id}: {prev[0]} weight_scale_2 "
f"{prev[1]} != {shard_id} weight_scale_2 {value}; the kernel "
"takes one L1 scale per expert."
)
def load_shared_expert_weight(self, name: str, loaded_weight: torch.Tensor) -> str:
"""Load one shared-expert checkpoint tensor; return the parameter name."""
if ".w2." in name or ".down_proj." in name:
param_name = "shared_l2_weight"
elif ".w1." in name or ".w3." in name:
param_name = "shared_l1_weight"
else:
raise ValueError(f"Unexpected shared-expert tensor: {name}")
if "weight_scale" in name:
param_name += "_scale"
param = getattr(self, param_name)
param.weight_loader(param, loaded_weight, name)
return f"{name.partition('shared_experts.')[0]}experts.{param_name}"
def shared_weight_loader(
self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str
) -> None:
shard_id = (
"w1" if ".w1." in weight_name else "w3" if ".w3." in weight_name else "w2"
)
is_scale = "weight_scale" in weight_name
self._load_weight(param.data, loaded_weight, shard_id, is_scale)
def _load_weight(
self,
dst: torch.Tensor,
src: torch.Tensor,
shard_id: str,
is_scale: bool,
) -> None:
"""Write checkpoint shard ``src`` into ``dst`` in the kernel layout.
``dst`` is one routed expert's slice or a shared-expert parameter.
"""
if shard_id not in ("w1", "w2", "w3"):
raise ValueError(f"Unsupported expert shard id: {shard_id}")
rows = dst.shape[0] // (1 if shard_id == "w2" else 2)
if is_scale:
src = self._pack_scale(src, rows, dst.shape[1] * 4)
else:
src = src.view(dst.dtype)
if tuple(src.shape) != (rows, dst.shape[1]):
raise ValueError(
f"MegaMoE {shard_id} weight shape {tuple(src.shape)} does not "
f"match the kernel shard ({rows}, {dst.shape[1]})"
)
dst, src = self._kernel_layout_shard(dst, src, shard_id, is_scale)
dst.copy_(src)
@staticmethod
def _pack_scale(sf: torch.Tensor, rows: int, cols: int) -> torch.Tensor:
"""Expand block scales to the kernel granularity, four per int32 along K."""
sf = sf.view(torch.uint8)
if rows % sf.shape[0] or cols % sf.shape[1]:
raise ValueError(
f"Scale shape {tuple(sf.shape)} does not tile ({rows}, {cols})"
)
sf = sf.repeat_interleave(rows // sf.shape[0], 0)
sf = sf.repeat_interleave(cols // sf.shape[1], 1)
return sf.contiguous().clone().view(torch.int32)
@staticmethod
def _kernel_layout_shard(
dst: torch.Tensor,
src: torch.Tensor,
shard_id: str,
is_scale: bool,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Pair a checkpoint shard with its destination in the MegaMoE layout."""
k = src.shape[-1]
if shard_id == "w2":
if not is_scale:
return dst, src
# UTCCP: scale row 128*b + 32*j + i -> 128*b + 4*i + j.
return dst.view(-1, 32, 4, k).transpose(1, 2), src.reshape(-1, 4, 32, k)
half = 0 if shard_id == "w1" else 1
if not is_scale:
# Interleave gate/up by 8 rows: r -> (r // 8) * 16 + 8 * half + r % 8.
return dst.view(-1, 2, 8, k)[:, half], src.reshape(-1, 8, k)
# Interleave + UTCCP: 64b + 16j + 8q + m -> 128b + 64q + 32half + 4m + j.
dst = dst.view(-1, 2, 2, 8, 4, k)[:, :, half].permute(0, 3, 1, 2, 4)
return dst, src.reshape(-1, 4, 2, 8, k)
def get_symm_buffer(self):
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
group = get_ep_group().device_group
key = (
id(group),
torch.accelerator.current_device_index(),
self.num_experts,
self.max_num_tokens,
self.top_k,
self.hidden_size,
self.intermediate_size,
self.num_shared_experts,
None if self.shared_spec is None else self.shared_spec.dtype,
self.weight_format.mma_type,
self.activation.name,
)
symm_buffer = self._symm_buffers.get(key)
if symm_buffer is None:
args = (
group,
self.num_experts,
self.max_num_tokens,
self.top_k,
self.hidden_size,
self.intermediate_size,
)
if self.weight_format.uses_nvfp4_api:
symm_buffer = deep_gemm.NVFP4SymmBuffer(
*args,
num_shared_experts=self.num_shared_experts,
shared_dtype=torch.float8_e4m3fn
if self.shared_spec is None
else self.shared_spec.dtype,
)
else:
symm_buffer = deep_gemm.get_symm_buffer_for_mega_moe(
*args,
num_shared_experts=self.num_shared_experts,
mma_type=self.weight_format.mma_type,
activation=self.activation.name,
)
self._symm_buffers[key] = symm_buffer
return symm_buffer
def get_block_m(self, num_tokens: int) -> int:
"""BLOCK_M the MegaMoE scheduler runs ``num_tokens`` local tokens with."""
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
args = (
get_ep_group().world_size,
self.num_experts,
self.get_symm_buffer().num_max_tokens_per_rank,
num_tokens,
self.top_k,
)
if self.weight_format.uses_nvfp4_api:
return deep_gemm.get_block_m_for_nvfp4_mega_moe(*args)
return deep_gemm.get_block_m_for_mega_moe(*args, self.weight_format.mma_type)
def set_eplb_state(
self,
moe_layer_idx: int,
expert_load_view: torch.Tensor,
logical_to_physical_map: torch.Tensor,
logical_replica_count: torch.Tensor,
) -> None:
self.eplb_state.set_layer_state(
moe_layer_idx,
expert_load_view,
logical_to_physical_map,
logical_replica_count,
)
def get_expert_weights(self) -> list[torch.Tensor]:
def _to_eplb_view(name: str, t: torch.Tensor) -> torch.Tensor:
"""Return a (num_local_experts, -1) view with contiguous memory layout."""
assert t.shape[0] == self.num_local_experts
if t.is_contiguous():
return t.view(self.num_local_experts, -1)
elif t.dim() == 3 and t.stride(1) == 1 and t.stride(2) == t.shape[1]:
# scales have shape (E, M, N) with memory layout (E, N, M)
back = torch.transpose(t, 1, 2)
assert back.is_contiguous()
return back.view(self.num_local_experts, -1)
raise AssertionError(
f"MegaMoE EPLB {name}: non-contiguous expert tensor with "
f"unexpected layout shape={tuple(t.shape)} "
f"stride={tuple(t.stride())} dtype={t.dtype}"
)
return [
_to_eplb_view(name, param)
for name, param in self.named_parameters(recurse=False)
if not name.startswith("shared_")
]
def update_expert_map(self) -> None:
pass
@property
def layer_id(self) -> int:
return extract_layer_index(self.prefix)
def _synchronize_first_launch(self) -> None:
ep_group = get_ep_group()
key = (id(ep_group.cpu_group), torch.accelerator.current_device_index())
if key in self._synchronized_ep_groups:
return
torch.accelerator.synchronize()
torch.distributed.barrier(group=ep_group.cpu_group)
self._synchronized_ep_groups.add(key)
def forward(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
shared_experts: nn.Module | None = None,
) -> torch.Tensor:
"""Run the routed and shared experts on this rank's tokens.
Args:
hidden_states: BF16 ``[num_tokens, hidden_size]`` activations.
topk_weights: ``[num_tokens, top_k]`` routing weights.
topk_ids: ``[num_tokens, top_k]`` logical expert ids.
shared_experts: Unfused shared MLP, run on ``hidden_states`` and
added to the output. Must be ``None`` when this layer fused the
shared expert at construction.
Returns:
BF16 ``[num_tokens, hidden_size]`` routed plus shared expert output.
"""
if shared_experts is not None and self.has_fused_shared_experts:
raise ValueError(
f"{self.prefix} fuses the shared expert into MegaMoE; "
"pass shared_experts=None."
)
num_tokens = hidden_states.shape[0]
if num_tokens > self.max_num_tokens:
raise ValueError(
f"MegaMoE got {num_tokens} tokens, but the symmetric buffer was "
f"sized for {self.max_num_tokens}."
)
self._synchronize_first_launch()
is_padding = None
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
is_padding = get_forward_context().is_padding
if is_padding is not None:
is_padding = is_padding[:num_tokens]
if self.capture_fn is not None:
self.capture_fn(topk_ids)
# EPLB: map logical expert IDs to physical replicas and record load.
eplb_state = self.eplb_state
if eplb_state.logical_to_physical_map is not None:
assert eplb_state.expert_load_view is not None
assert eplb_state.logical_replica_count is not None
assert eplb_state.should_record_tensor is not None
if is_padding is not None:
topk_ids = torch.where(is_padding.unsqueeze(1), -1, topk_ids)
topk_ids = eplb_map_to_physical_and_record(
topk_ids=topk_ids,
expert_load_view=eplb_state.expert_load_view,
logical_to_physical_map=eplb_state.logical_to_physical_map,
logical_replica_count=eplb_state.logical_replica_count,
record_enabled=eplb_state.should_record_tensor,
num_unpadded_tokens=eplb_state.num_unpadded_tokens_tensors[
dbo_current_ubatch_id()
]
if eplb_state.num_unpadded_tokens_tensors is not None
else None,
)
from vllm.utils.deep_gemm import _import_deep_gemm
deep_gemm = _import_deep_gemm()
fmt = self.weight_format
symm_buffer = self.get_symm_buffer()
shared_x_sf = None
shared_block_m = None
if self.shared_spec is not None and self.shared_spec.scale_group_k:
shared_x_sf = symm_buffer.shared_l1_acts_sf
shared_block_m = self.get_block_m(num_tokens)
kwargs = dict(self._kernel_kwargs)
if fmt.uses_nvfp4_api:
l2_activation_scale = self.l2_activation_scale
prepare_megamoe_nvfp4_inputs(
hidden_states,
topk_weights,
topk_ids,
symm_buffer.x[:num_tokens],
symm_buffer.x_sf[:num_tokens],
symm_buffer.x_scales[:num_tokens],
symm_buffer.topk_idx[:num_tokens],
symm_buffer.topk_weights[:num_tokens],
is_padding=is_padding,
shared_x=symm_buffer.shared_l1_acts[:num_tokens]
if self.has_fused_shared_experts
else None,
shared_x_sf=shared_x_sf,
shared_block_m=shared_block_m,
w2_global_scale=self.w2_weight_scale_2,
l2_alpha=self._l2_alpha,
l2_activation_scale=l2_activation_scale,
)
kwargs.update(
l1_alpha=self.w13_weight_scale_2,
l2_alpha=self._l2_alpha,
l2_activation_scale=l2_activation_scale,
use_x_scales=True,
)
else:
quantized = fmt.hidden_quant is not None
prepare_megamoe_inputs(
hidden_states,
topk_weights,
topk_ids,
symm_buffer.x[:num_tokens],
symm_buffer.x_sf[:num_tokens] if quantized else None,
symm_buffer.topk_idx[:num_tokens],
symm_buffer.topk_weights[:num_tokens],
is_padding=is_padding,
shared_x_sf=shared_x_sf,
shared_block_m=shared_block_m,
hidden_quant=fmt.hidden_quant,
)
if self.has_fused_shared_experts:
kwargs["shared_l1_weights"] = self._kernel_weights("shared_l1")
kwargs["shared_l2_weights"] = self._kernel_weights("shared_l2")
y = torch.empty_like(hidden_states, dtype=torch.bfloat16)
getattr(deep_gemm, fmt.kernel)(
y,
self.l1_weights,
self.l2_weights,
symm_buffer,
fast_math=self.fast_math,
**kwargs,
)
if shared_experts is not None:
y += shared_experts(hidden_states)
return y
DeepGemmMegaMoEExperts.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
@@ -1,79 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import abstractmethod
from collections.abc import Callable, Mapping
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
from vllm.model_executor.warmup.jit_warmup import (
VllmJitKernel,
)
DEFAULT_CUTEDSL_COMPILE_OPTIONS = "--enable-tvm-ffi"
CompileKeyT = TypeVar("CompileKeyT")
CuTeDSLLaunchSpec: TypeAlias = (
tuple[CompileKeyT, tuple[Any, ...]]
| tuple[CompileKeyT, tuple[Any, ...], Any]
| tuple[
CompileKeyT,
tuple[Any, ...],
Any,
Callable[[], Any],
]
)
def cutedsl_fake_stream(*, use_tvm_ffi_env_stream: bool = True) -> Any:
import cutlass.cute as cute
return cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=use_tvm_ffi_env_stream)
def compile_cutedsl(
entry: Callable[..., Any],
*args: Any,
options: str = DEFAULT_CUTEDSL_COMPILE_OPTIONS,
use_tvm_ffi_env_stream: bool = True,
) -> Any:
import cutlass.cute as cute
return cute.compile(
entry,
*args,
cutedsl_fake_stream(use_tvm_ffi_env_stream=use_tvm_ffi_env_stream),
options=options,
)
class VllmCuTeDSLJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
"""CuTeDSL owner whose compiled executor is shared by warmup and runtime."""
kernel: ClassVar[Any]
bind_launch_inputs = False
@abstractmethod
def warmup_inputs(self, compile_key: CompileKeyT) -> tuple[Any, ...]:
"""Return fake arguments that compile one executor specialization."""
raise NotImplementedError
def compile(self, compile_key: CompileKeyT) -> None:
if compile_key in self._compiled_cache:
return
self._compiled_cache[compile_key] = compile_cutedsl(
self.kernel(compile_key),
*self.warmup_inputs(compile_key),
)
def launch(
self,
launch_spec: CuTeDSLLaunchSpec[CompileKeyT],
_inputs: Mapping[str, Any],
) -> Any:
# (compile_key, args), optionally followed by output and epilogue.
compile_key, launch_args = launch_spec[:2]
executor = self._get_or_compile(compile_key)
result = executor(*launch_args)
if len(launch_spec) == 4:
return launch_spec[3]()
return launch_spec[2] if len(launch_spec) == 3 else result
@@ -1,171 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Compile-only helpers for TileLang JIT warmup."""
from __future__ import annotations
import logging
from abc import abstractmethod
from collections.abc import Callable, Iterator, Mapping, MutableMapping
from contextlib import contextmanager
from dataclasses import dataclass, field
from functools import wraps
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar, cast
import torch
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
CompileKeyT = TypeVar("CompileKeyT")
# (kernel_args, launch_args), optionally followed by keywords and a result.
TileLangLaunchSpec: TypeAlias = (
tuple[tuple[Any, ...], tuple[Any, ...]]
| tuple[tuple[Any, ...], tuple[Any, ...], Mapping[str, Any] | None]
| tuple[tuple[Any, ...], tuple[Any, ...], Mapping[str, Any] | None, Any]
)
@contextmanager
def _quiet_tilelang_warmup_logs() -> Iterator[None]:
loggers = [
logging.getLogger("tilelang.cache.kernel_cache"),
logging.getLogger("tilelang.jit.kernel"),
]
previous_levels = [logger.level for logger in loggers]
try:
for logger in loggers:
logger.setLevel(logging.ERROR)
yield
finally:
for logger, level in zip(loggers, previous_levels):
logger.setLevel(level)
@dataclass(frozen=True)
class TileLangWarmupTensor:
"""Minimal tensor-like object accepted by TileLang compile().
TileLang builds its cache key and TIR from tensor dtype, shape and stride.
This object deliberately has no storage, so compile-only warmup does not
allocate GPU memory and does not launch the kernel.
"""
dtype: torch.dtype
shape: tuple[int, ...] = (1,)
strides: tuple[int, ...] | None = field(default=None)
def stride(self) -> tuple[int, ...]:
if self.strides is not None:
return self.strides
strides: list[int] = []
stride = 1
for size in reversed(self.shape):
strides.append(stride)
stride *= size
return tuple(reversed(strides))
def new_empty(
self,
shape: tuple[int, ...],
*,
dtype: torch.dtype | None = None,
device: Any = None,
) -> TileLangWarmupTensor:
return TileLangWarmupTensor(dtype or self.dtype, shape)
def make_tilelang_warmup_tensor(
dtype: torch.dtype,
*shape: int,
strides: tuple[int, ...] | None = None,
) -> TileLangWarmupTensor:
return TileLangWarmupTensor(dtype=dtype, shape=tuple(shape), strides=strides)
def _tilelang_call_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
call_kwargs = dict(kwargs)
tune_params = call_kwargs.pop("__tune_params", {})
if isinstance(tune_params, dict):
call_kwargs.update(tune_params)
return call_kwargs
def compile_tilelang(jit_impl: Any, *args: Any, **kwargs: Any) -> None:
"""Compile one TileLang specialization and populate its call cache.
TileLang's ``compile()`` materializes the kernel without launching it.
We also store the compiled kernel in ``_kernel_cache`` using the same
parsed key as ``__call__`` so runtime does not report a cache miss for an
already materialized specialization.
"""
with _quiet_tilelang_warmup_logs():
compiled = jit_impl.compile(*args, **kwargs)
func = getattr(jit_impl, "func", None)
parse_args = getattr(func, "parse_args", None)
cache = getattr(jit_impl, "_kernel_cache", None)
if not callable(parse_args) or not isinstance(cache, MutableMapping):
return
key, _ = parse_args(*args, **_tilelang_call_kwargs(kwargs))
cache[key] = compiled
class VllmTileLangJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
"""TileLang owner whose runtime launch specification is reused for warmup."""
kernel: ClassVar[Any]
_warming_key: CompileKeyT | None = None
@abstractmethod
def warmup_inputs(self, compile_key: CompileKeyT) -> dict[str, Any]:
"""Return runtime-shaped inputs that reproduce one compile key."""
raise NotImplementedError
def compile(self, compile_key: CompileKeyT) -> None:
self._warming_key = compile_key
try:
cast(Callable[..., Any], self)(**self.warmup_inputs(compile_key))
finally:
self._warming_key = None
def launch(
self,
jit_impl: Any,
args: tuple[Any, ...],
kwargs: Mapping[str, Any] | None = None,
) -> Any:
call_kwargs = dict(kwargs or {})
if self._warming_key is not None:
return compile_tilelang(jit_impl, *args, **call_kwargs)
return jit_impl(*args, **call_kwargs)
def kernel_launcher(
call_fn: Callable[..., TileLangLaunchSpec],
) -> Callable[..., Any]:
"""Launch TileLang from declarative kernel and runtime argument tuples."""
@wraps(call_fn)
def wrapper(
self: VllmTileLangJitKernel[Any],
*args: Any,
**kwargs: Any,
) -> Any:
launch_spec = call_fn(self, *args, **kwargs)
kernel_args, launch_args = launch_spec[:2]
if self._warming_key is not None and kernel_args not in (
(),
(self._warming_key,),
):
raise RuntimeError(
"TileLang warmup inputs produced a different compile key: "
f"expected {self._warming_key!r}, got kernel arguments "
f"{kernel_args!r}"
)
launch_kwargs = launch_spec[2] if len(launch_spec) > 2 else None
output = launch_spec[3] if len(launch_spec) > 3 else None
result = self.launch(self.kernel(*kernel_args), launch_args, launch_kwargs)
return output if len(launch_spec) > 3 else result
return wrapper