[Mini-VLLM] Remove orphan subsystems: snapshot, CPU workers, dead decoders
- Delete the snapshot package, CLI command, and its launch tests - Delete CPU workers, cpu_communicator, and the hw_agnostic layer package - Delete the EC CPU connector, keeping the EC base glue intact - Delete dead composite/triton_flashinfer backends and third_party/flashmla - Delete dead video decoders, video-bound tests, and orphan utils - Remove the deep_gemm_mega_moe option and dead allreduce fusion pair Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,314 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import functools
|
||||
import inspect
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import get_cached_compilation_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.utils import maybe_disable_graph_partition
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Dictionary of all custom ops (classes, indexed by registered name).
|
||||
# To check if an op with a name is enabled, call .enabled() on the class.
|
||||
# Examples:
|
||||
# - MyOp.enabled()
|
||||
# - op_registry["my_op"].enabled()
|
||||
op_registry: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {}
|
||||
op_registry_oot: dict[str, type["CustomOp"] | type["PluggableLayer"]] = {}
|
||||
|
||||
|
||||
def maybe_get_oot_by_class(class_type: type) -> type:
|
||||
class_name = class_type.__name__
|
||||
if class_name in op_registry_oot:
|
||||
return op_registry_oot[class_name]
|
||||
return class_type
|
||||
|
||||
|
||||
class PluggableLayer(nn.Module):
|
||||
"""Base class for pluggable layers.
|
||||
|
||||
A PluggableLayer is a *module-composing* abstraction: it may instantiate other
|
||||
``torch.nn.Module`` objects as sub-layers, and its functionality depends on
|
||||
these sub-layers following a generalized invocation sequence. Also, it is stateful
|
||||
and may hold parameters or buffers.
|
||||
|
||||
Unlike :class:`CustomOp`, PluggableLayer does NOT provide per-platform
|
||||
``forward_*`` dispatch. Instead, it supports out-of-tree (OOT) replacement
|
||||
of the entire layer class at instantiation time, allowing customized
|
||||
initialization and submodule composition.
|
||||
"""
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
try:
|
||||
layer_class_name = cls.__name__
|
||||
except AttributeError:
|
||||
raise TypeError(
|
||||
f"Cannot instantiate '{cls.__name__}': its 'name' attribute "
|
||||
f"was not set, possibly because it was not decorated with "
|
||||
f"@PluggableLayer.register, or it's the PluggableLayer itself."
|
||||
) from None
|
||||
|
||||
if layer_class_name not in op_registry_oot:
|
||||
layer_cls_to_instantiate = cls
|
||||
else:
|
||||
layer_cls_to_instantiate = op_registry_oot[layer_class_name]
|
||||
logger.debug(
|
||||
"Instantiating pluggable layer: %s using %s",
|
||||
layer_class_name,
|
||||
str(layer_cls_to_instantiate),
|
||||
)
|
||||
return super().__new__(layer_cls_to_instantiate)
|
||||
|
||||
# Decorator to register pluggable layers.
|
||||
@classmethod
|
||||
def register(cls, name: str):
|
||||
def decorator(op_cls):
|
||||
assert name not in op_registry, f"Duplicate op name: {name}"
|
||||
op_cls.name = name
|
||||
op_registry[name] = op_cls
|
||||
return op_cls
|
||||
|
||||
return decorator
|
||||
|
||||
# Decorator to register out-of-tree(oot) pluggable layers.
|
||||
# For OOT pluggable layers:
|
||||
# if in-tree layer class is registered with an oot_custom_layer,
|
||||
# the oot_custom_layer will be used instead.
|
||||
@classmethod
|
||||
def register_oot(cls, _decorated_layer_cls=None, name: str | None = None):
|
||||
def decorator(layer_cls):
|
||||
reg_name = name if name is not None else cls.__name__
|
||||
assert reg_name not in op_registry_oot, f"Duplicate layer name: {reg_name}"
|
||||
layer_cls.name = reg_name
|
||||
op_registry_oot[reg_name] = layer_cls
|
||||
return layer_cls
|
||||
|
||||
if _decorated_layer_cls is None:
|
||||
# Called with parentheses: @PluggableLayer.register_oot()
|
||||
# or @PluggableLayer.register_oot(name="...")
|
||||
return decorator
|
||||
elif isinstance(_decorated_layer_cls, type): # Check if it's a class
|
||||
# Called without parentheses: @PluggableLayer.register_oot
|
||||
return decorator(_decorated_layer_cls)
|
||||
else:
|
||||
raise TypeError("Decorator can only be applied to classes.")
|
||||
|
||||
|
||||
class CustomOp(nn.Module):
|
||||
"""Base class for custom ops.
|
||||
Dispatches the forward method to the appropriate backend.
|
||||
"""
|
||||
|
||||
def __new__(cls, *args, **kwargs):
|
||||
try:
|
||||
op_name = cls.__name__
|
||||
except AttributeError:
|
||||
raise TypeError(
|
||||
f"Cannot instantiate '{cls.__name__}': its 'name' attribute "
|
||||
f"was not set, possibly because it was not decorated with "
|
||||
f"@CustomOp.register, or it's the CustomOp base class itself."
|
||||
) from None
|
||||
|
||||
if op_name not in op_registry_oot:
|
||||
op_cls_to_instantiate = cls
|
||||
else:
|
||||
op_cls_to_instantiate = op_registry_oot[op_name]
|
||||
logger.debug(
|
||||
"Instantiating custom op: %s using %s",
|
||||
op_name,
|
||||
str(op_cls_to_instantiate),
|
||||
)
|
||||
return super().__new__(op_cls_to_instantiate)
|
||||
|
||||
def __init__(self, *, enforce_enable: bool = False, compile_native: bool = False):
|
||||
super().__init__()
|
||||
self._enforce_enable = enforce_enable
|
||||
self._forward_method = self.dispatch_forward(compile_native=compile_native)
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
return self._forward_method(*args, **kwargs)
|
||||
|
||||
def forward_native(self, *args, **kwargs):
|
||||
"""PyTorch-native implementation; OOT plugins override via
|
||||
:meth:`forward_oot`."""
|
||||
raise NotImplementedError
|
||||
|
||||
def forward_oot(self, *args, **kwargs):
|
||||
# By default, OOT ops fall back to the PyTorch-native implementation.
|
||||
return self.forward_native(*args, **kwargs)
|
||||
|
||||
def dispatch_forward(self, compile_native: bool):
|
||||
# The hw-agnostic CustomOp dispatches only between forward_native
|
||||
# (in-tree, used on every platform) and forward_oot (overridden by
|
||||
# an OOT plugin via register_oot).
|
||||
compilation_config = get_cached_compilation_config()
|
||||
|
||||
# NOTE(shen-shanshan): CustomOp object can be enforce enabled, e.g.,
|
||||
# enable device-specific kernels in ViT models when enabling graph
|
||||
# mode. By default, it will follow the compilation_config to determine
|
||||
# whether enable itself.
|
||||
# This enforce_enable mechanism will be removed after we adding a
|
||||
# separate compilation_config for multi-modal part.
|
||||
enabled = self._enforce_enable or self.enabled()
|
||||
if enabled:
|
||||
compilation_config.enabled_custom_ops.update([self.__class__.name])
|
||||
else:
|
||||
compilation_config.disabled_custom_ops.update([self.__class__.name])
|
||||
|
||||
if not enabled:
|
||||
# Compile forward_native to avoid eager torch ops if inside
|
||||
# opaque torch custom op (e.g. fused_moe, unified_attention, etc.)
|
||||
return self.maybe_compile(self.forward_native, enable=compile_native)
|
||||
|
||||
if current_platform.is_out_of_tree():
|
||||
return self.forward_oot
|
||||
return self.forward_native
|
||||
|
||||
def maybe_compile(self, fn, *, enable: bool = True):
|
||||
"""Compile fn if compilation enabled.
|
||||
Useful for CustomOp instances called from within a torch custom op,
|
||||
meaning the forward call is hidden from the model-level torch.compile.
|
||||
|
||||
NOTE: this does not enable fusion across ops, so opaque custom ops
|
||||
should still be unwrapped wherever possible.
|
||||
"""
|
||||
from vllm.config.compilation import CompilationMode
|
||||
|
||||
# Do not compile if compilation disabled
|
||||
if not enable:
|
||||
return fn
|
||||
|
||||
# Do not compile if global compilation disabled
|
||||
compilation_config = get_cached_compilation_config()
|
||||
if compilation_config.mode == CompilationMode.NONE:
|
||||
return fn
|
||||
|
||||
# If eager backend is used, do not compile either
|
||||
if compilation_config.backend == "eager":
|
||||
return fn
|
||||
|
||||
compile_options = maybe_disable_graph_partition(
|
||||
current_platform.simple_compile_backend
|
||||
)
|
||||
backend = current_platform.simple_compile_backend
|
||||
|
||||
dynamic_arg_dims = getattr(self.__class__, "_dynamic_arg_dims", None)
|
||||
if dynamic_arg_dims is not None:
|
||||
compiled_fn = torch.compile(
|
||||
fn,
|
||||
dynamic=False,
|
||||
backend=backend,
|
||||
options=compile_options,
|
||||
)
|
||||
sig = inspect.signature(fn)
|
||||
|
||||
@functools.wraps(fn)
|
||||
def wrapper(*args, **kwargs):
|
||||
bound = sig.bind(*args, **kwargs)
|
||||
bound.apply_defaults()
|
||||
for name, dims in dynamic_arg_dims.items():
|
||||
arg = bound.arguments.get(name)
|
||||
if arg is not None and isinstance(arg, torch.Tensor):
|
||||
dims_list = [dims] if isinstance(dims, int) else dims
|
||||
for d in dims_list:
|
||||
real_d = arg.ndim + d if d < 0 else d
|
||||
torch._dynamo.mark_dynamic(arg, real_d)
|
||||
return compiled_fn(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
# dynamic=True to avoid recompilations
|
||||
return torch.compile(
|
||||
fn,
|
||||
dynamic=True,
|
||||
backend=backend,
|
||||
options=compile_options,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def enabled(cls) -> bool:
|
||||
# if no name, then it was not registered
|
||||
compilation_config = get_cached_compilation_config()
|
||||
custom_ops = compilation_config.custom_ops
|
||||
if not hasattr(cls, "name"):
|
||||
logger.warning_once(
|
||||
"Custom op %s was not registered, which means it won't appear "
|
||||
"in the op registry. It will be enabled/disabled based on the "
|
||||
"global settings.",
|
||||
cls.__name__,
|
||||
)
|
||||
return CustomOp.default_on()
|
||||
|
||||
enabled = f"+{cls.name}" in custom_ops
|
||||
disabled = f"-{cls.name}" in custom_ops
|
||||
assert not (enabled and disabled), f"Cannot enable and disable {cls.name}"
|
||||
|
||||
return (CustomOp.default_on() or enabled) and not disabled
|
||||
|
||||
@staticmethod
|
||||
def default_on() -> bool:
|
||||
"""Behavior controlled by `CompilationConfig.custom_ops`: On by default if
|
||||
'all', off by default if 'none'.
|
||||
When PyTorch Inductor is used, 'none' is the default value,
|
||||
otherwise 'all'.
|
||||
"""
|
||||
compilation_config = get_cached_compilation_config()
|
||||
count_none = compilation_config.custom_ops.count("none")
|
||||
count_all = compilation_config.custom_ops.count("all")
|
||||
assert count_none + count_all == 1
|
||||
|
||||
return not count_none > 0 or count_all > 0
|
||||
|
||||
# Decorator to register custom ops.
|
||||
@classmethod
|
||||
def register(
|
||||
cls,
|
||||
name: str,
|
||||
dynamic_arg_dims: dict[str, int | list[int]] | None = None,
|
||||
):
|
||||
def decorator(op_cls):
|
||||
assert name not in op_registry, f"Duplicate op name: {name}"
|
||||
op_cls.name = name
|
||||
op_cls._dynamic_arg_dims = dynamic_arg_dims
|
||||
op_registry[name] = op_cls
|
||||
return op_cls
|
||||
|
||||
return decorator
|
||||
|
||||
# Decorator to register out-of-tree(oot) custom ops.
|
||||
# For OOT custom ops:
|
||||
# if in-tree layer class is registered with an oot_custom_op layer,
|
||||
# the oot_custom_op layer will be used instead.
|
||||
# Example:
|
||||
# - @UnquantizedFusedMoEMethod.register_oot
|
||||
# class HPUUnquantizedFusedMoEMethod(UnquantizedFusedMoEMethod)
|
||||
# or
|
||||
# - @CustomOP.register_oot(name="UnquantizedFusedMoEMethod")
|
||||
@classmethod
|
||||
def register_oot(cls, _decorated_op_cls=None, name: str | None = None):
|
||||
def decorator(op_cls):
|
||||
reg_name = name if name is not None else cls.__name__
|
||||
assert reg_name not in op_registry_oot, f"Duplicate op name: {reg_name}"
|
||||
op_cls.name = reg_name
|
||||
op_registry_oot[reg_name] = op_cls
|
||||
return op_cls
|
||||
|
||||
if _decorated_op_cls is None:
|
||||
# Called with parentheses: @CustomOP.register_oot()
|
||||
# or @CustomOP.register_oot(name="...")
|
||||
# So, _decorated_op_cls is None.
|
||||
# We return the actual decorator function.
|
||||
return decorator
|
||||
elif isinstance(_decorated_op_cls, type): # Check if it's a class
|
||||
# Called without parentheses: @CustomOP.register_oot
|
||||
# The first argument is the class itself.
|
||||
# We call the 'decorator' function immediately with the class.
|
||||
return decorator(_decorated_op_cls)
|
||||
else:
|
||||
# Handle other unexpected cases if necessary
|
||||
raise TypeError("Decorator can only be applied to classes.")
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,27 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.hw_agnostic.custom_op import CustomOp
|
||||
|
||||
|
||||
@CustomOp.register("silu_and_mul")
|
||||
class SiluAndMul(CustomOp):
|
||||
"""SwiGLU: ``x -> silu(x[:d]) * x[d:]`` where ``d = x.shape[-1] // 2``."""
|
||||
|
||||
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
|
||||
d = x.shape[-1] // 2
|
||||
return F.silu(x[..., :d]) * x[..., d:]
|
||||
|
||||
|
||||
# Activation-and-mul ops keyed by HF activation name.
|
||||
_ACTIVATION_AND_MUL_REGISTRY = {
|
||||
"silu": SiluAndMul,
|
||||
"swish": SiluAndMul,
|
||||
}
|
||||
|
||||
|
||||
def get_act_and_mul_fn(act_fn_name: str) -> CustomOp:
|
||||
"""Build the hw-agnostic activation-and-mul op named `act_fn_name`."""
|
||||
return _ACTIVATION_AND_MUL_REGISTRY[act_fn_name.lower()]()
|
||||
@@ -1,90 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch import Tensor
|
||||
|
||||
from vllm.model_executor.hw_agnostic.custom_op import CustomOp
|
||||
|
||||
|
||||
@CustomOp.register("rms_norm")
|
||||
class RMSNorm(CustomOp):
|
||||
"""``x -> w * x / sqrt(E[x^2] + eps)``. With ``residual``, fuses
|
||||
``residual += x`` then RMSNorm and returns ``(normalized, residual)``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
eps: float = 1e-6,
|
||||
var_hidden_size: int | None = None,
|
||||
has_weight: bool = True,
|
||||
dtype: torch.dtype | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.variance_epsilon = eps
|
||||
self.variance_size_override = (
|
||||
None if var_hidden_size == hidden_size else var_hidden_size
|
||||
)
|
||||
weight_dtype = dtype or torch.get_default_dtype()
|
||||
self.has_weight = has_weight
|
||||
weight = torch.ones(hidden_size, dtype=weight_dtype)
|
||||
if has_weight:
|
||||
self.weight = nn.Parameter(weight)
|
||||
else:
|
||||
self.register_buffer("weight", weight, persistent=False)
|
||||
|
||||
def _rms_norm(
|
||||
self,
|
||||
x: Tensor,
|
||||
weight: Tensor | None,
|
||||
epsilon: float,
|
||||
variance_size: int | None = None,
|
||||
) -> Tensor:
|
||||
"""Weighted root-mean-square layer normalization"""
|
||||
orig_dtype = x.dtype
|
||||
x = x.to(torch.float32)
|
||||
x_var = x if variance_size is None else x[..., :variance_size]
|
||||
variance = x_var.pow(2).mean(dim=-1, keepdim=True)
|
||||
x = x * torch.rsqrt(variance + epsilon)
|
||||
if weight is not None:
|
||||
x = x.to(weight.dtype) * weight
|
||||
return x.to(orig_dtype)
|
||||
|
||||
def _fused_add_rms_norm(
|
||||
self,
|
||||
x: Tensor,
|
||||
x_residual: Tensor,
|
||||
weight: Tensor | None,
|
||||
epsilon: float,
|
||||
variance_size: int | None = None,
|
||||
) -> tuple[Tensor, Tensor]:
|
||||
"""Fused add and weighted root-mean-square layer normalization"""
|
||||
orig_dtype = x.dtype
|
||||
x = x.to(torch.float32)
|
||||
x = x + x_residual.to(torch.float32)
|
||||
x_residual = x.to(orig_dtype)
|
||||
|
||||
x_var = x if variance_size is None else x[..., :variance_size]
|
||||
variance = x_var.pow(2).mean(dim=-1, keepdim=True)
|
||||
x = x * torch.rsqrt(variance + epsilon)
|
||||
if weight is not None:
|
||||
x = x.to(weight.dtype) * weight
|
||||
return x.to(orig_dtype), x_residual
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
residual: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
weight = self.weight if self.has_weight else None
|
||||
epsilon = self.variance_epsilon
|
||||
variance_size = self.variance_size_override
|
||||
|
||||
if residual is None:
|
||||
return self._rms_norm(x, weight, epsilon, variance_size)
|
||||
else:
|
||||
return self._fused_add_rms_norm(x, residual, weight, epsilon, variance_size)
|
||||
|
||||
def extra_repr(self) -> str:
|
||||
return f"hidden_size={self.hidden_size}, eps={self.variance_epsilon}"
|
||||
@@ -1,190 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Manual fusion of tensor-parallel all-reduce with the following GemmaRMSNorm.
|
||||
|
||||
Under tensor parallelism a ``RowParallelLinear`` (e.g. attention ``o_proj``)
|
||||
produces a per-rank partial sum that is all-reduced, and the result is then fed
|
||||
into a ``GemmaRMSNorm`` that adds the residual and normalizes. flashinfer ships a
|
||||
kernel that fuses all-reduce + residual-add + RMSNorm into a single launch; this
|
||||
helper drives it directly (no torch.compile pass) for models that run eager.
|
||||
|
||||
Scope: attention output only, no quantization. When the flashinfer fast path is
|
||||
not applicable (TP==1, flashinfer/NVSwitch unavailable, unsupported dtype, or an
|
||||
oversize batch) it falls back to ``all_reduce`` + ``GemmaRMSNorm``, which is
|
||||
numerically identical to the unfused model path.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.communication_op import tensor_model_parallel_all_reduce
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MiB = 1024 * 1024
|
||||
|
||||
# flashinfer fused all-reduce + RMSNorm is wired as a registered custom op in
|
||||
# allreduce_rms_fusion; both that op and the workspace helpers only exist when
|
||||
# flashinfer.comm.allreduce_fusion is importable.
|
||||
try:
|
||||
from vllm.compilation.passes.fusion.allreduce_rms_fusion import (
|
||||
flashinfer_trtllm_fused_allreduce_norm,
|
||||
)
|
||||
from vllm.distributed.device_communicators.flashinfer_all_reduce import (
|
||||
flashinfer_comm,
|
||||
get_fi_ar_workspace,
|
||||
)
|
||||
|
||||
_AR_RESIDUAL_RMS_NORM = (
|
||||
flashinfer_comm.AllReduceFusionPattern.kARResidualRMSNorm
|
||||
if flashinfer_comm is not None
|
||||
else None
|
||||
)
|
||||
except (ImportError, AttributeError):
|
||||
flashinfer_trtllm_fused_allreduce_norm = None # type: ignore[assignment]
|
||||
get_fi_ar_workspace = None # type: ignore[assignment]
|
||||
_AR_RESIDUAL_RMS_NORM = None
|
||||
|
||||
|
||||
_FI_SUPPORTED_DTYPES = (torch.bfloat16, torch.float16)
|
||||
|
||||
|
||||
def _max_token_num(tp_size: int, hidden_size: int, dtype: torch.dtype) -> int | None:
|
||||
"""Workspace token budget for flashinfer fused all-reduce, or None if the
|
||||
current world size / device is unsupported. Mirrors ``FlashInferAllReduce``."""
|
||||
from vllm.config.compilation import PassConfig
|
||||
|
||||
max_size_mb = PassConfig.default_fi_allreduce_fusion_max_size_mb().get(tp_size)
|
||||
if not max_size_mb:
|
||||
return None
|
||||
element_size = torch.tensor([], dtype=dtype).element_size()
|
||||
return int(max_size_mb * MiB) // (hidden_size * element_size)
|
||||
|
||||
|
||||
def _can_use_flashinfer(hidden_states: torch.Tensor, tp_size: int) -> tuple[bool, int]:
|
||||
"""Whether the flashinfer fused path applies; returns (ok, max_token_num)."""
|
||||
if (
|
||||
flashinfer_trtllm_fused_allreduce_norm is None
|
||||
or get_fi_ar_workspace is None
|
||||
or _AR_RESIDUAL_RMS_NORM is None
|
||||
):
|
||||
return False, 0
|
||||
if (
|
||||
not hidden_states.is_cuda
|
||||
or hidden_states.dim() != 2
|
||||
or not hidden_states.is_contiguous()
|
||||
or hidden_states.dtype not in _FI_SUPPORTED_DTYPES
|
||||
):
|
||||
return False, 0
|
||||
|
||||
num_tokens, hidden_size = hidden_states.shape
|
||||
max_token_num = _max_token_num(tp_size, hidden_size, hidden_states.dtype)
|
||||
if max_token_num is None or num_tokens > max_token_num:
|
||||
return False, 0
|
||||
|
||||
# Lazily create / fetch the (globally cached) workspace; returns None on
|
||||
# GPUs without NVSwitch, in which case we fall back gracefully.
|
||||
workspace = get_fi_ar_workspace(
|
||||
world_size=tp_size,
|
||||
rank=get_tensor_model_parallel_rank(),
|
||||
max_token_num=max_token_num,
|
||||
hidden_dim=hidden_size,
|
||||
dtype=hidden_states.dtype,
|
||||
group=get_tp_group().cpu_group,
|
||||
)
|
||||
if workspace is None:
|
||||
return False, 0
|
||||
# The token-count bound above uses the whole workspace budget, but a backend
|
||||
# may use only a fraction of it per call (mnnvl rotates through three Lamport
|
||||
# buffers). Ask the workspace so we don't admit tensors that the kernel will
|
||||
# reject with "The buffer size in the given workspace is insufficient".
|
||||
if not workspace.is_buffer_size_sufficient(
|
||||
tp_size=tp_size,
|
||||
num_tokens=num_tokens,
|
||||
hidden_dim=hidden_size,
|
||||
dtype=hidden_states.dtype,
|
||||
):
|
||||
return False, 0
|
||||
return True, max_token_num
|
||||
|
||||
|
||||
def _can_use_aiter_fused_ar_rms(hidden_states: torch.Tensor) -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if not rocm_aiter_ops.is_custom_all_reduce_enabled():
|
||||
return False
|
||||
if (
|
||||
hidden_states.dim() != 2
|
||||
or not hidden_states.is_contiguous()
|
||||
or hidden_states.dtype not in _FI_SUPPORTED_DTYPES
|
||||
):
|
||||
return False
|
||||
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
|
||||
if aiter_ar is None or aiter_ar.disabled:
|
||||
return False
|
||||
total_bytes = hidden_states.numel() * hidden_states.element_size()
|
||||
if total_bytes > aiter_ar.effective_max_size():
|
||||
return False
|
||||
if not aiter_ar.should_custom_ar(hidden_states):
|
||||
return False
|
||||
return aiter_ar.use_1stage_fused_ar_rms(hidden_states)
|
||||
|
||||
|
||||
def fused_allreduce_gemma_rms_norm(
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
norm: GemmaRMSNorm,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""All-reduce ``hidden_states`` + add ``residual`` + GemmaRMSNorm, fused.
|
||||
|
||||
``hidden_states`` is the per-rank *partial* (un-reduced) output of a
|
||||
row-parallel linear; ``norm`` is the GemmaRMSNorm applied right after.
|
||||
Returns ``(normed_output, new_residual)``, equivalent to
|
||||
``norm(all_reduce(hidden_states), residual)``.
|
||||
"""
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
if tp_size == 1:
|
||||
# No all-reduce needed; identical to the unfused path.
|
||||
return norm(hidden_states, residual)
|
||||
|
||||
ok, max_token_num = _can_use_flashinfer(hidden_states, tp_size)
|
||||
if ok:
|
||||
norm_out = torch.empty_like(hidden_states)
|
||||
# With norm_out provided, the kernel writes the new residual
|
||||
# (all_reduce(hidden_states) + residual) into the hidden_states buffer
|
||||
# and the normalized result into norm_out, leaving `residual` untouched.
|
||||
flashinfer_trtllm_fused_allreduce_norm(
|
||||
allreduce_in=hidden_states,
|
||||
residual=residual,
|
||||
rms_gamma=norm.weight,
|
||||
rms_eps=norm.variance_epsilon,
|
||||
world_size=tp_size,
|
||||
weight_bias=1.0, # GemmaRMSNorm-style
|
||||
launch_with_pdl=True,
|
||||
fp32_acc=True,
|
||||
max_token_num=max_token_num,
|
||||
pattern_code=_AR_RESIDUAL_RMS_NORM,
|
||||
norm_out=norm_out,
|
||||
)
|
||||
return norm_out, hidden_states
|
||||
|
||||
if _can_use_aiter_fused_ar_rms(hidden_states):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
return rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()(
|
||||
input_=hidden_states,
|
||||
residual=residual,
|
||||
weight=norm.weight,
|
||||
epsilon=norm.variance_epsilon,
|
||||
gemma_norm=True,
|
||||
)
|
||||
|
||||
# Fallback: explicit all-reduce + GemmaRMSNorm (matches the unfused model).
|
||||
reduced = tensor_model_parallel_all_reduce(hidden_states)
|
||||
return norm(reduced, residual)
|
||||
@@ -1,857 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepGEMM MegaMoE experts: fused EP dispatch, expert GEMMs and combine.
|
||||
|
||||
Each ``MEGA_MOE_*`` format is one row of the SM100 feature matrix. MX weights
|
||||
and activations carry one UE8M0 scale per 1x32 block. NVFP4 weights carry one
|
||||
E4M3 scale per 1x16 block plus a per-expert FP32 global scale; NVFP4 dispatch
|
||||
scales each token by its own FP32 scale.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from inspect import signature
|
||||
from typing import Any, ClassVar, Literal
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.forward_context import get_forward_context, is_forward_context_available
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe.prepare_megamoe import (
|
||||
prepare_megamoe_inputs,
|
||||
prepare_megamoe_nvfp4_inputs,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.base_router import (
|
||||
eplb_map_to_physical_and_record,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp8Dynamic,
|
||||
kNvfp4DynamicToken,
|
||||
)
|
||||
from vllm.model_executor.models.utils import extract_layer_index
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.ubatching import dbo_current_ubatch_id
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MegaMoEActivation:
|
||||
"""Gated activation fused into the MegaMoE L1 epilogue.
|
||||
|
||||
``swiglu`` computes ``gate * sigmoid(alpha * gate) * (up + beta)`` after
|
||||
clamping gate from above and up symmetrically to ``clamp``; this covers
|
||||
plain, DeepSeek-V4 (``clamp``) and GPT-OSS style (``alpha``/``beta``)
|
||||
experts. ``situ`` is Kimi K3's SiTU with gate tanh scale ``alpha`` and
|
||||
linear tanh scale ``beta``; ``beta=0`` leaves the linear branch unscaled.
|
||||
"""
|
||||
|
||||
name: Literal["swiglu", "situ"] = "swiglu"
|
||||
clamp: float | None = None
|
||||
alpha: float = 1.0
|
||||
beta: float = 0.0
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.name not in ("swiglu", "situ"):
|
||||
raise ValueError(f"Unsupported MegaMoE activation {self.name!r}.")
|
||||
if not (math.isfinite(self.alpha) and math.isfinite(self.beta)):
|
||||
raise ValueError(f"MegaMoE activation scales must be finite: {self}.")
|
||||
if self.clamp is not None and not self.clamp >= 0:
|
||||
raise ValueError(f"MegaMoE activation clamp must be >= 0: {self}.")
|
||||
if self.name == "situ" and (
|
||||
self.clamp is not None or self.alpha <= 0 or self.beta < 0
|
||||
):
|
||||
raise ValueError(
|
||||
"SiTU takes no clamp, a positive alpha and a non-negative beta; "
|
||||
f"got {self}."
|
||||
)
|
||||
|
||||
def kernel_kwargs(self) -> dict[str, Any]:
|
||||
# Defaults stay implicit so DeepGEMM builds without alpha/beta still work.
|
||||
kwargs: dict[str, Any] = {"activation_clamp": self.clamp}
|
||||
if self.name != "swiglu":
|
||||
kwargs["activation"] = self.name
|
||||
if self.alpha != 1.0:
|
||||
kwargs["activation_alpha"] = self.alpha
|
||||
if self.beta != 0.0:
|
||||
kwargs["activation_beta"] = self.beta
|
||||
return kwargs
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MegaMoEWeightSpec:
|
||||
"""Storage of one MegaMoE weight in the kernel layout."""
|
||||
|
||||
dtype: torch.dtype
|
||||
# Weight values packed per storage element along K (2 for FP4).
|
||||
values_per_element: int = 1
|
||||
# K values per block scale; None for unscaled weights.
|
||||
scale_group_k: int | None = None
|
||||
# Block scale dtype: UE8M0 (MX) or E4M3 (NVFP4).
|
||||
scale_dtype: torch.dtype | None = None
|
||||
|
||||
@property
|
||||
def has_global_scale(self) -> bool:
|
||||
"""E4M3 block scales pair with an FP32 per-tensor scale."""
|
||||
return self.scale_dtype == torch.float8_e4m3fn
|
||||
|
||||
|
||||
_MXFP4_WEIGHT = MegaMoEWeightSpec(
|
||||
torch.int8,
|
||||
values_per_element=2,
|
||||
scale_group_k=32,
|
||||
scale_dtype=torch.float8_e8m0fnu,
|
||||
)
|
||||
_MXFP8_WEIGHT = MegaMoEWeightSpec(
|
||||
torch.float8_e4m3fn, scale_group_k=32, scale_dtype=torch.float8_e8m0fnu
|
||||
)
|
||||
_NVFP4_WEIGHT = MegaMoEWeightSpec(
|
||||
torch.int8,
|
||||
values_per_element=2,
|
||||
scale_group_k=16,
|
||||
scale_dtype=torch.float8_e4m3fn,
|
||||
)
|
||||
_BF16_WEIGHT = MegaMoEWeightSpec(torch.bfloat16)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MegaMoEWeightFormat:
|
||||
"""One routed-expert precision of the MegaMoE feature matrix."""
|
||||
|
||||
name: str
|
||||
# DeepGEMM MMA tag; it also keys the symmetric-buffer layout.
|
||||
mma_type: str
|
||||
kernel: str
|
||||
routed: MegaMoEWeightSpec
|
||||
# Precisions a shared expert may have to run inside the same launch.
|
||||
shared: tuple[MegaMoEWeightSpec, ...]
|
||||
# Quantization of the dispatched hidden states; None dispatches BF16.
|
||||
hidden_quant: QuantKey | None
|
||||
activations: frozenset[str]
|
||||
|
||||
@property
|
||||
def uses_nvfp4_api(self) -> bool:
|
||||
"""``nvfp4_mega_moe`` has its own symmetric buffer, BLOCK_M and scales."""
|
||||
return self.kernel == "nvfp4_mega_moe"
|
||||
|
||||
|
||||
MEGA_MOE_MXFP4 = MegaMoEWeightFormat(
|
||||
name="mxfp4",
|
||||
mma_type="fp8xfp4",
|
||||
kernel="fp8_fp4_mega_moe",
|
||||
routed=_MXFP4_WEIGHT,
|
||||
shared=(_MXFP8_WEIGHT,),
|
||||
hidden_quant=kMxfp8Dynamic,
|
||||
activations=frozenset({"swiglu", "situ"}),
|
||||
)
|
||||
MEGA_MOE_MXFP8 = MegaMoEWeightFormat(
|
||||
name="mxfp8",
|
||||
mma_type="fp8xfp8",
|
||||
kernel="fp8_fp4_mega_moe",
|
||||
routed=_MXFP8_WEIGHT,
|
||||
shared=(_MXFP8_WEIGHT,),
|
||||
hidden_quant=kMxfp8Dynamic,
|
||||
activations=frozenset({"swiglu", "situ"}),
|
||||
)
|
||||
MEGA_MOE_NVFP4 = MegaMoEWeightFormat(
|
||||
name="nvfp4",
|
||||
mma_type="nvfp4",
|
||||
kernel="nvfp4_mega_moe",
|
||||
routed=_NVFP4_WEIGHT,
|
||||
shared=(_MXFP8_WEIGHT, _BF16_WEIGHT),
|
||||
hidden_quant=kNvfp4DynamicToken,
|
||||
activations=frozenset({"swiglu"}),
|
||||
)
|
||||
MEGA_MOE_BF16 = MegaMoEWeightFormat(
|
||||
name="bf16",
|
||||
mma_type="bf16xbf16",
|
||||
kernel="bf16_mega_moe",
|
||||
routed=_BF16_WEIGHT,
|
||||
shared=(_BF16_WEIGHT,),
|
||||
hidden_quant=None,
|
||||
activations=frozenset({"swiglu"}),
|
||||
)
|
||||
MEGA_MOE_WEIGHT_FORMATS = {
|
||||
fmt.name: fmt
|
||||
for fmt in (MEGA_MOE_MXFP4, MEGA_MOE_MXFP8, MEGA_MOE_NVFP4, MEGA_MOE_BF16)
|
||||
}
|
||||
|
||||
|
||||
def check_deep_gemm_mega_moe_supported(
|
||||
device: torch.device, hidden_size: int, intermediate_size: int
|
||||
) -> None:
|
||||
"""Raise unless DeepGEMM MegaMoE can run on ``device`` with these shapes.
|
||||
|
||||
Raises:
|
||||
NotImplementedError: If the device architecture is unsupported.
|
||||
ValueError: If the shapes are unsupported.
|
||||
|
||||
"""
|
||||
if not current_platform.is_device_capability_family(
|
||||
100, device_id=device.index or 0
|
||||
):
|
||||
raise NotImplementedError("DeepGEMM MegaMoE requires SM100 GPUs.")
|
||||
if hidden_size % 128 != 0 or intermediate_size % 128 != 0:
|
||||
raise ValueError(
|
||||
"DeepGEMM MegaMoE requires hidden and intermediate sizes "
|
||||
"to be multiples of 128."
|
||||
)
|
||||
|
||||
|
||||
def _linear_weight_scale(linear: nn.Module) -> torch.Tensor | None:
|
||||
scale = getattr(linear, "weight_scale", None)
|
||||
return getattr(linear, "weight_scale_inv", None) if scale is None else scale
|
||||
|
||||
|
||||
def _matches_weight_spec(linear: nn.Module, spec: MegaMoEWeightSpec) -> bool:
|
||||
"""Whether a checkpoint linear can be written into a ``spec`` weight."""
|
||||
weight, scale = linear.weight, _linear_weight_scale(linear)
|
||||
if weight.dtype != spec.dtype or weight.dim() != 2:
|
||||
return False
|
||||
if spec.scale_group_k is None:
|
||||
return scale is None
|
||||
if scale is None or scale.dim() != 2:
|
||||
return False
|
||||
(rows, cols), (scale_rows, scale_cols) = weight.shape, scale.shape
|
||||
scale_dtypes = {spec.scale_dtype}
|
||||
if spec.scale_dtype == torch.float8_e8m0fnu:
|
||||
scale_dtypes.add(torch.uint8)
|
||||
return (
|
||||
scale.dtype in scale_dtypes
|
||||
and rows % scale_rows == 0
|
||||
and cols % scale_cols == 0
|
||||
and (cols // scale_cols) % spec.scale_group_k == 0
|
||||
)
|
||||
|
||||
|
||||
class DeepGemmMegaMoEExperts(nn.Module):
|
||||
"""Routed experts, and optionally the shared expert, as one MegaMoE launch.
|
||||
|
||||
Parameters are registered in the kernel layout and every checkpoint shard is
|
||||
written straight into it, so loading (and reloading) needs no post-load
|
||||
transform. Whether the shared expert runs inside the launch is decided here,
|
||||
before any weight loads; a fused layer owns the shared-expert weights and
|
||||
the caller drops its shared MLP.
|
||||
|
||||
NVFP4 experts also take per-expert ``weight_scale_2`` (``w1`` and ``w3``
|
||||
must match: one L1 scale per expert) and ``w2`` ``input_scale``; the kernel
|
||||
quantizes every L2 input with the max of the latter over all loaded
|
||||
experts. Dispatch uses per-token scales, so ``w1``/``w3`` ``input_scale``
|
||||
are not loaded.
|
||||
|
||||
Args:
|
||||
num_experts: Physical experts across EP ranks, redundant included.
|
||||
activation: Activation fused into the L1 epilogue.
|
||||
weight_format: Routed-expert precision, a row of the feature matrix.
|
||||
shared_experts: Shared MLP to fuse if it is unsharded and in one of
|
||||
the ``weight_format.shared`` precisions; ``None`` never fuses.
|
||||
|
||||
"""
|
||||
|
||||
_symm_buffers: ClassVar[dict[tuple, Any]] = {}
|
||||
_synchronized_ep_groups: ClassVar[set[tuple[int, int]]] = set()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
*,
|
||||
num_experts: int,
|
||||
num_local_experts: int,
|
||||
experts_start_idx: int,
|
||||
top_k: int,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
activation: MegaMoEActivation,
|
||||
weight_format: MegaMoEWeightFormat,
|
||||
shared_experts: nn.Module | None = None,
|
||||
num_logical_experts: int | None = None,
|
||||
fast_math: bool = True,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
if activation.name not in weight_format.activations:
|
||||
raise NotImplementedError(
|
||||
f"DeepGEMM MegaMoE {weight_format.name} experts support "
|
||||
f"{sorted(weight_format.activations)}, not {activation.name!r}."
|
||||
)
|
||||
device = torch.get_default_device()
|
||||
if device.type != current_platform.device_type: # meta under ipc_cache
|
||||
device = torch.device(
|
||||
current_platform.device_type, torch.accelerator.current_device_index()
|
||||
)
|
||||
check_deep_gemm_mega_moe_supported(device, hidden_size, intermediate_size)
|
||||
|
||||
self.prefix = prefix
|
||||
self.capture_fn: Callable[[torch.Tensor], None] | None = None
|
||||
self.num_experts = num_experts
|
||||
self.num_local_experts = num_local_experts
|
||||
self.experts_start_idx = experts_start_idx
|
||||
self.experts_end_idx = experts_start_idx + num_local_experts
|
||||
self.num_logical_experts = (
|
||||
num_logical_experts if num_logical_experts is not None else num_experts
|
||||
)
|
||||
self.top_k = top_k
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.max_num_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
self.weight_format = weight_format
|
||||
self.activation = activation
|
||||
self.fast_math = fast_math
|
||||
self._kernel_kwargs = activation.kernel_kwargs()
|
||||
self.eplb_state = EplbLayerState()
|
||||
# L2 input scale of every logical expert; the NVFP4 kernel takes the max.
|
||||
self._w2_input_scales = torch.zeros(self.num_logical_experts, device="cpu")
|
||||
self.l2_activation_scale = 1.0
|
||||
# The first of each expert's w1/w3 weight_scale_2 until the other loads.
|
||||
self._pending_l1_scales: dict[int, tuple[str, float]] = {}
|
||||
|
||||
self.num_shared_experts, self.shared_spec = self._fusable_shared_experts(
|
||||
shared_experts
|
||||
)
|
||||
self._create_weights()
|
||||
for name, param in self.named_parameters(recurse=False):
|
||||
shared = name.startswith("shared_")
|
||||
loader = self.shared_weight_loader if shared else self.weight_loader
|
||||
set_weight_attrs(param, {"weight_loader": loader})
|
||||
if not shared and name.endswith("weight_scale"):
|
||||
param.quant_method = "block"
|
||||
|
||||
# Register in the static forward context so the custom-op wrapper
|
||||
# can look up this module by name from within a torch.compile graph.
|
||||
compilation_config = vllm_config.compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
raise ValueError(f"Duplicate layer name: {prefix}")
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
@property
|
||||
def has_fused_shared_experts(self) -> bool:
|
||||
return self.num_shared_experts > 0
|
||||
|
||||
def _fusable_shared_experts(
|
||||
self, shared_experts: nn.Module | None
|
||||
) -> tuple[int, MegaMoEWeightSpec | None]:
|
||||
"""How many shared experts the launch fuses, and in which precision."""
|
||||
if shared_experts is None:
|
||||
return 0, None
|
||||
gate_up, down = shared_experts.gate_up_proj, shared_experts.down_proj
|
||||
hidden, inter = self.hidden_size, self.intermediate_size
|
||||
shared_size = down.weight.shape[-1]
|
||||
spec = next(
|
||||
(
|
||||
spec
|
||||
for spec in self.weight_format.shared
|
||||
if all(_matches_weight_spec(lin, spec) for lin in (gate_up, down))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if (
|
||||
tuple(gate_up.weight.shape) != (2 * shared_size, hidden)
|
||||
or tuple(down.weight.shape) != (hidden, shared_size)
|
||||
or shared_size % inter
|
||||
):
|
||||
reason = (
|
||||
f"needs an unsharded shared MLP over hidden size {hidden} with a "
|
||||
f"multiple of {inter} intermediate; got {tuple(gate_up.weight.shape)}"
|
||||
f" and {tuple(down.weight.shape)}"
|
||||
)
|
||||
elif spec is None:
|
||||
scale = _linear_weight_scale(gate_up)
|
||||
supported = ", ".join(
|
||||
f"{s.dtype}{' with 1x32 UE8M0 scales' if s.scale_group_k else ''}"
|
||||
for s in self.weight_format.shared
|
||||
)
|
||||
reason = (
|
||||
f"{self.weight_format.name} experts fuse {supported} shared "
|
||||
f"experts; got weight {gate_up.weight.dtype} and scale "
|
||||
f"{None if scale is None else (tuple(scale.shape), scale.dtype)}"
|
||||
)
|
||||
elif not self._kernel_fuses_shared_experts(spec):
|
||||
reason = (
|
||||
"the installed DeepGEMM Python API is older than the vLLM source. "
|
||||
"Rebuild the vendored _deep_gemm_C extension to enable it"
|
||||
)
|
||||
else:
|
||||
return shared_size // inter, spec
|
||||
logger.warning_once("Disabling MegaMoE shared-expert fusion: %s.", reason)
|
||||
return 0, None
|
||||
|
||||
def _kernel_fuses_shared_experts(self, spec: MegaMoEWeightSpec) -> bool:
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
fmt = self.weight_format
|
||||
buffer_api = "get_symm_buffer_for_mega_moe"
|
||||
block_m_api = "get_block_m_for_mega_moe"
|
||||
if fmt.uses_nvfp4_api:
|
||||
buffer_api = "NVFP4SymmBuffer"
|
||||
block_m_api = "get_block_m_for_nvfp4_mega_moe"
|
||||
try:
|
||||
api = {
|
||||
*signature(getattr(deep_gemm, buffer_api)).parameters,
|
||||
*signature(getattr(deep_gemm, fmt.kernel)).parameters,
|
||||
}
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
return False
|
||||
# Scaled shared inputs are laid out by the runtime BLOCK_M.
|
||||
return {
|
||||
"num_shared_experts",
|
||||
"shared_l1_weights",
|
||||
"shared_l2_weights",
|
||||
} <= api and (spec.scale_group_k is None or hasattr(deep_gemm, block_m_api))
|
||||
|
||||
def _create_weights(self) -> None:
|
||||
"""Register the routed and fused shared-expert parameters.
|
||||
|
||||
Routed parameters are indexed by local expert on dim 0, which EPLB moves.
|
||||
"""
|
||||
e, h, i = self.num_local_experts, self.hidden_size, self.intermediate_size
|
||||
fmt = self.weight_format
|
||||
self._register_weight("w13", (e, 2 * i, h), fmt.routed)
|
||||
self._register_weight("w2", (e, h, i), fmt.routed)
|
||||
if fmt.routed.has_global_scale:
|
||||
# w13_weight_scale_2 is the kernel's L1 alpha (per-token dispatch).
|
||||
for name in ("w13_weight_scale_2", "w2_weight_scale_2", "w2_input_scale"):
|
||||
scale = torch.zeros(e, dtype=torch.float32)
|
||||
self.register_parameter(name, nn.Parameter(scale, requires_grad=False))
|
||||
# w2 global weight scale times the L2 activation scale, per launch.
|
||||
l2_alpha = torch.zeros(e, dtype=torch.float32)
|
||||
self.register_buffer("_l2_alpha", l2_alpha, persistent=False)
|
||||
if self.shared_spec is not None:
|
||||
# The shared expert has the layout of one routed expert slice.
|
||||
s = i * self.num_shared_experts
|
||||
self._register_weight("shared_l1", (2 * s, h), self.shared_spec)
|
||||
self._register_weight("shared_l2", (h, s), self.shared_spec)
|
||||
|
||||
def _register_weight(
|
||||
self, name: str, shape: tuple[int, ...], spec: MegaMoEWeightSpec
|
||||
) -> None:
|
||||
def register(name: str, data: torch.Tensor) -> None:
|
||||
self.register_parameter(name, nn.Parameter(data, requires_grad=False))
|
||||
|
||||
*outer, n, k = shape
|
||||
register(
|
||||
f"{name}_weight",
|
||||
torch.zeros(*outer, n, k // spec.values_per_element, dtype=spec.dtype),
|
||||
)
|
||||
if spec.scale_group_k is not None:
|
||||
# Block scales, MN-major, four per int32 along K.
|
||||
scale_k = k // (4 * spec.scale_group_k)
|
||||
scale = torch.zeros(*outer, scale_k, n, dtype=torch.int32)
|
||||
register(f"{name}_weight_scale", scale.transpose(-1, -2))
|
||||
|
||||
def _kernel_weights(
|
||||
self, name: str
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
weight = getattr(self, f"{name}_weight")
|
||||
scale = getattr(self, f"{name}_weight_scale", None)
|
||||
return weight if scale is None else (weight, scale)
|
||||
|
||||
@property
|
||||
def l1_weights(self) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
return self._kernel_weights("w13")
|
||||
|
||||
@property
|
||||
def l2_weights(self) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
return self._kernel_weights("w2")
|
||||
|
||||
def _map_global_expert_id(self, expert_id: int) -> list[int]:
|
||||
"""Local slots that logical expert ``expert_id`` occupies on this rank."""
|
||||
physical_ids: list[int] = []
|
||||
for p in range(self.experts_start_idx, self.experts_end_idx):
|
||||
if p % self.num_logical_experts == expert_id:
|
||||
physical_ids.append(p - self.experts_start_idx)
|
||||
return physical_ids
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
return_success: bool = False,
|
||||
) -> bool | None:
|
||||
if not loaded_weight.is_meta:
|
||||
self._record_global_scale(weight_name, loaded_weight, shard_id, expert_id)
|
||||
local_expert_ids = self._map_global_expert_id(expert_id)
|
||||
param_prefix = "w2_" if shard_id == "w2" else "w13_"
|
||||
if not local_expert_ids or param_prefix not in weight_name:
|
||||
return False if return_success else None
|
||||
is_global_scale = weight_name.endswith(("weight_scale_2", "input_scale"))
|
||||
is_scale = weight_name.endswith("weight_scale")
|
||||
for local_expert_id in local_expert_ids:
|
||||
if is_global_scale:
|
||||
param.data[local_expert_id].copy_(loaded_weight.reshape(()))
|
||||
else:
|
||||
self._load_weight(
|
||||
param.data[local_expert_id], loaded_weight, shard_id, is_scale
|
||||
)
|
||||
return True if return_success else None
|
||||
|
||||
def _record_global_scale(
|
||||
self, weight_name: str, scale: torch.Tensor, shard_id: str, expert_id: int
|
||||
) -> None:
|
||||
"""Track NVFP4 global scales the kernel takes one of per launch or expert.
|
||||
|
||||
Every rank sees the scales of all experts, so ranks agree and EPLB can
|
||||
move experts.
|
||||
"""
|
||||
if weight_name.endswith("w2_input_scale"):
|
||||
self._w2_input_scales[expert_id] = scale.max()
|
||||
self.l2_activation_scale = float(self._w2_input_scales.max()) or 1.0
|
||||
elif weight_name.endswith("w13_weight_scale_2"):
|
||||
# Compare w1 and w3 of the same load, not the live (maybe stale) slot.
|
||||
prev = self._pending_l1_scales.pop(expert_id, None)
|
||||
value = float(scale)
|
||||
if prev is None or prev[0] == shard_id:
|
||||
self._pending_l1_scales[expert_id] = (shard_id, value)
|
||||
elif not math.isclose(value, prev[1], rel_tol=1e-5):
|
||||
raise ValueError(
|
||||
f"MegaMoE NVFP4 expert {expert_id}: {prev[0]} weight_scale_2 "
|
||||
f"{prev[1]} != {shard_id} weight_scale_2 {value}; the kernel "
|
||||
"takes one L1 scale per expert."
|
||||
)
|
||||
|
||||
def load_shared_expert_weight(self, name: str, loaded_weight: torch.Tensor) -> str:
|
||||
"""Load one shared-expert checkpoint tensor; return the parameter name."""
|
||||
if ".w2." in name or ".down_proj." in name:
|
||||
param_name = "shared_l2_weight"
|
||||
elif ".w1." in name or ".w3." in name:
|
||||
param_name = "shared_l1_weight"
|
||||
else:
|
||||
raise ValueError(f"Unexpected shared-expert tensor: {name}")
|
||||
if "weight_scale" in name:
|
||||
param_name += "_scale"
|
||||
param = getattr(self, param_name)
|
||||
param.weight_loader(param, loaded_weight, name)
|
||||
return f"{name.partition('shared_experts.')[0]}experts.{param_name}"
|
||||
|
||||
def shared_weight_loader(
|
||||
self, param: nn.Parameter, loaded_weight: torch.Tensor, weight_name: str
|
||||
) -> None:
|
||||
shard_id = (
|
||||
"w1" if ".w1." in weight_name else "w3" if ".w3." in weight_name else "w2"
|
||||
)
|
||||
is_scale = "weight_scale" in weight_name
|
||||
self._load_weight(param.data, loaded_weight, shard_id, is_scale)
|
||||
|
||||
def _load_weight(
|
||||
self,
|
||||
dst: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
shard_id: str,
|
||||
is_scale: bool,
|
||||
) -> None:
|
||||
"""Write checkpoint shard ``src`` into ``dst`` in the kernel layout.
|
||||
|
||||
``dst`` is one routed expert's slice or a shared-expert parameter.
|
||||
"""
|
||||
if shard_id not in ("w1", "w2", "w3"):
|
||||
raise ValueError(f"Unsupported expert shard id: {shard_id}")
|
||||
rows = dst.shape[0] // (1 if shard_id == "w2" else 2)
|
||||
if is_scale:
|
||||
src = self._pack_scale(src, rows, dst.shape[1] * 4)
|
||||
else:
|
||||
src = src.view(dst.dtype)
|
||||
if tuple(src.shape) != (rows, dst.shape[1]):
|
||||
raise ValueError(
|
||||
f"MegaMoE {shard_id} weight shape {tuple(src.shape)} does not "
|
||||
f"match the kernel shard ({rows}, {dst.shape[1]})"
|
||||
)
|
||||
dst, src = self._kernel_layout_shard(dst, src, shard_id, is_scale)
|
||||
dst.copy_(src)
|
||||
|
||||
@staticmethod
|
||||
def _pack_scale(sf: torch.Tensor, rows: int, cols: int) -> torch.Tensor:
|
||||
"""Expand block scales to the kernel granularity, four per int32 along K."""
|
||||
sf = sf.view(torch.uint8)
|
||||
if rows % sf.shape[0] or cols % sf.shape[1]:
|
||||
raise ValueError(
|
||||
f"Scale shape {tuple(sf.shape)} does not tile ({rows}, {cols})"
|
||||
)
|
||||
sf = sf.repeat_interleave(rows // sf.shape[0], 0)
|
||||
sf = sf.repeat_interleave(cols // sf.shape[1], 1)
|
||||
return sf.contiguous().clone().view(torch.int32)
|
||||
|
||||
@staticmethod
|
||||
def _kernel_layout_shard(
|
||||
dst: torch.Tensor,
|
||||
src: torch.Tensor,
|
||||
shard_id: str,
|
||||
is_scale: bool,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Pair a checkpoint shard with its destination in the MegaMoE layout."""
|
||||
k = src.shape[-1]
|
||||
if shard_id == "w2":
|
||||
if not is_scale:
|
||||
return dst, src
|
||||
# UTCCP: scale row 128*b + 32*j + i -> 128*b + 4*i + j.
|
||||
return dst.view(-1, 32, 4, k).transpose(1, 2), src.reshape(-1, 4, 32, k)
|
||||
half = 0 if shard_id == "w1" else 1
|
||||
if not is_scale:
|
||||
# Interleave gate/up by 8 rows: r -> (r // 8) * 16 + 8 * half + r % 8.
|
||||
return dst.view(-1, 2, 8, k)[:, half], src.reshape(-1, 8, k)
|
||||
# Interleave + UTCCP: 64b + 16j + 8q + m -> 128b + 64q + 32half + 4m + j.
|
||||
dst = dst.view(-1, 2, 2, 8, 4, k)[:, :, half].permute(0, 3, 1, 2, 4)
|
||||
return dst, src.reshape(-1, 4, 2, 8, k)
|
||||
|
||||
def get_symm_buffer(self):
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
group = get_ep_group().device_group
|
||||
key = (
|
||||
id(group),
|
||||
torch.accelerator.current_device_index(),
|
||||
self.num_experts,
|
||||
self.max_num_tokens,
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
self.num_shared_experts,
|
||||
None if self.shared_spec is None else self.shared_spec.dtype,
|
||||
self.weight_format.mma_type,
|
||||
self.activation.name,
|
||||
)
|
||||
symm_buffer = self._symm_buffers.get(key)
|
||||
if symm_buffer is None:
|
||||
args = (
|
||||
group,
|
||||
self.num_experts,
|
||||
self.max_num_tokens,
|
||||
self.top_k,
|
||||
self.hidden_size,
|
||||
self.intermediate_size,
|
||||
)
|
||||
if self.weight_format.uses_nvfp4_api:
|
||||
symm_buffer = deep_gemm.NVFP4SymmBuffer(
|
||||
*args,
|
||||
num_shared_experts=self.num_shared_experts,
|
||||
shared_dtype=torch.float8_e4m3fn
|
||||
if self.shared_spec is None
|
||||
else self.shared_spec.dtype,
|
||||
)
|
||||
else:
|
||||
symm_buffer = deep_gemm.get_symm_buffer_for_mega_moe(
|
||||
*args,
|
||||
num_shared_experts=self.num_shared_experts,
|
||||
mma_type=self.weight_format.mma_type,
|
||||
activation=self.activation.name,
|
||||
)
|
||||
self._symm_buffers[key] = symm_buffer
|
||||
return symm_buffer
|
||||
|
||||
def get_block_m(self, num_tokens: int) -> int:
|
||||
"""BLOCK_M the MegaMoE scheduler runs ``num_tokens`` local tokens with."""
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
args = (
|
||||
get_ep_group().world_size,
|
||||
self.num_experts,
|
||||
self.get_symm_buffer().num_max_tokens_per_rank,
|
||||
num_tokens,
|
||||
self.top_k,
|
||||
)
|
||||
if self.weight_format.uses_nvfp4_api:
|
||||
return deep_gemm.get_block_m_for_nvfp4_mega_moe(*args)
|
||||
return deep_gemm.get_block_m_for_mega_moe(*args, self.weight_format.mma_type)
|
||||
|
||||
def set_eplb_state(
|
||||
self,
|
||||
moe_layer_idx: int,
|
||||
expert_load_view: torch.Tensor,
|
||||
logical_to_physical_map: torch.Tensor,
|
||||
logical_replica_count: torch.Tensor,
|
||||
) -> None:
|
||||
self.eplb_state.set_layer_state(
|
||||
moe_layer_idx,
|
||||
expert_load_view,
|
||||
logical_to_physical_map,
|
||||
logical_replica_count,
|
||||
)
|
||||
|
||||
def get_expert_weights(self) -> list[torch.Tensor]:
|
||||
def _to_eplb_view(name: str, t: torch.Tensor) -> torch.Tensor:
|
||||
"""Return a (num_local_experts, -1) view with contiguous memory layout."""
|
||||
assert t.shape[0] == self.num_local_experts
|
||||
if t.is_contiguous():
|
||||
return t.view(self.num_local_experts, -1)
|
||||
elif t.dim() == 3 and t.stride(1) == 1 and t.stride(2) == t.shape[1]:
|
||||
# scales have shape (E, M, N) with memory layout (E, N, M)
|
||||
back = torch.transpose(t, 1, 2)
|
||||
assert back.is_contiguous()
|
||||
return back.view(self.num_local_experts, -1)
|
||||
|
||||
raise AssertionError(
|
||||
f"MegaMoE EPLB {name}: non-contiguous expert tensor with "
|
||||
f"unexpected layout shape={tuple(t.shape)} "
|
||||
f"stride={tuple(t.stride())} dtype={t.dtype}"
|
||||
)
|
||||
|
||||
return [
|
||||
_to_eplb_view(name, param)
|
||||
for name, param in self.named_parameters(recurse=False)
|
||||
if not name.startswith("shared_")
|
||||
]
|
||||
|
||||
def update_expert_map(self) -> None:
|
||||
pass
|
||||
|
||||
@property
|
||||
def layer_id(self) -> int:
|
||||
return extract_layer_index(self.prefix)
|
||||
|
||||
def _synchronize_first_launch(self) -> None:
|
||||
ep_group = get_ep_group()
|
||||
key = (id(ep_group.cpu_group), torch.accelerator.current_device_index())
|
||||
if key in self._synchronized_ep_groups:
|
||||
return
|
||||
torch.accelerator.synchronize()
|
||||
torch.distributed.barrier(group=ep_group.cpu_group)
|
||||
self._synchronized_ep_groups.add(key)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: nn.Module | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run the routed and shared experts on this rank's tokens.
|
||||
|
||||
Args:
|
||||
hidden_states: BF16 ``[num_tokens, hidden_size]`` activations.
|
||||
topk_weights: ``[num_tokens, top_k]`` routing weights.
|
||||
topk_ids: ``[num_tokens, top_k]`` logical expert ids.
|
||||
shared_experts: Unfused shared MLP, run on ``hidden_states`` and
|
||||
added to the output. Must be ``None`` when this layer fused the
|
||||
shared expert at construction.
|
||||
|
||||
Returns:
|
||||
BF16 ``[num_tokens, hidden_size]`` routed plus shared expert output.
|
||||
|
||||
"""
|
||||
if shared_experts is not None and self.has_fused_shared_experts:
|
||||
raise ValueError(
|
||||
f"{self.prefix} fuses the shared expert into MegaMoE; "
|
||||
"pass shared_experts=None."
|
||||
)
|
||||
num_tokens = hidden_states.shape[0]
|
||||
if num_tokens > self.max_num_tokens:
|
||||
raise ValueError(
|
||||
f"MegaMoE got {num_tokens} tokens, but the symmetric buffer was "
|
||||
f"sized for {self.max_num_tokens}."
|
||||
)
|
||||
self._synchronize_first_launch()
|
||||
|
||||
is_padding = None
|
||||
if envs.VLLM_MOE_SKIP_PADDING and is_forward_context_available():
|
||||
is_padding = get_forward_context().is_padding
|
||||
if is_padding is not None:
|
||||
is_padding = is_padding[:num_tokens]
|
||||
|
||||
if self.capture_fn is not None:
|
||||
self.capture_fn(topk_ids)
|
||||
|
||||
# EPLB: map logical expert IDs to physical replicas and record load.
|
||||
eplb_state = self.eplb_state
|
||||
if eplb_state.logical_to_physical_map is not None:
|
||||
assert eplb_state.expert_load_view is not None
|
||||
assert eplb_state.logical_replica_count is not None
|
||||
assert eplb_state.should_record_tensor is not None
|
||||
if is_padding is not None:
|
||||
topk_ids = torch.where(is_padding.unsqueeze(1), -1, topk_ids)
|
||||
topk_ids = eplb_map_to_physical_and_record(
|
||||
topk_ids=topk_ids,
|
||||
expert_load_view=eplb_state.expert_load_view,
|
||||
logical_to_physical_map=eplb_state.logical_to_physical_map,
|
||||
logical_replica_count=eplb_state.logical_replica_count,
|
||||
record_enabled=eplb_state.should_record_tensor,
|
||||
num_unpadded_tokens=eplb_state.num_unpadded_tokens_tensors[
|
||||
dbo_current_ubatch_id()
|
||||
]
|
||||
if eplb_state.num_unpadded_tokens_tensors is not None
|
||||
else None,
|
||||
)
|
||||
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
deep_gemm = _import_deep_gemm()
|
||||
fmt = self.weight_format
|
||||
symm_buffer = self.get_symm_buffer()
|
||||
shared_x_sf = None
|
||||
shared_block_m = None
|
||||
if self.shared_spec is not None and self.shared_spec.scale_group_k:
|
||||
shared_x_sf = symm_buffer.shared_l1_acts_sf
|
||||
shared_block_m = self.get_block_m(num_tokens)
|
||||
kwargs = dict(self._kernel_kwargs)
|
||||
if fmt.uses_nvfp4_api:
|
||||
l2_activation_scale = self.l2_activation_scale
|
||||
prepare_megamoe_nvfp4_inputs(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
symm_buffer.x[:num_tokens],
|
||||
symm_buffer.x_sf[:num_tokens],
|
||||
symm_buffer.x_scales[:num_tokens],
|
||||
symm_buffer.topk_idx[:num_tokens],
|
||||
symm_buffer.topk_weights[:num_tokens],
|
||||
is_padding=is_padding,
|
||||
shared_x=symm_buffer.shared_l1_acts[:num_tokens]
|
||||
if self.has_fused_shared_experts
|
||||
else None,
|
||||
shared_x_sf=shared_x_sf,
|
||||
shared_block_m=shared_block_m,
|
||||
w2_global_scale=self.w2_weight_scale_2,
|
||||
l2_alpha=self._l2_alpha,
|
||||
l2_activation_scale=l2_activation_scale,
|
||||
)
|
||||
kwargs.update(
|
||||
l1_alpha=self.w13_weight_scale_2,
|
||||
l2_alpha=self._l2_alpha,
|
||||
l2_activation_scale=l2_activation_scale,
|
||||
use_x_scales=True,
|
||||
)
|
||||
else:
|
||||
quantized = fmt.hidden_quant is not None
|
||||
prepare_megamoe_inputs(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
symm_buffer.x[:num_tokens],
|
||||
symm_buffer.x_sf[:num_tokens] if quantized else None,
|
||||
symm_buffer.topk_idx[:num_tokens],
|
||||
symm_buffer.topk_weights[:num_tokens],
|
||||
is_padding=is_padding,
|
||||
shared_x_sf=shared_x_sf,
|
||||
shared_block_m=shared_block_m,
|
||||
hidden_quant=fmt.hidden_quant,
|
||||
)
|
||||
|
||||
if self.has_fused_shared_experts:
|
||||
kwargs["shared_l1_weights"] = self._kernel_weights("shared_l1")
|
||||
kwargs["shared_l2_weights"] = self._kernel_weights("shared_l2")
|
||||
y = torch.empty_like(hidden_states, dtype=torch.bfloat16)
|
||||
getattr(deep_gemm, fmt.kernel)(
|
||||
y,
|
||||
self.l1_weights,
|
||||
self.l2_weights,
|
||||
symm_buffer,
|
||||
fast_math=self.fast_math,
|
||||
**kwargs,
|
||||
)
|
||||
if shared_experts is not None:
|
||||
y += shared_experts(hidden_states)
|
||||
return y
|
||||
|
||||
|
||||
DeepGemmMegaMoEExperts.weight_loader.supports_moe_loading = True # type: ignore[attr-defined]
|
||||
@@ -1,79 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Mapping
|
||||
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import (
|
||||
VllmJitKernel,
|
||||
)
|
||||
|
||||
DEFAULT_CUTEDSL_COMPILE_OPTIONS = "--enable-tvm-ffi"
|
||||
|
||||
CompileKeyT = TypeVar("CompileKeyT")
|
||||
CuTeDSLLaunchSpec: TypeAlias = (
|
||||
tuple[CompileKeyT, tuple[Any, ...]]
|
||||
| tuple[CompileKeyT, tuple[Any, ...], Any]
|
||||
| tuple[
|
||||
CompileKeyT,
|
||||
tuple[Any, ...],
|
||||
Any,
|
||||
Callable[[], Any],
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def cutedsl_fake_stream(*, use_tvm_ffi_env_stream: bool = True) -> Any:
|
||||
import cutlass.cute as cute
|
||||
|
||||
return cute.runtime.make_fake_stream(use_tvm_ffi_env_stream=use_tvm_ffi_env_stream)
|
||||
|
||||
|
||||
def compile_cutedsl(
|
||||
entry: Callable[..., Any],
|
||||
*args: Any,
|
||||
options: str = DEFAULT_CUTEDSL_COMPILE_OPTIONS,
|
||||
use_tvm_ffi_env_stream: bool = True,
|
||||
) -> Any:
|
||||
import cutlass.cute as cute
|
||||
|
||||
return cute.compile(
|
||||
entry,
|
||||
*args,
|
||||
cutedsl_fake_stream(use_tvm_ffi_env_stream=use_tvm_ffi_env_stream),
|
||||
options=options,
|
||||
)
|
||||
|
||||
|
||||
class VllmCuTeDSLJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
"""CuTeDSL owner whose compiled executor is shared by warmup and runtime."""
|
||||
|
||||
kernel: ClassVar[Any]
|
||||
bind_launch_inputs = False
|
||||
|
||||
@abstractmethod
|
||||
def warmup_inputs(self, compile_key: CompileKeyT) -> tuple[Any, ...]:
|
||||
"""Return fake arguments that compile one executor specialization."""
|
||||
raise NotImplementedError
|
||||
|
||||
def compile(self, compile_key: CompileKeyT) -> None:
|
||||
if compile_key in self._compiled_cache:
|
||||
return
|
||||
self._compiled_cache[compile_key] = compile_cutedsl(
|
||||
self.kernel(compile_key),
|
||||
*self.warmup_inputs(compile_key),
|
||||
)
|
||||
|
||||
def launch(
|
||||
self,
|
||||
launch_spec: CuTeDSLLaunchSpec[CompileKeyT],
|
||||
_inputs: Mapping[str, Any],
|
||||
) -> Any:
|
||||
# (compile_key, args), optionally followed by output and epilogue.
|
||||
compile_key, launch_args = launch_spec[:2]
|
||||
executor = self._get_or_compile(compile_key)
|
||||
result = executor(*launch_args)
|
||||
if len(launch_spec) == 4:
|
||||
return launch_spec[3]()
|
||||
return launch_spec[2] if len(launch_spec) == 3 else result
|
||||
@@ -1,171 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Compile-only helpers for TileLang JIT warmup."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from abc import abstractmethod
|
||||
from collections.abc import Callable, Iterator, Mapping, MutableMapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field
|
||||
from functools import wraps
|
||||
from typing import Any, ClassVar, Generic, TypeAlias, TypeVar, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup import VllmJitKernel
|
||||
|
||||
CompileKeyT = TypeVar("CompileKeyT")
|
||||
# (kernel_args, launch_args), optionally followed by keywords and a result.
|
||||
TileLangLaunchSpec: TypeAlias = (
|
||||
tuple[tuple[Any, ...], tuple[Any, ...]]
|
||||
| tuple[tuple[Any, ...], tuple[Any, ...], Mapping[str, Any] | None]
|
||||
| tuple[tuple[Any, ...], tuple[Any, ...], Mapping[str, Any] | None, Any]
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _quiet_tilelang_warmup_logs() -> Iterator[None]:
|
||||
loggers = [
|
||||
logging.getLogger("tilelang.cache.kernel_cache"),
|
||||
logging.getLogger("tilelang.jit.kernel"),
|
||||
]
|
||||
previous_levels = [logger.level for logger in loggers]
|
||||
try:
|
||||
for logger in loggers:
|
||||
logger.setLevel(logging.ERROR)
|
||||
yield
|
||||
finally:
|
||||
for logger, level in zip(loggers, previous_levels):
|
||||
logger.setLevel(level)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TileLangWarmupTensor:
|
||||
"""Minimal tensor-like object accepted by TileLang compile().
|
||||
|
||||
TileLang builds its cache key and TIR from tensor dtype, shape and stride.
|
||||
This object deliberately has no storage, so compile-only warmup does not
|
||||
allocate GPU memory and does not launch the kernel.
|
||||
"""
|
||||
|
||||
dtype: torch.dtype
|
||||
shape: tuple[int, ...] = (1,)
|
||||
strides: tuple[int, ...] | None = field(default=None)
|
||||
|
||||
def stride(self) -> tuple[int, ...]:
|
||||
if self.strides is not None:
|
||||
return self.strides
|
||||
|
||||
strides: list[int] = []
|
||||
stride = 1
|
||||
for size in reversed(self.shape):
|
||||
strides.append(stride)
|
||||
stride *= size
|
||||
return tuple(reversed(strides))
|
||||
|
||||
def new_empty(
|
||||
self,
|
||||
shape: tuple[int, ...],
|
||||
*,
|
||||
dtype: torch.dtype | None = None,
|
||||
device: Any = None,
|
||||
) -> TileLangWarmupTensor:
|
||||
return TileLangWarmupTensor(dtype or self.dtype, shape)
|
||||
|
||||
|
||||
def make_tilelang_warmup_tensor(
|
||||
dtype: torch.dtype,
|
||||
*shape: int,
|
||||
strides: tuple[int, ...] | None = None,
|
||||
) -> TileLangWarmupTensor:
|
||||
return TileLangWarmupTensor(dtype=dtype, shape=tuple(shape), strides=strides)
|
||||
|
||||
|
||||
def _tilelang_call_kwargs(kwargs: dict[str, Any]) -> dict[str, Any]:
|
||||
call_kwargs = dict(kwargs)
|
||||
tune_params = call_kwargs.pop("__tune_params", {})
|
||||
if isinstance(tune_params, dict):
|
||||
call_kwargs.update(tune_params)
|
||||
return call_kwargs
|
||||
|
||||
|
||||
def compile_tilelang(jit_impl: Any, *args: Any, **kwargs: Any) -> None:
|
||||
"""Compile one TileLang specialization and populate its call cache.
|
||||
|
||||
TileLang's ``compile()`` materializes the kernel without launching it.
|
||||
We also store the compiled kernel in ``_kernel_cache`` using the same
|
||||
parsed key as ``__call__`` so runtime does not report a cache miss for an
|
||||
already materialized specialization.
|
||||
"""
|
||||
with _quiet_tilelang_warmup_logs():
|
||||
compiled = jit_impl.compile(*args, **kwargs)
|
||||
func = getattr(jit_impl, "func", None)
|
||||
parse_args = getattr(func, "parse_args", None)
|
||||
cache = getattr(jit_impl, "_kernel_cache", None)
|
||||
if not callable(parse_args) or not isinstance(cache, MutableMapping):
|
||||
return
|
||||
|
||||
key, _ = parse_args(*args, **_tilelang_call_kwargs(kwargs))
|
||||
cache[key] = compiled
|
||||
|
||||
|
||||
class VllmTileLangJitKernel(VllmJitKernel[CompileKeyT], Generic[CompileKeyT]):
|
||||
"""TileLang owner whose runtime launch specification is reused for warmup."""
|
||||
|
||||
kernel: ClassVar[Any]
|
||||
_warming_key: CompileKeyT | None = None
|
||||
|
||||
@abstractmethod
|
||||
def warmup_inputs(self, compile_key: CompileKeyT) -> dict[str, Any]:
|
||||
"""Return runtime-shaped inputs that reproduce one compile key."""
|
||||
raise NotImplementedError
|
||||
|
||||
def compile(self, compile_key: CompileKeyT) -> None:
|
||||
self._warming_key = compile_key
|
||||
try:
|
||||
cast(Callable[..., Any], self)(**self.warmup_inputs(compile_key))
|
||||
finally:
|
||||
self._warming_key = None
|
||||
|
||||
def launch(
|
||||
self,
|
||||
jit_impl: Any,
|
||||
args: tuple[Any, ...],
|
||||
kwargs: Mapping[str, Any] | None = None,
|
||||
) -> Any:
|
||||
call_kwargs = dict(kwargs or {})
|
||||
if self._warming_key is not None:
|
||||
return compile_tilelang(jit_impl, *args, **call_kwargs)
|
||||
return jit_impl(*args, **call_kwargs)
|
||||
|
||||
|
||||
def kernel_launcher(
|
||||
call_fn: Callable[..., TileLangLaunchSpec],
|
||||
) -> Callable[..., Any]:
|
||||
"""Launch TileLang from declarative kernel and runtime argument tuples."""
|
||||
|
||||
@wraps(call_fn)
|
||||
def wrapper(
|
||||
self: VllmTileLangJitKernel[Any],
|
||||
*args: Any,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
launch_spec = call_fn(self, *args, **kwargs)
|
||||
kernel_args, launch_args = launch_spec[:2]
|
||||
if self._warming_key is not None and kernel_args not in (
|
||||
(),
|
||||
(self._warming_key,),
|
||||
):
|
||||
raise RuntimeError(
|
||||
"TileLang warmup inputs produced a different compile key: "
|
||||
f"expected {self._warming_key!r}, got kernel arguments "
|
||||
f"{kernel_args!r}"
|
||||
)
|
||||
launch_kwargs = launch_spec[2] if len(launch_spec) > 2 else None
|
||||
output = launch_spec[3] if len(launch_spec) > 3 else None
|
||||
result = self.launch(self.kernel(*kernel_args), launch_args, launch_kwargs)
|
||||
return output if len(launch_spec) > 3 else result
|
||||
|
||||
return wrapper
|
||||
Reference in New Issue
Block a user