[MoE] Generalize masked activation for padded layouts (#51217)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
a232e29e9d
commit
82b7d49a6e
@@ -9,6 +9,8 @@ from typing import TYPE_CHECKING
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -134,11 +136,31 @@ _APPLY_MOE_ACTIVATIONS = frozenset(
|
||||
)
|
||||
|
||||
|
||||
_MASKED_MOE_ACTIVATION_NAMES: dict[MoEActivation, str] = {
|
||||
MoEActivation.SILU: "silu",
|
||||
MoEActivation.GELU: "gelu",
|
||||
MoEActivation.GELU_TANH: "gelu_tanh",
|
||||
MoEActivation.SITU: "situ",
|
||||
MoEActivation.SWIGLUOAI: "swigluoai",
|
||||
MoEActivation.SWIGLUOAI_UNINTERLEAVE: "swigluoai_uninterleave",
|
||||
MoEActivation.SWIGLUSTEP: "swiglustep",
|
||||
MoEActivation.SILU_NO_MUL: "silu_no_mul",
|
||||
MoEActivation.GELU_NO_MUL: "gelu_no_mul",
|
||||
MoEActivation.GELU_TANH_NO_MUL: "gelu_tanh_no_mul",
|
||||
MoEActivation.RELU2_NO_MUL: "relu2_no_mul",
|
||||
}
|
||||
|
||||
|
||||
def apply_moe_activation_supported(activation: MoEActivation) -> bool:
|
||||
"""Whether ``apply_moe_activation`` supports an activation."""
|
||||
return activation in _APPLY_MOE_ACTIVATIONS
|
||||
|
||||
|
||||
def apply_moe_activation_masked_supported(activation: MoEActivation) -> bool:
|
||||
"""Whether the masked ``apply_moe_activation`` path supports an activation."""
|
||||
return activation in _MASKED_MOE_ACTIVATION_NAMES
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ApplyMoEActivationConfig:
|
||||
"""Configuration forwarded to ``apply_moe_activation``."""
|
||||
@@ -177,6 +199,96 @@ class ApplyMoEActivationConfig:
|
||||
_DEFAULT_APPLY_MOE_ACTIVATION_CONFIG = ApplyMoEActivationConfig()
|
||||
|
||||
|
||||
def _validate_moe_activation_shapes(
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
expected_dim: int,
|
||||
) -> None:
|
||||
assert input.dim() == expected_dim, f"Input must be {expected_dim}D"
|
||||
assert output.dim() == expected_dim, f"Output must be {expected_dim}D"
|
||||
assert input.shape[:-1] == output.shape[:-1], (
|
||||
f"Input/output leading shapes must match: {input.shape} vs {output.shape}"
|
||||
)
|
||||
if activation.is_gated:
|
||||
assert output.size(-1) * 2 == input.size(-1), (
|
||||
f"{activation.value} expects 2x ratio: "
|
||||
f"{output.size(-1) * 2} vs {input.size(-1)}"
|
||||
)
|
||||
else:
|
||||
assert output.size(-1) == input.size(-1), (
|
||||
f"{activation.value} expects equal sizes: "
|
||||
f"{output.size(-1)} vs {input.size(-1)}"
|
||||
)
|
||||
|
||||
|
||||
def _apply_moe_activation_masked(
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
valid_token_counts: torch.Tensor,
|
||||
config: ApplyMoEActivationConfig,
|
||||
) -> torch.Tensor:
|
||||
masked_activation = _MASKED_MOE_ACTIVATION_NAMES.get(activation)
|
||||
if masked_activation is None:
|
||||
raise NotImplementedError(
|
||||
f"Masked MoE activation is not implemented for {activation.value}"
|
||||
)
|
||||
|
||||
assert input.dim() in (2, 3), "Masked input must be 2D or 3D"
|
||||
_validate_moe_activation_shapes(activation, output, input, expected_dim=input.dim())
|
||||
assert input.dtype == output.dtype, "Input and output dtypes must match"
|
||||
assert input.device == output.device, "Input and output devices must match"
|
||||
assert input.is_contiguous(), "Input must be contiguous"
|
||||
assert output.is_contiguous(), "Output must be contiguous"
|
||||
assert valid_token_counts.dtype == torch.int32, (
|
||||
"valid_token_counts must use torch.int32"
|
||||
)
|
||||
assert valid_token_counts.dim() == 1, "valid_token_counts must be 1D"
|
||||
assert valid_token_counts.device == input.device, (
|
||||
"valid_token_counts must be on the input device"
|
||||
)
|
||||
assert valid_token_counts.is_contiguous(), "valid_token_counts must be contiguous"
|
||||
expected_counts = input.size(0) if input.dim() == 3 else 1
|
||||
assert valid_token_counts.size(0) == expected_counts, (
|
||||
f"valid_token_counts must have {expected_counts} element(s) for "
|
||||
f"{input.dim()}D input"
|
||||
)
|
||||
|
||||
masked_clamp_limit = 0.0 if config.clamp_limit is None else config.clamp_limit
|
||||
masked_alpha = config.alpha
|
||||
if activation == MoEActivation.SILU and config.clamp_limit is not None:
|
||||
masked_activation = "silu_with_clamp"
|
||||
elif activation == MoEActivation.SITU:
|
||||
assert config.activation_situ_beta is not None, (
|
||||
"SITU requires activation_situ_beta from FusedMoEConfig"
|
||||
)
|
||||
elif activation == MoEActivation.SWIGLUOAI:
|
||||
masked_clamp_limit = 7.0
|
||||
masked_alpha = 1.702
|
||||
elif activation == MoEActivation.SWIGLUOAI_UNINTERLEAVE:
|
||||
assert config.clamp_limit is not None, (
|
||||
"SWIGLUOAI_UNINTERLEAVE requires clamp_limit"
|
||||
)
|
||||
elif activation == MoEActivation.SWIGLUSTEP:
|
||||
masked_clamp_limit = 7.0
|
||||
|
||||
torch.ops._C.masked_moe_activation(
|
||||
output,
|
||||
input,
|
||||
valid_token_counts,
|
||||
masked_activation,
|
||||
masked_clamp_limit,
|
||||
masked_alpha,
|
||||
config.beta,
|
||||
1.0 if config.activation_situ_beta is None else config.activation_situ_beta,
|
||||
-1.0
|
||||
if config.activation_situ_linear_beta is None
|
||||
else config.activation_situ_linear_beta,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
def silu_and_mul_with_clamp(
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
@@ -184,7 +296,12 @@ def silu_and_mul_with_clamp(
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
if topk_ids is not None and expert_map is not None:
|
||||
if current_platform.is_xpu():
|
||||
d = input.shape[-1] // 2
|
||||
gate = torch.clamp(input[..., :d], max=clamp_limit)
|
||||
up = torch.clamp(input[..., d:], min=-clamp_limit, max=clamp_limit)
|
||||
output.copy_(gate * torch.sigmoid(gate) * up)
|
||||
elif topk_ids is not None and expert_map is not None:
|
||||
from vllm.model_executor.layers.fused_moe.utils import swiglu_limit_func
|
||||
|
||||
swiglu_limit_func(output, input, clamp_limit, topk_ids, expert_map)
|
||||
@@ -201,12 +318,15 @@ def apply_moe_activation(
|
||||
activation_config: ApplyMoEActivationConfig | None = None,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
valid_rows: torch.Tensor | None = None,
|
||||
valid_token_counts: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply MoE activation function.
|
||||
|
||||
The configuration drives specialized activation behavior. Routing tensors
|
||||
remain per-call inputs because they depend on the current token assignment.
|
||||
and valid token counts remain per-call inputs because they depend on the
|
||||
current token assignment. A single token count masks a flat ``[T, D]``
|
||||
buffer; one count per expert masks each prefix in a padded ``[E, T, D]``
|
||||
buffer.
|
||||
"""
|
||||
config = (
|
||||
_DEFAULT_APPLY_MOE_ACTIVATION_CONFIG
|
||||
@@ -214,19 +334,17 @@ def apply_moe_activation(
|
||||
else activation_config
|
||||
)
|
||||
|
||||
assert input.dim() == 2, "Input must be 2D"
|
||||
assert output.dim() == 2, "Output must be 2D"
|
||||
if activation.is_gated:
|
||||
assert output.size(-1) * 2 == input.size(-1), (
|
||||
f"{activation.value} expects 2x ratio: "
|
||||
f"{output.size(-1) * 2} vs {input.size(-1)}"
|
||||
)
|
||||
else:
|
||||
assert output.size(-1) == input.size(-1), (
|
||||
f"{activation.value} expects equal sizes: "
|
||||
f"{output.size(-1)} vs {input.size(-1)}"
|
||||
if valid_token_counts is not None:
|
||||
return _apply_moe_activation_masked(
|
||||
activation,
|
||||
output,
|
||||
input,
|
||||
valid_token_counts,
|
||||
config,
|
||||
)
|
||||
|
||||
_validate_moe_activation_shapes(activation, output, input, expected_dim=2)
|
||||
|
||||
# Activations with gated multiplication (gate × activation(up))
|
||||
if activation == MoEActivation.SILU:
|
||||
if config.clamp_limit is not None:
|
||||
@@ -257,7 +375,6 @@ def apply_moe_activation(
|
||||
-1.0
|
||||
if config.activation_situ_linear_beta is None
|
||||
else config.activation_situ_linear_beta,
|
||||
valid_rows,
|
||||
)
|
||||
elif activation == MoEActivation.SWIGLUOAI:
|
||||
torch.ops._C.swigluoai_and_mul(output, input)
|
||||
@@ -298,7 +415,7 @@ def situ_and_mul_quant(
|
||||
beta: float,
|
||||
linear_beta: float | None,
|
||||
group_size: int = 0,
|
||||
valid_rows: torch.Tensor | None = None,
|
||||
num_valid_tokens: torch.Tensor | None = None,
|
||||
topk: int = 1,
|
||||
) -> None:
|
||||
"""Fused Kimi SITU activation + dynamic FP8 quantization.
|
||||
@@ -314,11 +431,11 @@ def situ_and_mul_quant(
|
||||
[M, d // 128], matching humming ``quant_input(group_size=128, float32)``.
|
||||
|
||||
``linear_beta`` <= 0 (or None) means "unset" (up passed through), matching
|
||||
``SituAndMul(linear_beta=None)``. ``valid_rows`` (int32 scalar tensor) is the
|
||||
DeepEP v2 contiguous-layout valid *token* count (e.g. ``psum[-1:]``); the
|
||||
kernel multiplies it by ``topk`` to bound rows, so no host-side cast/multiply
|
||||
is needed. Padding rows are skipped and receive a benign scale of 1.0. Kept
|
||||
in this module so that every ``torch.ops._C.situ*`` call lives in one file.
|
||||
``SituAndMul(linear_beta=None)``. ``num_valid_tokens`` (int32 scalar tensor)
|
||||
is the DeepEP v2 contiguous-layout token count (e.g. ``psum[-1:]``); the
|
||||
kernel multiplies it by ``topk`` to bound rows. Padding rows are skipped and
|
||||
receive a benign scale of 1.0. Kept in this module so that every
|
||||
``torch.ops._C.situ*`` call lives in one file.
|
||||
"""
|
||||
torch.ops._C.situ_and_mul_quant(
|
||||
output,
|
||||
@@ -327,6 +444,6 @@ def situ_and_mul_quant(
|
||||
beta,
|
||||
-1.0 if linear_beta is None else linear_beta,
|
||||
group_size,
|
||||
valid_rows,
|
||||
num_valid_tokens,
|
||||
topk,
|
||||
)
|
||||
|
||||
@@ -5,7 +5,10 @@
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.activation import (
|
||||
MoEActivation,
|
||||
apply_moe_activation_masked_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
@@ -19,7 +22,6 @@ from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
moe_kernel_quantize_input,
|
||||
normalize_batched_scales_shape,
|
||||
swiglu_limit_func,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
@@ -839,16 +841,18 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
MoEActivation.GELU_TANH_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
]
|
||||
if current_platform.is_xpu():
|
||||
return activation in [
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU_TANH,
|
||||
MoEActivation.SWIGLUOAI,
|
||||
MoEActivation.SILU_NO_MUL,
|
||||
MoEActivation.GELU_NO_MUL,
|
||||
MoEActivation.GELU_TANH_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
]
|
||||
return apply_moe_activation_masked_supported(activation)
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
@@ -858,20 +862,6 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
# Let PrepareAndFinalize::finalize() decide the impl.
|
||||
return TopKWeightAndReduceDelegate()
|
||||
|
||||
def activation(
|
||||
self,
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
|
||||
if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
|
||||
swiglu_limit_func(output, input, float(gemm1_clamp_limit))
|
||||
return
|
||||
|
||||
super().activation(activation, output, input)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
@@ -998,11 +988,19 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
|
||||
intermediate_cache2.fill_(0)
|
||||
|
||||
# TODO (bnell): use triton utility from batched deep gemm.
|
||||
self.activation(
|
||||
activation,
|
||||
intermediate_cache2.view(-1, activation_out_dim),
|
||||
intermediate_cache1.view(-1, N),
|
||||
)
|
||||
if current_platform.is_xpu():
|
||||
self.activation(
|
||||
activation,
|
||||
intermediate_cache2.view(-1, activation_out_dim),
|
||||
intermediate_cache1.view(-1, N),
|
||||
)
|
||||
else:
|
||||
self.activation(
|
||||
activation,
|
||||
intermediate_cache2,
|
||||
intermediate_cache1,
|
||||
valid_token_counts=expert_num_tokens,
|
||||
)
|
||||
|
||||
qintermediate_cache2, a2q_scale = batched_moe_kernel_quantize_input(
|
||||
intermediate_cache2,
|
||||
|
||||
@@ -35,10 +35,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceDelegate,
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
swiglu_limit_func,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.utils import _resize_cache
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
INT4_DTYPE,
|
||||
INT8_DTYPE,
|
||||
@@ -103,17 +100,18 @@ def _is_supported_wna16_weight_key(weight_key: QuantKey | None) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def get_humming_moe_gemm_type() -> str:
|
||||
def get_humming_moe_gemm_type(moe_config: FusedMoEConfig | None = None) -> str:
|
||||
env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE
|
||||
gemm_type = "indexed"
|
||||
if env_gemm_type is not None:
|
||||
if env_gemm_type is not None and env_gemm_type.lower() != "auto":
|
||||
env_gemm_type = env_gemm_type.lower()
|
||||
if env_gemm_type == "indexed":
|
||||
gemm_type = env_gemm_type
|
||||
elif env_gemm_type in ["grouped_contiguous", "grouped"]:
|
||||
if env_gemm_type == "grouped":
|
||||
gemm_type = "grouped_contiguous"
|
||||
else:
|
||||
gemm_type = "indexed"
|
||||
gemm_type = env_gemm_type
|
||||
elif moe_config is not None and moe_config.moe_parallel_config.use_ep:
|
||||
gemm_type = "grouped_contiguous"
|
||||
else:
|
||||
gemm_type = "indexed"
|
||||
|
||||
logger.info_once(f"Using {gemm_type} gemm for humming moe") # noqa
|
||||
return gemm_type
|
||||
@@ -178,7 +176,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
max_num_tokens=max_num_tokens,
|
||||
num_dispatchers=num_dispatchers,
|
||||
)
|
||||
self._permute_scratch: MoEPermuteScratch | None = None
|
||||
self._permute_scratch: dict[int, MoEPermuteScratch] = {}
|
||||
|
||||
def init_humming_moe(self):
|
||||
from vllm.utils.humming import get_heuristics_config
|
||||
@@ -257,26 +255,36 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def _get_permute_scratch(self) -> MoEPermuteScratch | None:
|
||||
if self._permute_scratch is None and moe_permute_unpermute_supported():
|
||||
self._permute_scratch = MoEPermuteScratch(
|
||||
max_num_tokens=self.moe_config.max_num_tokens,
|
||||
topk=self.moe_config.experts_per_token,
|
||||
def _get_permute_scratch(self, topk: int) -> MoEPermuteScratch | None:
|
||||
if not moe_permute_unpermute_supported():
|
||||
return None
|
||||
|
||||
scratch = self._permute_scratch.get(topk)
|
||||
if scratch is None:
|
||||
max_expanded_rows = (
|
||||
self.moe_config.max_num_tokens
|
||||
* self.moe_config.dp_size
|
||||
* self.moe_config.experts_per_token
|
||||
)
|
||||
scratch = MoEPermuteScratch(
|
||||
max_num_tokens=math.ceil(max_expanded_rows / topk),
|
||||
topk=topk,
|
||||
num_experts=self.moe_config.num_experts,
|
||||
num_local_experts=self.moe_config.num_local_experts,
|
||||
device=torch.device(self.moe_config.device),
|
||||
hidden_size=self.moe_config.hidden_dim,
|
||||
hidden_dtype=self.moe_config.in_dtype,
|
||||
)
|
||||
return self._permute_scratch
|
||||
self._permute_scratch[topk] = scratch
|
||||
return scratch
|
||||
|
||||
def get_global_valid_shape_m(self, topk_ids: torch.Tensor):
|
||||
num_tokens = topk_ids.size(0)
|
||||
ctx = get_forward_context()
|
||||
if ctx.dp_metadata is not None:
|
||||
num_tokens = ctx.dp_metadata.num_tokens_across_dp_cpu.sum().item()
|
||||
return num_tokens * self.moe_config.experts_per_token
|
||||
|
||||
return num_tokens * topk_ids.size(1)
|
||||
return topk_ids.size(0) * topk_ids.size(1)
|
||||
|
||||
def estimate_local_valid_shape_m(self, topk_ids: torch.Tensor):
|
||||
# estimate shape_m for kernel tuning
|
||||
@@ -645,7 +653,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
if supported:
|
||||
assert hasattr(cls, "humming_gemm_type")
|
||||
gemm_type = cls.humming_gemm_type().value.lower()
|
||||
preferred_gemm_type = get_humming_moe_gemm_type()
|
||||
preferred_gemm_type = get_humming_moe_gemm_type(moe_config)
|
||||
supported = preferred_gemm_type.lower() == gemm_type
|
||||
if not supported:
|
||||
reason = (
|
||||
@@ -660,25 +668,16 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
activation: MoEActivation,
|
||||
output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
valid_rows: torch.Tensor | None = None,
|
||||
valid_token_counts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
activation_config = self.activation_config
|
||||
if (
|
||||
activation == MoEActivation.SILU
|
||||
and activation_config.clamp_limit is not None
|
||||
):
|
||||
swiglu_limit_func(
|
||||
output=output,
|
||||
input=input,
|
||||
swiglu_limit=activation_config.clamp_limit,
|
||||
)
|
||||
else:
|
||||
self.activation(
|
||||
activation=activation,
|
||||
input=input,
|
||||
output=output,
|
||||
valid_rows=valid_rows,
|
||||
)
|
||||
activation_kwargs: dict[str, Any] = dict(
|
||||
activation=activation,
|
||||
input=input,
|
||||
output=output,
|
||||
)
|
||||
if valid_token_counts is not None:
|
||||
activation_kwargs["valid_token_counts"] = valid_token_counts
|
||||
self.activation(**activation_kwargs)
|
||||
|
||||
def fused_situ_quant_enabled(self, activation: MoEActivation) -> bool:
|
||||
"""Whether the SITU activation + w2 quant can be fused into one kernel.
|
||||
@@ -718,7 +717,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
self,
|
||||
gate_up_output: torch.Tensor,
|
||||
quanted_down_input: torch.Tensor,
|
||||
valid_rows: torch.Tensor | None,
|
||||
num_valid_tokens: torch.Tensor | None,
|
||||
topk: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Run the fused SITU activation + FP8 quant for the w2 input.
|
||||
@@ -750,7 +749,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
beta=cfg.activation_situ_beta,
|
||||
linear_beta=cfg.activation_situ_linear_beta,
|
||||
group_size=group_size,
|
||||
valid_rows=valid_rows,
|
||||
num_valid_tokens=num_valid_tokens,
|
||||
topk=topk,
|
||||
)
|
||||
return quanted_down_input, input_scale
|
||||
@@ -878,8 +877,8 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
)
|
||||
|
||||
# psum[-1:] is the DeepEP valid *token* count as a zero-cost int32 view.
|
||||
# situ (rows = tokens*topk) multiplies by topk on-device; mul_sum bounds
|
||||
# on tokens, so both read this pointer directly -- no host cast/multiply.
|
||||
# The fused path and mul_sum consume tokens directly. The unfused
|
||||
# activation consumes the expanded row count (tokens * topk).
|
||||
valid_tokens = None
|
||||
topk = topk_ids.size(1)
|
||||
if (
|
||||
@@ -894,21 +893,18 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
inputs, input_scale = self.fused_situ_quant(
|
||||
gate_up_output=buffers["gate_up_output"],
|
||||
quanted_down_input=buffers["quanted_down_input"],
|
||||
valid_rows=valid_tokens,
|
||||
num_valid_tokens=valid_tokens,
|
||||
topk=topk,
|
||||
)
|
||||
else:
|
||||
# Fallback situ_and_mul takes int64 row counts (tokens*topk).
|
||||
valid_rows = (
|
||||
valid_tokens.to(torch.int64) * topk
|
||||
if valid_tokens is not None
|
||||
else None
|
||||
valid_token_counts = (
|
||||
valid_tokens * topk if valid_tokens is not None else None
|
||||
)
|
||||
self.apply_activation(
|
||||
activation=activation,
|
||||
input=buffers["gate_up_output"],
|
||||
output=buffers["activation_output"],
|
||||
valid_rows=valid_rows,
|
||||
valid_token_counts=valid_token_counts,
|
||||
)
|
||||
|
||||
inputs, input_scale = self.quantize_input(
|
||||
@@ -1002,7 +998,7 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
n_expert=global_num_experts,
|
||||
n_local_expert=self.num_experts,
|
||||
expert_map=expert_map,
|
||||
scratch=self._get_permute_scratch(),
|
||||
scratch=self._get_permute_scratch(topk_ids.size(1)),
|
||||
)
|
||||
|
||||
inputs, input_scale = self.quantize_input(
|
||||
@@ -1028,6 +1024,11 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
activation=activation,
|
||||
input=buffers["gate_up_output"],
|
||||
output=buffers["activation_output"],
|
||||
valid_token_counts=(
|
||||
expert_first_token_offset[-1:].to(torch.int32)
|
||||
if expert_tokens_meta is None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
inputs, input_scale = self.quantize_input(
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe.activation import (
|
||||
ApplyMoEActivationConfig,
|
||||
MoEActivation,
|
||||
apply_moe_activation,
|
||||
apply_moe_activation_masked_supported,
|
||||
apply_moe_activation_supported,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
@@ -955,6 +956,10 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.BatchedExperts
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return apply_moe_activation_masked_supported(activation)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
@@ -1000,26 +1005,18 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
||||
act: MoEActivation,
|
||||
act_output: torch.Tensor,
|
||||
act_input: torch.Tensor,
|
||||
**kwargs,
|
||||
*,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
if act != MoEActivation.SITU:
|
||||
self.activation(
|
||||
act,
|
||||
act_output,
|
||||
act_input,
|
||||
**kwargs,
|
||||
)
|
||||
return
|
||||
|
||||
num_experts, max_num_tokens = hidden_states.shape[:2]
|
||||
beta = self.activation_config.activation_situ_beta
|
||||
linear_beta = self.activation_config.activation_situ_linear_beta
|
||||
torch.ops._C.masked_situ_and_mul(
|
||||
self.activation(
|
||||
act,
|
||||
act_output.view(num_experts, max_num_tokens, -1),
|
||||
act_input.view(num_experts, max_num_tokens, -1),
|
||||
expert_tokens_meta.expert_num_tokens,
|
||||
1.0 if beta is None else beta,
|
||||
-1.0 if linear_beta is None else linear_beta,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
valid_token_counts=expert_tokens_meta.expert_num_tokens,
|
||||
)
|
||||
|
||||
return batched_fused_marlin_moe(
|
||||
|
||||
@@ -902,7 +902,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
*,
|
||||
topk_ids: torch.Tensor | None = None,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
valid_rows: torch.Tensor | None = None,
|
||||
valid_token_counts: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
apply_moe_activation(
|
||||
activation,
|
||||
@@ -911,7 +911,7 @@ class FusedMoEExpertsModular(FusedMoEExperts):
|
||||
activation_config=self.activation_config,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=expert_map,
|
||||
valid_rows=valid_rows,
|
||||
valid_token_counts=valid_token_counts,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
|
||||
Reference in New Issue
Block a user