Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-16 17:35:42 +07:00
347 changed files with 29531 additions and 3472 deletions
@@ -427,7 +427,7 @@ def sparse_mqa_logits_paged_decode(
q: [rows, 1, H, D] packed Q values (MXFP4 viewed as int8).
q_scale: [rows, 1, H] int32 packed UE8M0 Q scales.
kv_cache: 4D [num_pages, page_kv, 1, head_bytes] uint8 fused cache
view; the page stride must be 512B-aligned.
view; the page stride must be 256B-aligned.
weights: [rows, H] bf16 per-head weights.
context_lens: [rows] int32 per-row (compressed) context lengths.
block_table: [rows, P] int32 per-row page table, ``stride(-1) == 1``.
@@ -451,8 +451,8 @@ def sparse_mqa_logits_paged_decode(
rows = q.shape[0]
page_kv = kv_cache.shape[1]
assert weights.dtype == torch.bfloat16, weights.dtype
assert kv_cache.stride(0) % 512 == 0, (
"paged cache page stride must be 512B-aligned: "
assert kv_cache.stride(0) % 256 == 0, (
"paged cache page stride must be 256B-aligned: "
f"shape={tuple(kv_cache.shape)} stride={kv_cache.stride()}"
)
assert page_kv % sparse_block_kv == 0, (page_kv, sparse_block_kv)
@@ -88,6 +88,9 @@ from vllm.model_executor.kernels.linear.mxfp4.humming import (
from vllm.model_executor.kernels.linear.mxfp4.marlin import (
MarlinMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp4.pytorch import (
TorchMxFp4LinearKernel,
)
from vllm.model_executor.kernels.linear.mxfp4.xpu import (
XPUMxFp4LinearKernel,
)
@@ -341,6 +344,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
ChannelWiseTorchFP8ScaledMMLinearKernel,
RowWiseTorchFP8ScaledMMLinearKernel,
BlockWiseTorchFP8ScaledMMLinearKernel,
TorchMxFp4LinearKernel,
TorchMxFp8LinearKernel,
TorchNvFp4LinearKernel,
},
@@ -624,6 +628,7 @@ _POSSIBLE_MXFP4_KERNELS: dict[PlatformEnum, list[type[MxFp4LinearKernel]]] = {
],
PlatformEnum.XPU: [
XPUMxFp4LinearKernel,
TorchMxFp4LinearKernel,
],
}
@@ -0,0 +1,71 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
xpu_mxfp4_quantize,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kMxfp4Dynamic,
)
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
_E8M0 = torch.float8_e8m0fnu
_FP4X2 = torch.float4_e2m1fn_x2
class TorchMxFp4LinearKernel(MxFp4LinearKernel):
"""MXFP4 W4A4 GEMM on XPU using the native ``torch._scaled_mm`` dispatch."""
@classmethod
def is_supported(
cls, compute_capability: int | None = None
) -> tuple[bool, str | None]:
if not current_platform.is_xpu():
return False, "TorchMxFp4 only supports XPU"
return True, None
@classmethod
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
if config.activation_quant_key != kMxfp4Dynamic:
return False, "only supports MXFP4 dynamic activation"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
# weight: [N, K_packed] packed-fp4; weight_scale: [N, K//32] E8M0.
# oneDNN wants un-swizzled scales and a column-major weight (B operand).
weight = layer.weight.view(_FP4X2)
replace_parameter(layer, "weight", weight.data.t())
weight_scale = layer.weight_scale.view(_E8M0)
weight_scale = weight_scale.t().contiguous()
replace_parameter(layer, "weight_scale", weight_scale.data)
def apply_weights(
self,
layer: torch.nn.Module,
x: torch.Tensor,
bias: torch.Tensor | None = None,
) -> torch.Tensor:
out_dtype = x.dtype
weight = layer.weight # column-major [K_packed, N]
N = weight.shape[1]
input_2d = x.reshape(-1, x.shape[-1])
x_fp4, x_scale = xpu_mxfp4_quantize(input_2d)
x_scale = x_scale.view(_E8M0)
out = torch._scaled_mm(
x_fp4,
weight,
scale_a=x_scale,
scale_b=layer.weight_scale,
bias=bias,
out_dtype=out_dtype,
)
return out.reshape(*x.shape[:-1], N)
@@ -9,6 +9,7 @@ from vllm._aiter_ops import (
rocm_aiter_ops,
)
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
_upcast_e8m0_to_fp32,
)
@@ -399,6 +400,26 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
rocm_aiter_ops.is_triton_gemm_w8a8_tuned(n, k) or _on_gfx1250
)
self.use_bpreshuffle = (
_on_gfx1250
and n % 16 == 0
and k % 128 == 0
and bool(rocm_aiter_ops.is_triton_blockscale_bpreshuffle_gemm_tuned(n, k))
)
if self.use_bpreshuffle:
# The tuned preshuffle configs expect a column-major group scale.
act_scale_descriptor = config.activation_quant_key.scale
self.quant_fp8 = QuantFP8(
static=act_scale_descriptor.static,
group_shape=act_scale_descriptor.group_shape,
num_token_padding=self.get_output_padding(),
use_ue8m0=False,
column_major_scales=True,
)
logger.info_once(
"Using aiter A8W8 blockscale B-preshuffle GEMM for N=%d K=%d", n, k
)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
super().process_weights_after_loading(layer)
@@ -410,6 +431,18 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
if ws is not None and ws.dtype == torch.float8_e8m0fnu:
replace_parameter(layer, attr, _upcast_e8m0_to_fp32(ws).contiguous())
if getattr(layer, "is_bmm", False):
self.use_bpreshuffle = False
if self.use_bpreshuffle:
w = FP8BlockParams.from_layer(layer).weight
replace_parameter(
layer,
FP8BlockParams.WEIGHT,
rocm_aiter_ops.shuffle_weight(w.data, layout=(16, 16)),
)
layer.aiter_bpreshuffled = True
@classmethod
def is_supported(cls, compute_capability=None):
if (
@@ -464,7 +497,11 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
Bs = Bs.to(torch.float32)
out_dtype = self.config.out_dtype
if self.use_triton:
if self.use_bpreshuffle:
gemm_a8w8_blockscale_op = (
rocm_aiter_ops.triton_gemm_a8w8_blockscale_preshuffle
)
elif self.use_triton:
gemm_a8w8_blockscale_op = rocm_aiter_ops.triton_gemm_a8w8_blockscale
else:
gemm_a8w8_blockscale_op = rocm_aiter_ops.gemm_a8w8_blockscale
@@ -249,6 +249,8 @@ class FusedMoEQuantConfig:
_w1: FusedMoEQuantDesc
_w2: FusedMoEQuantDesc
is_scale_swizzled: bool = True
# Emit block-quant activation scales column-major (physical [G, M] layout).
is_scale_transposed: bool = False
# MXFP4-specific TRTLLM parameters for SwiGLU activation clamping.
# These correspond to gemm1_alpha, gemm1_beta, gemm1_clamp_limit
@@ -503,6 +505,7 @@ class FusedMoEQuantConfig:
w2_zp: torch.Tensor | None = None,
weight_dtype: torch.dtype | str | None = None,
is_scale_swizzled: bool = True,
is_scale_transposed: bool = False,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
@@ -537,6 +540,9 @@ class FusedMoEQuantConfig:
- is_scale_swizzled: Whether the activation scale-factor layout is
swizzled. Pass through to the underlying quantization kernel for
dtypes that distinguish layouts (nvfp4, mxfp8). Defaults to True.
- is_scale_transposed: Whether block-quantized activation scales are
emitted column-major (physical [num_groups, num_tokens] layout) by
the quantize step. Defaults to False.
- gemm1_alpha: Optional MXFP4 TRTLLM SwiGLU alpha parameter.
- gemm1_beta: Optional MXFP4 TRTLLM SwiGLU beta parameter.
- gemm1_clamp_limit: Optional MXFP4 TRTLLM SwiGLU clamp limit.
@@ -573,6 +579,7 @@ class FusedMoEQuantConfig:
weight_dtype, w_shape, w2_scale, g2_alphas, w2_zp, w2_bias
),
is_scale_swizzled=is_scale_swizzled,
is_scale_transposed=is_scale_transposed,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
@@ -597,6 +604,7 @@ def fp8_w8a8_moe_quant_config(
a2_gscale: torch.Tensor | None = None,
g1_alphas: torch.Tensor | None = None,
g2_alphas: torch.Tensor | None = None,
is_scale_transposed: bool = False,
gemm1_alpha: float | None = None,
gemm1_beta: float | None = None,
gemm1_clamp_limit: float | None = None,
@@ -617,6 +625,7 @@ def fp8_w8a8_moe_quant_config(
per_act_token_quant=per_act_token_quant,
per_out_ch_quant=per_out_ch_quant,
block_shape=block_shape,
is_scale_transposed=is_scale_transposed,
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=gemm1_clamp_limit,
File diff suppressed because it is too large Load Diff
@@ -16,6 +16,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
)
from vllm.triton_utils import triton
__all__ = [
"AiterW4A16ExpertsMonolithic",
@@ -52,10 +53,10 @@ def _aiter_w4a16_silu_via_a8w4(
from aiter.ops.triton.moe_op_gemm_a8w4 import moe_gemm_a8w4
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
should_use_cdna4_mx_scale_swizzle,
get_aiter_mx_scale_swizzle,
)
swz = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None
swz = get_aiter_mx_scale_swizzle()
quant_dtype = torch.float8_e4m3fn
g1_gammas = gammas if apply_router_weight_on_input else None
@@ -134,6 +135,8 @@ def aiter_triton_kernel_w4a16_moe_forward(
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
score_mode: str | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
):
assert quant_config is not None and rocm_aiter_ops.is_enabled()
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
@@ -154,17 +157,37 @@ def aiter_triton_kernel_w4a16_moe_forward(
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
if score_mode is not None:
scale = routed_scaling_factor if routed_scaling_factor is not None else 1.0
if hash_indices_table is not None:
assert score_mode == "sqrtsoftplus" and input_ids is not None
# Same block_m heuristic as aiter's routing().
n_tokens, n_experts = gating_output.shape
block_m = max(
16, min(triton.next_power_of_2(max(1, n_tokens * topk // n_experts)), 128)
)
routing_data, gather_idx, scatter_idx = _routing_mod.routing_from_hash(
gating_output,
hash_indices_table,
input_ids,
topk,
block_m,
score_mode=score_mode,
renorm=renormalize,
routed_scaling_factor=scale,
)
elif score_mode is not None:
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output,
topk,
score_mode=score_mode,
bias=e_score_correction_bias,
renorm=renormalize,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
bias=(
e_score_correction_bias.float()
if e_score_correction_bias is not None
else None
),
renorm=renormalize,
routed_scaling_factor=scale,
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
@@ -348,6 +371,10 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
RoutingMethodType.DeepSeekV3,
]
@staticmethod
def _supports_hash_routing() -> bool:
return True
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
@@ -375,6 +402,8 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor:
routing_method = self.moe_config.routing_method
if routing_method == RoutingMethodType.DeepseekV4:
@@ -411,4 +440,6 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
score_mode=score_mode,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
@@ -0,0 +1,373 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
RoutingMethodType,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Dynamic,
kMxfp4Static,
)
__all__ = [
"AiterW4A4ExpertsMonolithic",
"aiter_triton_kernel_w4a4_moe_forward",
]
def _aiter_raw(t):
if t is None or isinstance(t, torch.Tensor):
return t
return t.storage.data if hasattr(t, "storage") else t
def aiter_triton_kernel_w4a4_moe_forward(
hidden_states: torch.Tensor,
w1,
w2,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
activation: MoEActivation = MoEActivation.SWIGLUOAI,
quant_config: FusedMoEQuantConfig | None = None,
apply_router_weight_on_input: bool = False,
global_num_experts: int = -1,
expert_map: torch.Tensor | None = None,
unpadded_N_w1=None,
unpadded_K_w1=None,
unpadded_N_w2=None,
unpadded_K_w2=None,
num_expert_group: int | None = None,
topk_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
score_mode: str | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
):
assert quant_config is not None and rocm_aiter_ops.is_enabled()
from aiter.ops.triton.moe.moe_op_gemm_a4w4 import moe_gemm_a4w4, mxfp4_quant
try:
from aiter.ops.triton.moe.moe_routing import routing as _routing_mod
except ImportError:
from aiter.ops.triton.moe_routing import routing as _routing_mod
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
get_aiter_mx_scale_swizzle,
)
from vllm.platforms.rocm import on_gfx1250
if on_gfx1250():
_routing_mod.is_tdm_avail = lambda: False
aiter_routing = _routing_mod.routing
gating_output = torch.nan_to_num(gating_output, nan=0.0, posinf=0.0, neginf=0.0)
if hash_indices_table is not None:
assert input_ids is not None, "hash routing requires input_ids"
n_tokens, n_expts_tot = gating_output.shape
tokens_per_expt = max(1, n_tokens * topk // n_expts_tot)
block_m = max(16, min(1 << (tokens_per_expt - 1).bit_length(), 128))
routing_data, gather_idx, scatter_idx = _routing_mod.routing_from_hash(
gating_output,
hash_indices_table,
input_ids.to(hash_indices_table.dtype),
topk,
block_m,
score_mode=score_mode or "sqrtsoftplus",
renorm=renormalize,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
)
elif score_mode is not None:
use_grouped_topk = num_expert_group is not None and num_expert_group > 1
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output,
topk,
score_mode=score_mode,
bias=(
e_score_correction_bias.float()
if e_score_correction_bias is not None
else None
),
renorm=renormalize,
routed_scaling_factor=(
routed_scaling_factor if routed_scaling_factor is not None else 1.0
),
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
)
else:
routing_data, gather_idx, scatter_idx = aiter_routing(
gating_output, topk, sm_first=not renormalize
)
if on_gfx1250():
gather_src = gather_idx.to(torch.long) // topk
hidden_states = hidden_states[gather_src]
gather_idx = None
w1_data = _aiter_raw(w1)
w2_data = _aiter_raw(w2)
w1_scale = _aiter_raw(quant_config.w1_scale)
w2_scale = _aiter_raw(quant_config.w2_scale)
gammas = routing_data.gate_scal if routing_data else None
swiglu_limit = (
quant_config.gemm1_clamp_limit
if quant_config.gemm1_clamp_limit is not None
else 7.0
)
# SWIGLUOAI needs the alpha/residual swiglu fused into the GEMM1 epilogue,
# which reads gate/up interleaved along N (see the oracle's weight prep).
fused_swiglu = activation in (
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
)
swiglu_kwargs: dict[str, object] = {}
if fused_swiglu:
assert quant_config.gemm1_beta in (None, 1.0), (
"aiter's fused swiglu hardcodes the residual to (up + 1); "
f"gemm1_beta={quant_config.gemm1_beta} cannot be expressed"
)
swiglu_kwargs = {
"alpha": (
quant_config.gemm1_alpha
if quant_config.gemm1_alpha is not None
else 1.0
),
"limit": swiglu_limit,
"swiglu_add_residual": quant_config.gemm1_beta == 1.0,
}
swizzle_mx_scale = get_aiter_mx_scale_swizzle()
x_q, x_scale = mxfp4_quant(hidden_states.to(torch.bfloat16))
# GEMM1: gate+up projection.
raw_intermediate = moe_gemm_a4w4(
x_q,
w1_data,
x_scale,
w1_scale,
bias=quant_config.w1_bias,
routing_data=routing_data,
gather_indx=gather_idx,
gammas=gammas if apply_router_weight_on_input else None,
swizzle_mx_scale=swizzle_mx_scale,
apply_swiglu=fused_swiglu,
**swiglu_kwargs,
)
if fused_swiglu:
# The swiglu epilogue already halved N.
intermediate = (
raw_intermediate
if unpadded_N_w1 is None
else raw_intermediate[:, : unpadded_N_w1 // 2]
)
else:
if unpadded_N_w1 is not None:
raw_intermediate = raw_intermediate[:, :unpadded_N_w1]
# SiLU(gate) * up on the concatenated [gate | up] halves
from aiter.ops.triton.fusions.fused_clamp_act_mul import fused_clamp_act_mul
half_n = raw_intermediate.shape[-1] // 2
intermediate = torch.empty(
raw_intermediate.shape[0],
half_n,
dtype=raw_intermediate.dtype,
device=raw_intermediate.device,
)
fused_clamp_act_mul(
raw_intermediate,
out=intermediate,
swiglu_limit=swiglu_limit,
activation="silu",
dtype_quant=None,
)
# GEMM2: down projection with scatter-reduce
mid_q, mid_scale = mxfp4_quant(intermediate.to(torch.bfloat16))
out = moe_gemm_a4w4(
mid_q,
w2_data,
mid_scale,
w2_scale,
bias=quant_config.w2_bias,
routing_data=routing_data,
scatter_indx=scatter_idx,
gammas=None if apply_router_weight_on_input else gammas,
swizzle_mx_scale=swizzle_mx_scale,
apply_swiglu=False,
)
return out
class AiterW4A4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
"""Monolithic MXFP4 W4A4 expert using AITER Triton/Gluon kernels (gfx1250).
Uses moe_gemm_a4w4 (auto-selects gluon backend on gfx1250) with dynamic
MXFP4 activation quantization via mxfp4_quant.
"""
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
self.topk = moe_config.experts_per_token
self.renormalize = moe_config.routing_method in (
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
RoutingMethodType.DeepSeekV3,
RoutingMethodType.MiniMax2,
)
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_hash_routing() -> bool:
return True
@staticmethod
def _supports_current_device() -> bool:
if not rocm_aiter_ops.is_enabled():
return False
from vllm.platforms.rocm import on_gfx1250
return on_gfx1250()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return (weight_key, activation_key) == (kMxfp4Static, kMxfp4Dynamic)
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in (
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
MoEActivation.SILU,
)
@staticmethod
def _supports_parallel_config(
moe_parallel_config: FusedMoEParallelConfig,
) -> bool:
return (
not moe_parallel_config.use_all2all_kernels
and not moe_parallel_config.enable_eplb
and moe_parallel_config.dp_size <= 1
)
@staticmethod
def _supports_routing_method(
routing_method: RoutingMethodType,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return routing_method in [
RoutingMethodType.Renormalize,
RoutingMethodType.RenormalizeNaive,
RoutingMethodType.DeepseekV4,
RoutingMethodType.DeepSeekV3,
RoutingMethodType.MiniMax2,
]
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
routing_method: RoutingMethodType,
) -> bool:
return True
@property
def expects_unquantized_inputs(self) -> bool:
return True
def apply(
self,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
router_logits: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
apply_router_weight_on_input: bool,
num_expert_group: int | None = None,
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.moe_config.intermediate_size_per_partition_unpadded is not None
assert self.moe_config.hidden_dim_unpadded is not None
routing_method = self.moe_config.routing_method
if routing_method == RoutingMethodType.DeepseekV4:
score_mode = "sqrtsoftplus"
elif routing_method in (
RoutingMethodType.DeepSeekV3,
RoutingMethodType.MiniMax2,
):
# Sigmoid scores + routing bias; grouped for DeepSeek-R1/V3,
# ungrouped for MiniMax-M3.
score_mode = "sigmoid"
else:
score_mode = None
return aiter_triton_kernel_w4a4_moe_forward(
hidden_states=hidden_states,
w1=w1,
w2=w2,
gating_output=router_logits,
topk=self.topk,
renormalize=self.renormalize,
activation=activation,
global_num_experts=global_num_experts,
expert_map=expert_map,
quant_config=self.quant_config,
apply_router_weight_on_input=apply_router_weight_on_input,
unpadded_N_w1=(
self.moe_config.intermediate_size_per_partition_unpadded * 2
),
unpadded_K_w1=self.moe_config.hidden_dim_unpadded,
unpadded_N_w2=self.moe_config.hidden_dim_unpadded,
unpadded_K_w2=(self.moe_config.intermediate_size_per_partition_unpadded),
num_expert_group=num_expert_group,
topk_group=topk_group,
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
score_mode=score_mode,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
@@ -61,12 +61,6 @@ def aiter_triton_kernel_w4a8_moe_forward(
gating_output, topk, sm_first=not renormalize
)
# gfx1250: aiter's in-kernel gather is numerically broken
if on_gfx1250():
gather_src = gather_idx.to(torch.long) // topk
hidden_states = hidden_states[gather_src]
gather_idx = None
return triton_kernel_fused_mxfp4_w4a8_experts(
None,
hidden_states,
@@ -130,11 +124,11 @@ def triton_kernel_fused_mxfp4_w4a8_experts(
from aiter.ops.triton.quant_moe import downcast_to_static_fp8
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
should_use_cdna4_mx_scale_swizzle,
get_aiter_mx_scale_swizzle,
weight_mx_scale,
)
_swizzle_mx_scale = "CDNA4_SCALE" if should_use_cdna4_mx_scale_swizzle() else None
_swizzle_mx_scale = get_aiter_mx_scale_swizzle()
assert quant_config.w1_precision is not None, (
"w1_precision in quant config can't be None"
@@ -436,6 +436,7 @@ def FusedMoEFactory(
# TODO get from router? needs to be truncated?
e_score_correction_bias=e_score_correction_bias,
apply_router_weight_on_input=apply_router_weight_on_input,
hash_indices_table=hash_indices_table,
**routed_experts_args if routed_experts_args is not None else {},
)
@@ -5,7 +5,7 @@ from collections.abc import Callable
from dataclasses import dataclass
from enum import Enum
from math import prod
from typing import final
from typing import Any, final
import torch
@@ -579,7 +579,11 @@ class FusedMoEExperts(ABC):
return False, _make_reason(
f"parallel config {moe_config.moe_parallel_config}"
)
elif moe_config.has_hash_routing and cls.is_monolithic():
elif (
moe_config.has_hash_routing
and cls.is_monolithic()
and not cls._supports_hash_routing()
):
return False, _make_reason("hash routing")
elif moe_config.require_decomposed_backend and cls.is_monolithic():
return False, _make_reason("routes adjusted after routing")
@@ -659,6 +663,15 @@ class FusedMoEExperts(ABC):
"""
return True
@staticmethod
def _supports_hash_routing() -> bool:
"""Whether a monolithic kernel implements hash routing itself.
Kernels that return True receive `input_ids` and `hash_indices_table`
in `apply()` for layers with a hash-routing table.
"""
return False
@staticmethod
def _supports_router_logits_dtype(
router_logits_dtype: torch.dtype | None,
@@ -1584,11 +1597,20 @@ class FusedMoEKernelMonolithicImpl:
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Same as forward(), except uses router_logits as opposed
to the topk_ids and topk_weights. This is used for kernels
that have fused router + experts (e.g. FLASHINFER_TRTLLM).
"""
# Only experts with _supports_hash_routing() accept these.
hash_kwargs: dict[str, Any] = {}
if hash_indices_table is not None:
hash_kwargs = {
"input_ids": input_ids,
"hash_indices_table": hash_indices_table,
}
a1q, a1q_scale, router_logits = self.prepare_finalize.prepare(
hidden_states,
router_logits=router_logits,
@@ -1605,6 +1627,11 @@ class FusedMoEKernelMonolithicImpl:
f"kernel {type(self.fused_experts).__name__}."
)
routing_replay_out = routing_sink.buffer[: len(a1q)]
hash_kwargs = (
{"input_ids": input_ids, "hash_indices_table": hash_indices_table}
if hash_indices_table is not None
else {}
)
fused_out = self.fused_experts.apply(
hidden_states=a1q,
w1=w1,
@@ -1621,6 +1648,7 @@ class FusedMoEKernelMonolithicImpl:
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_replay_out=routing_replay_out,
**hash_kwargs,
)
if routing_sink is not None:
routing_sink.capture_fn(routing_replay_out)
@@ -1733,6 +1761,8 @@ class FusedMoEKernel:
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
input_ids: torch.Tensor | None = None,
hash_indices_table: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
return self.impl.apply(
@@ -1749,6 +1779,8 @@ class FusedMoEKernel:
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_sink=routing_sink,
input_ids=input_ids,
hash_indices_table=hash_indices_table,
)
def apply(
@@ -841,6 +841,11 @@ def make_fp8_moe_quant_config(
block_shape=block_shape,
per_act_token_quant=per_act_token_quant,
per_out_ch_quant=per_out_ch_quant,
# TRTLLM fp8-block consumes activation scales as [K/128, M]; emit them
# transposed from the quant kernel instead of copying afterwards.
is_scale_transposed=(
fp8_backend == Fp8MoeBackend.FLASHINFER_TRTLLM and block_shape == [128, 128]
),
gemm1_alpha=gemm1_alpha,
gemm1_beta=gemm1_beta,
gemm1_clamp_limit=swiglu_limit,
@@ -152,7 +152,9 @@ class Mxfp4MoeBackend(Enum):
# W4A16: aiter Triton moe_gemm_a16w4 kernel (gfx942/gfx950/gfx1250)
AITER_TRITON_MXFP4_BF16 = "AITER_TRITON_MXFP4_BF16"
AITER_MXFP4_FP8 = "AITER_MXFP4_FP8" # W4A8: triton kernel
AITER_MXFP4_MXFP4 = "AITER_MXFP4_MXFP4" # W4A4: CK kernel
AITER_MXFP4_MXFP4 = "AITER_MXFP4_MXFP4" # W4A4: CK kernel (gfx950)
# W4A4: aiter Triton/Gluon moe_gemm_a4w4 kernel (gfx1250)
AITER_TRITON_MXFP4_MXFP4 = "AITER_TRITON_MXFP4_MXFP4"
# Triton
TRITON = "TRITON"
TRITON_UNFUSED = "TRITON_UNFUSED"
@@ -310,6 +312,13 @@ def backend_to_kernel_cls(
return [AiterMxfp4Experts]
elif backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4:
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a4_moe import (
AiterW4A4ExpertsMonolithic,
)
return [AiterW4A4ExpertsMonolithic]
elif backend == Mxfp4MoeBackend.XPU:
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import XPUExpertsMxFp4
@@ -370,9 +379,11 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16,
Mxfp4MoeBackend.AITER_MXFP4_FP8,
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4,
],
"aiter_triton_mxfp4_bf16": [Mxfp4MoeBackend.AITER_TRITON_MXFP4_BF16],
"aiter_mxfp4_fp8": [Mxfp4MoeBackend.AITER_MXFP4_FP8],
"aiter_triton_mxfp4_mxfp4": [Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4],
"aiter_mxfp4_mxfp4": [Mxfp4MoeBackend.AITER_MXFP4_MXFP4],
"xpu": [Mxfp4MoeBackend.XPU],
"cpu": [Mxfp4MoeBackend.CPU],
@@ -464,6 +475,7 @@ def _backend_activation_key(backend: Mxfp4MoeBackend) -> QuantKey | None:
return kFp8StaticTensorSym
if backend in (
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4,
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
):
return kMxfp4Dynamic
@@ -896,6 +908,18 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
return hidden_size, intermediate_size
def _interleave_gate_up_rows(t: torch.Tensor) -> torch.Tensor:
"""Repack [E, 2n, X] from [all gates | all ups] to [g0, u0, g1, u1, ...]."""
num_experts, two_n, inner = t.shape
assert two_n % 2 == 0, f"expected an even gate/up row count, got {two_n}"
return (
t.view(num_experts, 2, two_n // 2, inner)
.transpose(1, 2)
.contiguous()
.view(num_experts, two_n, inner)
)
def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend: Mxfp4MoeBackend,
layer: torch.nn.Module,
@@ -1192,9 +1216,56 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
w2_bias,
)
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
from vllm._aiter_ops import rocm_aiter_ops
elif mxfp4_backend == Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4:
if w13_bias is not None:
w13_bias = w13_bias.data.to(torch.float32)
if w2_bias is not None:
w2_bias = w2_bias.data.to(torch.float32)
# Triton moe_gemm_a4w4: uint8 column-major weights.
w13_data = w13_weight.data.view(torch.uint8)
w2_data = w2_weight.data.view(torch.uint8)
w13_scale_data = w13_weight_scale.data.view(torch.uint8)
# SILU applies the activation outside the GEMM and keeps the
# checkpoint's packed halves; the SWIGLUOAI epilogue needs them
# interleaved.
if getattr(layer, "activation", None) in (
MoEActivation.SWIGLUOAI,
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
):
w13_data = _interleave_gate_up_rows(w13_data)
w13_scale_data = _interleave_gate_up_rows(w13_scale_data)
# Transpose to [E, K, N] — the view has stride(-2)==1 (column-major),
# which is what moe_gemm_a4w4 asserts.
w13_data = w13_data.transpose(1, 2)
w2_data = w2_data.transpose(1, 2)
# Scales are [E, N, K_scale]; both kernels index them as
# [E, K_scale, N] with K innermost, so transpose(1, 2)
# Keep them uint8: Triton moe_gemm_a4w4 takes e8m0 scales
w13_scale = w13_scale_data.transpose(1, 2)
w2_scale = w2_weight_scale.data.view(torch.uint8).transpose(1, 2)
from aiter.ops.triton.utils.shuffle import shuffle_scale_moe
w13_scale = shuffle_scale_moe(
w13_scale,
arch="gfx1250",
preshuffle_factor=32,
scale_kwidth=4,
)
w2_scale = shuffle_scale_moe(
w2_scale,
arch="gfx1250",
preshuffle_factor=32,
scale_kwidth=4,
)
return (w13_data, w2_data, w13_scale, w2_scale, w13_bias, w2_bias)
elif mxfp4_backend == Mxfp4MoeBackend.AITER_MXFP4_MXFP4:
if w13_bias is not None:
w13_bias = w13_bias.data.to(torch.float32)
if w2_bias is not None:
@@ -1203,6 +1274,8 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
# e8m0_shuffle on weight scales (GFX950 swizzle layout)
from aiter.utility.fp4_utils import e8m0_shuffle
from vllm._aiter_ops import rocm_aiter_ops
s0, s1, _ = w13_weight_scale.shape
w13_weight_scale.data = e8m0_shuffle(w13_weight_scale.view(s0 * s1, -1)).view(
s0, s1, -1
@@ -1935,6 +2008,7 @@ def convert_weight_to_mxfp4_moe_kernel_format(
)
elif mxfp4_backend in (
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4,
Mxfp4MoeBackend.AITER_MXFP4_FP8,
):
return convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
@@ -2088,6 +2162,7 @@ def make_mxfp4_moe_quant_config(
)
elif mxfp4_backend in (
Mxfp4MoeBackend.AITER_MXFP4_MXFP4,
Mxfp4MoeBackend.AITER_TRITON_MXFP4_MXFP4,
Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4,
):
return ocp_mx_moe_quant_config(
@@ -31,6 +31,7 @@ def _quantize_input(
per_act_token_quant=quant_config.per_act_token_quant,
block_shape=quant_config.block_shape,
is_scale_swizzled=quant_config.is_scale_swizzled,
is_scale_transposed=quant_config.is_scale_transposed,
mx_alignment=quant_config.mx_alignment,
)
@@ -0,0 +1,682 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton input-staging kernels for DeepGEMM MegaMoE.
Quantize hidden states to fp8 with E8M0 group scales, to NVFP4 with per-token
global scales, or copy them as BF16, and repack the routing top-k tensors into
the int64/float32 layout that the DeepGEMM MegaMoE kernels consume.
"""
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP8_DTYPE,
QuantKey,
kMxfp8Dynamic,
)
from vllm.triton_utils import tl, triton
_E8M0_SCALE_DTYPES = (torch.uint8, torch.float8_e8m0fnu)
# Inverse of the NVFP4 range: E2M1 max (6) times E4M3 max (448).
_INV_NVFP4_MAX = tl.constexpr(1.0 / 2688.0)
@triton.jit
def _quantize_fp8_groups(
hidden,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
GROUP_K: tl.constexpr,
USE_UE8M0: tl.constexpr,
):
"""Return fp8 values, fp32 group scales and, for UE8M0, the scale exponents."""
num_groups: tl.constexpr = BLOCK_K // GROUP_K
hidden_groups = tl.reshape(tl.abs(hidden), [BLOCK_M, num_groups, GROUP_K])
amax = tl.max(hidden_groups, axis=2)
amax = tl.maximum(amax, 1.0e-4)
scale = amax / 448.0
if USE_UE8M0:
scale_bits = scale.to(tl.uint32, bitcast=True)
scale_exp = ((scale_bits >> 23) & 0xFF) + ((scale_bits & 0x7FFFFF) != 0).to(
tl.uint32
)
scale_exp = tl.minimum(tl.maximum(scale_exp, 1), 254)
quant_scale = (scale_exp << 23).to(tl.float32, bitcast=True)
else:
scale_exp = scale.to(tl.uint32, bitcast=True)
quant_scale = scale
hidden_groups = tl.reshape(hidden, [BLOCK_M, num_groups, GROUP_K])
scaled = hidden_groups * (1.0 / quant_scale)[:, :, None]
scaled = tl.reshape(scaled, [BLOCK_M, BLOCK_K])
return scaled.to(tl.float8e4nv), quant_scale, scale_exp
@triton.jit
def _pack_bytes(values, NUM_BYTES: tl.constexpr):
"""Pack the last dim of ``[..., NUM_BYTES]`` byte values into int32 words."""
offsets = tl.arange(0, NUM_BYTES)
return tl.sum(values.to(tl.uint32) << (offsets * 8), axis=-1).to(tl.int32)
@triton.jit
def _shared_sf_row(token_id, SHARED_BLOCK_M: tl.constexpr):
"""Row of ``token_id`` in the shared-expert MN-major UTCCP scale pages."""
m_block_id = token_id // SHARED_BLOCK_M
m_in_block = token_id % SHARED_BLOCK_M
aligned_block_m: tl.constexpr = triton.cdiv(SHARED_BLOCK_M, 128) * 128
transposed_m = (
(m_in_block // 128) * 128 + (m_in_block % 32) * 4 + (m_in_block % 128) // 32
)
return m_block_id * aligned_block_m + transposed_m
@triton.jit
def _store_topk(
token_id,
token_mask,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
topk_ids_stride_m: tl.constexpr,
topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr,
topk_weights_out_stride_k: tl.constexpr,
top_k: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
) -> None:
"""Repack top-k routing as int64 ids and fp32 weights; padding is unrouted."""
topk_offsets = tl.arange(0, BLOCK_TOPK)
topk_mask = token_mask[:, None] & (topk_offsets[None, :] < top_k)
token_is_padding = tl.full((BLOCK_M,), False, tl.int1)
if is_padding is not None:
token_is_padding = tl.load(
is_padding + token_id * is_padding_stride_m,
mask=token_mask,
other=True,
)
ids = tl.load(
topk_ids
+ token_id[:, None] * topk_ids_stride_m
+ topk_offsets[None, :] * topk_ids_stride_k,
mask=topk_mask,
other=0,
).to(tl.int64)
ids = tl.where(token_is_padding[:, None], -1, ids)
tl.store(
topk_idx_out
+ token_id[:, None] * topk_idx_stride_m
+ topk_offsets[None, :] * topk_idx_stride_k,
ids,
mask=topk_mask,
)
weights = tl.load(
topk_weights
+ token_id[:, None] * topk_weights_stride_m
+ topk_offsets[None, :] * topk_weights_stride_k,
mask=topk_mask,
other=0.0,
)
weights = tl.where(token_is_padding[:, None], 0.0, weights)
tl.store(
topk_weights_out
+ token_id[:, None] * topk_weights_out_stride_m
+ topk_offsets[None, :] * topk_weights_out_stride_k,
weights,
mask=topk_mask,
)
@triton.jit(do_not_specialize=["num_tokens"])
def _prepare_megamoe_inputs_kernel(
hidden_states,
x_out,
x_sf,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
hidden_stride_m: tl.constexpr,
hidden_stride_k: tl.constexpr,
x_stride_m: tl.constexpr,
x_stride_k: tl.constexpr,
x_sf_stride_m: tl.constexpr,
x_sf_stride_k: tl.constexpr,
shared_x_sf_stride_m: tl.constexpr,
shared_x_sf_stride_k: tl.constexpr,
topk_ids_stride_m: tl.constexpr,
topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr,
topk_weights_out_stride_k: tl.constexpr,
num_tokens,
hidden_size: tl.constexpr,
top_k: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
GROUP_K: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
SHARED_BLOCK_M: tl.constexpr,
QUANTIZE: tl.constexpr,
USE_UE8M0: tl.constexpr,
) -> None:
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
token_mask = token_id < num_tokens
k_block_id = tl.program_id(1)
k_offsets = k_block_id * BLOCK_K + tl.arange(0, BLOCK_K)
k_mask = token_mask[:, None] & (k_offsets[None, :] < hidden_size)
hidden = tl.load(
hidden_states
+ token_id[:, None] * hidden_stride_m
+ k_offsets[None, :] * hidden_stride_k,
mask=k_mask,
other=0.0,
).to(tl.float32)
if not QUANTIZE:
tl.store(
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
hidden.to(x_out.dtype.element_ty),
mask=k_mask,
)
else:
num_groups: tl.constexpr = BLOCK_K // GROUP_K
fp8, quant_scale, scale_exp = _quantize_fp8_groups(
hidden, BLOCK_M, BLOCK_K, GROUP_K, USE_UE8M0
)
tl.store(
x_out + token_id[:, None] * x_stride_m + k_offsets[None, :] * x_stride_k,
fp8,
mask=k_mask,
)
if USE_UE8M0:
packed_scale = _pack_bytes(scale_exp, num_groups)
tl.store(
x_sf + token_id * x_sf_stride_m + k_block_id * x_sf_stride_k,
packed_scale,
mask=token_mask,
)
else:
sf_offset = tl.arange(0, num_groups)
tl.store(
x_sf
+ token_id[:, None] * x_sf_stride_m
+ k_block_id * x_sf_stride_k
+ sf_offset[None, :],
quant_scale,
mask=token_mask[:, None],
)
# DeepGEMM's SM100 shared-expert TMA loads require the activation scales
# in an MN-major layout whose row permutation depends on the MegaMoE
# scheduler's runtime BLOCK_M. Write that view while the packed UE8M0 scale
# is already resident, avoiding another kernel and temporary tensor.
if USE_UE8M0 and shared_x_sf is not None:
shared_row = _shared_sf_row(token_id, SHARED_BLOCK_M)
tl.store(
shared_x_sf
+ shared_row * shared_x_sf_stride_m
+ k_block_id * shared_x_sf_stride_k,
packed_scale,
mask=token_mask,
)
if k_block_id == 0:
_store_topk(
token_id,
token_mask,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
topk_ids_stride_m,
topk_ids_stride_k,
topk_weights_stride_m,
topk_weights_stride_k,
is_padding_stride_m,
topk_idx_stride_m,
topk_idx_stride_k,
topk_weights_out_stride_m,
topk_weights_out_stride_k,
top_k,
BLOCK_M,
BLOCK_TOPK,
)
@triton.jit(do_not_specialize=["num_tokens", "l2_activation_scale"])
def _prepare_megamoe_nvfp4_inputs_kernel(
hidden_states,
x_out,
x_sf,
x_scales,
shared_x,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
w2_global_scale,
l2_alpha,
hidden_stride_m: tl.constexpr,
hidden_stride_k: tl.constexpr,
x_stride_m: tl.constexpr,
x_sf_stride_m: tl.constexpr,
shared_x_stride_m: tl.constexpr,
shared_x_sf_stride_m: tl.constexpr,
shared_x_sf_stride_k: tl.constexpr,
topk_ids_stride_m: tl.constexpr,
topk_ids_stride_k: tl.constexpr,
topk_weights_stride_m: tl.constexpr,
topk_weights_stride_k: tl.constexpr,
is_padding_stride_m: tl.constexpr,
topk_idx_stride_m: tl.constexpr,
topk_idx_stride_k: tl.constexpr,
topk_weights_out_stride_m: tl.constexpr,
topk_weights_out_stride_k: tl.constexpr,
num_tokens,
l2_activation_scale,
hidden_size: tl.constexpr,
top_k: tl.constexpr,
num_local_experts: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
BLOCK_TOPK: tl.constexpr,
BLOCK_E: tl.constexpr,
SHARED_BLOCK_M: tl.constexpr,
SHARED_MXFP8: tl.constexpr,
) -> None:
if l2_alpha is not None:
experts = tl.arange(0, BLOCK_E)
expert_mask = (experts < num_local_experts) & (tl.program_id(0) == 0)
alpha = tl.load(w2_global_scale + experts, mask=expert_mask)
tl.store(l2_alpha + experts, alpha * l2_activation_scale, mask=expert_mask)
token_id = tl.program_id(0) * BLOCK_M + tl.arange(0, BLOCK_M)
token_mask = token_id < num_tokens
k_offsets = tl.arange(0, BLOCK_K)
hidden_ptrs = hidden_states + token_id[:, None] * hidden_stride_m
amax = tl.zeros((BLOCK_M,), tl.float32)
for k_block_id in range(hidden_size // BLOCK_K):
hidden = tl.load(
hidden_ptrs + (k_block_id * BLOCK_K + k_offsets[None, :]) * hidden_stride_k,
mask=token_mask[:, None],
other=0.0,
).to(tl.float32)
amax = tl.maximum(amax, tl.max(tl.abs(hidden), axis=1))
x_scale = tl.maximum(amax, 1.0e-12) * _INV_NVFP4_MAX
tl.store(x_scales + token_id, x_scale, mask=token_mask)
for k_block_id in range(hidden_size // BLOCK_K):
k_ids = k_block_id * BLOCK_K + k_offsets
hidden = tl.load(
hidden_ptrs + k_ids[None, :] * hidden_stride_k,
mask=token_mask[:, None],
other=0.0,
)
# E2M1 values with one E4M3 scale per 16, relative to the token scale.
values = tl.reshape(
tl.div_rn(hidden.to(tl.float32), x_scale[:, None]),
[BLOCK_M, BLOCK_K // 16, 16],
)
block_scale = tl.max(tl.abs(values), axis=2) * (1.0 / 6.0)
block_scale = tl.minimum(tl.maximum(block_scale, 2.0**-9), 448.0)
block_scale = block_scale.to(tl.float8e4nv)
values = tl.div_rn(values, block_scale.to(tl.float32)[:, :, None])
magnitude = tl.abs(values)
# Round to nearest even on the grid {0, 0.5, 1, 1.5, 2, 3, 4, 6}.
code = (
(magnitude > 0.25).to(tl.int32)
+ (magnitude >= 0.75).to(tl.int32)
+ (magnitude > 1.25).to(tl.int32)
+ (magnitude >= 1.75).to(tl.int32)
+ (magnitude > 2.5).to(tl.int32)
+ (magnitude >= 3.5).to(tl.int32)
+ (magnitude > 5.0).to(tl.int32)
)
code = tl.where((values < 0) & (code != 0), code | 8, code)
low, high = tl.split(tl.reshape(code, [BLOCK_M, BLOCK_K // 2, 2]))
packed_offsets = k_block_id * (BLOCK_K // 2) + tl.arange(0, BLOCK_K // 2)
tl.store(
x_out + token_id[:, None] * x_stride_m + packed_offsets[None, :],
(low | (high << 4)).to(tl.uint8),
mask=token_mask[:, None],
)
sf_offsets = k_block_id * (BLOCK_K // 64) + tl.arange(0, BLOCK_K // 64)
sf_bytes = tl.reshape(
block_scale.to(tl.uint8, bitcast=True), [BLOCK_M, BLOCK_K // 64, 4]
)
tl.store(
x_sf + token_id[:, None] * x_sf_stride_m + sf_offsets[None, :],
_pack_bytes(sf_bytes, 4),
mask=token_mask[:, None],
)
if shared_x is not None:
shared_ptrs = shared_x + token_id[:, None] * shared_x_stride_m
if SHARED_MXFP8:
fp8, _, scale_exp = _quantize_fp8_groups(
hidden.to(tl.float32), BLOCK_M, BLOCK_K, 32, True
)
tl.store(shared_ptrs + k_ids[None, :], fp8, mask=token_mask[:, None])
tl.store(
shared_x_sf
+ _shared_sf_row(token_id, SHARED_BLOCK_M) * shared_x_sf_stride_m
+ k_block_id * shared_x_sf_stride_k,
_pack_bytes(scale_exp, BLOCK_K // 32),
mask=token_mask,
)
else:
tl.store(shared_ptrs + k_ids[None, :], hidden, mask=token_mask[:, None])
_store_topk(
token_id,
token_mask,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
topk_ids_stride_m,
topk_ids_stride_k,
topk_weights_stride_m,
topk_weights_stride_k,
is_padding_stride_m,
topk_idx_stride_m,
topk_idx_stride_k,
topk_weights_out_stride_m,
topk_weights_out_stride_k,
top_k,
BLOCK_M,
BLOCK_TOPK,
)
def _resolve_hidden_quant(hidden_quant: QuantKey, block_k: int) -> tuple[int, bool]:
"""Map a hidden-state ``QuantKey`` to the kernel's ``(GROUP_K, USE_UE8M0)``."""
scale = hidden_quant.scale
group_shape = scale.group_shape
if (
hidden_quant.dtype != FP8_DTYPE
or not hidden_quant.symmetric
or hidden_quant.scale2 is not None
or scale.static
or not group_shape.is_per_group()
or block_k % group_shape.col != 0
or scale.dtype not in (*_E8M0_SCALE_DTYPES, torch.float32)
):
raise ValueError(
"MegaMoE input staging requires symmetric dynamic fp8 "
f"per-group quantization with a group size dividing {block_k} and "
f"E8M0 or fp32 scales, got {hidden_quant}."
)
return group_shape.col, scale.dtype in _E8M0_SCALE_DTYPES
def prepare_megamoe_inputs(
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
x: torch.Tensor,
x_sf: torch.Tensor | None,
topk_idx_out: torch.Tensor,
topk_weights_out: torch.Tensor,
is_padding: torch.Tensor | None = None,
shared_x_sf: torch.Tensor | None = None,
shared_block_m: int | None = None,
hidden_quant: QuantKey | None = kMxfp8Dynamic,
) -> None:
"""Quantize hidden states and repack top-k routing for DeepGEMM MegaMoE.
Args:
hidden_states: Input activations of shape ``[num_tokens, hidden_dim]``.
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
x: Output buffer for the dispatched hidden states.
x_sf: Output buffer for the hidden-state scale factors; ``None`` when
``hidden_quant`` is ``None``.
topk_idx_out: Output buffer for the repacked top-k expert ids.
topk_weights_out: Output buffer for the repacked top-k weights.
is_padding: Optional per-token mask; padded tokens are not routed.
shared_x_sf: Optional output buffer for shared-expert scale factors.
Must be given together with ``shared_block_m``.
shared_block_m: Block M used to lay out ``shared_x_sf``.
hidden_quant: Hidden-state quantization scheme. E8M0 scales are packed
four per int32 into ``x_sf``; fp32 scales are stored directly.
``None`` copies the hidden states into ``x`` unquantized.
"""
num_tokens, hidden_size = hidden_states.shape
if num_tokens == 0:
return
if hidden_size % 128 != 0:
raise ValueError(
"MegaMoE input staging requires hidden_size to be a multiple of 128."
)
block_k = 128
quantize = hidden_quant is not None
if quantize != (x_sf is not None):
raise ValueError("MegaMoE input staging needs x_sf iff hidden_quant is set.")
group_k, use_ue8m0 = (
(block_k, False)
if hidden_quant is None
else _resolve_hidden_quant(hidden_quant, block_k)
)
top_k = topk_ids.shape[1]
if topk_weights.shape != topk_ids.shape:
raise ValueError(
"MegaMoE input staging requires topk_weights and "
"topk_ids to have the same shape."
)
if (shared_x_sf is None) != (shared_block_m is None):
raise ValueError(
"MegaMoE shared input staging requires both shared_x_sf and shared_block_m."
)
if shared_x_sf is not None and not use_ue8m0:
raise ValueError(
"MegaMoE shared input staging currently requires "
"UE8M0-packed hidden scales."
)
if shared_x_sf is not None:
assert shared_block_m is not None
if shared_block_m <= 0:
raise ValueError("MegaMoE shared_block_m must be positive.")
expected_sf_k = hidden_size // 128
if shared_x_sf.ndim != 2 or shared_x_sf.shape[1] != expected_sf_k:
raise ValueError(
"MegaMoE shared_x_sf must have shape "
f"(*, {expected_sf_k}), got {tuple(shared_x_sf.shape)}."
)
aligned_block_m = triton.cdiv(shared_block_m, 128) * 128
required_rows = triton.cdiv(num_tokens, shared_block_m) * aligned_block_m
if shared_x_sf.shape[0] < required_rows:
raise ValueError(
"MegaMoE shared_x_sf has insufficient rows: requires "
f"{required_rows}, got {shared_x_sf.shape[0]}."
)
# On GB200, eight-row tiles win from 64 tokens; keep smaller batches untiled.
block_m = 8 if num_tokens >= 64 else 1
grid = (triton.cdiv(num_tokens, block_m), triton.cdiv(hidden_size, block_k))
block_topk = triton.next_power_of_2(top_k)
padding_stride_m = is_padding.stride(0) if is_padding is not None else 0
_prepare_megamoe_inputs_kernel[grid](
hidden_states,
x,
x_sf,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
hidden_states.stride(0),
hidden_states.stride(1),
x.stride(0),
x.stride(1),
x_sf.stride(0) if x_sf is not None else 0,
x_sf.stride(1) if x_sf is not None else 0,
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
topk_ids.stride(0),
topk_ids.stride(1),
topk_weights.stride(0),
topk_weights.stride(1),
padding_stride_m,
topk_idx_out.stride(0),
topk_idx_out.stride(1),
topk_weights_out.stride(0),
topk_weights_out.stride(1),
num_tokens,
hidden_size,
top_k,
BLOCK_M=block_m,
BLOCK_K=block_k,
GROUP_K=group_k,
BLOCK_TOPK=block_topk,
SHARED_BLOCK_M=shared_block_m or 1,
QUANTIZE=quantize,
USE_UE8M0=use_ue8m0,
num_warps=4,
)
def prepare_megamoe_nvfp4_inputs(
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
x: torch.Tensor,
x_sf: torch.Tensor,
x_scales: torch.Tensor,
topk_idx_out: torch.Tensor,
topk_weights_out: torch.Tensor,
is_padding: torch.Tensor | None = None,
shared_x: torch.Tensor | None = None,
shared_x_sf: torch.Tensor | None = None,
shared_block_m: int | None = None,
w2_global_scale: torch.Tensor | None = None,
l2_alpha: torch.Tensor | None = None,
l2_activation_scale: float = 1.0,
) -> None:
"""Stage DeepGEMM NVFP4 MegaMoE inputs (``kNvfp4DynamicToken``).
Each token is scaled by its own fp32 ``x_scales`` (row amax / 2688) and
quantized to E2M1 with one E4M3 scale per 16 values.
Args:
hidden_states: BF16 activations of shape ``[num_tokens, hidden_dim]``.
topk_weights: Router top-k weights of shape ``[num_tokens, top_k]``.
topk_ids: Router top-k expert ids of shape ``[num_tokens, top_k]``.
x: Packed E2M1 output, ``[num_tokens, hidden_dim // 2]``.
x_sf: E4M3 scales packed four per int32, ``[num_tokens, hidden_dim // 64]``.
x_scales: Per-token fp32 dequantization scales, ``[num_tokens]``.
topk_idx_out: Output buffer for the repacked top-k expert ids.
topk_weights_out: Output buffer for the repacked top-k weights.
is_padding: Optional per-token mask; padded tokens are not routed.
shared_x: Optional shared-expert input, fp8 (MXFP8 with ``shared_x_sf``)
or a BF16 copy of the hidden states.
shared_x_sf: UE8M0 scales of an fp8 ``shared_x`` in the shared-expert
TMA layout for ``shared_block_m``.
shared_block_m: Block M used to lay out ``shared_x_sf``.
w2_global_scale: Per-local-expert L2 weight global scales.
l2_alpha: Output ``w2_global_scale * l2_activation_scale``, the L2
dequantization scale of each local expert. Written even without
local tokens, as other ranks' tokens still reach these experts.
l2_activation_scale: Global scale the kernel quantizes L2 inputs with.
"""
num_tokens, hidden_size = hidden_states.shape
if hidden_size % 128 != 0:
raise ValueError(
"MegaMoE input staging requires hidden_size to be a multiple of 128."
)
if topk_weights.shape != topk_ids.shape:
raise ValueError(
"MegaMoE input staging requires topk_weights and "
"topk_ids to have the same shape."
)
shared_mxfp8 = shared_x is not None and shared_x.dtype == FP8_DTYPE
if shared_mxfp8 != (shared_x_sf is not None) or (shared_x_sf is None) != (
shared_block_m is None
):
raise ValueError(
"MegaMoE NVFP4 staging needs shared_x_sf and shared_block_m exactly "
"for an fp8 shared_x."
)
if (w2_global_scale is None) != (l2_alpha is None):
raise ValueError("MegaMoE NVFP4 staging needs w2_global_scale with l2_alpha.")
for name, tensor in (("x", x), ("x_sf", x_sf), ("shared_x", shared_x)):
if tensor is not None and tensor.stride(-1) != 1:
raise ValueError(f"MegaMoE NVFP4 staging requires a row-major {name}.")
block_m = 8 if num_tokens >= 64 else 1
top_k = topk_ids.shape[1]
num_local_experts = 0 if l2_alpha is None else l2_alpha.numel()
# One program even without tokens, to write l2_alpha.
grid = (max(triton.cdiv(num_tokens, block_m), 1),)
_prepare_megamoe_nvfp4_inputs_kernel[grid](
hidden_states,
x.view(torch.uint8),
x_sf,
x_scales,
shared_x,
shared_x_sf,
topk_ids,
topk_weights,
is_padding,
topk_idx_out,
topk_weights_out,
w2_global_scale,
l2_alpha,
hidden_states.stride(0),
hidden_states.stride(1),
x.stride(0),
x_sf.stride(0),
shared_x.stride(0) if shared_x is not None else 0,
shared_x_sf.stride(0) if shared_x_sf is not None else 0,
shared_x_sf.stride(1) if shared_x_sf is not None else 0,
topk_ids.stride(0),
topk_ids.stride(1),
topk_weights.stride(0),
topk_weights.stride(1),
is_padding.stride(0) if is_padding is not None else 0,
topk_idx_out.stride(0),
topk_idx_out.stride(1),
topk_weights_out.stride(0),
topk_weights_out.stride(1),
num_tokens,
l2_activation_scale,
hidden_size,
top_k,
num_local_experts,
BLOCK_M=block_m,
BLOCK_K=128,
BLOCK_TOPK=triton.next_power_of_2(top_k),
BLOCK_E=triton.next_power_of_2(max(num_local_experts, 1)),
SHARED_BLOCK_M=shared_block_m or 1,
SHARED_MXFP8=shared_mxfp8,
num_warps=4,
)
@@ -79,6 +79,9 @@ class RoutedExperts(PluggableLayer):
- Executing routed experts via quant_method.apply()
"""
# Hash-routing table, owned by the gate.
hash_indices_table: torch.Tensor | None = None
def __init__(
self,
layer_name: str,
@@ -107,6 +110,7 @@ class RoutedExperts(PluggableLayer):
swiglu_beta: float | None = None,
e_score_correction_bias: torch.Tensor | None = None,
apply_router_weight_on_input: bool = False,
hash_indices_table: torch.Tensor | None = None,
):
super().__init__()
self.layer_name = layer_name
@@ -152,6 +156,8 @@ class RoutedExperts(PluggableLayer):
# Set by bind_routed_experts_capturer for monolithic kernels.
self.routing_sink: RoutedExpertsSink | None = None
self.apply_router_weight_on_input = apply_router_weight_on_input
# The gate owns this Parameter: keep it out of this module's parameters.
object.__setattr__(self, "hash_indices_table", hash_indices_table)
# End random parameters
self._loaded_expert_biases: set[str] = set()
self._fused_shared_expert_quantizer = self._get_fused_shared_expert_quantizer()
@@ -14,7 +14,7 @@ import torch
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.platforms import current_platform
@@ -85,103 +85,74 @@ class RoutedExpertsCapturer:
self.tp_size = vllm_config.parallel_config.tensor_parallel_size
def capture(self, layer_id: int, topk_ids: torch.Tensor) -> None:
"""Capture expert routing decisions for a specific layer.
Under data parallelism, ``topk_ids`` may have four different batch
layouts depending on where the DP combine happens and whether
Expert Parallelism (EP) or Sequence Parallelism (SP) is active for the
MoE layer:
- ``n == total`` (naive dispatch): all DP ranks' tokens are
concatenated before routing; we slice out this rank's span
using the cumulative per-rank counts.
- ``n == token_num_per_dp`` (modular-kernel path): DP combine
happens inside ``quant_method.apply``; ``select_experts`` only
ever sees this rank's tokens, so we take the whole tensor.
- ``n == sum(dp_metadata.local_sizes)`` (naive DP+EP dispatch):
sequence-parallel shards from every DP rank are gathered through
the flattened EP group. The shard sizes include CUDA-graph / SP
padding, so we use them to locate this DP rank's unpadded rows.
- ``n == ceil(token_num_per_dp / tp_size)`` (SP + modular-kernel
path): tokens were split along dim=0 across the TP group by
``_sequence_parallel_context``
(``moe_runner_base.py:_sequence_parallel_context``), so each
TP rank only sees its shard. We all-gather along dim=0 to
reconstruct this DP rank's full routing tensor. SP pads with
ceil-div (see ``_compute_sp_num_tokens`` in
``forward_context.py``), so the gathered tensor may contain a
few trailing padding rows which are trimmed by the downstream
``[:token_num_per_dp]`` slice.
Args:
layer_id: The layer index.
topk_ids: Tensor of shape (batch_size, num_routed_experts).
"""
ctx = get_forward_context()
if ctx.dp_metadata is None:
local_topk_ids = topk_ids
else:
num_tokens_dp = ctx.dp_metadata.num_tokens_across_dp_cpu
num_local_tokens = int(num_tokens_dp[self.dp_rank].item())
total = int(num_tokens_dp.sum().item())
n = topk_ids.shape[0]
shard_sizes = getattr(ctx.dp_metadata, "local_sizes", None)
gathered_size = sum(shard_sizes) if shard_sizes is not None else -1
if n == total:
end = int(num_tokens_dp[: self.dp_rank + 1].sum().item())
local_topk_ids = topk_ids[end - num_local_tokens : end]
elif n == num_local_tokens:
local_topk_ids = topk_ids
elif shard_sizes is not None and n == gathered_size:
# Naive DP+EP dispatch gathers the sequence-parallel shards in
# flattened DP-then-TP order. ``local_sizes`` is the exact
# all-gatherv layout, including padding. Select this DP rank's
# contiguous shard group, then trim its trailing padding.
num_dp_ranks = len(num_tokens_dp)
assert len(shard_sizes) % num_dp_ranks == 0
shards_per_dp_rank = len(shard_sizes) // num_dp_ranks
first_shard = self.dp_rank * shards_per_dp_rank
start = sum(shard_sizes[:first_shard])
local_topk_ids = topk_ids[start : start + num_local_tokens]
elif (
self.tp_size > 1
and n == (num_local_tokens + self.tp_size - 1) // self.tp_size
):
# SP + modular-kernel path. All-gather across the TP
# group along dim=0 to reconstruct the full per-DP-rank
# tensor; keep only the first ``token_num_per_dp`` rows
# (trailing rows are SP ceil-div padding). The TP group
# is always initialized on real rollout workers, and
# every rank in the group reaches this branch in
# lockstep (bind is per-FusedMoEFactory layer, SP is a global
# condition), so a bare all_gather here will not
# deadlock -- let it raise if the precondition is
# violated rather than skip silently.
#
# ``topk_ids`` is already whatever the router produced
# (typically int32/int64, both supported by NCCL); the
# downstream buffer assignment narrows to the capture dtype.
local_topk_ids = get_tp_group().all_gather(topk_ids, dim=0)[
:num_local_tokens
]
else:
sp_expected = (num_local_tokens + self.tp_size - 1) // self.tp_size
raise AssertionError(
"RoutedExpertsCapturer: unexpected topk_ids batch "
f"dim {n} (expected {total}, {num_local_tokens}, "
f"{gathered_size}, or {sp_expected} for "
f"dp_rank={self.dp_rank}, tp_size={self.tp_size})"
)
if not 0 <= layer_id < self.device_buffer.shape[1]:
raise IndexError(
f"routed-experts layer {layer_id} exceeds capture buffer "
f"layer count {self.device_buffer.shape[1]}"
)
local_topk_ids = self._restore_local_topk_ids(topk_ids, get_forward_context())
self.device_buffer[: len(local_topk_ids), layer_id] = local_topk_ids
def _restore_local_topk_ids(
self, topk_ids: torch.Tensor, ctx: ForwardContext
) -> torch.Tensor:
"""Restore this DP rank's token rows, trimming trailing SP padding.
Layout priority: cross-DP batch, local batch, gathered DP/TP shards,
then a local TP shard. Gathered rows are DP-major/TP-minor; preserve
this priority when layout sizes coincide. PCP is unsupported.
Without DP metadata, use the pre-SP batch descriptor. Unknown counts
pass through; known counts must match a full, sharded or padded batch.
A full local batch avoids gathering, including zero/one-token batches
where only the publishing TP0 rank needs the real rows.
"""
n = topk_ids.shape[0]
dp_metadata = ctx.dp_metadata
if dp_metadata is None:
if ctx.batch_descriptor is None:
return topk_ids
num_local_tokens = ctx.batch_descriptor.num_tokens
else:
num_tokens_dp = dp_metadata.num_tokens_across_dp_cpu
num_local_tokens = int(num_tokens_dp[self.dp_rank].item())
if n == int(num_tokens_dp.sum().item()):
end = int(num_tokens_dp[: self.dp_rank + 1].sum().item())
return topk_ids[end - num_local_tokens : end]
if n == num_local_tokens:
return topk_ids
if dp_metadata is not None:
shard_sizes = dp_metadata.local_sizes
if shard_sizes is not None and n == sum(shard_sizes):
num_dp_ranks = len(num_tokens_dp)
if len(shard_sizes) % num_dp_ranks != 0:
raise ValueError(
"RoutedExpertsCapturer: gathered shard count "
f"{len(shard_sizes)} is not divisible by DP rank count "
f"{num_dp_ranks}"
)
shards_per_dp_rank = len(shard_sizes) // num_dp_ranks
first_shard = self.dp_rank * shards_per_dp_rank
start = sum(shard_sizes[:first_shard])
return topk_ids[start : start + num_local_tokens]
shard_n = (num_local_tokens + self.tp_size - 1) // self.tp_size
if n == shard_n:
return get_tp_group().all_gather(topk_ids.contiguous(), dim=0)[
:num_local_tokens
]
if dp_metadata is None and n == shard_n * self.tp_size:
return topk_ids[:num_local_tokens]
raise AssertionError(
f"RoutedExpertsCapturer: unexpected topk_ids batch dim {n} "
f"(local_tokens={num_local_tokens}, "
f"dp_rank={self.dp_rank}, tp_size={self.tp_size})"
)
def snapshot_routing_data(self, num_tokens: int) -> torch.Tensor:
"""Return a stable snapshot of the current routing data."""
return self.device_buffer[:num_tokens].to(self.output_dtype)
@@ -84,7 +84,14 @@ class AiterSharedRoutedFusedMoERouter(BaseRouter):
total_topk_weights_slice = total_topk_weights[:M]
topk_ids_slice = total_topk_ids[:M, :topk]
topk_func = dispatch_topk_softmax_func(use_rocm_aiter=True)
topk_func = dispatch_topk_softmax_func(
use_rocm_aiter=True,
topk_weights=total_topk_weights_slice,
topk_indices=topk_ids_slice,
gating_output=router_logits,
num_shared_experts=num_fse,
shared_expert_scoring_func="sigmoid",
)
topk_func(
total_topk_weights_slice,
topk_ids_slice,
@@ -110,7 +117,10 @@ class AiterSharedRoutedFusedMoERouter(BaseRouter):
)
topk_func = dispatch_topk_softmax_func(
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled()
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled(),
topk_weights=topk_weights,
topk_indices=topk_ids,
gating_output=routing_logits,
)
topk_weights, topk_ids = topk_func(
topk_weights,
@@ -61,10 +61,73 @@ def vllm_topk_sigmoid(
return topk_weights, topk_indices
# At E=512 / top-10, AITER's topk_gating is faster than the legacy launcher
# through this many tokens; its one-row generic prefill path regresses above it.
_AITER_TOPK_GATING_MAX_TOKENS = 4096
def _aiter_topk_gating_supported(
topk_weights: torch.Tensor,
topk_indices: torch.Tensor,
gating_output: torch.Tensor,
num_shared_experts: int,
shared_expert_scoring_func: str,
) -> bool:
"""Whether AITER's ``topk_gating`` can serve this softmax top-k launch.
It needs contiguous gating rows and at most ``_AITER_TOPK_GATING_MAX_TOKENS``
tokens. Fused shared experts are scored with sigmoid after the routed
top-k: 1, 2, 4 or 8 of them, with ``topk_weights`` and ``topk_indices``
sharing a row stride and ``topk_weights`` wide enough for the shared
columns. Shared expert ids stay as the caller prefilled them.
"""
if (
not gating_output.is_contiguous()
or gating_output.shape[0] > _AITER_TOPK_GATING_MAX_TOKENS
):
return False
if num_shared_experts == 0:
return not shared_expert_scoring_func
return (
shared_expert_scoring_func == "sigmoid"
and num_shared_experts in (1, 2, 4, 8)
and topk_weights.stride(0) == topk_indices.stride(0)
and topk_weights.shape[-1] >= topk_indices.shape[-1] + num_shared_experts
)
def dispatch_topk_softmax_func(
use_rocm_aiter: bool = False,
*,
topk_weights: torch.Tensor | None = None,
topk_indices: torch.Tensor | None = None,
gating_output: torch.Tensor | None = None,
num_shared_experts: int = 0,
shared_expert_scoring_func: str = "",
) -> Callable[..., tuple[torch.Tensor, ...]]:
"""Pick the softmax top-k implementation for one launch.
On ROCm with AITER, ``topk_gating`` is chosen on gfx942 and gfx950 when the
launch described by the tensors and shared-expert arguments is one it supports;
every other AITER launch uses the legacy ``topk_softmax``. The tensors are
optional: without them the launch cannot be judged and the legacy launcher
is used.
"""
if use_rocm_aiter:
if (
rocm_aiter_ops.is_topk_gating_enabled()
and topk_weights is not None
and topk_indices is not None
and gating_output is not None
and _aiter_topk_gating_supported(
topk_weights,
topk_indices,
gating_output,
num_shared_experts,
shared_expert_scoring_func,
)
):
return rocm_aiter_ops.topk_gating
return rocm_aiter_ops.topk_softmax
return vllm_topk_softmax
@@ -104,7 +167,10 @@ def fused_topk(
if scoring_func == "softmax":
topk_func = dispatch_topk_softmax_func(
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled()
use_rocm_aiter=rocm_aiter_ops.is_fused_moe_enabled(),
topk_weights=topk_weights,
topk_indices=topk_ids,
gating_output=gating_output,
)
topk_weights, topk_ids = topk_func(
topk_weights, topk_ids, token_expert_indices, gating_output, renormalize
@@ -24,14 +24,15 @@ class GateLinear(ReplicatedLinear):
1. cuteDSL ll_bf16_gemm (SM90+, M<=16, bf16 in, fp32 out,
any shape w/ K divisible by 8)
2. fp32 specialized kernel (SM90+ or gfx950, bf16/fp32 in, fp32 out,
M<=32, model-specific shapes)
2. fp32 specialized kernel (bf16/fp32 in, fp32 out, M<=32; SM90+
model-specific shapes, gfx950 any shape)
3. cuteDSL ll_fp32w_gemm (SM90+, M<=32, bf16/fp32 in, fp32 out,
any shape)
4. bf16x3 CuteDSL kernel (SM100, bf16 in, fp32 weight)
5. ROCm bf16x3 router GEMM (gfx950, bf16 in, fp32 weight, fp32 out,
M>=2048)
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype)
6. cuBLAS bf16×bf16→fp32 (SM90+ + bf16 weight + fp32 out_dtype); gfx950
routes low M through the tier 2 kernel
7. F.linear via ReplicatedLinear (ultimate fallback)
The ``out_dtype`` attribute is mutable and can be set after init
@@ -73,16 +74,6 @@ class GateLinear(ReplicatedLinear):
from vllm.platforms.rocm import on_gfx950
is_gfx950 = on_gfx950()
is_rocm_fp32_shape = False
if is_gfx950:
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES,
)
is_rocm_fp32_shape = (
input_size,
output_size,
) in ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES
batch_invariant = envs.VLLM_BATCH_INVARIANT
can_use_specialized_kernels = (
current_platform.is_cuda()
@@ -107,6 +98,7 @@ class GateLinear(ReplicatedLinear):
return_bias=return_bias,
)
self.out_dtype = out_dtype
self.is_gfx950 = is_gfx950
# A quantized gate exposes no plain ``weight``, so every specialized
# tier below is disabled and forward falls back to ReplicatedLinear.
@@ -127,7 +119,7 @@ class GateLinear(ReplicatedLinear):
and (is_hopper or is_blackwell)
and (input_size, output_size) in self.FP32_SUPPORTED_SHAPES
)
or (is_gfx950 and is_rocm_fp32_shape)
or is_gfx950
)
)
self.allow_bf16x3_router_gemm = (
@@ -349,7 +341,10 @@ class GateLinear(ReplicatedLinear):
# Tier 6: cuBLAS bf16→fp32
if self.allow_cublas_router_gemm and x.dtype == torch.bfloat16:
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
if self.is_gfx950:
output = torch.ops.vllm.fp32_router_gemm_dispatch(x, self.weight, False)
else:
output = torch.mm(x, self.weight.T, out_dtype=torch.float32)
return self._return(output)
# Tier 7: F.linear (ReplicatedLinear)
@@ -465,17 +460,20 @@ def fp32_router_gemm_dispatch_impl(
This must be wrapped in a custom op because our torch.compile integration
does not support runtime dispatching on num_tokens.
"""
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
if current_platform.is_rocm():
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
can_use_rocm_fp32_router_gemm,
rocm_fp32_router_gemm,
)
if current_platform.is_rocm():
from vllm.model_executor.layers.fused_moe.router.rocm_fp32_router_gemm import ( # noqa: E501
can_use_rocm_fp32_router_gemm,
rocm_fp32_router_gemm,
)
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
x = x.contiguous()
if can_use_rocm_fp32_router_gemm(x, weight):
return rocm_fp32_router_gemm(x, weight)
return torch.nn.functional.linear(x.float(), weight)
if weight.dtype == torch.bfloat16:
return torch.mm(x, weight.T, out_dtype=torch.float32)
return torch.nn.functional.linear(x.float(), weight)
if x.shape[0] <= _FP32_ROUTER_GEMM_MAX_TOKENS:
return ops.fp32_router_gemm(x, weight)
if allow_bf16x3_router_gemm and x.dtype == torch.bfloat16:
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Low-token FP32 router GEMM for gfx950."""
"""Low-token router GEMM with FP32 output for gfx950 (FP32 or BF16 weights)."""
import torch
@@ -9,15 +9,10 @@ from vllm.platforms.rocm import on_gfx950
from vllm.triton_utils import tl, triton
_MAX_TOKENS = 32
ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES = frozenset(
{
(3072, 256),
(4096, 8),
(4096, 192),
(6144, 128),
(6144, 256),
}
)
# From this many multiply-adds on, F.linear wins on MI355.
_MAX_MACS = 1 << 27
# Above this num_tokens * num_experts, hipBLASLt wins for BF16 weights.
_MAX_BF16_TOKEN_EXPERTS = 8192
@triton.jit
@@ -30,6 +25,7 @@ def _rocm_fp32_router_gemm_kernel(
N: tl.constexpr,
BLOCK_M: tl.constexpr,
BLOCK_K: tl.constexpr,
EVEN_K: tl.constexpr,
):
pid = tl.program_id(0)
pid_m = pid // N
@@ -40,12 +36,27 @@ def _rocm_fp32_router_gemm_kernel(
for k_start in range(0, K, BLOCK_K):
offsets_k_block = k_start + offsets_k
hidden_states = tl.load(
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
mask=offsets_m[:, None] < M,
other=0.0,
).to(tl.float32)
router_weight = tl.load(router_weight_ptr + pid_n * K + offsets_k_block)
if EVEN_K:
hidden_states = tl.load(
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
mask=offsets_m[:, None] < M,
other=0.0,
).to(tl.float32)
router_weight = tl.load(router_weight_ptr + pid_n * K + offsets_k_block).to(
tl.float32
)
else:
k_mask = offsets_k_block < K
hidden_states = tl.load(
hidden_states_ptr + offsets_m[:, None] * K + offsets_k_block[None, :],
mask=(offsets_m[:, None] < M) & k_mask[None, :],
other=0.0,
).to(tl.float32)
router_weight = tl.load(
router_weight_ptr + pid_n * K + offsets_k_block,
mask=k_mask,
other=0.0,
).to(tl.float32)
partials += hidden_states * router_weight[None, :]
accumulator = tl.sum(partials, axis=1)
@@ -57,28 +68,36 @@ def _rocm_fp32_router_gemm_kernel(
def _launch_config(
hidden_size: int,
num_experts: int,
num_tokens: int,
num_experts: int, num_tokens: int, bf16_weight: bool
) -> tuple[int, int, int]:
if (hidden_size, num_experts) == (4096, 8):
return 1, 1024, 4
"""Return (BLOCK_M, BLOCK_K, num_warps), swept on MI355 for M<=32.
One program computes BLOCK_M tokens for one expert, so narrow routers keep
BLOCK_M=1 to fill the GPU and wide routers grow BLOCK_M to reuse weights.
"""
if bf16_weight:
return (1 if num_tokens <= 4 else 2), 1024, 2
if num_experts < 64:
return 1, 1024, 4
wide = num_experts > 256
if num_tokens <= 2:
return 1, 512, 2
if num_tokens <= 4:
return 1, 1024, 8
return (2, 1024, 2) if wide else (2, 2048, 4)
if num_tokens <= 8:
return (4, 512, 2) if wide else (4, 1024, 4)
if num_tokens <= 16:
block_m = 8 if (hidden_size, num_experts) == (6144, 256) else 4
return block_m, 1024, 8
if (hidden_size, num_experts) == (3072, 256):
return 8, 1024, 8
return 4, 2048, 8
return (8, 512, 2) if wide else (4, 1024, 4)
if num_experts >= 768:
return 16, 256, 2
return (8, 256, 2) if wide else (4, 512, 2)
def can_use_rocm_fp32_router_gemm(
hidden_states: torch.Tensor,
router_weight: torch.Tensor,
) -> bool:
"""Return whether the tensors match the tuned gfx950 fast path."""
"""Return whether the tensors can use the gfx950 fast path."""
try:
_validate_inputs(hidden_states, router_weight)
except (RuntimeError, ValueError):
@@ -96,26 +115,30 @@ def _validate_inputs(
raise ValueError("hidden_states and router_weight must be 2D tensors")
if hidden_states.dtype not in (torch.bfloat16, torch.float32):
raise ValueError("hidden_states must have dtype bfloat16 or float32")
if router_weight.dtype != torch.float32:
raise ValueError("router_weight must have dtype float32")
if router_weight.dtype not in (torch.float32, torch.bfloat16):
raise ValueError("router_weight must have dtype float32 or bfloat16")
if hidden_states.device.type != "cuda" or router_weight.device.type != "cuda":
raise ValueError("hidden_states and router_weight must be GPU tensors")
if hidden_states.device != router_weight.device:
raise ValueError("hidden_states and router_weight must be on the same device")
if not hidden_states.is_contiguous() or not router_weight.is_contiguous():
raise ValueError("hidden_states and router_weight must be contiguous")
shape = (hidden_states.shape[1], router_weight.shape[0])
if (
shape not in ROCM_FP32_ROUTER_GEMM_SUPPORTED_SHAPES
or router_weight.shape[1] != shape[0]
):
raise ValueError(
"supported (hidden_size, num_experts) shape pairs are "
"(3072, 256), (4096, 8), (4096, 192), (6144, 128), "
"and (6144, 256)"
)
if router_weight.shape[1] != hidden_states.shape[1]:
raise ValueError("router_weight and hidden_states hidden sizes differ")
if not 0 <= hidden_states.shape[0] <= _MAX_TOKENS:
raise ValueError(f"num_tokens must be in [0, {_MAX_TOKENS}]")
if hidden_states.shape[0] * router_weight.numel() >= _MAX_MACS:
raise ValueError(
f"num_tokens * num_experts * hidden_size must be < {_MAX_MACS}"
)
if (
router_weight.dtype == torch.bfloat16
and hidden_states.shape[0] * router_weight.shape[0] > _MAX_BF16_TOKEN_EXPERTS
):
raise ValueError(
f"num_tokens * num_experts must be <= {_MAX_BF16_TOKEN_EXPERTS} "
"for bfloat16 weights"
)
def rocm_fp32_router_gemm(
@@ -131,7 +154,10 @@ def rocm_fp32_router_gemm(
if num_tokens == 0:
return output
block_m, block_k, num_warps = _launch_config(hidden_size, num_experts, num_tokens)
block_m, block_k, num_warps = _launch_config(
num_experts, num_tokens, router_weight.dtype == torch.bfloat16
)
block_k = min(block_k, triton.next_power_of_2(hidden_size))
grid = (triton.cdiv(num_tokens, block_m) * num_experts,)
_rocm_fp32_router_gemm_kernel[grid](
hidden_states,
@@ -142,6 +168,7 @@ def rocm_fp32_router_gemm(
N=num_experts,
BLOCK_M=block_m,
BLOCK_K=block_k,
EVEN_K=hidden_size % block_k == 0,
num_warps=num_warps,
num_stages=1,
)
@@ -325,6 +325,7 @@ def _fp8_quantize(
A_scale: torch.Tensor | None,
per_act_token: bool,
block_shape: list[int] | None = None,
is_scale_transposed: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Perform fp8 quantization on the inputs. If a block_shape
is provided, the output will be blocked.
@@ -339,7 +340,9 @@ def _fp8_quantize(
assert not per_act_token
assert len(block_shape) == 2
_, block_k = block_shape[0], block_shape[1]
A, A_scale = per_token_group_quant_fp8(A, block_k)
A, A_scale = per_token_group_quant_fp8(
A, block_k, column_major_scales=is_scale_transposed
)
assert cdiv(A.size(-1), block_k) == A_scale.size(-1)
return A, A_scale
@@ -462,6 +465,7 @@ def moe_kernel_quantize_input(
per_act_token_quant: bool,
block_shape: list[int] | None = None,
is_scale_swizzled: bool = True,
is_scale_transposed: bool = False,
quantization_emulation: bool = False,
mx_alignment: int = 0,
) -> tuple[torch.Tensor, torch.Tensor | None]:
@@ -469,7 +473,9 @@ def moe_kernel_quantize_input(
if quantization_emulation:
return _fp8_quantize_dequantize(A, A_scale)
else:
return _fp8_quantize(A, A_scale, per_act_token_quant, block_shape)
return _fp8_quantize(
A, A_scale, per_act_token_quant, block_shape, is_scale_transposed
)
elif quant_dtype == torch.int8:
if quantization_emulation:
raise NotImplementedError(
+3
View File
@@ -31,6 +31,7 @@ from vllm.model_executor.layers.quantization.base_config import (
)
from vllm.model_executor.layers.utils import (
dispatch_unquantized_gemm,
maybe_pad_weight_avoid_cache_aliasing,
)
from vllm.model_executor.parameter import (
BasevLLMParameter,
@@ -225,6 +226,8 @@ class UnquantizedLinearMethod(LinearMethodBase):
and weight.stride(0) != 1
):
layer.weight.data = weight.t().contiguous().t()
elif current_platform.is_rocm():
layer.weight.data = maybe_pad_weight_avoid_cache_aliasing(layer.weight.data)
def apply(
self,
@@ -479,6 +479,8 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
input_ids=input_ids,
hash_indices_table=layer.hash_indices_table,
)
@@ -1132,4 +1134,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
input_ids=input_ids,
hash_indices_table=layer.hash_indices_table,
)
@@ -0,0 +1,204 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Online-quantized copies of an already-loaded lm_head."""
from itertools import chain
from typing import Literal
from weakref import WeakKeyDictionary
import torch
from torch.nn import Module, Parameter
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import init_nvfp4_linear_kernel
from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PtpcOnlineLinearMethod,
OnlineLinearBase,
)
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
FLOAT4_E2M1_MAX,
ref_nvfp4_quant,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import weight_amax
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
UnquantizedEmbeddingMethod,
)
from vllm.model_executor.model_loader.reload.layerwise import get_layerwise_info
from vllm.model_executor.utils import replace_parameter
from vllm.utils.torch_utils import set_default_torch_dtype
logger = init_logger(__name__)
def _quantize_nvfp4(
weight: torch.Tensor, chunk_rows: int = 8192
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""NVFP4-quantize the rows of `weight` without the SM100+ quant kernel.
Returns packed E2M1 codes (low nibble first), E4M3 scales per 16 values and
the fp32 dequantization global scale, in the checkpoint layout.
"""
fp8_max = torch.finfo(torch.float8_e4m3fn).max
global_scale = (weight_amax(weight).float() / (FLOAT4_E2M1_MAX * fp8_max)).clamp(
min=1e-12
)
inv_global_scale = (1.0 / global_scale).reshape(())
# E2M1 magnitudes 0, 0.5, ..., 6 doubled to integers, mapped to their codes.
code_of = torch.zeros(13, dtype=torch.uint8, device=weight.device)
code_of[torch.tensor([0, 1, 2, 3, 4, 6, 8, 12])] = torch.arange(
8, dtype=torch.uint8
).to(weight.device)
n, k = weight.shape
packed = torch.empty(n, k // 2, dtype=torch.uint8, device=weight.device)
scales = torch.empty(n, k // 16, dtype=torch.float8_e4m3fn, device=weight.device)
for start in range(0, n, chunk_rows):
rows = slice(start, start + chunk_rows)
values, block_scales = ref_nvfp4_quant(weight[rows], inv_global_scale, 16)
codes = code_of[(values.abs() * 2).long()] | ((values < 0).to(torch.uint8) << 3)
packed[rows] = codes[:, 0::2] | (codes[:, 1::2] << 4)
scales[rows] = block_scales.to(torch.float8_e4m3fn)
return packed, scales, global_scale
class _LMHeadNvfp4Method(OnlineLinearBase):
"""Weight-only NVFP4 lm_head copy on the platform's W4A16 kernel;
activations stay in the model dtype."""
def process_weights_after_loading(self, layer: Module) -> None:
packed, scales, global_scale = _quantize_nvfp4(layer.weight)
replace_parameter(layer, "weight", packed)
layer.weight_scale = Parameter(scales, requires_grad=False)
layer.weight_global_scale = Parameter(global_scale, requires_grad=False)
self.kernel = init_nvfp4_linear_kernel(use_a16=True)
self.kernel.process_weights_after_loading(layer)
def apply(
self, layer: Module, x: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
return self.kernel.apply_weights(layer, x, bias)
_LM_HEAD_METHODS: dict[str, type[OnlineLinearBase]] = {
"fp8": Fp8PtpcOnlineLinearMethod,
"nvfp4": _LMHeadNvfp4Method,
}
def _quantize_copy(lm_head: ParallelLMHead, quantization: str) -> ParallelLMHead:
if (
not isinstance(
lm_head.quant_method, (UnquantizedEmbeddingMethod, UnquantizedLinearMethod)
)
or getattr(lm_head, "bias", None) is not None
or lm_head.weight.dtype not in (torch.float16, torch.bfloat16)
):
raise ValueError(
"Quantizing an lm_head copy needs an unquantized fp16/bf16 lm_head "
"without bias."
)
# Online methods take their output dtype from the default dtype, which is
# only the model dtype while the model itself is being built.
dtype = lm_head.weight.dtype
with set_default_torch_dtype(dtype):
head = ParallelLMHead(
lm_head.num_embeddings,
lm_head.embedding_dim,
params_dtype=dtype,
org_num_embeddings=lm_head.org_vocab_size,
padding_size=lm_head.padding_size,
prefix=lm_head.prefix,
disable_tp=lm_head.disable_tp,
quant_method=_LM_HEAD_METHODS[quantization](),
)
replace_parameter(head, "weight", lm_head.weight.data)
head.quant_method.process_weights_after_loading(head)
# The copy is never loaded. The online loader wrapper captures the layer,
# so keeping it would leave each copy to the cycle collector.
for param in head.parameters(recurse=False):
if hasattr(param, "weight_loader"):
del param.weight_loader
return head
# Each quantized copy maps to the head it is derived from. Weak keys, and no
# module attribute, so the copy never owns the source as a submodule.
_SOURCES: WeakKeyDictionary[Module, tuple[ParallelLMHead, str]] = WeakKeyDictionary()
def quantized_lm_head_copy(
lm_head: ParallelLMHead, quantization: Literal["fp8", "nvfp4"]
) -> ParallelLMHead:
"""A ParallelLMHead holding this rank's shard of `lm_head`, quantized.
"fp8" uses per-row weight and per-token activation scales on the platform's
FP8 linear kernel; "nvfp4" is weight-only on the
platform's W4A16 kernel. `lm_head` is left
unchanged and no bf16 copy of its weight is made. The copy is derived state:
`refresh_quantized_lm_heads` re-derives it after `lm_head` changes.
"""
head = _quantize_copy(lm_head, quantization)
# Keep it out of layerwise reload, which would re-run
# process_weights_after_loading on the already-quantized weight.
get_layerwise_info(head).reset()
_SOURCES[head] = (lm_head, quantization)
logger.info(
"Quantized a %s copy of the lm_head (%d x %d per rank).",
quantization,
head.output_size_per_partition,
head.input_size_per_partition,
)
return head
@torch.no_grad()
def refresh_quantized_lm_heads(model: Module) -> None:
"""Re-derive the quantized lm_head copies in `model` from their source heads.
Copies in place, so CUDA graphs keep pointing at valid tensors.
"""
for module in model.modules():
if module not in _SOURCES:
continue
fresh = _quantize_copy(*_SOURCES[module])
for name, tensor in chain(
fresh.named_parameters(recurse=False), fresh.named_buffers(recurse=False)
):
getattr(module, name).copy_(tensor)
# (parent, attribute, copy) for each copy swapped out by share_source_lm_heads.
_SWAPPED: dict[Module, list[tuple[Module, str, Module]]] = {}
def share_source_lm_heads(model: Module) -> None:
"""Replace each quantized copy in `model` with its source head.
`model` then loads weights like a drafter sharing the unquantized head,
until `restore_quantized_lm_heads` swaps the copies back in.
"""
swapped = [
(module, name, child)
for module in model.modules()
for name, child in module.named_children()
if child in _SOURCES
]
for module, name, child in swapped:
setattr(module, name, _SOURCES[child][0])
_SWAPPED[model] = swapped
def restore_quantized_lm_heads(model: Module, refresh: bool = True) -> None:
"""Undo `share_source_lm_heads`, then re-derive the copies in `model`.
Args:
model: The model passed to `share_source_lm_heads`, if any.
refresh: Whether to re-derive the copies; False when the source heads
may be mid-reload.
"""
for module, name, head in _SWAPPED.pop(model, ()):
setattr(module, name, head)
if refresh:
refresh_quantized_lm_heads(model)
@@ -1647,8 +1647,10 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
# If no native backend available, use emulation.
if self.mxfp4_backend is Mxfp4MoeBackend.NONE:
self.mxfp4_backend = Mxfp4MoeBackend.EMULATION
self.experts_cls = None
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
if self.experts_cls is None:
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
logger.info_once(
f"Using {self.mxfp4_backend.value} backend for {self.ocp_mx_scheme}"
@@ -379,6 +379,79 @@ def align_moe_weights_for_fi(
return padded_w13, padded_w2, padded_intermediate
def align_mxfp8_moe_weights_for_fi_trtllm(
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
is_act_and_mul: bool,
min_alignment: int,
hidden_alignment: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
"""align_moe_weights_for_fi for MXFP8 weights and their UE8M0 block scales
for the FlashInfer TRT-LLM MoE kernel, which also pads the hidden size to a
multiple of hidden_alignment.
Padded weights are zero and padded scales are 127 (UE8M0 for 1.0), so the
padded rows and columns contribute nothing. Scales may be 3D
[E, rows, cols] or flattened [E, rows * cols]; their layout is kept.
Returns the padded intermediate and hidden sizes.
"""
num_experts, hidden_size, intermediate = w2.shape
padded_hidden = round_up(hidden_size, hidden_alignment)
w13, w2, padded_intermediate = align_moe_weights_for_fi(
w13, w2, is_act_and_mul, min_alignment
)
if padded_intermediate == intermediate and padded_hidden == hidden_size:
return w13, w2, w13_scale, w2_scale, intermediate, hidden_size
if padded_hidden != hidden_size:
logger.info_once(
"Padding hidden size from %d to %d for MXFP8 TRTLLM MoE kernel.",
hidden_size,
padded_hidden,
)
padded_w13 = w13.new_zeros((num_experts, w13.shape[1], padded_hidden))
padded_w13[:, :, :hidden_size] = w13
padded_w2 = w2.new_zeros((num_experts, padded_hidden, w2.shape[2]))
padded_w2[:, :hidden_size] = w2
w13, w2 = padded_w13, padded_w2
up_mult = 2 if is_act_and_mul else 1
w13_scale_3d = w13_scale.reshape(num_experts, up_mult * intermediate, -1)
w2_scale_3d = w2_scale.reshape(num_experts, hidden_size, -1)
block = hidden_size // w13_scale_3d.shape[2]
assert padded_hidden % block == 0 and padded_intermediate % block == 0
# Pad each of the gate and up halves (or the single non-gated block) on its
# own, like the weights.
padded_w13_scale = w13_scale_3d.new_full(
(num_experts, up_mult * padded_intermediate, padded_hidden // block), 127
)
for i in range(up_mult):
dst = i * padded_intermediate
padded_w13_scale[:, dst : dst + intermediate, : w13_scale_3d.shape[2]] = (
w13_scale_3d[:, i * intermediate : (i + 1) * intermediate]
)
padded_w2_scale = w2_scale_3d.new_full(
(num_experts, padded_hidden, padded_intermediate // block), 127
)
padded_w2_scale[:, :hidden_size, : w2_scale_3d.shape[2]] = w2_scale_3d
if w13_scale.ndim == 2:
padded_w13_scale = padded_w13_scale.reshape(num_experts, -1)
if w2_scale.ndim == 2:
padded_w2_scale = padded_w2_scale.reshape(num_experts, -1)
return (
w13,
w2,
padded_w13_scale,
padded_w2_scale,
padded_intermediate,
padded_hidden,
)
def _shuffle_deepseek_fp8_moe_weights(
w13: torch.Tensor,
w2: torch.Tensor,
@@ -486,8 +559,9 @@ def prepare_fp8_moe_layer_for_fi(
Note that for trtllm we update the model state dict
with the scale format needed for these kernels.
Note that for per-tensor, we update the layer's
intermediate size if the weights needed padding.
Note that for per-tensor and for MXFP8 TRT-LLM, we update the layer's
intermediate size (and for MXFP8 TRT-LLM its hidden size) if the weights
needed padding.
"""
assert hasattr(layer.moe_config, "is_act_and_mul")
block_quant = (
@@ -499,6 +573,21 @@ def prepare_fp8_moe_layer_for_fi(
# MXFP8 TRT-LLM requires W31 swap + reorder + shuffle.
if is_mxfp8 and is_trtllm:
# The kernel needs intermediate % 128 == 0 and hidden % 256 == 0. The
# MoE runner pads the activations to the padded hidden size.
w13, w2, w13_scale, w2_scale, new_intermediate, new_hidden = (
align_mxfp8_moe_weights_for_fi_trtllm(
w13,
w2,
w13_scale,
w2_scale,
layer.moe_config.is_act_and_mul,
min_alignment=128,
hidden_alignment=256,
)
)
layer.moe_config.intermediate_size_per_partition = new_intermediate
layer.moe_config.hidden_dim = new_hidden
# FlashInfer TRT-LLM SwiGLU expects [up; gate] but vLLM stores
# [gate; up]. Swap both weights and scales before interleaving.
if layer.moe_config.is_act_and_mul:
@@ -13,6 +13,14 @@ from vllm.utils.torch_utils import direct_register_custom_op, is_torch_equal_or_
logger = init_logger(__name__)
def should_use_gfx1250_mx_scale_swizzle() -> bool:
if not current_platform.is_rocm():
return False
from vllm.platforms.rocm import on_gfx1250
return on_gfx1250()
def should_use_cdna4_mx_scale_swizzle() -> bool:
"""Whether to use the CDNA4 swizzled scale layout for mxfp4 on gfx950.
@@ -27,6 +35,15 @@ def should_use_cdna4_mx_scale_swizzle() -> bool:
return on_gfx950() and get_tensor_model_parallel_world_size() <= 2
def get_aiter_mx_scale_swizzle() -> str | None:
"""AITER `swizzle_mx_scale` arg matching the load-time scale swizzle."""
if should_use_gfx1250_mx_scale_swizzle():
return "GFX1250_SCALE"
if should_use_cdna4_mx_scale_swizzle():
return "CDNA4_SCALE"
return None
def weight_mx_scale(precision_config):
"""mxfp4 weight scale from a triton_kernels PrecisionConfig: 3.8 exposes it
as b_mx_scale, 3.5.1/3.6 as weight_scale.
@@ -136,6 +153,17 @@ def _swizzle_mxfp4(quant_tensor, scale, num_warps=8):
# transpose the tensor so that the quantization axis is on dim1
quant_tensor = quant_tensor.transpose(-2, -1)
scale = scale.transpose(-2, -1)
if should_use_gfx1250_mx_scale_swizzle():
from aiter.ops.triton.utils.shuffle import shuffle_scale_moe
assert (
scale.dim() == 3 and scale.shape[-1] % 32 == 0 and scale.shape[-2] % 4 == 0
), f"GFX1250 scale swizzle needs (E,K_SCALE%4,N%32); got {tuple(scale.shape)}"
scale = shuffle_scale_moe(
scale, arch="gfx1250", preshuffle_factor=32, scale_kwidth=4
)
quant_tensor = convert_layout(
wrap_torch_tensor(quant_tensor, dtype=FP4), value_layout, **value_layout_opts
)
+77 -18
View File
@@ -225,6 +225,8 @@ direct_register_custom_op(
def use_aiter_triton_gemm(n, m, k, dtype):
from vllm.platforms.rocm import on_gfx1250
if (
not rocm_aiter_ops.is_triton_gemm_enabled()
# MI300's - fp8nuz=True
@@ -233,6 +235,9 @@ def use_aiter_triton_gemm(n, m, k, dtype):
):
return False
if on_gfx1250():
return True
# use hipblaslt for the larger GEMMs
if n > 2048 and m > 512:
return False
@@ -285,6 +290,44 @@ def wvsplitkrc_dispatch(n: int, k: int, m: int, cu_count: int) -> tuple[int, boo
return chunkk, fits
# On gfx11 a weight row stride that is a multiple of 2048 B makes the multi-row global
# loads of a GEMM collide on the L2/MALL channel hash. Offsetting the stride by one
# 128 B cache line dodges the collision and keeps each row cache-line aligned.
def cache_aliasing_pad_elems(k: int, element_size: int) -> int:
"""Row-stride pad, in elements, that stops an ``[n, k]`` weight aliasing on gfx11,
or 0 when the stride does not alias.
Args:
k: Number of elements per weight row.
element_size: Size of one element in bytes.
Returns:
The number of elements to add to the row stride.
"""
if (k * element_size) % 2048 != 0:
return 0
return 128 // element_size
def maybe_pad_weight_avoid_cache_aliasing(weight: torch.Tensor) -> torch.Tensor:
"""Offset a dense weight's row stride so it stops aliasing.
Returns a view with the original shape whose row stride is one cache line
wider, or ``weight`` unchanged when the pad does not apply.
"""
from vllm.platforms.rocm import on_gfx11
if not (current_platform.is_rocm() and on_gfx11()):
return weight
if weight.dim() != 2 or not weight.is_contiguous():
return weight
pad = cache_aliasing_pad_elems(weight.shape[1], weight.element_size())
if pad == 0:
return weight
return torch.nn.functional.pad(weight, (0, pad))[..., :-pad]
def rocm_unquantized_gemm_impl(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None
) -> torch.Tensor:
@@ -295,7 +338,7 @@ def rocm_unquantized_gemm_impl(
return linear_batch_invariant(x, weight, bias)
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx950, on_gfx1250
from vllm.platforms.rocm import on_gfx1x, on_gfx9, on_gfx11, on_gfx950
n = x.numel() // x.size(-1)
m = weight.shape[0]
@@ -328,38 +371,54 @@ def rocm_unquantized_gemm_impl(
x_view = x.reshape(-1, x.size(-1)).contiguous()
return ops.wvSplitKrc(x_view, weight, cu_count, bias)
# gfx1250's aiter gemm_a16w16 uses the gluon backend, which requires
# K % 256 == 0 (it walks K with fixed-size descriptors and won't pad a
# partial last tile). Some whitelisted shapes have K=2880 (e.g. gpt-oss-120b
# hidden), so skip aiter there and fall back to the torch GEMM path below.
if use_aiter_triton_gemm(n, m, k, x.dtype) and not (on_gfx1250() and k % 256 != 0):
if use_aiter_triton_gemm(n, m, k, x.dtype):
from aiter.ops.triton.gemm_a16w16 import gemm_a16w16
return gemm_a16w16(x, weight, bias)
use_skinny = (
skinny_gemm_enabled = (
envs.VLLM_ROCM_USE_SKINNY_GEMM
and (on_gfx9() or on_gfx1x())
# build (gfx9/gfx11 ISA); fall back to torch GEMM there.
# TODO GFX1250: Include once skinny GEMM is supported on gfx1250
and x.dtype in [torch.float16, torch.bfloat16]
and k % 8 == 0
and skinny_operands_compatible
)
if use_skinny:
# wvSplitK allows row strides. Perf impact is only validated on gfx11.
weight_ok_for_wvsplitk = weight.is_contiguous() or (
on_gfx11() and weight.stride(1) == 1
)
use_wvsplitk = (
skinny_gemm_enabled
and weight_ok_for_wvsplitk
and (bias is None or bias.is_contiguous())
and (m == 1 or m > 8)
and 0 < n <= 5
)
use_llmm1 = (
skinny_gemm_enabled
and weight.is_contiguous()
and bias is None
and m % 4 == 0
and n == 1
and k <= 8192
)
if use_wvsplitk:
# The skinny kernels assume contiguous K elements. A shape-preserving
# reshape can retain a transposed activation's non-contiguous strides.
# Note: Only build that view inside the branches that consume it.
if (m == 1 or m > 8) and 0 < n <= 5:
x_view = x.reshape(-1, x.size(-1)).contiguous()
cu_count = num_compute_units()
out = ops.wvSplitK(weight, x_view, cu_count, bias)
return out.reshape(*x.shape[:-1], weight.shape[0])
elif m % 4 == 0 and n == 1 and k <= 8192 and bias is None:
x_view = x.reshape(-1, x.size(-1)).contiguous()
out = ops.LLMM1(weight, x_view, 4)
return out.reshape(*x.shape[:-1], weight.shape[0])
x_view = x.reshape(-1, x.size(-1)).contiguous()
out = ops.wvSplitK(weight, x_view, cu_count, bias)
return out.reshape(*x.shape[:-1], weight.shape[0])
if use_llmm1:
x_view = x.reshape(-1, x.size(-1)).contiguous()
out = ops.LLMM1(weight, x_view, 4)
return out.reshape(*x.shape[:-1], weight.shape[0])
if rocm_aiter_ops.is_tgemm_enabled():
from aiter.tuned_gemm import tgemm
@@ -607,6 +607,7 @@ class ParallelLMHead(VocabParallelEmbedding):
prefix: str = "",
*,
disable_tp: bool = False,
quant_method: QuantizeMethodBase | None = None,
):
self.has_bias = bias
super().__init__(
@@ -618,6 +619,7 @@ class ParallelLMHead(VocabParallelEmbedding):
quant_config,
prefix,
disable_tp=disable_tp,
quant_method=quant_method,
)
self.quant_config = quant_config
if bias:
+1 -1
View File
@@ -992,7 +992,7 @@ class Qwen4ExpForConditionalGenerationConfig(Qwen3_5ForConditionalGenerationConf
parallel_config = vllm_config.parallel_config
# MoE SP also enables HC SP without the explicit HC flag.
if (
current_platform.is_cuda()
current_platform.is_cuda_alike()
and parallel_config.use_sequence_parallel_moe
and parallel_config.pipeline_parallel_size != 1
):
+5
View File
@@ -81,3 +81,8 @@ class D3ForSequenceClassification(_D3ForSequenceClassificationBase):
allow_patterns_overrides=[READOUT_WEIGHTS],
)
]
def is_unused_checkpoint_weight(self, name: str) -> bool:
# The readout's only tensor, loaded through `secondary_weights`. A
# checkpoint without a weight index lists its file with the backbone's.
return name == "weight"
+193 -4
View File
@@ -4,9 +4,10 @@
# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py
"""Inference-only Deepseek-VL2 model compatible with HuggingFace weights."""
import functools
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, TypeAlias
from collections.abc import Hashable, Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, TypeAlias
import torch
import torch.nn as nn
@@ -51,7 +52,12 @@ from vllm.transformers_utils.configs.deepseek_vl2 import (
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import set_default_torch_dtype
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
from .interfaces import (
MultiModalEmbeddings,
SupportsEncoderCudaGraph,
SupportsMultiModal,
SupportsPP,
)
from .utils import (
AutoWeightsLoader,
WeightsMapper,
@@ -319,7 +325,9 @@ class DeepseekVL2MultiModalProcessor(
info=DeepseekVL2ProcessingInfo,
dummy_inputs=DeepseekVL2DummyInputsBuilder,
)
class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
class DeepseekVLV2ForCausalLM(
nn.Module, SupportsMultiModal, SupportsPP, SupportsEncoderCudaGraph
):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"language.": "language_model.",
@@ -340,6 +348,7 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
multimodal_config = vllm_config.model_config.multimodal_config
self.config = config
self.model_config = vllm_config.model_config
self.multimodal_config = multimodal_config
self.vision_config = config.vision_config
@@ -583,6 +592,186 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
pixel_values=pixel_values, images_spatial_crop=images_spatial_crop
)
# -- SupportsEncoderCudaGraph protocol methods --
@functools.cached_property
def _proj_hw(self) -> tuple[int, int]:
"""Return (h, w) of the projector output per tile."""
h = w = math.ceil(
(self.vision_config.image_size // self.vision_config.patch_size)
/ self.projector_config.downsample_ratio
)
return h, w
def get_encoder_cudagraph_config(self):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphConfig
return EncoderCudaGraphConfig(
modalities=["image"],
buffer_keys=["pixel_values"],
out_hidden_size=self.projector_config.n_embed,
)
def get_encoder_cudagraph_budget_range(
self, vllm_config: "VllmConfig"
) -> tuple[int, int]:
h, w = self._proj_hw
# Smallest valid image: 1 global + 1×1 local tile → 2 tiles total.
min_budget = h * (w + 1) * 2 + 1
max_budget = min(
vllm_config.scheduler_config.max_num_batched_tokens,
self.model_config.max_model_len,
)
return min_budget, max_budget
def get_encoder_cudagraph_item_specs(self, mm_kwargs: dict[str, Any]):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderItemSpec
h, w = self._proj_hw
specs = []
for row in mm_kwargs["images_spatial_crop"].tolist():
tw, th = int(row[0]), int(row[1])
if tw == 0 or th == 0:
break
n_tiles = 1 + tw * th
output_tokens = h * (w + 1) + th * h * (tw * w + 1) + 1
specs.append(
EncoderItemSpec(input_size=n_tiles, output_tokens=output_tokens)
)
return specs
def select_encoder_cudagraph_items(
self, mm_kwargs: dict[str, Any], indices: list[int]
) -> dict[str, Any]:
pixel_values = mm_kwargs["pixel_values"]
images_spatial_crop = mm_kwargs["images_spatial_crop"]
if not indices:
return {
"pixel_values": pixel_values[:0],
"images_spatial_crop": images_spatial_crop[:0],
}
# Build per-image tile counts (1 global + tw*th local).
tiles_per_image = []
for row in images_spatial_crop.tolist():
tw, th = int(row[0]), int(row[1])
if tw == 0 or th == 0:
break
tiles_per_image.append(1 + tw * th)
cum_tiles = [0]
for t in tiles_per_image:
cum_tiles.append(cum_tiles[-1] + t)
selected_pv = torch.cat(
[pixel_values[cum_tiles[i] : cum_tiles[i + 1]] for i in indices]
)
return {
"pixel_values": selected_pv,
"images_spatial_crop": images_spatial_crop[list(indices)],
}
def prepare_encoder_cudagraph_capture_inputs(
self,
token_budget: int,
max_batch_size: int,
max_frames_per_batch: int,
device: torch.device,
dtype: torch.dtype,
path: str = "default",
axis_keys: tuple[Hashable, ...] | None = None,
):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphCaptureInputs
h, w = self._proj_hw
# Each tile contributes at least h*w output tokens (before newlines).
# Using h*w (not h*(w+1)) guarantees we never under-allocate.
max_tiles = (token_budget + h * w - 1) // (h * w)
dummy_pixel_values = torch.randn(
max_tiles,
3,
self.vision_config.image_size,
self.vision_config.image_size,
device=device,
dtype=dtype,
)
return EncoderCudaGraphCaptureInputs(
values={"pixel_values": dummy_pixel_values}
)
def prepare_encoder_cudagraph_replay_buffers(
self,
mm_kwargs: dict[str, Any],
max_batch_size: int,
max_frames_per_batch: int,
path: str = "default",
):
from vllm.v1.worker.encoder_cudagraph_defs import EncoderCudaGraphReplayBuffers
return EncoderCudaGraphReplayBuffers(
values={"pixel_values": mm_kwargs["pixel_values"]}
)
def encoder_cudagraph_forward(
self, values: dict[str, torch.Tensor], path: str = "default"
) -> torch.Tensor:
pixel_values = values["pixel_values"]
features = self.vision.forward_features(pixel_values)
return self.projector(features)
def postprocess_encoder_output(
self,
outputs: dict[str, torch.Tensor],
indices: list[int],
per_item_out_tokens: list[int],
dest: dict[int, torch.Tensor] | list[torch.Tensor | None],
clone: bool = False,
batch_mm_kwargs: dict[str, Any] | None = None,
) -> None:
"""Assemble per-image embeddings from raw per-tile projector output.
``outputs["default"]`` has shape ``[N_tiles_captured, hw, n_embed]``.
This method re-applies newlines and the view-separator that the
eager ``_pixel_values_to_embedding`` path inserts in-line.
"""
assert batch_mm_kwargs is not None
output = outputs["default"]
images_spatial_crop = batch_mm_kwargs["images_spatial_crop"]
_, hw, n_dim = output.shape
h = w = int(hw**0.5)
tile_offset = 0
for rank, img_idx in enumerate(indices):
tw, th = (
int(images_spatial_crop[rank][0]),
int(images_spatial_crop[rank][1]),
)
n_tiles = 1 + tw * th
tiles = output[tile_offset : tile_offset + n_tiles]
tile_offset += n_tiles
# Global tile: add one newline per row.
global_f = tiles[0].view(h, w, n_dim)
newline_g = self.image_newline.view(1, 1, n_dim).expand(h, 1, n_dim)
global_f = torch.cat([global_f, newline_g], dim=1).view(-1, n_dim)
# Local tiles: rearrange grid, then add one newline per row.
local_f = tiles[1:].view(th, tw, h, w, n_dim)
local_f = local_f.permute(0, 2, 1, 3, 4).reshape(th * h, tw * w, n_dim)
newline_l = self.image_newline.view(1, 1, n_dim).expand(th * h, 1, n_dim)
local_f = torch.cat([local_f, newline_l], dim=1).view(-1, n_dim)
if self.global_view_pos == "head":
emb = torch.cat([global_f, self.view_seperator[None], local_f])
else:
emb = torch.cat([local_f, self.view_seperator[None], global_f])
if isinstance(dest, dict):
dest[img_idx] = emb.clone() if clone else emb
else:
dest[rank] = emb.clone() if clone else emb
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
image_input = self._parse_and_validate_image_input(**kwargs)
if image_input is None:
+1 -10
View File
@@ -14,7 +14,6 @@ from tqdm import tqdm
import vllm.envs as envs
from vllm.distributed.parallel_state import (
get_dp_group,
get_ep_group,
is_global_first_rank,
)
from vllm.forward_context import ForwardContext, override_forward_context
@@ -356,7 +355,6 @@ def _get_fp8_fp4_mega_moe_m_values(
dg = _import_deep_gemm()
gate, experts = module.gate, module.experts
uses_mega_gate = _mega_moe_uses_mega_gate(module)
num_max_tokens_per_rank = experts.get_symm_buffer().num_max_tokens_per_rank
if module.use_sequence_parallel:
max_tokens = cdiv(max_tokens, module.tp_size)
@@ -369,14 +367,7 @@ def _get_fp8_fp4_mega_moe_m_values(
if uses_mega_gate and hasattr(dg, "get_bf16_mega_gate_config")
else {}
)
block_m = dg.get_block_m_for_mega_moe(
get_ep_group().world_size,
experts.num_experts,
num_max_tokens_per_rank,
num_tokens,
experts.top_k,
"fp8xfp4",
)
block_m = experts.get_block_m(num_tokens)
# Keep the largest M: DeepSeek-V4 only calls bf16_mega_gate above a
# small token threshold, and a config's largest M is always above it.
m_values[(tuple(gate_config.items()), block_m)] = num_tokens
@@ -30,6 +30,7 @@ _DEEPSEEK_V4_SPARSE_MLA_BACKENDS = frozenset(
"ROCM_FLASHMLA_SPARSE_DSV4",
"FLASHMLA_SPARSE_DSV41",
"FLASHINFER_MLA_SPARSE_DSV41",
"ROCM_FLASHMLA_SPARSE_DSV41",
"DEEPSEEK_SPARSE_SWA",
}
)