Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 22:06:10 +07:00
327 changed files with 13704 additions and 5433 deletions
@@ -31,6 +31,9 @@ from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_one
from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_two_sided import ( # noqa: E501
FlashInferNVLinkTwoSidedPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.passthrough import (
PassThroughPrepareAndFinalize,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import (
has_deep_ep,
@@ -185,6 +188,10 @@ def maybe_make_prepare_finalize(
all2all_manager: Any | None = None,
input_dtype: torch.dtype | None = None,
) -> FusedMoEPrepareAndFinalize | None:
if moe.moe_parallel_config.use_passthrough_all2all:
# The experts dispatch and combine themselves, in every EP topology.
return PassThroughPrepareAndFinalize()
if not moe.moe_parallel_config.use_all2all_kernels:
if not allow_new_interface:
return None
@@ -1068,6 +1068,12 @@ class FusedMoEParallelConfig:
and self.all2all_backend == "allgather_reducescatter"
)
@property
def use_passthrough_all2all(self):
# Not gated on use_all2all_kernels: the experts dispatch and combine
# internally in every EP topology, including TP-only.
return self.all2all_backend == "passthrough"
@property
def use_mori_kernels(self):
return self.use_all2all_kernels and self.all2all_backend in (
@@ -1530,6 +1536,10 @@ class FusedMoEConfig:
def use_deepep_ll_kernels(self):
return self.moe_parallel_config.use_deepep_ll_kernels
@property
def use_passthrough_all2all(self):
return self.moe_parallel_config.use_passthrough_all2all
@property
def use_mori_kernels(self):
return self.moe_parallel_config.use_mori_kernels
@@ -374,6 +374,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
routing_method = self.moe_config.routing_method
if routing_method == RoutingMethodType.DeepseekV4:
@@ -289,6 +289,7 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.moe_config.intermediate_size_per_partition_unpadded is not None
assert self.moe_config.hidden_dim_unpadded is not None
@@ -198,6 +198,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Apply the monolithic 4-bit INT MoE forward pass.
@@ -215,6 +216,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: Bias for expert scores
routed_scaling_factor: Scaling factor for routing
topk_group: Group size for topk
routing_replay_out: Unused; this kernel does not capture routing
Returns:
Output tensor after MoE computation
@@ -0,0 +1,210 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlashInfer MoE-EP megakernels as a modular-kernel experts implementation.
The megakernel routes on vLLM's top-k output and then dispatches, computes and
combines in one launch, so it is paired with a pass-through prepare/finalize.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
FlashInferMoeEp,
FlashInferMoeEpEpilogue,
FlashInferMoeEpWeights,
apply_topk_in_fc1,
flashinfer_moe_ep_unsupported_reasons,
supports_current_device,
validate_flashinfer_moe_ep_layer,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
kNvfp4Static,
require_finite_positive_scales,
)
from vllm.model_executor.utils import replace_parameter
if TYPE_CHECKING:
pass
# Layer parameters the megakernel reads. After ``process_weights_after_loading``
# they alias the megakernel's own tensors, so EPLB permutes exactly what it uses.
KERNEL_WEIGHT_NAMES = ("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale")
_supported_weight_quant_schemes: frozenset[QuantKey] = frozenset(
(kNvfp4Static, kMxfp4Static)
)
_supported_activations: frozenset[MoEActivation] = frozenset((MoEActivation.SILU,))
class FlashInferMoeEpExperts(mk.FusedMoEExpertsModular):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
) -> None:
super().__init__(moe_config, quant_config)
self._adapter: FlashInferMoeEp | None = None
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
supported, reason = mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
if not supported:
return False, reason
reasons = flashinfer_moe_ep_unsupported_reasons(
moe_config, weight_key, activation_key
)
if reasons:
return False, f"kernel does not support {', '.join(reasons)}"
return True, None
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return supports_current_device()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return weight_key in _supported_weight_quant_schemes
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in _supported_activations
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
return (
moe_parallel_config.use_ep
and not moe_parallel_config.use_batched_activation_format
)
@property
def expects_unquantized_inputs(self) -> bool:
return True
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
return (0,), (0,), (M, K)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Build the megakernel from the layer's canonical weights.
The oracle has already put the weights in kernel format; the per-expert
weight global scales arrive through the quant config and become the
epilogue alphas.
"""
validate_flashinfer_moe_ep_layer(layer) # type: ignore[arg-type]
adapter = FlashInferMoeEp(
self.moe_config,
FlashInferMoeEpWeights(
w13=layer.w13_weight,
w2=layer.w2_weight,
w13_scale=getattr(layer, "w13_weight_scale", None),
w2_scale=getattr(layer, "w2_weight_scale", None),
),
epilogue_from_quant_config(self.quant_config),
apply_topk_in_fc1=apply_topk_in_fc1(
self.moe_config,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
),
)
adapter.warmup()
for name, tensor in zip(KERNEL_WEIGHT_NAMES, adapter.kernel_weights()):
replace_parameter(layer, name, tensor)
self._adapter = adapter
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
if self._adapter is None:
raise RuntimeError("FlashInfer MoE-EP weights have not been processed")
output.copy_(
self._adapter(hidden_states, topk_ids.to(torch.int32), topk_weights)
)
def epilogue_from_quant_config(
quant_config: FusedMoEQuantConfig,
) -> FlashInferMoeEpEpilogue:
"""Per-expert epilogue constants from the canonical quant config.
The weight global scales (``g1_alphas``/``g2_alphas``) become the fc1/fc2
alphas. Activations are quantized dynamically inside the megakernel, so the
checkpoint's input scales drop out. MXFP4 checkpoints carry no global scales
and keep the default epilogue. The alphas alias the layer's parameters, so
EPLB permutations reach the kernel.
"""
fc1_alpha, fc2_alpha = quant_config.g1_alphas, quant_config.g2_alphas
if fc1_alpha is None and fc2_alpha is None:
return FlashInferMoeEpEpilogue()
assert fc1_alpha is not None and fc2_alpha is not None
require_finite_positive_scales("g1_alphas", fc1_alpha)
require_finite_positive_scales("g2_alphas", fc2_alpha)
if fc1_alpha.dim() != 1 or fc2_alpha.dim() != 1:
raise ValueError("the megakernel needs one weight global scale per expert")
return FlashInferMoeEpEpilogue(
fc1_alpha=fc1_alpha.float().contiguous(),
fc2_alpha=fc2_alpha.float().contiguous(),
)
@@ -1580,6 +1580,7 @@ class OAITritonMxfp4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
return triton_kernel_moe_forward(
hidden_states=hidden_states,
@@ -272,6 +272,7 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
import flashinfer
@@ -282,10 +283,6 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
# dummy 0-token forward has to keep the finalized (empty) form.
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=num_tokens,
device=hidden_states.device,
)
flashinfer_output = flashinfer.fused_moe.trtllm_bf16_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -312,5 +309,4 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
num_tokens=num_tokens,
top_k=self.topk,
)
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
return routed_output
@@ -450,6 +450,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
import flashinfer
from flashinfer.fused_moe import Fp8QuantizationType, WeightLayout
@@ -487,11 +488,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
n_group = num_expert_group or 0
selected_topk_group = topk_group or 0
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
kwargs = dict(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -522,9 +518,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
if is_mxfp8 or activation == MoEActivation.RELU2_NO_MUL:
kwargs["activation_type"] = activation_type
result = flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return result
def _apply_per_tensor(
@@ -543,6 +536,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
# Delay import for non-CUDA.
import flashinfer
@@ -558,11 +552,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
else:
assert not apply_router_weight_on_input
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
out = flashinfer.fused_moe.trtllm_fp8_per_tensor_scale_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -586,9 +575,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
routing_replay_out=routing_replay_out,
)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return out
def apply(
@@ -607,6 +593,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
if self.quant_config.block_shape is not None:
return self._apply_block_scale(
@@ -623,6 +610,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_replay_out=routing_replay_out,
)
elif self.quant_config.is_per_tensor:
return self._apply_per_tensor(
@@ -638,6 +626,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
num_expert_group=num_expert_group,
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
routing_replay_out=routing_replay_out,
)
else:
raise NotImplementedError(
@@ -215,6 +215,7 @@ class TrtLlmMxfp4ExpertsMonolithic(
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
from flashinfer import trtllm_fp4_block_scale_moe
@@ -236,10 +237,6 @@ class TrtLlmMxfp4ExpertsMonolithic(
device=hidden_states.device,
)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=num_tokens,
device=hidden_states.device,
)
flashinfer_output = trtllm_fp4_block_scale_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -279,7 +276,6 @@ class TrtLlmMxfp4ExpertsMonolithic(
top_k=self.topk,
finalized_output=finalized_output,
)
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
return routed_output
@@ -139,6 +139,7 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.quantization.utils.flashinfer_mxint4_moe import ( # noqa: E501
flashinfer_trtllm_mxint4_moe,
@@ -147,10 +148,6 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
assert self.w1_scale is not None
assert self.w2_scale is not None
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
result = flashinfer_trtllm_mxint4_moe(
x=hidden_states,
router_logits=router_logits,
@@ -169,7 +166,4 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
routing_method_type=self.routing_method,
routing_replay_out=routing_replay_out,
)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return result
@@ -515,6 +515,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
import flashinfer
@@ -545,10 +546,6 @@ class TrtLlmNvFp4ExpertsMonolithic(
# dummy 0-token forward has to keep the finalized (empty) form.
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=num_tokens,
device=hidden_states.device,
)
# Invoke kernel.
# NOTE: Activation padding and output
# truncation are handled by the MoE runner's
@@ -592,5 +589,4 @@ class TrtLlmNvFp4ExpertsMonolithic(
num_tokens=num_tokens,
top_k=self.topk,
)
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
return routed_output
@@ -151,6 +151,7 @@ class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
if apply_router_weight_on_input:
raise NotImplementedError(
@@ -0,0 +1,481 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import contextlib
import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Literal, cast
import torch
from vllm.config import get_current_vllm_config
from vllm.config.kernel import (
FLASHINFER_MOE_EP_BACKENDS,
FLASHINFER_MOE_EP_CUTEDSL,
FLASHINFER_MOE_EP_DEEP_GEMM,
)
from vllm.distributed import get_ep_group
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
kNvfp4Static,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import _import_deep_gemm
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
FlashInferMoeEpKernel = Literal["cutedsl", "deep_gemm"]
FlashInferMoeEpWeightFormat = Literal["nvfp4", "mxfp4"]
_WEIGHT_FORMATS: dict[QuantKey | None, FlashInferMoeEpWeightFormat] = {
kNvfp4Static: "nvfp4",
kMxfp4Static: "mxfp4",
}
_SUPPORTED_CAPABILITIES = frozenset({(10, 0), (10, 3)})
def supports_current_device() -> bool:
capability = current_platform.get_device_capability()
return capability is not None and (
(capability.major, capability.minor) in _SUPPORTED_CAPABILITIES
)
_E2M1_LUT = (
0.0,
0.5,
1.0,
1.5,
2.0,
3.0,
4.0,
6.0,
-0.0,
-0.5,
-1.0,
-1.5,
-2.0,
-3.0,
-4.0,
-6.0,
)
@dataclass(frozen=True)
class FlashInferMoeEpBackendSpec:
kernel: FlashInferMoeEpKernel
weight_formats: frozenset[FlashInferMoeEpWeightFormat]
@dataclass(frozen=True)
class FlashInferMoeEpWeights:
w13: torch.Tensor
w2: torch.Tensor
w13_scale: torch.Tensor | None = None
w2_scale: torch.Tensor | None = None
@dataclass(frozen=True)
class FlashInferMoeEpEpilogue:
input_norm_const: float = 1.0
fc1_alpha: torch.Tensor | None = None
fc2_alpha: torch.Tensor | None = None
fc1_norm_const: torch.Tensor | None = None
_BACKEND_SPECS = {
FLASHINFER_MOE_EP_CUTEDSL: FlashInferMoeEpBackendSpec(
kernel="cutedsl",
weight_formats=frozenset({"nvfp4", "mxfp4"}),
),
FLASHINFER_MOE_EP_DEEP_GEMM: FlashInferMoeEpBackendSpec(
kernel="deep_gemm",
weight_formats=frozenset({"mxfp4"}),
),
}
assert frozenset(_BACKEND_SPECS) == FLASHINFER_MOE_EP_BACKENDS
def is_flashinfer_moe_ep_backend(moe_backend: str) -> bool:
return moe_backend in _BACKEND_SPECS
def flashinfer_moe_ep_backend_spec(
moe_backend: str,
) -> FlashInferMoeEpBackendSpec:
try:
return _BACKEND_SPECS[moe_backend]
except KeyError:
raise ValueError(
f"{moe_backend!r} is not a FlashInfer MoE-EP backend; expected "
f"one of {sorted(_BACKEND_SPECS)}"
) from None
def flashinfer_moe_ep_unsupported_reasons(
moe: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> tuple[str, ...]:
"""Deployment constraints of the megakernel beyond the framework's checks.
``activation_key`` is ``None`` for MXFP4 checkpoints (the kernel quantizes
bf16 activations itself) but marks a W4A16 recipe for NVFP4 weights.
"""
spec = flashinfer_moe_ep_backend_spec(moe.moe_backend)
unsupported: list[str] = []
weight_format = _WEIGHT_FORMATS.get(weight_key)
if weight_format is None:
unsupported.append(f"weight scheme {weight_key}")
elif weight_format not in spec.weight_formats:
unsupported.append(f"{weight_format.upper()} weights")
if weight_key == kNvfp4Static and activation_key is None:
unsupported.append("A16 activations with NVFP4 weights")
if moe.skip_final_all_reduce:
unsupported.append("skip_final_all_reduce")
if moe.in_dtype != torch.bfloat16:
unsupported.append(f"activation dtype {moe.in_dtype}")
if moe.has_bias:
unsupported.append("expert bias")
if moe.swiglu_alpha is not None or moe.swiglu_beta is not None:
unsupported.append("custom SwiGLU alpha or beta")
if (
spec.kernel == "deep_gemm"
and moe.routing_method is not RoutingMethodType.DeepseekV4
):
unsupported.append(f"routing method {moe.routing_method.name}")
vllm_config = get_current_vllm_config()
if vllm_config.weight_transfer_config is not None:
unsupported.append("runtime weight transfer")
if vllm_config.parallel_config.enable_dbo:
unsupported.append("dual batch overlap")
if spec.kernel == "deep_gemm" and vllm_config.parallel_config.enable_eplb:
unsupported.append("EPLB")
return tuple(unsupported)
def validate_flashinfer_moe_ep_layer(layer: RoutedExperts) -> None:
if layer.expert_map_manager.placement_strategy != "linear":
raise ValueError(
f"{layer.moe_config.moe_backend} requires contiguous linear "
"expert placement"
)
def apply_topk_in_fc1(
moe: FusedMoEConfig,
*,
apply_router_weight_on_input: bool = False,
) -> bool:
return (
apply_router_weight_on_input
or moe.routing_method is RoutingMethodType.DeepseekV4
)
def _dequant_mxfp4_ue8m0_gran32(
packed: torch.Tensor,
scale: torch.Tensor,
) -> torch.Tensor:
raw = packed.view(torch.uint8)
lut = torch.tensor(_E2M1_LUT, dtype=torch.float32, device=raw.device)
values = torch.empty(
raw.shape[0],
raw.shape[1] * 2,
dtype=torch.float32,
device=raw.device,
)
values[:, ::2] = lut[(raw & 0x0F).to(torch.int64)]
values[:, 1::2] = lut[(raw >> 4).to(torch.int64)]
encoded_scale = (
scale
if scale.dtype == torch.float8_e8m0fnu
else scale.view(torch.float8_e8m0fnu)
)
decoded_scale = encoded_scale.float()
return (values * decoded_scale.repeat_interleave(32, dim=-1)).to(torch.bfloat16)
def _dequant_mxfp4_experts(
weight: torch.Tensor,
scale: torch.Tensor,
) -> torch.Tensor:
num_experts, output_size, packed_input_size = weight.shape
result = torch.empty(
num_experts,
output_size,
packed_input_size * 2,
dtype=torch.bfloat16,
device=weight.device,
)
for expert_id in range(num_experts):
result[expert_id] = _dequant_mxfp4_ue8m0_gran32(
weight[expert_id], scale[expert_id]
)
return result
def mxfp4_moe_ep_weights(
moe_backend: str,
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
) -> FlashInferMoeEpWeights:
spec = flashinfer_moe_ep_backend_spec(moe_backend)
if spec.kernel == "deep_gemm":
return FlashInferMoeEpWeights(w13, w2, w13_scale, w2_scale)
return FlashInferMoeEpWeights(
w13=_dequant_mxfp4_experts(w13, w13_scale),
w2=_dequant_mxfp4_experts(w2, w2_scale),
)
class _FlashInferMoeEpApi:
def __init__(self) -> None:
try:
from flashinfer.moe_ep import (
BootstrapConfig,
DeepGemmMegaMoeConfig,
FleetParams,
MegaConfig,
MoEEpMegaLayer,
MoEEpTensors,
MoEWeightPack,
Nvfp4CutedslMegaMoeConfig,
)
except ImportError as err:
raise RuntimeError(
"FlashInfer MoE-EP requires the public API shipped with "
"FlashInfer 0.7.0 or newer"
) from err
self.BootstrapConfig = BootstrapConfig
self.DeepGemmMegaMoeConfig = DeepGemmMegaMoeConfig
self.FleetParams = FleetParams
self.MegaConfig = MegaConfig
self.MoEEpMegaLayer = MoEEpMegaLayer
self.MoEEpTensors = MoEEpTensors
self.MoEWeightPack = MoEWeightPack
self.Nvfp4CutedslMegaMoeConfig = Nvfp4CutedslMegaMoeConfig
def _load_flashinfer_moe_ep_api() -> _FlashInferMoeEpApi:
return _FlashInferMoeEpApi()
def _expose_deep_gemm_to_flashinfer() -> None:
deep_gemm = _import_deep_gemm()
if deep_gemm is None:
raise RuntimeError(
"FlashInfer MoE-EP DeepGEMM requires a usable DeepGEMM installation"
)
sys.modules.setdefault("deep_gemm", deep_gemm)
class FlashInferMoeEp:
def __init__(
self,
moe: FusedMoEConfig,
weights: FlashInferMoeEpWeights,
epilogue: FlashInferMoeEpEpilogue | None = None,
*,
apply_topk_in_fc1: bool,
) -> None:
spec = flashinfer_moe_ep_backend_spec(moe.moe_backend)
if spec.kernel == "deep_gemm":
_expose_deep_gemm_to_flashinfer()
api = _load_flashinfer_moe_ep_api()
if epilogue is None:
epilogue = FlashInferMoeEpEpilogue()
ep_group = get_ep_group()
bootstrap = api.BootstrapConfig(
world_size=ep_group.world_size,
rank=ep_group.rank_in_group,
process_group=ep_group.device_group,
device=torch.accelerator.current_device_index(),
)
fleet_params = api.FleetParams(
num_experts=moe.num_experts,
max_tokens_per_rank=moe.max_num_tokens,
token_hidden_size=moe.hidden_dim,
)
weight_pack = api.MoEWeightPack(
w13=weights.w13,
w2=weights.w2,
w13_scale=weights.w13_scale,
w2_scale=weights.w2_scale,
)
if spec.kernel == "cutedsl":
megakernel = api.Nvfp4CutedslMegaMoeConfig(
intermediate_size=moe.intermediate_size,
top_k=moe.experts_per_token,
gate_up_clamp=moe.swiglu_limit,
fast_math=True,
apply_topk_in_fc1=apply_topk_in_fc1,
enable_in_kernel_fc2_reduce=False,
combine_dtype="bf16",
input_norm_const=epilogue.input_norm_const,
fc1_alpha=None,
fc2_alpha=None,
fc1_norm_const=None,
knobs=None,
)
else:
megakernel = api.DeepGemmMegaMoeConfig(
intermediate_size=moe.intermediate_size,
top_k=moe.experts_per_token,
activation_clamp=moe.swiglu_limit,
fast_math=True,
)
backend = api.MegaConfig(
megakernel=megakernel,
quantize_input=True,
preprocess_weights=True,
)
self._moe_ep_tensors_cls = api.MoEEpTensors
self._mega_layer: Any | None = api.MoEEpMegaLayer(
bootstrap,
fleet_params,
weight_pack,
backend,
)
self._device = weights.w13.device
self._hidden_size = moe.hidden_dim
self._max_num_tokens = moe.max_num_tokens
self._num_experts = moe.num_experts
self._top_k = moe.experts_per_token
self._epilogue: FlashInferMoeEpEpilogue | None = epilogue
self._return_workspace_view = moe.routing_method is RoutingMethodType.DeepseekV4
@property
def can_overlap_shared_experts(self) -> bool:
return False
@property
def output_is_reduced(self) -> bool:
return True
@property
def topk_indices_dtype(self) -> torch.dtype:
return torch.int32
@property
def is_monolithic(self) -> bool:
return False
def _tensors(
self,
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
) -> Any:
epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
return self._moe_ep_tensors_cls(
hidden_states=hidden_states,
topk_ids=topk_ids,
topk_weights=topk_weights,
fc1_alpha=epilogue.fc1_alpha,
fc2_alpha=epilogue.fc2_alpha,
fc1_norm_const=epilogue.fc1_norm_const,
)
def __call__(
self,
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
) -> torch.Tensor:
if self._mega_layer is None:
raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
if hidden_states.shape[0] > self._max_num_tokens:
raise ValueError(
f"FlashInfer MoE-EP got {hidden_states.shape[0]} tokens, but "
f"the workspace supports at most {self._max_num_tokens}"
)
tensors = self._tensors(hidden_states, topk_ids, topk_weights)
if self._return_workspace_view and getattr(
self._mega_layer, "supports_output_view", False
):
return self._mega_layer.forward(tensors, return_workspace_view=True)
return self._mega_layer(tensors)
@torch.inference_mode()
def warmup(self) -> None:
if self._mega_layer is None:
return
hidden_states = torch.zeros(
1,
self._hidden_size,
dtype=torch.bfloat16,
device=self._device,
)
topk_ids = (
torch.arange(self._top_k, dtype=torch.int32, device=self._device)
.mul_(self._num_experts // self._top_k)
.view(1, self._top_k)
)
topk_weights = torch.full(
(1, self._top_k),
1.0 / self._top_k,
dtype=torch.float32,
device=self._device,
)
self._mega_layer.warmup(self._tensors(hidden_states, topk_ids, topk_weights))
def kernel_weights(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Kernel-resident fc1 weight, fc1 scales, fc2 weight, fc2 scales.
Contiguous per-expert byte tensors that alias the memory the megakernel
reads, so a caller may permute experts in place (EPLB).
"""
if self._mega_layer is None:
raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
(fc1_weight, fc1_sf), (fc2_weight, fc2_sf) = self._mega_layer._transformed
return (
_expert_major_bytes(fc1_weight),
_expert_major_bytes(fc1_sf),
_expert_major_bytes(fc2_weight),
_expert_major_bytes(fc2_sf),
)
def expert_weight_views(self) -> tuple[torch.Tensor, ...]:
"""Every per-expert tensor the kernel reads: weights plus epilogue."""
epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
vectors = (epilogue.fc1_alpha, epilogue.fc2_alpha, epilogue.fc1_norm_const)
return (*self.kernel_weights(), *(v for v in vectors if v is not None))
def destroy(self) -> None:
mega_layer = self._mega_layer
if mega_layer is None:
return
mega_layer.destroy()
self._mega_layer = None
self._epilogue = None
def __del__(self) -> None:
with contextlib.suppress(Exception):
self.destroy()
def _expert_major_bytes(tensor: torch.Tensor) -> torch.Tensor:
"""Contiguous storage of a per-expert kernel tensor, 1-byte dtypes as uint8.
The CuTeDSL kernel keeps (E, K, N) transpose views over (E, N, K) storage.
"""
if not tensor.is_contiguous():
tensor = tensor.transpose(1, 2)
if not tensor.is_contiguous():
raise ValueError(f"expert tensor is not expert-major: {tensor.stride()}")
return tensor.view(torch.uint8) if tensor.element_size() == 1 else tensor
@@ -23,6 +23,9 @@ from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsSink,
)
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
SharedExperts,
SharedExpertsOrder,
@@ -979,9 +982,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
def is_monolithic() -> bool:
return True
routing_replay_capture_fn: Callable[[torch.Tensor], None] | None = None
_routing_replay_buffer: torch.Tensor | None = None
def supports_routing_replay_capture(self) -> bool:
"""Whether this expert supports routing replay capture.
@@ -990,53 +990,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
"""
return False
def set_capture_fn(
self,
capture_fn: Callable[[torch.Tensor], None] | None,
) -> None:
self.routing_replay_capture_fn = capture_fn
if capture_fn is None:
self._routing_replay_buffer = None
return
# Allocate for per-rank batches gathered across the DP or EP group.
dispatch_group_size = (
self.moe_config.ep_size
if self.moe_config.use_ep
else self.moe_config.dp_size
)
max_num_replay_tokens = self.moe_config.max_num_tokens * dispatch_group_size
self._routing_replay_buffer = torch.empty(
(max_num_replay_tokens, self.moe_config.experts_per_token),
dtype=torch.int16,
device=self.moe_config.device,
)
def _maybe_make_routing_replay_buffer(
self,
num_tokens: int,
device: torch.device,
) -> torch.Tensor | None:
if self.routing_replay_capture_fn is None:
return None
buf = self._routing_replay_buffer
assert buf is not None
if buf.shape[0] < num_tokens or buf.device != device:
raise ValueError(
"Routing replay buffer was initialized for "
f"{buf.shape[0]} tokens on {buf.device}, but the kernel "
f"received {num_tokens} tokens on {device}."
)
return buf
def _maybe_dispatch_routing_replay(
self,
routing_replay_out: torch.Tensor | None,
num_tokens: int,
) -> None:
if routing_replay_out is None or self.routing_replay_capture_fn is None:
return
self.routing_replay_capture_fn(routing_replay_out[:num_tokens])
def apply(
self,
hidden_states: torch.Tensor,
@@ -1053,10 +1006,14 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Same as ``FusedMoEExperts.apply``, except uses router_logits as opposed
to the topk_ids and topk_weights. This is useful for kernels
with fused router and fused_experts (e.g. FLASHINFER_TRTLLM).
Kernels that ``supports_routing_replay_capture`` write each token's
routed expert ids into the leading rows of ``routing_replay_out``.
"""
raise NotImplementedError
@@ -1563,6 +1520,8 @@ class FusedMoEKernelMonolithicImpl:
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Same as forward(), except uses router_logits as opposed
to the topk_ids and topk_weights. This is used for kernels
@@ -1575,6 +1534,15 @@ class FusedMoEKernelMonolithicImpl:
defer_input_quant=self.fused_experts.expects_unquantized_inputs,
)
routing_replay_out = None
if routing_sink is not None:
# Checked per call: a weight reload may rebuild a different kernel.
if not self.fused_experts.supports_routing_replay_capture():
raise ValueError(
"Routed-experts capture is not supported with monolithic MoE "
f"kernel {type(self.fused_experts).__name__}."
)
routing_replay_out = routing_sink.buffer[: len(a1q)]
fused_out = self.fused_experts.apply(
hidden_states=a1q,
w1=w1,
@@ -1590,7 +1558,10 @@ class FusedMoEKernelMonolithicImpl:
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_replay_out=routing_replay_out,
)
if routing_sink is not None:
routing_sink.capture_fn(routing_replay_out)
if isinstance(fused_out, UnfinalizedMoEOutput):
if not self.prepare_finalize.supports_deferred_moe_finalize():
@@ -1698,6 +1669,8 @@ class FusedMoEKernel:
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
return self.impl.apply(
@@ -1713,6 +1686,7 @@ class FusedMoEKernel:
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_sink=routing_sink,
)
def apply(
@@ -3,6 +3,7 @@
import torch
from torch._subclasses.fake_tensor import FakeTensor
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@@ -75,11 +76,18 @@ def moe_fused_mul_sum_kernel(
def _heuristic_config(
hidden_size: int,
element_size: int,
device_index: int = 0,
):
is_fp32 = element_size > 2
max_block_k = 256 if is_fp32 else 512
BLOCK_K = max(128, min(triton.next_power_of_2(hidden_size), max_block_k))
num_warps = 4 if is_fp32 else 2
if not is_fp32 and hidden_size >= 1024 and current_platform.is_cuda():
capability = current_platform.get_device_capability(device_index)
if capability is not None and capability >= (7, 5):
# Larger tiles reduce the serial hidden-dimension loop.
max_block_k = 4096
num_warps = 8 if capability == (7, 5) else 16
BLOCK_K = max(128, min(triton.next_power_of_2(hidden_size), max_block_k))
num_stages = 3
return BLOCK_K, num_warps, num_stages
@@ -149,6 +157,7 @@ def moe_fused_mul_sum(
BLOCK_K, num_warps, num_stages = _heuristic_config(
hidden_size,
inputs.element_size(),
inputs.device.index or 0,
)
grid = (num_tokens,)
moe_fused_mul_sum_kernel[grid](
@@ -8,7 +8,11 @@ import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.config import get_current_vllm_config
from vllm.config.kernel import MoEBackend
from vllm.config.kernel import (
FLASHINFER_MOE_EP_CUTEDSL,
FLASHINFER_MOE_EP_DEEP_GEMM,
MoEBackend,
)
from vllm.config.quantization import QuantizationConfigArgs
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
@@ -145,6 +149,9 @@ class Mxfp4MoeBackend(Enum):
EMULATION = "EMULATION"
# Humming
HUMMING = "HUMMING"
# FlashInfer MoE-EP megakernels: routing outside, dispatch/GEMMs/combine fused.
FLASHINFER_MOE_EP_CUTEDSL = "FLASHINFER_MOE_EP_CUTEDSL"
FLASHINFER_MOE_EP_DEEP_GEMM = "FLASHINFER_MOE_EP_DEEP_GEMM"
# Backends that share the same TRTLLM weight format
@@ -165,6 +172,13 @@ B12X_BACKENDS = (
Mxfp4MoeBackend.B12X_MXFP4_BF16,
)
# Megakernels consume the checkpoint's MXFP4 tensors through the shared
# FlashInfer MoE-EP adapter; vLLM does not convert their weight layout.
FLASHINFER_MOE_EP_MXFP4_BACKENDS = (
Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM,
)
def backend_to_kernel_cls(
backend: Mxfp4MoeBackend,
@@ -291,6 +305,13 @@ def backend_to_kernel_cls(
return [OCP_MXQuantizationEmulationTritonExperts]
elif backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import ( # noqa: E501
FlashInferMoeEpExperts,
)
return [FlashInferMoeEpExperts]
else:
raise ValueError(f"Unknown MXFP4 MoE backend: {backend.value}")
@@ -314,6 +335,10 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
],
"flashinfer_cutlass_afp8": [Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8],
"flashinfer_moe_ep_cutedsl": [Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL],
"flashinfer_moe_ep_mega_deep_gemm": [
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM
],
"triton": [Mxfp4MoeBackend.TRITON],
"triton_unfused": [Mxfp4MoeBackend.TRITON_UNFUSED],
"humming": [Mxfp4MoeBackend.HUMMING],
@@ -735,6 +760,9 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
activation: MoEActivation | None = None,
) -> tuple[int, int]:
"""Round up hidden_size and intermediate_size based on backend requirements."""
if backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
# The megakernel takes the model's dimensions and validates them itself.
return hidden_size, intermediate_size
if backend in B12X_BACKENDS:
# b12x plans for the exact model dimensions. B12xExperts validates the
# required MXFP4 block alignment before selecting the backend.
@@ -1827,6 +1855,29 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias=w2_bias,
_cache_permute_indices=_cache_permute_indices,
)
elif mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
mxfp4_moe_ep_weights,
)
# DeepGEMM consumes MXFP4 directly; CuTeDSL gets bf16 and requantizes.
weights = mxfp4_moe_ep_weights(
FLASHINFER_MOE_EP_CUTEDSL
if mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL
else FLASHINFER_MOE_EP_DEEP_GEMM,
w13_weight,
w2_weight,
w13_weight_scale,
w2_weight_scale,
)
return (
weights.w13,
weights.w2,
weights.w13_scale,
weights.w2_scale,
w13_bias,
w2_bias,
)
elif mxfp4_backend == Mxfp4MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_mxfp4_moe_layer_for_cpu,
@@ -1874,6 +1925,9 @@ def make_mxfp4_moe_quant_config(
layer: "RoutedExperts | None" = None,
) -> FusedMoEQuantConfig | None:
"""Create a FusedMoEQuantConfig for the given MXFP4 backend."""
if mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
# The megakernel quantizes activations itself and owns its scales.
return FusedMoEQuantConfig.make("nvfp4", weight_dtype="mxfp4")
if mxfp4_backend == Mxfp4MoeBackend.B12X_MXFP4_MXFP8:
return mxfp4_mxfp8_moe_quant_config(
w1_bias=w1_bias,
@@ -50,6 +50,7 @@ class NvFp4MoeBackend(Enum):
MARLIN = "MARLIN"
HUMMING = "HUMMING"
EMULATION = "EMULATION"
FLASHINFER_MOE_EP_CUTEDSL = "FLASHINFER_MOE_EP_CUTEDSL"
FLASHINFER_NVFP4_MOE_BACKENDS = [
@@ -148,6 +149,12 @@ def backend_to_kernel_cls(
)
return [Nvfp4QuantizationEmulationTritonExperts]
elif backend == NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL:
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import ( # noqa: E501
FlashInferMoeEpExperts,
)
return [FlashInferMoeEpExperts]
else:
raise ValueError(f"Unknown NvFP4 MoE backend: {backend.value}")
@@ -160,6 +167,7 @@ def map_nvfp4_backend(runner_backend: MoEBackend) -> NvFp4MoeBackend:
"flashinfer_trtllm": NvFp4MoeBackend.FLASHINFER_TRTLLM,
"flashinfer_cutlass": NvFp4MoeBackend.FLASHINFER_CUTLASS,
"flashinfer_cutedsl": NvFp4MoeBackend.FLASHINFER_CUTEDSL,
"flashinfer_moe_ep_cutedsl": NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
"flashinfer_b12x": NvFp4MoeBackend.FLASHINFER_B12X,
"marlin": NvFp4MoeBackend.MARLIN,
"humming": NvFp4MoeBackend.HUMMING,
@@ -207,6 +215,7 @@ def select_nvfp4_moe_backend(
NvFp4MoeBackend.FLASHINFER_TRTLLM,
NvFp4MoeBackend.FLASHINFER_CUTLASS,
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
NvFp4MoeBackend.VLLM_CUTLASS,
NvFp4MoeBackend.MARLIN,
NvFp4MoeBackend.EMULATION,
@@ -281,8 +290,9 @@ def select_nvfp4_moe_backend(
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
f"explicitly requested moe_backend={runner_backend!r} does "
f"not apply the SwiGLU clamp. Use 'flashinfer_trtllm', "
f"'flashinfer_cutlass', 'flashinfer_cutedsl', 'cutlass', "
f"'b12x', 'marlin', or 'humming' instead."
f"'flashinfer_cutlass', 'flashinfer_cutedsl', "
f"'flashinfer_moe_ep_cutedsl', 'cutlass', 'b12x', 'marlin', "
f"or 'humming' instead."
)
return _return_or_raise(
requested_backend, config, weight_key, activation_key, activation_format
@@ -491,6 +501,9 @@ def convert_to_nvfp4_moe_kernel_format(
# for other experts - other selection strategies may be used.
a13_scale = 1.0 / a13_scale.max().to(torch.float32)
a2_scale = 1.0 / a2_scale.max().to(torch.float32)
elif nvfp4_backend == NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL:
# The megakernel transforms the checkpoint tensors itself at init.
pass
else:
raise ValueError(f"Unknown NvFp4 backend for MoE: {nvfp4_backend}")
@@ -0,0 +1,41 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Prepare/finalize stages that do no expert-parallel communication.
Selected by ``all2all_backend="passthrough"``: the experts implementation
dispatches, combines and reduces itself, so nothing is left for these stages.
"""
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.prepare_finalize.no_dp_ep import (
MoEPrepareAndFinalizeNoDPEPModular,
)
class PassThroughPrepareAndFinalize(MoEPrepareAndFinalizeNoDPEPModular):
"""Hands activations and routing straight to experts that own their EP
communication."""
def output_is_reduced(self) -> bool:
return True
def supports_deferred_moe_finalize(self) -> bool:
# The experts apply the top-k weights and combine internally; there
# is no finalize left for a consumer to take over.
return False
def prepare(
self,
a1: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
num_experts: int,
expert_map: torch.Tensor | None,
apply_router_weight_on_input: bool,
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool = False,
) -> mk.PrepareResultType:
return a1, None, None, None, None
@@ -20,6 +20,9 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsSink,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
@@ -138,6 +141,8 @@ class RoutedExperts(PluggableLayer):
self.swiglu_alpha = swiglu_alpha
self.swiglu_beta = swiglu_beta
self.e_score_correction_bias = e_score_correction_bias
# Set by bind_routed_experts_capturer for monolithic kernels.
self.routing_sink: RoutedExpertsSink | None = None
self.apply_router_weight_on_input = apply_router_weight_on_input
# End random parameters
self._loaded_expert_biases: set[str] = set()
@@ -15,6 +15,7 @@ import torch
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.platforms import current_platform
logger = logging.getLogger(__name__)
@@ -186,6 +187,23 @@ class RoutedExpertsCapturer:
return self.device_buffer[:num_tokens].to(self.output_dtype)
class RoutedExpertsSink:
"""Layer-owned buffer and callback for the expert ids a monolithic kernel
routes to; it outlives kernel rebuilds on weight reload."""
def __init__(
self, moe_config: FusedMoEConfig, capture_fn: Callable[[torch.Tensor], None]
) -> None:
# Sized for per-rank batches gathered across the DP or EP group.
group_size = moe_config.ep_size if moe_config.use_ep else moe_config.dp_size
self.buffer = torch.empty(
(moe_config.max_num_tokens * group_size, moe_config.experts_per_token),
dtype=torch.int16,
device=moe_config.device,
)
self.capture_fn = capture_fn
def bind_routed_experts_capturer(
model: torch.nn.Module,
capturer: RoutedExpertsCapturer,
@@ -219,7 +237,9 @@ def bind_routed_experts_capturer(
"Routed-experts capture is not supported with monolithic "
f"MoE kernel {type(fused_experts).__name__}."
)
fused_experts.set_capture_fn(capture_fn)
module.routed_experts.routing_sink = RoutedExpertsSink(
fused_experts.moe_config, capture_fn
)
num_bound += 1
elif isinstance(module.router, BaseRouter):
module.router.set_capture_fn(capture_fn)
@@ -362,4 +362,5 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
+57 -10
View File
@@ -163,6 +163,14 @@ class SparseIndexerTopk(torch.nn.Module):
current_platform.has_device_capability(90)
and not current_platform.is_device_capability_family(120)
)
self._is_rocm = current_platform.is_rocm()
self._aiter_enabled = False
self._aiter_capable = False
if self._is_rocm:
from vllm._aiter_ops import rocm_aiter_ops
self._aiter_enabled = bool(rocm_aiter_ops.is_enabled())
self._aiter_capable = bool(rocm_aiter_ops.is_indexer_top_k_enabled())
def resolve_backend(
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
@@ -193,6 +201,19 @@ class SparseIndexerTopk(torch.nn.Module):
f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
f" stride={logits.stride()}, topk={topk_tokens}"
)
elif self._backend == "aiter":
if not self._is_rocm:
failures.append("requires a ROCm platform")
elif not self._aiter_enabled:
failures.append("requires AITER (VLLM_ROCM_USE_AITER=1)")
elif not self._aiter_capable:
failures.append(
"AITER has no tuned indexer top-k kernels for this GPU arch"
)
if topk_tokens not in (512, 1024, 2048, 4096):
failures.append(
f"topk_tokens must be in (512, 1024, 2048, 4096), got {topk_tokens}"
)
elif self._backend == "flashinfer":
if not self._is_cuda:
failures.append("requires a CUDA platform")
@@ -216,11 +237,14 @@ class SparseIndexerTopk(torch.nn.Module):
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> str:
"""The priority chain: cooperative -> persistent -> per_row.
deep_select/flashinfer/torch are opt-in only.
aiter/deep_select/flashinfer/torch are opt-in only.
cooperative_topk is preferred whenever it is applicable, i.e. within
its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to
persistent_topk.
aiter not auto-resolved as it does not outperform the per-row on every shape,
hence we delegate to the caller to decide when to use it.
"""
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
return "cooperative"
@@ -285,7 +309,8 @@ class SparseIndexerTopk(torch.nn.Module):
max_seq_len: int,
) -> None:
"""Run the resolved decode top-k implementation, writing into
topk_indices (int32, -1 fill for rows shorter than topk_tokens)."""
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
"""
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
if backend == "deep_select":
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
@@ -344,15 +369,37 @@ class SparseIndexerTopk(torch.nn.Module):
) < row_ends.unsqueeze(1)
indices = torch.where(in_range, indices, -1)
topk_indices.copy_(indices)
else:
assert backend == "per_row", f"unknown topk backend: {backend}"
ops.top_k_per_row_decode(
elif backend == "aiter":
from vllm._aiter_ops import rocm_aiter_ops
# AITER's decode kernel only implements the 1D/next_n == 1 form,
# which is equivalent once the ragged ends are expanded.
rocm_aiter_ops.indexer_top_k_decode(
logits,
next_n,
seq_lens,
1,
self._row_ends(seq_lens, next_n, logits.shape[0]),
topk_indices,
logits.shape[0],
logits.stride(0),
logits.stride(1),
topk_tokens,
)
else:
assert backend == "per_row", f"unknown topk backend: {backend}"
self._per_row(logits, seq_lens, next_n, topk_indices, topk_tokens)
@staticmethod
def _per_row(
logits: torch.Tensor,
seq_lens: torch.Tensor,
next_n: int,
topk_indices: torch.Tensor,
topk_tokens: int,
) -> None:
ops.top_k_per_row_decode(
logits,
next_n,
seq_lens,
topk_indices,
logits.shape[0],
logits.stride(0),
logits.stride(1),
topk_tokens,
)
@@ -551,28 +551,33 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
raise ValueError(
f"VLLM_GDN_DECODE_KERNEL=cuda is not supported: {reason}"
)
logger.info_once(
"Falling back to the Triton GDN decode path: %s", reason
)
if self.speculative_config is not None:
logger.info_once(
"Falling back to the Triton GDN spec decode path: %s",
reason,
)
self.gdn_decode_kernel = "triton"
elif current_platform.is_cpu():
self.gdn_decode_kernel = "CPU"
elif current_platform.is_xpu():
self.gdn_decode_kernel = "XPU"
self.enable_fused_gdn_decode = self.gdn_decode_kernel == "cuda"
self.enable_fused_gdn_spec_decode = (
self.gdn_decode_kernel == "cuda" and self.speculative_config is not None
)
# RecoverSSM verify replaces the per-draft snapshot path (Triton decode
# only)
self.use_gdn_recoverssm = self._uses_gdn_recoverssm()
if self.use_gdn_recoverssm:
self.gdn_decode_kernel = "triton"
self.enable_fused_gdn_decode = False
self.enable_fused_gdn_spec_decode = False
self.enable_packed_recurrent_decode = False
logger.info_once(
"GDN RecoverSSM speculative verify active (spec_query_len %d)",
self.num_spec + 1,
)
logger.info_once("GDN decode kernel: %s", self.gdn_decode_kernel)
if self.speculative_config is not None:
logger.info_once("GDN spec decode kernel: %s", self.gdn_decode_kernel)
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
@@ -969,12 +974,12 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
mixed_qkvz, _ = self.in_proj_qkvz(hidden_states)
ba, _ = self.in_proj_ba(hidden_states)
use_fused_gdn_decode = (
self.enable_fused_gdn_decode
use_fused_gdn_spec_decode = (
self.enable_fused_gdn_spec_decode
and hidden_states.dtype == torch.bfloat16
and self.norm.weight.dtype in (torch.bfloat16, torch.float32)
)
if use_fused_gdn_decode:
if use_fused_gdn_spec_decode:
core_attn_out = torch.zeros(
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
dtype=hidden_states.dtype,
@@ -790,6 +790,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -779,4 +779,5 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -24,10 +24,15 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsMxFp4,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
is_flashinfer_moe_ep_backend,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
B12X_BACKENDS,
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
Mxfp4MoeBackend,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
select_mxfp4_moe_backend,
@@ -38,7 +43,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_moe_fp4_layer_for_marlin,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -49,12 +54,14 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
super().__init__(moe)
self.group_size = 32
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
self.experts_cls: type[mk.FusedMoEExperts] | None = None
# Backend selection must match the weight preparation below: CUTLASS
# swizzles scales, b12x and XPU consume checkpoint packing, and Marlin
# repacks weights and scales.
# swizzles scales, b12x and XPU consume checkpoint packing, Marlin
# repacks weights and scales, and the megakernels take the checkpoint
# tensors as they are.
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
self.experts_cls: type[mk.FusedMoEExperts]
if moe.moe_backend == "b12x":
if moe.moe_backend == "b12x" or is_flashinfer_moe_ep_backend(moe.moe_backend):
self.mxfp4_backend, experts_cls = select_mxfp4_moe_backend(moe)
assert experts_cls is not None
self.experts_cls = experts_cls
@@ -70,6 +77,10 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
logger.info_once("Using MarlinExperts for MXFP4 MoE")
self.experts_cls = MarlinExperts
@property
def supports_eplb(self) -> bool:
return self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS
def create_weights(
self,
layer: torch.nn.Module,
@@ -198,6 +209,21 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
layer.w2_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w2), requires_grad=False
)
elif self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
w13, w2, w13_scale, w2_scale, _, _ = (
convert_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=self.mxfp4_backend,
layer=layer,
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
w13_weight_scale=layer.w13_weight_scale,
w2_weight_scale=layer.w2_weight_scale,
)
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
pass
else:
@@ -211,6 +237,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None:
assert self.experts_cls is not None
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
@@ -38,6 +38,7 @@ _EPLB_SUPPORTED_NVFP4_BACKENDS = frozenset(
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
NvFp4MoeBackend.FLASHINFER_CUTEDSL_BATCHED,
NvFp4MoeBackend.FLASHINFER_TRTLLM,
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
}
)
@@ -61,7 +62,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
weight_key=kNvfp4Static,
activation_key=None if use_a16 else kNvfp4Dynamic,
)
self.use_global_sf = is_global_sf_supported_for_nvfp4_backend(
self.nvfp4_backend
)
@@ -155,7 +155,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
set_weight_attrs(w13_weight_scale_2, extra_weight_attrs)
w2_weight_scale_2 = torch.nn.Parameter(
torch.empty(num_experts, dtype=torch.float32), requires_grad=False
torch.empty(num_experts, dtype=torch.float32),
requires_grad=False,
)
layer.register_parameter("w2_weight_global_scale", w2_weight_scale_2)
extra_weight_attrs.update(
@@ -175,7 +176,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
set_weight_attrs(w13_input_scale, extra_weight_attrs)
w2_input_scale = torch.nn.Parameter(
torch.empty(num_experts, dtype=torch.float32), requires_grad=False
torch.empty(num_experts, dtype=torch.float32),
requires_grad=False,
)
layer.register_parameter("w2_input_global_scale", w2_input_scale)
extra_weight_attrs.update(
@@ -293,6 +295,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -287,6 +287,7 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -366,6 +366,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -291,4 +291,5 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -179,6 +179,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -528,6 +528,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -818,6 +818,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -4,6 +4,7 @@
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config.kernel import FLASHINFER_MOE_EP_BACKENDS
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
@@ -53,6 +54,11 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
def __init__(self, moe) -> None:
super().__init__(moe)
if moe.moe_backend in FLASHINFER_MOE_EP_BACKENDS:
raise ValueError(
f"moe_backend={moe.moe_backend!r} is not supported for AutoRound "
"(INC) MXFP4 MoE checkpoints"
)
self.group_size = 32
# Backend selection must stay consistent with the weight preparation in
# process_weights_after_loading / get_fused_moe_quant_config, which use
@@ -201,7 +201,7 @@ class QuantFP8(CustomOp):
if scale is None:
if self.group_shape == GroupShape.PER_TOKEN:
x_max, _ = x.abs().max(dim=-1)
x_max = x.abs().amax(dim=-1)
x_max = x_max.unsqueeze(-1).to(torch.float32)
if scale_ub is not None:
x_max = x_max.clamp(max=scale_ub)
@@ -680,6 +680,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -1092,6 +1093,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -1461,6 +1463,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -2314,9 +2317,9 @@ def select_linear_kernel(
w = spec.weight
assert isinstance(w, QuantKey), f"resolve() must supply a weight key, got {w!r}"
if w.dtype == FP4_DTYPE:
# W4A16 (activation is None) → use_a16=True defaults to Marlin *and*
# honors --linear-backend (matches upstream ModelOptNvFp4W4A16LinearMethod
# after #50273); W4A4 → use_a16=False.
# W4A16 (activation is None) → use_a16=True, auto prefers FlashInfer
# CuTe-DSL backend on SM100/103 when available and Marlin otherwise,
# and honors --linear-backend (#50273); W4A4 → use_a16=False.
return init_nvfp4_linear_kernel(use_a16=spec.activation is None)
if w.scale.dtype == MXFP8_SCALE_DTYPE:
return init_mxfp8_linear_kernel(
@@ -531,6 +531,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@staticmethod
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import torch
from vllm.logger import init_logger
@@ -21,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
backend_to_kernel_cls,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
@@ -472,9 +475,67 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool:
"""Route Kimi-K3 weight-only MXFP4 SiTU MoE to AITER A16W4 on gfx950."""
from vllm.platforms import current_platform
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.platforms.rocm import on_gfx950
return (
rocm_aiter_ops.is_fused_moe_enabled()
and on_gfx950()
and moe.activation == MoEActivation.SITU
and moe.activation_situ_linear_beta is not None
and rocm_aiter_ops.get_aiter_activation_type("situ") is not None
)
def _use_k3_situ_int4_gfx942(moe: FusedMoEConfig) -> bool:
"""Opt-in gfx942 MXFP4-to-int4 requant for Kimi-K3 SiTU experts."""
from vllm.platforms import current_platform
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.platforms.rocm import on_gfx942
return (
rocm_aiter_ops.is_fused_moe_enabled()
and on_gfx942()
and moe.activation == MoEActivation.SITU
and moe.activation_situ_linear_beta is not None
and _moe_weight_override_is_int4()
)
def _moe_weight_override_is_int4() -> bool:
"""True when --quantization-config.moe.weight int4_per_group_32 was set."""
from vllm.config import get_current_vllm_config
vllm_config = get_current_vllm_config()
if vllm_config is None:
return False
quant_args = getattr(vllm_config.model_config, "quantization_config", None)
moe_spec = getattr(quant_args, "moe", None)
weight = getattr(moe_spec, "weight", None)
if weight is None:
return False
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt4Static32,
)
return weight == kInt4Static32
class Mxfp4MoEMethod(FusedMoEMethodBase):
"""MXFP4 MoE quantization method."""
@@ -482,9 +543,33 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_dtype = "mxfp4"
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
self.is_k3_situ_int4_gfx942 = _use_k3_situ_int4_gfx942(moe)
self.experts_cls: type[mk.FusedMoEExperts] | None
if self.is_k3_situ_aiter or self.is_k3_situ_int4_gfx942:
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
if self.is_k3_situ_int4_gfx942:
logger.warning_once(
"Requantizing Kimi-K3 MXFP4 experts to groupwise int4 "
"for gfx942. This conversion is lossy and was explicitly "
"enabled by --quantization-config.moe.weight "
"int4_per_group_32."
)
else:
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
from vllm._aiter_ops import rocm_aiter_ops
if rocm_aiter_ops.is_fused_moe_situv2_enabled():
# a16w4 does not read this bound. Force 0 so leftover a8w4
# config cannot keep bf16 activations.
# TODO: Remove once AITER takes this as a kernel argument.
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
else:
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
moe
)
self.max_capture_size = moe.max_capture_size
@@ -782,7 +867,142 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def process_weights_after_loading(self, layer):
def _setup_kernel_k3_situ_gfx942(self, layer: RoutedExperts) -> None:
import inspect
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
flydsl_a16w4_gemm1,
)
# Fail closed if this AITER build would silently run SiLU.
if "act" not in inspect.signature(flydsl_a16w4_gemm1).parameters:
raise RuntimeError(
"This AITER build ignores the SiTUv2 activation on the "
"replacement packed-int4 MoE path and would silently compute "
"SiLU. Rebuild with the replacement A16W4/int4 pipeline."
)
from aiter import dtypes as aiter_dtypes
from aiter.ops.quant import per_1x32_i4_quant
from aiter.ops.shuffle import (
pack_int8_to_packed_int4,
shuffle_scale_for_int4,
shuffle_weight,
)
from aiter.utility import fp4_utils
fp4_dtype = torch.float4_e2m1fn_x2
e8m0_dtype = torch.float8_e8m0fnu
# mxfp4_to_f32 grows the packed weight 8x and peaks above 20 GiB per
# rank if all experts convert at once. Convert 8 experts at a time.
_CONVERT_CHUNK = 8
def convert(
weight: torch.nn.Parameter,
scale: torch.nn.Parameter,
) -> tuple[torch.Tensor, torch.Tensor]:
# Drop the source before returning. Packed int4 is the same size as
# packed MXFP4, so holding both doubles the footprint. That fits
# under expert parallel (1/ep_size experts per rank) and does not
# fit under pure tensor parallel.
w_all = weight.data.view(fp4_dtype)
s_all = scale.data.view(e8m0_dtype)
num_experts = w_all.shape[0]
# Write chunks into a preallocated output. torch.cat of a list
# would hold the full result twice at the join.
out_packed: torch.Tensor | None = None
out_scale: torch.Tensor | None = None
scale_stride = 0
for lo in range(0, num_experts, _CONVERT_CHUNK):
hi = min(lo + _CONVERT_CHUNK, num_experts)
weight_f32 = fp4_utils.mxfp4_to_f32(w_all[lo:hi])
scale_f32 = fp4_utils.e8m0_to_f32(s_all[lo:hi])
chunk_experts, output_size, input_size = weight_f32.shape
weight_bf16 = (
(
weight_f32.view(
chunk_experts, output_size, input_size // 32, 32
)
* scale_f32.view(
chunk_experts, output_size, input_size // 32, 1
)
)
.view(chunk_experts, output_size, input_size)
.to(torch.bfloat16)
)
del weight_f32, scale_f32
weight_int4, weight_scale = per_1x32_i4_quant(weight_bf16)
del weight_bf16
weight_int4 = weight_int4.view(aiter_dtypes.i4x2).view(
chunk_experts, output_size, input_size
)
weight_packed = pack_int8_to_packed_int4(
shuffle_weight(weight_int4.view(aiter_dtypes.i8), (16, 16))
)
del weight_int4
chunk_packed = weight_packed.view(
chunk_experts, output_size, input_size // 2
).view(aiter_dtypes.i4x2)
chunk_scale = (
shuffle_scale_for_int4(weight_scale, group_size=32)
.view(-1)
.contiguous()
)
if out_packed is None:
out_packed = torch.empty(
(num_experts,) + tuple(chunk_packed.shape[1:]),
dtype=chunk_packed.dtype,
device=chunk_packed.device,
)
scale_stride = chunk_scale.numel() // chunk_experts
out_scale = torch.empty(
num_experts * scale_stride,
dtype=chunk_scale.dtype,
device=chunk_scale.device,
)
assert out_packed is not None
assert out_scale is not None
out_packed[lo:hi].copy_(chunk_packed)
out_scale[lo * scale_stride : hi * scale_stride].copy_(chunk_scale)
del weight_packed, weight_scale, chunk_packed, chunk_scale
torch.accelerator.empty_cache()
del w_all, s_all
assert out_packed is not None
assert out_scale is not None
weight.data = torch.empty(0, dtype=torch.uint8, device=out_packed.device)
scale.data = torch.empty(0, dtype=torch.uint8, device=out_packed.device)
torch.accelerator.empty_cache()
return out_packed, out_scale
w13, w13_scale = convert(layer.w13_weight, layer.w13_weight_scale)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w13_weight_scale", w13_scale)
del w13, w13_scale
torch.accelerator.empty_cache()
w2, w2_scale = convert(layer.w2_weight, layer.w2_weight_scale)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w2_weight_scale", w2_scale)
del w2, w2_scale
torch.accelerator.empty_cache()
layer.w13_weight.is_shuffled = True
layer.w2_weight.is_shuffled = True
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None and self.experts_cls is not None:
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
mxfp4_backend=self.mxfp4_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
return
@@ -795,6 +1015,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
self._build_moe_kernel(layer)
return
if self.is_k3_situ_int4_gfx942:
self._setup_kernel_k3_situ_gfx942(layer)
return
w13 = layer.w13_weight
w2 = layer.w2_weight
w13_scale = layer.w13_weight_scale
@@ -900,4 +1124,5 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -65,6 +65,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4Static32,
kInt8StaticChannelSym,
kMxfp4Static,
kMxfp8Dynamic,
@@ -231,6 +232,9 @@ class OnlineQuantizationConfig(QuantizationConfig):
"""
if spec is None or spec.weight is None:
return None
# Load-time gfx942 requant, not online conversion. Mxfp4MoEMethod owns it.
if spec.weight == kInt4Static32:
return None
cls = table.get(spec.weight)
if cls is None:
raise ValueError(
@@ -156,6 +156,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -1976,6 +1976,11 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
}
}
@@ -25,6 +25,11 @@ INT4_DTYPE = scalar_types.uint4b8
INT8_DTYPE = scalar_types.uint8b128
def require_finite_positive_scales(name: str, value: torch.Tensor) -> None:
if not torch.isfinite(value).all() or not (value > 0).all():
raise ValueError(f"{name} must contain finite positive scales")
def _dtype_abbr(dtype: torch.dtype | ScalarType) -> str:
"""Return a stable short name for torch and ScalarType dtypes."""
if isinstance(dtype, ScalarType):