Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 22:06:10 +07:00
327 changed files with 13704 additions and 5433 deletions
@@ -31,6 +31,9 @@ from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_one
from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_two_sided import ( # noqa: E501
FlashInferNVLinkTwoSidedPrepareAndFinalize,
)
from vllm.model_executor.layers.fused_moe.prepare_finalize.passthrough import (
PassThroughPrepareAndFinalize,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import (
has_deep_ep,
@@ -185,6 +188,10 @@ def maybe_make_prepare_finalize(
all2all_manager: Any | None = None,
input_dtype: torch.dtype | None = None,
) -> FusedMoEPrepareAndFinalize | None:
if moe.moe_parallel_config.use_passthrough_all2all:
# The experts dispatch and combine themselves, in every EP topology.
return PassThroughPrepareAndFinalize()
if not moe.moe_parallel_config.use_all2all_kernels:
if not allow_new_interface:
return None
@@ -1068,6 +1068,12 @@ class FusedMoEParallelConfig:
and self.all2all_backend == "allgather_reducescatter"
)
@property
def use_passthrough_all2all(self):
# Not gated on use_all2all_kernels: the experts dispatch and combine
# internally in every EP topology, including TP-only.
return self.all2all_backend == "passthrough"
@property
def use_mori_kernels(self):
return self.use_all2all_kernels and self.all2all_backend in (
@@ -1530,6 +1536,10 @@ class FusedMoEConfig:
def use_deepep_ll_kernels(self):
return self.moe_parallel_config.use_deepep_ll_kernels
@property
def use_passthrough_all2all(self):
return self.moe_parallel_config.use_passthrough_all2all
@property
def use_mori_kernels(self):
return self.moe_parallel_config.use_mori_kernels
@@ -374,6 +374,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
routing_method = self.moe_config.routing_method
if routing_method == RoutingMethodType.DeepseekV4:
@@ -289,6 +289,7 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
assert self.moe_config.intermediate_size_per_partition_unpadded is not None
assert self.moe_config.hidden_dim_unpadded is not None
@@ -198,6 +198,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Apply the monolithic 4-bit INT MoE forward pass.
@@ -215,6 +216,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: Bias for expert scores
routed_scaling_factor: Scaling factor for routing
topk_group: Group size for topk
routing_replay_out: Unused; this kernel does not capture routing
Returns:
Output tensor after MoE computation
@@ -0,0 +1,210 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlashInfer MoE-EP megakernels as a modular-kernel experts implementation.
The megakernel routes on vLLM's top-k output and then dispatches, computes and
combines in one launch, so it is paired with a pass-through prepare/finalize.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
FusedMoEQuantConfig,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
FlashInferMoeEp,
FlashInferMoeEpEpilogue,
FlashInferMoeEpWeights,
apply_topk_in_fc1,
flashinfer_moe_ep_unsupported_reasons,
supports_current_device,
validate_flashinfer_moe_ep_layer,
)
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
TopKWeightAndReduceNoOP,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
kNvfp4Static,
require_finite_positive_scales,
)
from vllm.model_executor.utils import replace_parameter
if TYPE_CHECKING:
pass
# Layer parameters the megakernel reads. After ``process_weights_after_loading``
# they alias the megakernel's own tensors, so EPLB permutes exactly what it uses.
KERNEL_WEIGHT_NAMES = ("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale")
_supported_weight_quant_schemes: frozenset[QuantKey] = frozenset(
(kNvfp4Static, kMxfp4Static)
)
_supported_activations: frozenset[MoEActivation] = frozenset((MoEActivation.SILU,))
class FlashInferMoeEpExperts(mk.FusedMoEExpertsModular):
def __init__(
self,
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
) -> None:
super().__init__(moe_config, quant_config)
self._adapter: FlashInferMoeEp | None = None
@staticmethod
def is_supported_config(
cls: type[mk.FusedMoEExperts],
moe_config: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
activation_format: mk.FusedMoEActivationFormat,
) -> tuple[bool, str | None]:
supported, reason = mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
)
if not supported:
return False, reason
reasons = flashinfer_moe_ep_unsupported_reasons(
moe_config, weight_key, activation_key
)
if reasons:
return False, f"kernel does not support {', '.join(reasons)}"
return True, None
@staticmethod
def activation_format() -> mk.FusedMoEActivationFormat:
return mk.FusedMoEActivationFormat.Standard
@staticmethod
def _supports_current_device() -> bool:
return supports_current_device()
@staticmethod
def _supports_no_act_and_mul() -> bool:
return False
@staticmethod
def _supports_quant_scheme(
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> bool:
return weight_key in _supported_weight_quant_schemes
@staticmethod
def _supports_activation(activation: MoEActivation) -> bool:
return activation in _supported_activations
@staticmethod
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
return (
moe_parallel_config.use_ep
and not moe_parallel_config.use_batched_activation_format
)
@property
def expects_unquantized_inputs(self) -> bool:
return True
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
return TopKWeightAndReduceNoOP()
def workspace_shapes(
self,
M: int,
N: int,
K: int,
topk: int,
global_num_experts: int,
local_num_experts: int,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
activation: MoEActivation,
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
return (0,), (0,), (M, K)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
"""Build the megakernel from the layer's canonical weights.
The oracle has already put the weights in kernel format; the per-expert
weight global scales arrive through the quant config and become the
epilogue alphas.
"""
validate_flashinfer_moe_ep_layer(layer) # type: ignore[arg-type]
adapter = FlashInferMoeEp(
self.moe_config,
FlashInferMoeEpWeights(
w13=layer.w13_weight,
w2=layer.w2_weight,
w13_scale=getattr(layer, "w13_weight_scale", None),
w2_scale=getattr(layer, "w2_weight_scale", None),
),
epilogue_from_quant_config(self.quant_config),
apply_topk_in_fc1=apply_topk_in_fc1(
self.moe_config,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
),
)
adapter.warmup()
for name, tensor in zip(KERNEL_WEIGHT_NAMES, adapter.kernel_weights()):
replace_parameter(layer, name, tensor)
self._adapter = adapter
def apply(
self,
output: torch.Tensor,
hidden_states: torch.Tensor,
w1: torch.Tensor,
w2: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
activation: MoEActivation,
global_num_experts: int,
expert_map: torch.Tensor | None,
a1q_scale: torch.Tensor | None,
a2_scale: torch.Tensor | None,
workspace13: torch.Tensor,
workspace2: torch.Tensor,
expert_tokens_meta: mk.ExpertTokensMetadata | None,
apply_router_weight_on_input: bool,
) -> None:
if self._adapter is None:
raise RuntimeError("FlashInfer MoE-EP weights have not been processed")
output.copy_(
self._adapter(hidden_states, topk_ids.to(torch.int32), topk_weights)
)
def epilogue_from_quant_config(
quant_config: FusedMoEQuantConfig,
) -> FlashInferMoeEpEpilogue:
"""Per-expert epilogue constants from the canonical quant config.
The weight global scales (``g1_alphas``/``g2_alphas``) become the fc1/fc2
alphas. Activations are quantized dynamically inside the megakernel, so the
checkpoint's input scales drop out. MXFP4 checkpoints carry no global scales
and keep the default epilogue. The alphas alias the layer's parameters, so
EPLB permutations reach the kernel.
"""
fc1_alpha, fc2_alpha = quant_config.g1_alphas, quant_config.g2_alphas
if fc1_alpha is None and fc2_alpha is None:
return FlashInferMoeEpEpilogue()
assert fc1_alpha is not None and fc2_alpha is not None
require_finite_positive_scales("g1_alphas", fc1_alpha)
require_finite_positive_scales("g2_alphas", fc2_alpha)
if fc1_alpha.dim() != 1 or fc2_alpha.dim() != 1:
raise ValueError("the megakernel needs one weight global scale per expert")
return FlashInferMoeEpEpilogue(
fc1_alpha=fc1_alpha.float().contiguous(),
fc2_alpha=fc2_alpha.float().contiguous(),
)
@@ -1580,6 +1580,7 @@ class OAITritonMxfp4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
return triton_kernel_moe_forward(
hidden_states=hidden_states,
@@ -272,6 +272,7 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
import flashinfer
@@ -282,10 +283,6 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
# dummy 0-token forward has to keep the finalized (empty) form.
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=num_tokens,
device=hidden_states.device,
)
flashinfer_output = flashinfer.fused_moe.trtllm_bf16_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -312,5 +309,4 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
num_tokens=num_tokens,
top_k=self.topk,
)
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
return routed_output
@@ -450,6 +450,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
import flashinfer
from flashinfer.fused_moe import Fp8QuantizationType, WeightLayout
@@ -487,11 +488,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
n_group = num_expert_group or 0
selected_topk_group = topk_group or 0
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
kwargs = dict(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -522,9 +518,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
if is_mxfp8 or activation == MoEActivation.RELU2_NO_MUL:
kwargs["activation_type"] = activation_type
result = flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return result
def _apply_per_tensor(
@@ -543,6 +536,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
# Delay import for non-CUDA.
import flashinfer
@@ -558,11 +552,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
else:
assert not apply_router_weight_on_input
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
out = flashinfer.fused_moe.trtllm_fp8_per_tensor_scale_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -586,9 +575,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
routing_replay_out=routing_replay_out,
)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return out
def apply(
@@ -607,6 +593,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
if self.quant_config.block_shape is not None:
return self._apply_block_scale(
@@ -623,6 +610,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_replay_out=routing_replay_out,
)
elif self.quant_config.is_per_tensor:
return self._apply_per_tensor(
@@ -638,6 +626,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
num_expert_group=num_expert_group,
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
routing_replay_out=routing_replay_out,
)
else:
raise NotImplementedError(
@@ -215,6 +215,7 @@ class TrtLlmMxfp4ExpertsMonolithic(
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
from flashinfer import trtllm_fp4_block_scale_moe
@@ -236,10 +237,6 @@ class TrtLlmMxfp4ExpertsMonolithic(
device=hidden_states.device,
)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=num_tokens,
device=hidden_states.device,
)
flashinfer_output = trtllm_fp4_block_scale_moe(
routing_logits=router_logits,
routing_bias=e_score_correction_bias,
@@ -279,7 +276,6 @@ class TrtLlmMxfp4ExpertsMonolithic(
top_k=self.topk,
finalized_output=finalized_output,
)
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
return routed_output
@@ -139,6 +139,7 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
from vllm.model_executor.layers.quantization.utils.flashinfer_mxint4_moe import ( # noqa: E501
flashinfer_trtllm_mxint4_moe,
@@ -147,10 +148,6 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
assert self.w1_scale is not None
assert self.w2_scale is not None
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=hidden_states.shape[0],
device=hidden_states.device,
)
result = flashinfer_trtllm_mxint4_moe(
x=hidden_states,
router_logits=router_logits,
@@ -169,7 +166,4 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
routing_method_type=self.routing_method,
routing_replay_out=routing_replay_out,
)
self._maybe_dispatch_routing_replay(
routing_replay_out, num_tokens=hidden_states.shape[0]
)
return result
@@ -515,6 +515,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
import flashinfer
@@ -545,10 +546,6 @@ class TrtLlmNvFp4ExpertsMonolithic(
# dummy 0-token forward has to keep the finalized (empty) form.
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
routing_replay_out = self._maybe_make_routing_replay_buffer(
num_tokens=num_tokens,
device=hidden_states.device,
)
# Invoke kernel.
# NOTE: Activation padding and output
# truncation are handled by the MoE runner's
@@ -592,5 +589,4 @@ class TrtLlmNvFp4ExpertsMonolithic(
num_tokens=num_tokens,
top_k=self.topk,
)
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
return routed_output
@@ -151,6 +151,7 @@ class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor:
if apply_router_weight_on_input:
raise NotImplementedError(
@@ -0,0 +1,481 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import contextlib
import sys
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Literal, cast
import torch
from vllm.config import get_current_vllm_config
from vllm.config.kernel import (
FLASHINFER_MOE_EP_BACKENDS,
FLASHINFER_MOE_EP_CUTEDSL,
FLASHINFER_MOE_EP_DEEP_GEMM,
)
from vllm.distributed import get_ep_group
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kMxfp4Static,
kNvfp4Static,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import _import_deep_gemm
if TYPE_CHECKING:
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
FlashInferMoeEpKernel = Literal["cutedsl", "deep_gemm"]
FlashInferMoeEpWeightFormat = Literal["nvfp4", "mxfp4"]
_WEIGHT_FORMATS: dict[QuantKey | None, FlashInferMoeEpWeightFormat] = {
kNvfp4Static: "nvfp4",
kMxfp4Static: "mxfp4",
}
_SUPPORTED_CAPABILITIES = frozenset({(10, 0), (10, 3)})
def supports_current_device() -> bool:
capability = current_platform.get_device_capability()
return capability is not None and (
(capability.major, capability.minor) in _SUPPORTED_CAPABILITIES
)
_E2M1_LUT = (
0.0,
0.5,
1.0,
1.5,
2.0,
3.0,
4.0,
6.0,
-0.0,
-0.5,
-1.0,
-1.5,
-2.0,
-3.0,
-4.0,
-6.0,
)
@dataclass(frozen=True)
class FlashInferMoeEpBackendSpec:
kernel: FlashInferMoeEpKernel
weight_formats: frozenset[FlashInferMoeEpWeightFormat]
@dataclass(frozen=True)
class FlashInferMoeEpWeights:
w13: torch.Tensor
w2: torch.Tensor
w13_scale: torch.Tensor | None = None
w2_scale: torch.Tensor | None = None
@dataclass(frozen=True)
class FlashInferMoeEpEpilogue:
input_norm_const: float = 1.0
fc1_alpha: torch.Tensor | None = None
fc2_alpha: torch.Tensor | None = None
fc1_norm_const: torch.Tensor | None = None
_BACKEND_SPECS = {
FLASHINFER_MOE_EP_CUTEDSL: FlashInferMoeEpBackendSpec(
kernel="cutedsl",
weight_formats=frozenset({"nvfp4", "mxfp4"}),
),
FLASHINFER_MOE_EP_DEEP_GEMM: FlashInferMoeEpBackendSpec(
kernel="deep_gemm",
weight_formats=frozenset({"mxfp4"}),
),
}
assert frozenset(_BACKEND_SPECS) == FLASHINFER_MOE_EP_BACKENDS
def is_flashinfer_moe_ep_backend(moe_backend: str) -> bool:
return moe_backend in _BACKEND_SPECS
def flashinfer_moe_ep_backend_spec(
moe_backend: str,
) -> FlashInferMoeEpBackendSpec:
try:
return _BACKEND_SPECS[moe_backend]
except KeyError:
raise ValueError(
f"{moe_backend!r} is not a FlashInfer MoE-EP backend; expected "
f"one of {sorted(_BACKEND_SPECS)}"
) from None
def flashinfer_moe_ep_unsupported_reasons(
moe: FusedMoEConfig,
weight_key: QuantKey | None,
activation_key: QuantKey | None,
) -> tuple[str, ...]:
"""Deployment constraints of the megakernel beyond the framework's checks.
``activation_key`` is ``None`` for MXFP4 checkpoints (the kernel quantizes
bf16 activations itself) but marks a W4A16 recipe for NVFP4 weights.
"""
spec = flashinfer_moe_ep_backend_spec(moe.moe_backend)
unsupported: list[str] = []
weight_format = _WEIGHT_FORMATS.get(weight_key)
if weight_format is None:
unsupported.append(f"weight scheme {weight_key}")
elif weight_format not in spec.weight_formats:
unsupported.append(f"{weight_format.upper()} weights")
if weight_key == kNvfp4Static and activation_key is None:
unsupported.append("A16 activations with NVFP4 weights")
if moe.skip_final_all_reduce:
unsupported.append("skip_final_all_reduce")
if moe.in_dtype != torch.bfloat16:
unsupported.append(f"activation dtype {moe.in_dtype}")
if moe.has_bias:
unsupported.append("expert bias")
if moe.swiglu_alpha is not None or moe.swiglu_beta is not None:
unsupported.append("custom SwiGLU alpha or beta")
if (
spec.kernel == "deep_gemm"
and moe.routing_method is not RoutingMethodType.DeepseekV4
):
unsupported.append(f"routing method {moe.routing_method.name}")
vllm_config = get_current_vllm_config()
if vllm_config.weight_transfer_config is not None:
unsupported.append("runtime weight transfer")
if vllm_config.parallel_config.enable_dbo:
unsupported.append("dual batch overlap")
if spec.kernel == "deep_gemm" and vllm_config.parallel_config.enable_eplb:
unsupported.append("EPLB")
return tuple(unsupported)
def validate_flashinfer_moe_ep_layer(layer: RoutedExperts) -> None:
if layer.expert_map_manager.placement_strategy != "linear":
raise ValueError(
f"{layer.moe_config.moe_backend} requires contiguous linear "
"expert placement"
)
def apply_topk_in_fc1(
moe: FusedMoEConfig,
*,
apply_router_weight_on_input: bool = False,
) -> bool:
return (
apply_router_weight_on_input
or moe.routing_method is RoutingMethodType.DeepseekV4
)
def _dequant_mxfp4_ue8m0_gran32(
packed: torch.Tensor,
scale: torch.Tensor,
) -> torch.Tensor:
raw = packed.view(torch.uint8)
lut = torch.tensor(_E2M1_LUT, dtype=torch.float32, device=raw.device)
values = torch.empty(
raw.shape[0],
raw.shape[1] * 2,
dtype=torch.float32,
device=raw.device,
)
values[:, ::2] = lut[(raw & 0x0F).to(torch.int64)]
values[:, 1::2] = lut[(raw >> 4).to(torch.int64)]
encoded_scale = (
scale
if scale.dtype == torch.float8_e8m0fnu
else scale.view(torch.float8_e8m0fnu)
)
decoded_scale = encoded_scale.float()
return (values * decoded_scale.repeat_interleave(32, dim=-1)).to(torch.bfloat16)
def _dequant_mxfp4_experts(
weight: torch.Tensor,
scale: torch.Tensor,
) -> torch.Tensor:
num_experts, output_size, packed_input_size = weight.shape
result = torch.empty(
num_experts,
output_size,
packed_input_size * 2,
dtype=torch.bfloat16,
device=weight.device,
)
for expert_id in range(num_experts):
result[expert_id] = _dequant_mxfp4_ue8m0_gran32(
weight[expert_id], scale[expert_id]
)
return result
def mxfp4_moe_ep_weights(
moe_backend: str,
w13: torch.Tensor,
w2: torch.Tensor,
w13_scale: torch.Tensor,
w2_scale: torch.Tensor,
) -> FlashInferMoeEpWeights:
spec = flashinfer_moe_ep_backend_spec(moe_backend)
if spec.kernel == "deep_gemm":
return FlashInferMoeEpWeights(w13, w2, w13_scale, w2_scale)
return FlashInferMoeEpWeights(
w13=_dequant_mxfp4_experts(w13, w13_scale),
w2=_dequant_mxfp4_experts(w2, w2_scale),
)
class _FlashInferMoeEpApi:
def __init__(self) -> None:
try:
from flashinfer.moe_ep import (
BootstrapConfig,
DeepGemmMegaMoeConfig,
FleetParams,
MegaConfig,
MoEEpMegaLayer,
MoEEpTensors,
MoEWeightPack,
Nvfp4CutedslMegaMoeConfig,
)
except ImportError as err:
raise RuntimeError(
"FlashInfer MoE-EP requires the public API shipped with "
"FlashInfer 0.7.0 or newer"
) from err
self.BootstrapConfig = BootstrapConfig
self.DeepGemmMegaMoeConfig = DeepGemmMegaMoeConfig
self.FleetParams = FleetParams
self.MegaConfig = MegaConfig
self.MoEEpMegaLayer = MoEEpMegaLayer
self.MoEEpTensors = MoEEpTensors
self.MoEWeightPack = MoEWeightPack
self.Nvfp4CutedslMegaMoeConfig = Nvfp4CutedslMegaMoeConfig
def _load_flashinfer_moe_ep_api() -> _FlashInferMoeEpApi:
return _FlashInferMoeEpApi()
def _expose_deep_gemm_to_flashinfer() -> None:
deep_gemm = _import_deep_gemm()
if deep_gemm is None:
raise RuntimeError(
"FlashInfer MoE-EP DeepGEMM requires a usable DeepGEMM installation"
)
sys.modules.setdefault("deep_gemm", deep_gemm)
class FlashInferMoeEp:
def __init__(
self,
moe: FusedMoEConfig,
weights: FlashInferMoeEpWeights,
epilogue: FlashInferMoeEpEpilogue | None = None,
*,
apply_topk_in_fc1: bool,
) -> None:
spec = flashinfer_moe_ep_backend_spec(moe.moe_backend)
if spec.kernel == "deep_gemm":
_expose_deep_gemm_to_flashinfer()
api = _load_flashinfer_moe_ep_api()
if epilogue is None:
epilogue = FlashInferMoeEpEpilogue()
ep_group = get_ep_group()
bootstrap = api.BootstrapConfig(
world_size=ep_group.world_size,
rank=ep_group.rank_in_group,
process_group=ep_group.device_group,
device=torch.accelerator.current_device_index(),
)
fleet_params = api.FleetParams(
num_experts=moe.num_experts,
max_tokens_per_rank=moe.max_num_tokens,
token_hidden_size=moe.hidden_dim,
)
weight_pack = api.MoEWeightPack(
w13=weights.w13,
w2=weights.w2,
w13_scale=weights.w13_scale,
w2_scale=weights.w2_scale,
)
if spec.kernel == "cutedsl":
megakernel = api.Nvfp4CutedslMegaMoeConfig(
intermediate_size=moe.intermediate_size,
top_k=moe.experts_per_token,
gate_up_clamp=moe.swiglu_limit,
fast_math=True,
apply_topk_in_fc1=apply_topk_in_fc1,
enable_in_kernel_fc2_reduce=False,
combine_dtype="bf16",
input_norm_const=epilogue.input_norm_const,
fc1_alpha=None,
fc2_alpha=None,
fc1_norm_const=None,
knobs=None,
)
else:
megakernel = api.DeepGemmMegaMoeConfig(
intermediate_size=moe.intermediate_size,
top_k=moe.experts_per_token,
activation_clamp=moe.swiglu_limit,
fast_math=True,
)
backend = api.MegaConfig(
megakernel=megakernel,
quantize_input=True,
preprocess_weights=True,
)
self._moe_ep_tensors_cls = api.MoEEpTensors
self._mega_layer: Any | None = api.MoEEpMegaLayer(
bootstrap,
fleet_params,
weight_pack,
backend,
)
self._device = weights.w13.device
self._hidden_size = moe.hidden_dim
self._max_num_tokens = moe.max_num_tokens
self._num_experts = moe.num_experts
self._top_k = moe.experts_per_token
self._epilogue: FlashInferMoeEpEpilogue | None = epilogue
self._return_workspace_view = moe.routing_method is RoutingMethodType.DeepseekV4
@property
def can_overlap_shared_experts(self) -> bool:
return False
@property
def output_is_reduced(self) -> bool:
return True
@property
def topk_indices_dtype(self) -> torch.dtype:
return torch.int32
@property
def is_monolithic(self) -> bool:
return False
def _tensors(
self,
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
) -> Any:
epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
return self._moe_ep_tensors_cls(
hidden_states=hidden_states,
topk_ids=topk_ids,
topk_weights=topk_weights,
fc1_alpha=epilogue.fc1_alpha,
fc2_alpha=epilogue.fc2_alpha,
fc1_norm_const=epilogue.fc1_norm_const,
)
def __call__(
self,
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
) -> torch.Tensor:
if self._mega_layer is None:
raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
if hidden_states.shape[0] > self._max_num_tokens:
raise ValueError(
f"FlashInfer MoE-EP got {hidden_states.shape[0]} tokens, but "
f"the workspace supports at most {self._max_num_tokens}"
)
tensors = self._tensors(hidden_states, topk_ids, topk_weights)
if self._return_workspace_view and getattr(
self._mega_layer, "supports_output_view", False
):
return self._mega_layer.forward(tensors, return_workspace_view=True)
return self._mega_layer(tensors)
@torch.inference_mode()
def warmup(self) -> None:
if self._mega_layer is None:
return
hidden_states = torch.zeros(
1,
self._hidden_size,
dtype=torch.bfloat16,
device=self._device,
)
topk_ids = (
torch.arange(self._top_k, dtype=torch.int32, device=self._device)
.mul_(self._num_experts // self._top_k)
.view(1, self._top_k)
)
topk_weights = torch.full(
(1, self._top_k),
1.0 / self._top_k,
dtype=torch.float32,
device=self._device,
)
self._mega_layer.warmup(self._tensors(hidden_states, topk_ids, topk_weights))
def kernel_weights(
self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
"""Kernel-resident fc1 weight, fc1 scales, fc2 weight, fc2 scales.
Contiguous per-expert byte tensors that alias the memory the megakernel
reads, so a caller may permute experts in place (EPLB).
"""
if self._mega_layer is None:
raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
(fc1_weight, fc1_sf), (fc2_weight, fc2_sf) = self._mega_layer._transformed
return (
_expert_major_bytes(fc1_weight),
_expert_major_bytes(fc1_sf),
_expert_major_bytes(fc2_weight),
_expert_major_bytes(fc2_sf),
)
def expert_weight_views(self) -> tuple[torch.Tensor, ...]:
"""Every per-expert tensor the kernel reads: weights plus epilogue."""
epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
vectors = (epilogue.fc1_alpha, epilogue.fc2_alpha, epilogue.fc1_norm_const)
return (*self.kernel_weights(), *(v for v in vectors if v is not None))
def destroy(self) -> None:
mega_layer = self._mega_layer
if mega_layer is None:
return
mega_layer.destroy()
self._mega_layer = None
self._epilogue = None
def __del__(self) -> None:
with contextlib.suppress(Exception):
self.destroy()
def _expert_major_bytes(tensor: torch.Tensor) -> torch.Tensor:
"""Contiguous storage of a per-expert kernel tensor, 1-byte dtypes as uint8.
The CuTeDSL kernel keeps (E, K, N) transpose views over (E, N, K) storage.
"""
if not tensor.is_contiguous():
tensor = tensor.transpose(1, 2)
if not tensor.is_contiguous():
raise ValueError(f"expert tensor is not expert-major: {tensor.stride()}")
return tensor.view(torch.uint8) if tensor.element_size() == 1 else tensor
@@ -23,6 +23,9 @@ from vllm.model_executor.layers.fused_moe.config import (
RoutingMethodType,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsSink,
)
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
SharedExperts,
SharedExpertsOrder,
@@ -979,9 +982,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
def is_monolithic() -> bool:
return True
routing_replay_capture_fn: Callable[[torch.Tensor], None] | None = None
_routing_replay_buffer: torch.Tensor | None = None
def supports_routing_replay_capture(self) -> bool:
"""Whether this expert supports routing replay capture.
@@ -990,53 +990,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
"""
return False
def set_capture_fn(
self,
capture_fn: Callable[[torch.Tensor], None] | None,
) -> None:
self.routing_replay_capture_fn = capture_fn
if capture_fn is None:
self._routing_replay_buffer = None
return
# Allocate for per-rank batches gathered across the DP or EP group.
dispatch_group_size = (
self.moe_config.ep_size
if self.moe_config.use_ep
else self.moe_config.dp_size
)
max_num_replay_tokens = self.moe_config.max_num_tokens * dispatch_group_size
self._routing_replay_buffer = torch.empty(
(max_num_replay_tokens, self.moe_config.experts_per_token),
dtype=torch.int16,
device=self.moe_config.device,
)
def _maybe_make_routing_replay_buffer(
self,
num_tokens: int,
device: torch.device,
) -> torch.Tensor | None:
if self.routing_replay_capture_fn is None:
return None
buf = self._routing_replay_buffer
assert buf is not None
if buf.shape[0] < num_tokens or buf.device != device:
raise ValueError(
"Routing replay buffer was initialized for "
f"{buf.shape[0]} tokens on {buf.device}, but the kernel "
f"received {num_tokens} tokens on {device}."
)
return buf
def _maybe_dispatch_routing_replay(
self,
routing_replay_out: torch.Tensor | None,
num_tokens: int,
) -> None:
if routing_replay_out is None or self.routing_replay_capture_fn is None:
return
self.routing_replay_capture_fn(routing_replay_out[:num_tokens])
def apply(
self,
hidden_states: torch.Tensor,
@@ -1053,10 +1006,14 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
routing_replay_out: torch.Tensor | None = None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Same as ``FusedMoEExperts.apply``, except uses router_logits as opposed
to the topk_ids and topk_weights. This is useful for kernels
with fused router and fused_experts (e.g. FLASHINFER_TRTLLM).
Kernels that ``supports_routing_replay_capture`` write each token's
routed expert ids into the leading rows of ``routing_replay_out``.
"""
raise NotImplementedError
@@ -1563,6 +1520,8 @@ class FusedMoEKernelMonolithicImpl:
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
"""Same as forward(), except uses router_logits as opposed
to the topk_ids and topk_weights. This is used for kernels
@@ -1575,6 +1534,15 @@ class FusedMoEKernelMonolithicImpl:
defer_input_quant=self.fused_experts.expects_unquantized_inputs,
)
routing_replay_out = None
if routing_sink is not None:
# Checked per call: a weight reload may rebuild a different kernel.
if not self.fused_experts.supports_routing_replay_capture():
raise ValueError(
"Routed-experts capture is not supported with monolithic MoE "
f"kernel {type(self.fused_experts).__name__}."
)
routing_replay_out = routing_sink.buffer[: len(a1q)]
fused_out = self.fused_experts.apply(
hidden_states=a1q,
w1=w1,
@@ -1590,7 +1558,10 @@ class FusedMoEKernelMonolithicImpl:
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_replay_out=routing_replay_out,
)
if routing_sink is not None:
routing_sink.capture_fn(routing_replay_out)
if isinstance(fused_out, UnfinalizedMoEOutput):
if not self.prepare_finalize.supports_deferred_moe_finalize():
@@ -1698,6 +1669,8 @@ class FusedMoEKernel:
e_score_correction_bias: torch.Tensor | None = None,
routed_scaling_factor: float | None = None,
topk_group: int | None = None,
*,
routing_sink: RoutedExpertsSink | None,
) -> torch.Tensor | UnfinalizedMoEOutput:
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
return self.impl.apply(
@@ -1713,6 +1686,7 @@ class FusedMoEKernel:
e_score_correction_bias=e_score_correction_bias,
routed_scaling_factor=routed_scaling_factor,
topk_group=topk_group,
routing_sink=routing_sink,
)
def apply(
@@ -3,6 +3,7 @@
import torch
from torch._subclasses.fake_tensor import FakeTensor
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
@@ -75,11 +76,18 @@ def moe_fused_mul_sum_kernel(
def _heuristic_config(
hidden_size: int,
element_size: int,
device_index: int = 0,
):
is_fp32 = element_size > 2
max_block_k = 256 if is_fp32 else 512
BLOCK_K = max(128, min(triton.next_power_of_2(hidden_size), max_block_k))
num_warps = 4 if is_fp32 else 2
if not is_fp32 and hidden_size >= 1024 and current_platform.is_cuda():
capability = current_platform.get_device_capability(device_index)
if capability is not None and capability >= (7, 5):
# Larger tiles reduce the serial hidden-dimension loop.
max_block_k = 4096
num_warps = 8 if capability == (7, 5) else 16
BLOCK_K = max(128, min(triton.next_power_of_2(hidden_size), max_block_k))
num_stages = 3
return BLOCK_K, num_warps, num_stages
@@ -149,6 +157,7 @@ def moe_fused_mul_sum(
BLOCK_K, num_warps, num_stages = _heuristic_config(
hidden_size,
inputs.element_size(),
inputs.device.index or 0,
)
grid = (num_tokens,)
moe_fused_mul_sum_kernel[grid](
@@ -8,7 +8,11 @@ import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm import envs
from vllm.config import get_current_vllm_config
from vllm.config.kernel import MoEBackend
from vllm.config.kernel import (
FLASHINFER_MOE_EP_CUTEDSL,
FLASHINFER_MOE_EP_DEEP_GEMM,
MoEBackend,
)
from vllm.config.quantization import QuantizationConfigArgs
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
@@ -145,6 +149,9 @@ class Mxfp4MoeBackend(Enum):
EMULATION = "EMULATION"
# Humming
HUMMING = "HUMMING"
# FlashInfer MoE-EP megakernels: routing outside, dispatch/GEMMs/combine fused.
FLASHINFER_MOE_EP_CUTEDSL = "FLASHINFER_MOE_EP_CUTEDSL"
FLASHINFER_MOE_EP_DEEP_GEMM = "FLASHINFER_MOE_EP_DEEP_GEMM"
# Backends that share the same TRTLLM weight format
@@ -165,6 +172,13 @@ B12X_BACKENDS = (
Mxfp4MoeBackend.B12X_MXFP4_BF16,
)
# Megakernels consume the checkpoint's MXFP4 tensors through the shared
# FlashInfer MoE-EP adapter; vLLM does not convert their weight layout.
FLASHINFER_MOE_EP_MXFP4_BACKENDS = (
Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM,
)
def backend_to_kernel_cls(
backend: Mxfp4MoeBackend,
@@ -291,6 +305,13 @@ def backend_to_kernel_cls(
return [OCP_MXQuantizationEmulationTritonExperts]
elif backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import ( # noqa: E501
FlashInferMoeEpExperts,
)
return [FlashInferMoeEpExperts]
else:
raise ValueError(f"Unknown MXFP4 MoE backend: {backend.value}")
@@ -314,6 +335,10 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
],
"flashinfer_cutlass_afp8": [Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8],
"flashinfer_moe_ep_cutedsl": [Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL],
"flashinfer_moe_ep_mega_deep_gemm": [
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM
],
"triton": [Mxfp4MoeBackend.TRITON],
"triton_unfused": [Mxfp4MoeBackend.TRITON_UNFUSED],
"humming": [Mxfp4MoeBackend.HUMMING],
@@ -735,6 +760,9 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
activation: MoEActivation | None = None,
) -> tuple[int, int]:
"""Round up hidden_size and intermediate_size based on backend requirements."""
if backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
# The megakernel takes the model's dimensions and validates them itself.
return hidden_size, intermediate_size
if backend in B12X_BACKENDS:
# b12x plans for the exact model dimensions. B12xExperts validates the
# required MXFP4 block alignment before selecting the backend.
@@ -1827,6 +1855,29 @@ def convert_weight_to_mxfp4_moe_kernel_format(
w2_bias=w2_bias,
_cache_permute_indices=_cache_permute_indices,
)
elif mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
mxfp4_moe_ep_weights,
)
# DeepGEMM consumes MXFP4 directly; CuTeDSL gets bf16 and requantizes.
weights = mxfp4_moe_ep_weights(
FLASHINFER_MOE_EP_CUTEDSL
if mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL
else FLASHINFER_MOE_EP_DEEP_GEMM,
w13_weight,
w2_weight,
w13_weight_scale,
w2_weight_scale,
)
return (
weights.w13,
weights.w2,
weights.w13_scale,
weights.w2_scale,
w13_bias,
w2_bias,
)
elif mxfp4_backend == Mxfp4MoeBackend.CPU:
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
prepare_mxfp4_moe_layer_for_cpu,
@@ -1874,6 +1925,9 @@ def make_mxfp4_moe_quant_config(
layer: "RoutedExperts | None" = None,
) -> FusedMoEQuantConfig | None:
"""Create a FusedMoEQuantConfig for the given MXFP4 backend."""
if mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
# The megakernel quantizes activations itself and owns its scales.
return FusedMoEQuantConfig.make("nvfp4", weight_dtype="mxfp4")
if mxfp4_backend == Mxfp4MoeBackend.B12X_MXFP4_MXFP8:
return mxfp4_mxfp8_moe_quant_config(
w1_bias=w1_bias,
@@ -50,6 +50,7 @@ class NvFp4MoeBackend(Enum):
MARLIN = "MARLIN"
HUMMING = "HUMMING"
EMULATION = "EMULATION"
FLASHINFER_MOE_EP_CUTEDSL = "FLASHINFER_MOE_EP_CUTEDSL"
FLASHINFER_NVFP4_MOE_BACKENDS = [
@@ -148,6 +149,12 @@ def backend_to_kernel_cls(
)
return [Nvfp4QuantizationEmulationTritonExperts]
elif backend == NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL:
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import ( # noqa: E501
FlashInferMoeEpExperts,
)
return [FlashInferMoeEpExperts]
else:
raise ValueError(f"Unknown NvFP4 MoE backend: {backend.value}")
@@ -160,6 +167,7 @@ def map_nvfp4_backend(runner_backend: MoEBackend) -> NvFp4MoeBackend:
"flashinfer_trtllm": NvFp4MoeBackend.FLASHINFER_TRTLLM,
"flashinfer_cutlass": NvFp4MoeBackend.FLASHINFER_CUTLASS,
"flashinfer_cutedsl": NvFp4MoeBackend.FLASHINFER_CUTEDSL,
"flashinfer_moe_ep_cutedsl": NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
"flashinfer_b12x": NvFp4MoeBackend.FLASHINFER_B12X,
"marlin": NvFp4MoeBackend.MARLIN,
"humming": NvFp4MoeBackend.HUMMING,
@@ -207,6 +215,7 @@ def select_nvfp4_moe_backend(
NvFp4MoeBackend.FLASHINFER_TRTLLM,
NvFp4MoeBackend.FLASHINFER_CUTLASS,
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
NvFp4MoeBackend.VLLM_CUTLASS,
NvFp4MoeBackend.MARLIN,
NvFp4MoeBackend.EMULATION,
@@ -281,8 +290,9 @@ def select_nvfp4_moe_backend(
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
f"explicitly requested moe_backend={runner_backend!r} does "
f"not apply the SwiGLU clamp. Use 'flashinfer_trtllm', "
f"'flashinfer_cutlass', 'flashinfer_cutedsl', 'cutlass', "
f"'b12x', 'marlin', or 'humming' instead."
f"'flashinfer_cutlass', 'flashinfer_cutedsl', "
f"'flashinfer_moe_ep_cutedsl', 'cutlass', 'b12x', 'marlin', "
f"or 'humming' instead."
)
return _return_or_raise(
requested_backend, config, weight_key, activation_key, activation_format
@@ -491,6 +501,9 @@ def convert_to_nvfp4_moe_kernel_format(
# for other experts - other selection strategies may be used.
a13_scale = 1.0 / a13_scale.max().to(torch.float32)
a2_scale = 1.0 / a2_scale.max().to(torch.float32)
elif nvfp4_backend == NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL:
# The megakernel transforms the checkpoint tensors itself at init.
pass
else:
raise ValueError(f"Unknown NvFp4 backend for MoE: {nvfp4_backend}")
@@ -0,0 +1,41 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Prepare/finalize stages that do no expert-parallel communication.
Selected by ``all2all_backend="passthrough"``: the experts implementation
dispatches, combines and reduces itself, so nothing is left for these stages.
"""
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
from vllm.model_executor.layers.fused_moe.prepare_finalize.no_dp_ep import (
MoEPrepareAndFinalizeNoDPEPModular,
)
class PassThroughPrepareAndFinalize(MoEPrepareAndFinalizeNoDPEPModular):
"""Hands activations and routing straight to experts that own their EP
communication."""
def output_is_reduced(self) -> bool:
return True
def supports_deferred_moe_finalize(self) -> bool:
# The experts apply the top-k weights and combine internally; there
# is no finalize left for a consumer to take over.
return False
def prepare(
self,
a1: torch.Tensor,
topk_weights: torch.Tensor,
topk_ids: torch.Tensor,
num_experts: int,
expert_map: torch.Tensor | None,
apply_router_weight_on_input: bool,
quant_config: FusedMoEQuantConfig,
defer_input_quant: bool = False,
) -> mk.PrepareResultType:
return a1, None, None, None, None
@@ -20,6 +20,9 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
FusedMoEMethodBase,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
RoutedExpertsSink,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
@@ -138,6 +141,8 @@ class RoutedExperts(PluggableLayer):
self.swiglu_alpha = swiglu_alpha
self.swiglu_beta = swiglu_beta
self.e_score_correction_bias = e_score_correction_bias
# Set by bind_routed_experts_capturer for monolithic kernels.
self.routing_sink: RoutedExpertsSink | None = None
self.apply_router_weight_on_input = apply_router_weight_on_input
# End random parameters
self._loaded_expert_biases: set[str] = set()
@@ -15,6 +15,7 @@ import torch
from vllm.config import VllmConfig
from vllm.distributed.parallel_state import get_tp_group
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
from vllm.platforms import current_platform
logger = logging.getLogger(__name__)
@@ -186,6 +187,23 @@ class RoutedExpertsCapturer:
return self.device_buffer[:num_tokens].to(self.output_dtype)
class RoutedExpertsSink:
"""Layer-owned buffer and callback for the expert ids a monolithic kernel
routes to; it outlives kernel rebuilds on weight reload."""
def __init__(
self, moe_config: FusedMoEConfig, capture_fn: Callable[[torch.Tensor], None]
) -> None:
# Sized for per-rank batches gathered across the DP or EP group.
group_size = moe_config.ep_size if moe_config.use_ep else moe_config.dp_size
self.buffer = torch.empty(
(moe_config.max_num_tokens * group_size, moe_config.experts_per_token),
dtype=torch.int16,
device=moe_config.device,
)
self.capture_fn = capture_fn
def bind_routed_experts_capturer(
model: torch.nn.Module,
capturer: RoutedExpertsCapturer,
@@ -219,7 +237,9 @@ def bind_routed_experts_capturer(
"Routed-experts capture is not supported with monolithic "
f"MoE kernel {type(fused_experts).__name__}."
)
fused_experts.set_capture_fn(capture_fn)
module.routed_experts.routing_sink = RoutedExpertsSink(
fused_experts.moe_config, capture_fn
)
num_bound += 1
elif isinstance(module.router, BaseRouter):
module.router.set_capture_fn(capture_fn)
@@ -362,4 +362,5 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
+57 -10
View File
@@ -163,6 +163,14 @@ class SparseIndexerTopk(torch.nn.Module):
current_platform.has_device_capability(90)
and not current_platform.is_device_capability_family(120)
)
self._is_rocm = current_platform.is_rocm()
self._aiter_enabled = False
self._aiter_capable = False
if self._is_rocm:
from vllm._aiter_ops import rocm_aiter_ops
self._aiter_enabled = bool(rocm_aiter_ops.is_enabled())
self._aiter_capable = bool(rocm_aiter_ops.is_indexer_top_k_enabled())
def resolve_backend(
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
@@ -193,6 +201,19 @@ class SparseIndexerTopk(torch.nn.Module):
f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
f" stride={logits.stride()}, topk={topk_tokens}"
)
elif self._backend == "aiter":
if not self._is_rocm:
failures.append("requires a ROCm platform")
elif not self._aiter_enabled:
failures.append("requires AITER (VLLM_ROCM_USE_AITER=1)")
elif not self._aiter_capable:
failures.append(
"AITER has no tuned indexer top-k kernels for this GPU arch"
)
if topk_tokens not in (512, 1024, 2048, 4096):
failures.append(
f"topk_tokens must be in (512, 1024, 2048, 4096), got {topk_tokens}"
)
elif self._backend == "flashinfer":
if not self._is_cuda:
failures.append("requires a CUDA platform")
@@ -216,11 +237,14 @@ class SparseIndexerTopk(torch.nn.Module):
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
) -> str:
"""The priority chain: cooperative -> persistent -> per_row.
deep_select/flashinfer/torch are opt-in only.
aiter/deep_select/flashinfer/torch are opt-in only.
cooperative_topk is preferred whenever it is applicable, i.e. within
its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to
persistent_topk.
aiter not auto-resolved as it does not outperform the per-row on every shape,
hence we delegate to the caller to decide when to use it.
"""
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
return "cooperative"
@@ -285,7 +309,8 @@ class SparseIndexerTopk(torch.nn.Module):
max_seq_len: int,
) -> None:
"""Run the resolved decode top-k implementation, writing into
topk_indices (int32, -1 fill for rows shorter than topk_tokens)."""
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
"""
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
if backend == "deep_select":
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
@@ -344,15 +369,37 @@ class SparseIndexerTopk(torch.nn.Module):
) < row_ends.unsqueeze(1)
indices = torch.where(in_range, indices, -1)
topk_indices.copy_(indices)
else:
assert backend == "per_row", f"unknown topk backend: {backend}"
ops.top_k_per_row_decode(
elif backend == "aiter":
from vllm._aiter_ops import rocm_aiter_ops
# AITER's decode kernel only implements the 1D/next_n == 1 form,
# which is equivalent once the ragged ends are expanded.
rocm_aiter_ops.indexer_top_k_decode(
logits,
next_n,
seq_lens,
1,
self._row_ends(seq_lens, next_n, logits.shape[0]),
topk_indices,
logits.shape[0],
logits.stride(0),
logits.stride(1),
topk_tokens,
)
else:
assert backend == "per_row", f"unknown topk backend: {backend}"
self._per_row(logits, seq_lens, next_n, topk_indices, topk_tokens)
@staticmethod
def _per_row(
logits: torch.Tensor,
seq_lens: torch.Tensor,
next_n: int,
topk_indices: torch.Tensor,
topk_tokens: int,
) -> None:
ops.top_k_per_row_decode(
logits,
next_n,
seq_lens,
topk_indices,
logits.shape[0],
logits.stride(0),
logits.stride(1),
topk_tokens,
)
@@ -551,28 +551,33 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
raise ValueError(
f"VLLM_GDN_DECODE_KERNEL=cuda is not supported: {reason}"
)
logger.info_once(
"Falling back to the Triton GDN decode path: %s", reason
)
if self.speculative_config is not None:
logger.info_once(
"Falling back to the Triton GDN spec decode path: %s",
reason,
)
self.gdn_decode_kernel = "triton"
elif current_platform.is_cpu():
self.gdn_decode_kernel = "CPU"
elif current_platform.is_xpu():
self.gdn_decode_kernel = "XPU"
self.enable_fused_gdn_decode = self.gdn_decode_kernel == "cuda"
self.enable_fused_gdn_spec_decode = (
self.gdn_decode_kernel == "cuda" and self.speculative_config is not None
)
# RecoverSSM verify replaces the per-draft snapshot path (Triton decode
# only)
self.use_gdn_recoverssm = self._uses_gdn_recoverssm()
if self.use_gdn_recoverssm:
self.gdn_decode_kernel = "triton"
self.enable_fused_gdn_decode = False
self.enable_fused_gdn_spec_decode = False
self.enable_packed_recurrent_decode = False
logger.info_once(
"GDN RecoverSSM speculative verify active (spec_query_len %d)",
self.num_spec + 1,
)
logger.info_once("GDN decode kernel: %s", self.gdn_decode_kernel)
if self.speculative_config is not None:
logger.info_once("GDN spec decode kernel: %s", self.gdn_decode_kernel)
compilation_config = get_current_vllm_config().compilation_config
if prefix in compilation_config.static_forward_context:
@@ -969,12 +974,12 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
mixed_qkvz, _ = self.in_proj_qkvz(hidden_states)
ba, _ = self.in_proj_ba(hidden_states)
use_fused_gdn_decode = (
self.enable_fused_gdn_decode
use_fused_gdn_spec_decode = (
self.enable_fused_gdn_spec_decode
and hidden_states.dtype == torch.bfloat16
and self.norm.weight.dtype in (torch.bfloat16, torch.float32)
)
if use_fused_gdn_decode:
if use_fused_gdn_spec_decode:
core_attn_out = torch.zeros(
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
dtype=hidden_states.dtype,
@@ -790,6 +790,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -779,4 +779,5 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -24,10 +24,15 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
XPUExpertsMxFp4,
)
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
is_flashinfer_moe_ep_backend,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
B12X_BACKENDS,
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
Mxfp4MoeBackend,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
make_mxfp4_moe_quant_config,
select_mxfp4_moe_backend,
@@ -38,7 +43,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
prepare_moe_fp4_layer_for_marlin,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -49,12 +54,14 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
super().__init__(moe)
self.group_size = 32
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
self.experts_cls: type[mk.FusedMoEExperts] | None = None
# Backend selection must match the weight preparation below: CUTLASS
# swizzles scales, b12x and XPU consume checkpoint packing, and Marlin
# repacks weights and scales.
# swizzles scales, b12x and XPU consume checkpoint packing, Marlin
# repacks weights and scales, and the megakernels take the checkpoint
# tensors as they are.
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
self.experts_cls: type[mk.FusedMoEExperts]
if moe.moe_backend == "b12x":
if moe.moe_backend == "b12x" or is_flashinfer_moe_ep_backend(moe.moe_backend):
self.mxfp4_backend, experts_cls = select_mxfp4_moe_backend(moe)
assert experts_cls is not None
self.experts_cls = experts_cls
@@ -70,6 +77,10 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
logger.info_once("Using MarlinExperts for MXFP4 MoE")
self.experts_cls = MarlinExperts
@property
def supports_eplb(self) -> bool:
return self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS
def create_weights(
self,
layer: torch.nn.Module,
@@ -198,6 +209,21 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
layer.w2_weight_scale = torch.nn.Parameter(
torch.stack(swizzled_w2), requires_grad=False
)
elif self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
w13, w2, w13_scale, w2_scale, _, _ = (
convert_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=self.mxfp4_backend,
layer=layer,
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
w13_weight_scale=layer.w13_weight_scale,
w2_weight_scale=layer.w2_weight_scale,
)
)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w13_weight_scale", w13_scale)
replace_parameter(layer, "w2_weight_scale", w2_scale)
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
pass
else:
@@ -211,6 +237,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None:
assert self.experts_cls is not None
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
@@ -38,6 +38,7 @@ _EPLB_SUPPORTED_NVFP4_BACKENDS = frozenset(
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
NvFp4MoeBackend.FLASHINFER_CUTEDSL_BATCHED,
NvFp4MoeBackend.FLASHINFER_TRTLLM,
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
}
)
@@ -61,7 +62,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
weight_key=kNvfp4Static,
activation_key=None if use_a16 else kNvfp4Dynamic,
)
self.use_global_sf = is_global_sf_supported_for_nvfp4_backend(
self.nvfp4_backend
)
@@ -155,7 +155,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
set_weight_attrs(w13_weight_scale_2, extra_weight_attrs)
w2_weight_scale_2 = torch.nn.Parameter(
torch.empty(num_experts, dtype=torch.float32), requires_grad=False
torch.empty(num_experts, dtype=torch.float32),
requires_grad=False,
)
layer.register_parameter("w2_weight_global_scale", w2_weight_scale_2)
extra_weight_attrs.update(
@@ -175,7 +176,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
set_weight_attrs(w13_input_scale, extra_weight_attrs)
w2_input_scale = torch.nn.Parameter(
torch.empty(num_experts, dtype=torch.float32), requires_grad=False
torch.empty(num_experts, dtype=torch.float32),
requires_grad=False,
)
layer.register_parameter("w2_input_global_scale", w2_input_scale)
extra_weight_attrs.update(
@@ -293,6 +295,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -287,6 +287,7 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -366,6 +366,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -291,4 +291,5 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -179,6 +179,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -528,6 +528,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -818,6 +818,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -4,6 +4,7 @@
import torch
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
from vllm.config.kernel import FLASHINFER_MOE_EP_BACKENDS
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import (
FusedMoeWeightScaleSupported,
@@ -53,6 +54,11 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
def __init__(self, moe) -> None:
super().__init__(moe)
if moe.moe_backend in FLASHINFER_MOE_EP_BACKENDS:
raise ValueError(
f"moe_backend={moe.moe_backend!r} is not supported for AutoRound "
"(INC) MXFP4 MoE checkpoints"
)
self.group_size = 32
# Backend selection must stay consistent with the weight preparation in
# process_weights_after_loading / get_fused_moe_quant_config, which use
@@ -201,7 +201,7 @@ class QuantFP8(CustomOp):
if scale is None:
if self.group_shape == GroupShape.PER_TOKEN:
x_max, _ = x.abs().max(dim=-1)
x_max = x.abs().amax(dim=-1)
x_max = x_max.unsqueeze(-1).to(torch.float32)
if scale_ub is not None:
x_max = x_max.clamp(max=scale_ub)
@@ -680,6 +680,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -1092,6 +1093,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -1461,6 +1463,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -2314,9 +2317,9 @@ def select_linear_kernel(
w = spec.weight
assert isinstance(w, QuantKey), f"resolve() must supply a weight key, got {w!r}"
if w.dtype == FP4_DTYPE:
# W4A16 (activation is None) → use_a16=True defaults to Marlin *and*
# honors --linear-backend (matches upstream ModelOptNvFp4W4A16LinearMethod
# after #50273); W4A4 → use_a16=False.
# W4A16 (activation is None) → use_a16=True, auto prefers FlashInfer
# CuTe-DSL backend on SM100/103 when available and Marlin otherwise,
# and honors --linear-backend (#50273); W4A4 → use_a16=False.
return init_nvfp4_linear_kernel(use_a16=spec.activation is None)
if w.scale.dtype == MXFP8_SCALE_DTYPE:
return init_mxfp8_linear_kernel(
@@ -531,6 +531,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@staticmethod
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import torch
from vllm.logger import init_logger
@@ -21,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
TRITON_BACKENDS,
Mxfp4MoeBackend,
backend_to_kernel_cls,
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
convert_weight_to_mxfp4_moe_kernel_format,
make_mxfp4_moe_kernel,
@@ -472,9 +475,67 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool:
"""Route Kimi-K3 weight-only MXFP4 SiTU MoE to AITER A16W4 on gfx950."""
from vllm.platforms import current_platform
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.platforms.rocm import on_gfx950
return (
rocm_aiter_ops.is_fused_moe_enabled()
and on_gfx950()
and moe.activation == MoEActivation.SITU
and moe.activation_situ_linear_beta is not None
and rocm_aiter_ops.get_aiter_activation_type("situ") is not None
)
def _use_k3_situ_int4_gfx942(moe: FusedMoEConfig) -> bool:
"""Opt-in gfx942 MXFP4-to-int4 requant for Kimi-K3 SiTU experts."""
from vllm.platforms import current_platform
if not current_platform.is_rocm():
return False
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.platforms.rocm import on_gfx942
return (
rocm_aiter_ops.is_fused_moe_enabled()
and on_gfx942()
and moe.activation == MoEActivation.SITU
and moe.activation_situ_linear_beta is not None
and _moe_weight_override_is_int4()
)
def _moe_weight_override_is_int4() -> bool:
"""True when --quantization-config.moe.weight int4_per_group_32 was set."""
from vllm.config import get_current_vllm_config
vllm_config = get_current_vllm_config()
if vllm_config is None:
return False
quant_args = getattr(vllm_config.model_config, "quantization_config", None)
moe_spec = getattr(quant_args, "moe", None)
weight = getattr(moe_spec, "weight", None)
if weight is None:
return False
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt4Static32,
)
return weight == kInt4Static32
class Mxfp4MoEMethod(FusedMoEMethodBase):
"""MXFP4 MoE quantization method."""
@@ -482,9 +543,33 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
def __init__(self, moe: FusedMoEConfig):
super().__init__(moe)
self.weight_dtype = "mxfp4"
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
self.is_k3_situ_int4_gfx942 = _use_k3_situ_int4_gfx942(moe)
self.experts_cls: type[mk.FusedMoEExperts] | None
if self.is_k3_situ_aiter or self.is_k3_situ_int4_gfx942:
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
if self.is_k3_situ_int4_gfx942:
logger.warning_once(
"Requantizing Kimi-K3 MXFP4 experts to groupwise int4 "
"for gfx942. This conversion is lossy and was explicitly "
"enabled by --quantization-config.moe.weight "
"int4_per_group_32."
)
else:
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
from vllm._aiter_ops import rocm_aiter_ops
if rocm_aiter_ops.is_fused_moe_situv2_enabled():
# a16w4 does not read this bound. Force 0 so leftover a8w4
# config cannot keep bf16 activations.
# TODO: Remove once AITER takes this as a kernel argument.
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
else:
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
moe
)
self.max_capture_size = moe.max_capture_size
@@ -782,7 +867,142 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
)
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
def process_weights_after_loading(self, layer):
def _setup_kernel_k3_situ_gfx942(self, layer: RoutedExperts) -> None:
import inspect
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
flydsl_a16w4_gemm1,
)
# Fail closed if this AITER build would silently run SiLU.
if "act" not in inspect.signature(flydsl_a16w4_gemm1).parameters:
raise RuntimeError(
"This AITER build ignores the SiTUv2 activation on the "
"replacement packed-int4 MoE path and would silently compute "
"SiLU. Rebuild with the replacement A16W4/int4 pipeline."
)
from aiter import dtypes as aiter_dtypes
from aiter.ops.quant import per_1x32_i4_quant
from aiter.ops.shuffle import (
pack_int8_to_packed_int4,
shuffle_scale_for_int4,
shuffle_weight,
)
from aiter.utility import fp4_utils
fp4_dtype = torch.float4_e2m1fn_x2
e8m0_dtype = torch.float8_e8m0fnu
# mxfp4_to_f32 grows the packed weight 8x and peaks above 20 GiB per
# rank if all experts convert at once. Convert 8 experts at a time.
_CONVERT_CHUNK = 8
def convert(
weight: torch.nn.Parameter,
scale: torch.nn.Parameter,
) -> tuple[torch.Tensor, torch.Tensor]:
# Drop the source before returning. Packed int4 is the same size as
# packed MXFP4, so holding both doubles the footprint. That fits
# under expert parallel (1/ep_size experts per rank) and does not
# fit under pure tensor parallel.
w_all = weight.data.view(fp4_dtype)
s_all = scale.data.view(e8m0_dtype)
num_experts = w_all.shape[0]
# Write chunks into a preallocated output. torch.cat of a list
# would hold the full result twice at the join.
out_packed: torch.Tensor | None = None
out_scale: torch.Tensor | None = None
scale_stride = 0
for lo in range(0, num_experts, _CONVERT_CHUNK):
hi = min(lo + _CONVERT_CHUNK, num_experts)
weight_f32 = fp4_utils.mxfp4_to_f32(w_all[lo:hi])
scale_f32 = fp4_utils.e8m0_to_f32(s_all[lo:hi])
chunk_experts, output_size, input_size = weight_f32.shape
weight_bf16 = (
(
weight_f32.view(
chunk_experts, output_size, input_size // 32, 32
)
* scale_f32.view(
chunk_experts, output_size, input_size // 32, 1
)
)
.view(chunk_experts, output_size, input_size)
.to(torch.bfloat16)
)
del weight_f32, scale_f32
weight_int4, weight_scale = per_1x32_i4_quant(weight_bf16)
del weight_bf16
weight_int4 = weight_int4.view(aiter_dtypes.i4x2).view(
chunk_experts, output_size, input_size
)
weight_packed = pack_int8_to_packed_int4(
shuffle_weight(weight_int4.view(aiter_dtypes.i8), (16, 16))
)
del weight_int4
chunk_packed = weight_packed.view(
chunk_experts, output_size, input_size // 2
).view(aiter_dtypes.i4x2)
chunk_scale = (
shuffle_scale_for_int4(weight_scale, group_size=32)
.view(-1)
.contiguous()
)
if out_packed is None:
out_packed = torch.empty(
(num_experts,) + tuple(chunk_packed.shape[1:]),
dtype=chunk_packed.dtype,
device=chunk_packed.device,
)
scale_stride = chunk_scale.numel() // chunk_experts
out_scale = torch.empty(
num_experts * scale_stride,
dtype=chunk_scale.dtype,
device=chunk_scale.device,
)
assert out_packed is not None
assert out_scale is not None
out_packed[lo:hi].copy_(chunk_packed)
out_scale[lo * scale_stride : hi * scale_stride].copy_(chunk_scale)
del weight_packed, weight_scale, chunk_packed, chunk_scale
torch.accelerator.empty_cache()
del w_all, s_all
assert out_packed is not None
assert out_scale is not None
weight.data = torch.empty(0, dtype=torch.uint8, device=out_packed.device)
scale.data = torch.empty(0, dtype=torch.uint8, device=out_packed.device)
torch.accelerator.empty_cache()
return out_packed, out_scale
w13, w13_scale = convert(layer.w13_weight, layer.w13_weight_scale)
replace_parameter(layer, "w13_weight", w13)
replace_parameter(layer, "w13_weight_scale", w13_scale)
del w13, w13_scale
torch.accelerator.empty_cache()
w2, w2_scale = convert(layer.w2_weight, layer.w2_weight_scale)
replace_parameter(layer, "w2_weight", w2)
replace_parameter(layer, "w2_weight_scale", w2_scale)
del w2, w2_scale
torch.accelerator.empty_cache()
layer.w13_weight.is_shuffled = True
layer.w2_weight.is_shuffled = True
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
if self.moe_quant_config is not None and self.experts_cls is not None:
self.moe_kernel = make_mxfp4_moe_kernel(
moe_quant_config=self.moe_quant_config,
moe_config=self.moe,
mxfp4_backend=self.mxfp4_backend,
experts_cls=self.experts_cls,
routing_tables=layer._expert_routing_tables(),
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
return
@@ -795,6 +1015,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
self._build_moe_kernel(layer)
return
if self.is_k3_situ_int4_gfx942:
self._setup_kernel_k3_situ_gfx942(layer)
return
w13 = layer.w13_weight
w2 = layer.w2_weight
w13_scale = layer.w13_weight_scale
@@ -900,4 +1124,5 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -65,6 +65,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Static128BlockSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kInt4Static32,
kInt8StaticChannelSym,
kMxfp4Static,
kMxfp8Dynamic,
@@ -231,6 +232,9 @@ class OnlineQuantizationConfig(QuantizationConfig):
"""
if spec is None or spec.weight is None:
return None
# Load-time gfx942 requant, not online conversion. Mxfp4MoEMethod owns it.
if spec.weight == kInt4Static32:
return None
cls = table.get(spec.weight)
if cls is None:
raise ValueError(
@@ -156,6 +156,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
def apply(
@@ -1976,6 +1976,11 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
global_num_experts=layer.global_num_experts,
expert_map=layer.expert_map,
apply_router_weight_on_input=layer.apply_router_weight_on_input,
num_expert_group=layer.num_expert_group,
topk_group=layer.topk_group,
e_score_correction_bias=layer.e_score_correction_bias,
routed_scaling_factor=layer.routed_scaling_factor,
routing_sink=layer.routing_sink,
)
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 128,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 2
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 2
},
"2048": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 16,
"num_stages": 3
}
}
@@ -0,0 +1,82 @@
{
"1": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 3
},
"4": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"16": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 256,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 16,
"num_stages": 2
},
"32": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
},
"64": {
"BLOCK_SIZE_M": 32,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 1,
"num_warps": 8,
"num_stages": 3
},
"128": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"512": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"1024": {
"BLOCK_SIZE_M": 64,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 3
},
"2048": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 8,
"num_stages": 2
},
"4096": {
"BLOCK_SIZE_M": 128,
"BLOCK_SIZE_N": 128,
"BLOCK_SIZE_K": 64,
"GROUP_SIZE_M": 32,
"num_warps": 32,
"num_stages": 3
}
}
@@ -25,6 +25,11 @@ INT4_DTYPE = scalar_types.uint4b8
INT8_DTYPE = scalar_types.uint8b128
def require_finite_positive_scales(name: str, value: torch.Tensor) -> None:
if not torch.isfinite(value).all() or not (value > 0).all():
raise ValueError(f"{name} must contain finite positive scales")
def _dtype_abbr(dtype: torch.dtype | ScalarType) -> str:
"""Return a stable short name for torch and ScalarType dtypes."""
if isinstance(dtype, ScalarType):
@@ -457,8 +457,6 @@ class DeepseekOCRForCausalLM(
return None
assert isinstance(pixel_values, torch.Tensor)
assert images_crop is None or isinstance(images_crop, torch.Tensor)
if torch.sum(pixel_values).item() == 0:
return None
# Use actual tensor spatial dim instead of hardcoded
# vision_config.image_size (1024). The vision encoders (SAM & CLIP)
+1 -1
View File
@@ -318,7 +318,7 @@ class DeepseekOCR2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Support
images_spatial_crop = kwargs.pop("images_spatial_crop", None)
images_crop = kwargs.pop("images_crop", None)
if pixel_values is None or torch.sum(pixel_values).item() == 0:
if pixel_values is None:
return None
base_size = self.vision_config.image_size
-324
View File
@@ -1,324 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt_neox/modeling_gpt_neox.py
# Copyright 2023 The vLLM team.
# Copyright 2022 EleutherAI The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only GPT-NeoX model compatible with HuggingFace weights."""
from collections.abc import Iterable
from itertools import islice
import torch
from torch import nn
from transformers import GPTNeoXConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsPP
from .utils import (
AutoWeightsLoader,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
class GPTNeoXAttention(nn.Module):
def __init__(
self,
config: GPTNeoXConfig,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.total_num_heads = config.num_attention_heads
self.hidden_size = config.hidden_size
self.head_size = self.hidden_size // self.total_num_heads
self.bias = getattr(config, "attention_bias", True)
tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
assert self.total_num_heads % tensor_model_parallel_world_size == 0
self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
self.query_key_value = QKVParallelLinear(
config.hidden_size,
self.head_size,
self.total_num_heads,
bias=self.bias,
quant_config=quant_config,
prefix=f"{prefix}.query_key_value",
)
self.dense = RowParallelLinear(
config.hidden_size,
config.hidden_size,
bias=self.bias,
quant_config=quant_config,
prefix=f"{prefix}.dense",
)
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
self.rotary_emb = get_rope(
self.head_size,
max_position=max_position_embeddings,
rope_parameters=config.rope_parameters,
)
scaling = self.head_size**-0.5
self.attn = Attention(
self.num_heads,
self.head_size,
scaling,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
def forward(
self,
position_ids: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
qkv, _ = self.query_key_value(hidden_states)
q, k, v = qkv.chunk(chunks=3, dim=-1)
q, k = self.rotary_emb(position_ids, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.dense(attn_output)
return output
class GPTNeoXMLP(nn.Module):
def __init__(
self,
config: GPTNeoXConfig,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.dense_h_to_4h = ColumnParallelLinear(
config.hidden_size,
config.intermediate_size,
quant_config=quant_config,
prefix=f"{prefix}.dense_h_to_4h",
)
self.dense_4h_to_h = RowParallelLinear(
config.intermediate_size,
config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.dense_4h_to_h",
)
self.act = get_act_fn(config.hidden_act)
def forward(self, hidden_states):
hidden_states, _ = self.dense_h_to_4h(hidden_states)
hidden_states = self.act(hidden_states)
hidden_states, _ = self.dense_4h_to_h(hidden_states)
return hidden_states
class GPTNeoXLayer(nn.Module):
def __init__(
self,
config: GPTNeoXConfig,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.use_parallel_residual = config.use_parallel_residual
self.input_layernorm = nn.LayerNorm(
config.hidden_size, eps=config.layer_norm_eps
)
self.post_attention_layernorm = nn.LayerNorm(
config.hidden_size, eps=config.layer_norm_eps
)
self.attention = GPTNeoXAttention(
config, cache_config, quant_config, prefix=f"{prefix}.attention"
)
self.mlp = GPTNeoXMLP(config, quant_config, prefix=f"{prefix}.mlp")
def forward(
self,
position_ids: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
attn_input = self.input_layernorm(hidden_states)
attn_output = self.attention(
position_ids=position_ids,
hidden_states=attn_input,
)
if self.use_parallel_residual:
# pseudocode:
# x = x + attn(ln1(x)) + mlp(ln2(x))
mlp_input = self.post_attention_layernorm(hidden_states)
mlp_output = self.mlp(mlp_input)
hidden_states = mlp_output + attn_output + hidden_states
else:
# pseudocode:
# x = x + attn(ln1(x))
# x = x + mlp(ln2(x))
attn_output = attn_output + hidden_states
mlp_input = self.post_attention_layernorm(attn_output)
mlp_output = self.mlp(mlp_input)
hidden_states = mlp_output + attn_output
return hidden_states
@support_torch_compile
class GPTNeoXModel(nn.Module):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_substr={"attention.bias": None, "attention.masked_bias": None}
)
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.config = config
self.embed_in = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
)
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: GPTNeoXLayer(
config, cache_config, quant_config, prefix=prefix
),
prefix=f"{prefix}.layers",
)
self.final_layer_norm = nn.LayerNorm(
config.hidden_size, eps=config.layer_norm_eps
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.hidden_size
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_in(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
position_ids: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_input_ids(input_ids)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
for layer in islice(self.layers, self.start_layer, self.end_layer):
hidden_states = layer(position_ids, hidden_states)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
hidden_states = self.final_layer_norm(hidden_states)
return hidden_states
def _repack_qkv(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[tuple[str, torch.Tensor]]:
# GPT-NeoX's fused QKV is laid out as (num_heads * 3 * head_size) on
# its output dim (0), while vLLM expects (3 * num_heads * head_size).
num_heads = self.config.num_attention_heads
for name, loaded_weight in weights:
if "query_key_value" in name:
shape = loaded_weight.shape
loaded_weight = loaded_weight.view((num_heads, 3, -1) + shape[1:])
loaded_weight = loaded_weight.transpose(0, 1)
loaded_weight = loaded_weight.reshape(shape)
yield name, loaded_weight
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(
self._repack_qkv(weights), mapper=self.hf_to_vllm_mapper
)
class GPTNeoXForCausalLM(nn.Module, SupportsPP):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
self.gpt_neox = GPTNeoXModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "gpt_neox")
)
self.embed_out = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "embed_out"),
)
if self.config.tie_word_embeddings:
self.embed_out = self.embed_out.tie_weights(self.gpt_neox.embed_in)
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.gpt_neox.make_empty_intermediate_tensors
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.gpt_neox.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.gpt_neox(
input_ids, positions, intermediate_tensors, inputs_embeds
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
logits = self.logits_processor(self.embed_out, hidden_states)
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights)
-441
View File
@@ -1,441 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
# Copyright 2023 The vLLM team.
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only Jais2 model compatible with HuggingFace weights."""
from collections.abc import Iterable
import torch
from torch import nn
from transformers import Jais2Config
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import (
get_pp_group,
get_tensor_model_parallel_world_size,
)
from vllm.model_executor.layers.activation import ReLUSquaredActivation
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
extract_layer_index,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
class Jais2MLP(nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
prefix: str = "",
) -> None:
super().__init__()
self.up_proj = ColumnParallelLinear(
input_size=hidden_size,
output_size=intermediate_size,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.up_proj",
)
self.down_proj = RowParallelLinear(
input_size=intermediate_size,
output_size=hidden_size,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
)
self.act_fn = ReLUSquaredActivation()
def forward(self, x):
x, _ = self.up_proj(x)
x = self.act_fn(x)
x, _ = self.down_proj(x)
return x
class Jais2Attention(nn.Module):
def __init__(
self,
config: Jais2Config,
hidden_size: int,
num_heads: int,
num_kv_heads: int,
max_position_embeddings: int = 8192,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
cache_config: CacheConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
layer_idx = extract_layer_index(prefix)
self.hidden_size = hidden_size
tp_size = get_tensor_model_parallel_world_size()
self.total_num_heads = num_heads
assert self.total_num_heads % tp_size == 0
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
if self.total_num_kv_heads >= tp_size:
# Number of KV heads is greater than TP size, so we partition
# the KV heads across multiple tensor parallel GPUs.
assert self.total_num_kv_heads % tp_size == 0
else:
# Number of KV heads is less than TP size, so we replicate
# the KV heads across multiple tensor parallel GPUs.
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
# MistralConfig has an optional head_dim introduced by Mistral-Nemo
self.head_dim = getattr(
config, "head_dim", self.hidden_size // self.total_num_heads
)
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = self.head_dim**-0.5
self.max_position_embeddings = max_position_embeddings
self.qkv_proj = QKVParallelLinear(
hidden_size=hidden_size,
head_size=self.head_dim,
total_num_heads=self.total_num_heads,
total_num_kv_heads=self.total_num_kv_heads,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.o_proj = RowParallelLinear(
input_size=self.total_num_heads * self.head_dim,
output_size=hidden_size,
bias=bias,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
is_neox_style = True
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position_embeddings,
rope_parameters=getattr(config, "rope_parameters", None),
is_neox_style=is_neox_style,
)
if hasattr(config, "interleaved_sliding_window"):
interleaved_sliding_window = config.interleaved_sliding_window
if isinstance(interleaved_sliding_window, int):
sliding_window = interleaved_sliding_window
elif isinstance(interleaved_sliding_window, list):
sw_idx = layer_idx % len(interleaved_sliding_window)
sliding_window = interleaved_sliding_window[sw_idx]
else:
raise ValueError(
f"{type(interleaved_sliding_window)} is not supported."
)
else:
sliding_window = None
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
per_layer_sliding_window=sliding_window,
prefix=f"{prefix}.attn",
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.o_proj(attn_output)
return output
class Jais2DecoderLayer(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
config: Jais2Config,
prefix: str = "",
) -> None:
super().__init__()
config = config or vllm_config.model_config.hf_config
cache_config = vllm_config.cache_config
quant_config = self.get_quant_config(vllm_config)
self.hidden_size = config.hidden_size
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
# Support abacusai/Smaug-72B-v0.1 with attention_bias
attention_bias = getattr(config, "attention_bias", False) or getattr(
config, "bias", False
)
self.self_attn = Jais2Attention(
config=config,
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
num_kv_heads=getattr(
config, "num_key_value_heads", config.num_attention_heads
),
max_position_embeddings=max_position_embeddings,
quant_config=quant_config,
bias=attention_bias,
cache_config=cache_config,
prefix=f"{prefix}.self_attn",
)
self.mlp = Jais2MLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
bias=getattr(config, "mlp_bias", False),
prefix=f"{prefix}.mlp",
)
self.input_layernorm = nn.LayerNorm(
config.hidden_size, eps=config.layer_norm_eps
)
self.post_attention_layernorm = nn.LayerNorm(
config.hidden_size, eps=config.layer_norm_eps
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
else:
hidden_states, residual = (
self.input_layernorm(hidden_states + residual),
hidden_states + residual,
)
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
)
# Fully Connected
hidden_states, residual = (
self.post_attention_layernorm(hidden_states + residual),
hidden_states + residual,
)
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
def get_quant_config(self, vllm_config: VllmConfig) -> QuantizationConfig | None:
"""Get quantization config for this layer. Override in subclasses."""
return vllm_config.quant_config
@support_torch_compile
class Jais2Model(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
prefix: str = "",
layer_type: type[nn.Module] = Jais2DecoderLayer,
):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
self.vocab_size = config.vocab_size
self.org_vocab_size = config.vocab_size
if get_pp_group().is_first_rank or (
config.tie_word_embeddings and get_pp_group().is_last_rank
):
self.embed_tokens = VocabParallelEmbedding(
self.vocab_size,
config.hidden_size,
org_num_embeddings=config.vocab_size,
quant_config=quant_config,
)
else:
self.embed_tokens = PPMissingLayer()
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: layer_type(
config=config,
vllm_config=vllm_config,
prefix=prefix,
),
prefix=f"{prefix}.layers",
)
if get_pp_group().is_last_rank:
self.norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
else:
self.norm = PPMissingLayer()
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_input_ids(input_ids)
residual = None
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
for i in range(self.start_layer, self.end_layer):
layer = self.layers[i]
hidden_states, residual = layer(positions, hidden_states, residual)
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
hidden_states, _ = self.norm(hidden_states + residual), residual
return hidden_states
class Jais2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
}
embedding_modules = {
"embed_tokens": "input_embeddings",
"lm_head": "output_embeddings",
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
self.model = self._init_model(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
if config.tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
logit_scale = getattr(config, "logit_scale", 1.0)
self.logits_processor = LogitsProcessor(
config.vocab_size, scale=logit_scale
)
else:
self.lm_head = PPMissingLayer()
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
def _init_model(self, vllm_config: VllmConfig, prefix: str = ""):
return Jais2Model(vllm_config=vllm_config, prefix=prefix)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
model_output = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
return model_output
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
-13
View File
@@ -7,8 +7,6 @@ from typing import Annotated, Any, Literal, cast
import torch
import torch.nn as nn
import transformers
from packaging.version import Version
from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig
from transformers.models.pixtral import PixtralProcessor
@@ -59,10 +57,6 @@ from .utils import (
maybe_prefix,
)
TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY = Version(transformers.__version__) >= Version(
"5.15.0"
)
class Mistral3ImagePixelInputs(TensorSchema):
"""Dimensions:
@@ -284,13 +278,6 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
# PixtralProcessor supports image-only calls starting in transformers 5.15.
# Older releases need dummy text and take the slower tokenization path.
if not TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY:
return self.dummy_inputs.get_dummy_text(mm_counts)
return None
def _postprocess_hf_mm_data(
self,
mm_data: Mapping[str, object],
@@ -568,15 +568,6 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
self.lm_head = ParallelLMHead(
config.decoder.vocab_size, config.decoder.d_model, quant_config=quant_config
)
# Some checkpoints (e.g. compact exports) tie the output head to the
# decoder's input embeddings instead of materializing a separate
# lm_head.weight tensor.
tie_word_embeddings = bool(
getattr(config, "tie_word_embeddings", False)
or getattr(config.decoder, "tie_word_embeddings", False)
)
if tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(self.decoder.embed_tokens)
self.logits_processor = LogitsProcessor(
self.vocab_size, config.decoder.vocab_size
)
-325
View File
@@ -1,325 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from
# https://huggingface.co/microsoft/phi-1_5/blob/main/modeling_phi.py
# Copyright 2023 The vLLM team.
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
#
# BSD 3-Clause License
#
# Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""Inference-only Phi-1.5 model compatible with HuggingFace weights."""
from collections.abc import Iterable
from itertools import islice
import torch
from torch import nn
from transformers import PhiConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
class PhiAttention(nn.Module):
def __init__(
self,
config: PhiConfig,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.hidden_size = config.hidden_size
self.head_size = self.hidden_size // config.num_attention_heads
tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
assert config.num_attention_heads % tensor_model_parallel_world_size == 0
self.num_heads = config.num_attention_heads // tensor_model_parallel_world_size
# pylint: disable=C0103
self.qkv_proj = QKVParallelLinear(
self.hidden_size,
self.head_size,
config.num_attention_heads,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.dense = RowParallelLinear(
self.hidden_size,
self.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.dense",
)
scaling = self.head_size**-0.5
max_position_embeddings = getattr(config, "max_position_embeddings", 2048)
self.rotary_emb = get_rope(
self.head_size,
max_position=max_position_embeddings,
rope_parameters=config.rope_parameters,
)
self.attn = Attention(
self.num_heads,
self.head_size,
scaling,
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
)
def forward(
self,
position_ids: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.chunk(chunks=3, dim=-1)
q, k = self.rotary_emb(position_ids, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.dense(attn_output)
return output
class PhiMLP(nn.Module):
def __init__(
self,
config: PhiConfig,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
n_inner = getattr(config, "n_inner", None)
n_inner = n_inner if n_inner is not None else 4 * config.hidden_size
self.fc1 = ColumnParallelLinear(
config.hidden_size,
n_inner,
quant_config=quant_config,
prefix=f"{prefix}.fc1",
)
self.fc2 = RowParallelLinear(
n_inner,
config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.fc2",
)
self.act = get_act_fn(config.hidden_act)
def forward(self, hidden_states):
hidden_states, _ = self.fc1(hidden_states)
hidden_states = self.act(hidden_states)
hidden_states, _ = self.fc2(hidden_states)
return hidden_states
class PhiLayer(nn.Module):
def __init__(
self,
config: PhiConfig,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
self.input_layernorm = nn.LayerNorm(
config.hidden_size, eps=config.layer_norm_eps
)
self.self_attn = PhiAttention(
config, cache_config, quant_config, prefix=f"{prefix}.self_attn"
)
self.mlp = PhiMLP(config, quant_config, prefix=f"{prefix}.mlp")
def forward(
self,
position_ids: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
attn_outputs = self.self_attn(
position_ids=position_ids,
hidden_states=hidden_states,
)
feed_forward_hidden_states = self.mlp(hidden_states)
hidden_states = attn_outputs + feed_forward_hidden_states + residual
return hidden_states
@support_torch_compile
class PhiModel(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size, config.hidden_size
)
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: PhiLayer(config, cache_config, quant_config, prefix=prefix),
prefix=f"{prefix}.layers",
)
self.final_layernorm = nn.LayerNorm(
config.hidden_size, eps=config.layer_norm_eps
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], config.hidden_size
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_input_ids(input_ids)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
for layer in islice(self.layers, self.start_layer, self.end_layer):
hidden_states = layer(positions, hidden_states)
if not get_pp_group().is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
hidden_states = self.final_layernorm(hidden_states)
return hidden_states
class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
}
)
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
# lm_head use bias, cannot share word embeddings
assert not config.tie_word_embeddings
self.quant_config = quant_config
self.model = PhiModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
bias=True,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias)
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@@ -2260,10 +2260,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
)
st_idx += text_len
bos_pos = np.broadcast_to(np.array([st_idx]), (3, 1))
llm_pos_ids_list.append(bos_pos)
st_idx += 1
if modality == "audio":
audio_tokens = self._compute_audio_token_count(
data["audio_feature_length"]
@@ -2276,7 +2272,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
eos_pos = np.broadcast_to(np.array([st_idx]), (3, 1))
llm_pos_ids_list.append(eos_pos)
st = offset + 1 + audio_tokens + 1
st = offset + audio_tokens + 1
elif modality == "image":
grid_t = data["grid_t"]
@@ -2294,7 +2290,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
eos_pos = np.broadcast_to(np.array([st_idx]), (3, 1))
llm_pos_ids_list.append(eos_pos)
st = offset + 1 + image_len + 1
st = offset + image_len + 1
elif modality == "video":
grid_t = data["grid_t"]
@@ -2313,7 +2309,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
eos_pos = np.broadcast_to(np.array([st_idx]), (3, 1))
llm_pos_ids_list.append(eos_pos)
st = offset + 1 + video_len + 1
st = offset + video_len + 1
else:
audio_bos_pos = np.broadcast_to(np.array([st_idx - 1]), (3, 1))
llm_pos_ids_list.append(audio_bos_pos)
@@ -2330,7 +2326,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
audio_len = self._compute_audio_token_count(
data["audio_feature_length"]
)
st = offset + 2 + video_len + audio_len + 2
st = offset + 1 + video_len + audio_len + 2
if st < seq_len:
st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
+4 -4
View File
@@ -120,7 +120,6 @@ _TEXT_GENERATION_MODELS = {
"GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
"GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
"GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
"GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
"GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),
@@ -136,7 +135,6 @@ _TEXT_GENERATION_MODELS = {
"InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
"IQuestCoderForCausalLM": ("llama", "LlamaForCausalLM"),
"IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
"Jais2ForCausalLM": ("jais2", "Jais2ForCausalLM"),
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
"K2HorizonForCausalLM": ("k2_horizon", "K2HorizonForCausalLM"),
"KimiLinearForCausalLM": (
@@ -188,7 +186,6 @@ _TEXT_GENERATION_MODELS = {
"PanguProMoEV2ForCausalLM": ("openpangu", "PanguProMoEV2ForCausalLM"),
"PanguUltraMoEForCausalLM": ("openpangu", "PanguUltraMoEForCausalLM"),
"Param2MoEForCausalLM": ("param2moe", "Param2MoEForCausalLM"),
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
"PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
"Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"),
@@ -200,7 +197,6 @@ _TEXT_GENERATION_MODELS = {
"Qwen3_5MoeForCausalLM": ("qwen3_5", "Qwen3_5MoeForCausalLM"),
"SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
"SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
"SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
"Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
"Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
"Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
@@ -709,13 +705,17 @@ _TRANSFORMERS_SUPPORTED_MODELS = {
"FlexOlmoForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"GlmForCausalLM": ("transformers", "TransformersForCausalLM"),
"GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"),
"GPTNeoXForCausalLM": ("transformers", "TransformersForCausalLM"),
"HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"),
"HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"),
"MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"),
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
"Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"),
"Olmo3ForCausalLM": ("transformers", "TransformersForCausalLM"),
"PhiForCausalLM": ("transformers", "TransformersForCausalLM"),
"SeedOssForCausalLM": ("transformers", "TransformersForCausalLM"),
"SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
"Starcoder2ForCausalLM": ("transformers", "TransformersForCausalLM"),
"VaultGemmaForCausalLM": ("transformers", "TransformersForCausalLM"),
-432
View File
@@ -1,432 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2025 The Seed team.
# Copyright 2023 The vLLM team.
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
#
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
# and OPT implementations in this library. It has been modified from its
# original forms to accommodate minor architectural differences compared
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Inference-only SeedOss model compatible with HuggingFace weights."""
from collections.abc import Iterable
from itertools import islice
import torch
from torch import nn
from transformers import PreTrainedConfig as SeedOssConfig
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
QKVParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.config import set_default_rope_theta
from vllm.v1.attention.backend import AttentionType
from .interfaces import SupportsLoRA, SupportsPP
from .utils import (
AutoWeightsLoader,
PPMissingLayer,
WeightsMapper,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
logger = init_logger(__name__)
class SeedOssMLP(nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.gate_up_proj",
)
self.down_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.down_proj",
)
if hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
)
self.act_fn = SiluAndMul()
def forward(self, x):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
class SeedOssAttention(nn.Module):
def __init__(
self,
hidden_size: int,
num_heads: int,
num_kv_heads: int,
head_dim: int,
rope_parameters: dict,
max_position: int = 4096 * 32,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
attn_type: str = AttentionType.DECODER,
) -> None:
super().__init__()
self.hidden_size = hidden_size
tp_size = get_tensor_model_parallel_world_size()
self.total_num_heads = num_heads
assert self.total_num_heads % tp_size == 0
self.num_heads = self.total_num_heads // tp_size
self.total_num_kv_heads = num_kv_heads
self.head_dim = head_dim
if self.total_num_kv_heads >= tp_size:
# Number of KV heads is greater than TP size, so we partition
# the KV heads across multiple tensor parallel GPUs.
assert self.total_num_kv_heads % tp_size == 0
else:
# Number of KV heads is less than TP size, so we replicate
# the KV heads across multiple tensor parallel GPUs.
assert tp_size % self.total_num_kv_heads == 0
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
self.q_size = self.num_heads * self.head_dim
self.kv_size = self.num_kv_heads * self.head_dim
self.scaling = self.head_dim**-0.5
self.qkv_proj = QKVParallelLinear(
hidden_size,
self.head_dim,
self.total_num_heads,
self.total_num_kv_heads,
bias=True,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
)
self.o_proj = RowParallelLinear(
self.total_num_heads * self.head_dim,
hidden_size,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.o_proj",
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position,
rope_parameters=rope_parameters,
)
self.attn = Attention(
self.num_heads,
self.head_dim,
self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
attn_type=attn_type,
prefix=f"{prefix}.attn",
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
attn_output = self.attn(q, k, v)
output, _ = self.o_proj(attn_output)
return output
class SeedOssDecoderLayer(nn.Module):
def __init__(
self,
config: SeedOssConfig,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
set_default_rope_theta(config, default_theta=1000000)
# By default, SeedOss uses causal attention as it is a
# decoder-only model.
# You can override the HF config with `is_causal=False` to enable
# bidirectional attention, which is used in some embedding models
if getattr(config, "is_causal", True):
attn_type = AttentionType.DECODER
else:
attn_type = AttentionType.ENCODER_ONLY
self.self_attn = SeedOssAttention(
hidden_size=self.hidden_size,
num_heads=config.num_attention_heads,
max_position=config.max_position_embeddings,
num_kv_heads=config.num_key_value_heads,
head_dim=config.head_dim,
cache_config=cache_config,
quant_config=quant_config,
rope_parameters=config.rope_parameters,
prefix=f"{prefix}.self_attn",
attn_type=attn_type,
)
self.mlp = SeedOssMLP(
hidden_size=self.hidden_size,
intermediate_size=config.intermediate_size,
hidden_act=config.hidden_act,
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps
)
def forward(
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
else:
hidden_states, residual = self.input_layernorm(hidden_states, residual)
hidden_states = self.self_attn(
positions=positions,
hidden_states=hidden_states,
)
# Fully Connected
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
@support_torch_compile(
dynamic_arg_dims={
"input_ids": 0,
"positions": -1,
"intermediate_tensors": 0,
"inputs_embeds": 0,
}
)
class SeedOssModel(nn.Module):
def __init__(
self,
*,
vllm_config: VllmConfig,
prefix: str = "",
decoder_layer_type: type[nn.Module] = SeedOssDecoderLayer,
):
super().__init__()
config = vllm_config.model_config.hf_config
cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config
# TODO (@robertgshaw2): see if this can be moved out
if cache_config.sliding_window is not None and hasattr(
config, "max_window_layers"
):
assert config.max_window_layers == config.num_hidden_layers, (
"Sliding window for some but all layers is not supported. "
"This model uses sliding window but `max_window_layers` = {} "
"is less than `num_hidden_layers` = {}. Please open an issue "
"to discuss this feature.".format(
config.max_window_layers,
config.num_hidden_layers,
)
)
self.config = config
self.quant_config = quant_config
self.vocab_size = config.vocab_size
if get_pp_group().is_first_rank or (
config.tie_word_embeddings and get_pp_group().is_last_rank
):
self.embed_tokens = VocabParallelEmbedding(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=f"{prefix}.embed_tokens",
)
else:
self.embed_tokens = PPMissingLayer()
# Use the provided decoder layer type or default to SeedDecoderLayer
decoder_layer_type = decoder_layer_type or SeedOssDecoderLayer
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
lambda prefix: decoder_layer_type(
config=config,
cache_config=cache_config,
quant_config=quant_config,
prefix=prefix,
),
prefix=f"{prefix}.layers",
)
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
if get_pp_group().is_last_rank:
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
self.norm = PPMissingLayer()
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.embed_input_ids(input_ids)
residual = None
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
for layer in islice(self.layers, self.start_layer, self.end_layer):
hidden_states, residual = layer(
positions,
hidden_states,
residual,
)
if not get_pp_group().is_last_rank:
return IntermediateTensors(
{"hidden_states": hidden_states, "residual": residual}
)
hidden_states, _ = self.norm(hidden_states, residual)
return hidden_states
class SeedOssForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_stacked={
# weight_name: (param_name, shard_id)
".q_proj": (".qkv_proj", "q"),
".k_proj": (".qkv_proj", "k"),
".v_proj": (".qkv_proj", "v"),
".gate_proj": (".gate_up_proj", 0),
".up_proj": (".gate_up_proj", 1),
}
)
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
self.config = config
self.quant_config = quant_config
self.model = SeedOssModel(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
)
if get_pp_group().is_last_rank:
self.lm_head = ParallelLMHead(
config.vocab_size,
config.hidden_size,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
if config.tie_word_embeddings:
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
else:
self.lm_head = PPMissingLayer()
self.logits_processor = LogitsProcessor(config.vocab_size)
self.make_empty_intermediate_tensors = (
self.model.make_empty_intermediate_tensors
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
return hidden_states
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(self)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
+18 -12
View File
@@ -32,6 +32,7 @@ from torch import nn
from transformers import AutoModel, PreTrainedConfig
from transformers.conversion_mapping import (
WeightRenaming,
get_checkpoint_conversion_mapping,
get_model_conversion_mapping,
)
@@ -184,7 +185,7 @@ class Base(
# Substitute remaining layers with vLLM's layers as needed
self.recursive_replace()
# Create attention instances for KV cache allocation
self._create_attention_instances()
self._create_attention_instances(prefix)
# Initialize any parameters that have not had their modules replaced
self.init_parameters(self.model)
@@ -311,7 +312,14 @@ class Base(
orig_to_new_renaming: list[WeightRenaming] = []
orig_to_new_regex: dict[re.Pattern, str | None] = {}
for mapping in get_model_conversion_mapping(self.model):
mappings = get_model_conversion_mapping(self.model)
# Ensure layers that are part of the model, but not AutoModel, get their mapping
instantiated = {type(m).__name__ for m in self.model.modules()}
for arch in self.config.architectures or []:
if arch not in instantiated:
mappings.extend(get_checkpoint_conversion_mapping(arch) or [])
for mapping in mappings:
# Handle weights which have been renamed in Transformers
if isinstance(mapping, WeightRenaming):
orig_to_new_renaming.append(mapping)
@@ -579,7 +587,7 @@ class Base(
self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked)
def _create_attention_instances(self):
def _create_attention_instances(self, prefix: str = ""):
"""Create `Attention` instances to inform KV cache allocation."""
text_config = self.text_config
attn_cls = self._get_attn_cls()
@@ -613,8 +621,8 @@ class Base(
f"Layer {layer} does not dispatch through the Transformers "
"attention interface and vLLM has no other way to handle it."
)
prefix, attn_fuser = self.attention_fusers[i]
attn_module = self.get_submodule(prefix)
attn_prefix, attn_fuser = self.attention_fusers[i]
attn_module = self.get_submodule(attn_prefix)
# `[i]` is the whole-model config unless the checkpoint is
# heterogeneous, in which case it is this layer's own geometry.
@@ -632,17 +640,18 @@ class Base(
scale=scale,
cache_config=self.cache_config,
quant_config=self.quant_config,
prefix=f"{i}.attn",
prefix=maybe_prefix(prefix, f"{attn_prefix}.{VLLM_ATTN_ATTR}"),
)
if attn_cls is MLAAttention:
mla_fuser = next(
(f for f in self.fusers[prefix] if isinstance(f, MLAFuser)), None
(f for f in self.fusers[attn_prefix] if isinstance(f, MLAFuser)),
None,
)
if mla_fuser is None:
raise ValueError(
f"Layer {i} ({prefix}) did not fuse as MLA, but the rest of "
"the model did, so its latent KV cache cannot be allocated."
f"Layer {i} ({attn_prefix}) did not fuse as MLA, but the rest "
"of the model did, so its latent KV cache cannot be allocated."
)
dims = get_mla_dims(self.model_config)
kwargs.update(
@@ -690,10 +699,8 @@ class Base(
# vLLM does not support encoder-decoder models, so if any encoder layer is
# found in a text only model, we assume the whole model is an encoder model
if has_encoder(self.model) and not is_multimodal(self.config):
self.check_version("5.0.0", "encoder models support")
return EncoderOnlyAttention
if self.model_config.use_mla:
self.check_version("5.15.0.dev0", "optimized MLA support")
if any(
isinstance(fuser, MLAFuser)
for fusers in self.fusers.values()
@@ -819,7 +826,6 @@ class Base(
)
def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
self.check_version("5.2.0", "Eagle3 support")
from transformers.utils.output_capturing import (
OutputRecorder,
maybe_install_capturing_hooks,
@@ -19,6 +19,8 @@
from collections.abc import Iterable
from typing import TYPE_CHECKING
import torch
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
@@ -30,8 +32,6 @@ from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
from .base import Base
if TYPE_CHECKING:
import torch
from vllm.config import VllmConfig
@@ -68,11 +68,15 @@ class CausalMixin(VllmModelForTextGeneration, Base):
def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]:
"""A thin wrapper around `Base.load_weights` to handle the lm_head bias."""
lm_head_bias = set()
load_device = self.vllm_config.load_config.device or self.device_config.device
def auto_load_lm_head_bias(weights):
for name, weight in weights:
if name.endswith("lm_head.bias") and self.pp_group.is_last_rank:
self.lm_head._register_bias()
# load_weights is called outside vLLM's device context,
# so we must enter it again when registering the bias.
with torch.device(load_device):
self.lm_head._register_bias()
self.lm_head.bias.weight_loader(self.lm_head.bias, weight)
lm_head_bias.add(name)
else:
@@ -156,7 +156,6 @@ class MoEMixin(MixtureOfExperts, Base):
num_local_physical_experts: int
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
self.check_version("5.0.0", "MoE models support")
# Skip MixtureOfExperts.__init__ and call the next class in MRO
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
@@ -23,17 +23,11 @@ from typing import TYPE_CHECKING, Any
import torch
import transformers
from packaging.version import Version
from transformers.utils.generic import ModelOutput
from vllm.compilation.decorators import should_torch_compile_mm_encoder
from vllm.config.utils import getattr_iter
from vllm.inputs import (
MultiModalDataBuiltins,
MultiModalDataDict,
MultiModalInput,
mm_input,
)
from vllm.inputs import MultiModalDataBuiltins, MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.models.interfaces import (
MultiModalEmbeddings,
@@ -42,26 +36,15 @@ from vllm.model_executor.models.interfaces import (
)
from vllm.model_executor.models.module_mapping import MultiModelKeys
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
from vllm.multimodal.inputs import (
MultiModalFeatureSpec,
MultiModalFieldConfig,
PlaceholderRange,
)
from vllm.multimodal.parse import (
ImageProcessorItems,
ImageSize,
MultiModalDataItems,
MultiModalDataParser,
)
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig
from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseMultiModalProcessor,
BaseProcessingInfo,
ProcessorInputs,
PromptReplacement,
PromptUpdate,
PromptUpdateDetails,
TimingContext,
cached_encode,
)
from vllm.sequence import IntermediateTensors
@@ -219,11 +202,13 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
return data
class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]):
"""Processing common to both Transformers backend processors: calling the HF
processor, sizing images, and attributing its outputs to a modality.
class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
"""Locates placeholders from the `text_replacement_offsets` the HF processor
reports, expressing each one as a `PromptUpdate`.
Subclasses add the strategy for locating placeholders in the prompt.
Stating the expansion as an update is what lets it be rebuilt from an
unexpanded prompt, so this processor takes the base class's processing path
and with it the multi-modal processor cache.
"""
def _get_modality_field_names(self, modality: str) -> set[str]:
@@ -415,301 +400,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
for image, (height, width) in zip(pixel_values, image_sizes.tolist())
]
class LegacyMultiModalProcessor(_MultiModalProcessorBase):
"""Locates placeholders by searching the prompt the HF processor has already
expanded for the tokens of each modality.
Serves transformers versions with no `return_text_replacement_offsets`.
Placeholders found this way cannot be rebuilt from an unexpanded prompt, so
this processor overrides `apply` and gets no multi-modal processor cache.
Remove it once `requirements/common.txt` requires `transformers>=5.15.0`.
"""
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
out_mm_kwargs: MultiModalKwargsItems,
) -> Sequence[PromptUpdate]:
"""Empty, because `apply` writes the placeholder ranges itself rather than
deriving them from updates."""
return []
def _get_mm_token_ids(self, modality: str) -> list[int]:
"""Token ids marking where `modality` sits in the prompt, which for some
processors differ from the placeholder written into it.
The expanded prompt is all this path has to go on, so it takes the
processor at its word about which tokens belong to the modality.
"""
info = self.info
processor = info.get_hf_processor()
declared = getattr(processor, f"{modality}_token_ids", None) or ()
if ids := [token_id for token_id in declared if token_id is not None]:
return ids
config = info.get_hf_config()
names = (f"{modality}_token_id", f"{modality}_token_index")
token_id = getattr(processor, names[0], getattr_iter(config, names))
if token_id is None:
token = getattr(processor, f"{modality}_token", None)
if token is not None:
token_id = info.get_tokenizer().get_vocab().get(token)
if token_id is None:
raise ValueError(
f"Cannot find {modality}_token_id on processor or model config"
)
return [token_id]
def _apply_audio(
self,
prompt_ids: list[int],
processed_data: "BatchFeature",
num_audios: int,
) -> dict[str, list[PlaceholderRange]]:
"""Take each contiguous run of the audio token as one item's placeholder,
and record how many tokens the run holds."""
audio_token_ids = self._get_mm_token_ids("audio")
prompt_tensor = torch.tensor(prompt_ids)
is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
if not is_audio.any():
raise ValueError(
f"{num_audios} audio item(s) were passed but the prompt "
"contains no audio token. Add one placeholder per audio item."
)
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
transitions = padded.int().diff()
offsets = torch.where(transitions == 1)[0]
lengths = torch.where(transitions == -1)[0] - offsets
if len(offsets) != num_audios:
raise ValueError(
f"Found {len(offsets)} run(s) of the audio token in the prompt but "
f"{num_audios} audio item(s) were passed. The Transformers backend "
"locates audio placeholders by finding contiguous runs of the audio "
"token, so placeholders with no text between them cannot yet be told "
"apart. Separate them in the prompt to work around this."
)
ranges = [
PlaceholderRange(offset=offset.item(), length=length.item())
for offset, length in zip(offsets, lengths)
]
processed_data["num_audio_tokens"] = lengths
return {"audio": ranges}
def _get_num_multimodal_tokens(
self,
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, list[int]]:
"""Ask the HF processor how many tokens and patches each image expands to."""
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
images = mm_items.get_items("image", ImageProcessorItems)
image_sizes = [
(size.height, size.width)
for size in map(images.get_image_size, range(len(images)))
]
return processor._get_num_multimodal_tokens(
image_sizes=image_sizes,
**self.info.ctx.get_merged_mm_kwargs({}),
)
def _apply_vision(
self,
prompt_ids: list[int],
processed_data: "BatchFeature",
mm_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
mm_token_type_ids: torch.Tensor | None,
) -> dict[str, list[PlaceholderRange]]:
"""Split the positions the processor marks as image into one placeholder per
item, sized by the token count it reports for each image."""
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
if mm_token_type_ids is None:
raise ValueError(
f"{type(hf_processor).__name__} returned no `mm_token_type_ids`, so "
"the Transformers modeling backend cannot locate the placeholder of "
"each image."
)
# These mark the tokens structuring an image as well as the image itself,
# which is what the counts below are over
mm_positions = torch.where(mm_token_type_ids[0] == 1)[0]
mm_tokens_per_modality = self._get_num_multimodal_tokens(
mm_items, hf_processor_mm_kwargs
)
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
split_sizes = mm_tokens_per_modality["num_image_tokens"]
if sum(split_sizes) != len(mm_positions):
raise ValueError(
f"The expanded prompt holds {len(mm_positions)} image token(s) but "
f"{type(hf_processor).__name__} accounts for {sum(split_sizes)} "
f"across {mm_items.get_count('image')} image item(s)."
)
if split_sizes:
image_token_ids = torch.tensor(self._get_mm_token_ids("image"))
mm_tokens = torch.tensor(prompt_ids)[mm_positions]
ranges = [
PlaceholderRange(
offset=positions[0].item(),
length=positions.shape[0],
# Only some of the span carries embeddings
is_embed=torch.isin(tokens, image_token_ids),
)
for positions, tokens in zip(
torch.split(mm_positions, split_sizes),
torch.split(mm_tokens, split_sizes),
)
]
mm_placeholders = {"image": ranges}
processed_data["num_image_patches"] = torch.tensor(
mm_tokens_per_modality["num_image_patches"]
)
return mm_placeholders
def apply(
self,
inputs: ProcessorInputs,
timing_ctx: TimingContext,
) -> MultiModalInput:
"""Process the prompt and every multi-modal item in one HF processor call,
then read the placeholder ranges out of the token ids it returns."""
prompt = inputs.prompt
mm_items = inputs.mm_data_items
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
with timing_ctx.record("apply_hf_processor"):
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
prompt_text = hf_processor.decode(prompt)
# Bypass cached processor and always apply to the full set of mm inputs
# NOTE: we can't just set caching=False because base class method
# transforms outputs to `MultiModalKwargs` which is not going to
# work for Transformers. The vision path has logic tied to
# `mm_tokens_per_modality` in _apply_vision()
hf_processor_mm_kwargs = {
# vLLM needs the untruncated sequence to keep placeholder
# tokens aligned. Note that the text inputs are just dummy
# text, not the original prompt. The original prompt is
# already tokenized by the renderer.
"truncation": False,
# HF processors only accept text, and the decoded string already
# contains any special tokens, so don't let them be added again
# (the HF processor call disables `add_special_tokens`).
"add_special_tokens": False,
**hf_processor_mm_kwargs,
}
processor_data, _, passthrough_data = self._get_hf_mm_inputs(
mm_items, hf_processor_mm_kwargs
)
# The real prompt is fed to the processor below and the placeholder
# ranges are read back out of its output, so the dummy text meant for
# media-only callers is discarded here.
processor_data.pop("text", None)
has_mm_data = any(processor_data.values())
# Ask which modality owns each token of the expanded prompt, which
# is the only marking of the tokens an expansion adds around an item
if has_mm_data:
processor_data = {**processor_data, "return_mm_token_type_ids": True}
try:
processed_data = self.info.ctx.call_hf_processor(
self.info.get_hf_processor(**hf_processor_mm_kwargs),
dict(text=prompt_text, **processor_data),
hf_processor_mm_kwargs,
)
except ValueError:
if has_mm_data:
raise
# Some processors reject a prompt holding placeholders with
# no data to go with them, so tokenize it without them
prompt_ids = self.info.get_tokenizer().encode(prompt_text)
processed_data = transformers.BatchFeature(
dict(input_ids=[prompt_ids]), tensor_type="pt"
)
self._unpad_images(processed_data)
self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs)
processed_data.update(passthrough_data)
input_ids = processed_data.pop("input_ids")
if not isinstance(input_ids, list):
input_ids = input_ids.tolist()
(prompt_ids,) = input_ids
# Use overrides if provided; fallback to data-dependent hashing.
with timing_ctx.record("get_mm_hashes"):
mm_hashes = inputs.get_mm_hashes(
self.info.model_id,
self.info.ctx.get_mm_config().mm_hasher_algorithm,
)
# Gemma3 reports them under the key the model uses for its own token types
mm_token_type_ids = processed_data.pop("token_type_ids", None)
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
if num_audios := mm_items.get_count("audio", strict=False):
mm_placeholders.update(
self._apply_audio(prompt_ids, processed_data, num_audios)
)
if mm_items.get_count("image", strict=False):
mm_placeholders.update(
self._apply_vision(
prompt_ids,
processed_data,
mm_items,
hf_processor_mm_kwargs,
mm_token_type_ids,
)
)
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
processed_data,
self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
)
# Bypassing `_maybe_apply_prompt_updates` also bypasses its validation.
# `_validate_mm_placeholders` can't be reused because it is typed for the
# `PlaceholderFeaturesInfo` the prompt update machinery produces.
mm_item_counts = mm_items.get_all_counts()
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
for modality, item_count in mm_item_counts.items():
num_placeholders = len(mm_placeholders.get(modality, []))
if num_placeholders != item_count:
raise RuntimeError(
f"Expected there to be {item_count} prompt placeholders "
f"corresponding to {item_count} {modality} items, but instead "
f"found {num_placeholders} prompt placeholders! Make sure the "
"prompt contains a placeholder token for each item."
)
return mm_input(
prompt_token_ids=prompt_ids,
mm_kwargs=mm_kwargs,
mm_hashes=mm_hashes,
mm_placeholders=mm_placeholders,
)
class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"""Locates placeholders from the `text_replacement_offsets` the HF processor
reports, expressing each one as a `PromptUpdate`.
Stating the expansion as an update is what lets it be rebuilt from an
unexpanded prompt, so this processor takes the base class's processing path
and with it the multi-modal processor cache.
"""
def _get_prompt_updates(
self,
mm_items: MultiModalDataItems,
@@ -800,9 +490,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
except (AttributeError, KeyError, TypeError):
return None
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
def _apply_hf_processor_main(
self,
mm_items: MultiModalDataItems,
@@ -856,8 +543,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
"to reach `ProcessorMixin.get_text_with_replacements` with one "
"replacement per item, which usually means implementing "
"`replace_<modality>_token`. Please report this to transformers "
"so it can be fixed, and install `transformers<5.15.0` in the "
"meantime to locate placeholders in the expanded prompt instead."
"so it can be fixed."
)
hf_inputs.pop("input_ids", None)
return self._finalize_hf_mm_data(
@@ -896,15 +582,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
)
# From this version on, a processor reporting no offsets is an error rather than a
# fallback to searching the expanded prompt
MultiModalProcessor = (
LegacyMultiModalProcessor
if Version(transformers.__version__) < Version("5.15.0")
else OffsetsMultiModalProcessor
)
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
# Skip SupportsMRoPE.__init__ and call the next class in MRO
@@ -982,7 +659,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
super()._decorate_for_torch_compile()
# Decorate the encoder model classes to support torch compile if needed
if self.compilation_config.compile_mm_encoder:
self.check_version("5.0.0", "multimodal encoder compilation support")
encoder_classes = self._pre_trained_model_classes.encoders
if not encoder_classes:
raise ValueError(
@@ -1113,7 +789,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
if input_features is None:
return None
self.check_version("5.13.0", "audio models support")
num_audio_tokens = kwargs.pop("num_audio_tokens")
kwargs.pop("token_type_ids", None)
kwargs.pop("mm_token_type_ids", None)