Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -31,6 +31,9 @@ from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_one
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_two_sided import ( # noqa: E501
|
||||
FlashInferNVLinkTwoSidedPrepareAndFinalize,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.passthrough import (
|
||||
PassThroughPrepareAndFinalize,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import (
|
||||
has_deep_ep,
|
||||
@@ -185,6 +188,10 @@ def maybe_make_prepare_finalize(
|
||||
all2all_manager: Any | None = None,
|
||||
input_dtype: torch.dtype | None = None,
|
||||
) -> FusedMoEPrepareAndFinalize | None:
|
||||
if moe.moe_parallel_config.use_passthrough_all2all:
|
||||
# The experts dispatch and combine themselves, in every EP topology.
|
||||
return PassThroughPrepareAndFinalize()
|
||||
|
||||
if not moe.moe_parallel_config.use_all2all_kernels:
|
||||
if not allow_new_interface:
|
||||
return None
|
||||
|
||||
@@ -1068,6 +1068,12 @@ class FusedMoEParallelConfig:
|
||||
and self.all2all_backend == "allgather_reducescatter"
|
||||
)
|
||||
|
||||
@property
|
||||
def use_passthrough_all2all(self):
|
||||
# Not gated on use_all2all_kernels: the experts dispatch and combine
|
||||
# internally in every EP topology, including TP-only.
|
||||
return self.all2all_backend == "passthrough"
|
||||
|
||||
@property
|
||||
def use_mori_kernels(self):
|
||||
return self.use_all2all_kernels and self.all2all_backend in (
|
||||
@@ -1530,6 +1536,10 @@ class FusedMoEConfig:
|
||||
def use_deepep_ll_kernels(self):
|
||||
return self.moe_parallel_config.use_deepep_ll_kernels
|
||||
|
||||
@property
|
||||
def use_passthrough_all2all(self):
|
||||
return self.moe_parallel_config.use_passthrough_all2all
|
||||
|
||||
@property
|
||||
def use_mori_kernels(self):
|
||||
return self.moe_parallel_config.use_mori_kernels
|
||||
|
||||
@@ -374,6 +374,7 @@ class AiterW4A16ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
routing_method = self.moe_config.routing_method
|
||||
if routing_method == RoutingMethodType.DeepseekV4:
|
||||
|
||||
@@ -289,6 +289,7 @@ class AiterW4A8ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
assert self.moe_config.intermediate_size_per_partition_unpadded is not None
|
||||
assert self.moe_config.hidden_dim_unpadded is not None
|
||||
|
||||
@@ -198,6 +198,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Apply the monolithic 4-bit INT MoE forward pass.
|
||||
|
||||
@@ -215,6 +216,7 @@ class CPUExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: Bias for expert scores
|
||||
routed_scaling_factor: Scaling factor for routing
|
||||
topk_group: Group size for topk
|
||||
routing_replay_out: Unused; this kernel does not capture routing
|
||||
|
||||
Returns:
|
||||
Output tensor after MoE computation
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FlashInfer MoE-EP megakernels as a modular-kernel experts implementation.
|
||||
|
||||
The megakernel routes on vLLM's top-k output and then dispatches, computes and
|
||||
combines in one launch, so it is paired with a pass-through prepare/finalize.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
|
||||
FlashInferMoeEp,
|
||||
FlashInferMoeEpEpilogue,
|
||||
FlashInferMoeEpWeights,
|
||||
apply_topk_in_fc1,
|
||||
flashinfer_moe_ep_unsupported_reasons,
|
||||
supports_current_device,
|
||||
validate_flashinfer_moe_ep_layer,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
|
||||
TopKWeightAndReduceNoOP,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
kNvfp4Static,
|
||||
require_finite_positive_scales,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
|
||||
if TYPE_CHECKING:
|
||||
pass
|
||||
|
||||
# Layer parameters the megakernel reads. After ``process_weights_after_loading``
|
||||
# they alias the megakernel's own tensors, so EPLB permutes exactly what it uses.
|
||||
KERNEL_WEIGHT_NAMES = ("w13_weight", "w13_weight_scale", "w2_weight", "w2_weight_scale")
|
||||
|
||||
|
||||
_supported_weight_quant_schemes: frozenset[QuantKey] = frozenset(
|
||||
(kNvfp4Static, kMxfp4Static)
|
||||
)
|
||||
|
||||
_supported_activations: frozenset[MoEActivation] = frozenset((MoEActivation.SILU,))
|
||||
|
||||
|
||||
class FlashInferMoeEpExperts(mk.FusedMoEExpertsModular):
|
||||
def __init__(
|
||||
self,
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
) -> None:
|
||||
super().__init__(moe_config, quant_config)
|
||||
self._adapter: FlashInferMoeEp | None = None
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
cls: type[mk.FusedMoEExperts],
|
||||
moe_config: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
supported, reason = mk.FusedMoEExperts.is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
)
|
||||
if not supported:
|
||||
return False, reason
|
||||
reasons = flashinfer_moe_ep_unsupported_reasons(
|
||||
moe_config, weight_key, activation_key
|
||||
)
|
||||
if reasons:
|
||||
return False, f"kernel does not support {', '.join(reasons)}"
|
||||
return True, None
|
||||
|
||||
@staticmethod
|
||||
def activation_format() -> mk.FusedMoEActivationFormat:
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
return supports_current_device()
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _supports_quant_scheme(
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
return weight_key in _supported_weight_quant_schemes
|
||||
|
||||
@staticmethod
|
||||
def _supports_activation(activation: MoEActivation) -> bool:
|
||||
return activation in _supported_activations
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
return (
|
||||
moe_parallel_config.use_ep
|
||||
and not moe_parallel_config.use_batched_activation_format
|
||||
)
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
return True
|
||||
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
return TopKWeightAndReduceNoOP()
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
M: int,
|
||||
N: int,
|
||||
K: int,
|
||||
topk: int,
|
||||
global_num_experts: int,
|
||||
local_num_experts: int,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
activation: MoEActivation,
|
||||
) -> tuple[tuple[int, ...], tuple[int, ...], tuple[int, ...]]:
|
||||
return (0,), (0,), (M, K)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
"""Build the megakernel from the layer's canonical weights.
|
||||
|
||||
The oracle has already put the weights in kernel format; the per-expert
|
||||
weight global scales arrive through the quant config and become the
|
||||
epilogue alphas.
|
||||
"""
|
||||
validate_flashinfer_moe_ep_layer(layer) # type: ignore[arg-type]
|
||||
adapter = FlashInferMoeEp(
|
||||
self.moe_config,
|
||||
FlashInferMoeEpWeights(
|
||||
w13=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
w13_scale=getattr(layer, "w13_weight_scale", None),
|
||||
w2_scale=getattr(layer, "w2_weight_scale", None),
|
||||
),
|
||||
epilogue_from_quant_config(self.quant_config),
|
||||
apply_topk_in_fc1=apply_topk_in_fc1(
|
||||
self.moe_config,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
),
|
||||
)
|
||||
adapter.warmup()
|
||||
for name, tensor in zip(KERNEL_WEIGHT_NAMES, adapter.kernel_weights()):
|
||||
replace_parameter(layer, name, tensor)
|
||||
self._adapter = adapter
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
if self._adapter is None:
|
||||
raise RuntimeError("FlashInfer MoE-EP weights have not been processed")
|
||||
output.copy_(
|
||||
self._adapter(hidden_states, topk_ids.to(torch.int32), topk_weights)
|
||||
)
|
||||
|
||||
|
||||
def epilogue_from_quant_config(
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
) -> FlashInferMoeEpEpilogue:
|
||||
"""Per-expert epilogue constants from the canonical quant config.
|
||||
|
||||
The weight global scales (``g1_alphas``/``g2_alphas``) become the fc1/fc2
|
||||
alphas. Activations are quantized dynamically inside the megakernel, so the
|
||||
checkpoint's input scales drop out. MXFP4 checkpoints carry no global scales
|
||||
and keep the default epilogue. The alphas alias the layer's parameters, so
|
||||
EPLB permutations reach the kernel.
|
||||
"""
|
||||
fc1_alpha, fc2_alpha = quant_config.g1_alphas, quant_config.g2_alphas
|
||||
if fc1_alpha is None and fc2_alpha is None:
|
||||
return FlashInferMoeEpEpilogue()
|
||||
assert fc1_alpha is not None and fc2_alpha is not None
|
||||
require_finite_positive_scales("g1_alphas", fc1_alpha)
|
||||
require_finite_positive_scales("g2_alphas", fc2_alpha)
|
||||
if fc1_alpha.dim() != 1 or fc2_alpha.dim() != 1:
|
||||
raise ValueError("the megakernel needs one weight global scale per expert")
|
||||
return FlashInferMoeEpEpilogue(
|
||||
fc1_alpha=fc1_alpha.float().contiguous(),
|
||||
fc2_alpha=fc2_alpha.float().contiguous(),
|
||||
)
|
||||
@@ -1580,6 +1580,7 @@ class OAITritonMxfp4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return triton_kernel_moe_forward(
|
||||
hidden_states=hidden_states,
|
||||
|
||||
@@ -272,6 +272,7 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
import flashinfer
|
||||
|
||||
@@ -282,10 +283,6 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
|
||||
# dummy 0-token forward has to keep the finalized (empty) form.
|
||||
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=num_tokens,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
flashinfer_output = flashinfer.fused_moe.trtllm_bf16_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -312,5 +309,4 @@ class TrtLlmBf16ExpertsMonolithic(TrtLlmBf16ExpertsBase, mk.FusedMoEExpertsMonol
|
||||
num_tokens=num_tokens,
|
||||
top_k=self.topk,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
|
||||
return routed_output
|
||||
|
||||
@@ -450,6 +450,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
import flashinfer
|
||||
from flashinfer.fused_moe import Fp8QuantizationType, WeightLayout
|
||||
@@ -487,11 +488,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
n_group = num_expert_group or 0
|
||||
selected_topk_group = topk_group or 0
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
kwargs = dict(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -522,9 +518,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
if is_mxfp8 or activation == MoEActivation.RELU2_NO_MUL:
|
||||
kwargs["activation_type"] = activation_type
|
||||
result = flashinfer.fused_moe.trtllm_fp8_block_scale_moe(**kwargs)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return result
|
||||
|
||||
def _apply_per_tensor(
|
||||
@@ -543,6 +536,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# Delay import for non-CUDA.
|
||||
import flashinfer
|
||||
@@ -558,11 +552,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
else:
|
||||
assert not apply_router_weight_on_input
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
out = flashinfer.fused_moe.trtllm_fp8_per_tensor_scale_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -586,9 +575,6 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
tune_max_num_tokens=fi_moe_largest_bucket(self.moe_config),
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return out
|
||||
|
||||
def apply(
|
||||
@@ -607,6 +593,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.quant_config.block_shape is not None:
|
||||
return self._apply_block_scale(
|
||||
@@ -623,6 +610,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
elif self.quant_config.is_per_tensor:
|
||||
return self._apply_per_tensor(
|
||||
@@ -638,6 +626,7 @@ class TrtLlmFp8ExpertsMonolithic(TrtLlmFp8ExpertsBase, mk.FusedMoEExpertsMonolit
|
||||
num_expert_group=num_expert_group,
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -215,6 +215,7 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
from flashinfer import trtllm_fp4_block_scale_moe
|
||||
|
||||
@@ -236,10 +237,6 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=num_tokens,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
flashinfer_output = trtllm_fp4_block_scale_moe(
|
||||
routing_logits=router_logits,
|
||||
routing_bias=e_score_correction_bias,
|
||||
@@ -279,7 +276,6 @@ class TrtLlmMxfp4ExpertsMonolithic(
|
||||
top_k=self.topk,
|
||||
finalized_output=finalized_output,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
|
||||
return routed_output
|
||||
|
||||
|
||||
|
||||
@@ -139,6 +139,7 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_mxint4_moe import ( # noqa: E501
|
||||
flashinfer_trtllm_mxint4_moe,
|
||||
@@ -147,10 +148,6 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
assert self.w1_scale is not None
|
||||
assert self.w2_scale is not None
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=hidden_states.shape[0],
|
||||
device=hidden_states.device,
|
||||
)
|
||||
result = flashinfer_trtllm_mxint4_moe(
|
||||
x=hidden_states,
|
||||
router_logits=router_logits,
|
||||
@@ -169,7 +166,4 @@ class TrtLlmMxint4ExpertsMonolithic(mk.FusedMoEExpertsMonolithic):
|
||||
routing_method_type=self.routing_method,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(
|
||||
routing_replay_out, num_tokens=hidden_states.shape[0]
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -515,6 +515,7 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
import flashinfer
|
||||
|
||||
@@ -545,10 +546,6 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
# dummy 0-token forward has to keep the finalized (empty) form.
|
||||
defer = self.moe_config.should_defer_moe_finalize(num_tokens)
|
||||
|
||||
routing_replay_out = self._maybe_make_routing_replay_buffer(
|
||||
num_tokens=num_tokens,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
# Invoke kernel.
|
||||
# NOTE: Activation padding and output
|
||||
# truncation are handled by the MoE runner's
|
||||
@@ -592,5 +589,4 @@ class TrtLlmNvFp4ExpertsMonolithic(
|
||||
num_tokens=num_tokens,
|
||||
top_k=self.topk,
|
||||
)
|
||||
self._maybe_dispatch_routing_replay(routing_replay_out, num_tokens=num_tokens)
|
||||
return routed_output
|
||||
|
||||
@@ -151,6 +151,7 @@ class ZentorchExpertsInt4(mk.FusedMoEExpertsMonolithic):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if apply_router_weight_on_input:
|
||||
raise NotImplementedError(
|
||||
|
||||
@@ -0,0 +1,481 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Literal, cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config.kernel import (
|
||||
FLASHINFER_MOE_EP_BACKENDS,
|
||||
FLASHINFER_MOE_EP_CUTEDSL,
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
)
|
||||
from vllm.distributed import get_ep_group
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kMxfp4Static,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import _import_deep_gemm
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
|
||||
|
||||
FlashInferMoeEpKernel = Literal["cutedsl", "deep_gemm"]
|
||||
FlashInferMoeEpWeightFormat = Literal["nvfp4", "mxfp4"]
|
||||
_WEIGHT_FORMATS: dict[QuantKey | None, FlashInferMoeEpWeightFormat] = {
|
||||
kNvfp4Static: "nvfp4",
|
||||
kMxfp4Static: "mxfp4",
|
||||
}
|
||||
|
||||
_SUPPORTED_CAPABILITIES = frozenset({(10, 0), (10, 3)})
|
||||
|
||||
|
||||
def supports_current_device() -> bool:
|
||||
capability = current_platform.get_device_capability()
|
||||
return capability is not None and (
|
||||
(capability.major, capability.minor) in _SUPPORTED_CAPABILITIES
|
||||
)
|
||||
|
||||
|
||||
_E2M1_LUT = (
|
||||
0.0,
|
||||
0.5,
|
||||
1.0,
|
||||
1.5,
|
||||
2.0,
|
||||
3.0,
|
||||
4.0,
|
||||
6.0,
|
||||
-0.0,
|
||||
-0.5,
|
||||
-1.0,
|
||||
-1.5,
|
||||
-2.0,
|
||||
-3.0,
|
||||
-4.0,
|
||||
-6.0,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FlashInferMoeEpBackendSpec:
|
||||
kernel: FlashInferMoeEpKernel
|
||||
weight_formats: frozenset[FlashInferMoeEpWeightFormat]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FlashInferMoeEpWeights:
|
||||
w13: torch.Tensor
|
||||
w2: torch.Tensor
|
||||
w13_scale: torch.Tensor | None = None
|
||||
w2_scale: torch.Tensor | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FlashInferMoeEpEpilogue:
|
||||
input_norm_const: float = 1.0
|
||||
fc1_alpha: torch.Tensor | None = None
|
||||
fc2_alpha: torch.Tensor | None = None
|
||||
fc1_norm_const: torch.Tensor | None = None
|
||||
|
||||
|
||||
_BACKEND_SPECS = {
|
||||
FLASHINFER_MOE_EP_CUTEDSL: FlashInferMoeEpBackendSpec(
|
||||
kernel="cutedsl",
|
||||
weight_formats=frozenset({"nvfp4", "mxfp4"}),
|
||||
),
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM: FlashInferMoeEpBackendSpec(
|
||||
kernel="deep_gemm",
|
||||
weight_formats=frozenset({"mxfp4"}),
|
||||
),
|
||||
}
|
||||
|
||||
assert frozenset(_BACKEND_SPECS) == FLASHINFER_MOE_EP_BACKENDS
|
||||
|
||||
|
||||
def is_flashinfer_moe_ep_backend(moe_backend: str) -> bool:
|
||||
return moe_backend in _BACKEND_SPECS
|
||||
|
||||
|
||||
def flashinfer_moe_ep_backend_spec(
|
||||
moe_backend: str,
|
||||
) -> FlashInferMoeEpBackendSpec:
|
||||
try:
|
||||
return _BACKEND_SPECS[moe_backend]
|
||||
except KeyError:
|
||||
raise ValueError(
|
||||
f"{moe_backend!r} is not a FlashInfer MoE-EP backend; expected "
|
||||
f"one of {sorted(_BACKEND_SPECS)}"
|
||||
) from None
|
||||
|
||||
|
||||
def flashinfer_moe_ep_unsupported_reasons(
|
||||
moe: FusedMoEConfig,
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> tuple[str, ...]:
|
||||
"""Deployment constraints of the megakernel beyond the framework's checks.
|
||||
|
||||
``activation_key`` is ``None`` for MXFP4 checkpoints (the kernel quantizes
|
||||
bf16 activations itself) but marks a W4A16 recipe for NVFP4 weights.
|
||||
"""
|
||||
spec = flashinfer_moe_ep_backend_spec(moe.moe_backend)
|
||||
unsupported: list[str] = []
|
||||
weight_format = _WEIGHT_FORMATS.get(weight_key)
|
||||
if weight_format is None:
|
||||
unsupported.append(f"weight scheme {weight_key}")
|
||||
elif weight_format not in spec.weight_formats:
|
||||
unsupported.append(f"{weight_format.upper()} weights")
|
||||
if weight_key == kNvfp4Static and activation_key is None:
|
||||
unsupported.append("A16 activations with NVFP4 weights")
|
||||
if moe.skip_final_all_reduce:
|
||||
unsupported.append("skip_final_all_reduce")
|
||||
if moe.in_dtype != torch.bfloat16:
|
||||
unsupported.append(f"activation dtype {moe.in_dtype}")
|
||||
if moe.has_bias:
|
||||
unsupported.append("expert bias")
|
||||
if moe.swiglu_alpha is not None or moe.swiglu_beta is not None:
|
||||
unsupported.append("custom SwiGLU alpha or beta")
|
||||
if (
|
||||
spec.kernel == "deep_gemm"
|
||||
and moe.routing_method is not RoutingMethodType.DeepseekV4
|
||||
):
|
||||
unsupported.append(f"routing method {moe.routing_method.name}")
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
if vllm_config.weight_transfer_config is not None:
|
||||
unsupported.append("runtime weight transfer")
|
||||
if vllm_config.parallel_config.enable_dbo:
|
||||
unsupported.append("dual batch overlap")
|
||||
if spec.kernel == "deep_gemm" and vllm_config.parallel_config.enable_eplb:
|
||||
unsupported.append("EPLB")
|
||||
return tuple(unsupported)
|
||||
|
||||
|
||||
def validate_flashinfer_moe_ep_layer(layer: RoutedExperts) -> None:
|
||||
if layer.expert_map_manager.placement_strategy != "linear":
|
||||
raise ValueError(
|
||||
f"{layer.moe_config.moe_backend} requires contiguous linear "
|
||||
"expert placement"
|
||||
)
|
||||
|
||||
|
||||
def apply_topk_in_fc1(
|
||||
moe: FusedMoEConfig,
|
||||
*,
|
||||
apply_router_weight_on_input: bool = False,
|
||||
) -> bool:
|
||||
return (
|
||||
apply_router_weight_on_input
|
||||
or moe.routing_method is RoutingMethodType.DeepseekV4
|
||||
)
|
||||
|
||||
|
||||
def _dequant_mxfp4_ue8m0_gran32(
|
||||
packed: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
raw = packed.view(torch.uint8)
|
||||
lut = torch.tensor(_E2M1_LUT, dtype=torch.float32, device=raw.device)
|
||||
values = torch.empty(
|
||||
raw.shape[0],
|
||||
raw.shape[1] * 2,
|
||||
dtype=torch.float32,
|
||||
device=raw.device,
|
||||
)
|
||||
values[:, ::2] = lut[(raw & 0x0F).to(torch.int64)]
|
||||
values[:, 1::2] = lut[(raw >> 4).to(torch.int64)]
|
||||
encoded_scale = (
|
||||
scale
|
||||
if scale.dtype == torch.float8_e8m0fnu
|
||||
else scale.view(torch.float8_e8m0fnu)
|
||||
)
|
||||
decoded_scale = encoded_scale.float()
|
||||
return (values * decoded_scale.repeat_interleave(32, dim=-1)).to(torch.bfloat16)
|
||||
|
||||
|
||||
def _dequant_mxfp4_experts(
|
||||
weight: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
num_experts, output_size, packed_input_size = weight.shape
|
||||
result = torch.empty(
|
||||
num_experts,
|
||||
output_size,
|
||||
packed_input_size * 2,
|
||||
dtype=torch.bfloat16,
|
||||
device=weight.device,
|
||||
)
|
||||
for expert_id in range(num_experts):
|
||||
result[expert_id] = _dequant_mxfp4_ue8m0_gran32(
|
||||
weight[expert_id], scale[expert_id]
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def mxfp4_moe_ep_weights(
|
||||
moe_backend: str,
|
||||
w13: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w13_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
) -> FlashInferMoeEpWeights:
|
||||
spec = flashinfer_moe_ep_backend_spec(moe_backend)
|
||||
if spec.kernel == "deep_gemm":
|
||||
return FlashInferMoeEpWeights(w13, w2, w13_scale, w2_scale)
|
||||
return FlashInferMoeEpWeights(
|
||||
w13=_dequant_mxfp4_experts(w13, w13_scale),
|
||||
w2=_dequant_mxfp4_experts(w2, w2_scale),
|
||||
)
|
||||
|
||||
|
||||
class _FlashInferMoeEpApi:
|
||||
def __init__(self) -> None:
|
||||
try:
|
||||
from flashinfer.moe_ep import (
|
||||
BootstrapConfig,
|
||||
DeepGemmMegaMoeConfig,
|
||||
FleetParams,
|
||||
MegaConfig,
|
||||
MoEEpMegaLayer,
|
||||
MoEEpTensors,
|
||||
MoEWeightPack,
|
||||
Nvfp4CutedslMegaMoeConfig,
|
||||
)
|
||||
except ImportError as err:
|
||||
raise RuntimeError(
|
||||
"FlashInfer MoE-EP requires the public API shipped with "
|
||||
"FlashInfer 0.7.0 or newer"
|
||||
) from err
|
||||
|
||||
self.BootstrapConfig = BootstrapConfig
|
||||
self.DeepGemmMegaMoeConfig = DeepGemmMegaMoeConfig
|
||||
self.FleetParams = FleetParams
|
||||
self.MegaConfig = MegaConfig
|
||||
self.MoEEpMegaLayer = MoEEpMegaLayer
|
||||
self.MoEEpTensors = MoEEpTensors
|
||||
self.MoEWeightPack = MoEWeightPack
|
||||
self.Nvfp4CutedslMegaMoeConfig = Nvfp4CutedslMegaMoeConfig
|
||||
|
||||
|
||||
def _load_flashinfer_moe_ep_api() -> _FlashInferMoeEpApi:
|
||||
return _FlashInferMoeEpApi()
|
||||
|
||||
|
||||
def _expose_deep_gemm_to_flashinfer() -> None:
|
||||
deep_gemm = _import_deep_gemm()
|
||||
if deep_gemm is None:
|
||||
raise RuntimeError(
|
||||
"FlashInfer MoE-EP DeepGEMM requires a usable DeepGEMM installation"
|
||||
)
|
||||
sys.modules.setdefault("deep_gemm", deep_gemm)
|
||||
|
||||
|
||||
class FlashInferMoeEp:
|
||||
def __init__(
|
||||
self,
|
||||
moe: FusedMoEConfig,
|
||||
weights: FlashInferMoeEpWeights,
|
||||
epilogue: FlashInferMoeEpEpilogue | None = None,
|
||||
*,
|
||||
apply_topk_in_fc1: bool,
|
||||
) -> None:
|
||||
spec = flashinfer_moe_ep_backend_spec(moe.moe_backend)
|
||||
if spec.kernel == "deep_gemm":
|
||||
_expose_deep_gemm_to_flashinfer()
|
||||
api = _load_flashinfer_moe_ep_api()
|
||||
if epilogue is None:
|
||||
epilogue = FlashInferMoeEpEpilogue()
|
||||
ep_group = get_ep_group()
|
||||
bootstrap = api.BootstrapConfig(
|
||||
world_size=ep_group.world_size,
|
||||
rank=ep_group.rank_in_group,
|
||||
process_group=ep_group.device_group,
|
||||
device=torch.accelerator.current_device_index(),
|
||||
)
|
||||
fleet_params = api.FleetParams(
|
||||
num_experts=moe.num_experts,
|
||||
max_tokens_per_rank=moe.max_num_tokens,
|
||||
token_hidden_size=moe.hidden_dim,
|
||||
)
|
||||
weight_pack = api.MoEWeightPack(
|
||||
w13=weights.w13,
|
||||
w2=weights.w2,
|
||||
w13_scale=weights.w13_scale,
|
||||
w2_scale=weights.w2_scale,
|
||||
)
|
||||
if spec.kernel == "cutedsl":
|
||||
megakernel = api.Nvfp4CutedslMegaMoeConfig(
|
||||
intermediate_size=moe.intermediate_size,
|
||||
top_k=moe.experts_per_token,
|
||||
gate_up_clamp=moe.swiglu_limit,
|
||||
fast_math=True,
|
||||
apply_topk_in_fc1=apply_topk_in_fc1,
|
||||
enable_in_kernel_fc2_reduce=False,
|
||||
combine_dtype="bf16",
|
||||
input_norm_const=epilogue.input_norm_const,
|
||||
fc1_alpha=None,
|
||||
fc2_alpha=None,
|
||||
fc1_norm_const=None,
|
||||
knobs=None,
|
||||
)
|
||||
else:
|
||||
megakernel = api.DeepGemmMegaMoeConfig(
|
||||
intermediate_size=moe.intermediate_size,
|
||||
top_k=moe.experts_per_token,
|
||||
activation_clamp=moe.swiglu_limit,
|
||||
fast_math=True,
|
||||
)
|
||||
backend = api.MegaConfig(
|
||||
megakernel=megakernel,
|
||||
quantize_input=True,
|
||||
preprocess_weights=True,
|
||||
)
|
||||
|
||||
self._moe_ep_tensors_cls = api.MoEEpTensors
|
||||
self._mega_layer: Any | None = api.MoEEpMegaLayer(
|
||||
bootstrap,
|
||||
fleet_params,
|
||||
weight_pack,
|
||||
backend,
|
||||
)
|
||||
self._device = weights.w13.device
|
||||
self._hidden_size = moe.hidden_dim
|
||||
self._max_num_tokens = moe.max_num_tokens
|
||||
self._num_experts = moe.num_experts
|
||||
self._top_k = moe.experts_per_token
|
||||
self._epilogue: FlashInferMoeEpEpilogue | None = epilogue
|
||||
self._return_workspace_view = moe.routing_method is RoutingMethodType.DeepseekV4
|
||||
|
||||
@property
|
||||
def can_overlap_shared_experts(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def output_is_reduced(self) -> bool:
|
||||
return True
|
||||
|
||||
@property
|
||||
def topk_indices_dtype(self) -> torch.dtype:
|
||||
return torch.int32
|
||||
|
||||
@property
|
||||
def is_monolithic(self) -> bool:
|
||||
return False
|
||||
|
||||
def _tensors(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
) -> Any:
|
||||
epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
|
||||
return self._moe_ep_tensors_cls(
|
||||
hidden_states=hidden_states,
|
||||
topk_ids=topk_ids,
|
||||
topk_weights=topk_weights,
|
||||
fc1_alpha=epilogue.fc1_alpha,
|
||||
fc2_alpha=epilogue.fc2_alpha,
|
||||
fc1_norm_const=epilogue.fc1_norm_const,
|
||||
)
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if self._mega_layer is None:
|
||||
raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
|
||||
if hidden_states.shape[0] > self._max_num_tokens:
|
||||
raise ValueError(
|
||||
f"FlashInfer MoE-EP got {hidden_states.shape[0]} tokens, but "
|
||||
f"the workspace supports at most {self._max_num_tokens}"
|
||||
)
|
||||
tensors = self._tensors(hidden_states, topk_ids, topk_weights)
|
||||
if self._return_workspace_view and getattr(
|
||||
self._mega_layer, "supports_output_view", False
|
||||
):
|
||||
return self._mega_layer.forward(tensors, return_workspace_view=True)
|
||||
return self._mega_layer(tensors)
|
||||
|
||||
@torch.inference_mode()
|
||||
def warmup(self) -> None:
|
||||
if self._mega_layer is None:
|
||||
return
|
||||
hidden_states = torch.zeros(
|
||||
1,
|
||||
self._hidden_size,
|
||||
dtype=torch.bfloat16,
|
||||
device=self._device,
|
||||
)
|
||||
topk_ids = (
|
||||
torch.arange(self._top_k, dtype=torch.int32, device=self._device)
|
||||
.mul_(self._num_experts // self._top_k)
|
||||
.view(1, self._top_k)
|
||||
)
|
||||
topk_weights = torch.full(
|
||||
(1, self._top_k),
|
||||
1.0 / self._top_k,
|
||||
dtype=torch.float32,
|
||||
device=self._device,
|
||||
)
|
||||
self._mega_layer.warmup(self._tensors(hidden_states, topk_ids, topk_weights))
|
||||
|
||||
def kernel_weights(
|
||||
self,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Kernel-resident fc1 weight, fc1 scales, fc2 weight, fc2 scales.
|
||||
|
||||
Contiguous per-expert byte tensors that alias the memory the megakernel
|
||||
reads, so a caller may permute experts in place (EPLB).
|
||||
"""
|
||||
if self._mega_layer is None:
|
||||
raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
|
||||
(fc1_weight, fc1_sf), (fc2_weight, fc2_sf) = self._mega_layer._transformed
|
||||
return (
|
||||
_expert_major_bytes(fc1_weight),
|
||||
_expert_major_bytes(fc1_sf),
|
||||
_expert_major_bytes(fc2_weight),
|
||||
_expert_major_bytes(fc2_sf),
|
||||
)
|
||||
|
||||
def expert_weight_views(self) -> tuple[torch.Tensor, ...]:
|
||||
"""Every per-expert tensor the kernel reads: weights plus epilogue."""
|
||||
epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
|
||||
vectors = (epilogue.fc1_alpha, epilogue.fc2_alpha, epilogue.fc1_norm_const)
|
||||
return (*self.kernel_weights(), *(v for v in vectors if v is not None))
|
||||
|
||||
def destroy(self) -> None:
|
||||
mega_layer = self._mega_layer
|
||||
if mega_layer is None:
|
||||
return
|
||||
mega_layer.destroy()
|
||||
self._mega_layer = None
|
||||
self._epilogue = None
|
||||
|
||||
def __del__(self) -> None:
|
||||
with contextlib.suppress(Exception):
|
||||
self.destroy()
|
||||
|
||||
|
||||
def _expert_major_bytes(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Contiguous storage of a per-expert kernel tensor, 1-byte dtypes as uint8.
|
||||
|
||||
The CuTeDSL kernel keeps (E, K, N) transpose views over (E, N, K) storage.
|
||||
"""
|
||||
if not tensor.is_contiguous():
|
||||
tensor = tensor.transpose(1, 2)
|
||||
if not tensor.is_contiguous():
|
||||
raise ValueError(f"expert tensor is not expert-major: {tensor.stride()}")
|
||||
return tensor.view(torch.uint8) if tensor.element_size() == 1 else tensor
|
||||
@@ -23,6 +23,9 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
RoutingMethodType,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsSink,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.runner.shared_experts import (
|
||||
SharedExperts,
|
||||
SharedExpertsOrder,
|
||||
@@ -979,9 +982,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
def is_monolithic() -> bool:
|
||||
return True
|
||||
|
||||
routing_replay_capture_fn: Callable[[torch.Tensor], None] | None = None
|
||||
_routing_replay_buffer: torch.Tensor | None = None
|
||||
|
||||
def supports_routing_replay_capture(self) -> bool:
|
||||
"""Whether this expert supports routing replay capture.
|
||||
|
||||
@@ -990,53 +990,6 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
"""
|
||||
return False
|
||||
|
||||
def set_capture_fn(
|
||||
self,
|
||||
capture_fn: Callable[[torch.Tensor], None] | None,
|
||||
) -> None:
|
||||
self.routing_replay_capture_fn = capture_fn
|
||||
if capture_fn is None:
|
||||
self._routing_replay_buffer = None
|
||||
return
|
||||
# Allocate for per-rank batches gathered across the DP or EP group.
|
||||
dispatch_group_size = (
|
||||
self.moe_config.ep_size
|
||||
if self.moe_config.use_ep
|
||||
else self.moe_config.dp_size
|
||||
)
|
||||
max_num_replay_tokens = self.moe_config.max_num_tokens * dispatch_group_size
|
||||
self._routing_replay_buffer = torch.empty(
|
||||
(max_num_replay_tokens, self.moe_config.experts_per_token),
|
||||
dtype=torch.int16,
|
||||
device=self.moe_config.device,
|
||||
)
|
||||
|
||||
def _maybe_make_routing_replay_buffer(
|
||||
self,
|
||||
num_tokens: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor | None:
|
||||
if self.routing_replay_capture_fn is None:
|
||||
return None
|
||||
buf = self._routing_replay_buffer
|
||||
assert buf is not None
|
||||
if buf.shape[0] < num_tokens or buf.device != device:
|
||||
raise ValueError(
|
||||
"Routing replay buffer was initialized for "
|
||||
f"{buf.shape[0]} tokens on {buf.device}, but the kernel "
|
||||
f"received {num_tokens} tokens on {device}."
|
||||
)
|
||||
return buf
|
||||
|
||||
def _maybe_dispatch_routing_replay(
|
||||
self,
|
||||
routing_replay_out: torch.Tensor | None,
|
||||
num_tokens: int,
|
||||
) -> None:
|
||||
if routing_replay_out is None or self.routing_replay_capture_fn is None:
|
||||
return
|
||||
self.routing_replay_capture_fn(routing_replay_out[:num_tokens])
|
||||
|
||||
def apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
@@ -1053,10 +1006,14 @@ class FusedMoEExpertsMonolithic(FusedMoEExperts):
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
routing_replay_out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""Same as ``FusedMoEExperts.apply``, except uses router_logits as opposed
|
||||
to the topk_ids and topk_weights. This is useful for kernels
|
||||
with fused router and fused_experts (e.g. FLASHINFER_TRTLLM).
|
||||
|
||||
Kernels that ``supports_routing_replay_capture`` write each token's
|
||||
routed expert ids into the leading rows of ``routing_replay_out``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1563,6 +1520,8 @@ class FusedMoEKernelMonolithicImpl:
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
*,
|
||||
routing_sink: RoutedExpertsSink | None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
"""Same as forward(), except uses router_logits as opposed
|
||||
to the topk_ids and topk_weights. This is used for kernels
|
||||
@@ -1575,6 +1534,15 @@ class FusedMoEKernelMonolithicImpl:
|
||||
defer_input_quant=self.fused_experts.expects_unquantized_inputs,
|
||||
)
|
||||
|
||||
routing_replay_out = None
|
||||
if routing_sink is not None:
|
||||
# Checked per call: a weight reload may rebuild a different kernel.
|
||||
if not self.fused_experts.supports_routing_replay_capture():
|
||||
raise ValueError(
|
||||
"Routed-experts capture is not supported with monolithic MoE "
|
||||
f"kernel {type(self.fused_experts).__name__}."
|
||||
)
|
||||
routing_replay_out = routing_sink.buffer[: len(a1q)]
|
||||
fused_out = self.fused_experts.apply(
|
||||
hidden_states=a1q,
|
||||
w1=w1,
|
||||
@@ -1590,7 +1558,10 @@ class FusedMoEKernelMonolithicImpl:
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_replay_out=routing_replay_out,
|
||||
)
|
||||
if routing_sink is not None:
|
||||
routing_sink.capture_fn(routing_replay_out)
|
||||
|
||||
if isinstance(fused_out, UnfinalizedMoEOutput):
|
||||
if not self.prepare_finalize.supports_deferred_moe_finalize():
|
||||
@@ -1698,6 +1669,8 @@ class FusedMoEKernel:
|
||||
e_score_correction_bias: torch.Tensor | None = None,
|
||||
routed_scaling_factor: float | None = None,
|
||||
topk_group: int | None = None,
|
||||
*,
|
||||
routing_sink: RoutedExpertsSink | None,
|
||||
) -> torch.Tensor | UnfinalizedMoEOutput:
|
||||
assert isinstance(self.impl, FusedMoEKernelMonolithicImpl)
|
||||
return self.impl.apply(
|
||||
@@ -1713,6 +1686,7 @@ class FusedMoEKernel:
|
||||
e_score_correction_bias=e_score_correction_bias,
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
topk_group=topk_group,
|
||||
routing_sink=routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import torch
|
||||
from torch._subclasses.fake_tensor import FakeTensor
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@@ -75,11 +76,18 @@ def moe_fused_mul_sum_kernel(
|
||||
def _heuristic_config(
|
||||
hidden_size: int,
|
||||
element_size: int,
|
||||
device_index: int = 0,
|
||||
):
|
||||
is_fp32 = element_size > 2
|
||||
max_block_k = 256 if is_fp32 else 512
|
||||
BLOCK_K = max(128, min(triton.next_power_of_2(hidden_size), max_block_k))
|
||||
num_warps = 4 if is_fp32 else 2
|
||||
if not is_fp32 and hidden_size >= 1024 and current_platform.is_cuda():
|
||||
capability = current_platform.get_device_capability(device_index)
|
||||
if capability is not None and capability >= (7, 5):
|
||||
# Larger tiles reduce the serial hidden-dimension loop.
|
||||
max_block_k = 4096
|
||||
num_warps = 8 if capability == (7, 5) else 16
|
||||
BLOCK_K = max(128, min(triton.next_power_of_2(hidden_size), max_block_k))
|
||||
num_stages = 3
|
||||
return BLOCK_K, num_warps, num_stages
|
||||
|
||||
@@ -149,6 +157,7 @@ def moe_fused_mul_sum(
|
||||
BLOCK_K, num_warps, num_stages = _heuristic_config(
|
||||
hidden_size,
|
||||
inputs.element_size(),
|
||||
inputs.device.index or 0,
|
||||
)
|
||||
grid = (num_tokens,)
|
||||
moe_fused_mul_sum_kernel[grid](
|
||||
|
||||
@@ -8,7 +8,11 @@ import torch
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm import envs
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config.kernel import MoEBackend
|
||||
from vllm.config.kernel import (
|
||||
FLASHINFER_MOE_EP_CUTEDSL,
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
MoEBackend,
|
||||
)
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -145,6 +149,9 @@ class Mxfp4MoeBackend(Enum):
|
||||
EMULATION = "EMULATION"
|
||||
# Humming
|
||||
HUMMING = "HUMMING"
|
||||
# FlashInfer MoE-EP megakernels: routing outside, dispatch/GEMMs/combine fused.
|
||||
FLASHINFER_MOE_EP_CUTEDSL = "FLASHINFER_MOE_EP_CUTEDSL"
|
||||
FLASHINFER_MOE_EP_DEEP_GEMM = "FLASHINFER_MOE_EP_DEEP_GEMM"
|
||||
|
||||
|
||||
# Backends that share the same TRTLLM weight format
|
||||
@@ -165,6 +172,13 @@ B12X_BACKENDS = (
|
||||
Mxfp4MoeBackend.B12X_MXFP4_BF16,
|
||||
)
|
||||
|
||||
# Megakernels consume the checkpoint's MXFP4 tensors through the shared
|
||||
# FlashInfer MoE-EP adapter; vLLM does not convert their weight layout.
|
||||
FLASHINFER_MOE_EP_MXFP4_BACKENDS = (
|
||||
Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
|
||||
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
)
|
||||
|
||||
|
||||
def backend_to_kernel_cls(
|
||||
backend: Mxfp4MoeBackend,
|
||||
@@ -291,6 +305,13 @@ def backend_to_kernel_cls(
|
||||
|
||||
return [OCP_MXQuantizationEmulationTritonExperts]
|
||||
|
||||
elif backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import ( # noqa: E501
|
||||
FlashInferMoeEpExperts,
|
||||
)
|
||||
|
||||
return [FlashInferMoeEpExperts]
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unknown MXFP4 MoE backend: {backend.value}")
|
||||
|
||||
@@ -314,6 +335,10 @@ def map_mxfp4_backend(runner_backend: MoEBackend) -> list[Mxfp4MoeBackend]:
|
||||
Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
|
||||
],
|
||||
"flashinfer_cutlass_afp8": [Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8],
|
||||
"flashinfer_moe_ep_cutedsl": [Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL],
|
||||
"flashinfer_moe_ep_mega_deep_gemm": [
|
||||
Mxfp4MoeBackend.FLASHINFER_MOE_EP_DEEP_GEMM
|
||||
],
|
||||
"triton": [Mxfp4MoeBackend.TRITON],
|
||||
"triton_unfused": [Mxfp4MoeBackend.TRITON_UNFUSED],
|
||||
"humming": [Mxfp4MoeBackend.HUMMING],
|
||||
@@ -735,6 +760,9 @@ def mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
activation: MoEActivation | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""Round up hidden_size and intermediate_size based on backend requirements."""
|
||||
if backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
# The megakernel takes the model's dimensions and validates them itself.
|
||||
return hidden_size, intermediate_size
|
||||
if backend in B12X_BACKENDS:
|
||||
# b12x plans for the exact model dimensions. B12xExperts validates the
|
||||
# required MXFP4 block alignment before selecting the backend.
|
||||
@@ -1827,6 +1855,29 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
w2_bias=w2_bias,
|
||||
_cache_permute_indices=_cache_permute_indices,
|
||||
)
|
||||
elif mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
|
||||
mxfp4_moe_ep_weights,
|
||||
)
|
||||
|
||||
# DeepGEMM consumes MXFP4 directly; CuTeDSL gets bf16 and requantizes.
|
||||
weights = mxfp4_moe_ep_weights(
|
||||
FLASHINFER_MOE_EP_CUTEDSL
|
||||
if mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL
|
||||
else FLASHINFER_MOE_EP_DEEP_GEMM,
|
||||
w13_weight,
|
||||
w2_weight,
|
||||
w13_weight_scale,
|
||||
w2_weight_scale,
|
||||
)
|
||||
return (
|
||||
weights.w13,
|
||||
weights.w2,
|
||||
weights.w13_scale,
|
||||
weights.w2_scale,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
)
|
||||
elif mxfp4_backend == Mxfp4MoeBackend.CPU:
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
prepare_mxfp4_moe_layer_for_cpu,
|
||||
@@ -1874,6 +1925,9 @@ def make_mxfp4_moe_quant_config(
|
||||
layer: "RoutedExperts | None" = None,
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
"""Create a FusedMoEQuantConfig for the given MXFP4 backend."""
|
||||
if mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
# The megakernel quantizes activations itself and owns its scales.
|
||||
return FusedMoEQuantConfig.make("nvfp4", weight_dtype="mxfp4")
|
||||
if mxfp4_backend == Mxfp4MoeBackend.B12X_MXFP4_MXFP8:
|
||||
return mxfp4_mxfp8_moe_quant_config(
|
||||
w1_bias=w1_bias,
|
||||
|
||||
@@ -50,6 +50,7 @@ class NvFp4MoeBackend(Enum):
|
||||
MARLIN = "MARLIN"
|
||||
HUMMING = "HUMMING"
|
||||
EMULATION = "EMULATION"
|
||||
FLASHINFER_MOE_EP_CUTEDSL = "FLASHINFER_MOE_EP_CUTEDSL"
|
||||
|
||||
|
||||
FLASHINFER_NVFP4_MOE_BACKENDS = [
|
||||
@@ -148,6 +149,12 @@ def backend_to_kernel_cls(
|
||||
)
|
||||
|
||||
return [Nvfp4QuantizationEmulationTritonExperts]
|
||||
elif backend == NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL:
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_moe_ep import ( # noqa: E501
|
||||
FlashInferMoeEpExperts,
|
||||
)
|
||||
|
||||
return [FlashInferMoeEpExperts]
|
||||
else:
|
||||
raise ValueError(f"Unknown NvFP4 MoE backend: {backend.value}")
|
||||
|
||||
@@ -160,6 +167,7 @@ def map_nvfp4_backend(runner_backend: MoEBackend) -> NvFp4MoeBackend:
|
||||
"flashinfer_trtllm": NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
"flashinfer_cutlass": NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
||||
"flashinfer_cutedsl": NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
"flashinfer_moe_ep_cutedsl": NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
|
||||
"flashinfer_b12x": NvFp4MoeBackend.FLASHINFER_B12X,
|
||||
"marlin": NvFp4MoeBackend.MARLIN,
|
||||
"humming": NvFp4MoeBackend.HUMMING,
|
||||
@@ -207,6 +215,7 @@ def select_nvfp4_moe_backend(
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
NvFp4MoeBackend.FLASHINFER_CUTLASS,
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
|
||||
NvFp4MoeBackend.VLLM_CUTLASS,
|
||||
NvFp4MoeBackend.MARLIN,
|
||||
NvFp4MoeBackend.EMULATION,
|
||||
@@ -281,8 +290,9 @@ def select_nvfp4_moe_backend(
|
||||
f"Model sets swiglu_limit={config.swiglu_limit}, but the "
|
||||
f"explicitly requested moe_backend={runner_backend!r} does "
|
||||
f"not apply the SwiGLU clamp. Use 'flashinfer_trtllm', "
|
||||
f"'flashinfer_cutlass', 'flashinfer_cutedsl', 'cutlass', "
|
||||
f"'b12x', 'marlin', or 'humming' instead."
|
||||
f"'flashinfer_cutlass', 'flashinfer_cutedsl', "
|
||||
f"'flashinfer_moe_ep_cutedsl', 'cutlass', 'b12x', 'marlin', "
|
||||
f"or 'humming' instead."
|
||||
)
|
||||
return _return_or_raise(
|
||||
requested_backend, config, weight_key, activation_key, activation_format
|
||||
@@ -491,6 +501,9 @@ def convert_to_nvfp4_moe_kernel_format(
|
||||
# for other experts - other selection strategies may be used.
|
||||
a13_scale = 1.0 / a13_scale.max().to(torch.float32)
|
||||
a2_scale = 1.0 / a2_scale.max().to(torch.float32)
|
||||
elif nvfp4_backend == NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL:
|
||||
# The megakernel transforms the checkpoint tensors itself at init.
|
||||
pass
|
||||
else:
|
||||
raise ValueError(f"Unknown NvFp4 backend for MoE: {nvfp4_backend}")
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Prepare/finalize stages that do no expert-parallel communication.
|
||||
|
||||
Selected by ``all2all_backend="passthrough"``: the experts implementation
|
||||
dispatches, combines and reduces itself, so nothing is left for these stages.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantConfig
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.no_dp_ep import (
|
||||
MoEPrepareAndFinalizeNoDPEPModular,
|
||||
)
|
||||
|
||||
|
||||
class PassThroughPrepareAndFinalize(MoEPrepareAndFinalizeNoDPEPModular):
|
||||
"""Hands activations and routing straight to experts that own their EP
|
||||
communication."""
|
||||
|
||||
def output_is_reduced(self) -> bool:
|
||||
return True
|
||||
|
||||
def supports_deferred_moe_finalize(self) -> bool:
|
||||
# The experts apply the top-k weights and combine internally; there
|
||||
# is no finalize left for a consumer to take over.
|
||||
return False
|
||||
|
||||
def prepare(
|
||||
self,
|
||||
a1: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
defer_input_quant: bool = False,
|
||||
) -> mk.PrepareResultType:
|
||||
return a1, None, None, None, None
|
||||
@@ -20,6 +20,9 @@ from vllm.model_executor.layers.fused_moe.fused_moe_method_base import (
|
||||
FusedMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
|
||||
RoutedExpertsSink,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
@@ -138,6 +141,8 @@ class RoutedExperts(PluggableLayer):
|
||||
self.swiglu_alpha = swiglu_alpha
|
||||
self.swiglu_beta = swiglu_beta
|
||||
self.e_score_correction_bias = e_score_correction_bias
|
||||
# Set by bind_routed_experts_capturer for monolithic kernels.
|
||||
self.routing_sink: RoutedExpertsSink | None = None
|
||||
self.apply_router_weight_on_input = apply_router_weight_on_input
|
||||
# End random parameters
|
||||
self._loaded_expert_biases: set[str] = set()
|
||||
|
||||
@@ -15,6 +15,7 @@ import torch
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import get_tp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -186,6 +187,23 @@ class RoutedExpertsCapturer:
|
||||
return self.device_buffer[:num_tokens].to(self.output_dtype)
|
||||
|
||||
|
||||
class RoutedExpertsSink:
|
||||
"""Layer-owned buffer and callback for the expert ids a monolithic kernel
|
||||
routes to; it outlives kernel rebuilds on weight reload."""
|
||||
|
||||
def __init__(
|
||||
self, moe_config: FusedMoEConfig, capture_fn: Callable[[torch.Tensor], None]
|
||||
) -> None:
|
||||
# Sized for per-rank batches gathered across the DP or EP group.
|
||||
group_size = moe_config.ep_size if moe_config.use_ep else moe_config.dp_size
|
||||
self.buffer = torch.empty(
|
||||
(moe_config.max_num_tokens * group_size, moe_config.experts_per_token),
|
||||
dtype=torch.int16,
|
||||
device=moe_config.device,
|
||||
)
|
||||
self.capture_fn = capture_fn
|
||||
|
||||
|
||||
def bind_routed_experts_capturer(
|
||||
model: torch.nn.Module,
|
||||
capturer: RoutedExpertsCapturer,
|
||||
@@ -219,7 +237,9 @@ def bind_routed_experts_capturer(
|
||||
"Routed-experts capture is not supported with monolithic "
|
||||
f"MoE kernel {type(fused_experts).__name__}."
|
||||
)
|
||||
fused_experts.set_capture_fn(capture_fn)
|
||||
module.routed_experts.routing_sink = RoutedExpertsSink(
|
||||
fused_experts.moe_config, capture_fn
|
||||
)
|
||||
num_bound += 1
|
||||
elif isinstance(module.router, BaseRouter):
|
||||
module.router.set_capture_fn(capture_fn)
|
||||
|
||||
@@ -362,4 +362,5 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
@@ -163,6 +163,14 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
current_platform.has_device_capability(90)
|
||||
and not current_platform.is_device_capability_family(120)
|
||||
)
|
||||
self._is_rocm = current_platform.is_rocm()
|
||||
self._aiter_enabled = False
|
||||
self._aiter_capable = False
|
||||
if self._is_rocm:
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
self._aiter_enabled = bool(rocm_aiter_ops.is_enabled())
|
||||
self._aiter_capable = bool(rocm_aiter_ops.is_indexer_top_k_enabled())
|
||||
|
||||
def resolve_backend(
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
@@ -193,6 +201,19 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
f"inputs violate DeepSelect's constraints: dtype={logits.dtype},"
|
||||
f" stride={logits.stride()}, topk={topk_tokens}"
|
||||
)
|
||||
elif self._backend == "aiter":
|
||||
if not self._is_rocm:
|
||||
failures.append("requires a ROCm platform")
|
||||
elif not self._aiter_enabled:
|
||||
failures.append("requires AITER (VLLM_ROCM_USE_AITER=1)")
|
||||
elif not self._aiter_capable:
|
||||
failures.append(
|
||||
"AITER has no tuned indexer top-k kernels for this GPU arch"
|
||||
)
|
||||
if topk_tokens not in (512, 1024, 2048, 4096):
|
||||
failures.append(
|
||||
f"topk_tokens must be in (512, 1024, 2048, 4096), got {topk_tokens}"
|
||||
)
|
||||
elif self._backend == "flashinfer":
|
||||
if not self._is_cuda:
|
||||
failures.append("requires a CUDA platform")
|
||||
@@ -216,11 +237,14 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
self, logits: torch.Tensor, topk_tokens: int, num_rows: int
|
||||
) -> str:
|
||||
"""The priority chain: cooperative -> persistent -> per_row.
|
||||
deep_select/flashinfer/torch are opt-in only.
|
||||
aiter/deep_select/flashinfer/torch are opt-in only.
|
||||
|
||||
cooperative_topk is preferred whenever it is applicable, i.e. within
|
||||
its AUTO_COOPERATIVE_MAX_ROWS row limit; larger batches go to
|
||||
persistent_topk.
|
||||
|
||||
aiter not auto-resolved as it does not outperform the per-row on every shape,
|
||||
hence we delegate to the caller to decide when to use it.
|
||||
"""
|
||||
if not self._cooperative_constraints(logits, topk_tokens, num_rows):
|
||||
return "cooperative"
|
||||
@@ -285,7 +309,8 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
max_seq_len: int,
|
||||
) -> None:
|
||||
"""Run the resolved decode top-k implementation, writing into
|
||||
topk_indices (int32, -1 fill for rows shorter than topk_tokens)."""
|
||||
topk_indices (int32, -1 fill for rows shorter than topk_tokens).
|
||||
"""
|
||||
backend = self.resolve_backend(logits, topk_tokens, logits.shape[0])
|
||||
if backend == "deep_select":
|
||||
row_ends = self._row_ends(seq_lens, next_n, logits.shape[0])
|
||||
@@ -344,15 +369,37 @@ class SparseIndexerTopk(torch.nn.Module):
|
||||
) < row_ends.unsqueeze(1)
|
||||
indices = torch.where(in_range, indices, -1)
|
||||
topk_indices.copy_(indices)
|
||||
else:
|
||||
assert backend == "per_row", f"unknown topk backend: {backend}"
|
||||
ops.top_k_per_row_decode(
|
||||
elif backend == "aiter":
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
# AITER's decode kernel only implements the 1D/next_n == 1 form,
|
||||
# which is equivalent once the ragged ends are expanded.
|
||||
rocm_aiter_ops.indexer_top_k_decode(
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
1,
|
||||
self._row_ends(seq_lens, next_n, logits.shape[0]),
|
||||
topk_indices,
|
||||
logits.shape[0],
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
)
|
||||
else:
|
||||
assert backend == "per_row", f"unknown topk backend: {backend}"
|
||||
self._per_row(logits, seq_lens, next_n, topk_indices, topk_tokens)
|
||||
|
||||
@staticmethod
|
||||
def _per_row(
|
||||
logits: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
next_n: int,
|
||||
topk_indices: torch.Tensor,
|
||||
topk_tokens: int,
|
||||
) -> None:
|
||||
ops.top_k_per_row_decode(
|
||||
logits,
|
||||
next_n,
|
||||
seq_lens,
|
||||
topk_indices,
|
||||
logits.shape[0],
|
||||
logits.stride(0),
|
||||
logits.stride(1),
|
||||
topk_tokens,
|
||||
)
|
||||
|
||||
@@ -551,28 +551,33 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
raise ValueError(
|
||||
f"VLLM_GDN_DECODE_KERNEL=cuda is not supported: {reason}"
|
||||
)
|
||||
logger.info_once(
|
||||
"Falling back to the Triton GDN decode path: %s", reason
|
||||
)
|
||||
if self.speculative_config is not None:
|
||||
logger.info_once(
|
||||
"Falling back to the Triton GDN spec decode path: %s",
|
||||
reason,
|
||||
)
|
||||
self.gdn_decode_kernel = "triton"
|
||||
elif current_platform.is_cpu():
|
||||
self.gdn_decode_kernel = "CPU"
|
||||
elif current_platform.is_xpu():
|
||||
self.gdn_decode_kernel = "XPU"
|
||||
|
||||
self.enable_fused_gdn_decode = self.gdn_decode_kernel == "cuda"
|
||||
self.enable_fused_gdn_spec_decode = (
|
||||
self.gdn_decode_kernel == "cuda" and self.speculative_config is not None
|
||||
)
|
||||
# RecoverSSM verify replaces the per-draft snapshot path (Triton decode
|
||||
# only)
|
||||
self.use_gdn_recoverssm = self._uses_gdn_recoverssm()
|
||||
if self.use_gdn_recoverssm:
|
||||
self.gdn_decode_kernel = "triton"
|
||||
self.enable_fused_gdn_decode = False
|
||||
self.enable_fused_gdn_spec_decode = False
|
||||
self.enable_packed_recurrent_decode = False
|
||||
logger.info_once(
|
||||
"GDN RecoverSSM speculative verify active (spec_query_len %d)",
|
||||
self.num_spec + 1,
|
||||
)
|
||||
logger.info_once("GDN decode kernel: %s", self.gdn_decode_kernel)
|
||||
if self.speculative_config is not None:
|
||||
logger.info_once("GDN spec decode kernel: %s", self.gdn_decode_kernel)
|
||||
|
||||
compilation_config = get_current_vllm_config().compilation_config
|
||||
if prefix in compilation_config.static_forward_context:
|
||||
@@ -969,12 +974,12 @@ class QwenGatedDeltaNetAttention(GatedDeltaNetAttention):
|
||||
mixed_qkvz, _ = self.in_proj_qkvz(hidden_states)
|
||||
ba, _ = self.in_proj_ba(hidden_states)
|
||||
|
||||
use_fused_gdn_decode = (
|
||||
self.enable_fused_gdn_decode
|
||||
use_fused_gdn_spec_decode = (
|
||||
self.enable_fused_gdn_spec_decode
|
||||
and hidden_states.dtype == torch.bfloat16
|
||||
and self.norm.weight.dtype in (torch.bfloat16, torch.float32)
|
||||
)
|
||||
if use_fused_gdn_decode:
|
||||
if use_fused_gdn_spec_decode:
|
||||
core_attn_out = torch.zeros(
|
||||
(num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
|
||||
dtype=hidden_states.dtype,
|
||||
|
||||
@@ -790,6 +790,7 @@ class AutoAWQMoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -779,4 +779,5 @@ class AutoGPTQMoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
+32
-5
@@ -24,10 +24,15 @@ from vllm.model_executor.layers.fused_moe.experts.marlin_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.xpu_moe import (
|
||||
XPUExpertsMxFp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.flashinfer_moe_ep import (
|
||||
is_flashinfer_moe_ep_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
B12X_BACKENDS,
|
||||
FLASHINFER_MOE_EP_MXFP4_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
convert_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
make_mxfp4_moe_quant_config,
|
||||
select_mxfp4_moe_backend,
|
||||
@@ -38,7 +43,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
|
||||
prepare_moe_fp4_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -49,12 +54,14 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
super().__init__(moe)
|
||||
self.group_size = 32
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.MARLIN
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None = None
|
||||
|
||||
# Backend selection must match the weight preparation below: CUTLASS
|
||||
# swizzles scales, b12x and XPU consume checkpoint packing, and Marlin
|
||||
# repacks weights and scales.
|
||||
# swizzles scales, b12x and XPU consume checkpoint packing, Marlin
|
||||
# repacks weights and scales, and the megakernels take the checkpoint
|
||||
# tensors as they are.
|
||||
self.use_cutlass_mxfp4 = CutlassExpertsMxfp4._supports_current_device()
|
||||
self.experts_cls: type[mk.FusedMoEExperts]
|
||||
if moe.moe_backend == "b12x":
|
||||
if moe.moe_backend == "b12x" or is_flashinfer_moe_ep_backend(moe.moe_backend):
|
||||
self.mxfp4_backend, experts_cls = select_mxfp4_moe_backend(moe)
|
||||
assert experts_cls is not None
|
||||
self.experts_cls = experts_cls
|
||||
@@ -70,6 +77,10 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
logger.info_once("Using MarlinExperts for MXFP4 MoE")
|
||||
self.experts_cls = MarlinExperts
|
||||
|
||||
@property
|
||||
def supports_eplb(self) -> bool:
|
||||
return self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -198,6 +209,21 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
layer.w2_weight_scale = torch.nn.Parameter(
|
||||
torch.stack(swizzled_w2), requires_grad=False
|
||||
)
|
||||
elif self.mxfp4_backend in FLASHINFER_MOE_EP_MXFP4_BACKENDS:
|
||||
w13, w2, w13_scale, w2_scale, _, _ = (
|
||||
convert_weight_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
layer=layer,
|
||||
w13_weight=layer.w13_weight,
|
||||
w2_weight=layer.w2_weight,
|
||||
w13_weight_scale=layer.w13_weight_scale,
|
||||
w2_weight_scale=layer.w2_weight_scale,
|
||||
)
|
||||
)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
elif self.mxfp4_backend in B12X_BACKENDS or current_platform.is_xpu():
|
||||
pass
|
||||
else:
|
||||
@@ -211,6 +237,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None:
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
|
||||
+6
-3
@@ -38,6 +38,7 @@ _EPLB_SUPPORTED_NVFP4_BACKENDS = frozenset(
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
NvFp4MoeBackend.FLASHINFER_CUTEDSL_BATCHED,
|
||||
NvFp4MoeBackend.FLASHINFER_TRTLLM,
|
||||
NvFp4MoeBackend.FLASHINFER_MOE_EP_CUTEDSL,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -61,7 +62,6 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
weight_key=kNvfp4Static,
|
||||
activation_key=None if use_a16 else kNvfp4Dynamic,
|
||||
)
|
||||
|
||||
self.use_global_sf = is_global_sf_supported_for_nvfp4_backend(
|
||||
self.nvfp4_backend
|
||||
)
|
||||
@@ -155,7 +155,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
set_weight_attrs(w13_weight_scale_2, extra_weight_attrs)
|
||||
|
||||
w2_weight_scale_2 = torch.nn.Parameter(
|
||||
torch.empty(num_experts, dtype=torch.float32), requires_grad=False
|
||||
torch.empty(num_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_weight_global_scale", w2_weight_scale_2)
|
||||
extra_weight_attrs.update(
|
||||
@@ -175,7 +176,8 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
set_weight_attrs(w13_input_scale, extra_weight_attrs)
|
||||
|
||||
w2_input_scale = torch.nn.Parameter(
|
||||
torch.empty(num_experts, dtype=torch.float32), requires_grad=False
|
||||
torch.empty(num_experts, dtype=torch.float32),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.register_parameter("w2_input_global_scale", w2_input_scale)
|
||||
extra_weight_attrs.update(
|
||||
@@ -293,6 +295,7 @@ class CompressedTensorsW4A4Nvfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+1
@@ -287,6 +287,7 @@ class CompressedTensorsW4A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+1
@@ -366,6 +366,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+1
@@ -291,4 +291,5 @@ class CompressedTensorsW8A8Int8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
+1
@@ -179,6 +179,7 @@ class CompressedTensorsW8A8Mxfp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
+1
@@ -528,6 +528,7 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -818,6 +818,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config.kernel import FLASHINFER_MOE_EP_BACKENDS
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoeWeightScaleSupported,
|
||||
@@ -53,6 +54,11 @@ class INCMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def __init__(self, moe) -> None:
|
||||
super().__init__(moe)
|
||||
if moe.moe_backend in FLASHINFER_MOE_EP_BACKENDS:
|
||||
raise ValueError(
|
||||
f"moe_backend={moe.moe_backend!r} is not supported for AutoRound "
|
||||
"(INC) MXFP4 MoE checkpoints"
|
||||
)
|
||||
self.group_size = 32
|
||||
# Backend selection must stay consistent with the weight preparation in
|
||||
# process_weights_after_loading / get_fused_moe_quant_config, which use
|
||||
|
||||
@@ -201,7 +201,7 @@ class QuantFP8(CustomOp):
|
||||
|
||||
if scale is None:
|
||||
if self.group_shape == GroupShape.PER_TOKEN:
|
||||
x_max, _ = x.abs().max(dim=-1)
|
||||
x_max = x.abs().amax(dim=-1)
|
||||
x_max = x_max.unsqueeze(-1).to(torch.float32)
|
||||
if scale_ub is not None:
|
||||
x_max = x_max.clamp(max=scale_ub)
|
||||
|
||||
@@ -680,6 +680,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
@@ -1092,6 +1093,7 @@ class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
@@ -1461,6 +1463,7 @@ class ModelOptMxFp8FusedMoE(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
@@ -2314,9 +2317,9 @@ def select_linear_kernel(
|
||||
w = spec.weight
|
||||
assert isinstance(w, QuantKey), f"resolve() must supply a weight key, got {w!r}"
|
||||
if w.dtype == FP4_DTYPE:
|
||||
# W4A16 (activation is None) → use_a16=True defaults to Marlin *and*
|
||||
# honors --linear-backend (matches upstream ModelOptNvFp4W4A16LinearMethod
|
||||
# after #50273); W4A4 → use_a16=False.
|
||||
# W4A16 (activation is None) → use_a16=True, auto prefers FlashInfer
|
||||
# CuTe-DSL backend on SM100/103 when available and Marlin otherwise,
|
||||
# and honors --linear-backend (#50273); W4A4 → use_a16=False.
|
||||
return init_nvfp4_linear_kernel(use_a16=spec.activation is None)
|
||||
if w.scale.dtype == MXFP8_SCALE_DTYPE:
|
||||
return init_mxfp8_linear_kernel(
|
||||
|
||||
@@ -531,6 +531,7 @@ class MoeWNA16Method(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
@@ -21,6 +23,7 @@ from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
TRITON_BACKENDS,
|
||||
Mxfp4MoeBackend,
|
||||
backend_to_kernel_cls,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
convert_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_kernel,
|
||||
@@ -472,9 +475,67 @@ class GptOssMxfp4MoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
|
||||
def _use_k3_situ_aiter(moe: FusedMoEConfig) -> bool:
|
||||
"""Route Kimi-K3 weight-only MXFP4 SiTU MoE to AITER A16W4 on gfx950."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
return (
|
||||
rocm_aiter_ops.is_fused_moe_enabled()
|
||||
and on_gfx950()
|
||||
and moe.activation == MoEActivation.SITU
|
||||
and moe.activation_situ_linear_beta is not None
|
||||
and rocm_aiter_ops.get_aiter_activation_type("situ") is not None
|
||||
)
|
||||
|
||||
|
||||
def _use_k3_situ_int4_gfx942(moe: FusedMoEConfig) -> bool:
|
||||
"""Opt-in gfx942 MXFP4-to-int4 requant for Kimi-K3 SiTU experts."""
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.platforms.rocm import on_gfx942
|
||||
|
||||
return (
|
||||
rocm_aiter_ops.is_fused_moe_enabled()
|
||||
and on_gfx942()
|
||||
and moe.activation == MoEActivation.SITU
|
||||
and moe.activation_situ_linear_beta is not None
|
||||
and _moe_weight_override_is_int4()
|
||||
)
|
||||
|
||||
|
||||
def _moe_weight_override_is_int4() -> bool:
|
||||
"""True when --quantization-config.moe.weight int4_per_group_32 was set."""
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
if vllm_config is None:
|
||||
return False
|
||||
quant_args = getattr(vllm_config.model_config, "quantization_config", None)
|
||||
moe_spec = getattr(quant_args, "moe", None)
|
||||
weight = getattr(moe_spec, "weight", None)
|
||||
if weight is None:
|
||||
return False
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4Static32,
|
||||
)
|
||||
|
||||
return weight == kInt4Static32
|
||||
|
||||
|
||||
class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
"""MXFP4 MoE quantization method."""
|
||||
|
||||
@@ -482,9 +543,33 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def __init__(self, moe: FusedMoEConfig):
|
||||
super().__init__(moe)
|
||||
|
||||
self.weight_dtype = "mxfp4"
|
||||
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(moe)
|
||||
self.is_k3_situ_aiter = _use_k3_situ_aiter(moe)
|
||||
self.is_k3_situ_int4_gfx942 = _use_k3_situ_int4_gfx942(moe)
|
||||
self.experts_cls: type[mk.FusedMoEExperts] | None
|
||||
if self.is_k3_situ_aiter or self.is_k3_situ_int4_gfx942:
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.AITER_MXFP4_BF16
|
||||
self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]
|
||||
if self.is_k3_situ_int4_gfx942:
|
||||
logger.warning_once(
|
||||
"Requantizing Kimi-K3 MXFP4 experts to groupwise int4 "
|
||||
"for gfx942. This conversion is lossy and was explicitly "
|
||||
"enabled by --quantization-config.moe.weight "
|
||||
"int4_per_group_32."
|
||||
)
|
||||
else:
|
||||
logger.info_once("Using AITER_MXFP4_BF16 for Kimi-K3 SiTU MXFP4 MoE.")
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if rocm_aiter_ops.is_fused_moe_situv2_enabled():
|
||||
# a16w4 does not read this bound. Force 0 so leftover a8w4
|
||||
# config cannot keep bf16 activations.
|
||||
# TODO: Remove once AITER takes this as a kernel argument.
|
||||
os.environ["AITER_BF16_FP8_MOE_BOUND"] = "0"
|
||||
else:
|
||||
self.mxfp4_backend, self.experts_cls = select_deepseek_v4_mxfp4_moe_backend(
|
||||
moe
|
||||
)
|
||||
|
||||
self.max_capture_size = moe.max_capture_size
|
||||
|
||||
@@ -782,7 +867,142 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
self.moe_kernel.fused_experts.process_weights_after_loading(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
def _setup_kernel_k3_situ_gfx942(self, layer: RoutedExperts) -> None:
|
||||
import inspect
|
||||
|
||||
from aiter.ops.flydsl.kernels.moe_2stage_a16wmix import (
|
||||
flydsl_a16w4_gemm1,
|
||||
)
|
||||
|
||||
# Fail closed if this AITER build would silently run SiLU.
|
||||
if "act" not in inspect.signature(flydsl_a16w4_gemm1).parameters:
|
||||
raise RuntimeError(
|
||||
"This AITER build ignores the SiTUv2 activation on the "
|
||||
"replacement packed-int4 MoE path and would silently compute "
|
||||
"SiLU. Rebuild with the replacement A16W4/int4 pipeline."
|
||||
)
|
||||
|
||||
from aiter import dtypes as aiter_dtypes
|
||||
from aiter.ops.quant import per_1x32_i4_quant
|
||||
from aiter.ops.shuffle import (
|
||||
pack_int8_to_packed_int4,
|
||||
shuffle_scale_for_int4,
|
||||
shuffle_weight,
|
||||
)
|
||||
from aiter.utility import fp4_utils
|
||||
|
||||
fp4_dtype = torch.float4_e2m1fn_x2
|
||||
e8m0_dtype = torch.float8_e8m0fnu
|
||||
|
||||
# mxfp4_to_f32 grows the packed weight 8x and peaks above 20 GiB per
|
||||
# rank if all experts convert at once. Convert 8 experts at a time.
|
||||
_CONVERT_CHUNK = 8
|
||||
|
||||
def convert(
|
||||
weight: torch.nn.Parameter,
|
||||
scale: torch.nn.Parameter,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Drop the source before returning. Packed int4 is the same size as
|
||||
# packed MXFP4, so holding both doubles the footprint. That fits
|
||||
# under expert parallel (1/ep_size experts per rank) and does not
|
||||
# fit under pure tensor parallel.
|
||||
w_all = weight.data.view(fp4_dtype)
|
||||
s_all = scale.data.view(e8m0_dtype)
|
||||
num_experts = w_all.shape[0]
|
||||
|
||||
# Write chunks into a preallocated output. torch.cat of a list
|
||||
# would hold the full result twice at the join.
|
||||
out_packed: torch.Tensor | None = None
|
||||
out_scale: torch.Tensor | None = None
|
||||
scale_stride = 0
|
||||
for lo in range(0, num_experts, _CONVERT_CHUNK):
|
||||
hi = min(lo + _CONVERT_CHUNK, num_experts)
|
||||
weight_f32 = fp4_utils.mxfp4_to_f32(w_all[lo:hi])
|
||||
scale_f32 = fp4_utils.e8m0_to_f32(s_all[lo:hi])
|
||||
chunk_experts, output_size, input_size = weight_f32.shape
|
||||
weight_bf16 = (
|
||||
(
|
||||
weight_f32.view(
|
||||
chunk_experts, output_size, input_size // 32, 32
|
||||
)
|
||||
* scale_f32.view(
|
||||
chunk_experts, output_size, input_size // 32, 1
|
||||
)
|
||||
)
|
||||
.view(chunk_experts, output_size, input_size)
|
||||
.to(torch.bfloat16)
|
||||
)
|
||||
del weight_f32, scale_f32
|
||||
|
||||
weight_int4, weight_scale = per_1x32_i4_quant(weight_bf16)
|
||||
del weight_bf16
|
||||
weight_int4 = weight_int4.view(aiter_dtypes.i4x2).view(
|
||||
chunk_experts, output_size, input_size
|
||||
)
|
||||
weight_packed = pack_int8_to_packed_int4(
|
||||
shuffle_weight(weight_int4.view(aiter_dtypes.i8), (16, 16))
|
||||
)
|
||||
del weight_int4
|
||||
chunk_packed = weight_packed.view(
|
||||
chunk_experts, output_size, input_size // 2
|
||||
).view(aiter_dtypes.i4x2)
|
||||
chunk_scale = (
|
||||
shuffle_scale_for_int4(weight_scale, group_size=32)
|
||||
.view(-1)
|
||||
.contiguous()
|
||||
)
|
||||
if out_packed is None:
|
||||
out_packed = torch.empty(
|
||||
(num_experts,) + tuple(chunk_packed.shape[1:]),
|
||||
dtype=chunk_packed.dtype,
|
||||
device=chunk_packed.device,
|
||||
)
|
||||
scale_stride = chunk_scale.numel() // chunk_experts
|
||||
out_scale = torch.empty(
|
||||
num_experts * scale_stride,
|
||||
dtype=chunk_scale.dtype,
|
||||
device=chunk_scale.device,
|
||||
)
|
||||
assert out_packed is not None
|
||||
assert out_scale is not None
|
||||
out_packed[lo:hi].copy_(chunk_packed)
|
||||
out_scale[lo * scale_stride : hi * scale_stride].copy_(chunk_scale)
|
||||
del weight_packed, weight_scale, chunk_packed, chunk_scale
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
del w_all, s_all
|
||||
assert out_packed is not None
|
||||
assert out_scale is not None
|
||||
weight.data = torch.empty(0, dtype=torch.uint8, device=out_packed.device)
|
||||
scale.data = torch.empty(0, dtype=torch.uint8, device=out_packed.device)
|
||||
torch.accelerator.empty_cache()
|
||||
return out_packed, out_scale
|
||||
|
||||
w13, w13_scale = convert(layer.w13_weight, layer.w13_weight_scale)
|
||||
replace_parameter(layer, "w13_weight", w13)
|
||||
replace_parameter(layer, "w13_weight_scale", w13_scale)
|
||||
del w13, w13_scale
|
||||
torch.accelerator.empty_cache()
|
||||
|
||||
w2, w2_scale = convert(layer.w2_weight, layer.w2_weight_scale)
|
||||
replace_parameter(layer, "w2_weight", w2)
|
||||
replace_parameter(layer, "w2_weight_scale", w2_scale)
|
||||
del w2, w2_scale
|
||||
torch.accelerator.empty_cache()
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config is not None and self.experts_cls is not None:
|
||||
self.moe_kernel = make_mxfp4_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
if self.mxfp4_backend == Mxfp4MoeBackend.NONE:
|
||||
return
|
||||
|
||||
@@ -795,6 +1015,10 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
self._build_moe_kernel(layer)
|
||||
return
|
||||
|
||||
if self.is_k3_situ_int4_gfx942:
|
||||
self._setup_kernel_k3_situ_gfx942(layer)
|
||||
return
|
||||
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
w13_scale = layer.w13_weight_scale
|
||||
@@ -900,4 +1124,5 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
@@ -65,6 +65,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kInt4Static32,
|
||||
kInt8StaticChannelSym,
|
||||
kMxfp4Static,
|
||||
kMxfp8Dynamic,
|
||||
@@ -231,6 +232,9 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
"""
|
||||
if spec is None or spec.weight is None:
|
||||
return None
|
||||
# Load-time gfx942 requant, not online conversion. Mxfp4MoEMethod owns it.
|
||||
if spec.weight == kInt4Static32:
|
||||
return None
|
||||
cls = table.get(spec.weight)
|
||||
if cls is None:
|
||||
raise ValueError(
|
||||
|
||||
@@ -156,6 +156,7 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
def apply(
|
||||
|
||||
@@ -1976,6 +1976,11 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
num_expert_group=layer.num_expert_group,
|
||||
topk_group=layer.topk_group,
|
||||
e_score_correction_bias=layer.e_score_correction_bias,
|
||||
routed_scaling_factor=layer.routed_scaling_factor,
|
||||
routing_sink=layer.routing_sink,
|
||||
)
|
||||
|
||||
|
||||
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 128,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 2
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"1": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 3
|
||||
},
|
||||
"4": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"16": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 256,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 16,
|
||||
"num_stages": 2
|
||||
},
|
||||
"32": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
},
|
||||
"64": {
|
||||
"BLOCK_SIZE_M": 32,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 1,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"128": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"512": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"1024": {
|
||||
"BLOCK_SIZE_M": 64,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 3
|
||||
},
|
||||
"2048": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 8,
|
||||
"num_stages": 2
|
||||
},
|
||||
"4096": {
|
||||
"BLOCK_SIZE_M": 128,
|
||||
"BLOCK_SIZE_N": 128,
|
||||
"BLOCK_SIZE_K": 64,
|
||||
"GROUP_SIZE_M": 32,
|
||||
"num_warps": 32,
|
||||
"num_stages": 3
|
||||
}
|
||||
}
|
||||
@@ -25,6 +25,11 @@ INT4_DTYPE = scalar_types.uint4b8
|
||||
INT8_DTYPE = scalar_types.uint8b128
|
||||
|
||||
|
||||
def require_finite_positive_scales(name: str, value: torch.Tensor) -> None:
|
||||
if not torch.isfinite(value).all() or not (value > 0).all():
|
||||
raise ValueError(f"{name} must contain finite positive scales")
|
||||
|
||||
|
||||
def _dtype_abbr(dtype: torch.dtype | ScalarType) -> str:
|
||||
"""Return a stable short name for torch and ScalarType dtypes."""
|
||||
if isinstance(dtype, ScalarType):
|
||||
|
||||
@@ -457,8 +457,6 @@ class DeepseekOCRForCausalLM(
|
||||
return None
|
||||
assert isinstance(pixel_values, torch.Tensor)
|
||||
assert images_crop is None or isinstance(images_crop, torch.Tensor)
|
||||
if torch.sum(pixel_values).item() == 0:
|
||||
return None
|
||||
|
||||
# Use actual tensor spatial dim instead of hardcoded
|
||||
# vision_config.image_size (1024). The vision encoders (SAM & CLIP)
|
||||
|
||||
@@ -318,7 +318,7 @@ class DeepseekOCR2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, Support
|
||||
images_spatial_crop = kwargs.pop("images_spatial_crop", None)
|
||||
images_crop = kwargs.pop("images_crop", None)
|
||||
|
||||
if pixel_values is None or torch.sum(pixel_values).item() == 0:
|
||||
if pixel_values is None:
|
||||
return None
|
||||
|
||||
base_size = self.vision_config.image_size
|
||||
|
||||
@@ -1,324 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from
|
||||
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/gpt_neox/modeling_gpt_neox.py
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright 2022 EleutherAI The HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only GPT-NeoX model compatible with HuggingFace weights."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import GPTNeoXConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
|
||||
class GPTNeoXAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: GPTNeoXConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.total_num_heads = config.num_attention_heads
|
||||
self.hidden_size = config.hidden_size
|
||||
self.head_size = self.hidden_size // self.total_num_heads
|
||||
self.bias = getattr(config, "attention_bias", True)
|
||||
|
||||
tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
|
||||
assert self.total_num_heads % tensor_model_parallel_world_size == 0
|
||||
self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
|
||||
|
||||
self.query_key_value = QKVParallelLinear(
|
||||
config.hidden_size,
|
||||
self.head_size,
|
||||
self.total_num_heads,
|
||||
bias=self.bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.query_key_value",
|
||||
)
|
||||
self.dense = RowParallelLinear(
|
||||
config.hidden_size,
|
||||
config.hidden_size,
|
||||
bias=self.bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.dense",
|
||||
)
|
||||
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_size,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
)
|
||||
scaling = self.head_size**-0.5
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
scaling,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
position_ids: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.query_key_value(hidden_states)
|
||||
q, k, v = qkv.chunk(chunks=3, dim=-1)
|
||||
q, k = self.rotary_emb(position_ids, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.dense(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class GPTNeoXMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: GPTNeoXConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.dense_h_to_4h = ColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
config.intermediate_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.dense_h_to_4h",
|
||||
)
|
||||
self.dense_4h_to_h = RowParallelLinear(
|
||||
config.intermediate_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.dense_4h_to_h",
|
||||
)
|
||||
self.act = get_act_fn(config.hidden_act)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
hidden_states, _ = self.dense_h_to_4h(hidden_states)
|
||||
hidden_states = self.act(hidden_states)
|
||||
hidden_states, _ = self.dense_4h_to_h(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class GPTNeoXLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: GPTNeoXConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.use_parallel_residual = config.use_parallel_residual
|
||||
self.input_layernorm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.layer_norm_eps
|
||||
)
|
||||
self.post_attention_layernorm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.layer_norm_eps
|
||||
)
|
||||
self.attention = GPTNeoXAttention(
|
||||
config, cache_config, quant_config, prefix=f"{prefix}.attention"
|
||||
)
|
||||
self.mlp = GPTNeoXMLP(config, quant_config, prefix=f"{prefix}.mlp")
|
||||
|
||||
def forward(
|
||||
self,
|
||||
position_ids: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
attn_input = self.input_layernorm(hidden_states)
|
||||
attn_output = self.attention(
|
||||
position_ids=position_ids,
|
||||
hidden_states=attn_input,
|
||||
)
|
||||
|
||||
if self.use_parallel_residual:
|
||||
# pseudocode:
|
||||
# x = x + attn(ln1(x)) + mlp(ln2(x))
|
||||
mlp_input = self.post_attention_layernorm(hidden_states)
|
||||
mlp_output = self.mlp(mlp_input)
|
||||
hidden_states = mlp_output + attn_output + hidden_states
|
||||
else:
|
||||
# pseudocode:
|
||||
# x = x + attn(ln1(x))
|
||||
# x = x + mlp(ln2(x))
|
||||
attn_output = attn_output + hidden_states
|
||||
mlp_input = self.post_attention_layernorm(attn_output)
|
||||
mlp_output = self.mlp(mlp_input)
|
||||
hidden_states = mlp_output + attn_output
|
||||
return hidden_states
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class GPTNeoXModel(nn.Module):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_substr={"attention.bias": None, "attention.masked_bias": None}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
|
||||
self.embed_in = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
)
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: GPTNeoXLayer(
|
||||
config, cache_config, quant_config, prefix=prefix
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.final_layer_norm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.layer_norm_eps
|
||||
)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], config.hidden_size
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_in(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
position_ids: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
hidden_states = layer(position_ids, hidden_states)
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
hidden_states = self.final_layer_norm(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
def _repack_qkv(
|
||||
self, weights: Iterable[tuple[str, torch.Tensor]]
|
||||
) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
# GPT-NeoX's fused QKV is laid out as (num_heads * 3 * head_size) on
|
||||
# its output dim (0), while vLLM expects (3 * num_heads * head_size).
|
||||
num_heads = self.config.num_attention_heads
|
||||
for name, loaded_weight in weights:
|
||||
if "query_key_value" in name:
|
||||
shape = loaded_weight.shape
|
||||
loaded_weight = loaded_weight.view((num_heads, 3, -1) + shape[1:])
|
||||
loaded_weight = loaded_weight.transpose(0, 1)
|
||||
loaded_weight = loaded_weight.reshape(shape)
|
||||
yield name, loaded_weight
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(
|
||||
self._repack_qkv(weights), mapper=self.hf_to_vllm_mapper
|
||||
)
|
||||
|
||||
|
||||
class GPTNeoXForCausalLM(nn.Module, SupportsPP):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.gpt_neox = GPTNeoXModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "gpt_neox")
|
||||
)
|
||||
self.embed_out = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "embed_out"),
|
||||
)
|
||||
if self.config.tie_word_embeddings:
|
||||
self.embed_out = self.embed_out.tie_weights(self.gpt_neox.embed_in)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.gpt_neox.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.gpt_neox.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
hidden_states = self.gpt_neox(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.embed_out, hidden_states)
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
@@ -1,441 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from
|
||||
# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
||||
# and OPT implementations in this library. It has been modified from its
|
||||
# original forms to accommodate minor architectural differences compared
|
||||
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Inference-only Jais2 model compatible with HuggingFace weights."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Jais2Config
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import ReLUSquaredActivation
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
extract_layer_index,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
|
||||
class Jais2MLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
hidden_act: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
bias: bool = False,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.up_proj = ColumnParallelLinear(
|
||||
input_size=hidden_size,
|
||||
output_size=intermediate_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
input_size=intermediate_size,
|
||||
output_size=hidden_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
self.act_fn = ReLUSquaredActivation()
|
||||
|
||||
def forward(self, x):
|
||||
x, _ = self.up_proj(x)
|
||||
x = self.act_fn(x)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class Jais2Attention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: Jais2Config,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
max_position_embeddings: int = 8192,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
bias: bool = False,
|
||||
cache_config: CacheConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
layer_idx = extract_layer_index(prefix)
|
||||
self.hidden_size = hidden_size
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.total_num_heads = num_heads
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.total_num_kv_heads = num_kv_heads
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
# Number of KV heads is greater than TP size, so we partition
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert self.total_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
# Number of KV heads is less than TP size, so we replicate
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
# MistralConfig has an optional head_dim introduced by Mistral-Nemo
|
||||
self.head_dim = getattr(
|
||||
config, "head_dim", self.hidden_size // self.total_num_heads
|
||||
)
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
self.scaling = self.head_dim**-0.5
|
||||
self.max_position_embeddings = max_position_embeddings
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size=hidden_size,
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=self.total_num_heads,
|
||||
total_num_kv_heads=self.total_num_kv_heads,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
|
||||
self.o_proj = RowParallelLinear(
|
||||
input_size=self.total_num_heads * self.head_dim,
|
||||
output_size=hidden_size,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
is_neox_style = True
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=getattr(config, "rope_parameters", None),
|
||||
is_neox_style=is_neox_style,
|
||||
)
|
||||
|
||||
if hasattr(config, "interleaved_sliding_window"):
|
||||
interleaved_sliding_window = config.interleaved_sliding_window
|
||||
if isinstance(interleaved_sliding_window, int):
|
||||
sliding_window = interleaved_sliding_window
|
||||
elif isinstance(interleaved_sliding_window, list):
|
||||
sw_idx = layer_idx % len(interleaved_sliding_window)
|
||||
sliding_window = interleaved_sliding_window[sw_idx]
|
||||
else:
|
||||
raise ValueError(
|
||||
f"{type(interleaved_sliding_window)} is not supported."
|
||||
)
|
||||
else:
|
||||
sliding_window = None
|
||||
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
per_layer_sliding_window=sliding_window,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class Jais2DecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
config: Jais2Config,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
config = config or vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = self.get_quant_config(vllm_config)
|
||||
|
||||
self.hidden_size = config.hidden_size
|
||||
max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
|
||||
# Support abacusai/Smaug-72B-v0.1 with attention_bias
|
||||
attention_bias = getattr(config, "attention_bias", False) or getattr(
|
||||
config, "bias", False
|
||||
)
|
||||
self.self_attn = Jais2Attention(
|
||||
config=config,
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
num_kv_heads=getattr(
|
||||
config, "num_key_value_heads", config.num_attention_heads
|
||||
),
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
quant_config=quant_config,
|
||||
bias=attention_bias,
|
||||
cache_config=cache_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
self.mlp = Jais2MLP(
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
bias=getattr(config, "mlp_bias", False),
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.input_layernorm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.layer_norm_eps
|
||||
)
|
||||
self.post_attention_layernorm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.layer_norm_eps
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Self Attention
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = (
|
||||
self.input_layernorm(hidden_states + residual),
|
||||
hidden_states + residual,
|
||||
)
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
)
|
||||
|
||||
# Fully Connected
|
||||
hidden_states, residual = (
|
||||
self.post_attention_layernorm(hidden_states + residual),
|
||||
hidden_states + residual,
|
||||
)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
def get_quant_config(self, vllm_config: VllmConfig) -> QuantizationConfig | None:
|
||||
"""Get quantization config for this layer. Override in subclasses."""
|
||||
return vllm_config.quant_config
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class Jais2Model(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
layer_type: type[nn.Module] = Jais2DecoderLayer,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
self.org_vocab_size = config.vocab_size
|
||||
if get_pp_group().is_first_rank or (
|
||||
config.tie_word_embeddings and get_pp_group().is_last_rank
|
||||
):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
self.vocab_size,
|
||||
config.hidden_size,
|
||||
org_num_embeddings=config.vocab_size,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: layer_type(
|
||||
config=config,
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
residual = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
hidden_states, residual = layer(positions, hidden_states, residual)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
|
||||
hidden_states, _ = self.norm(hidden_states + residual), residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Jais2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
}
|
||||
|
||||
embedding_modules = {
|
||||
"embed_tokens": "input_embeddings",
|
||||
"lm_head": "output_embeddings",
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
self.config = config
|
||||
self.model = self._init_model(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
if config.tie_word_embeddings:
|
||||
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
|
||||
|
||||
logit_scale = getattr(config, "logit_scale", 1.0)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
config.vocab_size, scale=logit_scale
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def _init_model(self, vllm_config: VllmConfig, prefix: str = ""):
|
||||
return Jais2Model(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
model_output = self.model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
return model_output
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
@@ -7,8 +7,6 @@ from typing import Annotated, Any, Literal, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
from transformers import BatchFeature, Mistral3Config, PixtralVisionConfig
|
||||
from transformers.models.pixtral import PixtralProcessor
|
||||
|
||||
@@ -59,10 +57,6 @@ from .utils import (
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY = Version(transformers.__version__) >= Version(
|
||||
"5.15.0"
|
||||
)
|
||||
|
||||
|
||||
class Mistral3ImagePixelInputs(TensorSchema):
|
||||
"""Dimensions:
|
||||
@@ -284,13 +278,6 @@ class Mistral3DummyInputsBuilder(BaseDummyInputsBuilder):
|
||||
|
||||
|
||||
class Mistral3MultiModalProcessor(BaseMultiModalProcessor[Mistral3ProcessingInfo]):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None:
|
||||
# PixtralProcessor supports image-only calls starting in transformers 5.15.
|
||||
# Older releases need dummy text and take the slower tokenization path.
|
||||
if not TRANSFORMERS_SUPPORTS_PIXTRAL_IMAGE_ONLY:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
return None
|
||||
|
||||
def _postprocess_hf_mm_data(
|
||||
self,
|
||||
mm_data: Mapping[str, object],
|
||||
|
||||
@@ -568,15 +568,6 @@ class NemotronParseForConditionalGeneration(nn.Module, SupportsMultiModal):
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.decoder.vocab_size, config.decoder.d_model, quant_config=quant_config
|
||||
)
|
||||
# Some checkpoints (e.g. compact exports) tie the output head to the
|
||||
# decoder's input embeddings instead of materializing a separate
|
||||
# lm_head.weight tensor.
|
||||
tie_word_embeddings = bool(
|
||||
getattr(config, "tie_word_embeddings", False)
|
||||
or getattr(config.decoder, "tie_word_embeddings", False)
|
||||
)
|
||||
if tie_word_embeddings:
|
||||
self.lm_head = self.lm_head.tie_weights(self.decoder.embed_tokens)
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.vocab_size, config.decoder.vocab_size
|
||||
)
|
||||
|
||||
@@ -1,325 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Adapted from
|
||||
# https://huggingface.co/microsoft/phi-1_5/blob/main/modeling_phi.py
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT license.
|
||||
#
|
||||
# BSD 3-Clause License
|
||||
#
|
||||
# Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu.
|
||||
# All rights reserved.
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions are met:
|
||||
#
|
||||
# * Redistributions of source code must retain the above copyright notice, this
|
||||
# list of conditions and the following disclaimer.
|
||||
#
|
||||
# * Redistributions in binary form must reproduce the above copyright notice,
|
||||
# this list of conditions and the following disclaimer in the documentation
|
||||
# and/or other materials provided with the distribution.
|
||||
#
|
||||
# * Neither the name of the copyright holder nor the names of its
|
||||
# contributors may be used to endorse or promote products derived from
|
||||
# this software without specific prior written permission.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
"""Inference-only Phi-1.5 model compatible with HuggingFace weights."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PhiConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.model_executor.layers.activation import get_act_fn
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
|
||||
class PhiAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PhiConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
self.head_size = self.hidden_size // config.num_attention_heads
|
||||
|
||||
tensor_model_parallel_world_size = get_tensor_model_parallel_world_size()
|
||||
assert config.num_attention_heads % tensor_model_parallel_world_size == 0
|
||||
self.num_heads = config.num_attention_heads // tensor_model_parallel_world_size
|
||||
|
||||
# pylint: disable=C0103
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
self.hidden_size,
|
||||
self.head_size,
|
||||
config.num_attention_heads,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.dense = RowParallelLinear(
|
||||
self.hidden_size,
|
||||
self.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.dense",
|
||||
)
|
||||
|
||||
scaling = self.head_size**-0.5
|
||||
|
||||
max_position_embeddings = getattr(config, "max_position_embeddings", 2048)
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_size,
|
||||
max_position=max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
)
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
scaling,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
position_ids: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.chunk(chunks=3, dim=-1)
|
||||
q, k = self.rotary_emb(position_ids, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.dense(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class PhiMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PhiConfig,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
n_inner = getattr(config, "n_inner", None)
|
||||
n_inner = n_inner if n_inner is not None else 4 * config.hidden_size
|
||||
|
||||
self.fc1 = ColumnParallelLinear(
|
||||
config.hidden_size,
|
||||
n_inner,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.fc1",
|
||||
)
|
||||
self.fc2 = RowParallelLinear(
|
||||
n_inner,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.fc2",
|
||||
)
|
||||
self.act = get_act_fn(config.hidden_act)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
hidden_states, _ = self.fc1(hidden_states)
|
||||
hidden_states = self.act(hidden_states)
|
||||
hidden_states, _ = self.fc2(hidden_states)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class PhiLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: PhiConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.input_layernorm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.layer_norm_eps
|
||||
)
|
||||
self.self_attn = PhiAttention(
|
||||
config, cache_config, quant_config, prefix=f"{prefix}.self_attn"
|
||||
)
|
||||
self.mlp = PhiMLP(config, quant_config, prefix=f"{prefix}.mlp")
|
||||
|
||||
def forward(
|
||||
self,
|
||||
position_ids: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
attn_outputs = self.self_attn(
|
||||
position_ids=position_ids,
|
||||
hidden_states=hidden_states,
|
||||
)
|
||||
feed_forward_hidden_states = self.mlp(hidden_states)
|
||||
hidden_states = attn_outputs + feed_forward_hidden_states + residual
|
||||
return hidden_states
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class PhiModel(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size, config.hidden_size
|
||||
)
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: PhiLayer(config, cache_config, quant_config, prefix=prefix),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
self.final_layernorm = nn.LayerNorm(
|
||||
config.hidden_size, eps=config.layer_norm_eps
|
||||
)
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], config.hidden_size
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
hidden_states = layer(positions, hidden_states)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
|
||||
hidden_states = self.final_layernorm(hidden_states)
|
||||
|
||||
return hidden_states
|
||||
|
||||
|
||||
class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
}
|
||||
)
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
# lm_head use bias, cannot share word embeddings
|
||||
assert not config.tie_word_embeddings
|
||||
|
||||
self.quant_config = quant_config
|
||||
|
||||
self.model = PhiModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias)
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
@@ -2260,10 +2260,6 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
)
|
||||
st_idx += text_len
|
||||
|
||||
bos_pos = np.broadcast_to(np.array([st_idx]), (3, 1))
|
||||
llm_pos_ids_list.append(bos_pos)
|
||||
st_idx += 1
|
||||
|
||||
if modality == "audio":
|
||||
audio_tokens = self._compute_audio_token_count(
|
||||
data["audio_feature_length"]
|
||||
@@ -2276,7 +2272,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
|
||||
eos_pos = np.broadcast_to(np.array([st_idx]), (3, 1))
|
||||
llm_pos_ids_list.append(eos_pos)
|
||||
st = offset + 1 + audio_tokens + 1
|
||||
st = offset + audio_tokens + 1
|
||||
|
||||
elif modality == "image":
|
||||
grid_t = data["grid_t"]
|
||||
@@ -2294,7 +2290,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
|
||||
eos_pos = np.broadcast_to(np.array([st_idx]), (3, 1))
|
||||
llm_pos_ids_list.append(eos_pos)
|
||||
st = offset + 1 + image_len + 1
|
||||
st = offset + image_len + 1
|
||||
|
||||
elif modality == "video":
|
||||
grid_t = data["grid_t"]
|
||||
@@ -2313,7 +2309,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
|
||||
eos_pos = np.broadcast_to(np.array([st_idx]), (3, 1))
|
||||
llm_pos_ids_list.append(eos_pos)
|
||||
st = offset + 1 + video_len + 1
|
||||
st = offset + video_len + 1
|
||||
else:
|
||||
audio_bos_pos = np.broadcast_to(np.array([st_idx - 1]), (3, 1))
|
||||
llm_pos_ids_list.append(audio_bos_pos)
|
||||
@@ -2330,7 +2326,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
audio_len = self._compute_audio_token_count(
|
||||
data["audio_feature_length"]
|
||||
)
|
||||
st = offset + 2 + video_len + audio_len + 2
|
||||
st = offset + 1 + video_len + audio_len + 2
|
||||
|
||||
if st < seq_len:
|
||||
st_idx = int(llm_pos_ids_list[-1].max()) + 1 if llm_pos_ids_list else 0
|
||||
|
||||
@@ -120,7 +120,6 @@ _TEXT_GENERATION_MODELS = {
|
||||
"GptOssForCausalLM": ("gpt_oss", "GptOssForCausalLM"),
|
||||
"GPT2LMHeadModel": ("gpt2", "GPT2LMHeadModel"),
|
||||
"GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
|
||||
"GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
|
||||
"GraniteForCausalLM": ("granite", "GraniteForCausalLM"),
|
||||
"GraniteMoeForCausalLM": ("granitemoe", "GraniteMoeForCausalLM"),
|
||||
"GraniteMoeHybridForCausalLM": ("granitemoehybrid", "GraniteMoeHybridForCausalLM"),
|
||||
@@ -136,7 +135,6 @@ _TEXT_GENERATION_MODELS = {
|
||||
"InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
|
||||
"IQuestCoderForCausalLM": ("llama", "LlamaForCausalLM"),
|
||||
"IQuestLoopCoderForCausalLM": ("iquest_loopcoder", "IQuestLoopCoderForCausalLM"),
|
||||
"Jais2ForCausalLM": ("jais2", "Jais2ForCausalLM"),
|
||||
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
|
||||
"K2HorizonForCausalLM": ("k2_horizon", "K2HorizonForCausalLM"),
|
||||
"KimiLinearForCausalLM": (
|
||||
@@ -188,7 +186,6 @@ _TEXT_GENERATION_MODELS = {
|
||||
"PanguProMoEV2ForCausalLM": ("openpangu", "PanguProMoEV2ForCausalLM"),
|
||||
"PanguUltraMoEForCausalLM": ("openpangu", "PanguUltraMoEForCausalLM"),
|
||||
"Param2MoEForCausalLM": ("param2moe", "Param2MoEForCausalLM"),
|
||||
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
|
||||
"Phi3ForCausalLM": ("phi3", "Phi3ForCausalLM"),
|
||||
"PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
|
||||
"Plamo3ForCausalLM": ("plamo3", "Plamo3ForCausalLM"),
|
||||
@@ -200,7 +197,6 @@ _TEXT_GENERATION_MODELS = {
|
||||
"Qwen3_5MoeForCausalLM": ("qwen3_5", "Qwen3_5MoeForCausalLM"),
|
||||
"SarvamMoEForCausalLM": ("sarvam", "SarvamMoEForCausalLM"),
|
||||
"SarvamMLAForCausalLM": ("sarvam", "SarvamMLAForCausalLM"),
|
||||
"SeedOssForCausalLM": ("seed_oss", "SeedOssForCausalLM"),
|
||||
"Step1ForCausalLM": ("step1", "Step1ForCausalLM"),
|
||||
"Step3TextForCausalLM": ("step3_text", "Step3TextForCausalLM"),
|
||||
"Step3p5ForCausalLM": ("step3p5", "Step3p5ForCausalLM"),
|
||||
@@ -709,13 +705,17 @@ _TRANSFORMERS_SUPPORTED_MODELS = {
|
||||
"FlexOlmoForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"GlmForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"GPTNeoXForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Olmo3ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"PhiForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"SeedOssForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Starcoder2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"VaultGemmaForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
|
||||
@@ -1,432 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2025 The Seed team.
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
|
||||
#
|
||||
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
||||
# and OPT implementations in this library. It has been modified from its
|
||||
# original forms to accommodate minor architectural differences compared
|
||||
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Inference-only SeedOss model compatible with HuggingFace weights."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from itertools import islice
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import PreTrainedConfig as SeedOssConfig
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.config import set_default_rope_theta
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
from .interfaces import SupportsLoRA, SupportsPP
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SeedOssMLP(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
hidden_act: str,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
class SeedOssAttention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
rope_parameters: dict,
|
||||
max_position: int = 4096 * 32,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
attn_type: str = AttentionType.DECODER,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
self.total_num_heads = num_heads
|
||||
assert self.total_num_heads % tp_size == 0
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
self.total_num_kv_heads = num_kv_heads
|
||||
self.head_dim = head_dim
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
# Number of KV heads is greater than TP size, so we partition
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert self.total_num_kv_heads % tp_size == 0
|
||||
else:
|
||||
# Number of KV heads is less than TP size, so we replicate
|
||||
# the KV heads across multiple tensor parallel GPUs.
|
||||
assert tp_size % self.total_num_kv_heads == 0
|
||||
self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
|
||||
self.q_size = self.num_heads * self.head_dim
|
||||
self.kv_size = self.num_kv_heads * self.head_dim
|
||||
self.scaling = self.head_dim**-0.5
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size,
|
||||
self.head_dim,
|
||||
self.total_num_heads,
|
||||
self.total_num_kv_heads,
|
||||
bias=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj",
|
||||
)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * self.head_dim,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position,
|
||||
rope_parameters=rope_parameters,
|
||||
)
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
self.scaling,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
attn_type=attn_type,
|
||||
prefix=f"{prefix}.attn",
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
attn_output = self.attn(q, k, v)
|
||||
output, _ = self.o_proj(attn_output)
|
||||
return output
|
||||
|
||||
|
||||
class SeedOssDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: SeedOssConfig,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
set_default_rope_theta(config, default_theta=1000000)
|
||||
|
||||
# By default, SeedOss uses causal attention as it is a
|
||||
# decoder-only model.
|
||||
# You can override the HF config with `is_causal=False` to enable
|
||||
# bidirectional attention, which is used in some embedding models
|
||||
if getattr(config, "is_causal", True):
|
||||
attn_type = AttentionType.DECODER
|
||||
else:
|
||||
attn_type = AttentionType.ENCODER_ONLY
|
||||
|
||||
self.self_attn = SeedOssAttention(
|
||||
hidden_size=self.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
max_position=config.max_position_embeddings,
|
||||
num_kv_heads=config.num_key_value_heads,
|
||||
head_dim=config.head_dim,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
rope_parameters=config.rope_parameters,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
attn_type=attn_type,
|
||||
)
|
||||
self.mlp = SeedOssMLP(
|
||||
hidden_size=self.hidden_size,
|
||||
intermediate_size=config.intermediate_size,
|
||||
hidden_act=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mlp",
|
||||
)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
residual: torch.Tensor | None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Self Attention
|
||||
if residual is None:
|
||||
residual = hidden_states
|
||||
hidden_states = self.input_layernorm(hidden_states)
|
||||
else:
|
||||
hidden_states, residual = self.input_layernorm(hidden_states, residual)
|
||||
hidden_states = self.self_attn(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
)
|
||||
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.post_attention_layernorm(hidden_states, residual)
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@support_torch_compile(
|
||||
dynamic_arg_dims={
|
||||
"input_ids": 0,
|
||||
"positions": -1,
|
||||
"intermediate_tensors": 0,
|
||||
"inputs_embeds": 0,
|
||||
}
|
||||
)
|
||||
class SeedOssModel(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
decoder_layer_type: type[nn.Module] = SeedOssDecoderLayer,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
# TODO (@robertgshaw2): see if this can be moved out
|
||||
if cache_config.sliding_window is not None and hasattr(
|
||||
config, "max_window_layers"
|
||||
):
|
||||
assert config.max_window_layers == config.num_hidden_layers, (
|
||||
"Sliding window for some but all layers is not supported. "
|
||||
"This model uses sliding window but `max_window_layers` = {} "
|
||||
"is less than `num_hidden_layers` = {}. Please open an issue "
|
||||
"to discuss this feature.".format(
|
||||
config.max_window_layers,
|
||||
config.num_hidden_layers,
|
||||
)
|
||||
)
|
||||
|
||||
self.config = config
|
||||
self.quant_config = quant_config
|
||||
self.vocab_size = config.vocab_size
|
||||
|
||||
if get_pp_group().is_first_rank or (
|
||||
config.tie_word_embeddings and get_pp_group().is_last_rank
|
||||
):
|
||||
self.embed_tokens = VocabParallelEmbedding(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.embed_tokens",
|
||||
)
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
# Use the provided decoder layer type or default to SeedDecoderLayer
|
||||
decoder_layer_type = decoder_layer_type or SeedOssDecoderLayer
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
config.num_hidden_layers,
|
||||
lambda prefix: decoder_layer_type(
|
||||
config=config,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
),
|
||||
prefix=f"{prefix}.layers",
|
||||
)
|
||||
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
if get_pp_group().is_last_rank:
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
residual = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
hidden_states, residual = layer(
|
||||
positions,
|
||||
hidden_states,
|
||||
residual,
|
||||
)
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors(
|
||||
{"hidden_states": hidden_states, "residual": residual}
|
||||
)
|
||||
hidden_states, _ = self.norm(hidden_states, residual)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class SeedOssForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_stacked={
|
||||
# weight_name: (param_name, shard_id)
|
||||
".q_proj": (".qkv_proj", "q"),
|
||||
".k_proj": (".qkv_proj", "k"),
|
||||
".v_proj": (".qkv_proj", "v"),
|
||||
".gate_proj": (".gate_up_proj", 0),
|
||||
".up_proj": (".gate_up_proj", 1),
|
||||
}
|
||||
)
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
self.config = config
|
||||
|
||||
self.quant_config = quant_config
|
||||
self.model = SeedOssModel(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
|
||||
)
|
||||
|
||||
if get_pp_group().is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
config.vocab_size,
|
||||
config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
if config.tie_word_embeddings:
|
||||
self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size)
|
||||
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
hidden_states = self.model(
|
||||
input_ids, positions, intermediate_tensors, inputs_embeds
|
||||
)
|
||||
return hidden_states
|
||||
|
||||
def compute_logits(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor | None:
|
||||
logits = self.logits_processor(self.lm_head, hidden_states)
|
||||
return logits
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
@@ -32,6 +32,7 @@ from torch import nn
|
||||
from transformers import AutoModel, PreTrainedConfig
|
||||
from transformers.conversion_mapping import (
|
||||
WeightRenaming,
|
||||
get_checkpoint_conversion_mapping,
|
||||
get_model_conversion_mapping,
|
||||
)
|
||||
|
||||
@@ -184,7 +185,7 @@ class Base(
|
||||
# Substitute remaining layers with vLLM's layers as needed
|
||||
self.recursive_replace()
|
||||
# Create attention instances for KV cache allocation
|
||||
self._create_attention_instances()
|
||||
self._create_attention_instances(prefix)
|
||||
|
||||
# Initialize any parameters that have not had their modules replaced
|
||||
self.init_parameters(self.model)
|
||||
@@ -311,7 +312,14 @@ class Base(
|
||||
orig_to_new_renaming: list[WeightRenaming] = []
|
||||
orig_to_new_regex: dict[re.Pattern, str | None] = {}
|
||||
|
||||
for mapping in get_model_conversion_mapping(self.model):
|
||||
mappings = get_model_conversion_mapping(self.model)
|
||||
# Ensure layers that are part of the model, but not AutoModel, get their mapping
|
||||
instantiated = {type(m).__name__ for m in self.model.modules()}
|
||||
for arch in self.config.architectures or []:
|
||||
if arch not in instantiated:
|
||||
mappings.extend(get_checkpoint_conversion_mapping(arch) or [])
|
||||
|
||||
for mapping in mappings:
|
||||
# Handle weights which have been renamed in Transformers
|
||||
if isinstance(mapping, WeightRenaming):
|
||||
orig_to_new_renaming.append(mapping)
|
||||
@@ -579,7 +587,7 @@ class Base(
|
||||
|
||||
self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked)
|
||||
|
||||
def _create_attention_instances(self):
|
||||
def _create_attention_instances(self, prefix: str = ""):
|
||||
"""Create `Attention` instances to inform KV cache allocation."""
|
||||
text_config = self.text_config
|
||||
attn_cls = self._get_attn_cls()
|
||||
@@ -613,8 +621,8 @@ class Base(
|
||||
f"Layer {layer} does not dispatch through the Transformers "
|
||||
"attention interface and vLLM has no other way to handle it."
|
||||
)
|
||||
prefix, attn_fuser = self.attention_fusers[i]
|
||||
attn_module = self.get_submodule(prefix)
|
||||
attn_prefix, attn_fuser = self.attention_fusers[i]
|
||||
attn_module = self.get_submodule(attn_prefix)
|
||||
|
||||
# `[i]` is the whole-model config unless the checkpoint is
|
||||
# heterogeneous, in which case it is this layer's own geometry.
|
||||
@@ -632,17 +640,18 @@ class Base(
|
||||
scale=scale,
|
||||
cache_config=self.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
prefix=f"{i}.attn",
|
||||
prefix=maybe_prefix(prefix, f"{attn_prefix}.{VLLM_ATTN_ATTR}"),
|
||||
)
|
||||
|
||||
if attn_cls is MLAAttention:
|
||||
mla_fuser = next(
|
||||
(f for f in self.fusers[prefix] if isinstance(f, MLAFuser)), None
|
||||
(f for f in self.fusers[attn_prefix] if isinstance(f, MLAFuser)),
|
||||
None,
|
||||
)
|
||||
if mla_fuser is None:
|
||||
raise ValueError(
|
||||
f"Layer {i} ({prefix}) did not fuse as MLA, but the rest of "
|
||||
"the model did, so its latent KV cache cannot be allocated."
|
||||
f"Layer {i} ({attn_prefix}) did not fuse as MLA, but the rest "
|
||||
"of the model did, so its latent KV cache cannot be allocated."
|
||||
)
|
||||
dims = get_mla_dims(self.model_config)
|
||||
kwargs.update(
|
||||
@@ -690,10 +699,8 @@ class Base(
|
||||
# vLLM does not support encoder-decoder models, so if any encoder layer is
|
||||
# found in a text only model, we assume the whole model is an encoder model
|
||||
if has_encoder(self.model) and not is_multimodal(self.config):
|
||||
self.check_version("5.0.0", "encoder models support")
|
||||
return EncoderOnlyAttention
|
||||
if self.model_config.use_mla:
|
||||
self.check_version("5.15.0.dev0", "optimized MLA support")
|
||||
if any(
|
||||
isinstance(fuser, MLAFuser)
|
||||
for fusers in self.fusers.values()
|
||||
@@ -819,7 +826,6 @@ class Base(
|
||||
)
|
||||
|
||||
def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
|
||||
self.check_version("5.2.0", "Eagle3 support")
|
||||
from transformers.utils.output_capturing import (
|
||||
OutputRecorder,
|
||||
maybe_install_capturing_hooks,
|
||||
|
||||
@@ -19,6 +19,8 @@
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -30,8 +32,6 @@ from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
|
||||
from .base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
@@ -68,11 +68,15 @@ class CausalMixin(VllmModelForTextGeneration, Base):
|
||||
def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]:
|
||||
"""A thin wrapper around `Base.load_weights` to handle the lm_head bias."""
|
||||
lm_head_bias = set()
|
||||
load_device = self.vllm_config.load_config.device or self.device_config.device
|
||||
|
||||
def auto_load_lm_head_bias(weights):
|
||||
for name, weight in weights:
|
||||
if name.endswith("lm_head.bias") and self.pp_group.is_last_rank:
|
||||
self.lm_head._register_bias()
|
||||
# load_weights is called outside vLLM's device context,
|
||||
# so we must enter it again when registering the bias.
|
||||
with torch.device(load_device):
|
||||
self.lm_head._register_bias()
|
||||
self.lm_head.bias.weight_loader(self.lm_head.bias, weight)
|
||||
lm_head_bias.add(name)
|
||||
else:
|
||||
|
||||
@@ -156,7 +156,6 @@ class MoEMixin(MixtureOfExperts, Base):
|
||||
num_local_physical_experts: int
|
||||
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
self.check_version("5.0.0", "MoE models support")
|
||||
# Skip MixtureOfExperts.__init__ and call the next class in MRO
|
||||
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
|
||||
@@ -23,17 +23,11 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
from transformers.utils.generic import ModelOutput
|
||||
|
||||
from vllm.compilation.decorators import should_torch_compile_mm_encoder
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.inputs import (
|
||||
MultiModalDataBuiltins,
|
||||
MultiModalDataDict,
|
||||
MultiModalInput,
|
||||
mm_input,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataBuiltins, MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
MultiModalEmbeddings,
|
||||
@@ -42,26 +36,15 @@ from vllm.model_executor.models.interfaces import (
|
||||
)
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
|
||||
from vllm.multimodal.inputs import (
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldConfig,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.multimodal.parse import (
|
||||
ImageProcessorItems,
|
||||
ImageSize,
|
||||
MultiModalDataItems,
|
||||
MultiModalDataParser,
|
||||
)
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalFieldConfig
|
||||
from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseMultiModalProcessor,
|
||||
BaseProcessingInfo,
|
||||
ProcessorInputs,
|
||||
PromptReplacement,
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
TimingContext,
|
||||
cached_encode,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -219,11 +202,13 @@ class MultiModalDummyInputsBuilder(BaseDummyInputsBuilder[MultiModalProcessingIn
|
||||
return data
|
||||
|
||||
|
||||
class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
"""Processing common to both Transformers backend processors: calling the HF
|
||||
processor, sizing images, and attributing its outputs to a modality.
|
||||
class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
"""Locates placeholders from the `text_replacement_offsets` the HF processor
|
||||
reports, expressing each one as a `PromptUpdate`.
|
||||
|
||||
Subclasses add the strategy for locating placeholders in the prompt.
|
||||
Stating the expansion as an update is what lets it be rebuilt from an
|
||||
unexpanded prompt, so this processor takes the base class's processing path
|
||||
and with it the multi-modal processor cache.
|
||||
"""
|
||||
|
||||
def _get_modality_field_names(self, modality: str) -> set[str]:
|
||||
@@ -415,301 +400,6 @@ class _MultiModalProcessorBase(BaseMultiModalProcessor[MultiModalProcessingInfo]
|
||||
for image, (height, width) in zip(pixel_values, image_sizes.tolist())
|
||||
]
|
||||
|
||||
|
||||
class LegacyMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"""Locates placeholders by searching the prompt the HF processor has already
|
||||
expanded for the tokens of each modality.
|
||||
|
||||
Serves transformers versions with no `return_text_replacement_offsets`.
|
||||
Placeholders found this way cannot be rebuilt from an unexpanded prompt, so
|
||||
this processor overrides `apply` and gets no multi-modal processor cache.
|
||||
Remove it once `requirements/common.txt` requires `transformers>=5.15.0`.
|
||||
"""
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
out_mm_kwargs: MultiModalKwargsItems,
|
||||
) -> Sequence[PromptUpdate]:
|
||||
"""Empty, because `apply` writes the placeholder ranges itself rather than
|
||||
deriving them from updates."""
|
||||
return []
|
||||
|
||||
def _get_mm_token_ids(self, modality: str) -> list[int]:
|
||||
"""Token ids marking where `modality` sits in the prompt, which for some
|
||||
processors differ from the placeholder written into it.
|
||||
|
||||
The expanded prompt is all this path has to go on, so it takes the
|
||||
processor at its word about which tokens belong to the modality.
|
||||
"""
|
||||
info = self.info
|
||||
processor = info.get_hf_processor()
|
||||
declared = getattr(processor, f"{modality}_token_ids", None) or ()
|
||||
if ids := [token_id for token_id in declared if token_id is not None]:
|
||||
return ids
|
||||
config = info.get_hf_config()
|
||||
names = (f"{modality}_token_id", f"{modality}_token_index")
|
||||
token_id = getattr(processor, names[0], getattr_iter(config, names))
|
||||
if token_id is None:
|
||||
token = getattr(processor, f"{modality}_token", None)
|
||||
if token is not None:
|
||||
token_id = info.get_tokenizer().get_vocab().get(token)
|
||||
if token_id is None:
|
||||
raise ValueError(
|
||||
f"Cannot find {modality}_token_id on processor or model config"
|
||||
)
|
||||
return [token_id]
|
||||
|
||||
def _apply_audio(
|
||||
self,
|
||||
prompt_ids: list[int],
|
||||
processed_data: "BatchFeature",
|
||||
num_audios: int,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
"""Take each contiguous run of the audio token as one item's placeholder,
|
||||
and record how many tokens the run holds."""
|
||||
audio_token_ids = self._get_mm_token_ids("audio")
|
||||
prompt_tensor = torch.tensor(prompt_ids)
|
||||
is_audio = torch.isin(prompt_tensor, torch.tensor(audio_token_ids))
|
||||
|
||||
if not is_audio.any():
|
||||
raise ValueError(
|
||||
f"{num_audios} audio item(s) were passed but the prompt "
|
||||
"contains no audio token. Add one placeholder per audio item."
|
||||
)
|
||||
|
||||
padded = torch.cat([torch.tensor([False]), is_audio, torch.tensor([False])])
|
||||
transitions = padded.int().diff()
|
||||
offsets = torch.where(transitions == 1)[0]
|
||||
lengths = torch.where(transitions == -1)[0] - offsets
|
||||
|
||||
if len(offsets) != num_audios:
|
||||
raise ValueError(
|
||||
f"Found {len(offsets)} run(s) of the audio token in the prompt but "
|
||||
f"{num_audios} audio item(s) were passed. The Transformers backend "
|
||||
"locates audio placeholders by finding contiguous runs of the audio "
|
||||
"token, so placeholders with no text between them cannot yet be told "
|
||||
"apart. Separate them in the prompt to work around this."
|
||||
)
|
||||
|
||||
ranges = [
|
||||
PlaceholderRange(offset=offset.item(), length=length.item())
|
||||
for offset, length in zip(offsets, lengths)
|
||||
]
|
||||
processed_data["num_audio_tokens"] = lengths
|
||||
return {"audio": ranges}
|
||||
|
||||
def _get_num_multimodal_tokens(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, list[int]]:
|
||||
"""Ask the HF processor how many tokens and patches each image expands to."""
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
images = mm_items.get_items("image", ImageProcessorItems)
|
||||
image_sizes = [
|
||||
(size.height, size.width)
|
||||
for size in map(images.get_image_size, range(len(images)))
|
||||
]
|
||||
return processor._get_num_multimodal_tokens(
|
||||
image_sizes=image_sizes,
|
||||
**self.info.ctx.get_merged_mm_kwargs({}),
|
||||
)
|
||||
|
||||
def _apply_vision(
|
||||
self,
|
||||
prompt_ids: list[int],
|
||||
processed_data: "BatchFeature",
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
mm_token_type_ids: torch.Tensor | None,
|
||||
) -> dict[str, list[PlaceholderRange]]:
|
||||
"""Split the positions the processor marks as image into one placeholder per
|
||||
item, sized by the token count it reports for each image."""
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
if mm_token_type_ids is None:
|
||||
raise ValueError(
|
||||
f"{type(hf_processor).__name__} returned no `mm_token_type_ids`, so "
|
||||
"the Transformers modeling backend cannot locate the placeholder of "
|
||||
"each image."
|
||||
)
|
||||
|
||||
# These mark the tokens structuring an image as well as the image itself,
|
||||
# which is what the counts below are over
|
||||
mm_positions = torch.where(mm_token_type_ids[0] == 1)[0]
|
||||
mm_tokens_per_modality = self._get_num_multimodal_tokens(
|
||||
mm_items, hf_processor_mm_kwargs
|
||||
)
|
||||
|
||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
||||
split_sizes = mm_tokens_per_modality["num_image_tokens"]
|
||||
if sum(split_sizes) != len(mm_positions):
|
||||
raise ValueError(
|
||||
f"The expanded prompt holds {len(mm_positions)} image token(s) but "
|
||||
f"{type(hf_processor).__name__} accounts for {sum(split_sizes)} "
|
||||
f"across {mm_items.get_count('image')} image item(s)."
|
||||
)
|
||||
|
||||
if split_sizes:
|
||||
image_token_ids = torch.tensor(self._get_mm_token_ids("image"))
|
||||
mm_tokens = torch.tensor(prompt_ids)[mm_positions]
|
||||
ranges = [
|
||||
PlaceholderRange(
|
||||
offset=positions[0].item(),
|
||||
length=positions.shape[0],
|
||||
# Only some of the span carries embeddings
|
||||
is_embed=torch.isin(tokens, image_token_ids),
|
||||
)
|
||||
for positions, tokens in zip(
|
||||
torch.split(mm_positions, split_sizes),
|
||||
torch.split(mm_tokens, split_sizes),
|
||||
)
|
||||
]
|
||||
mm_placeholders = {"image": ranges}
|
||||
|
||||
processed_data["num_image_patches"] = torch.tensor(
|
||||
mm_tokens_per_modality["num_image_patches"]
|
||||
)
|
||||
return mm_placeholders
|
||||
|
||||
def apply(
|
||||
self,
|
||||
inputs: ProcessorInputs,
|
||||
timing_ctx: TimingContext,
|
||||
) -> MultiModalInput:
|
||||
"""Process the prompt and every multi-modal item in one HF processor call,
|
||||
then read the placeholder ranges out of the token ids it returns."""
|
||||
prompt = inputs.prompt
|
||||
mm_items = inputs.mm_data_items
|
||||
hf_processor_mm_kwargs = inputs.hf_processor_mm_kwargs
|
||||
|
||||
with timing_ctx.record("apply_hf_processor"):
|
||||
hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
prompt_text = hf_processor.decode(prompt)
|
||||
|
||||
# Bypass cached processor and always apply to the full set of mm inputs
|
||||
# NOTE: we can't just set caching=False because base class method
|
||||
# transforms outputs to `MultiModalKwargs` which is not going to
|
||||
# work for Transformers. The vision path has logic tied to
|
||||
# `mm_tokens_per_modality` in _apply_vision()
|
||||
hf_processor_mm_kwargs = {
|
||||
# vLLM needs the untruncated sequence to keep placeholder
|
||||
# tokens aligned. Note that the text inputs are just dummy
|
||||
# text, not the original prompt. The original prompt is
|
||||
# already tokenized by the renderer.
|
||||
"truncation": False,
|
||||
# HF processors only accept text, and the decoded string already
|
||||
# contains any special tokens, so don't let them be added again
|
||||
# (the HF processor call disables `add_special_tokens`).
|
||||
"add_special_tokens": False,
|
||||
**hf_processor_mm_kwargs,
|
||||
}
|
||||
|
||||
processor_data, _, passthrough_data = self._get_hf_mm_inputs(
|
||||
mm_items, hf_processor_mm_kwargs
|
||||
)
|
||||
|
||||
# The real prompt is fed to the processor below and the placeholder
|
||||
# ranges are read back out of its output, so the dummy text meant for
|
||||
# media-only callers is discarded here.
|
||||
processor_data.pop("text", None)
|
||||
has_mm_data = any(processor_data.values())
|
||||
|
||||
# Ask which modality owns each token of the expanded prompt, which
|
||||
# is the only marking of the tokens an expansion adds around an item
|
||||
if has_mm_data:
|
||||
processor_data = {**processor_data, "return_mm_token_type_ids": True}
|
||||
|
||||
try:
|
||||
processed_data = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**hf_processor_mm_kwargs),
|
||||
dict(text=prompt_text, **processor_data),
|
||||
hf_processor_mm_kwargs,
|
||||
)
|
||||
except ValueError:
|
||||
if has_mm_data:
|
||||
raise
|
||||
# Some processors reject a prompt holding placeholders with
|
||||
# no data to go with them, so tokenize it without them
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt_text)
|
||||
processed_data = transformers.BatchFeature(
|
||||
dict(input_ids=[prompt_ids]), tensor_type="pt"
|
||||
)
|
||||
self._unpad_images(processed_data)
|
||||
self._unpad_audios(processed_data, processor_data, hf_processor_mm_kwargs)
|
||||
processed_data.update(passthrough_data)
|
||||
|
||||
input_ids = processed_data.pop("input_ids")
|
||||
if not isinstance(input_ids, list):
|
||||
input_ids = input_ids.tolist()
|
||||
|
||||
(prompt_ids,) = input_ids
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
|
||||
# Gemma3 reports them under the key the model uses for its own token types
|
||||
mm_token_type_ids = processed_data.pop("token_type_ids", None)
|
||||
mm_token_type_ids = processed_data.pop("mm_token_type_ids", mm_token_type_ids)
|
||||
|
||||
mm_placeholders: dict[str, list[PlaceholderRange]] = {}
|
||||
if num_audios := mm_items.get_count("audio", strict=False):
|
||||
mm_placeholders.update(
|
||||
self._apply_audio(prompt_ids, processed_data, num_audios)
|
||||
)
|
||||
if mm_items.get_count("image", strict=False):
|
||||
mm_placeholders.update(
|
||||
self._apply_vision(
|
||||
prompt_ids,
|
||||
processed_data,
|
||||
mm_items,
|
||||
hf_processor_mm_kwargs,
|
||||
mm_token_type_ids,
|
||||
)
|
||||
)
|
||||
|
||||
mm_kwargs = MultiModalKwargsItems.from_hf_inputs(
|
||||
processed_data,
|
||||
self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
|
||||
)
|
||||
|
||||
# Bypassing `_maybe_apply_prompt_updates` also bypasses its validation.
|
||||
# `_validate_mm_placeholders` can't be reused because it is typed for the
|
||||
# `PlaceholderFeaturesInfo` the prompt update machinery produces.
|
||||
mm_item_counts = mm_items.get_all_counts()
|
||||
self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
|
||||
for modality, item_count in mm_item_counts.items():
|
||||
num_placeholders = len(mm_placeholders.get(modality, []))
|
||||
if num_placeholders != item_count:
|
||||
raise RuntimeError(
|
||||
f"Expected there to be {item_count} prompt placeholders "
|
||||
f"corresponding to {item_count} {modality} items, but instead "
|
||||
f"found {num_placeholders} prompt placeholders! Make sure the "
|
||||
"prompt contains a placeholder token for each item."
|
||||
)
|
||||
|
||||
return mm_input(
|
||||
prompt_token_ids=prompt_ids,
|
||||
mm_kwargs=mm_kwargs,
|
||||
mm_hashes=mm_hashes,
|
||||
mm_placeholders=mm_placeholders,
|
||||
)
|
||||
|
||||
|
||||
class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"""Locates placeholders from the `text_replacement_offsets` the HF processor
|
||||
reports, expressing each one as a `PromptUpdate`.
|
||||
|
||||
Stating the expansion as an update is what lets it be rebuilt from an
|
||||
unexpanded prompt, so this processor takes the base class's processing path
|
||||
and with it the multi-modal processor cache.
|
||||
"""
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
@@ -800,9 +490,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
except (AttributeError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
def _apply_hf_processor_main(
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
@@ -856,8 +543,7 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
"to reach `ProcessorMixin.get_text_with_replacements` with one "
|
||||
"replacement per item, which usually means implementing "
|
||||
"`replace_<modality>_token`. Please report this to transformers "
|
||||
"so it can be fixed, and install `transformers<5.15.0` in the "
|
||||
"meantime to locate placeholders in the expanded prompt instead."
|
||||
"so it can be fixed."
|
||||
)
|
||||
hf_inputs.pop("input_ids", None)
|
||||
return self._finalize_hf_mm_data(
|
||||
@@ -896,15 +582,6 @@ class OffsetsMultiModalProcessor(_MultiModalProcessorBase):
|
||||
)
|
||||
|
||||
|
||||
# From this version on, a processor reporting no offsets is an error rather than a
|
||||
# fallback to searching the expanded prompt
|
||||
MultiModalProcessor = (
|
||||
LegacyMultiModalProcessor
|
||||
if Version(transformers.__version__) < Version("5.15.0")
|
||||
else OffsetsMultiModalProcessor
|
||||
)
|
||||
|
||||
|
||||
class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
# Skip SupportsMRoPE.__init__ and call the next class in MRO
|
||||
@@ -982,7 +659,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
|
||||
super()._decorate_for_torch_compile()
|
||||
# Decorate the encoder model classes to support torch compile if needed
|
||||
if self.compilation_config.compile_mm_encoder:
|
||||
self.check_version("5.0.0", "multimodal encoder compilation support")
|
||||
encoder_classes = self._pre_trained_model_classes.encoders
|
||||
if not encoder_classes:
|
||||
raise ValueError(
|
||||
@@ -1113,7 +789,6 @@ class MultiModalMixin(SupportsMultiModal, SupportsMRoPE, Base):
|
||||
if input_features is None:
|
||||
return None
|
||||
|
||||
self.check_version("5.13.0", "audio models support")
|
||||
num_audio_tokens = kwargs.pop("num_audio_tokens")
|
||||
kwargs.pop("token_type_ids", None)
|
||||
kwargs.pop("mm_token_type_ids", None)
|
||||
|
||||
Reference in New Issue
Block a user