[Refactor] Move FusedMoE hidden_size roundup to quant_method (#34285)
Signed-off-by: Bowen Bao <bowenbao@amd.com>
This commit is contained in:
@@ -715,8 +715,6 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
layer.intermediate_size_per_partition = intermediate_size_per_partition
|
||||
layer.hidden_size = hidden_size
|
||||
layer.num_experts = num_experts
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = None
|
||||
@@ -2274,8 +2272,6 @@ class CompressedTensorsW4A8Fp8MoEMethod(CompressedTensorsMoEMethod):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
layer.intermediate_size_per_partition = intermediate_size_per_partition
|
||||
layer.hidden_size = hidden_size
|
||||
layer.num_experts = num_experts
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = None
|
||||
|
||||
@@ -672,8 +672,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
layer.intermediate_size_per_partition = intermediate_size_per_partition
|
||||
layer.hidden_size = hidden_size
|
||||
layer.num_experts = num_experts
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = None
|
||||
@@ -1011,8 +1009,6 @@ class Fp8OnlineMoEMethod(Fp8MoEMethod):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
layer.intermediate_size_per_partition = intermediate_size_per_partition
|
||||
layer.hidden_size = hidden_size
|
||||
layer.num_experts = num_experts
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = None
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
@@ -107,18 +108,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
self._cache_permute_indices: dict[torch.Size, torch.Tensor] = {}
|
||||
self.moe_kernel: mk.FusedMoEKernel | None = None
|
||||
|
||||
# Round up dims once based on backend. This mutates the shared
|
||||
# FusedMoEConfig in-place so that create_weights() and all
|
||||
# downstream code see the padded dimensions. This must happen
|
||||
# before create_weights() is called.
|
||||
self.moe.hidden_dim, self.moe.intermediate_size_per_partition = (
|
||||
mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
self.mxfp4_backend,
|
||||
self.moe.hidden_dim,
|
||||
self.moe.intermediate_size_per_partition,
|
||||
)
|
||||
)
|
||||
|
||||
# Used for triton kernel precision configs
|
||||
self.w13_precision_config = None
|
||||
self.w2_precision_config = None
|
||||
@@ -129,6 +118,23 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
# so can skip the padding in the forward before applying the moe method
|
||||
return self.mxfp4_backend == Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8
|
||||
|
||||
def maybe_roundup_sizes(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
act_dtype: torch.dtype,
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> tuple[int, int]:
|
||||
hidden_size, intermediate_size_per_partition = super().maybe_roundup_sizes(
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size_per_partition=intermediate_size_per_partition,
|
||||
act_dtype=act_dtype,
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
)
|
||||
return mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
self.mxfp4_backend, hidden_size, intermediate_size_per_partition
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -143,32 +149,16 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
scale_dtype = torch.uint8
|
||||
mxfp4_block = 32
|
||||
|
||||
# Use pre-rounded sizes from config
|
||||
self.intermediate_size = intermediate_size_per_partition_after_pad = (
|
||||
self.moe.intermediate_size_per_partition
|
||||
)
|
||||
self.hidden_size = hidden_size = self.moe.hidden_dim
|
||||
|
||||
# Expose padded dimensions on the layer for LoRA and Marlin code
|
||||
# that reads layer.hidden_size / layer.intermediate_size_per_partition.
|
||||
layer.params_dtype = params_dtype
|
||||
layer.num_experts = num_experts
|
||||
layer.hidden_size = hidden_size
|
||||
layer.intermediate_size_per_partition = (
|
||||
intermediate_size_per_partition_after_pad
|
||||
)
|
||||
|
||||
# CK (gfx950) padding info for rocm_aiter_ops.fused_moe()
|
||||
self.hidden_pad = extra_weight_attrs.get("hidden_pad", 0)
|
||||
self.intermediate_pad = (
|
||||
intermediate_size_per_partition_after_pad - intermediate_size_per_partition
|
||||
)
|
||||
self.intermediate_size = intermediate_size_per_partition
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
# Fused gate_up_proj (column parallel)
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition_after_pad,
|
||||
2 * intermediate_size_per_partition,
|
||||
hidden_size // 2,
|
||||
dtype=weight_dtype,
|
||||
),
|
||||
@@ -180,7 +170,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition_after_pad,
|
||||
2 * intermediate_size_per_partition,
|
||||
hidden_size // mxfp4_block,
|
||||
dtype=scale_dtype,
|
||||
),
|
||||
@@ -194,7 +184,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition_after_pad // 2,
|
||||
intermediate_size_per_partition // 2,
|
||||
dtype=weight_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -206,7 +196,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition_after_pad // mxfp4_block,
|
||||
intermediate_size_per_partition // mxfp4_block,
|
||||
dtype=scale_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -218,7 +208,7 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition_after_pad,
|
||||
2 * intermediate_size_per_partition,
|
||||
dtype=torch.bfloat16,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -368,8 +358,6 @@ class Mxfp4MoEMethod(FusedMoEMethodBase):
|
||||
w2_scale=w2_scale,
|
||||
w1_bias=w1_bias,
|
||||
w2_bias=w2_bias,
|
||||
hidden_pad=self.hidden_pad,
|
||||
intermediate_pad=self.intermediate_pad,
|
||||
)
|
||||
|
||||
def select_gemm_impl(
|
||||
|
||||
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
MoEActivation,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
fp8_w8a8_moe_quant_config,
|
||||
mxfp4_w4a8_moe_quant_config,
|
||||
@@ -27,13 +28,13 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.fused_marlin_moe import fused_marlin_moe
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
mxfp4_round_up_hidden_size_and_intermediate_size,
|
||||
select_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
|
||||
prepare_fp8_moe_layer_for_marlin,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
CK_MXFP4_MOE_DIM_ALIGNMENT,
|
||||
_swizzle_mxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import (
|
||||
@@ -49,7 +50,6 @@ from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -173,8 +173,6 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
layer.intermediate_size_per_partition = intermediate_size_per_partition
|
||||
layer.hidden_size = hidden_size
|
||||
layer.num_experts = num_experts
|
||||
layer.orig_dtype = params_dtype
|
||||
layer.weight_block_size = None
|
||||
@@ -182,7 +180,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
|
||||
# WEIGHTS
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
hidden_size,
|
||||
@@ -194,7 +192,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition,
|
||||
@@ -461,6 +459,7 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
activation=layer.activation,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
quant_config=self.moe_quant_config,
|
||||
moe_config=layer.moe_config,
|
||||
expert_map=layer.expert_map,
|
||||
)
|
||||
elif self.use_marlin:
|
||||
@@ -527,7 +526,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
):
|
||||
params_dtype = torch.uint32
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition,
|
||||
hidden_size // 8, # INT32 packing for W4
|
||||
@@ -536,7 +535,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
requires_grad=False,
|
||||
)
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition // 8, # INT32 packing for W4
|
||||
@@ -649,6 +648,7 @@ class QuarkW4A8Fp8MoEMethod(QuarkMoEMethod):
|
||||
activation=layer.activation,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
quant_config=self.moe_quant_config,
|
||||
moe_config=layer.moe_config,
|
||||
expert_map=layer.expert_map,
|
||||
)
|
||||
|
||||
@@ -702,6 +702,9 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
self.mxfp4_backend: Mxfp4MoeBackend | None = None
|
||||
if self.ocp_mx_scheme == "w_mxfp4":
|
||||
self.mxfp4_backend, _ = select_mxfp4_moe_backend(moe)
|
||||
elif self.ocp_mx_scheme.startswith("w_mxfp4"):
|
||||
# TODO(bowenbao): refactor and introduce backends for other OCP MX schemes.
|
||||
self.mxfp4_backend = Mxfp4MoeBackend.NONE
|
||||
|
||||
if self.input_quant is not None:
|
||||
self.static_input_scales = not self.input_quant.get("is_dynamic")
|
||||
@@ -734,36 +737,11 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
self.emulate = (
|
||||
not current_platform.supports_mx()
|
||||
or not self.ocp_mx_scheme.startswith("w_mxfp4")
|
||||
) and (self.mxfp4_backend is None or not self.use_rocm_aiter_moe)
|
||||
|
||||
# CK's pre-compiled MXFP4 MoE GEMM kernel instances have dimension
|
||||
# alignment requirements. When violated (e.g. MiniMax-M2.1 with
|
||||
# TP=4 yields intermediate_size_per_partition=384), AITER raises:
|
||||
# "device_gemm ... does not support this GEMM problem".
|
||||
# Fall back to emulation in that case.
|
||||
# For gpt_oss models, create_weights rounds up the dimensions
|
||||
# internally, so the alignment check is skipped.
|
||||
if (
|
||||
not self.emulate
|
||||
and self.use_rocm_aiter_moe
|
||||
and self.ocp_mx_scheme is not None
|
||||
and self.ocp_mx_scheme.startswith("w_mxfp4")
|
||||
and self.model_type != "gpt_oss"
|
||||
and moe.intermediate_size_per_partition % CK_MXFP4_MOE_DIM_ALIGNMENT != 0
|
||||
):
|
||||
logger.warning_once(
|
||||
"AITER CK MXFP4 MoE GEMM does not support "
|
||||
"intermediate_size_per_partition=%d (not a multiple of %d). "
|
||||
"This typically happens when intermediate_size / "
|
||||
"tensor_parallel_size produces an incompatible dimension. "
|
||||
"Falling back to emulation mode. To avoid this overhead, "
|
||||
"use a compatible tensor_parallel_size or set "
|
||||
"VLLM_ROCM_USE_AITER_MOE=0.",
|
||||
moe.intermediate_size_per_partition,
|
||||
CK_MXFP4_MOE_DIM_ALIGNMENT,
|
||||
)
|
||||
self.use_rocm_aiter_moe = False
|
||||
self.emulate = True
|
||||
) and (
|
||||
self.mxfp4_backend is None
|
||||
or self.mxfp4_backend is Mxfp4MoeBackend.NONE
|
||||
or not self.use_rocm_aiter_moe
|
||||
)
|
||||
|
||||
if self.emulate:
|
||||
logger.warning_once(
|
||||
@@ -780,6 +758,27 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
"The current mode supports native MoE MXFP4 computation"
|
||||
)
|
||||
|
||||
def maybe_roundup_sizes(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
act_dtype: torch.dtype,
|
||||
moe_parallel_config: FusedMoEParallelConfig,
|
||||
) -> tuple[int, int]:
|
||||
hidden_size, intermediate_size_per_partition = super().maybe_roundup_sizes(
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size_per_partition=intermediate_size_per_partition,
|
||||
act_dtype=act_dtype,
|
||||
moe_parallel_config=moe_parallel_config,
|
||||
)
|
||||
if self.mxfp4_backend is not None:
|
||||
hidden_size, intermediate_size_per_partition = (
|
||||
mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
self.mxfp4_backend, hidden_size, intermediate_size_per_partition
|
||||
)
|
||||
)
|
||||
return hidden_size, intermediate_size_per_partition
|
||||
|
||||
def get_packed_dim(self, dim: int, quant_dtype: str):
|
||||
if quant_dtype == "mxfp4":
|
||||
assert dim % 2 == 0
|
||||
@@ -805,40 +804,12 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
)
|
||||
|
||||
params_dtype = torch.uint8
|
||||
self.intermediate_size_per_partition = intermediate_size_per_partition
|
||||
if self.model_type == "gpt_oss":
|
||||
if current_platform.is_rocm():
|
||||
intermediate_size_per_partition_after_pad = round_up(
|
||||
intermediate_size_per_partition, 256
|
||||
)
|
||||
else:
|
||||
intermediate_size_per_partition_after_pad = round_up(
|
||||
intermediate_size_per_partition, 64
|
||||
)
|
||||
else:
|
||||
intermediate_size_per_partition_after_pad = intermediate_size_per_partition
|
||||
|
||||
self.unpadded_hidden_size = extra_weight_attrs.get(
|
||||
"unpadded_hidden_size", hidden_size
|
||||
)
|
||||
|
||||
# On GFX950, the GFX950MXScaleLayout swizzle requires
|
||||
# hidden_size to be a multiple of 256 (SCALE_K = hidden_size / 32
|
||||
# must be divisible by 8). Pad hidden_size for weight/scale
|
||||
# allocation; the original value is preserved in unpadded_hidden_size.
|
||||
# Only applies to the native (non-emulated) CK path on GFX950.
|
||||
if (
|
||||
self.model_type == "gpt_oss"
|
||||
and current_platform.is_rocm()
|
||||
and not self.emulate
|
||||
):
|
||||
hidden_size = round_up(hidden_size, 256)
|
||||
|
||||
# WEIGHTS
|
||||
w13_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition_after_pad,
|
||||
2 * intermediate_size_per_partition,
|
||||
self.get_packed_dim(hidden_size, self.weight_dtype),
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -849,12 +820,10 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
set_weight_attrs(w13_weight, extra_weight_attrs)
|
||||
|
||||
w2_weight = torch.nn.Parameter(
|
||||
torch.empty(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
self.get_packed_dim(
|
||||
intermediate_size_per_partition_after_pad, self.weight_dtype
|
||||
),
|
||||
self.get_packed_dim(intermediate_size_per_partition, self.weight_dtype),
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -867,7 +836,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
w13_weight_scale = torch.nn.Parameter(
|
||||
torch.ones(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition_after_pad,
|
||||
2 * intermediate_size_per_partition,
|
||||
hidden_size // OCP_MX_BLOCK_SIZE,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
@@ -877,7 +846,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
torch.ones(
|
||||
num_experts,
|
||||
hidden_size,
|
||||
intermediate_size_per_partition_after_pad // OCP_MX_BLOCK_SIZE,
|
||||
intermediate_size_per_partition // OCP_MX_BLOCK_SIZE,
|
||||
dtype=params_dtype,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -892,7 +861,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
w13_bias = torch.nn.Parameter(
|
||||
torch.zeros(
|
||||
num_experts,
|
||||
2 * intermediate_size_per_partition_after_pad,
|
||||
2 * intermediate_size_per_partition,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
requires_grad=False,
|
||||
@@ -1072,6 +1041,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
topk_ids=topk_ids,
|
||||
activation=layer.activation,
|
||||
quant_config=self.moe_quant_config,
|
||||
moe_config=layer.moe_config,
|
||||
expert_map=layer.expert_map,
|
||||
)
|
||||
else:
|
||||
@@ -1204,6 +1174,8 @@ class QuarkOCP_MX_MoEMethod_OSS(QuarkOCP_MX_MoEMethod):
|
||||
triton_kernel_moe_forward,
|
||||
)
|
||||
|
||||
assert self.moe.hidden_dim_unpadded is not None
|
||||
assert self.moe.intermediate_size_per_partition_unpadded is not None
|
||||
return triton_kernel_moe_forward(
|
||||
hidden_states=x,
|
||||
w1=self.w13_weight_triton_tensor,
|
||||
@@ -1215,8 +1187,8 @@ class QuarkOCP_MX_MoEMethod_OSS(QuarkOCP_MX_MoEMethod):
|
||||
expert_map=expert_map,
|
||||
quant_config=self.moe_quant_config,
|
||||
apply_router_weight_on_input=layer.apply_router_weight_on_input,
|
||||
unpadded_N_w1=self.intermediate_size_per_partition * 2,
|
||||
unpadded_K_w1=self.unpadded_hidden_size,
|
||||
unpadded_N_w2=self.unpadded_hidden_size,
|
||||
unpadded_K_w2=self.intermediate_size_per_partition,
|
||||
unpadded_N_w1=self.moe.intermediate_size_per_partition_unpadded * 2,
|
||||
unpadded_K_w1=self.moe.hidden_dim_unpadded,
|
||||
unpadded_N_w2=self.moe.hidden_dim_unpadded,
|
||||
unpadded_K_w2=self.moe.intermediate_size_per_partition_unpadded,
|
||||
)
|
||||
|
||||
@@ -254,7 +254,6 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
|
||||
w13, w13_scale, w2, w2_scale, is_act_and_mul, min_alignment
|
||||
)
|
||||
)
|
||||
layer.intermediate_size_per_partition = padded_intermediate
|
||||
layer.moe_config.intermediate_size_per_partition = padded_intermediate
|
||||
|
||||
w13, w13_scale, w2, w2_scale = prepare_static_weights_for_trtllm_fp4_moe(
|
||||
|
||||
@@ -439,7 +439,6 @@ def prepare_fp8_moe_layer_for_fi(
|
||||
layer.moe_config.is_act_and_mul,
|
||||
min_alignment,
|
||||
)
|
||||
layer.intermediate_size_per_partition = new_intermediate
|
||||
layer.moe_config.intermediate_size_per_partition = new_intermediate
|
||||
|
||||
# FI kernels require W31 layout rather than W13.
|
||||
|
||||
Reference in New Issue
Block a user