Support kimi k3 nvfp4 checkpoint (#53132)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
@@ -371,8 +371,17 @@ def _get_kv_b_proj_input_dtype(
|
||||
return kv_b_proj.params_dtype
|
||||
if weight_dtype == torch.uint8:
|
||||
return None
|
||||
if weight_dtype == current_platform.fp8_dtype() and not use_fp8_prefill:
|
||||
return None
|
||||
if weight_dtype == current_platform.fp8_dtype():
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptFp8PbWoLinearMethod,
|
||||
)
|
||||
|
||||
quant_method = getattr(kv_b_proj, "quant_method", None)
|
||||
# FP8_PB_WO dynamically quantizes BF16/FP16 inputs in the linear method.
|
||||
if isinstance(quant_method, ModelOptFp8PbWoLinearMethod):
|
||||
return quant_method.input_dtype
|
||||
if not use_fp8_prefill:
|
||||
return None
|
||||
return weight_dtype
|
||||
|
||||
|
||||
|
||||
@@ -70,16 +70,9 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
)
|
||||
self.local_num_experts = moe_config.num_local_experts
|
||||
self.ep_rank = moe_config.moe_parallel_config.ep_rank
|
||||
self.is_situ = moe_config.activation == MoEActivation.SITU
|
||||
|
||||
assert self.quant_config.g1_alphas is not None
|
||||
assert self.quant_config.a2_gscale is not None
|
||||
if moe_config.is_act_and_mul:
|
||||
# g1_alpha_s = a13_scale * w13_scale_2
|
||||
# a2_gscale = (1 / a2_scale)
|
||||
# g1_scale_c = a13_scale * w13_scale_2 / a2_scale
|
||||
self.g1_scale_c = self.quant_config.g1_alphas * self.quant_config.a2_gscale
|
||||
else:
|
||||
self.g1_scale_c = self.quant_config.a2_gscale.clone()
|
||||
self.g1_scale_c = self._compute_g1_scale_c()
|
||||
|
||||
# Fall back to moe_config.swiglu_* when quant_config doesn't carry them
|
||||
# (ModelOpt NVFP4 checkpoints store these on moe_config, not quant_config).
|
||||
@@ -120,7 +113,6 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
# (gemm1_alpha) and situ linear_beta -> gatedActBeta (gemm1_beta).
|
||||
# These operate on the dequantized gate/up, so they are NOT folded by
|
||||
# g1_alphas in process_weights_after_loading.
|
||||
self.is_situ = moe_config.activation == MoEActivation.SITU
|
||||
if self.is_situ:
|
||||
situ_beta = moe_config.activation_situ_beta
|
||||
situ_linear_beta = moe_config.activation_situ_linear_beta
|
||||
@@ -143,18 +135,28 @@ class TrtLlmNvFp4ExpertsBase:
|
||||
clamp,
|
||||
)
|
||||
|
||||
def _compute_g1_scale_c(self) -> torch.Tensor:
|
||||
assert self.quant_config.g1_alphas is not None
|
||||
assert self.quant_config.a2_gscale is not None
|
||||
if not self.moe_config.is_act_and_mul:
|
||||
return self.quant_config.a2_gscale.clone()
|
||||
if self.is_situ:
|
||||
# SITU applies its nonlinear activation after g1_alphas, so only
|
||||
# the output quantization factor belongs in g1_scale_c.
|
||||
return self.quant_config.a2_gscale.clone()
|
||||
|
||||
# g1_alphas = a13_scale * w13_scale_2
|
||||
# a2_gscale = 1 / a2_scale
|
||||
# g1_scale_c = a13_scale * w13_scale_2 / a2_scale
|
||||
return self.quant_config.g1_alphas * self.quant_config.a2_gscale
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.w13_weight_scale_2.data.mul_(layer.w13_input_scale)
|
||||
layer.w2_weight_scale_2.data.mul_(layer.w2_input_scale)
|
||||
# Recompute g1_scale_c since g1_alphas was just fused in-place.
|
||||
# Register as a layer parameter so EPLB rearranges it alongside
|
||||
# other expert weights.
|
||||
assert self.quant_config.g1_alphas is not None
|
||||
assert self.quant_config.a2_gscale is not None
|
||||
if self.moe_config.is_act_and_mul:
|
||||
g1_scale_c = self.quant_config.g1_alphas * self.quant_config.a2_gscale
|
||||
else:
|
||||
g1_scale_c = self.quant_config.a2_gscale.clone()
|
||||
g1_scale_c = self._compute_g1_scale_c()
|
||||
layer.register_parameter(
|
||||
"g1_scale_c",
|
||||
torch.nn.Parameter(g1_scale_c, requires_grad=False),
|
||||
|
||||
@@ -624,6 +624,8 @@ class ModelOptFp8PbWoLinearMethod(LinearMethodBase):
|
||||
where block size is typically 128 for both dims.
|
||||
|
||||
vLLM executes it as FP8 GEMM with *dynamic per-token* activation quant.
|
||||
Output widths that are not block-aligned are padded and restored
|
||||
to their logical width before returning to the model.
|
||||
"""
|
||||
|
||||
_WEIGHT_BLOCK_SIZE: tuple[int, int] = (128, 128)
|
||||
@@ -670,6 +672,12 @@ class ModelOptFp8PbWoLinearMethod(LinearMethodBase):
|
||||
# element-space -> block-space for BlockQuantScaleParameter.
|
||||
layer.weight_block_size = self.weight_block_size
|
||||
|
||||
block_n, block_k = self._WEIGHT_BLOCK_SIZE
|
||||
remainder = output_size_per_partition % block_n
|
||||
self.output_padding = 0 if remainder == 0 else block_n - remainder
|
||||
self.logical_output_size = output_size_per_partition
|
||||
physical_output_size = output_size_per_partition + self.output_padding
|
||||
|
||||
weight = ModelWeightParameter(
|
||||
data=torch.empty(
|
||||
output_size_per_partition,
|
||||
@@ -682,19 +690,13 @@ class ModelOptFp8PbWoLinearMethod(LinearMethodBase):
|
||||
)
|
||||
layer.register_parameter("weight", weight)
|
||||
|
||||
block_n, block_k = self._WEIGHT_BLOCK_SIZE
|
||||
if output_size_per_partition % block_n != 0:
|
||||
raise ValueError(
|
||||
"ModelOpt FP8_PB_WO requires out_features divisible by "
|
||||
f"{block_n}, got {output_size_per_partition}."
|
||||
)
|
||||
if input_size_per_partition % block_k != 0:
|
||||
raise ValueError(
|
||||
"ModelOpt FP8_PB_WO requires in_features divisible by "
|
||||
f"{block_k}, got {input_size_per_partition}."
|
||||
)
|
||||
|
||||
out_blks = output_size_per_partition // block_n
|
||||
out_blks = physical_output_size // block_n
|
||||
in_blks = input_size_per_partition // block_k
|
||||
|
||||
# Match ModelOpt's exported shape so weight loading works without a
|
||||
@@ -711,7 +713,7 @@ class ModelOptFp8PbWoLinearMethod(LinearMethodBase):
|
||||
self.w8a8_block_fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
weight_shape=layer.weight.shape,
|
||||
weight_shape=(physical_output_size, input_size_per_partition),
|
||||
input_dtype=self.input_dtype,
|
||||
out_dtype=self.out_dtype,
|
||||
module_name=self.__class__.__name__,
|
||||
@@ -719,7 +721,15 @@ class ModelOptFp8PbWoLinearMethod(LinearMethodBase):
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
# Keep weight in [out, in] layout for Fp8BlockScaledMMLinearKernel.
|
||||
layer.weight = Parameter(layer.weight.data, requires_grad=False)
|
||||
weight = layer.weight.data
|
||||
if self.output_padding:
|
||||
padded_weight = weight.new_zeros(
|
||||
self.logical_output_size + self.output_padding,
|
||||
weight.shape[1],
|
||||
)
|
||||
padded_weight[: self.logical_output_size].copy_(weight)
|
||||
weight = padded_weight
|
||||
layer.weight = Parameter(weight, requires_grad=False)
|
||||
|
||||
scale = layer.weight_scale
|
||||
if scale.dim() == 4:
|
||||
@@ -733,8 +743,7 @@ class ModelOptFp8PbWoLinearMethod(LinearMethodBase):
|
||||
|
||||
layer.weight_scale = Parameter(scale.contiguous(), requires_grad=False)
|
||||
|
||||
if hasattr(self, "fp8_linear"):
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
self.w8a8_block_fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -742,7 +751,15 @@ class ModelOptFp8PbWoLinearMethod(LinearMethodBase):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.w8a8_block_fp8_linear.apply_weights(layer, x, bias)
|
||||
kernel_bias = None if self.output_padding else bias
|
||||
output = self.w8a8_block_fp8_linear.apply_weights(layer, x, kernel_bias)
|
||||
if not self.output_padding:
|
||||
return output
|
||||
|
||||
output = output[..., : self.logical_output_size].contiguous()
|
||||
if bias is not None:
|
||||
output.add_(bias)
|
||||
return output
|
||||
|
||||
|
||||
class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
@@ -2390,6 +2407,8 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
|
||||
if isinstance(layer, (LinearBase, ParallelLMHead)):
|
||||
if quant_algo == "FP8":
|
||||
return ModelOptFp8LinearMethod(self.fp8_config)
|
||||
if quant_algo == "FP8_PB_WO":
|
||||
return ModelOptFp8PbWoLinearMethod(self.fp8_config)
|
||||
if quant_algo == "NVFP4":
|
||||
return ModelOptNvFp4LinearMethod(self.nvfp4_config)
|
||||
if quant_algo == "W4A16_NVFP4":
|
||||
|
||||
Reference in New Issue
Block a user