Support kimi k3 nvfp4 checkpoint (#53132)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Wei Zhao
2026-08-21 05:59:44 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent b389ac2946
commit f8e0602713
6 changed files with 151 additions and 35 deletions
@@ -371,8 +371,17 @@ def _get_kv_b_proj_input_dtype(
return kv_b_proj.params_dtype
if weight_dtype == torch.uint8:
return None
if weight_dtype == current_platform.fp8_dtype() and not use_fp8_prefill:
return None
if weight_dtype == current_platform.fp8_dtype():
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptFp8PbWoLinearMethod,
)
quant_method = getattr(kv_b_proj, "quant_method", None)
# FP8_PB_WO dynamically quantizes BF16/FP16 inputs in the linear method.
if isinstance(quant_method, ModelOptFp8PbWoLinearMethod):
return quant_method.input_dtype
if not use_fp8_prefill:
return None
return weight_dtype