Support kimi k3 nvfp4 checkpoint (#53132)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Wei Zhao
2026-08-21 05:59:44 +00:00
committed by GitHub
co-authored by OpenAI Codex
parent b389ac2946
commit f8e0602713
6 changed files with 151 additions and 35 deletions
+59
View File
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.linear import UnquantizedLinearMethod
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptFp8Config,
ModelOptFp8LinearMethod,
ModelOptFp8PbWoLinearMethod,
ModelOptMixedPrecisionConfig,
ModelOptMxFp8Config,
ModelOptNvFp4Config,
@@ -137,6 +138,64 @@ def test_modelopt_fp8_updates_weight_dims_after_transpose():
method.fp8_linear.process_weights_after_loading.assert_called_once_with(layer)
def test_modelopt_fp8_pb_wo_hides_output_padding(default_vllm_config):
config = ModelOptFp8Config(
quant_method="FP8_PB_WO",
is_checkpoint_fp8_serialized=True,
kv_cache_quant_method=None,
exclude_modules=[],
)
default_vllm_config.model_config = Mock(dtype=torch.bfloat16)
kernel = Mock()
with (
set_current_vllm_config(default_vllm_config),
patch(
"vllm.model_executor.layers.quantization.modelopt.init_fp8_linear_kernel",
return_value=kernel,
),
patch(
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
return_value=0,
),
patch(
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
return_value=1,
),
):
method = ModelOptFp8PbWoLinearMethod(config)
layer = torch.nn.Module()
method.create_weights(
layer,
input_size_per_partition=128,
output_partition_sizes=[128, 65],
input_size=128,
output_size=193,
params_dtype=torch.bfloat16,
)
assert layer.weight.shape == (193, 128)
assert layer.weight_scale.shape == (2, 1, 1, 1)
layer.weight.data.fill_(1)
method.process_weights_after_loading(layer)
assert layer.weight.shape == (256, 128)
assert torch.count_nonzero(layer.weight[193:].float()) == 0
kernel.process_weights_after_loading.assert_called_once_with(layer)
physical_output = torch.randn(2, 256, dtype=torch.bfloat16)
kernel.apply_weights.return_value = physical_output
bias = torch.randn(193, dtype=torch.bfloat16)
output = method.apply(layer, torch.randn(2, 128), bias)
torch.testing.assert_close(output, physical_output[:, :193] + bias)
assert output.shape == (2, 193)
assert output.is_contiguous()
kernel.apply_weights.assert_called_once()
assert kernel.apply_weights.call_args.args[2] is None
def test_modelopt_nvfp4_leaves_excluded_parallel_lm_head_unquantized():
config = ModelOptNvFp4Config(
is_checkpoint_nvfp4_serialized=True,