Support kimi k3 nvfp4 checkpoint (#53132)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
@@ -23,7 +23,7 @@ from tests.kernels.utils import torch_moe
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.custom_op import CustomOp, op_registry
|
||||
from vllm.model_executor.layers.activation import SiluAndMulWithClamp
|
||||
from vllm.model_executor.layers.activation import SiluAndMulWithClamp, SituAndMul
|
||||
from vllm.model_executor.layers.fused_moe import fused_topk
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
@@ -63,6 +63,7 @@ MNK_FACTORS = [
|
||||
_SWIGLU_LIMIT = 0.1
|
||||
_LARGE_OUTPUT1_SCALE = 32768.0
|
||||
_CLAMP_OP_NAME = "test_silu_and_mul_with_clamp"
|
||||
_SITU_OP_NAME = "test_situ_and_mul"
|
||||
|
||||
# Test-only fixed-limit clamp. ``custom_op_name`` makes the class itself
|
||||
# valid as an ``activation=`` argument to ``torch_moe`` (which only looks
|
||||
@@ -78,12 +79,24 @@ if _CLAMP_OP_NAME not in op_registry:
|
||||
super().__init__(_SWIGLU_LIMIT, compile_native=compile_native)
|
||||
|
||||
|
||||
if _SITU_OP_NAME not in op_registry:
|
||||
|
||||
@CustomOp.register(_SITU_OP_NAME)
|
||||
class _SituAndMulTest(SituAndMul):
|
||||
custom_op_name = _SITU_OP_NAME
|
||||
|
||||
def __init__(self, *, compile_native: bool = True) -> None:
|
||||
super().__init__(4.0, 25.0, compile_native=compile_native)
|
||||
|
||||
|
||||
SILU_WITH_CLAMP = op_registry[_CLAMP_OP_NAME]
|
||||
SITU = op_registry[_SITU_OP_NAME]
|
||||
|
||||
|
||||
ACTIVATION_CASES = [
|
||||
pytest.param(MoEActivation.SILU, MoEActivation.SILU, None, id="silu"),
|
||||
pytest.param(MoEActivation.SILU, SILU_WITH_CLAMP, _SWIGLU_LIMIT, id="silu_clamp"),
|
||||
pytest.param(MoEActivation.SITU, SITU, None, id="situ"),
|
||||
pytest.param(
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
@@ -108,7 +121,7 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
topk: int,
|
||||
dtype: torch.dtype,
|
||||
activation: MoEActivation,
|
||||
torch_activation: MoEActivation | type[SiluAndMulWithClamp],
|
||||
torch_activation: MoEActivation | type[SiluAndMulWithClamp] | type[SituAndMul],
|
||||
swiglu_limit: float | None,
|
||||
workspace_init,
|
||||
):
|
||||
@@ -173,6 +186,10 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
in_dtype=dtype,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
activation_situ_beta=4.0 if activation == MoEActivation.SITU else None,
|
||||
activation_situ_linear_beta=(
|
||||
25.0 if activation == MoEActivation.SITU else None
|
||||
),
|
||||
)
|
||||
|
||||
trtllm_inner = TrtLlmNvFp4ExpertsModular(
|
||||
@@ -191,6 +208,8 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
fake_layer.w13_input_scale = torch.ones_like(quant_config.g1_alphas)
|
||||
fake_layer.w2_input_scale = torch.ones_like(quant_config.g2_alphas)
|
||||
trtllm_inner.process_weights_after_loading(fake_layer)
|
||||
if activation == MoEActivation.SITU:
|
||||
torch.testing.assert_close(trtllm_inner.g1_scale_c, quant_config.a2_gscale)
|
||||
|
||||
trtllm_experts = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.linear import UnquantizedLinearMethod
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptFp8Config,
|
||||
ModelOptFp8LinearMethod,
|
||||
ModelOptFp8PbWoLinearMethod,
|
||||
ModelOptMixedPrecisionConfig,
|
||||
ModelOptMxFp8Config,
|
||||
ModelOptNvFp4Config,
|
||||
@@ -137,6 +138,64 @@ def test_modelopt_fp8_updates_weight_dims_after_transpose():
|
||||
method.fp8_linear.process_weights_after_loading.assert_called_once_with(layer)
|
||||
|
||||
|
||||
def test_modelopt_fp8_pb_wo_hides_output_padding(default_vllm_config):
|
||||
config = ModelOptFp8Config(
|
||||
quant_method="FP8_PB_WO",
|
||||
is_checkpoint_fp8_serialized=True,
|
||||
kv_cache_quant_method=None,
|
||||
exclude_modules=[],
|
||||
)
|
||||
default_vllm_config.model_config = Mock(dtype=torch.bfloat16)
|
||||
kernel = Mock()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(default_vllm_config),
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.modelopt.init_fp8_linear_kernel",
|
||||
return_value=kernel,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
):
|
||||
method = ModelOptFp8PbWoLinearMethod(config)
|
||||
layer = torch.nn.Module()
|
||||
method.create_weights(
|
||||
layer,
|
||||
input_size_per_partition=128,
|
||||
output_partition_sizes=[128, 65],
|
||||
input_size=128,
|
||||
output_size=193,
|
||||
params_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
assert layer.weight.shape == (193, 128)
|
||||
assert layer.weight_scale.shape == (2, 1, 1, 1)
|
||||
|
||||
layer.weight.data.fill_(1)
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
assert layer.weight.shape == (256, 128)
|
||||
assert torch.count_nonzero(layer.weight[193:].float()) == 0
|
||||
kernel.process_weights_after_loading.assert_called_once_with(layer)
|
||||
|
||||
physical_output = torch.randn(2, 256, dtype=torch.bfloat16)
|
||||
kernel.apply_weights.return_value = physical_output
|
||||
bias = torch.randn(193, dtype=torch.bfloat16)
|
||||
output = method.apply(layer, torch.randn(2, 128), bias)
|
||||
|
||||
torch.testing.assert_close(output, physical_output[:, :193] + bias)
|
||||
assert output.shape == (2, 193)
|
||||
assert output.is_contiguous()
|
||||
kernel.apply_weights.assert_called_once()
|
||||
assert kernel.apply_weights.call_args.args[2] is None
|
||||
|
||||
|
||||
def test_modelopt_nvfp4_leaves_excluded_parallel_lm_head_unquantized():
|
||||
config = ModelOptNvFp4Config(
|
||||
is_checkpoint_nvfp4_serialized=True,
|
||||
|
||||
Reference in New Issue
Block a user