diff --git a/tests/kernels/test_bf16_skinny_gemm.py b/tests/kernels/test_bf16_skinny_gemm.py index d542f9f483..4988b4afd3 100644 --- a/tests/kernels/test_bf16_skinny_gemm.py +++ b/tests/kernels/test_bf16_skinny_gemm.py @@ -39,6 +39,7 @@ EXPECTED_SELECTIONS = { (7168, 8448): (set(range(1, 4)), set()), (8448, 7168): ({1, 2}, set()), (16896, 7168): ({1, 2}, set()), + (7168, 14336): ({1, 2}, set()), (20480, 7168): (set(range(1, 5)), set()), (40960, 7168): (set(range(1, 5)), set()), # TP16. @@ -96,6 +97,8 @@ EXPECTED_CUTE_CONFIGS = { (8448, 7168, 2): (32, 4, 4, 8), (16896, 7168, 1): (224, 6, 4, 8), (16896, 7168, 2): (32, 4, 4, 8), + (7168, 14336, 1): (256, 2, 1, 4), + (7168, 14336, 2): (224, 4, 2, 8), (20480, 7168, 1): (224, 4, 2, 8), (20480, 7168, 2): (64, 4, 2, 8), (20480, 7168, 3): (64, 2, 2, 8), @@ -218,7 +221,12 @@ def test_cute_configs_match_measured_table() -> None: for n, k, num_tokens, config in configs } assert actual == EXPECTED_CUTE_CONFIGS - assert all(config.static_k is None for *_, config in configs) + static_k_configs = { + (n, k, num_tokens, config.static_k) + for n, k, num_tokens, config in configs + if config.static_k is not None + } + assert static_k_configs == {(7168, 14336, 1, 14336)} def test_residual_cute_configs_match_measured_table() -> None: diff --git a/vllm/models/kimi_k3/nvidia/low_latency_gemm.py b/vllm/models/kimi_k3/nvidia/low_latency_gemm.py index 40dd51cb6d..834dc880ed 100644 --- a/vllm/models/kimi_k3/nvidia/low_latency_gemm.py +++ b/vllm/models/kimi_k3/nvidia/low_latency_gemm.py @@ -194,6 +194,14 @@ KIMI_K3_PROJECTIONS: dict[tuple[int, int], ProjectionSpec] = { ), name="dense_gate_up_proj", ), + (7168, 14336): ProjectionSpec( + 7168, + 14336, + cute_configs=( + (1, SkinnyGemmConfig(1, 256, 2, vector_width=4, static_k=14336)), + (2, _cute(2, 224, 4, 2)), + ), + ), (20480, 7168): ProjectionSpec( 20480, 7168, diff --git a/vllm/models/kimi_k3/nvidia/mtp.py b/vllm/models/kimi_k3/nvidia/mtp.py index 45cbe3767e..1677700202 100644 --- a/vllm/models/kimi_k3/nvidia/mtp.py +++ b/vllm/models/kimi_k3/nvidia/mtp.py @@ -16,6 +16,7 @@ from vllm.model_executor.layers.fused_moe import ( fused_moe_make_expert_params_mapping, ) from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.linear import ReplicatedLinear from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.vocab_parallel_embedding import ( @@ -80,7 +81,14 @@ class KimiK3MultiTokenPredictorLayer(nn.Module): self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.hnorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) - self.eh_proj = nn.Linear(config.hidden_size * 2, config.hidden_size, bias=False) + self.eh_proj = ReplicatedLinear( + config.hidden_size * 2, + config.hidden_size, + bias=False, + quant_config=None, + prefix=maybe_prefix(prefix, "eh_proj"), + return_bias=False, + ) self.shared_head = SharedHead( config=config, prefix=prefix, quant_config=quant_config