[Kimi-K3] Merge MLA gate into QKV-A projection (#54015)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
This commit is contained in:
Thien Tran
2026-08-27 17:47:18 -07:00
committed by GitHub
parent b2a6e9dca0
commit 6ec92bcbc8
4 changed files with 283 additions and 106 deletions
+90 -1
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import abstractmethod
from collections.abc import Iterable
from collections.abc import Callable, Iterable
from typing import Any
import torch
@@ -1512,6 +1512,95 @@ class MinimaxM3QKVParallelLinearWithIndexer(QKVParallelLinear):
param_data.copy_(loaded_weight)
class KimiK3MergedQKVGateLinear(MergedColumnParallelLinear):
"""Kimi-K3 QKV-A projection fused with its output-gate projection.
NOTE: Kimi-K3-specific. This is tailored to its replicated Q/KV latent
projections and tensor-parallel output gate; it is not a general-purpose
linear layer. It lives here alongside ``MergedColumnParallelLinear``, whose
sharding and weight-loading machinery it reuses.
The per-rank shard layout is ``[q_a | kv_a | gate]``. The latent shards
stay replicated across tensor-parallel ranks while the gate shard is
tensor-parallel, matching the standalone projections they replace.
"""
def __init__(
self,
hidden_size: int,
q_lora_rank: int,
kv_lora_rank: int,
qk_rope_head_dim: int,
total_num_heads: int,
v_head_dim: int,
bias: bool = False,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
tp_size = get_tensor_model_parallel_world_size()
# MergedColumnParallelLinear divides every logical shard by TP size.
# Inflate the replicated latent shards to retain their checkpoint widths.
output_sizes = [
q_lora_rank * tp_size,
(kv_lora_rank + qk_rope_head_dim) * tp_size,
total_num_heads * v_head_dim,
]
super().__init__(
input_size=hidden_size,
output_sizes=output_sizes,
bias=bias,
quant_config=quant_config,
prefix=prefix,
)
def _load_with_replicated_latents(
self,
loader: Callable[..., None],
param: Parameter,
loaded_weight: torch.Tensor,
loaded_shard_id: tuple[int, ...] | int | None,
) -> None:
is_replicated = isinstance(loaded_shard_id, int) and loaded_shard_id < 2
if not is_replicated:
loader(param, loaded_weight, loaded_shard_id)
return
# The base loader narrows every checkpoint shard using its TP rank.
# Replicated latent shards instead load the rank-zero slice on every rank.
tp_rank = self.tp_rank
param_tp_rank = getattr(param, "tp_rank", None)
self.tp_rank = 0
if param_tp_rank is not None:
param.tp_rank = 0
try:
loader(param, loaded_weight, loaded_shard_id)
finally:
self.tp_rank = tp_rank
if param_tp_rank is not None:
param.tp_rank = param_tp_rank
def weight_loader(
self,
param: Parameter,
loaded_weight: torch.Tensor,
loaded_shard_id: tuple[int, ...] | int | None = None,
) -> None:
self._load_with_replicated_latents(
super().weight_loader, param, loaded_weight, loaded_shard_id
)
def weight_loader_v2(
self,
param: BasevLLMParameter,
loaded_weight: torch.Tensor,
loaded_shard_id: tuple[int, ...] | int | None = None,
) -> None:
self._load_with_replicated_latents(
super().weight_loader_v2, param, loaded_weight, loaded_shard_id
)
# --8<-- [start:row_parallel_linear]
@PluggableLayer.register("row_parallel_linear")
class RowParallelLinear(LinearBase):