[Perf] Kimi K3 nvfp4 Align in_proj weights by 128 to avoid elementwise copy (#55242)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Wei Zhao
2026-09-05 10:46:47 +08:00
committed by GitHub
co-authored by OpenAI Codex
parent 385ba6b5b0
commit 16328c7a77
6 changed files with 53 additions and 21 deletions
+14 -8
View File
@@ -33,6 +33,9 @@ from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
from vllm.model_executor.layers.mamba.ops.gather_initial_states import (
gather_initial_states,
)
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptMixedPrecisionConfig,
)
from vllm.model_executor.model_loader.weight_utils import (
default_weight_loader,
sharded_weight_loader,
@@ -428,8 +431,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
)
self._projection_overlap_max_tokens = 0
# Keep f_a before the narrow beta shard, then pad each TP-local row
# to select the aligned BF16 GEMM path.
# Keep f_a before the narrow beta shard, then align each TP-local row.
qkvg_output_sizes = [self.projection_size] * 4
in_proj_output_sizes = qkvg_output_sizes + [
self.head_dim,
@@ -438,7 +440,14 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
local_output_size = (
4 * self.local_projection_size + self.head_dim + self.local_num_heads
)
self.in_proj_padding = -local_output_size % 16
in_proj_prefix = f"{prefix}.in_proj_qkvgfab"
alignment = (
128
if isinstance(self.quant_config, ModelOptMixedPrecisionConfig)
and self.quant_config._resolve_quant_algo(in_proj_prefix) == "FP8_PB_WO"
else 16
)
self.in_proj_padding = -local_output_size % alignment
if self.in_proj_padding:
in_proj_output_sizes.append(self.in_proj_padding * self.tp_size)
self.in_proj_qkvgfab = _KimiGDNMergedColumnParallelLinear(
@@ -448,7 +457,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
tp_size=self.tp_size,
bias=False,
quant_config=self.quant_config,
prefix=f"{prefix}.in_proj_qkvgfab",
prefix=in_proj_prefix,
)
if self.in_proj_padding:
self.in_proj_qkvgfab.weight.data[-self.in_proj_padding :].zero_()
@@ -588,10 +597,7 @@ class KimiK3DeltaAttention(GatedDeltaNetAttention):
if gemm_rs_ar.can_run(self.o_proj):
self.gemm_rs_ar = gemm_rs_ar
else:
logger.warning_once(
"GEMM-RS/AR is disabled for %s due to an incompatible projection.",
prefix,
)
gemm_rs_ar.warn_incompatible_projection()
compilation_config = vllm_config.compilation_config
if prefix in compilation_config.static_forward_context:
raise ValueError(f"Duplicate layer name: {prefix}")
+1 -4
View File
@@ -309,10 +309,7 @@ class MultiHeadLatentAttention(nn.Module, AttentionLayerBase):
if gemm_rs_ar.can_run(self.o_proj):
self.gemm_rs_ar = gemm_rs_ar
else:
logger.warning_once(
"GEMM-RS/AR is disabled for %s due to an incompatible projection.",
prefix,
)
gemm_rs_ar.warn_incompatible_projection()
# ---- Attention backend / impl / KV cache ----
self.quant_config = quant_config
+1 -4
View File
@@ -283,10 +283,7 @@ class KimiMLP(nn.Module):
if gemm_rs_ar.can_run(self.down_proj):
self.gemm_rs_ar = gemm_rs_ar
else:
logger.warning_once(
"GEMM-RS/AR is disabled for %s due to an incompatible projection.",
prefix,
)
gemm_rs_ar.warn_incompatible_projection()
if hidden_act == "silu":
self.act_fn = SiluAndMul()
elif hidden_act == "situ":
@@ -22,8 +22,11 @@ from cutlass.utils import get_smem_capacity_in_bytes
from vllm.cute_utils import _tcgen05, mbarrier, simple_tma_copy, to_cta0_smem
from vllm.distributed import get_tp_group
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
logger = init_logger(__name__)
@dsl_user_op
def nanosleep(ns: int, *, loc=None, ip=None) -> None:
@@ -779,6 +782,13 @@ class GemmRsAr:
and w.is_contiguous()
)
def warn_incompatible_projection(self) -> None:
logger.warning_once(
"Some projections are incompatible with GEMM-RS/AR; using the "
"unfused path instead.",
scope="global",
)
def should_run(self, x: torch.Tensor) -> bool:
# Use the same threshold for RS and AR for now. Small-M shapes are
# supported but faster on the existing LL path.