[Refactor][Quantization] Make FP8 and MLA weight transforms reusable pure functions (#57732)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
aoshen02
2026-09-23 01:16:20 -07:00
committed by GitHub
co-authored by Yongye Zhu
parent 9488318c6a
commit 973a3be780
6 changed files with 36 additions and 49 deletions
@@ -404,6 +404,24 @@ def _get_kv_b_proj_input_dtype(
return weight_dtype
def split_kv_b_proj(
kv_b_proj: nn.Module,
out_dtype: torch.dtype,
kv_lora_rank: int,
num_heads: int,
qk_nope_head_dim: int,
v_head_dim: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Dequantize ``kv_b_proj`` and return ``W_UK [L,N,P]``, ``W_UV [L,N,V]``."""
weight = get_and_maybe_dequant_weights(kv_b_proj, out_dtype=out_dtype).T
assert weight.shape == (
kv_lora_rank,
num_heads * (qk_nope_head_dim + v_head_dim),
), f"kv_b_proj weight {tuple(weight.shape)} vs {kv_lora_rank=} {num_heads=}"
weight = weight.view(kv_lora_rank, num_heads, qk_nope_head_dim + v_head_dim)
return weight.split([qk_nope_head_dim, v_head_dim], dim=-1)
class MLAAttention(nn.Module, AttentionLayerBase):
"""Multi-Head Latent Attention layer.
@@ -1221,13 +1239,6 @@ class MLAAttention(nn.Module, AttentionLayerBase):
)
return
# we currently do not have quantized bmm's which are needed for
# `W_UV` and `W_UK_T`, we just store fp16/bf16 copies and perform
# the bmm's in 16-bit, the extra memory overhead of this is fairly low
kv_b_proj_weight = get_and_maybe_dequant_weights(
self.kv_b_proj, out_dtype=act_dtype
).T
if self.dcp_q_replicate:
# qrep wired here: validate unsupported decode backends once.
assert self.q_pad_num_heads in (None, self.num_heads), (
@@ -1243,24 +1254,13 @@ class MLAAttention(nn.Module, AttentionLayerBase):
"FP4/FP8 MLA BMM paths."
)
assert kv_b_proj_weight.shape == (
self.kv_lora_rank,
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
), (
f"{kv_b_proj_weight.shape=}, "
f"{self.kv_lora_rank=}, "
f"{self.num_heads=}, "
f"{self.qk_nope_head_dim=}, "
f"{self.v_head_dim=}"
)
kv_b_proj_weight = kv_b_proj_weight.view(
W_UK, W_UV = split_kv_b_proj(
self.kv_b_proj,
act_dtype,
self.kv_lora_rank,
self.num_heads,
self.qk_nope_head_dim + self.v_head_dim,
)
W_UK, W_UV = kv_b_proj_weight.split(
[self.qk_nope_head_dim, self.v_head_dim], dim=-1
self.qk_nope_head_dim,
self.v_head_dim,
)
# If kv_b_proj_weight is unquantized, quantize it to mxfp4 if supported