[Bugfix][Model] Fix MiMo-V2.5 fused fp8 qkv_proj sharding (pre-shard count is num_key_value_heads; MTP path too) (#57508)

Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
This commit is contained in:
vllmellm
2026-09-19 06:05:00 -07:00
committed by GitHub
parent 4cc15f2121
commit 211e252d0b
3 changed files with 414 additions and 62 deletions
@@ -0,0 +1,225 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit test for MiMo-V2 fused fp8 ``qkv_proj`` sharding.
The checkpoint stores the fused QKV pre-sharded at ``num_key_value_heads``
chunks (``ckpt_tp``), each holding that slice's ``[Q_c | K_c | V_c]`` rows with
its own block scale grid. ``ckpt_tp`` is not the layer's KV-head count: a
MiMo-V2.5 SWA layer has 8 KV heads but 4 chunks of 3712 rows (= 29 blocks), so
each chunk carries two KV heads. Reading the scales as one slice per KV head
expands 14 x 128 = 1792 rows for an 1856-row slice and fails.
These tests build such checkpoints for the GA, SWA and Pro geometries and check
that every rank gets exactly the weights it should own, for TP sizes below,
equal to and above ``ckpt_tp`` (the last case replicating KV heads).
"""
import pytest
import torch
from vllm.model_executor.models.mimo_v2 import _shard_fp8_qkv_proj
from vllm.utils.math_utils import cdiv
pytestmark = pytest.mark.cpu_test
BLOCK = 128
FP8_DTYPE = torch.float8_e4m3fn
FP8_MAX = torch.finfo(FP8_DTYPE).max
COLS = 128 # one column block is enough to exercise the row mapping
# fp8 e4m3 has 3 mantissa bits, so a single block-quantization round trip costs
# ~2-3% relative L2; a wrong row -> scale pairing costs >= 10%.
TOL = 0.06
# (num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp): MiMo-V2.5 GA, its
# SWA/MTP layers, and MiMo-V2.5-Pro.
GEOMETRIES = {
"ga": (64, 4, 192, 128, 4),
"swa": (64, 8, 192, 128, 4),
"pro": (128, 8, 192, 128, 8),
}
def _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp):
return (
num_heads // ckpt_tp * head_dim
+ num_kv_heads // ckpt_tp * head_dim
+ num_kv_heads // ckpt_tp * v_head_dim
)
def _quantize_chunks(truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp):
"""Block-quantize each checkpoint chunk on its own grid, like the export."""
rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp)
weights, scales = [], []
for chunk in range(ckpt_tp):
rows = truth[chunk * rows_per_chunk : (chunk + 1) * rows_per_chunk]
quantized = torch.zeros_like(rows)
scale = torch.zeros(
cdiv(rows_per_chunk, BLOCK), rows.shape[1] // BLOCK, dtype=torch.float32
)
for row in range(0, rows_per_chunk, BLOCK):
for col in range(0, rows.shape[1], BLOCK):
block = rows[row : row + BLOCK, col : col + BLOCK]
amax = block.abs().max().clamp(min=1e-12) / FP8_MAX
scale[row // BLOCK, col // BLOCK] = amax
quantized[row : row + BLOCK, col : col + BLOCK] = block / amax
weights.append(quantized.to(FP8_DTYPE))
scales.append(scale)
return torch.cat(weights, dim=0), torch.cat(scales, dim=0)
def _owned_rows(
truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp, tp_rank, tp_size
):
"""Rows the rank must own, in the forward's `[Q | K | V]` order."""
rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp)
q_per_chunk = num_heads // ckpt_tp * head_dim
k_per_chunk = num_kv_heads // ckpt_tp * head_dim
kv_heads: list[int] = []
if tp_size <= num_kv_heads:
kv_heads = list(
range(
tp_rank * (num_kv_heads // tp_size),
(tp_rank + 1) * (num_kv_heads // tp_size),
)
)
else:
kv_heads = [tp_rank // (tp_size // num_kv_heads)]
index = []
for head in range(
tp_rank * (num_heads // tp_size), (tp_rank + 1) * (num_heads // tp_size)
):
chunk, offset = divmod(head, num_heads // ckpt_tp)
start = chunk * rows_per_chunk + offset * head_dim
index += list(range(start, start + head_dim))
for head in kv_heads:
chunk, offset = divmod(head, num_kv_heads // ckpt_tp)
start = chunk * rows_per_chunk + q_per_chunk + offset * head_dim
index += list(range(start, start + head_dim))
for head in kv_heads:
chunk, offset = divmod(head, num_kv_heads // ckpt_tp)
start = chunk * rows_per_chunk + q_per_chunk + k_per_chunk + offset * v_head_dim
index += list(range(start, start + v_head_dim))
return truth[torch.tensor(index)]
def _dequantize(weight, scale):
"""What the kernel computes: scale[row // block] applied per weight row."""
per_row = scale.repeat_interleave(BLOCK, dim=1)
return weight.to(torch.float32) * per_row[torch.arange(weight.shape[0]) // BLOCK]
def _relative_l2(got, expected):
return ((got - expected).norm() / expected.norm()).item()
@pytest.mark.parametrize("geometry", list(GEOMETRIES))
@pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
def test_fused_qkv_proj_sharding(geometry, tp_size):
num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp = GEOMETRIES[geometry]
if num_heads % tp_size or (tp_size > num_kv_heads and tp_size % num_kv_heads):
pytest.skip("TP size does not divide the head counts")
rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp)
truth = torch.randn(ckpt_tp * rows_per_chunk, COLS)
for chunk in range(ckpt_tp): # per-chunk magnitude spread, as in the export
truth[chunk * rows_per_chunk : (chunk + 1) * rows_per_chunk] *= 1.0 + chunk
weight, scale = _quantize_chunks(
truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp
)
kv_per_rank = (num_kv_heads // tp_size) if tp_size <= num_kv_heads else 1
rows_rank = (num_heads // tp_size) * head_dim + kv_per_rank * (
head_dim + v_head_dim
)
for tp_rank in range(tp_size):
w_rank, s_rank = _shard_fp8_qkv_proj(
weight,
scale,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
v_head_dim=v_head_dim,
tp_rank=tp_rank,
tp_size=tp_size,
ckpt_tp=ckpt_tp,
)
assert w_rank.shape == (rows_rank, COLS)
assert s_rank.shape == (cdiv(rows_rank, BLOCK), scale.shape[1])
expected = _owned_rows(
truth,
num_heads,
num_kv_heads,
head_dim,
v_head_dim,
ckpt_tp,
tp_rank,
tp_size,
)
assert _relative_l2(_dequantize(w_rank, s_rank), expected) <= TOL
@pytest.mark.parametrize("geometry", ["ga", "swa"])
@pytest.mark.parametrize("tp_size", [1, 2, 4])
def test_exact_chunk_matches_checkpoint(geometry, tp_size):
"""At tp_size <= ckpt_tp the chunk count is the only thing that matters."""
num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp = GEOMETRIES[geometry]
rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp)
truth = torch.randn(ckpt_tp * rows_per_chunk, COLS)
weight, scale = _quantize_chunks(
truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp
)
if tp_size == ckpt_tp:
w_rank, s_rank = _shard_fp8_qkv_proj(
weight,
scale,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
v_head_dim=v_head_dim,
tp_rank=0,
tp_size=tp_size,
ckpt_tp=ckpt_tp,
)
# The checkpoint chunk *is* the rank's slice: no re-quantization.
# (torch.equal has no fp8 kernel, so compare the underlying bytes.)
expected_w = weight.chunk(tp_size, dim=0)[0]
assert torch.equal(w_rank.view(torch.uint8), expected_w.view(torch.uint8))
assert torch.equal(s_rank, scale.chunk(tp_size, dim=0)[0])
def test_wrong_chunk_count_is_detected():
"""Reading SWA scales as one slice per KV head must not look correct.
This is the regression the fix addresses: 116 scale rows are 4 x 29 (29 =
ceil(3712 / 128)), not 8 x 15, so ``ckpt_tp = num_kv_heads`` pairs every
weight row with the wrong scale.
"""
num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp = GEOMETRIES["swa"]
rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp)
truth = torch.randn(ckpt_tp * rows_per_chunk, COLS)
for chunk in range(ckpt_tp):
truth[chunk * rows_per_chunk : (chunk + 1) * rows_per_chunk] *= 1.0 + chunk
weight, scale = _quantize_chunks(
truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp
)
assert scale.shape[0] == ckpt_tp * cdiv(rows_per_chunk, BLOCK)
assert scale.shape[0] != num_kv_heads * cdiv(rows_per_chunk, BLOCK)
kw = dict(
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
v_head_dim=v_head_dim,
tp_rank=0,
tp_size=1,
)
correct = _shard_fp8_qkv_proj(weight, scale, ckpt_tp=ckpt_tp, **kw)
wrong = _shard_fp8_qkv_proj(weight, scale, ckpt_tp=num_kv_heads, **kw)
assert correct[0].shape == wrong[0].shape
expected = _owned_rows(
truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp, 0, 1
)
assert _relative_l2(_dequantize(*correct), expected) <= TOL
assert _relative_l2(_dequantize(*wrong), expected) > 5 * TOL
+140 -54
View File
@@ -50,6 +50,7 @@ from vllm.model_executor.model_loader.weight_utils import (
)
from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.sequence import IntermediateTensors
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -460,6 +461,28 @@ class MiMoV2FlashDecoderLayer(nn.Module):
return self.config.hybrid_layer_pattern[self.layer_id] == 1
def _requantize_fp8(
grouped: torch.Tensor, rows_rank: int, block: int, dtype: torch.dtype
) -> tuple[torch.Tensor, torch.Tensor]:
"""Block-quantize a rank's ``[Q | K | V]`` rows back to fp8.
A rank's rows need not end on a block boundary (a single 1856-row slice is
14.5 blocks) while ``scaled_quantize`` requires both dims to be multiples of
the block size, so pad the tail with zeros: zero rows cannot raise a block's
amax, so every scale -- and the number of scale rows -- is unchanged. The
padding is dropped again here.
"""
padded = cdiv(rows_rank, block) * block
if padded != rows_rank:
grouped = torch.cat(
[grouped, grouped.new_zeros(padded - rows_rank, grouped.shape[1])], dim=0
)
w_rank, s_rank = scaled_quantize(
grouped, GroupShape(block, block), dtype, compute_dtype=torch.float32
)
return w_rank[:rows_rank], s_rank
def _shard_fp8_qkv_proj(
w_full: torch.Tensor,
s_full: torch.Tensor,
@@ -469,75 +492,136 @@ def _shard_fp8_qkv_proj(
v_head_dim: int,
tp_rank: int,
tp_size: int,
ckpt_tp: int,
block: int = 128,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Shard the fp8 qkv_proj weights for ``tp_rank``.
The checkpoint stores the fused QKV as ``num_kv_heads`` contiguous groups
(one per KV head; ``n`` below), each ordered ``[Q | K | V]``:
The checkpoint stores the fused QKV pre-sharded for ``ckpt_tp`` ranks (the
model config's ``num_key_value_heads``), each chunk holding that slice's
Q, K and V rows:
[Q_1 | K_1 | V_1 | Q_2 | K_2 | V_2 | ... | Q_n | K_n | V_n]
[Q_0 | K_0 | V_0 | Q_1 | K_1 | V_1 | ... | Q_n | K_n | V_n] (n = ckpt_tp)
Per group, Q has ``(num_heads / num_kv_heads) * head_dim`` rows, K has
``head_dim`` rows, and V has ``v_head_dim`` rows.
Per chunk, Q has ``(num_heads / ckpt_tp) * head_dim`` rows, K has
``(num_kv_heads / ckpt_tp) * head_dim`` rows, and V has
``(num_kv_heads / ckpt_tp) * v_head_dim`` rows, and the fp8 block scales
are tiled per chunk too (``ceil(rows_per_chunk / block)`` rows each).
Each TP rank owns ``g = num_kv_heads / tp_size`` of these groups, and the
forward expects them de-interleaved into a single Q, K, and V block:
``ckpt_tp`` is not the layer's KV-head count: a MiMo-V2.5 SWA layer has 8
KV heads over 4 chunks, so each chunk carries two KV heads (3712 rows = 29
blocks) while a GA layer has 4 KV heads over 4 chunks (3392 rows each).
The forward expects each rank's slice de-interleaved:
[Q_1 | Q_2 | ... | Q_g | K_1 | K_2 | ... | K_g | V_1 | V_2 | ... | V_g]
When ``g == 1`` the rank's slice is already ``[Q | K | V]``, so a plain
chunk suffices. When ``g > 1`` we cannot reach the de-interleaved layout by
re-permuting the fp8 block scales: each scale covers a 128-row block, and
since K is 192 rows (1.5 blocks) a block straddles the K/V boundary, so no
whole-block permutation produces it. Instead we dequantize this rank's
groups to float (dropping the block constraint), reorder the rows into the
layout above (Q, K, and V then each span a whole number of blocks), and
re-quantize to fp8.
When ``tp_size == ckpt_tp`` the checkpoint chunk *is* that layout, so a
plain chunk of both weight and scale suffices. Otherwise each rank's Q, K
and V rows are gathered from the chunks that hold them, dequantized with
the chunk's own scales, reordered, and re-quantized to fp8.
"""
assert tp_size <= num_kv_heads and num_kv_heads % tp_size == 0, (
"TP size must evenly split the number of KV heads."
assert num_heads % tp_size == 0, (
f"num_heads={num_heads} must be divisible by tp_size={tp_size}."
)
kv_heads_per_rank = num_kv_heads // tp_size
if kv_heads_per_rank == 1:
# One KV head per rank. The weights and scale can be trivially sharded
# without re-quantization.
w = w_full.chunk(tp_size, dim=0)[tp_rank]
s = s_full.chunk(tp_size, dim=0)[tp_rank]
return w, s
q_rows_per_group = (num_heads // num_kv_heads) * head_dim
k_rows_per_group = head_dim
v_rows_per_group = v_head_dim
rows_per_group = q_rows_per_group + k_rows_per_group + v_rows_per_group
scale_rows_per_group = s_full.shape[0] // num_kv_heads
qs, ks, vs = [], [], []
for g_idx in range(tp_rank * kv_heads_per_rank, (tp_rank + 1) * kv_heads_per_rank):
row_start = g_idx * rows_per_group
scale_row_start = g_idx * scale_rows_per_group
# Dequantize this group's weights.
w_g = w_full[row_start : row_start + rows_per_group].to(torch.float32)
s_g = s_full[scale_row_start : scale_row_start + scale_rows_per_group].to(
torch.float32
if ckpt_tp <= 0 or num_heads % ckpt_tp or num_kv_heads % ckpt_tp:
raise ValueError(
f"fused qkv_proj is pre-sharded at {ckpt_tp} chunks, which do not "
f"divide num_heads={num_heads} / num_kv_heads={num_kv_heads}."
)
s_g_expanded = s_g.repeat_interleave(block, dim=0).repeat_interleave(
block, dim=1
)[:rows_per_group]
w_g_dequant = w_g * s_g_expanded
# Track the dequantized q, k, and v weights separately.
qs.append(w_g_dequant[:q_rows_per_group])
ks.append(w_g_dequant[q_rows_per_group : q_rows_per_group + k_rows_per_group])
vs.append(w_g_dequant[q_rows_per_group + k_rows_per_group :])
# Combine the q, k, and v weights into the following layout:
# [Q_1, Q_2, .., Q_g, K_1, K_2, ..., K_g, V_1, V_2, ..., V_g]
grouped = torch.cat([torch.cat(qs), torch.cat(ks), torch.cat(vs)], dim=0)
# Quantize back to fp8.
return scaled_quantize(
grouped, GroupShape(block, block), w_full.dtype, compute_dtype=torch.float32
# When there are fewer KV heads than ranks, vLLM replicates them
# (`num_kv_head_replicas`) and rank r owns KV head r // replicas, which
# keeps every Q head grouped with the KV head it attends to.
if tp_size <= num_kv_heads:
assert num_kv_heads % tp_size == 0, (
f"num_kv_heads={num_kv_heads} must be divisible by tp_size={tp_size}."
)
kv_head_ids = list(
range(
tp_rank * (num_kv_heads // tp_size),
(tp_rank + 1) * (num_kv_heads // tp_size),
)
)
else:
assert tp_size % num_kv_heads == 0, (
f"tp_size={tp_size} must be divisible by num_kv_heads={num_kv_heads}."
)
kv_head_ids = [tp_rank // (tp_size // num_kv_heads)]
q_head_ids = list(
range(tp_rank * (num_heads // tp_size), (tp_rank + 1) * (num_heads // tp_size))
)
rows_per_chunk = w_full.shape[0] // ckpt_tp
q_per_chunk = (num_heads // ckpt_tp) * head_dim
k_per_chunk = (num_kv_heads // ckpt_tp) * head_dim
v_per_chunk = (num_kv_heads // ckpt_tp) * v_head_dim
if q_per_chunk + k_per_chunk + v_per_chunk != rows_per_chunk:
raise ValueError(
f"fused qkv_proj has {w_full.shape[0]} rows, not {ckpt_tp} chunks of "
f"{q_per_chunk} Q + {k_per_chunk} K + {v_per_chunk} V rows."
)
# The scales are tiled per chunk; they collapse to one continuous grid when
# a chunk is a whole number of blocks (the SWA chunks are: 3712 = 29 * 128).
chunk_scale_rows = cdiv(rows_per_chunk, block)
rows = torch.arange(w_full.shape[0])
per_chunk_scales = s_full.shape[0] == ckpt_tp * chunk_scale_rows
if per_chunk_scales:
scale_index = (rows // rows_per_chunk) * chunk_scale_rows + (
rows % rows_per_chunk
) // block
elif s_full.shape[0] == cdiv(w_full.shape[0], block):
scale_index = rows // block
else:
raise ValueError(
f"fused qkv_proj scale has {s_full.shape[0]} rows, expected either "
f"{ckpt_tp * chunk_scale_rows} ({ckpt_tp} chunks of "
f"{chunk_scale_rows} rows) or {cdiv(w_full.shape[0], block)} "
f"(one continuous grid)"
)
if tp_size == ckpt_tp and per_chunk_scales:
# One checkpoint chunk per rank: already [Q | K | V] for that rank, and
# its scale rows line up with ceil(rows_per_chunk / block).
return (
w_full.chunk(ckpt_tp, dim=0)[tp_rank],
s_full.chunk(ckpt_tp, dim=0)[tp_rank],
)
# Gather this rank's Q, K and V rows from the chunks that hold them.
q_heads_per_chunk = num_heads // ckpt_tp
kv_heads_per_chunk = num_kv_heads // ckpt_tp
head_rows = torch.arange(head_dim)
v_head_rows = torch.arange(v_head_dim)
row_index: list[torch.Tensor] = []
for head in q_head_ids:
chunk = head // q_heads_per_chunk
row_index.append(
chunk * rows_per_chunk + (head % q_heads_per_chunk) * head_dim + head_rows
)
for head in kv_head_ids:
chunk = head // kv_heads_per_chunk
row_index.append(
chunk * rows_per_chunk
+ q_per_chunk
+ (head % kv_heads_per_chunk) * head_dim
+ head_rows
)
for head in kv_head_ids:
chunk = head // kv_heads_per_chunk
row_index.append(
chunk * rows_per_chunk
+ q_per_chunk
+ k_per_chunk
+ (head % kv_heads_per_chunk) * v_head_dim
+ v_head_rows
)
index = torch.cat(row_index)
grouped = w_full[index].to(torch.float32) * s_full[
scale_index[index]
].repeat_interleave(block, dim=1)
return _requantize_fp8(grouped, index.numel(), block, w_full.dtype)
@support_torch_compile
class MiMoV2Model(nn.Module, EagleModelMixin):
@@ -819,6 +903,8 @@ class MiMoV2Model(nn.Module, EagleModelMixin):
v_head_dim=attn.v_head_dim,
tp_rank=tp_rank,
tp_size=tp_size,
# The fused qkv_proj is pre-sharded for this many ranks.
ckpt_tp=self.config.num_key_value_heads,
)
sharded = {"weight": w_rank, "weight_scale_inv": s_rank}
for kind, tensor in sharded.items():
+49 -8
View File
@@ -45,7 +45,7 @@ from .interfaces import (
SupportsMultiModal,
_require_is_multimodal,
)
from .mimo_v2 import MiMoV2Attention, MiMoV2MLP
from .mimo_v2 import MiMoV2Attention, MiMoV2MLP, _shard_fp8_qkv_proj
from .utils import _merge_multimodal_embeddings, maybe_prefix
# MiMo-V2 checkpoints contain multiple MTP layers, but vLLM currently supports
@@ -266,6 +266,7 @@ class MiMoV2MTP(nn.Module):
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
pending_qkv_proj: dict[str, dict[str, torch.Tensor]] = {}
for name, loaded_weight in weights:
if "rotary_emb.inv_freq" in name:
@@ -279,14 +280,54 @@ class MiMoV2MTP(nn.Module):
):
continue
# Support fused qkv_proj checkpoint (Pro format).
# The checkpoint is stored pre-sharded for TP=8 as
# [Q_rank0, K_rank0, V_rank0, Q_rank1, ...], so splitting along
# dim 0 with chunk(tp_size) gives each rank its Q+K+V slice for
# both the FP8 weight and the block weight_scale_inv. This matches
# how the main model loads the same layout.
# Fused qkv_proj: the fp8 weight and its block scales arrive as
# separate tensors and are pre-sharded at `config.num_key_value_heads`
# chunks of [Q_c | K_c | V_c] (see _shard_fp8_qkv_proj for the
# layout). Shard them together: a plain chunk(tp_size) only happens
# to be right when tp_size is that chunk count *and* the scales are
# tiled per chunk (they are 4 x 29 = 116 rows for MiMo-V2.5, not
# tp_size slices of an even split).
if "qkv_proj" in name:
if name in params_dict:
is_fp8_weight = (
name.endswith("qkv_proj.weight")
and loaded_weight.dtype == torch.float8_e4m3fn
)
is_fp8_scale = name.endswith("qkv_proj.weight_scale_inv")
if is_fp8_weight or is_fp8_scale:
prefix, kind = name.rsplit(".", 1)
if (
f"{prefix}.weight" not in params_dict
or f"{prefix}.weight_scale_inv" not in params_dict
):
continue
entry = pending_qkv_proj.setdefault(prefix, {})
entry[kind] = loaded_weight
if "weight" not in entry or "weight_scale_inv" not in entry:
# Waiting for the other half of the fused projection.
continue
del pending_qkv_proj[prefix]
# Geometry comes from the module the weights belong to.
attn = self.get_submodule(prefix.rsplit(".", 1)[0])
w_rank, s_rank = _shard_fp8_qkv_proj(
entry["weight"],
entry["weight_scale_inv"],
num_heads=attn.total_num_heads,
num_kv_heads=attn.total_num_kv_heads,
head_dim=attn.head_dim,
v_head_dim=attn.v_head_dim,
tp_rank=tp_rank,
tp_size=tp_size,
ckpt_tp=self.config.num_key_value_heads,
)
for k, tensor in (("weight", w_rank), ("weight_scale_inv", s_rank)):
param = params_dict[f"{prefix}.{k}"]
if tensor.shape[0] > param.shape[0]:
tensor = tensor[: param.shape[0]]
default_weight_loader(param, tensor)
loaded_params.add(f"{prefix}.{k}")
elif name in params_dict:
# Non-fp8 fused checkpoint: unchanged behaviour.
param = params_dict[name]
loaded_weight = loaded_weight.chunk(tp_size, dim=0)[tp_rank]
default_weight_loader(param, loaded_weight)