diff --git a/tests/models/quantization/test_mimo_v2_qkv_shard.py b/tests/models/quantization/test_mimo_v2_qkv_shard.py new file mode 100644 index 0000000000..91eb036bc7 --- /dev/null +++ b/tests/models/quantization/test_mimo_v2_qkv_shard.py @@ -0,0 +1,225 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit test for MiMo-V2 fused fp8 ``qkv_proj`` sharding. + +The checkpoint stores the fused QKV pre-sharded at ``num_key_value_heads`` +chunks (``ckpt_tp``), each holding that slice's ``[Q_c | K_c | V_c]`` rows with +its own block scale grid. ``ckpt_tp`` is not the layer's KV-head count: a +MiMo-V2.5 SWA layer has 8 KV heads but 4 chunks of 3712 rows (= 29 blocks), so +each chunk carries two KV heads. Reading the scales as one slice per KV head +expands 14 x 128 = 1792 rows for an 1856-row slice and fails. + +These tests build such checkpoints for the GA, SWA and Pro geometries and check +that every rank gets exactly the weights it should own, for TP sizes below, +equal to and above ``ckpt_tp`` (the last case replicating KV heads). +""" + +import pytest +import torch + +from vllm.model_executor.models.mimo_v2 import _shard_fp8_qkv_proj +from vllm.utils.math_utils import cdiv + +pytestmark = pytest.mark.cpu_test + +BLOCK = 128 +FP8_DTYPE = torch.float8_e4m3fn +FP8_MAX = torch.finfo(FP8_DTYPE).max +COLS = 128 # one column block is enough to exercise the row mapping +# fp8 e4m3 has 3 mantissa bits, so a single block-quantization round trip costs +# ~2-3% relative L2; a wrong row -> scale pairing costs >= 10%. +TOL = 0.06 + +# (num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp): MiMo-V2.5 GA, its +# SWA/MTP layers, and MiMo-V2.5-Pro. +GEOMETRIES = { + "ga": (64, 4, 192, 128, 4), + "swa": (64, 8, 192, 128, 4), + "pro": (128, 8, 192, 128, 8), +} + + +def _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp): + return ( + num_heads // ckpt_tp * head_dim + + num_kv_heads // ckpt_tp * head_dim + + num_kv_heads // ckpt_tp * v_head_dim + ) + + +def _quantize_chunks(truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp): + """Block-quantize each checkpoint chunk on its own grid, like the export.""" + rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp) + weights, scales = [], [] + for chunk in range(ckpt_tp): + rows = truth[chunk * rows_per_chunk : (chunk + 1) * rows_per_chunk] + quantized = torch.zeros_like(rows) + scale = torch.zeros( + cdiv(rows_per_chunk, BLOCK), rows.shape[1] // BLOCK, dtype=torch.float32 + ) + for row in range(0, rows_per_chunk, BLOCK): + for col in range(0, rows.shape[1], BLOCK): + block = rows[row : row + BLOCK, col : col + BLOCK] + amax = block.abs().max().clamp(min=1e-12) / FP8_MAX + scale[row // BLOCK, col // BLOCK] = amax + quantized[row : row + BLOCK, col : col + BLOCK] = block / amax + weights.append(quantized.to(FP8_DTYPE)) + scales.append(scale) + return torch.cat(weights, dim=0), torch.cat(scales, dim=0) + + +def _owned_rows( + truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp, tp_rank, tp_size +): + """Rows the rank must own, in the forward's `[Q | K | V]` order.""" + rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp) + q_per_chunk = num_heads // ckpt_tp * head_dim + k_per_chunk = num_kv_heads // ckpt_tp * head_dim + kv_heads: list[int] = [] + if tp_size <= num_kv_heads: + kv_heads = list( + range( + tp_rank * (num_kv_heads // tp_size), + (tp_rank + 1) * (num_kv_heads // tp_size), + ) + ) + else: + kv_heads = [tp_rank // (tp_size // num_kv_heads)] + index = [] + for head in range( + tp_rank * (num_heads // tp_size), (tp_rank + 1) * (num_heads // tp_size) + ): + chunk, offset = divmod(head, num_heads // ckpt_tp) + start = chunk * rows_per_chunk + offset * head_dim + index += list(range(start, start + head_dim)) + for head in kv_heads: + chunk, offset = divmod(head, num_kv_heads // ckpt_tp) + start = chunk * rows_per_chunk + q_per_chunk + offset * head_dim + index += list(range(start, start + head_dim)) + for head in kv_heads: + chunk, offset = divmod(head, num_kv_heads // ckpt_tp) + start = chunk * rows_per_chunk + q_per_chunk + k_per_chunk + offset * v_head_dim + index += list(range(start, start + v_head_dim)) + return truth[torch.tensor(index)] + + +def _dequantize(weight, scale): + """What the kernel computes: scale[row // block] applied per weight row.""" + per_row = scale.repeat_interleave(BLOCK, dim=1) + return weight.to(torch.float32) * per_row[torch.arange(weight.shape[0]) // BLOCK] + + +def _relative_l2(got, expected): + return ((got - expected).norm() / expected.norm()).item() + + +@pytest.mark.parametrize("geometry", list(GEOMETRIES)) +@pytest.mark.parametrize("tp_size", [1, 2, 4, 8]) +def test_fused_qkv_proj_sharding(geometry, tp_size): + num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp = GEOMETRIES[geometry] + if num_heads % tp_size or (tp_size > num_kv_heads and tp_size % num_kv_heads): + pytest.skip("TP size does not divide the head counts") + + rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp) + truth = torch.randn(ckpt_tp * rows_per_chunk, COLS) + for chunk in range(ckpt_tp): # per-chunk magnitude spread, as in the export + truth[chunk * rows_per_chunk : (chunk + 1) * rows_per_chunk] *= 1.0 + chunk + weight, scale = _quantize_chunks( + truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp + ) + + kv_per_rank = (num_kv_heads // tp_size) if tp_size <= num_kv_heads else 1 + rows_rank = (num_heads // tp_size) * head_dim + kv_per_rank * ( + head_dim + v_head_dim + ) + for tp_rank in range(tp_size): + w_rank, s_rank = _shard_fp8_qkv_proj( + weight, + scale, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + v_head_dim=v_head_dim, + tp_rank=tp_rank, + tp_size=tp_size, + ckpt_tp=ckpt_tp, + ) + assert w_rank.shape == (rows_rank, COLS) + assert s_rank.shape == (cdiv(rows_rank, BLOCK), scale.shape[1]) + expected = _owned_rows( + truth, + num_heads, + num_kv_heads, + head_dim, + v_head_dim, + ckpt_tp, + tp_rank, + tp_size, + ) + assert _relative_l2(_dequantize(w_rank, s_rank), expected) <= TOL + + +@pytest.mark.parametrize("geometry", ["ga", "swa"]) +@pytest.mark.parametrize("tp_size", [1, 2, 4]) +def test_exact_chunk_matches_checkpoint(geometry, tp_size): + """At tp_size <= ckpt_tp the chunk count is the only thing that matters.""" + num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp = GEOMETRIES[geometry] + rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp) + truth = torch.randn(ckpt_tp * rows_per_chunk, COLS) + weight, scale = _quantize_chunks( + truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp + ) + if tp_size == ckpt_tp: + w_rank, s_rank = _shard_fp8_qkv_proj( + weight, + scale, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + v_head_dim=v_head_dim, + tp_rank=0, + tp_size=tp_size, + ckpt_tp=ckpt_tp, + ) + # The checkpoint chunk *is* the rank's slice: no re-quantization. + # (torch.equal has no fp8 kernel, so compare the underlying bytes.) + expected_w = weight.chunk(tp_size, dim=0)[0] + assert torch.equal(w_rank.view(torch.uint8), expected_w.view(torch.uint8)) + assert torch.equal(s_rank, scale.chunk(tp_size, dim=0)[0]) + + +def test_wrong_chunk_count_is_detected(): + """Reading SWA scales as one slice per KV head must not look correct. + + This is the regression the fix addresses: 116 scale rows are 4 x 29 (29 = + ceil(3712 / 128)), not 8 x 15, so ``ckpt_tp = num_kv_heads`` pairs every + weight row with the wrong scale. + """ + num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp = GEOMETRIES["swa"] + rows_per_chunk = _chunk_rows(num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp) + truth = torch.randn(ckpt_tp * rows_per_chunk, COLS) + for chunk in range(ckpt_tp): + truth[chunk * rows_per_chunk : (chunk + 1) * rows_per_chunk] *= 1.0 + chunk + weight, scale = _quantize_chunks( + truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp + ) + assert scale.shape[0] == ckpt_tp * cdiv(rows_per_chunk, BLOCK) + assert scale.shape[0] != num_kv_heads * cdiv(rows_per_chunk, BLOCK) + + kw = dict( + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_dim=head_dim, + v_head_dim=v_head_dim, + tp_rank=0, + tp_size=1, + ) + correct = _shard_fp8_qkv_proj(weight, scale, ckpt_tp=ckpt_tp, **kw) + wrong = _shard_fp8_qkv_proj(weight, scale, ckpt_tp=num_kv_heads, **kw) + assert correct[0].shape == wrong[0].shape + + expected = _owned_rows( + truth, num_heads, num_kv_heads, head_dim, v_head_dim, ckpt_tp, 0, 1 + ) + assert _relative_l2(_dequantize(*correct), expected) <= TOL + assert _relative_l2(_dequantize(*wrong), expected) > 5 * TOL diff --git a/vllm/model_executor/models/mimo_v2.py b/vllm/model_executor/models/mimo_v2.py index ec67d72cf2..be665e7e06 100644 --- a/vllm/model_executor/models/mimo_v2.py +++ b/vllm/model_executor/models/mimo_v2.py @@ -50,6 +50,7 @@ from vllm.model_executor.model_loader.weight_utils import ( ) from vllm.model_executor.models.utils import sequence_parallel_chunk from vllm.sequence import IntermediateTensors +from vllm.utils.math_utils import cdiv from vllm.v1.attention.backend import AttentionType from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -460,6 +461,28 @@ class MiMoV2FlashDecoderLayer(nn.Module): return self.config.hybrid_layer_pattern[self.layer_id] == 1 +def _requantize_fp8( + grouped: torch.Tensor, rows_rank: int, block: int, dtype: torch.dtype +) -> tuple[torch.Tensor, torch.Tensor]: + """Block-quantize a rank's ``[Q | K | V]`` rows back to fp8. + + A rank's rows need not end on a block boundary (a single 1856-row slice is + 14.5 blocks) while ``scaled_quantize`` requires both dims to be multiples of + the block size, so pad the tail with zeros: zero rows cannot raise a block's + amax, so every scale -- and the number of scale rows -- is unchanged. The + padding is dropped again here. + """ + padded = cdiv(rows_rank, block) * block + if padded != rows_rank: + grouped = torch.cat( + [grouped, grouped.new_zeros(padded - rows_rank, grouped.shape[1])], dim=0 + ) + w_rank, s_rank = scaled_quantize( + grouped, GroupShape(block, block), dtype, compute_dtype=torch.float32 + ) + return w_rank[:rows_rank], s_rank + + def _shard_fp8_qkv_proj( w_full: torch.Tensor, s_full: torch.Tensor, @@ -469,75 +492,136 @@ def _shard_fp8_qkv_proj( v_head_dim: int, tp_rank: int, tp_size: int, + ckpt_tp: int, block: int = 128, ) -> tuple[torch.Tensor, torch.Tensor]: """Shard the fp8 qkv_proj weights for ``tp_rank``. - The checkpoint stores the fused QKV as ``num_kv_heads`` contiguous groups - (one per KV head; ``n`` below), each ordered ``[Q | K | V]``: + The checkpoint stores the fused QKV pre-sharded for ``ckpt_tp`` ranks (the + model config's ``num_key_value_heads``), each chunk holding that slice's + Q, K and V rows: - [Q_1 | K_1 | V_1 | Q_2 | K_2 | V_2 | ... | Q_n | K_n | V_n] + [Q_0 | K_0 | V_0 | Q_1 | K_1 | V_1 | ... | Q_n | K_n | V_n] (n = ckpt_tp) - Per group, Q has ``(num_heads / num_kv_heads) * head_dim`` rows, K has - ``head_dim`` rows, and V has ``v_head_dim`` rows. + Per chunk, Q has ``(num_heads / ckpt_tp) * head_dim`` rows, K has + ``(num_kv_heads / ckpt_tp) * head_dim`` rows, and V has + ``(num_kv_heads / ckpt_tp) * v_head_dim`` rows, and the fp8 block scales + are tiled per chunk too (``ceil(rows_per_chunk / block)`` rows each). - Each TP rank owns ``g = num_kv_heads / tp_size`` of these groups, and the - forward expects them de-interleaved into a single Q, K, and V block: + ``ckpt_tp`` is not the layer's KV-head count: a MiMo-V2.5 SWA layer has 8 + KV heads over 4 chunks, so each chunk carries two KV heads (3712 rows = 29 + blocks) while a GA layer has 4 KV heads over 4 chunks (3392 rows each). + + The forward expects each rank's slice de-interleaved: [Q_1 | Q_2 | ... | Q_g | K_1 | K_2 | ... | K_g | V_1 | V_2 | ... | V_g] - When ``g == 1`` the rank's slice is already ``[Q | K | V]``, so a plain - chunk suffices. When ``g > 1`` we cannot reach the de-interleaved layout by - re-permuting the fp8 block scales: each scale covers a 128-row block, and - since K is 192 rows (1.5 blocks) a block straddles the K/V boundary, so no - whole-block permutation produces it. Instead we dequantize this rank's - groups to float (dropping the block constraint), reorder the rows into the - layout above (Q, K, and V then each span a whole number of blocks), and - re-quantize to fp8. + When ``tp_size == ckpt_tp`` the checkpoint chunk *is* that layout, so a + plain chunk of both weight and scale suffices. Otherwise each rank's Q, K + and V rows are gathered from the chunks that hold them, dequantized with + the chunk's own scales, reordered, and re-quantized to fp8. """ - assert tp_size <= num_kv_heads and num_kv_heads % tp_size == 0, ( - "TP size must evenly split the number of KV heads." + assert num_heads % tp_size == 0, ( + f"num_heads={num_heads} must be divisible by tp_size={tp_size}." ) - - kv_heads_per_rank = num_kv_heads // tp_size - if kv_heads_per_rank == 1: - # One KV head per rank. The weights and scale can be trivially sharded - # without re-quantization. - w = w_full.chunk(tp_size, dim=0)[tp_rank] - s = s_full.chunk(tp_size, dim=0)[tp_rank] - return w, s - - q_rows_per_group = (num_heads // num_kv_heads) * head_dim - k_rows_per_group = head_dim - v_rows_per_group = v_head_dim - rows_per_group = q_rows_per_group + k_rows_per_group + v_rows_per_group - scale_rows_per_group = s_full.shape[0] // num_kv_heads - qs, ks, vs = [], [], [] - for g_idx in range(tp_rank * kv_heads_per_rank, (tp_rank + 1) * kv_heads_per_rank): - row_start = g_idx * rows_per_group - scale_row_start = g_idx * scale_rows_per_group - # Dequantize this group's weights. - w_g = w_full[row_start : row_start + rows_per_group].to(torch.float32) - s_g = s_full[scale_row_start : scale_row_start + scale_rows_per_group].to( - torch.float32 + if ckpt_tp <= 0 or num_heads % ckpt_tp or num_kv_heads % ckpt_tp: + raise ValueError( + f"fused qkv_proj is pre-sharded at {ckpt_tp} chunks, which do not " + f"divide num_heads={num_heads} / num_kv_heads={num_kv_heads}." ) - s_g_expanded = s_g.repeat_interleave(block, dim=0).repeat_interleave( - block, dim=1 - )[:rows_per_group] - w_g_dequant = w_g * s_g_expanded - # Track the dequantized q, k, and v weights separately. - qs.append(w_g_dequant[:q_rows_per_group]) - ks.append(w_g_dequant[q_rows_per_group : q_rows_per_group + k_rows_per_group]) - vs.append(w_g_dequant[q_rows_per_group + k_rows_per_group :]) - - # Combine the q, k, and v weights into the following layout: - # [Q_1, Q_2, .., Q_g, K_1, K_2, ..., K_g, V_1, V_2, ..., V_g] - grouped = torch.cat([torch.cat(qs), torch.cat(ks), torch.cat(vs)], dim=0) - # Quantize back to fp8. - return scaled_quantize( - grouped, GroupShape(block, block), w_full.dtype, compute_dtype=torch.float32 + # When there are fewer KV heads than ranks, vLLM replicates them + # (`num_kv_head_replicas`) and rank r owns KV head r // replicas, which + # keeps every Q head grouped with the KV head it attends to. + if tp_size <= num_kv_heads: + assert num_kv_heads % tp_size == 0, ( + f"num_kv_heads={num_kv_heads} must be divisible by tp_size={tp_size}." + ) + kv_head_ids = list( + range( + tp_rank * (num_kv_heads // tp_size), + (tp_rank + 1) * (num_kv_heads // tp_size), + ) + ) + else: + assert tp_size % num_kv_heads == 0, ( + f"tp_size={tp_size} must be divisible by num_kv_heads={num_kv_heads}." + ) + kv_head_ids = [tp_rank // (tp_size // num_kv_heads)] + q_head_ids = list( + range(tp_rank * (num_heads // tp_size), (tp_rank + 1) * (num_heads // tp_size)) ) + rows_per_chunk = w_full.shape[0] // ckpt_tp + q_per_chunk = (num_heads // ckpt_tp) * head_dim + k_per_chunk = (num_kv_heads // ckpt_tp) * head_dim + v_per_chunk = (num_kv_heads // ckpt_tp) * v_head_dim + if q_per_chunk + k_per_chunk + v_per_chunk != rows_per_chunk: + raise ValueError( + f"fused qkv_proj has {w_full.shape[0]} rows, not {ckpt_tp} chunks of " + f"{q_per_chunk} Q + {k_per_chunk} K + {v_per_chunk} V rows." + ) + + # The scales are tiled per chunk; they collapse to one continuous grid when + # a chunk is a whole number of blocks (the SWA chunks are: 3712 = 29 * 128). + chunk_scale_rows = cdiv(rows_per_chunk, block) + rows = torch.arange(w_full.shape[0]) + per_chunk_scales = s_full.shape[0] == ckpt_tp * chunk_scale_rows + if per_chunk_scales: + scale_index = (rows // rows_per_chunk) * chunk_scale_rows + ( + rows % rows_per_chunk + ) // block + elif s_full.shape[0] == cdiv(w_full.shape[0], block): + scale_index = rows // block + else: + raise ValueError( + f"fused qkv_proj scale has {s_full.shape[0]} rows, expected either " + f"{ckpt_tp * chunk_scale_rows} ({ckpt_tp} chunks of " + f"{chunk_scale_rows} rows) or {cdiv(w_full.shape[0], block)} " + f"(one continuous grid)" + ) + + if tp_size == ckpt_tp and per_chunk_scales: + # One checkpoint chunk per rank: already [Q | K | V] for that rank, and + # its scale rows line up with ceil(rows_per_chunk / block). + return ( + w_full.chunk(ckpt_tp, dim=0)[tp_rank], + s_full.chunk(ckpt_tp, dim=0)[tp_rank], + ) + + # Gather this rank's Q, K and V rows from the chunks that hold them. + q_heads_per_chunk = num_heads // ckpt_tp + kv_heads_per_chunk = num_kv_heads // ckpt_tp + head_rows = torch.arange(head_dim) + v_head_rows = torch.arange(v_head_dim) + row_index: list[torch.Tensor] = [] + for head in q_head_ids: + chunk = head // q_heads_per_chunk + row_index.append( + chunk * rows_per_chunk + (head % q_heads_per_chunk) * head_dim + head_rows + ) + for head in kv_head_ids: + chunk = head // kv_heads_per_chunk + row_index.append( + chunk * rows_per_chunk + + q_per_chunk + + (head % kv_heads_per_chunk) * head_dim + + head_rows + ) + for head in kv_head_ids: + chunk = head // kv_heads_per_chunk + row_index.append( + chunk * rows_per_chunk + + q_per_chunk + + k_per_chunk + + (head % kv_heads_per_chunk) * v_head_dim + + v_head_rows + ) + index = torch.cat(row_index) + grouped = w_full[index].to(torch.float32) * s_full[ + scale_index[index] + ].repeat_interleave(block, dim=1) + return _requantize_fp8(grouped, index.numel(), block, w_full.dtype) + @support_torch_compile class MiMoV2Model(nn.Module, EagleModelMixin): @@ -819,6 +903,8 @@ class MiMoV2Model(nn.Module, EagleModelMixin): v_head_dim=attn.v_head_dim, tp_rank=tp_rank, tp_size=tp_size, + # The fused qkv_proj is pre-sharded for this many ranks. + ckpt_tp=self.config.num_key_value_heads, ) sharded = {"weight": w_rank, "weight_scale_inv": s_rank} for kind, tensor in sharded.items(): diff --git a/vllm/model_executor/models/mimo_v2_mtp.py b/vllm/model_executor/models/mimo_v2_mtp.py index b08ba229f7..6fd8309350 100644 --- a/vllm/model_executor/models/mimo_v2_mtp.py +++ b/vllm/model_executor/models/mimo_v2_mtp.py @@ -45,7 +45,7 @@ from .interfaces import ( SupportsMultiModal, _require_is_multimodal, ) -from .mimo_v2 import MiMoV2Attention, MiMoV2MLP +from .mimo_v2 import MiMoV2Attention, MiMoV2MLP, _shard_fp8_qkv_proj from .utils import _merge_multimodal_embeddings, maybe_prefix # MiMo-V2 checkpoints contain multiple MTP layers, but vLLM currently supports @@ -266,6 +266,7 @@ class MiMoV2MTP(nn.Module): params_dict = dict(self.named_parameters()) loaded_params: set[str] = set() + pending_qkv_proj: dict[str, dict[str, torch.Tensor]] = {} for name, loaded_weight in weights: if "rotary_emb.inv_freq" in name: @@ -279,14 +280,54 @@ class MiMoV2MTP(nn.Module): ): continue - # Support fused qkv_proj checkpoint (Pro format). - # The checkpoint is stored pre-sharded for TP=8 as - # [Q_rank0, K_rank0, V_rank0, Q_rank1, ...], so splitting along - # dim 0 with chunk(tp_size) gives each rank its Q+K+V slice for - # both the FP8 weight and the block weight_scale_inv. This matches - # how the main model loads the same layout. + # Fused qkv_proj: the fp8 weight and its block scales arrive as + # separate tensors and are pre-sharded at `config.num_key_value_heads` + # chunks of [Q_c | K_c | V_c] (see _shard_fp8_qkv_proj for the + # layout). Shard them together: a plain chunk(tp_size) only happens + # to be right when tp_size is that chunk count *and* the scales are + # tiled per chunk (they are 4 x 29 = 116 rows for MiMo-V2.5, not + # tp_size slices of an even split). if "qkv_proj" in name: - if name in params_dict: + is_fp8_weight = ( + name.endswith("qkv_proj.weight") + and loaded_weight.dtype == torch.float8_e4m3fn + ) + is_fp8_scale = name.endswith("qkv_proj.weight_scale_inv") + if is_fp8_weight or is_fp8_scale: + prefix, kind = name.rsplit(".", 1) + if ( + f"{prefix}.weight" not in params_dict + or f"{prefix}.weight_scale_inv" not in params_dict + ): + continue + entry = pending_qkv_proj.setdefault(prefix, {}) + entry[kind] = loaded_weight + if "weight" not in entry or "weight_scale_inv" not in entry: + # Waiting for the other half of the fused projection. + continue + del pending_qkv_proj[prefix] + + # Geometry comes from the module the weights belong to. + attn = self.get_submodule(prefix.rsplit(".", 1)[0]) + w_rank, s_rank = _shard_fp8_qkv_proj( + entry["weight"], + entry["weight_scale_inv"], + num_heads=attn.total_num_heads, + num_kv_heads=attn.total_num_kv_heads, + head_dim=attn.head_dim, + v_head_dim=attn.v_head_dim, + tp_rank=tp_rank, + tp_size=tp_size, + ckpt_tp=self.config.num_key_value_heads, + ) + for k, tensor in (("weight", w_rank), ("weight_scale_inv", s_rank)): + param = params_dict[f"{prefix}.{k}"] + if tensor.shape[0] > param.shape[0]: + tensor = tensor[: param.shape[0]] + default_weight_loader(param, tensor) + loaded_params.add(f"{prefix}.{k}") + elif name in params_dict: + # Non-fp8 fused checkpoint: unchanged behaviour. param = params_dict[name] loaded_weight = loaded_weight.chunk(tp_size, dim=0)[tp_rank] default_weight_loader(param, loaded_weight)