[Models] Fix MiMo v2.x QKV TP sharding + FP4 support (#45200)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
64833f8158
commit
b5adb027ad
@@ -106,6 +106,7 @@ class Fp8Config(QuantizationConfig):
|
||||
activation_scheme: str = "dynamic",
|
||||
ignored_layers: list[str] | None = None,
|
||||
weight_block_size: list[int] | None = None,
|
||||
store_dtype: str | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
@@ -115,6 +116,7 @@ class Fp8Config(QuantizationConfig):
|
||||
raise ValueError(f"Unsupported activation scheme {activation_scheme}")
|
||||
self.activation_scheme = activation_scheme
|
||||
self.ignored_layers = ignored_layers or []
|
||||
self.store_dtype = store_dtype
|
||||
if weight_block_size is not None:
|
||||
if not is_checkpoint_fp8_serialized:
|
||||
raise ValueError(
|
||||
@@ -162,6 +164,7 @@ class Fp8Config(QuantizationConfig):
|
||||
activation_scheme = cls.get_from_keys(config, ["activation_scheme"])
|
||||
ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None)
|
||||
weight_block_size = cls.get_from_keys_or(config, ["weight_block_size"], None)
|
||||
store_dtype = cls.get_from_keys_or(config, ["store_dtype"], None)
|
||||
if not ignored_layers:
|
||||
ignored_layers = cls.get_from_keys_or(
|
||||
config, ["modules_to_not_convert"], None
|
||||
@@ -171,6 +174,7 @@ class Fp8Config(QuantizationConfig):
|
||||
activation_scheme=activation_scheme,
|
||||
ignored_layers=ignored_layers,
|
||||
weight_block_size=weight_block_size,
|
||||
store_dtype=store_dtype,
|
||||
)
|
||||
|
||||
def get_quant_method(
|
||||
@@ -198,6 +202,12 @@ class Fp8Config(QuantizationConfig):
|
||||
fused_mapping=self.packed_modules_mapping,
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(layer.moe_config)
|
||||
if self.store_dtype == "mxfp4":
|
||||
from vllm.model_executor.layers.quantization.mxfp4 import (
|
||||
Mxfp4MoEMethod,
|
||||
)
|
||||
|
||||
return Mxfp4MoEMethod(layer.moe_config)
|
||||
if self.is_checkpoint_fp8_serialized:
|
||||
moe_quant_method = Fp8MoEMethod(self, layer)
|
||||
else:
|
||||
|
||||
@@ -35,6 +35,10 @@ from vllm.model_executor.layers.linear import (
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
scaled_quantize,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
@@ -455,6 +459,85 @@ class MiMoV2FlashDecoderLayer(nn.Module):
|
||||
return self.config.hybrid_layer_pattern[self.layer_id] == 1
|
||||
|
||||
|
||||
def _shard_fp8_qkv_proj(
|
||||
w_full: torch.Tensor,
|
||||
s_full: torch.Tensor,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
v_head_dim: int,
|
||||
tp_rank: int,
|
||||
tp_size: int,
|
||||
block: int = 128,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Shard the fp8 qkv_proj weights for ``tp_rank``.
|
||||
|
||||
The checkpoint stores the fused QKV as ``num_kv_heads`` contiguous groups
|
||||
(one per KV head; ``n`` below), each ordered ``[Q | K | V]``:
|
||||
|
||||
[Q_1 | K_1 | V_1 | Q_2 | K_2 | V_2 | ... | Q_n | K_n | V_n]
|
||||
|
||||
Per group, Q has ``(num_heads / num_kv_heads) * head_dim`` rows, K has
|
||||
``head_dim`` rows, and V has ``v_head_dim`` rows.
|
||||
|
||||
Each TP rank owns ``g = num_kv_heads / tp_size`` of these groups, and the
|
||||
forward expects them de-interleaved into a single Q, K, and V block:
|
||||
|
||||
[Q_1 | Q_2 | ... | Q_g | K_1 | K_2 | ... | K_g | V_1 | V_2 | ... | V_g]
|
||||
|
||||
When ``g == 1`` the rank's slice is already ``[Q | K | V]``, so a plain
|
||||
chunk suffices. When ``g > 1`` we cannot reach the de-interleaved layout by
|
||||
re-permuting the fp8 block scales: each scale covers a 128-row block, and
|
||||
since K is 192 rows (1.5 blocks) a block straddles the K/V boundary, so no
|
||||
whole-block permutation produces it. Instead we dequantize this rank's
|
||||
groups to float (dropping the block constraint), reorder the rows into the
|
||||
layout above (Q, K, and V then each span a whole number of blocks), and
|
||||
re-quantize to fp8.
|
||||
"""
|
||||
assert tp_size <= num_kv_heads and num_kv_heads % tp_size == 0, (
|
||||
"TP size must evenly split the number of KV heads."
|
||||
)
|
||||
|
||||
kv_heads_per_rank = num_kv_heads // tp_size
|
||||
if kv_heads_per_rank == 1:
|
||||
# One KV head per rank. The weights and scale can be trivially sharded
|
||||
# without re-quantization.
|
||||
w = w_full.chunk(tp_size, dim=0)[tp_rank]
|
||||
s = s_full.chunk(tp_size, dim=0)[tp_rank]
|
||||
return w, s
|
||||
|
||||
q_rows_per_group = (num_heads // num_kv_heads) * head_dim
|
||||
k_rows_per_group = head_dim
|
||||
v_rows_per_group = v_head_dim
|
||||
rows_per_group = q_rows_per_group + k_rows_per_group + v_rows_per_group
|
||||
scale_rows_per_group = s_full.shape[0] // num_kv_heads
|
||||
qs, ks, vs = [], [], []
|
||||
for g_idx in range(tp_rank * kv_heads_per_rank, (tp_rank + 1) * kv_heads_per_rank):
|
||||
row_start = g_idx * rows_per_group
|
||||
scale_row_start = g_idx * scale_rows_per_group
|
||||
# Dequantize this group's weights.
|
||||
w_g = w_full[row_start : row_start + rows_per_group].to(torch.float32)
|
||||
s_g = s_full[scale_row_start : scale_row_start + scale_rows_per_group].to(
|
||||
torch.float32
|
||||
)
|
||||
s_g_expanded = s_g.repeat_interleave(block, dim=0).repeat_interleave(
|
||||
block, dim=1
|
||||
)[:rows_per_group]
|
||||
w_g_dequant = w_g * s_g_expanded
|
||||
# Track the dequantized q, k, and v weights separately.
|
||||
qs.append(w_g_dequant[:q_rows_per_group])
|
||||
ks.append(w_g_dequant[q_rows_per_group : q_rows_per_group + k_rows_per_group])
|
||||
vs.append(w_g_dequant[q_rows_per_group + k_rows_per_group :])
|
||||
|
||||
# Combine the q, k, and v weights into the following layout:
|
||||
# [Q_1, Q_2, .., Q_g, K_1, K_2, ..., K_g, V_1, V_2, ..., V_g]
|
||||
grouped = torch.cat([torch.cat(qs), torch.cat(ks), torch.cat(vs)], dim=0)
|
||||
# Quantize back to fp8.
|
||||
return scaled_quantize(
|
||||
grouped, GroupShape(block, block), w_full.dtype, compute_dtype=torch.float32
|
||||
)
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class MiMoV2Model(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
@@ -561,6 +644,10 @@ class MiMoV2Model(nn.Module):
|
||||
params_dict = dict(self.named_parameters(remove_duplicate=False))
|
||||
loaded_params: set[str] = set()
|
||||
expert_params_mapping = self.get_expert_mapping()
|
||||
# Pro-format fused qkv_proj arrives as two tensors (weight and
|
||||
# weight_scale_inv). Store them per-layer so that they can be
|
||||
# sharded together.
|
||||
pending_fp8_qkv_proj: dict[str, dict[str, torch.Tensor]] = {}
|
||||
for name, loaded_weight in weights:
|
||||
if "rotary_emb.inv_freq" in name:
|
||||
continue
|
||||
@@ -604,11 +691,15 @@ class MiMoV2Model(nn.Module):
|
||||
if expert_matched:
|
||||
continue
|
||||
# Support fused qkv_proj checkpoint (Pro format)
|
||||
if "qkv_proj" in name:
|
||||
if name in params_dict:
|
||||
param = params_dict[name]
|
||||
loaded_weight = loaded_weight.chunk(tp_size, dim=0)[tp_rank]
|
||||
default_weight_loader(param, loaded_weight)
|
||||
if self._try_load_fp8_qkv_proj(
|
||||
name,
|
||||
loaded_weight,
|
||||
pending_fp8_qkv_proj,
|
||||
params_dict,
|
||||
loaded_params,
|
||||
tp_rank,
|
||||
tp_size,
|
||||
):
|
||||
continue
|
||||
stacked_matched = False
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
@@ -666,6 +757,70 @@ class MiMoV2Model(nn.Module):
|
||||
|
||||
return loaded_params
|
||||
|
||||
def _try_load_fp8_qkv_proj(
|
||||
self,
|
||||
name: str,
|
||||
tensor: torch.Tensor,
|
||||
fp8_qkv_proj_dict: dict[str, dict[str, torch.Tensor]],
|
||||
params_dict: dict[str, torch.nn.Parameter],
|
||||
loaded_params: set[str],
|
||||
tp_rank: int,
|
||||
tp_size: int,
|
||||
) -> bool:
|
||||
"""
|
||||
The fused fp8 QKV projection weights and scale are stored separately.
|
||||
Special care must be taken while sharding these tensors across TP ranks.
|
||||
See _shard_fp8_qkv_proj for more details.
|
||||
|
||||
Returns:
|
||||
True if ``tensor`` was an fp8 qkv_proj weight/scale and was consumed
|
||||
(caller should skip it); False otherwise, so the caller falls
|
||||
through to its normal loading path.
|
||||
"""
|
||||
is_weight = (
|
||||
name.endswith("qkv_proj.weight") and tensor.dtype == torch.float8_e4m3fn
|
||||
)
|
||||
is_scale = name.endswith("qkv_proj.weight_scale_inv")
|
||||
if not is_weight and not is_scale:
|
||||
# Weight is not in FP8 format. Ignore.
|
||||
return False
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
# This qkv_proj is for a layer not on this PP rank.
|
||||
return True
|
||||
|
||||
prefix, qkv_kind = name.rsplit(".", 1)
|
||||
entry = fp8_qkv_proj_dict.setdefault(prefix, {})
|
||||
entry[qkv_kind] = tensor
|
||||
if "weight" not in entry or "weight_scale_inv" not in entry:
|
||||
# Still waiting for the other param.
|
||||
return True
|
||||
del fp8_qkv_proj_dict[prefix]
|
||||
|
||||
# Get self_attn module, which is a parent of qkv_proj.
|
||||
attn = self.get_submodule(prefix.rsplit(".", 1)[0])
|
||||
|
||||
# Shard the qkv_proj per-rank.
|
||||
w_rank, s_rank = _shard_fp8_qkv_proj(
|
||||
entry["weight"],
|
||||
entry["weight_scale_inv"],
|
||||
num_heads=attn.total_num_heads,
|
||||
num_kv_heads=attn.total_num_kv_heads,
|
||||
head_dim=attn.head_dim,
|
||||
v_head_dim=attn.v_head_dim,
|
||||
tp_rank=tp_rank,
|
||||
tp_size=tp_size,
|
||||
)
|
||||
sharded = {"weight": w_rank, "weight_scale_inv": s_rank}
|
||||
for kind, tensor in sharded.items():
|
||||
param_name = f"{prefix}.{kind}"
|
||||
param = params_dict[param_name]
|
||||
if tensor.shape[0] > param.shape[0]:
|
||||
tensor = tensor[: param.shape[0]]
|
||||
default_weight_loader(param, tensor)
|
||||
loaded_params.add(param_name)
|
||||
return True
|
||||
|
||||
|
||||
class MiMoV2FlashForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
|
||||
packed_modules_mapping = {
|
||||
|
||||
Reference in New Issue
Block a user