diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index 5143f3e61f..fcf14d66cd 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -106,6 +106,7 @@ class Fp8Config(QuantizationConfig): activation_scheme: str = "dynamic", ignored_layers: list[str] | None = None, weight_block_size: list[int] | None = None, + store_dtype: str | None = None, ) -> None: super().__init__() @@ -115,6 +116,7 @@ class Fp8Config(QuantizationConfig): raise ValueError(f"Unsupported activation scheme {activation_scheme}") self.activation_scheme = activation_scheme self.ignored_layers = ignored_layers or [] + self.store_dtype = store_dtype if weight_block_size is not None: if not is_checkpoint_fp8_serialized: raise ValueError( @@ -162,6 +164,7 @@ class Fp8Config(QuantizationConfig): activation_scheme = cls.get_from_keys(config, ["activation_scheme"]) ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None) weight_block_size = cls.get_from_keys_or(config, ["weight_block_size"], None) + store_dtype = cls.get_from_keys_or(config, ["store_dtype"], None) if not ignored_layers: ignored_layers = cls.get_from_keys_or( config, ["modules_to_not_convert"], None @@ -171,6 +174,7 @@ class Fp8Config(QuantizationConfig): activation_scheme=activation_scheme, ignored_layers=ignored_layers, weight_block_size=weight_block_size, + store_dtype=store_dtype, ) def get_quant_method( @@ -198,6 +202,12 @@ class Fp8Config(QuantizationConfig): fused_mapping=self.packed_modules_mapping, ): return UnquantizedFusedMoEMethod(layer.moe_config) + if self.store_dtype == "mxfp4": + from vllm.model_executor.layers.quantization.mxfp4 import ( + Mxfp4MoEMethod, + ) + + return Mxfp4MoEMethod(layer.moe_config) if self.is_checkpoint_fp8_serialized: moe_quant_method = Fp8MoEMethod(self, layer) else: diff --git a/vllm/model_executor/models/mimo_v2.py b/vllm/model_executor/models/mimo_v2.py index b5f618699c..84459df4d2 100644 --- a/vllm/model_executor/models/mimo_v2.py +++ b/vllm/model_executor/models/mimo_v2.py @@ -35,6 +35,10 @@ from vllm.model_executor.layers.linear import ( ) from vllm.model_executor.layers.logits_processor import LogitsProcessor from vllm.model_executor.layers.quantization import QuantizationConfig +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + GroupShape, + scaled_quantize, +) from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import ( ParallelLMHead, @@ -455,6 +459,85 @@ class MiMoV2FlashDecoderLayer(nn.Module): return self.config.hybrid_layer_pattern[self.layer_id] == 1 +def _shard_fp8_qkv_proj( + w_full: torch.Tensor, + s_full: torch.Tensor, + num_heads: int, + num_kv_heads: int, + head_dim: int, + v_head_dim: int, + tp_rank: int, + tp_size: int, + block: int = 128, +) -> tuple[torch.Tensor, torch.Tensor]: + """Shard the fp8 qkv_proj weights for ``tp_rank``. + + The checkpoint stores the fused QKV as ``num_kv_heads`` contiguous groups + (one per KV head; ``n`` below), each ordered ``[Q | K | V]``: + + [Q_1 | K_1 | V_1 | Q_2 | K_2 | V_2 | ... | Q_n | K_n | V_n] + + Per group, Q has ``(num_heads / num_kv_heads) * head_dim`` rows, K has + ``head_dim`` rows, and V has ``v_head_dim`` rows. + + Each TP rank owns ``g = num_kv_heads / tp_size`` of these groups, and the + forward expects them de-interleaved into a single Q, K, and V block: + + [Q_1 | Q_2 | ... | Q_g | K_1 | K_2 | ... | K_g | V_1 | V_2 | ... | V_g] + + When ``g == 1`` the rank's slice is already ``[Q | K | V]``, so a plain + chunk suffices. When ``g > 1`` we cannot reach the de-interleaved layout by + re-permuting the fp8 block scales: each scale covers a 128-row block, and + since K is 192 rows (1.5 blocks) a block straddles the K/V boundary, so no + whole-block permutation produces it. Instead we dequantize this rank's + groups to float (dropping the block constraint), reorder the rows into the + layout above (Q, K, and V then each span a whole number of blocks), and + re-quantize to fp8. + """ + assert tp_size <= num_kv_heads and num_kv_heads % tp_size == 0, ( + "TP size must evenly split the number of KV heads." + ) + + kv_heads_per_rank = num_kv_heads // tp_size + if kv_heads_per_rank == 1: + # One KV head per rank. The weights and scale can be trivially sharded + # without re-quantization. + w = w_full.chunk(tp_size, dim=0)[tp_rank] + s = s_full.chunk(tp_size, dim=0)[tp_rank] + return w, s + + q_rows_per_group = (num_heads // num_kv_heads) * head_dim + k_rows_per_group = head_dim + v_rows_per_group = v_head_dim + rows_per_group = q_rows_per_group + k_rows_per_group + v_rows_per_group + scale_rows_per_group = s_full.shape[0] // num_kv_heads + qs, ks, vs = [], [], [] + for g_idx in range(tp_rank * kv_heads_per_rank, (tp_rank + 1) * kv_heads_per_rank): + row_start = g_idx * rows_per_group + scale_row_start = g_idx * scale_rows_per_group + # Dequantize this group's weights. + w_g = w_full[row_start : row_start + rows_per_group].to(torch.float32) + s_g = s_full[scale_row_start : scale_row_start + scale_rows_per_group].to( + torch.float32 + ) + s_g_expanded = s_g.repeat_interleave(block, dim=0).repeat_interleave( + block, dim=1 + )[:rows_per_group] + w_g_dequant = w_g * s_g_expanded + # Track the dequantized q, k, and v weights separately. + qs.append(w_g_dequant[:q_rows_per_group]) + ks.append(w_g_dequant[q_rows_per_group : q_rows_per_group + k_rows_per_group]) + vs.append(w_g_dequant[q_rows_per_group + k_rows_per_group :]) + + # Combine the q, k, and v weights into the following layout: + # [Q_1, Q_2, .., Q_g, K_1, K_2, ..., K_g, V_1, V_2, ..., V_g] + grouped = torch.cat([torch.cat(qs), torch.cat(ks), torch.cat(vs)], dim=0) + # Quantize back to fp8. + return scaled_quantize( + grouped, GroupShape(block, block), w_full.dtype, compute_dtype=torch.float32 + ) + + @support_torch_compile class MiMoV2Model(nn.Module): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): @@ -561,6 +644,10 @@ class MiMoV2Model(nn.Module): params_dict = dict(self.named_parameters(remove_duplicate=False)) loaded_params: set[str] = set() expert_params_mapping = self.get_expert_mapping() + # Pro-format fused qkv_proj arrives as two tensors (weight and + # weight_scale_inv). Store them per-layer so that they can be + # sharded together. + pending_fp8_qkv_proj: dict[str, dict[str, torch.Tensor]] = {} for name, loaded_weight in weights: if "rotary_emb.inv_freq" in name: continue @@ -604,11 +691,15 @@ class MiMoV2Model(nn.Module): if expert_matched: continue # Support fused qkv_proj checkpoint (Pro format) - if "qkv_proj" in name: - if name in params_dict: - param = params_dict[name] - loaded_weight = loaded_weight.chunk(tp_size, dim=0)[tp_rank] - default_weight_loader(param, loaded_weight) + if self._try_load_fp8_qkv_proj( + name, + loaded_weight, + pending_fp8_qkv_proj, + params_dict, + loaded_params, + tp_rank, + tp_size, + ): continue stacked_matched = False for param_name, weight_name, shard_id in stacked_params_mapping: @@ -666,6 +757,70 @@ class MiMoV2Model(nn.Module): return loaded_params + def _try_load_fp8_qkv_proj( + self, + name: str, + tensor: torch.Tensor, + fp8_qkv_proj_dict: dict[str, dict[str, torch.Tensor]], + params_dict: dict[str, torch.nn.Parameter], + loaded_params: set[str], + tp_rank: int, + tp_size: int, + ) -> bool: + """ + The fused fp8 QKV projection weights and scale are stored separately. + Special care must be taken while sharding these tensors across TP ranks. + See _shard_fp8_qkv_proj for more details. + + Returns: + True if ``tensor`` was an fp8 qkv_proj weight/scale and was consumed + (caller should skip it); False otherwise, so the caller falls + through to its normal loading path. + """ + is_weight = ( + name.endswith("qkv_proj.weight") and tensor.dtype == torch.float8_e4m3fn + ) + is_scale = name.endswith("qkv_proj.weight_scale_inv") + if not is_weight and not is_scale: + # Weight is not in FP8 format. Ignore. + return False + + if is_pp_missing_parameter(name, self): + # This qkv_proj is for a layer not on this PP rank. + return True + + prefix, qkv_kind = name.rsplit(".", 1) + entry = fp8_qkv_proj_dict.setdefault(prefix, {}) + entry[qkv_kind] = tensor + if "weight" not in entry or "weight_scale_inv" not in entry: + # Still waiting for the other param. + return True + del fp8_qkv_proj_dict[prefix] + + # Get self_attn module, which is a parent of qkv_proj. + attn = self.get_submodule(prefix.rsplit(".", 1)[0]) + + # Shard the qkv_proj per-rank. + w_rank, s_rank = _shard_fp8_qkv_proj( + entry["weight"], + entry["weight_scale_inv"], + num_heads=attn.total_num_heads, + num_kv_heads=attn.total_num_kv_heads, + head_dim=attn.head_dim, + v_head_dim=attn.v_head_dim, + tp_rank=tp_rank, + tp_size=tp_size, + ) + sharded = {"weight": w_rank, "weight_scale_inv": s_rank} + for kind, tensor in sharded.items(): + param_name = f"{prefix}.{kind}" + param = params_dict[param_name] + if tensor.shape[0] > param.shape[0]: + tensor = tensor[: param.shape[0]] + default_weight_loader(param, tensor) + loaded_params.add(param_name) + return True + class MiMoV2FlashForCausalLM(nn.Module, SupportsPP, MixtureOfExperts): packed_modules_mapping = {