[Quantization] Add ModelOpt IQ2_XS, IQ2_XXS, and Q8_0 support via b12x (#59604)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Luke Alonso <lalonso@gmail.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Luke Alonso <lalonso@gmail.com>
This commit is contained in:
co-authored by
Andrii Skliar
Luke Alonso
parent
ac71b17517
commit
6849ca696a
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.b12x import (
|
||||
B12X_BLOCK_CODECS,
|
||||
B12xWarmupUnit,
|
||||
get_b12x_fused_moe,
|
||||
reuse_packed_weight_storage,
|
||||
@@ -38,6 +39,7 @@ _B12X_MOE_MODES: dict[
|
||||
tuple[torch.dtype | str | None, torch.dtype | str | None],
|
||||
tuple[str, str, str],
|
||||
] = {
|
||||
**{(codec, None): ("w4a16", codec, "w31") for codec in B12X_BLOCK_CODECS},
|
||||
("mxfp4", "mxfp8"): ("w4a8_mx", "fp4_e8m0_k32", "w31"),
|
||||
("mxfp4", None): ("w4a16", "fp4_e8m0_k32", "w31"),
|
||||
("nvfp4", "nvfp4"): ("nvfp4", "modelopt_nvfp4", "w31"),
|
||||
@@ -206,10 +208,13 @@ class B12xExperts(mk.FusedMoEExpertsModular):
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
):
|
||||
super().__init__(moe_config, quant_config)
|
||||
if quant_config.weight_quant_dtype not in ("mxfp4", "nvfp4"):
|
||||
if quant_config.weight_quant_dtype not in (
|
||||
"mxfp4",
|
||||
"nvfp4",
|
||||
*B12X_BLOCK_CODECS,
|
||||
):
|
||||
raise ValueError(
|
||||
"b12x MoE requires MXFP4 or NVFP4 weights, got "
|
||||
f"{quant_config.weight_quant_dtype}"
|
||||
f"unsupported b12x MoE weight format: {quant_config.weight_quant_dtype}"
|
||||
)
|
||||
scheme = (
|
||||
quant_config.weight_quant_dtype,
|
||||
@@ -293,6 +298,31 @@ class B12xExperts(mk.FusedMoEExpertsModular):
|
||||
raise RuntimeError(
|
||||
"b12x MoE weights must be prepared before CUDA graph capture"
|
||||
)
|
||||
if self._source_format in B12X_BLOCK_CODECS:
|
||||
fused_moe = _require_b12x_fused_moe()
|
||||
block_size, _ = B12X_BLOCK_CODECS[self._source_format]
|
||||
plan = fused_moe.plan_weights(
|
||||
source=fused_moe.PackedSource(
|
||||
format=self._source_format, w13_layout=self._w13_layout
|
||||
),
|
||||
activation=fused_moe.ActivationSpec(
|
||||
mode="a16",
|
||||
nonlinearity=_b12x_activation_name(activation),
|
||||
io_dtype=params_dtype,
|
||||
),
|
||||
geometry=fused_moe.MoEGeometry(
|
||||
num_experts=w1.shape[0],
|
||||
hidden_size=w2.shape[1],
|
||||
intermediate_size=w2.shape[2] * block_size,
|
||||
),
|
||||
)
|
||||
prepared = fused_moe.prepare_weights(
|
||||
plan=plan,
|
||||
weights=fused_moe.BlockQuantWeights(
|
||||
w13=w1, w2=w2, codec=self._source_format
|
||||
),
|
||||
)
|
||||
return prepared._impl
|
||||
if self.w1_scale is None or self.w2_scale is None:
|
||||
raise ValueError("b12x MoE requires w1 and w2 block scales")
|
||||
|
||||
@@ -345,6 +375,8 @@ class B12xExperts(mk.FusedMoEExpertsModular):
|
||||
)
|
||||
|
||||
def _refresh_quant_config(self, layer: torch.nn.Module) -> None:
|
||||
if self._source_format in B12X_BLOCK_CODECS:
|
||||
return
|
||||
self.quant_config._w1.scale = layer.w13_weight_scale
|
||||
self.quant_config._w2.scale = layer.w2_weight_scale
|
||||
if self._source_format != "modelopt_nvfp4":
|
||||
@@ -446,8 +478,10 @@ class B12xExperts(mk.FusedMoEExpertsModular):
|
||||
):
|
||||
return (
|
||||
False,
|
||||
"MXFP4 W4A8 requires hidden size divisible by 256 and "
|
||||
"per-rank intermediate size divisible by 32",
|
||||
(
|
||||
"MXFP4 W4A8 requires hidden size divisible by 256 and "
|
||||
"per-rank intermediate size divisible by 32"
|
||||
),
|
||||
)
|
||||
return mk.FusedMoEExperts.is_supported_config(
|
||||
cls, moe_config, weight_key, activation_key, activation_format
|
||||
@@ -530,14 +564,15 @@ class B12xExperts(mk.FusedMoEExpertsModular):
|
||||
w2: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
if w1.numel() and w2.numel():
|
||||
if w1.numel() and w2.numel() and self._source_format not in B12X_BLOCK_CODECS:
|
||||
return super().moe_problem_size(a1, w1, w2, topk_ids)
|
||||
prepared = self._prepared()
|
||||
tokens = int(a1.shape[0] if a1.ndim == 2 else a1.shape[1])
|
||||
return (
|
||||
int(prepared.num_experts),
|
||||
tokens,
|
||||
int(prepared.intermediate_size) * 2,
|
||||
int(prepared.intermediate_size)
|
||||
* (2 if self.moe_config.activation.is_gated else 1),
|
||||
int(a1.shape[-1]),
|
||||
int(topk_ids.shape[1]),
|
||||
)
|
||||
|
||||
@@ -986,12 +986,20 @@ class RoutedExperts(PluggableLayer):
|
||||
|
||||
for expert_name, loaded_weight in weights:
|
||||
qual_name = f"{self.layer_name}.{expert_name}"
|
||||
is_fused = loaded_weight.dim() == 3
|
||||
expert_key = qual_name.partition("experts.")[2].partition(".")[0]
|
||||
named_expert = (
|
||||
mapping_by_expert.get(expert_key)
|
||||
if qual_name.count("experts.") == 1
|
||||
else None
|
||||
)
|
||||
# Encoded per-expert [N, K / block_size, bytes_per_block] weights
|
||||
# have the same rank as fused [E, N, K] weights. An explicit expert
|
||||
# name identifies the per-expert layout without treating N as E.
|
||||
is_fused = loaded_weight.dim() == 3 and named_expert is None
|
||||
# Fused tensors and ambiguous names keep the full mapping.
|
||||
candidates = expert_mapping
|
||||
if not is_fused and qual_name.count("experts.") == 1:
|
||||
expert_key = qual_name.partition("experts.")[2].partition(".")[0]
|
||||
candidates = mapping_by_expert.get(expert_key, expert_mapping)
|
||||
candidates = named_expert or expert_mapping
|
||||
|
||||
matched = False
|
||||
for param_name, weight_name, expert_id, shard_id in candidates:
|
||||
|
||||
Reference in New Issue
Block a user