[Quantization] Add ModelOpt IQ2_XS, IQ2_XXS, and Q8_0 support via b12x (#59604)

Signed-off-by: Andrii Skliar <askliar@nvidia.com>
Signed-off-by: Luke Alonso <lalonso@gmail.com>
Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Co-authored-by: Luke Alonso <lalonso@gmail.com>
This commit is contained in:
Andrii Skliar
2026-10-13 09:06:33 -07:00
committed by GitHub
co-authored by Andrii Skliar Luke Alonso
parent ac71b17517
commit 6849ca696a
15 changed files with 1238 additions and 25 deletions
+42 -7
View File
@@ -27,6 +27,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.utils.b12x import (
B12X_BLOCK_CODECS,
B12xWarmupUnit,
get_b12x_fused_moe,
reuse_packed_weight_storage,
@@ -38,6 +39,7 @@ _B12X_MOE_MODES: dict[
tuple[torch.dtype | str | None, torch.dtype | str | None],
tuple[str, str, str],
] = {
**{(codec, None): ("w4a16", codec, "w31") for codec in B12X_BLOCK_CODECS},
("mxfp4", "mxfp8"): ("w4a8_mx", "fp4_e8m0_k32", "w31"),
("mxfp4", None): ("w4a16", "fp4_e8m0_k32", "w31"),
("nvfp4", "nvfp4"): ("nvfp4", "modelopt_nvfp4", "w31"),
@@ -206,10 +208,13 @@ class B12xExperts(mk.FusedMoEExpertsModular):
quant_config: FusedMoEQuantConfig,
):
super().__init__(moe_config, quant_config)
if quant_config.weight_quant_dtype not in ("mxfp4", "nvfp4"):
if quant_config.weight_quant_dtype not in (
"mxfp4",
"nvfp4",
*B12X_BLOCK_CODECS,
):
raise ValueError(
"b12x MoE requires MXFP4 or NVFP4 weights, got "
f"{quant_config.weight_quant_dtype}"
f"unsupported b12x MoE weight format: {quant_config.weight_quant_dtype}"
)
scheme = (
quant_config.weight_quant_dtype,
@@ -293,6 +298,31 @@ class B12xExperts(mk.FusedMoEExpertsModular):
raise RuntimeError(
"b12x MoE weights must be prepared before CUDA graph capture"
)
if self._source_format in B12X_BLOCK_CODECS:
fused_moe = _require_b12x_fused_moe()
block_size, _ = B12X_BLOCK_CODECS[self._source_format]
plan = fused_moe.plan_weights(
source=fused_moe.PackedSource(
format=self._source_format, w13_layout=self._w13_layout
),
activation=fused_moe.ActivationSpec(
mode="a16",
nonlinearity=_b12x_activation_name(activation),
io_dtype=params_dtype,
),
geometry=fused_moe.MoEGeometry(
num_experts=w1.shape[0],
hidden_size=w2.shape[1],
intermediate_size=w2.shape[2] * block_size,
),
)
prepared = fused_moe.prepare_weights(
plan=plan,
weights=fused_moe.BlockQuantWeights(
w13=w1, w2=w2, codec=self._source_format
),
)
return prepared._impl
if self.w1_scale is None or self.w2_scale is None:
raise ValueError("b12x MoE requires w1 and w2 block scales")
@@ -345,6 +375,8 @@ class B12xExperts(mk.FusedMoEExpertsModular):
)
def _refresh_quant_config(self, layer: torch.nn.Module) -> None:
if self._source_format in B12X_BLOCK_CODECS:
return
self.quant_config._w1.scale = layer.w13_weight_scale
self.quant_config._w2.scale = layer.w2_weight_scale
if self._source_format != "modelopt_nvfp4":
@@ -446,8 +478,10 @@ class B12xExperts(mk.FusedMoEExpertsModular):
):
return (
False,
"MXFP4 W4A8 requires hidden size divisible by 256 and "
"per-rank intermediate size divisible by 32",
(
"MXFP4 W4A8 requires hidden size divisible by 256 and "
"per-rank intermediate size divisible by 32"
),
)
return mk.FusedMoEExperts.is_supported_config(
cls, moe_config, weight_key, activation_key, activation_format
@@ -530,14 +564,15 @@ class B12xExperts(mk.FusedMoEExpertsModular):
w2: torch.Tensor,
topk_ids: torch.Tensor,
) -> tuple[int, int, int, int, int]:
if w1.numel() and w2.numel():
if w1.numel() and w2.numel() and self._source_format not in B12X_BLOCK_CODECS:
return super().moe_problem_size(a1, w1, w2, topk_ids)
prepared = self._prepared()
tokens = int(a1.shape[0] if a1.ndim == 2 else a1.shape[1])
return (
int(prepared.num_experts),
tokens,
int(prepared.intermediate_size) * 2,
int(prepared.intermediate_size)
* (2 if self.moe_config.activation.is_gated else 1),
int(a1.shape[-1]),
int(topk_ids.shape[1]),
)
@@ -986,12 +986,20 @@ class RoutedExperts(PluggableLayer):
for expert_name, loaded_weight in weights:
qual_name = f"{self.layer_name}.{expert_name}"
is_fused = loaded_weight.dim() == 3
expert_key = qual_name.partition("experts.")[2].partition(".")[0]
named_expert = (
mapping_by_expert.get(expert_key)
if qual_name.count("experts.") == 1
else None
)
# Encoded per-expert [N, K / block_size, bytes_per_block] weights
# have the same rank as fused [E, N, K] weights. An explicit expert
# name identifies the per-expert layout without treating N as E.
is_fused = loaded_weight.dim() == 3 and named_expert is None
# Fused tensors and ambiguous names keep the full mapping.
candidates = expert_mapping
if not is_fused and qual_name.count("experts.") == 1:
expert_key = qual_name.partition("experts.")[2].partition(".")[0]
candidates = mapping_by_expert.get(expert_key, expert_mapping)
candidates = named_expert or expert_mapping
matched = False
for param_name, weight_name, expert_id, shard_id in candidates: