From 70456e5e6fb4ee86b8ffd985f918e44ba632d925 Mon Sep 17 00:00:00 2001
From: Walter Beller-Morales <walterbm@users.noreply.github.com>
Date: Fri, 7 Aug 2026 15:45:52 -0400
Subject: [PATCH] [Bugfix] when loading weights skip empty expert bias if model
 does not support them (#50937)

Signed-off-by: walterbm <walter.beller.morales@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
---
 .../moe/test_moe_weight_loading_padded.py     | 81 +++++++++++++++++++
 .../layers/fused_moe/routed_experts.py        |  9 ++-
 2 files changed, 89 insertions(+), 1 deletion(-)

diff --git a/tests/kernels/moe/test_moe_weight_loading_padded.py b/tests/kernels/moe/test_moe_weight_loading_padded.py
index bb377049d9..d2e2d08dd9 100644
--- a/tests/kernels/moe/test_moe_weight_loading_padded.py
+++ b/tests/kernels/moe/test_moe_weight_loading_padded.py
@@ -14,6 +14,8 @@ import torch
 
 from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
 
+from .utils import make_dummy_moe_config
+
 
 class TestGetHiddenDim:
     """Unit tests for _get_hidden_dim."""
@@ -376,6 +378,85 @@ class TestWeightLoadingWithPaddedHiddenSize:
             )
 
 
+class TestLoadWeightsExpertBias:
+    """Some quantized exports (e.g. GPTQ, llm-compressor NVFP4) materialize
+    all-zero per-expert `.bias` tensors for models whose experts have no bias
+    params. `RoutedExperts.load_weights` needs to ignore them like
+    `AutoWeightsLoader` does, instead of raising AttributeError for the
+    nonexistent `w13_bias`/`w2_bias` params.
+    """
+
+    NUM_EXPERTS = 2
+
+    def _make_experts(self, has_bias: bool) -> torch.nn.Module:
+        experts = torch.nn.Module()
+        experts.layer_name = "model.layers.0.mlp.experts"
+        experts.moe_config = make_dummy_moe_config(num_experts=self.NUM_EXPERTS)
+        mapping = RoutedExperts.build_expert_params_mapping(
+            "gate_proj",
+            "down_proj",
+            "up_proj",
+            num_experts=self.NUM_EXPERTS,
+            routed_experts_prefix="",
+            include_fused=True,
+        )
+        experts.get_expert_mapping = lambda **_: mapping
+
+        def weight_loader(**_):
+            return True
+
+        names = ["w13_weight", "w2_weight"]
+        if has_bias:
+            names += ["w13_bias", "w2_bias"]
+        for name in names:
+            param = torch.nn.Parameter(torch.zeros(1), requires_grad=False)
+            param.weight_loader = weight_loader
+            setattr(experts, name, param)
+        return experts
+
+    def _checkpoint_weights(self) -> list[tuple[str, torch.Tensor]]:
+        return [
+            (f"{expert_id}.{proj}.{suffix}", torch.zeros(1, 1))
+            for expert_id in range(self.NUM_EXPERTS)
+            for proj in ("gate_proj", "up_proj", "down_proj")
+            for suffix in ("weight", "bias")
+        ]
+
+    def test_bias_free_experts_ignore_checkpoint_biases(self):
+        experts = self._make_experts(has_bias=False)
+        loaded = list(RoutedExperts.load_weights(experts, self._checkpoint_weights()))
+        assert set(loaded) == {"w13_weight", "w2_weight"}
+
+    def test_experts_with_bias_params_load_checkpoint_biases(self):
+        experts = self._make_experts(has_bias=True)
+        loaded = list(RoutedExperts.load_weights(experts, self._checkpoint_weights()))
+        assert set(loaded) == {"w13_weight", "w2_weight", "w13_bias", "w2_bias"}
+
+    def test_missing_non_bias_param_names_the_weight(self):
+        experts = self._make_experts(has_bias=False)
+        weights = [("0.down_proj.new_scale", torch.zeros(1, 1))]
+        with pytest.raises(AttributeError, match="w2_new_scale"):
+            list(RoutedExperts.load_weights(experts, weights))
+
+    @pytest.mark.parametrize(
+        "expert_name,param_name",
+        [
+            # Pre-fused checkpoints name biases `<proj>_bias` (gpt-oss) or
+            # `<proj>.bias` (quark), neither of which rewrites to a real param.
+            # Skipping them would silently drop a bias the layer does have, so
+            # they must raise; models rename them via WeightsMapper instead.
+            ("down_proj_bias", "w2_weight_bias"),
+            ("gate_up_proj_bias", "w13_weight_bias"),
+            ("down_proj.bias", "w2_weight.bias"),
+        ],
+    )
+    def test_fused_bias_names_are_not_skipped(self, expert_name, param_name):
+        experts = self._make_experts(has_bias=True)
+        weights = [(expert_name, torch.zeros(self.NUM_EXPERTS, 1))]
+        with pytest.raises(AttributeError, match=param_name.replace(".", r"\.")):
+            list(RoutedExperts.load_weights(experts, weights))
+
+
 class TestPerTensorScaleCoercion:
     """Regression test for shape-(1,) per-tensor scales (issue #43297).
 
diff --git a/vllm/model_executor/layers/fused_moe/routed_experts.py b/vllm/model_executor/layers/fused_moe/routed_experts.py
index 0cd0ad9950..6c31f817a3 100644
--- a/vllm/model_executor/layers/fused_moe/routed_experts.py
+++ b/vllm/model_executor/layers/fused_moe/routed_experts.py
@@ -936,7 +936,14 @@ class RoutedExperts(PluggableLayer):
                 matched = True
                 weight_name = qual_name.replace(weight_name, param_name)
                 param_name = weight_name.removeprefix(f"{self.layer_name}.")
-                param = getattr(self, param_name)
+                param = getattr(self, param_name, None)
+                if param is None:
+                    if param_name.endswith(("w13_bias", "w2_bias")):
+                        continue
+                    raise AttributeError(
+                        f"Layer {self.layer_name} has no parameter {param_name!r} "
+                        f"for checkpoint weight {qual_name!r}"
+                    )
                 if is_fused:
                     # w1 and w3 share one fused tensor; use a local copy so the
                     # transpose below doesn't mutate loaded_weight across