[Bugfix][CPU][MoE] Fix out-of-bounds write and segfault when router weights are fp32 (#56168)

Signed-off-by: Farzad Abdolhosseini <farzad@elastix.ai>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Farzad Abdolhosseini
2026-09-24 13:15:17 +08:00
committed by GitHub
co-authored by Claude Li, Jiang <jiang1.li@intel.com>
parent 7f1a5398e9
commit 8b06a343ec
2 changed files with 35 additions and 3 deletions
@@ -7,6 +7,7 @@ import torch
from vllm.model_executor.layers import utils
from vllm.platforms import current_platform
from vllm.utils.torch_utils import set_default_torch_dtype
@pytest.fixture(scope="module")
@@ -89,3 +90,25 @@ def test_dispatch_cpu_unquantized_gemm_logs_zentorch_dispatch(monkeypatch):
expected_prepacked,
)
]
@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
@pytest.mark.parametrize("weight_dtype", [torch.bfloat16, torch.float16, torch.float32])
def test_dispatch_cpu_unquantized_gemm_remove_weight_keeps_dtype(
monkeypatch, weight_dtype
):
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
layer = torch.nn.Linear(16, 8, bias=False, dtype=weight_dtype)
loading_dtype = (
torch.float32 if weight_dtype is not torch.float32 else torch.bfloat16
)
with set_default_torch_dtype(loading_dtype):
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
assert layer.weight.numel() == 0
assert layer.weight.dtype is weight_dtype
x = torch.randn(4, 16, dtype=weight_dtype)
output = layer.cpu_linear(x, layer.weight, None)
assert not output.isnan().any()
+12 -3
View File
@@ -513,7 +513,10 @@ def dispatch_cpu_unquantized_gemm(
)
)
if remove_weight:
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
layer.weight = torch.nn.Parameter(
torch.empty(0, dtype=dtype, device=layer.weight.device),
requires_grad=False,
)
logger.debug_once(
"CPU unquantized GEMM dispatch: using zentorch_linear_unary (prepacked=%s)",
is_prepacked,
@@ -539,7 +542,10 @@ def dispatch_cpu_unquantized_gemm(
bias,
)
if remove_weight:
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
layer.weight = torch.nn.Parameter(
torch.empty(0, dtype=dtype, device=layer.weight.device),
requires_grad=False,
)
logger.debug_once(
"CPU unquantized GEMM dispatch: using sgl-kernel weight_packed_linear"
)
@@ -554,7 +560,10 @@ def dispatch_cpu_unquantized_gemm(
handler = ops.create_onednn_mm(origin_weight.t(), 32)
layer.cpu_linear = lambda x, weight, bias: ops.onednn_mm(handler, x, bias)
if remove_weight:
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
layer.weight = torch.nn.Parameter(
torch.empty(0, dtype=dtype, device=layer.weight.device),
requires_grad=False,
)
logger.debug_once("CPU unquantized GEMM dispatch: using oneDNN onednn_mm")
return
except RuntimeError as e: