[Bugfix][CPU][MoE] Fix out-of-bounds write and segfault when router weights are fp32 (#56168)
Signed-off-by: Farzad Abdolhosseini <farzad@elastix.ai> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
co-authored by
Claude
Li, Jiang <jiang1.li@intel.com>
parent
7f1a5398e9
commit
8b06a343ec
@@ -7,6 +7,7 @@ import torch
|
||||
|
||||
from vllm.model_executor.layers import utils
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
@@ -89,3 +90,25 @@ def test_dispatch_cpu_unquantized_gemm_logs_zentorch_dispatch(monkeypatch):
|
||||
expected_prepacked,
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
|
||||
@pytest.mark.parametrize("weight_dtype", [torch.bfloat16, torch.float16, torch.float32])
|
||||
def test_dispatch_cpu_unquantized_gemm_remove_weight_keeps_dtype(
|
||||
monkeypatch, weight_dtype
|
||||
):
|
||||
monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
|
||||
|
||||
layer = torch.nn.Linear(16, 8, bias=False, dtype=weight_dtype)
|
||||
loading_dtype = (
|
||||
torch.float32 if weight_dtype is not torch.float32 else torch.bfloat16
|
||||
)
|
||||
with set_default_torch_dtype(loading_dtype):
|
||||
utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
|
||||
|
||||
assert layer.weight.numel() == 0
|
||||
assert layer.weight.dtype is weight_dtype
|
||||
|
||||
x = torch.randn(4, 16, dtype=weight_dtype)
|
||||
output = layer.cpu_linear(x, layer.weight, None)
|
||||
assert not output.isnan().any()
|
||||
|
||||
@@ -513,7 +513,10 @@ def dispatch_cpu_unquantized_gemm(
|
||||
)
|
||||
)
|
||||
if remove_weight:
|
||||
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.empty(0, dtype=dtype, device=layer.weight.device),
|
||||
requires_grad=False,
|
||||
)
|
||||
logger.debug_once(
|
||||
"CPU unquantized GEMM dispatch: using zentorch_linear_unary (prepacked=%s)",
|
||||
is_prepacked,
|
||||
@@ -539,7 +542,10 @@ def dispatch_cpu_unquantized_gemm(
|
||||
bias,
|
||||
)
|
||||
if remove_weight:
|
||||
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.empty(0, dtype=dtype, device=layer.weight.device),
|
||||
requires_grad=False,
|
||||
)
|
||||
logger.debug_once(
|
||||
"CPU unquantized GEMM dispatch: using sgl-kernel weight_packed_linear"
|
||||
)
|
||||
@@ -554,7 +560,10 @@ def dispatch_cpu_unquantized_gemm(
|
||||
handler = ops.create_onednn_mm(origin_weight.t(), 32)
|
||||
layer.cpu_linear = lambda x, weight, bias: ops.onednn_mm(handler, x, bias)
|
||||
if remove_weight:
|
||||
layer.weight = torch.nn.Parameter(torch.empty(0), requires_grad=False)
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.empty(0, dtype=dtype, device=layer.weight.device),
|
||||
requires_grad=False,
|
||||
)
|
||||
logger.debug_once("CPU unquantized GEMM dispatch: using oneDNN onednn_mm")
|
||||
return
|
||||
except RuntimeError as e:
|
||||
|
||||
Reference in New Issue
Block a user