[BugFxi] Fix DeepGEMM FP8 workspace over allocation (#53914)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
Lucas Wilkinson
2026-09-17 09:35:34 -04:00
committed by GitHub
co-authored by Robert Shaw
parent 75c71390d5
commit fe284ea17d
3 changed files with 73 additions and 3 deletions
+40
View File
@@ -26,6 +26,10 @@ from vllm.model_executor.layers.fused_moe.config import (
)
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
deepgemm_moe_permute,
ep_gather,
)
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
_fp8_workspace_shape,
)
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
TritonOrDeepGemmExperts,
@@ -43,6 +47,42 @@ from vllm.utils.deep_gemm import (
BLOCK_SIZE = [128, 128]
def test_ep_gather_uses_64_bit_row_offsets():
hidden_size = 4096
source_row = 1 << 18
input_tensor = torch.empty(
(source_row + 1, hidden_size),
device="cuda",
dtype=torch.bfloat16,
)
input_tensor[source_row].fill_(1)
output = torch.empty((1, hidden_size), device="cuda", dtype=torch.bfloat16)
ep_gather(
input_tensor=input_tensor,
recv_topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64),
recv_topk_weight=torch.ones((1, 1), device="cuda", dtype=torch.float32),
input_index=torch.full((1, 1), source_row, device="cuda", dtype=torch.int32),
expert_map=None,
output_tensor=output,
)
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
@pytest.mark.parametrize("workspace_dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("num_columns", [2048, 6144, 6145])
def test_fp8_workspace_shape(workspace_dtype, num_columns):
num_rows = 17
shape = _fp8_workspace_shape(num_rows, num_columns, workspace_dtype)
allocated_bytes = math.prod(shape) * workspace_dtype.itemsize
required_bytes = num_rows * num_columns * torch.float8_e4m3fn.itemsize
assert allocated_bytes >= required_bytes
assert allocated_bytes - required_bytes < num_rows * workspace_dtype.itemsize
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init):
hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16)