[BugFxi] Fix DeepGEMM FP8 workspace over allocation (#53914)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
75c71390d5
commit
fe284ea17d
@@ -26,6 +26,10 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.deep_gemm_utils import (
|
||||
deepgemm_moe_permute,
|
||||
ep_gather,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import (
|
||||
_fp8_workspace_shape,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
@@ -43,6 +47,42 @@ from vllm.utils.deep_gemm import (
|
||||
BLOCK_SIZE = [128, 128]
|
||||
|
||||
|
||||
def test_ep_gather_uses_64_bit_row_offsets():
|
||||
hidden_size = 4096
|
||||
source_row = 1 << 18
|
||||
input_tensor = torch.empty(
|
||||
(source_row + 1, hidden_size),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
input_tensor[source_row].fill_(1)
|
||||
output = torch.empty((1, hidden_size), device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
ep_gather(
|
||||
input_tensor=input_tensor,
|
||||
recv_topk_ids=torch.zeros((1, 1), device="cuda", dtype=torch.int64),
|
||||
recv_topk_weight=torch.ones((1, 1), device="cuda", dtype=torch.float32),
|
||||
input_index=torch.full((1, 1), source_row, device="cuda", dtype=torch.int32),
|
||||
expert_map=None,
|
||||
output_tensor=output,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("workspace_dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("num_columns", [2048, 6144, 6145])
|
||||
def test_fp8_workspace_shape(workspace_dtype, num_columns):
|
||||
num_rows = 17
|
||||
shape = _fp8_workspace_shape(num_rows, num_columns, workspace_dtype)
|
||||
|
||||
allocated_bytes = math.prod(shape) * workspace_dtype.itemsize
|
||||
required_bytes = num_rows * num_columns * torch.float8_e4m3fn.itemsize
|
||||
|
||||
assert allocated_bytes >= required_bytes
|
||||
assert allocated_bytes - required_bytes < num_rows * workspace_dtype.itemsize
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_moe_permute_initializes_padding_scales(workspace_init):
|
||||
hidden_states = torch.randn(2, 128, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
Reference in New Issue
Block a user