[BugFxi] Fix DeepGEMM FP8 workspace over allocation (#53914)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
Lucas Wilkinson
2026-09-17 09:35:34 -04:00
committed by GitHub
co-authored by Robert Shaw
parent 75c71390d5
commit fe284ea17d
3 changed files with 73 additions and 3 deletions
@@ -393,12 +393,13 @@ def _fwd_kernel_ep_gather(
source_token_index = tl.load(
input_index + cur_token * input_index_stride0 + topk_index
)
source_token_index_i64 = source_token_index.to(tl.int64)
acc_weight = tl.load(
recv_topk_weight + cur_token * recv_topk_weight_stride0 + topk_index
)
tmp = tl.load(
input_tensor
+ source_token_index * input_tensor_stride0
+ source_token_index_i64 * input_tensor_stride0
+ cur_block * BLOCK_D
+ off_d
)
@@ -50,6 +50,21 @@ from vllm.utils.import_utils import has_deep_gemm
logger = init_logger(__name__)
def _fp8_workspace_shape(
num_rows: int, num_columns: int, workspace_dtype: torch.dtype
) -> tuple[int, int]:
"""Size an FP8 byte buffer stored in a workspace of another dtype."""
bytes_per_workspace_element = workspace_dtype.itemsize
fp8_columns_per_workspace_element = (
bytes_per_workspace_element // torch.float8_e4m3fn.itemsize
)
assert bytes_per_workspace_element % torch.float8_e4m3fn.itemsize == 0
return (
num_rows,
-(-num_columns // fp8_columns_per_workspace_element),
)
def _valid_deep_gemm_shape(M: int, N: int, K: int) -> bool:
align = get_mk_alignment_for_contiguous_layout()[0]
return align <= M and N % align == 0 and K % align == 0
@@ -216,7 +231,14 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
assert M_sum % align_used == 0
activation_out_dim = self.adjust_N_for_activation(N, activation)
workspace1 = (M_sum, max(activation_out_dim, K))
# workspace1 is allocated in the activation dtype by the workspace
# manager, but is only ever viewed and used as FP8 in apply(). Size it
# by bytes instead of reserving one BF16/FP16 element per FP8 element.
workspace1 = _fp8_workspace_shape(
M_sum,
max(activation_out_dim, K),
self.workspace_dtype(self.moe_config.in_dtype),
)
workspace2 = (M_sum, max(N, K))
output = (M, K)
return (workspace1, workspace2, output)
@@ -480,7 +502,14 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
assert M_sum % align_used == 0
activation_out_dim = self.adjust_N_for_activation(N, activation)
workspace1 = (M_sum, max(activation_out_dim, K))
# workspace1 holds the permuted and requantized FP8 activations. The
# workspace manager allocates it in the model activation dtype, so
# account for the dtype sizes rather than overallocating BF16/FP16.
workspace1 = _fp8_workspace_shape(
M_sum,
max(activation_out_dim, K),
self.workspace_dtype(self.moe_config.in_dtype),
)
workspace2 = (M_sum, max(N, K))
output = (M, K)
return (workspace1, workspace2, output)