[BugFxi] Fix DeepGEMM FP8 workspace over allocation (#53914)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
75c71390d5
commit
fe284ea17d
@@ -393,12 +393,13 @@ def _fwd_kernel_ep_gather(
|
||||
source_token_index = tl.load(
|
||||
input_index + cur_token * input_index_stride0 + topk_index
|
||||
)
|
||||
source_token_index_i64 = source_token_index.to(tl.int64)
|
||||
acc_weight = tl.load(
|
||||
recv_topk_weight + cur_token * recv_topk_weight_stride0 + topk_index
|
||||
)
|
||||
tmp = tl.load(
|
||||
input_tensor
|
||||
+ source_token_index * input_tensor_stride0
|
||||
+ source_token_index_i64 * input_tensor_stride0
|
||||
+ cur_block * BLOCK_D
|
||||
+ off_d
|
||||
)
|
||||
|
||||
@@ -50,6 +50,21 @@ from vllm.utils.import_utils import has_deep_gemm
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _fp8_workspace_shape(
|
||||
num_rows: int, num_columns: int, workspace_dtype: torch.dtype
|
||||
) -> tuple[int, int]:
|
||||
"""Size an FP8 byte buffer stored in a workspace of another dtype."""
|
||||
bytes_per_workspace_element = workspace_dtype.itemsize
|
||||
fp8_columns_per_workspace_element = (
|
||||
bytes_per_workspace_element // torch.float8_e4m3fn.itemsize
|
||||
)
|
||||
assert bytes_per_workspace_element % torch.float8_e4m3fn.itemsize == 0
|
||||
return (
|
||||
num_rows,
|
||||
-(-num_columns // fp8_columns_per_workspace_element),
|
||||
)
|
||||
|
||||
|
||||
def _valid_deep_gemm_shape(M: int, N: int, K: int) -> bool:
|
||||
align = get_mk_alignment_for_contiguous_layout()[0]
|
||||
return align <= M and N % align == 0 and K % align == 0
|
||||
@@ -216,7 +231,14 @@ class DeepGemmExperts(mk.FusedMoEExpertsModular):
|
||||
assert M_sum % align_used == 0
|
||||
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
workspace1 = (M_sum, max(activation_out_dim, K))
|
||||
# workspace1 is allocated in the activation dtype by the workspace
|
||||
# manager, but is only ever viewed and used as FP8 in apply(). Size it
|
||||
# by bytes instead of reserving one BF16/FP16 element per FP8 element.
|
||||
workspace1 = _fp8_workspace_shape(
|
||||
M_sum,
|
||||
max(activation_out_dim, K),
|
||||
self.workspace_dtype(self.moe_config.in_dtype),
|
||||
)
|
||||
workspace2 = (M_sum, max(N, K))
|
||||
output = (M, K)
|
||||
return (workspace1, workspace2, output)
|
||||
@@ -480,7 +502,14 @@ class DeepGemmFP4Experts(mk.FusedMoEExpertsModular):
|
||||
assert M_sum % align_used == 0
|
||||
|
||||
activation_out_dim = self.adjust_N_for_activation(N, activation)
|
||||
workspace1 = (M_sum, max(activation_out_dim, K))
|
||||
# workspace1 holds the permuted and requantized FP8 activations. The
|
||||
# workspace manager allocates it in the model activation dtype, so
|
||||
# account for the dtype sizes rather than overallocating BF16/FP16.
|
||||
workspace1 = _fp8_workspace_shape(
|
||||
M_sum,
|
||||
max(activation_out_dim, K),
|
||||
self.workspace_dtype(self.moe_config.in_dtype),
|
||||
)
|
||||
workspace2 = (M_sum, max(N, K))
|
||||
output = (M, K)
|
||||
return (workspace1, workspace2, output)
|
||||
|
||||
Reference in New Issue
Block a user