[Feature] Enable DeepGEMM Linear on B200; 1.5% E2E throughput improvement (#23351)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -202,6 +202,12 @@ def calc_diff(x: torch.Tensor, y: torch.Tensor):
|
||||
return 1 - sim
|
||||
|
||||
|
||||
def should_use_deepgemm_for_fp8_linear(output_dtype: torch.dtype,
|
||||
weight: torch.Tensor):
|
||||
return (is_deep_gemm_supported() and output_dtype == torch.bfloat16
|
||||
and weight.shape[0] % 128 == 0 and weight.shape[1] % 128 == 0)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"calc_diff",
|
||||
"fp8_gemm_nt",
|
||||
@@ -210,4 +216,5 @@ __all__ = [
|
||||
"per_block_cast_to_fp8",
|
||||
"is_blackwell_deep_gemm_e8m0_used",
|
||||
"is_deep_gemm_supported",
|
||||
"should_use_deepgemm_for_fp8_linear",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user