[W8A8 Block Linear Refactor][2/N] Remove W8A8Fp8BlockLinearOp and adopt Fp8 block linear kernel selections. (#33892)
Signed-off-by: maral <maralbahari.98@gmail.com> Signed-off-by: Maral <maralbahari.98@gmail.com>
This commit is contained in:
@@ -413,7 +413,7 @@ def calc_diff(x: torch.Tensor, y: torch.Tensor):
|
||||
|
||||
def should_use_deepgemm_for_fp8_linear(
|
||||
output_dtype: torch.dtype,
|
||||
weight: torch.Tensor,
|
||||
weight_shape: tuple[int, int],
|
||||
supports_deep_gemm: bool | None = None,
|
||||
):
|
||||
if supports_deep_gemm is None:
|
||||
@@ -428,8 +428,8 @@ def should_use_deepgemm_for_fp8_linear(
|
||||
return (
|
||||
supports_deep_gemm
|
||||
and output_dtype == torch.bfloat16
|
||||
and weight.shape[0] % N_MULTIPLE == 0
|
||||
and weight.shape[1] % K_MULTIPLE == 0
|
||||
and weight_shape[0] % N_MULTIPLE == 0
|
||||
and weight_shape[1] % K_MULTIPLE == 0
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -748,8 +748,9 @@ def is_flashinfer_fp8_blockscale_gemm_supported() -> bool:
|
||||
def should_use_flashinfer_for_blockscale_fp8_gemm(
|
||||
is_flashinfer_supported: bool,
|
||||
output_dtype: torch.dtype,
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
input_dtype: torch.dtype,
|
||||
weight_dtype: torch.dtype,
|
||||
weight_shape: tuple[int, int],
|
||||
):
|
||||
if not is_flashinfer_supported:
|
||||
return False
|
||||
@@ -760,15 +761,12 @@ def should_use_flashinfer_for_blockscale_fp8_gemm(
|
||||
N_MULTIPLE = 64
|
||||
K_MULTIPLE = 128
|
||||
|
||||
weight_dtype = weight.dtype
|
||||
input_dtype = input.dtype
|
||||
|
||||
should_use_flashinfer = (
|
||||
output_dtype == torch.bfloat16
|
||||
and input_dtype == torch.bfloat16
|
||||
and weight_dtype == torch.float8_e4m3fn
|
||||
and weight.shape[0] % N_MULTIPLE == 0
|
||||
and weight.shape[1] % K_MULTIPLE == 0
|
||||
and weight_shape[0] % N_MULTIPLE == 0
|
||||
and weight_shape[1] % K_MULTIPLE == 0
|
||||
)
|
||||
|
||||
return should_use_flashinfer
|
||||
|
||||
Reference in New Issue
Block a user