[Performance][B200] silu_mul_quant: pack scales in int32 (#28358)
Signed-off-by: Varun Sundar Rabindranath <vsundarr@redhat.com> Co-authored-by: Varun Sundar Rabindranath <vsundarr@redhat.com>
This commit is contained in:
co-authored by
Varun Sundar Rabindranath
parent
fdfd5075aa
commit
fe1cd7704d
@@ -9,6 +9,7 @@ import functools
|
||||
import importlib
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from enum import Enum
|
||||
from typing import Any, NoReturn
|
||||
|
||||
import torch
|
||||
@@ -20,6 +21,28 @@ from vllm.utils.import_utils import has_deep_gemm
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
|
||||
class DeepGemmQuantScaleFMT(Enum):
|
||||
# Float32 scales in Float32 tensor
|
||||
FLOAT32 = 0
|
||||
# Compute float32 scales and ceil the scales to UE8M0.
|
||||
# Keep the scales in Float32 tensor.
|
||||
FLOAT32_CEIL_UE8M0 = 1
|
||||
# Compute float32 scales and ceil the scales to UE8M0.
|
||||
# Pack the scales into a int32 tensor where each int32
|
||||
# element contains 4 scale values.
|
||||
UE8M0 = 2
|
||||
|
||||
@staticmethod
|
||||
def from_oracle() -> "DeepGemmQuantScaleFMT":
|
||||
if not is_deep_gemm_e8m0_used():
|
||||
return DeepGemmQuantScaleFMT.FLOAT32
|
||||
return (
|
||||
DeepGemmQuantScaleFMT.UE8M0
|
||||
if current_platform.is_device_capability(100)
|
||||
else DeepGemmQuantScaleFMT.FLOAT32_CEIL_UE8M0
|
||||
)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def is_deep_gemm_supported() -> bool:
|
||||
"""Return `True` if DeepGEMM is supported on the current platform.
|
||||
|
||||
Reference in New Issue
Block a user