[Feature] Enable E8M0 by Default on Hopper for DeepGEMM, 5% E2E throughput improvement (#26197)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
335b28f7d1
commit
f8607863d8
+3
-15
@@ -29,12 +29,7 @@ def is_deep_gemm_supported() -> bool:
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability(100)
|
||||
)
|
||||
return (
|
||||
envs.VLLM_USE_DEEP_GEMM
|
||||
and has_deep_gemm()
|
||||
and is_supported_arch
|
||||
and not envs.VLLM_USE_FLASHINFER_MOE_FP8
|
||||
)
|
||||
return envs.VLLM_USE_DEEP_GEMM and has_deep_gemm() and is_supported_arch
|
||||
|
||||
|
||||
@functools.cache
|
||||
@@ -58,15 +53,8 @@ def is_deep_gemm_e8m0_used() -> bool:
|
||||
logger.info_once("DeepGEMM E8M0 disabled: FlashInfer MOE is enabled.")
|
||||
return False
|
||||
|
||||
if current_platform.is_device_capability(100) and envs.VLLM_USE_DEEP_GEMM_E8M0:
|
||||
logger.info_once("DeepGEMM E8M0 enabled on Blackwell GPU.")
|
||||
return True
|
||||
|
||||
if (
|
||||
current_platform.is_device_capability(90)
|
||||
and envs.VLLM_USE_DEEP_GEMM_E8M0_HOPPER
|
||||
):
|
||||
logger.info_once("DeepGEMM E8M0 enabled on Hopper GPU.")
|
||||
if envs.VLLM_USE_DEEP_GEMM_E8M0:
|
||||
logger.info_once("DeepGEMM E8M0 enabled on current platform.")
|
||||
return True
|
||||
|
||||
logger.info_once("DeepGEMM E8M0 disabled on current configuration.")
|
||||
|
||||
Reference in New Issue
Block a user