[Feature] Add VLLM_USE_DEEP_GEMM_E8M0 Env to Control E8M0 Scale (#21968)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2025-08-11 09:39:08 -07:00
committed by GitHub
parent 8e13d9fe6d
commit f7dcce7a4a
9 changed files with 65 additions and 39 deletions
+37 -10
View File
@@ -31,19 +31,37 @@ def is_deep_gemm_supported() -> bool:
@functools.cache
def is_blackwell_deep_gemm_used() -> bool:
"""Return ``True`` if vLLM is configured to use DeepGEMM on a
Blackwell-class GPU.
def is_blackwell_deep_gemm_e8m0_used() -> bool:
"""Return ``True`` if vLLM is configured to use DeepGEMM "
"E8M0 scale on a Blackwell-class GPU.
"""
if not (envs.VLLM_USE_DEEP_GEMM and has_deep_gemm()):
if not (envs.VLLM_USE_DEEP_GEMM):
logger.debug_once("DeepGEMM E8M0 disabled: VLLM_USE_DEEP_GEMM=0.")
return False
if not has_deep_gemm():
logger.debug_once("DeepGEMM E8M0 disabled: DeepGEMM backend missing.")
return False
if not envs.VLLM_USE_DEEP_GEMM_E8M0:
logger.debug_once("DeepGEMM E8M0 disabled: VLLM_USE_DEEP_GEMM_E8M0=0.")
return False
_lazy_init()
if _fp8_gemm_nt_impl is None:
logger.debug_once(
"DeepGEMM E8M0 disabled: _fp8_gemm_nt_impl not found")
return False
return (current_platform.is_cuda()
and current_platform.is_device_capability(100))
enabled = (current_platform.is_cuda()
and current_platform.has_device_capability(100))
if enabled:
logger.debug_once("DeepGEMM E8M0 enabled on Blackwell GPU.")
else:
logger.debug_once(
"DeepGEMM E8M0 disabled: not running on Blackwell GPU.")
return enabled
def _missing(*_: Any, **__: Any) -> NoReturn:
@@ -109,21 +127,30 @@ def fp8_gemm_nt(*args, **kwargs):
_lazy_init()
if _fp8_gemm_nt_impl is None:
return _missing(*args, **kwargs)
return _fp8_gemm_nt_impl(*args, **kwargs)
return _fp8_gemm_nt_impl(
*args,
disable_ue8m0_cast=not is_blackwell_deep_gemm_e8m0_used(),
**kwargs)
def m_grouped_fp8_gemm_nt_contiguous(*args, **kwargs):
_lazy_init()
if _grouped_impl is None:
return _missing(*args, **kwargs)
return _grouped_impl(*args, **kwargs)
return _grouped_impl(
*args,
disable_ue8m0_cast=not is_blackwell_deep_gemm_e8m0_used(),
**kwargs)
def fp8_m_grouped_gemm_nt_masked(*args, **kwargs):
_lazy_init()
if _grouped_masked_impl is None:
return _missing(*args, **kwargs)
return _grouped_masked_impl(*args, **kwargs)
return _grouped_masked_impl(
*args,
disable_ue8m0_cast=not is_blackwell_deep_gemm_e8m0_used(),
**kwargs)
def _ceil_to_ue8m0(x: torch.Tensor):
@@ -181,6 +208,6 @@ __all__ = [
"m_grouped_fp8_gemm_nt_contiguous",
"fp8_m_grouped_gemm_nt_masked",
"per_block_cast_to_fp8",
"is_blackwell_deep_gemm_used",
"is_blackwell_deep_gemm_e8m0_used",
"is_deep_gemm_supported",
]