[Feature] Add VLLM_USE_DEEP_GEMM_E8M0 Env to Control E8M0 Scale (#21968)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
+37
-10
@@ -31,19 +31,37 @@ def is_deep_gemm_supported() -> bool:
|
||||
|
||||
|
||||
@functools.cache
|
||||
def is_blackwell_deep_gemm_used() -> bool:
|
||||
"""Return ``True`` if vLLM is configured to use DeepGEMM on a
|
||||
Blackwell-class GPU.
|
||||
def is_blackwell_deep_gemm_e8m0_used() -> bool:
|
||||
"""Return ``True`` if vLLM is configured to use DeepGEMM "
|
||||
"E8M0 scale on a Blackwell-class GPU.
|
||||
"""
|
||||
if not (envs.VLLM_USE_DEEP_GEMM and has_deep_gemm()):
|
||||
if not (envs.VLLM_USE_DEEP_GEMM):
|
||||
logger.debug_once("DeepGEMM E8M0 disabled: VLLM_USE_DEEP_GEMM=0.")
|
||||
return False
|
||||
|
||||
if not has_deep_gemm():
|
||||
logger.debug_once("DeepGEMM E8M0 disabled: DeepGEMM backend missing.")
|
||||
return False
|
||||
|
||||
if not envs.VLLM_USE_DEEP_GEMM_E8M0:
|
||||
logger.debug_once("DeepGEMM E8M0 disabled: VLLM_USE_DEEP_GEMM_E8M0=0.")
|
||||
return False
|
||||
|
||||
_lazy_init()
|
||||
|
||||
if _fp8_gemm_nt_impl is None:
|
||||
logger.debug_once(
|
||||
"DeepGEMM E8M0 disabled: _fp8_gemm_nt_impl not found")
|
||||
return False
|
||||
|
||||
return (current_platform.is_cuda()
|
||||
and current_platform.is_device_capability(100))
|
||||
enabled = (current_platform.is_cuda()
|
||||
and current_platform.has_device_capability(100))
|
||||
if enabled:
|
||||
logger.debug_once("DeepGEMM E8M0 enabled on Blackwell GPU.")
|
||||
else:
|
||||
logger.debug_once(
|
||||
"DeepGEMM E8M0 disabled: not running on Blackwell GPU.")
|
||||
return enabled
|
||||
|
||||
|
||||
def _missing(*_: Any, **__: Any) -> NoReturn:
|
||||
@@ -109,21 +127,30 @@ def fp8_gemm_nt(*args, **kwargs):
|
||||
_lazy_init()
|
||||
if _fp8_gemm_nt_impl is None:
|
||||
return _missing(*args, **kwargs)
|
||||
return _fp8_gemm_nt_impl(*args, **kwargs)
|
||||
return _fp8_gemm_nt_impl(
|
||||
*args,
|
||||
disable_ue8m0_cast=not is_blackwell_deep_gemm_e8m0_used(),
|
||||
**kwargs)
|
||||
|
||||
|
||||
def m_grouped_fp8_gemm_nt_contiguous(*args, **kwargs):
|
||||
_lazy_init()
|
||||
if _grouped_impl is None:
|
||||
return _missing(*args, **kwargs)
|
||||
return _grouped_impl(*args, **kwargs)
|
||||
return _grouped_impl(
|
||||
*args,
|
||||
disable_ue8m0_cast=not is_blackwell_deep_gemm_e8m0_used(),
|
||||
**kwargs)
|
||||
|
||||
|
||||
def fp8_m_grouped_gemm_nt_masked(*args, **kwargs):
|
||||
_lazy_init()
|
||||
if _grouped_masked_impl is None:
|
||||
return _missing(*args, **kwargs)
|
||||
return _grouped_masked_impl(*args, **kwargs)
|
||||
return _grouped_masked_impl(
|
||||
*args,
|
||||
disable_ue8m0_cast=not is_blackwell_deep_gemm_e8m0_used(),
|
||||
**kwargs)
|
||||
|
||||
|
||||
def _ceil_to_ue8m0(x: torch.Tensor):
|
||||
@@ -181,6 +208,6 @@ __all__ = [
|
||||
"m_grouped_fp8_gemm_nt_contiguous",
|
||||
"fp8_m_grouped_gemm_nt_masked",
|
||||
"per_block_cast_to_fp8",
|
||||
"is_blackwell_deep_gemm_used",
|
||||
"is_blackwell_deep_gemm_e8m0_used",
|
||||
"is_deep_gemm_supported",
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user