[Kernel] Batch invariant NVFP4 linear using cutlass (#39912)
Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
co-authored by
Wentao Ye
Yongye Zhu
parent
3f3e862681
commit
5bb8d2767a
@@ -833,24 +833,41 @@ def init_nvfp4_linear_kernel() -> NvFp4LinearKernel:
|
||||
current platform."""
|
||||
config = NvFp4LinearLayerConfig()
|
||||
|
||||
# VLLM_BATCH_INVARIANT unconditionally forces emulation for deterministic
|
||||
# execution. It overrides both --linear-backend and the deprecated env
|
||||
# vars below.
|
||||
# VLLM_BATCH_INVARIANT forces deterministic execution. Prefer the
|
||||
# batch-invariant CUTLASS implementation when available, otherwise fall
|
||||
# back to emulation. It overrides both --linear-backend and the deprecated
|
||||
# env vars below.
|
||||
force_kernel: type[NvFp4LinearKernel] | None = None
|
||||
linear_backend = _get_linear_backend()
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
if linear_backend not in ("auto", "emulation"):
|
||||
logger.warning_once(
|
||||
"VLLM_BATCH_INVARIANT overrides --linear-backend=%s; using "
|
||||
"the emulation backend for deterministic execution.",
|
||||
linear_backend,
|
||||
)
|
||||
bi_supported, reason = CutlassNvFp4LinearKernel.is_supported()
|
||||
if bi_supported:
|
||||
if linear_backend not in ("auto", "cutlass"):
|
||||
logger.warning_once(
|
||||
"VLLM_BATCH_INVARIANT overrides --linear-backend=%s; "
|
||||
"using the CUTLASS backend for deterministic execution.",
|
||||
linear_backend,
|
||||
)
|
||||
else:
|
||||
logger.info_once(
|
||||
"VLLM_BATCH_INVARIANT forces NVFP4 linear to use the "
|
||||
"CUTLASS backend for deterministic execution."
|
||||
)
|
||||
force_kernel = CutlassNvFp4LinearKernel
|
||||
else:
|
||||
if linear_backend not in ("auto", "emulation"):
|
||||
logger.warning_once(
|
||||
"VLLM_BATCH_INVARIANT overrides --linear-backend=%s; "
|
||||
"using the emulation backend for deterministic execution.",
|
||||
linear_backend,
|
||||
)
|
||||
logger.info_once(
|
||||
"VLLM_BATCH_INVARIANT forces NVFP4 linear to use the "
|
||||
"emulation backend for deterministic execution."
|
||||
"VLLM_BATCH_INVARIANT is set but the batch-invariant NVFP4 "
|
||||
"kernel is not supported on this platform; falling back to "
|
||||
"emulation for deterministic execution. Reason: %s",
|
||||
reason,
|
||||
)
|
||||
force_kernel = EmulationNvFp4LinearKernel
|
||||
force_kernel = EmulationNvFp4LinearKernel
|
||||
elif linear_backend == "auto":
|
||||
# Deprecated env-var overrides — only honoured when --linear-backend
|
||||
# is "auto". Deprecation warnings are emitted from vllm/envs.py.
|
||||
|
||||
Reference in New Issue
Block a user