[Kernel] Batch invariant NVFP4 linear using cutlass (#39912)

Signed-off-by: Jakub Zakrzewski <jzakrzewski@nvidia.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
Jakub Zakrzewski
2026-05-23 09:41:12 -04:00
committed by GitHub
co-authored by Wentao Ye Yongye Zhu
parent 3f3e862681
commit 5bb8d2767a
5 changed files with 211 additions and 50 deletions
+29 -12
View File
@@ -833,24 +833,41 @@ def init_nvfp4_linear_kernel() -> NvFp4LinearKernel:
current platform."""
config = NvFp4LinearLayerConfig()
# VLLM_BATCH_INVARIANT unconditionally forces emulation for deterministic
# execution. It overrides both --linear-backend and the deprecated env
# vars below.
# VLLM_BATCH_INVARIANT forces deterministic execution. Prefer the
# batch-invariant CUTLASS implementation when available, otherwise fall
# back to emulation. It overrides both --linear-backend and the deprecated
# env vars below.
force_kernel: type[NvFp4LinearKernel] | None = None
linear_backend = _get_linear_backend()
if envs.VLLM_BATCH_INVARIANT:
if linear_backend not in ("auto", "emulation"):
logger.warning_once(
"VLLM_BATCH_INVARIANT overrides --linear-backend=%s; using "
"the emulation backend for deterministic execution.",
linear_backend,
)
bi_supported, reason = CutlassNvFp4LinearKernel.is_supported()
if bi_supported:
if linear_backend not in ("auto", "cutlass"):
logger.warning_once(
"VLLM_BATCH_INVARIANT overrides --linear-backend=%s; "
"using the CUTLASS backend for deterministic execution.",
linear_backend,
)
else:
logger.info_once(
"VLLM_BATCH_INVARIANT forces NVFP4 linear to use the "
"CUTLASS backend for deterministic execution."
)
force_kernel = CutlassNvFp4LinearKernel
else:
if linear_backend not in ("auto", "emulation"):
logger.warning_once(
"VLLM_BATCH_INVARIANT overrides --linear-backend=%s; "
"using the emulation backend for deterministic execution.",
linear_backend,
)
logger.info_once(
"VLLM_BATCH_INVARIANT forces NVFP4 linear to use the "
"emulation backend for deterministic execution."
"VLLM_BATCH_INVARIANT is set but the batch-invariant NVFP4 "
"kernel is not supported on this platform; falling back to "
"emulation for deterministic execution. Reason: %s",
reason,
)
force_kernel = EmulationNvFp4LinearKernel
force_kernel = EmulationNvFp4LinearKernel
elif linear_backend == "auto":
# Deprecated env-var overrides — only honoured when --linear-backend
# is "auto". Deprecation warnings are emitted from vllm/envs.py.