[Hardware][XPU] Register batch-invariant kernels for XPU (#41934)

Signed-off-by: tzielinski-habana <tomasz.zielinski@intel.com>
Signed-off-by: Tomasz Zielinski <85164140+tzielinski-habana@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Chendi.Xue <chendi.xue@intel.com>
This commit is contained in:
Tomasz Zielinski
2026-07-15 11:19:44 -04:00
committed by GitHub
co-authored by gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Chendi.Xue
parent 05eed72aec
commit 61141ed265
6 changed files with 127 additions and 55 deletions
+28 -23
View File
@@ -904,37 +904,41 @@ def enable_batch_invariant_mode():
_batch_invariant_MODE = True
_batch_invariant_LIB = torch.library.Library("aten", "IMPL")
if current_platform.is_device_capability_family(80):
# SM80 (Ampere) cannot rely on cuBLASLt-only determinism; install the
# triton persistent matmul overrides for mm/addmm/matmul/linear.
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, "CUDA")
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, "CUDA")
_batch_invariant_LIB.impl("aten::matmul", matmul_batch_invariant, "CUDA")
_batch_invariant_LIB.impl("aten::linear", linear_batch_invariant, "CUDA")
else:
# Hopper (SM90) and Blackwell (SM100): the only source of batch
# variance is split-k, which we disable via the cuBLAS workspace
# config.
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":16:8"
os.environ["CUBLASLT_WORKSPACE_SIZE"] = "1"
key = current_platform.dispatch_key
# Triton bmm/persistent-matmul kernels read this for the FP16 N-tile size;
# set unconditionally because bmm is overridden on all CUDA platforms.
if current_platform.is_cuda():
if current_platform.is_device_capability_family(80):
# SM80 (Ampere) cannot rely on cuBLASLt-only determinism; install the
# triton persistent matmul overrides for mm/addmm/matmul/linear.
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, key)
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, key)
_batch_invariant_LIB.impl("aten::matmul", matmul_batch_invariant, key)
_batch_invariant_LIB.impl("aten::linear", linear_batch_invariant, key)
else:
# Hopper (SM90) and Blackwell (SM100): the only source of batch
# variance is split-k, which we disable via the cuBLAS workspace
# config.
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":16:8"
os.environ["CUBLASLT_WORKSPACE_SIZE"] = "1"
_fp16_block_size_n = 256 if get_max_shared_memory_bytes() > 106496 else 128
elif current_platform.is_xpu():
_batch_invariant_LIB.impl("aten::mm", mm_batch_invariant, key)
_batch_invariant_LIB.impl("aten::addmm", addmm_batch_invariant, key)
# TODO: register matmul and linear for XPU
# once suitable Triton kernels are implemented
_batch_invariant_LIB.impl(
"aten::_log_softmax", _log_softmax_batch_invariant, "CUDA"
)
_batch_invariant_LIB.impl("aten::softmax", softmax_batch_invariant, "CUDA")
_batch_invariant_LIB.impl("aten::_softmax", softmax_batch_invariant, "CUDA")
_batch_invariant_LIB.impl("aten::mean.dim", mean_batch_invariant, "CUDA")
_fp16_block_size_n = 128
_batch_invariant_LIB.impl("aten::_log_softmax", _log_softmax_batch_invariant, key)
_batch_invariant_LIB.impl("aten::softmax", softmax_batch_invariant, key)
_batch_invariant_LIB.impl("aten::_softmax", softmax_batch_invariant, key)
_batch_invariant_LIB.impl("aten::mean.dim", mean_batch_invariant, key)
# torch 2.12+ registers a built-in Triton bmm kernel for CUDA
# (torch._native.ops.bmm_outer_product), so we need allow_override
# to replace it at the dispatcher level.
_batch_invariant_LIB.impl(
"aten::bmm", bmm_batch_invariant, "CUDA", allow_override=True
"aten::bmm", bmm_batch_invariant, key, allow_override=True
)
torch.bmm = bmm_batch_invariant
@@ -947,7 +951,8 @@ def enable_batch_invariant_mode():
torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = (
reduced_precision_val
)
torch.backends.cuda.preferred_blas_library(backend="cublaslt")
if current_platform.is_cuda():
torch.backends.cuda.preferred_blas_library(backend="cublaslt")
def override_envs_for_invariance():