[Bugfix] Make Attention Backend Auto-Selection Batch-Invariance-Aware (#40193)

Signed-off-by: Srreyansh Sethi <srreyansh.sethi@gmail.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Srreyansh Sethi
2026-04-23 14:57:03 +00:00
committed by GitHub
co-authored by Matthew Bonanni mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent d0009ddb0b
commit b7a2605020
9 changed files with 38 additions and 50 deletions
+3 -39
View File
@@ -14,7 +14,6 @@ from vllm.triton_utils import tl, triton
from vllm.utils.mem_utils import get_max_shared_memory_bytes
from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import is_torch_equal_or_newer
from vllm.v1.attention.backends.registry import AttentionBackendEnum
logger = init_logger(__name__)
@@ -991,40 +990,7 @@ def enable_batch_invariant_mode():
torch.backends.cuda.preferred_blas_library(backend="cublaslt")
def override_envs_for_invariance(
attention_backend: AttentionBackendEnum | None,
):
decode_invariant_backends = [
AttentionBackendEnum.FLASH_ATTN, # best supported backend
AttentionBackendEnum.TRITON_ATTN,
]
supported_backends = decode_invariant_backends + [
# FlashInfer temporarily disabled due to invariant CTA sizes.
# See FlashInfer issue #2424
# AttentionBackendEnum.FLASHINFER,
AttentionBackendEnum.FLASH_ATTN_MLA,
AttentionBackendEnum.TRITON_MLA,
# Not yet supported MLA backends
# AttentionBackendEnum.FLASHMLA,
# AttentionBackendEnum.FLEX_ATTENTION, # IMA issue
# AttentionBackendEnum.FLASHINFER_MLA, # PR #28967
]
if attention_backend not in supported_backends:
supported_names = [b.name for b in supported_backends]
backend_name = attention_backend.name if attention_backend else None
error = (
"VLLM batch_invariant mode requires an attention backend in "
f"{supported_names}, but got '{backend_name}'. "
"Please use --attention-backend or attention_config to set "
"one of the supported backends before enabling batch_invariant."
)
raise RuntimeError(error)
if attention_backend not in decode_invariant_backends:
warning = (
"You are using a non-decode-invariant form of batch invariance. "
"This will not be invariant between prefill and decode."
)
logger.warning_once(warning)
def override_envs_for_invariance():
os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0"
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
@@ -1045,12 +1011,10 @@ def override_envs_for_invariance(
os.environ["VLLM_USE_AOT_COMPILE"] = "0"
def init_batch_invariance(
attention_backend: AttentionBackendEnum | None,
):
def init_batch_invariance():
# this will hit all the csrc overrides as well
if envs.VLLM_BATCH_INVARIANT:
override_envs_for_invariance(attention_backend)
override_envs_for_invariance()
enable_batch_invariant_mode()
# Disable TF32 for batch invariance - it causes non-deterministic rounding