[Bugfix] Make Attention Backend Auto-Selection Batch-Invariance-Aware (#40193)

Signed-off-by: Srreyansh Sethi <srreyansh.sethi@gmail.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Srreyansh Sethi
2026-04-23 14:57:03 +00:00
committed by GitHub
co-authored by Matthew Bonanni mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent d0009ddb0b
commit b7a2605020
9 changed files with 38 additions and 50 deletions
+1 -8
View File
@@ -131,16 +131,9 @@ class TrainModel:
from vllm.model_executor.layers.batch_invariant import (
init_batch_invariance,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
# need to init all env vars for batch invariance which affect nccl ops
attn_backend = (
AttentionBackendEnum.TRITON_ATTN
if current_platform.is_rocm()
else AttentionBackendEnum.FLASH_ATTN
)
init_batch_invariance(attn_backend)
init_batch_invariance()
self.model = AutoModelForCausalLM.from_pretrained(
model_name, dtype=torch.bfloat16