[Bugfix] Make Attention Backend Auto-Selection Batch-Invariance-Aware (#40193)
Signed-off-by: Srreyansh Sethi <srreyansh.sethi@gmail.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Matthew Bonanni
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d0009ddb0b
commit
b7a2605020
@@ -1027,11 +1027,10 @@ def init_worker_distributed_environment(
|
||||
backend: str = "nccl",
|
||||
) -> None:
|
||||
"""Initialize the distributed environment."""
|
||||
attention_config = vllm_config.attention_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
from vllm.model_executor.layers.batch_invariant import init_batch_invariance
|
||||
|
||||
init_batch_invariance(attention_config.backend)
|
||||
init_batch_invariance()
|
||||
override_envs_for_eplb(parallel_config)
|
||||
set_custom_all_reduce(not parallel_config.disable_custom_all_reduce)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user