diff --git a/docs/features/batch_invariance.md b/docs/features/batch_invariance.md index 7353bd3594..0fad4682fc 100644 --- a/docs/features/batch_invariance.md +++ b/docs/features/batch_invariance.md @@ -144,7 +144,7 @@ When batch invariance is enabled, vLLM: 1. Uses deterministic kernel implementations for attention and other operations 2. Ensures consistent numerical behavior across different batch sizes -3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode) +3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant) 4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell), runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out (not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes). diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index c50edf4363..220da0c72b 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -1718,6 +1718,26 @@ class VllmConfig: ) self.compilation_config.mode = CompilationMode.NONE + # TODO: This is a stopgap that turns sequence parallelism / async TP + # off under batch invariance. Make the sequence-parallel reduce-scatter + # path batch-invariant instead so they can stay enabled (#56370). + # Sequence parallelism / async TP rewrite all_reduce + rms_norm into a + # reduce-scatter whose reduction order depends on the batch, so they + # are not batch-invariant. Decide this before the breakable-CUDA-graph + # default below, which declines to auto-enable when they are on. + pass_config = self.compilation_config.pass_config + if envs.VLLM_BATCH_INVARIANT and ( + pass_config.enable_sp or pass_config.fuse_gemm_comms + ): + logger.warning_once( + "Disabling sequence parallelism and async TP " + "(pass_config.enable_sp / fuse_gemm_comms) when " + "VLLM_BATCH_INVARIANT is enabled: the reduce-scatter path " + "is not batch-invariant (see vllm-project/vllm#56370)." + ) + pass_config.enable_sp = False + pass_config.fuse_gemm_comms = False + breakable_cudagraph_enabled = self._maybe_enable_breakable_cudagraph() if not breakable_cudagraph_enabled and (