From 785b990116ab2b7c77f5d946b2f52c2a8f9b9515 Mon Sep 17 00:00:00 2001 From: Flora Feng <4florafeng@gmail.com> Date: Thu, 1 Oct 2026 22:25:48 -0400 Subject: [PATCH] [Distributed] Enable custom all-reduce under VLLM_BATCH_INVARIANT (#58623) Signed-off-by: Flora Feng <4florafeng@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> --- docs/features/batch_invariance.md | 8 +- tests/distributed/test_custom_all_reduce.py | 90 +++++++++++++++++++ tests/v1/determinism/test_batch_invariance.py | 20 ----- vllm/config/parallel.py | 4 - .../device_communicators/cuda_communicator.py | 14 ++- .../device_communicators/custom_all_reduce.py | 23 ++++- .../determinism/batch_invariant.py | 2 + 7 files changed, 130 insertions(+), 31 deletions(-) diff --git a/docs/features/batch_invariance.md b/docs/features/batch_invariance.md index ff704348b5..182559141c 100644 --- a/docs/features/batch_invariance.md +++ b/docs/features/batch_invariance.md @@ -146,11 +146,15 @@ When batch invariance is enabled, vLLM: 1. Uses deterministic kernel implementations for attention and other operations 2. Ensures consistent numerical behavior across different batch sizes -3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant) -4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell), +3. Disables certain optimizations that may introduce non-determinism (such as sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant) +4. Under tensor parallelism, keeps custom all-reduce on with a fixed reduction order (the 1-stage kernel is pinned, and large inputs are reduced in fixed-size chunks), and disables FlashInfer, AITER and QuickReduce all-reduce +5. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell), runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out (not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes). +!!! warning + Batch invariance under tensor parallelism is not yet supported for all-reduces whose size isn't a multiple of 16 bytes, for example a hidden size that isn't a multiple of 8 in fp16/bf16. Such tensors can switch between NCCL and custom all-reduce depending on batch size. All validated models meet this requirement. + !!! note Enabling batch invariance may impact performance compared to the default non-deterministic mode. This trade-off is intentional to guarantee reproducibility. diff --git a/tests/distributed/test_custom_all_reduce.py b/tests/distributed/test_custom_all_reduce.py index 23d66ee973..84ff524499 100644 --- a/tests/distributed/test_custom_all_reduce.py +++ b/tests/distributed/test_custom_all_reduce.py @@ -205,6 +205,42 @@ def test_custom_allreduce_filters_dtype( assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected +@pytest.mark.parametrize("batch_invariant", [False, True]) +def test_custom_allreduce_size_gate_ignored_under_batch_invariance( + batch_invariant: bool, +) -> None: + """Batch invariance must not switch backends based on the input size.""" + communicator = car.CustomAllreduce.__new__(car.CustomAllreduce) + communicator.disabled = False + communicator.world_size = 2 + communicator.max_size = 1024 + communicator.batch_invariant = batch_invariant + communicator._ptr = 0 + + oversized = torch.empty(1024, dtype=torch.float16) + assert communicator.should_custom_ar(oversized) is batch_invariant + + +@pytest.mark.parametrize("batch_invariant", [False, True]) +def test_custom_reduce_scatter_disabled_under_batch_invariance( + monkeypatch, batch_invariant: bool +) -> None: + """Reduce-scatter stays on one backend under batch invariance.""" + monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True) + communicator = car.CustomAllreduce.__new__(car.CustomAllreduce) + communicator.disabled = False + communicator.world_size = 2 + communicator.fully_connected = True + communicator.mnnvl_only = False + communicator.mnnvl_multicast_ptr = 0 + communicator.max_reduce_scatter_size = 1024 + communicator.max_mnnvl_reduce_scatter_size = 1024 + communicator.batch_invariant = batch_invariant + + in_range = torch.empty(64, dtype=torch.float16) + assert communicator.should_custom_reduce_scatter(in_range) is not batch_invariant + + @pytest.mark.parametrize( ("major", "local_multicast", "expected"), [ @@ -622,6 +658,60 @@ def eager_allreduce( torch.testing.assert_close(out, inp * (tp_size**num_communication)) +@ray.remote(num_gpus=1, max_calls=1) +def chunked_allreduce( + monkeypatch: pytest.MonkeyPatch, + tp_size, + pp_size, + rank, + distributed_init_port, +): + """Inputs above max_size are reduced in chunks and match the fixed-order + reference bitwise, both eagerly and inside a CUDA graph.""" + with monkeypatch.context() as m: + m.delenv("CUDA_VISIBLE_DEVICES", raising=False) + m.delenv("HIP_VISIBLE_DEVICES", raising=False) + m.setenv("VLLM_CUSTOM_ALLREDUCE_ALGO", "1stage") + device = torch.device(f"cuda:{rank}") + torch.accelerator.set_device_index(device) + init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port) + ensure_model_parallel_initialized(tp_size, pp_size) + group = get_tp_group().device_group + fa = get_tp_group().device_communicator.ca_comm + + # Two full chunks plus a partial one. + dtype = torch.bfloat16 + chunk_numel = fa.max_size // dtype.itemsize + inp = torch.randn(2 * chunk_numel + 4096, device=device).to(dtype) + gathered = [torch.empty_like(inp) for _ in range(tp_size)] + dist.all_gather(gathered, inp, group=group) + ref = gathered[0].float() + for peer in gathered[1:]: + ref = ref + peer.float() + ref = ref.to(torch.bfloat16) + + out = fa.all_reduce(inp, registered=False) + assert torch.equal(out, ref) + + # Weak-contiguous but not C-contiguous: a transposed matrix. + inp_t, ref_t = inp.view(-1, 4096).t(), ref.view(-1, 4096).t() + assert torch.equal(fa.all_reduce(inp_t, registered=False), ref_t) + + with fa.capture(): + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + out = fa.all_reduce(inp, registered=True) + graph.replay() + assert torch.equal(out, ref) + + +@pytest.mark.parametrize("tp_size", [2, 4]) +def test_custom_allreduce_chunked(monkeypatch: pytest.MonkeyPatch, tp_size): + if tp_size > torch.accelerator.device_count(): + pytest.skip("Not enough GPUs to run the test.") + multi_process_parallel(monkeypatch, tp_size, 1, chunked_allreduce) + + @pytest.mark.parametrize("tp_size", [2]) @pytest.mark.parametrize("pipeline_parallel_size", [1, 2]) @pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce]) diff --git a/tests/v1/determinism/test_batch_invariance.py b/tests/v1/determinism/test_batch_invariance.py index b167078e06..f49d171f6f 100644 --- a/tests/v1/determinism/test_batch_invariance.py +++ b/tests/v1/determinism/test_batch_invariance.py @@ -176,17 +176,6 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN( random.seed(seed) tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1")) - # For batch invariance, disable custom all-reduce to ensure deterministic - # all-reduce operations (custom all-reduce may not be deterministic) - import vllm.envs as envs - - disable_custom_ar = envs.VLLM_BATCH_INVARIANT - - if disable_custom_ar: - print(f"\n{'=' * 80}") - print(f"BATCH INVARIANCE MODE: Disabling custom all-reduce (TP={tp_size})") - print(f"{'=' * 80}\n") - llm = LLM( model=TEST_MODEL, tensor_parallel_size=tp_size, @@ -744,15 +733,6 @@ def test_decode_logprobs_match_prefill_logprobs( random.seed(seed) tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1")) - import vllm.envs as envs - - disable_custom_ar = envs.VLLM_BATCH_INVARIANT - - if disable_custom_ar: - print(f"\n{'=' * 80}") - print(f"BATCH INVARIANCE MODE: Disabling custom all-reduce (TP={tp_size})") - print(f"{'=' * 80}\n") - llm = LLM( model=TEST_MODEL, tensor_parallel_size=tp_size, diff --git a/vllm/config/parallel.py b/vllm/config/parallel.py index b9136cfbb2..f4b825eb33 100644 --- a/vllm/config/parallel.py +++ b/vllm/config/parallel.py @@ -1087,10 +1087,6 @@ class ParallelConfig: # Lazy import to avoid circular import from vllm.v1.executor import Executor - # Enable batch invariance settings if requested - if envs.VLLM_BATCH_INVARIANT: - self.disable_custom_all_reduce = True - if ( self.distributed_executor_backend is not None and not isinstance(self.distributed_executor_backend, str) diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index fb0c281b4a..ef790f7732 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -68,8 +68,11 @@ class CudaCommunicator(DeviceCommunicatorBase): envs.VLLM_ALLREDUCE_USE_FLASHINFER_PCIE_IPC and not envs.VLLM_BATCH_INVARIANT ) - use_aiter_allreduce = use_custom_allreduce and bool( - rocm_aiter_ops.is_custom_all_reduce_enabled() + # Neither AITER nor QuickReduce all-reduce has a fixed reduction order. + use_aiter_allreduce = ( + use_custom_allreduce + and not envs.VLLM_BATCH_INVARIANT + and bool(rocm_aiter_ops.is_custom_all_reduce_enabled()) ) self.use_custom_allreduce = use_custom_allreduce @@ -166,7 +169,12 @@ class CudaCommunicator(DeviceCommunicatorBase): else: self.use_aiter_ag_rs = True - if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm(): + if ( + use_custom_allreduce + and self.world_size > 1 + and current_platform.is_rocm() + and not envs.VLLM_BATCH_INVARIANT + ): # Initialize a custom quick all-reduce implementation for AMD. # Quick reduce is designed as a complement to custom allreduce # (vLLM's or AITER's), so it is initialized for either backend. diff --git a/vllm/distributed/device_communicators/custom_all_reduce.py b/vllm/distributed/device_communicators/custom_all_reduce.py index b6fb540c7d..9e8477e494 100644 --- a/vllm/distributed/device_communicators/custom_all_reduce.py +++ b/vllm/distributed/device_communicators/custom_all_reduce.py @@ -119,6 +119,7 @@ from vllm.distributed.utils import is_weak_contiguous # noqa: E402 class CustomAllreduce: _SUPPORTED_WORLD_SIZES = [2, 4, 6, 8, 16] + batch_invariant = False _DEFAULT_ALL_GATHER_MAX_SIZE = 2 * 1024 * 1024 _DEFAULT_MNNVL_ALL_GATHER_MAX_SIZES = { 2: 8 * 1024 * 1024, @@ -179,6 +180,7 @@ class CustomAllreduce: self.mnnvl_multimem_rs_local_ptr = 0 self.mnnvl_multimem_rs_multicast_ptr = 0 self.mnnvl_only = False + self.batch_invariant = envs.VLLM_BATCH_INVARIANT if not custom_ar: # disable because of missing custom allreduce library @@ -504,7 +506,7 @@ class CustomAllreduce: # for 4 or more non NVLink-capable GPUs, custom allreduce provides # little performance improvement over NCCL. if self.world_size == 2 or self.fully_connected: - return inp_size < self.max_size + return self.batch_invariant or inp_size < self.max_size return False def all_reduce( @@ -518,13 +520,27 @@ class CustomAllreduce: """ if out is None: out = torch.empty_like(inp) + chunk_numel = self.max_size // inp.element_size() + if inp.numel() <= chunk_numel: + self._all_reduce_chunk(inp, out, registered) + return out + # Storage-order views: the gate only admits one contiguous block. + flat_inp = inp.as_strided((inp.numel(),), (1,), inp.storage_offset()) + flat_out = out.as_strided((out.numel(),), (1,), out.storage_offset()) + for start in range(0, flat_inp.numel(), chunk_numel): + end = start + chunk_numel + self._all_reduce_chunk(flat_inp[start:end], flat_out[start:end], registered) + return out + + def _all_reduce_chunk( + self, inp: torch.Tensor, out: torch.Tensor, registered: bool + ) -> None: if registered: ops.all_reduce(self._ptr, inp, out, 0, 0) else: ops.all_reduce( self._ptr, inp, out, self.buffer_ptrs[self.rank], self.max_size ) - return out def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor | None: """The main allreduce API that provides support for cuda graph.""" @@ -603,6 +619,9 @@ class CustomAllreduce: ) -> _ReduceScatterBackend | None: if self.disabled or not current_platform.is_cuda(): return None + # The size gates below would switch backends per batch. + if self.batch_invariant: + return None if self.world_size == 16 and not self.mnnvl_only: return None inp_size = inp.nbytes diff --git a/vllm/model_executor/determinism/batch_invariant.py b/vllm/model_executor/determinism/batch_invariant.py index 308e8c4f42..688e14d709 100644 --- a/vllm/model_executor/determinism/batch_invariant.py +++ b/vllm/model_executor/determinism/batch_invariant.py @@ -1139,6 +1139,8 @@ def enable_batch_invariant_mode(): def override_envs_for_invariance(): os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0" + # Only the 1-stage kernel has a size- and rank-independent reduction order. + os.environ["VLLM_CUSTOM_ALLREDUCE_ALGO"] = "1stage" os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"