[Distributed] Enable custom all-reduce under VLLM_BATCH_INVARIANT (#58623)

Signed-off-by: Flora Feng <4florafeng@gmail.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-10-02 02:25:48 +00:00
committed by GitHub
parent e3cae8d2ac
commit 785b990116
7 changed files with 130 additions and 31 deletions
+6 -2
View File
@@ -146,11 +146,15 @@ When batch invariance is enabled, vLLM:
1. Uses deterministic kernel implementations for attention and other operations
2. Ensures consistent numerical behavior across different batch sizes
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant)
4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell),
3. Disables certain optimizations that may introduce non-determinism (such as sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant)
4. Under tensor parallelism, keeps custom all-reduce on with a fixed reduction order (the 1-stage kernel is pinned, and large inputs are reduced in fixed-size chunks), and disables FlashInfer, AITER and QuickReduce all-reduce
5. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell),
runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out
(not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes).
!!! warning
Batch invariance under tensor parallelism is not yet supported for all-reduces whose size isn't a multiple of 16 bytes, for example a hidden size that isn't a multiple of 8 in fp16/bf16. Such tensors can switch between NCCL and custom all-reduce depending on batch size. All validated models meet this requirement.
!!! note
Enabling batch invariance may impact performance compared to the default non-deterministic mode. This trade-off is intentional to guarantee reproducibility.
@@ -205,6 +205,42 @@ def test_custom_allreduce_filters_dtype(
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
@pytest.mark.parametrize("batch_invariant", [False, True])
def test_custom_allreduce_size_gate_ignored_under_batch_invariance(
batch_invariant: bool,
) -> None:
"""Batch invariance must not switch backends based on the input size."""
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator.world_size = 2
communicator.max_size = 1024
communicator.batch_invariant = batch_invariant
communicator._ptr = 0
oversized = torch.empty(1024, dtype=torch.float16)
assert communicator.should_custom_ar(oversized) is batch_invariant
@pytest.mark.parametrize("batch_invariant", [False, True])
def test_custom_reduce_scatter_disabled_under_batch_invariance(
monkeypatch, batch_invariant: bool
) -> None:
"""Reduce-scatter stays on one backend under batch invariance."""
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator.world_size = 2
communicator.fully_connected = True
communicator.mnnvl_only = False
communicator.mnnvl_multicast_ptr = 0
communicator.max_reduce_scatter_size = 1024
communicator.max_mnnvl_reduce_scatter_size = 1024
communicator.batch_invariant = batch_invariant
in_range = torch.empty(64, dtype=torch.float16)
assert communicator.should_custom_reduce_scatter(in_range) is not batch_invariant
@pytest.mark.parametrize(
("major", "local_multicast", "expected"),
[
@@ -622,6 +658,60 @@ def eager_allreduce(
torch.testing.assert_close(out, inp * (tp_size**num_communication))
@ray.remote(num_gpus=1, max_calls=1)
def chunked_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
"""Inputs above max_size are reduced in chunks and match the fixed-order
reference bitwise, both eagerly and inside a CUDA graph."""
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
m.setenv("VLLM_CUSTOM_ALLREDUCE_ALGO", "1stage")
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
group = get_tp_group().device_group
fa = get_tp_group().device_communicator.ca_comm
# Two full chunks plus a partial one.
dtype = torch.bfloat16
chunk_numel = fa.max_size // dtype.itemsize
inp = torch.randn(2 * chunk_numel + 4096, device=device).to(dtype)
gathered = [torch.empty_like(inp) for _ in range(tp_size)]
dist.all_gather(gathered, inp, group=group)
ref = gathered[0].float()
for peer in gathered[1:]:
ref = ref + peer.float()
ref = ref.to(torch.bfloat16)
out = fa.all_reduce(inp, registered=False)
assert torch.equal(out, ref)
# Weak-contiguous but not C-contiguous: a transposed matrix.
inp_t, ref_t = inp.view(-1, 4096).t(), ref.view(-1, 4096).t()
assert torch.equal(fa.all_reduce(inp_t, registered=False), ref_t)
with fa.capture():
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
out = fa.all_reduce(inp, registered=True)
graph.replay()
assert torch.equal(out, ref)
@pytest.mark.parametrize("tp_size", [2, 4])
def test_custom_allreduce_chunked(monkeypatch: pytest.MonkeyPatch, tp_size):
if tp_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
multi_process_parallel(monkeypatch, tp_size, 1, chunked_allreduce)
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
@@ -176,17 +176,6 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
random.seed(seed)
tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1"))
# For batch invariance, disable custom all-reduce to ensure deterministic
# all-reduce operations (custom all-reduce may not be deterministic)
import vllm.envs as envs
disable_custom_ar = envs.VLLM_BATCH_INVARIANT
if disable_custom_ar:
print(f"\n{'=' * 80}")
print(f"BATCH INVARIANCE MODE: Disabling custom all-reduce (TP={tp_size})")
print(f"{'=' * 80}\n")
llm = LLM(
model=TEST_MODEL,
tensor_parallel_size=tp_size,
@@ -744,15 +733,6 @@ def test_decode_logprobs_match_prefill_logprobs(
random.seed(seed)
tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1"))
import vllm.envs as envs
disable_custom_ar = envs.VLLM_BATCH_INVARIANT
if disable_custom_ar:
print(f"\n{'=' * 80}")
print(f"BATCH INVARIANCE MODE: Disabling custom all-reduce (TP={tp_size})")
print(f"{'=' * 80}\n")
llm = LLM(
model=TEST_MODEL,
tensor_parallel_size=tp_size,
-4
View File
@@ -1087,10 +1087,6 @@ class ParallelConfig:
# Lazy import to avoid circular import
from vllm.v1.executor import Executor
# Enable batch invariance settings if requested
if envs.VLLM_BATCH_INVARIANT:
self.disable_custom_all_reduce = True
if (
self.distributed_executor_backend is not None
and not isinstance(self.distributed_executor_backend, str)
@@ -68,8 +68,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
envs.VLLM_ALLREDUCE_USE_FLASHINFER_PCIE_IPC
and not envs.VLLM_BATCH_INVARIANT
)
use_aiter_allreduce = use_custom_allreduce and bool(
rocm_aiter_ops.is_custom_all_reduce_enabled()
# Neither AITER nor QuickReduce all-reduce has a fixed reduction order.
use_aiter_allreduce = (
use_custom_allreduce
and not envs.VLLM_BATCH_INVARIANT
and bool(rocm_aiter_ops.is_custom_all_reduce_enabled())
)
self.use_custom_allreduce = use_custom_allreduce
@@ -166,7 +169,12 @@ class CudaCommunicator(DeviceCommunicatorBase):
else:
self.use_aiter_ag_rs = True
if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm():
if (
use_custom_allreduce
and self.world_size > 1
and current_platform.is_rocm()
and not envs.VLLM_BATCH_INVARIANT
):
# Initialize a custom quick all-reduce implementation for AMD.
# Quick reduce is designed as a complement to custom allreduce
# (vLLM's or AITER's), so it is initialized for either backend.
@@ -119,6 +119,7 @@ from vllm.distributed.utils import is_weak_contiguous # noqa: E402
class CustomAllreduce:
_SUPPORTED_WORLD_SIZES = [2, 4, 6, 8, 16]
batch_invariant = False
_DEFAULT_ALL_GATHER_MAX_SIZE = 2 * 1024 * 1024
_DEFAULT_MNNVL_ALL_GATHER_MAX_SIZES = {
2: 8 * 1024 * 1024,
@@ -179,6 +180,7 @@ class CustomAllreduce:
self.mnnvl_multimem_rs_local_ptr = 0
self.mnnvl_multimem_rs_multicast_ptr = 0
self.mnnvl_only = False
self.batch_invariant = envs.VLLM_BATCH_INVARIANT
if not custom_ar:
# disable because of missing custom allreduce library
@@ -504,7 +506,7 @@ class CustomAllreduce:
# for 4 or more non NVLink-capable GPUs, custom allreduce provides
# little performance improvement over NCCL.
if self.world_size == 2 or self.fully_connected:
return inp_size < self.max_size
return self.batch_invariant or inp_size < self.max_size
return False
def all_reduce(
@@ -518,13 +520,27 @@ class CustomAllreduce:
"""
if out is None:
out = torch.empty_like(inp)
chunk_numel = self.max_size // inp.element_size()
if inp.numel() <= chunk_numel:
self._all_reduce_chunk(inp, out, registered)
return out
# Storage-order views: the gate only admits one contiguous block.
flat_inp = inp.as_strided((inp.numel(),), (1,), inp.storage_offset())
flat_out = out.as_strided((out.numel(),), (1,), out.storage_offset())
for start in range(0, flat_inp.numel(), chunk_numel):
end = start + chunk_numel
self._all_reduce_chunk(flat_inp[start:end], flat_out[start:end], registered)
return out
def _all_reduce_chunk(
self, inp: torch.Tensor, out: torch.Tensor, registered: bool
) -> None:
if registered:
ops.all_reduce(self._ptr, inp, out, 0, 0)
else:
ops.all_reduce(
self._ptr, inp, out, self.buffer_ptrs[self.rank], self.max_size
)
return out
def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor | None:
"""The main allreduce API that provides support for cuda graph."""
@@ -603,6 +619,9 @@ class CustomAllreduce:
) -> _ReduceScatterBackend | None:
if self.disabled or not current_platform.is_cuda():
return None
# The size gates below would switch backends per batch.
if self.batch_invariant:
return None
if self.world_size == 16 and not self.mnnvl_only:
return None
inp_size = inp.nbytes
@@ -1139,6 +1139,8 @@ def enable_batch_invariant_mode():
def override_envs_for_invariance():
os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0"
# Only the 1-stage kernel has a size- and rank-independent reduction order.
os.environ["VLLM_CUSTOM_ALLREDUCE_ALGO"] = "1stage"
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"