[Distributed] Enable custom all-reduce under VLLM_BATCH_INVARIANT (#58623)
Signed-off-by: Flora Feng <4florafeng@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -146,11 +146,15 @@ When batch invariance is enabled, vLLM:
|
||||
|
||||
1. Uses deterministic kernel implementations for attention and other operations
|
||||
2. Ensures consistent numerical behavior across different batch sizes
|
||||
3. Disables certain optimizations that may introduce non-determinism (such as custom all-reduce operations in tensor parallel mode, and sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant)
|
||||
4. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell),
|
||||
3. Disables certain optimizations that may introduce non-determinism (such as sequence parallelism / async TP, whose reduce-scatter path is not batch-invariant)
|
||||
4. Under tensor parallelism, keeps custom all-reduce on with a fixed reduction order (the 1-stage kernel is pinned, and large inputs are reduced in fixed-size chunks), and disables FlashInfer, AITER and QuickReduce all-reduce
|
||||
5. On CUDA devices with tuned matmul table entries for the model's bf16 unquantized forward linear layers (Ada, Hopper, Blackwell),
|
||||
runs without `torch.compile` using breakable CUDA graphs so tile configs follow the runtime batch size; set `VLLM_USE_BREAKABLE_CUDAGRAPH=0` to opt out
|
||||
(not applied when sequence parallelism / async TP are enabled, since those are `torch.compile` passes).
|
||||
|
||||
!!! warning
|
||||
Batch invariance under tensor parallelism is not yet supported for all-reduces whose size isn't a multiple of 16 bytes, for example a hidden size that isn't a multiple of 8 in fp16/bf16. Such tensors can switch between NCCL and custom all-reduce depending on batch size. All validated models meet this requirement.
|
||||
|
||||
!!! note
|
||||
Enabling batch invariance may impact performance compared to the default non-deterministic mode. This trade-off is intentional to guarantee reproducibility.
|
||||
|
||||
|
||||
@@ -205,6 +205,42 @@ def test_custom_allreduce_filters_dtype(
|
||||
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_invariant", [False, True])
|
||||
def test_custom_allreduce_size_gate_ignored_under_batch_invariance(
|
||||
batch_invariant: bool,
|
||||
) -> None:
|
||||
"""Batch invariance must not switch backends based on the input size."""
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator.world_size = 2
|
||||
communicator.max_size = 1024
|
||||
communicator.batch_invariant = batch_invariant
|
||||
communicator._ptr = 0
|
||||
|
||||
oversized = torch.empty(1024, dtype=torch.float16)
|
||||
assert communicator.should_custom_ar(oversized) is batch_invariant
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_invariant", [False, True])
|
||||
def test_custom_reduce_scatter_disabled_under_batch_invariance(
|
||||
monkeypatch, batch_invariant: bool
|
||||
) -> None:
|
||||
"""Reduce-scatter stays on one backend under batch invariance."""
|
||||
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator.world_size = 2
|
||||
communicator.fully_connected = True
|
||||
communicator.mnnvl_only = False
|
||||
communicator.mnnvl_multicast_ptr = 0
|
||||
communicator.max_reduce_scatter_size = 1024
|
||||
communicator.max_mnnvl_reduce_scatter_size = 1024
|
||||
communicator.batch_invariant = batch_invariant
|
||||
|
||||
in_range = torch.empty(64, dtype=torch.float16)
|
||||
assert communicator.should_custom_reduce_scatter(in_range) is not batch_invariant
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("major", "local_multicast", "expected"),
|
||||
[
|
||||
@@ -622,6 +658,60 @@ def eager_allreduce(
|
||||
torch.testing.assert_close(out, inp * (tp_size**num_communication))
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def chunked_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
"""Inputs above max_size are reduced in chunks and match the fixed-order
|
||||
reference bitwise, both eagerly and inside a CUDA graph."""
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
m.setenv("VLLM_CUSTOM_ALLREDUCE_ALGO", "1stage")
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
group = get_tp_group().device_group
|
||||
fa = get_tp_group().device_communicator.ca_comm
|
||||
|
||||
# Two full chunks plus a partial one.
|
||||
dtype = torch.bfloat16
|
||||
chunk_numel = fa.max_size // dtype.itemsize
|
||||
inp = torch.randn(2 * chunk_numel + 4096, device=device).to(dtype)
|
||||
gathered = [torch.empty_like(inp) for _ in range(tp_size)]
|
||||
dist.all_gather(gathered, inp, group=group)
|
||||
ref = gathered[0].float()
|
||||
for peer in gathered[1:]:
|
||||
ref = ref + peer.float()
|
||||
ref = ref.to(torch.bfloat16)
|
||||
|
||||
out = fa.all_reduce(inp, registered=False)
|
||||
assert torch.equal(out, ref)
|
||||
|
||||
# Weak-contiguous but not C-contiguous: a transposed matrix.
|
||||
inp_t, ref_t = inp.view(-1, 4096).t(), ref.view(-1, 4096).t()
|
||||
assert torch.equal(fa.all_reduce(inp_t, registered=False), ref_t)
|
||||
|
||||
with fa.capture():
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
out = fa.all_reduce(inp, registered=True)
|
||||
graph.replay()
|
||||
assert torch.equal(out, ref)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
def test_custom_allreduce_chunked(monkeypatch: pytest.MonkeyPatch, tp_size):
|
||||
if tp_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, tp_size, 1, chunked_allreduce)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
|
||||
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
|
||||
|
||||
@@ -176,17 +176,6 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
|
||||
random.seed(seed)
|
||||
tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1"))
|
||||
|
||||
# For batch invariance, disable custom all-reduce to ensure deterministic
|
||||
# all-reduce operations (custom all-reduce may not be deterministic)
|
||||
import vllm.envs as envs
|
||||
|
||||
disable_custom_ar = envs.VLLM_BATCH_INVARIANT
|
||||
|
||||
if disable_custom_ar:
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f"BATCH INVARIANCE MODE: Disabling custom all-reduce (TP={tp_size})")
|
||||
print(f"{'=' * 80}\n")
|
||||
|
||||
llm = LLM(
|
||||
model=TEST_MODEL,
|
||||
tensor_parallel_size=tp_size,
|
||||
@@ -744,15 +733,6 @@ def test_decode_logprobs_match_prefill_logprobs(
|
||||
random.seed(seed)
|
||||
tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1"))
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
disable_custom_ar = envs.VLLM_BATCH_INVARIANT
|
||||
|
||||
if disable_custom_ar:
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f"BATCH INVARIANCE MODE: Disabling custom all-reduce (TP={tp_size})")
|
||||
print(f"{'=' * 80}\n")
|
||||
|
||||
llm = LLM(
|
||||
model=TEST_MODEL,
|
||||
tensor_parallel_size=tp_size,
|
||||
|
||||
@@ -1087,10 +1087,6 @@ class ParallelConfig:
|
||||
# Lazy import to avoid circular import
|
||||
from vllm.v1.executor import Executor
|
||||
|
||||
# Enable batch invariance settings if requested
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
self.disable_custom_all_reduce = True
|
||||
|
||||
if (
|
||||
self.distributed_executor_backend is not None
|
||||
and not isinstance(self.distributed_executor_backend, str)
|
||||
|
||||
@@ -68,8 +68,11 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
envs.VLLM_ALLREDUCE_USE_FLASHINFER_PCIE_IPC
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
)
|
||||
use_aiter_allreduce = use_custom_allreduce and bool(
|
||||
rocm_aiter_ops.is_custom_all_reduce_enabled()
|
||||
# Neither AITER nor QuickReduce all-reduce has a fixed reduction order.
|
||||
use_aiter_allreduce = (
|
||||
use_custom_allreduce
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
and bool(rocm_aiter_ops.is_custom_all_reduce_enabled())
|
||||
)
|
||||
|
||||
self.use_custom_allreduce = use_custom_allreduce
|
||||
@@ -166,7 +169,12 @@ class CudaCommunicator(DeviceCommunicatorBase):
|
||||
else:
|
||||
self.use_aiter_ag_rs = True
|
||||
|
||||
if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm():
|
||||
if (
|
||||
use_custom_allreduce
|
||||
and self.world_size > 1
|
||||
and current_platform.is_rocm()
|
||||
and not envs.VLLM_BATCH_INVARIANT
|
||||
):
|
||||
# Initialize a custom quick all-reduce implementation for AMD.
|
||||
# Quick reduce is designed as a complement to custom allreduce
|
||||
# (vLLM's or AITER's), so it is initialized for either backend.
|
||||
|
||||
@@ -119,6 +119,7 @@ from vllm.distributed.utils import is_weak_contiguous # noqa: E402
|
||||
|
||||
class CustomAllreduce:
|
||||
_SUPPORTED_WORLD_SIZES = [2, 4, 6, 8, 16]
|
||||
batch_invariant = False
|
||||
_DEFAULT_ALL_GATHER_MAX_SIZE = 2 * 1024 * 1024
|
||||
_DEFAULT_MNNVL_ALL_GATHER_MAX_SIZES = {
|
||||
2: 8 * 1024 * 1024,
|
||||
@@ -179,6 +180,7 @@ class CustomAllreduce:
|
||||
self.mnnvl_multimem_rs_local_ptr = 0
|
||||
self.mnnvl_multimem_rs_multicast_ptr = 0
|
||||
self.mnnvl_only = False
|
||||
self.batch_invariant = envs.VLLM_BATCH_INVARIANT
|
||||
|
||||
if not custom_ar:
|
||||
# disable because of missing custom allreduce library
|
||||
@@ -504,7 +506,7 @@ class CustomAllreduce:
|
||||
# for 4 or more non NVLink-capable GPUs, custom allreduce provides
|
||||
# little performance improvement over NCCL.
|
||||
if self.world_size == 2 or self.fully_connected:
|
||||
return inp_size < self.max_size
|
||||
return self.batch_invariant or inp_size < self.max_size
|
||||
return False
|
||||
|
||||
def all_reduce(
|
||||
@@ -518,13 +520,27 @@ class CustomAllreduce:
|
||||
"""
|
||||
if out is None:
|
||||
out = torch.empty_like(inp)
|
||||
chunk_numel = self.max_size // inp.element_size()
|
||||
if inp.numel() <= chunk_numel:
|
||||
self._all_reduce_chunk(inp, out, registered)
|
||||
return out
|
||||
# Storage-order views: the gate only admits one contiguous block.
|
||||
flat_inp = inp.as_strided((inp.numel(),), (1,), inp.storage_offset())
|
||||
flat_out = out.as_strided((out.numel(),), (1,), out.storage_offset())
|
||||
for start in range(0, flat_inp.numel(), chunk_numel):
|
||||
end = start + chunk_numel
|
||||
self._all_reduce_chunk(flat_inp[start:end], flat_out[start:end], registered)
|
||||
return out
|
||||
|
||||
def _all_reduce_chunk(
|
||||
self, inp: torch.Tensor, out: torch.Tensor, registered: bool
|
||||
) -> None:
|
||||
if registered:
|
||||
ops.all_reduce(self._ptr, inp, out, 0, 0)
|
||||
else:
|
||||
ops.all_reduce(
|
||||
self._ptr, inp, out, self.buffer_ptrs[self.rank], self.max_size
|
||||
)
|
||||
return out
|
||||
|
||||
def custom_all_reduce(self, input: torch.Tensor) -> torch.Tensor | None:
|
||||
"""The main allreduce API that provides support for cuda graph."""
|
||||
@@ -603,6 +619,9 @@ class CustomAllreduce:
|
||||
) -> _ReduceScatterBackend | None:
|
||||
if self.disabled or not current_platform.is_cuda():
|
||||
return None
|
||||
# The size gates below would switch backends per batch.
|
||||
if self.batch_invariant:
|
||||
return None
|
||||
if self.world_size == 16 and not self.mnnvl_only:
|
||||
return None
|
||||
inp_size = inp.nbytes
|
||||
|
||||
@@ -1139,6 +1139,8 @@ def enable_batch_invariant_mode():
|
||||
|
||||
def override_envs_for_invariance():
|
||||
os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0"
|
||||
# Only the 1-stage kernel has a size- and rank-independent reduction order.
|
||||
os.environ["VLLM_CUSTOM_ALLREDUCE_ALGO"] = "1stage"
|
||||
|
||||
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user