[Distributed] Enable custom all-reduce under VLLM_BATCH_INVARIANT (#58623)

Signed-off-by: Flora Feng <4florafeng@gmail.com>
Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
Flora Feng
2026-10-02 02:25:48 +00:00
committed by GitHub
parent e3cae8d2ac
commit 785b990116
7 changed files with 130 additions and 31 deletions
@@ -205,6 +205,42 @@ def test_custom_allreduce_filters_dtype(
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
@pytest.mark.parametrize("batch_invariant", [False, True])
def test_custom_allreduce_size_gate_ignored_under_batch_invariance(
batch_invariant: bool,
) -> None:
"""Batch invariance must not switch backends based on the input size."""
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator.world_size = 2
communicator.max_size = 1024
communicator.batch_invariant = batch_invariant
communicator._ptr = 0
oversized = torch.empty(1024, dtype=torch.float16)
assert communicator.should_custom_ar(oversized) is batch_invariant
@pytest.mark.parametrize("batch_invariant", [False, True])
def test_custom_reduce_scatter_disabled_under_batch_invariance(
monkeypatch, batch_invariant: bool
) -> None:
"""Reduce-scatter stays on one backend under batch invariance."""
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
communicator.disabled = False
communicator.world_size = 2
communicator.fully_connected = True
communicator.mnnvl_only = False
communicator.mnnvl_multicast_ptr = 0
communicator.max_reduce_scatter_size = 1024
communicator.max_mnnvl_reduce_scatter_size = 1024
communicator.batch_invariant = batch_invariant
in_range = torch.empty(64, dtype=torch.float16)
assert communicator.should_custom_reduce_scatter(in_range) is not batch_invariant
@pytest.mark.parametrize(
("major", "local_multicast", "expected"),
[
@@ -622,6 +658,60 @@ def eager_allreduce(
torch.testing.assert_close(out, inp * (tp_size**num_communication))
@ray.remote(num_gpus=1, max_calls=1)
def chunked_allreduce(
monkeypatch: pytest.MonkeyPatch,
tp_size,
pp_size,
rank,
distributed_init_port,
):
"""Inputs above max_size are reduced in chunks and match the fixed-order
reference bitwise, both eagerly and inside a CUDA graph."""
with monkeypatch.context() as m:
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
m.setenv("VLLM_CUSTOM_ALLREDUCE_ALGO", "1stage")
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
ensure_model_parallel_initialized(tp_size, pp_size)
group = get_tp_group().device_group
fa = get_tp_group().device_communicator.ca_comm
# Two full chunks plus a partial one.
dtype = torch.bfloat16
chunk_numel = fa.max_size // dtype.itemsize
inp = torch.randn(2 * chunk_numel + 4096, device=device).to(dtype)
gathered = [torch.empty_like(inp) for _ in range(tp_size)]
dist.all_gather(gathered, inp, group=group)
ref = gathered[0].float()
for peer in gathered[1:]:
ref = ref + peer.float()
ref = ref.to(torch.bfloat16)
out = fa.all_reduce(inp, registered=False)
assert torch.equal(out, ref)
# Weak-contiguous but not C-contiguous: a transposed matrix.
inp_t, ref_t = inp.view(-1, 4096).t(), ref.view(-1, 4096).t()
assert torch.equal(fa.all_reduce(inp_t, registered=False), ref_t)
with fa.capture():
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
out = fa.all_reduce(inp, registered=True)
graph.replay()
assert torch.equal(out, ref)
@pytest.mark.parametrize("tp_size", [2, 4])
def test_custom_allreduce_chunked(monkeypatch: pytest.MonkeyPatch, tp_size):
if tp_size > torch.accelerator.device_count():
pytest.skip("Not enough GPUs to run the test.")
multi_process_parallel(monkeypatch, tp_size, 1, chunked_allreduce)
@pytest.mark.parametrize("tp_size", [2])
@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
@@ -176,17 +176,6 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
random.seed(seed)
tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1"))
# For batch invariance, disable custom all-reduce to ensure deterministic
# all-reduce operations (custom all-reduce may not be deterministic)
import vllm.envs as envs
disable_custom_ar = envs.VLLM_BATCH_INVARIANT
if disable_custom_ar:
print(f"\n{'=' * 80}")
print(f"BATCH INVARIANCE MODE: Disabling custom all-reduce (TP={tp_size})")
print(f"{'=' * 80}\n")
llm = LLM(
model=TEST_MODEL,
tensor_parallel_size=tp_size,
@@ -744,15 +733,6 @@ def test_decode_logprobs_match_prefill_logprobs(
random.seed(seed)
tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1"))
import vllm.envs as envs
disable_custom_ar = envs.VLLM_BATCH_INVARIANT
if disable_custom_ar:
print(f"\n{'=' * 80}")
print(f"BATCH INVARIANCE MODE: Disabling custom all-reduce (TP={tp_size})")
print(f"{'=' * 80}\n")
llm = LLM(
model=TEST_MODEL,
tensor_parallel_size=tp_size,