[Distributed] Enable custom all-reduce under VLLM_BATCH_INVARIANT (#58623)
Signed-off-by: Flora Feng <4florafeng@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -205,6 +205,42 @@ def test_custom_allreduce_filters_dtype(
|
||||
assert communicator.should_custom_ar(torch.empty(16, dtype=dtype)) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_invariant", [False, True])
|
||||
def test_custom_allreduce_size_gate_ignored_under_batch_invariance(
|
||||
batch_invariant: bool,
|
||||
) -> None:
|
||||
"""Batch invariance must not switch backends based on the input size."""
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator.world_size = 2
|
||||
communicator.max_size = 1024
|
||||
communicator.batch_invariant = batch_invariant
|
||||
communicator._ptr = 0
|
||||
|
||||
oversized = torch.empty(1024, dtype=torch.float16)
|
||||
assert communicator.should_custom_ar(oversized) is batch_invariant
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_invariant", [False, True])
|
||||
def test_custom_reduce_scatter_disabled_under_batch_invariance(
|
||||
monkeypatch, batch_invariant: bool
|
||||
) -> None:
|
||||
"""Reduce-scatter stays on one backend under batch invariance."""
|
||||
monkeypatch.setattr(car.current_platform, "is_cuda", lambda: True)
|
||||
communicator = car.CustomAllreduce.__new__(car.CustomAllreduce)
|
||||
communicator.disabled = False
|
||||
communicator.world_size = 2
|
||||
communicator.fully_connected = True
|
||||
communicator.mnnvl_only = False
|
||||
communicator.mnnvl_multicast_ptr = 0
|
||||
communicator.max_reduce_scatter_size = 1024
|
||||
communicator.max_mnnvl_reduce_scatter_size = 1024
|
||||
communicator.batch_invariant = batch_invariant
|
||||
|
||||
in_range = torch.empty(64, dtype=torch.float16)
|
||||
assert communicator.should_custom_reduce_scatter(in_range) is not batch_invariant
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("major", "local_multicast", "expected"),
|
||||
[
|
||||
@@ -622,6 +658,60 @@ def eager_allreduce(
|
||||
torch.testing.assert_close(out, inp * (tp_size**num_communication))
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def chunked_allreduce(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
tp_size,
|
||||
pp_size,
|
||||
rank,
|
||||
distributed_init_port,
|
||||
):
|
||||
"""Inputs above max_size are reduced in chunks and match the fixed-order
|
||||
reference bitwise, both eagerly and inside a CUDA graph."""
|
||||
with monkeypatch.context() as m:
|
||||
m.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
m.delenv("HIP_VISIBLE_DEVICES", raising=False)
|
||||
m.setenv("VLLM_CUSTOM_ALLREDUCE_ALGO", "1stage")
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
init_test_distributed_environment(tp_size, pp_size, rank, distributed_init_port)
|
||||
ensure_model_parallel_initialized(tp_size, pp_size)
|
||||
group = get_tp_group().device_group
|
||||
fa = get_tp_group().device_communicator.ca_comm
|
||||
|
||||
# Two full chunks plus a partial one.
|
||||
dtype = torch.bfloat16
|
||||
chunk_numel = fa.max_size // dtype.itemsize
|
||||
inp = torch.randn(2 * chunk_numel + 4096, device=device).to(dtype)
|
||||
gathered = [torch.empty_like(inp) for _ in range(tp_size)]
|
||||
dist.all_gather(gathered, inp, group=group)
|
||||
ref = gathered[0].float()
|
||||
for peer in gathered[1:]:
|
||||
ref = ref + peer.float()
|
||||
ref = ref.to(torch.bfloat16)
|
||||
|
||||
out = fa.all_reduce(inp, registered=False)
|
||||
assert torch.equal(out, ref)
|
||||
|
||||
# Weak-contiguous but not C-contiguous: a transposed matrix.
|
||||
inp_t, ref_t = inp.view(-1, 4096).t(), ref.view(-1, 4096).t()
|
||||
assert torch.equal(fa.all_reduce(inp_t, registered=False), ref_t)
|
||||
|
||||
with fa.capture():
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
out = fa.all_reduce(inp, registered=True)
|
||||
graph.replay()
|
||||
assert torch.equal(out, ref)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2, 4])
|
||||
def test_custom_allreduce_chunked(monkeypatch: pytest.MonkeyPatch, tp_size):
|
||||
if tp_size > torch.accelerator.device_count():
|
||||
pytest.skip("Not enough GPUs to run the test.")
|
||||
multi_process_parallel(monkeypatch, tp_size, 1, chunked_allreduce)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_size", [2])
|
||||
@pytest.mark.parametrize("pipeline_parallel_size", [1, 2])
|
||||
@pytest.mark.parametrize("test_target", [eager_allreduce, graph_allreduce])
|
||||
|
||||
@@ -176,17 +176,6 @@ def test_logprobs_bitwise_batch_invariance_bs1_vs_bsN(
|
||||
random.seed(seed)
|
||||
tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1"))
|
||||
|
||||
# For batch invariance, disable custom all-reduce to ensure deterministic
|
||||
# all-reduce operations (custom all-reduce may not be deterministic)
|
||||
import vllm.envs as envs
|
||||
|
||||
disable_custom_ar = envs.VLLM_BATCH_INVARIANT
|
||||
|
||||
if disable_custom_ar:
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f"BATCH INVARIANCE MODE: Disabling custom all-reduce (TP={tp_size})")
|
||||
print(f"{'=' * 80}\n")
|
||||
|
||||
llm = LLM(
|
||||
model=TEST_MODEL,
|
||||
tensor_parallel_size=tp_size,
|
||||
@@ -744,15 +733,6 @@ def test_decode_logprobs_match_prefill_logprobs(
|
||||
random.seed(seed)
|
||||
tp_size = int(os.getenv("VLLM_TEST_TP_SIZE", "1"))
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
disable_custom_ar = envs.VLLM_BATCH_INVARIANT
|
||||
|
||||
if disable_custom_ar:
|
||||
print(f"\n{'=' * 80}")
|
||||
print(f"BATCH INVARIANCE MODE: Disabling custom all-reduce (TP={tp_size})")
|
||||
print(f"{'=' * 80}\n")
|
||||
|
||||
llm = LLM(
|
||||
model=TEST_MODEL,
|
||||
tensor_parallel_size=tp_size,
|
||||
|
||||
Reference in New Issue
Block a user