From 7f599d78546819948c32f2b23d913507bbb38875 Mon Sep 17 00:00:00 2001 From: haoyangli0109 Date: Mon, 27 Jul 2026 16:38:45 +0800 Subject: [PATCH] [communication] [bugfix] fix quickreduce acc error in cudagraph mode (#46913) Signed-off-by: Haoyang Li Signed-off-by: tjtanaa Co-authored-by: tjtanaa Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: Andreas Karatzas --- csrc/quickreduce/quick_reduce.h | 38 ++++-- tests/distributed/test_rocm_quick_reduce.py | 124 ++++++++++++++++++++ 2 files changed, 154 insertions(+), 8 deletions(-) diff --git a/csrc/quickreduce/quick_reduce.h b/csrc/quickreduce/quick_reduce.h index 7506329972..449721b571 100644 --- a/csrc/quickreduce/quick_reduce.h +++ b/csrc/quickreduce/quick_reduce.h @@ -22,17 +22,25 @@ template __global__ __quickreduce_launch_bounds_two_shot__ static void allreduce_prototype_twoshot(T const* A, T* B, uint32_t N, uint32_t num_blocks, int rank, uint8_t** dbuffer_list, - uint32_t data_offset, uint32_t flag_color, + uint32_t data_offset, uint32_t* d_flag_counters, int64_t data_size_per_phase) { int block = blockIdx.x; int grid = gridDim.x; + // Load this block's counter from device memory and advance it on-device, + // so the color keeps changing across graph replays instead of being frozen. + uint32_t flag_color = d_flag_counters[blockIdx.x]; + while (block < num_blocks) { AllReduceKernel::run(A, B, N, block, rank, dbuffer_list, data_offset, flag_color, data_size_per_phase); block += grid; flag_color++; } + // All threads compute the same final value; one writer per block is enough. + if (threadIdx.x == 0 && threadIdx.y == 0) { + d_flag_counters[blockIdx.x] = flag_color; + } } #define TWOSHOT_DISPATCH(__codec) \ @@ -42,21 +50,21 @@ allreduce_prototype_twoshot(T const* A, T* B, uint32_t N, uint32_t num_blocks, hipLaunchKernelGGL((allreduce_prototype_twoshot), \ dim3(grid), dim3(kBlockTwoShot), 0, stream, A, B, N, \ num_blocks, rank, dbuffer_list, data_offset, \ - flag_color, this->kMaxProblemSize); \ + d_flag_counters, this->kMaxProblemSize); \ } else if (world_size == 4) { \ using LineCodec = __codec; \ using AllReduceKernel = AllReduceTwoshot; \ hipLaunchKernelGGL((allreduce_prototype_twoshot), \ dim3(grid), dim3(kBlockTwoShot), 0, stream, A, B, N, \ num_blocks, rank, dbuffer_list, data_offset, \ - flag_color, this->kMaxProblemSize); \ + d_flag_counters, this->kMaxProblemSize); \ } else if (world_size == 8) { \ using LineCodec = __codec; \ using AllReduceKernel = AllReduceTwoshot; \ hipLaunchKernelGGL((allreduce_prototype_twoshot), \ dim3(grid), dim3(kBlockTwoShot), 0, stream, A, B, N, \ num_blocks, rank, dbuffer_list, data_offset, \ - flag_color, this->kMaxProblemSize); \ + d_flag_counters, this->kMaxProblemSize); \ } // INT3 only retains good performance on TP2 (world_size == 2). On TP4/TP8 @@ -69,7 +77,7 @@ allreduce_prototype_twoshot(T const* A, T* B, uint32_t N, uint32_t num_blocks, hipLaunchKernelGGL((allreduce_prototype_twoshot), \ dim3(grid), dim3(kBlockTwoShot), 0, stream, A, B, N, \ num_blocks, rank, dbuffer_list, data_offset, \ - flag_color, this->kMaxProblemSize); \ + d_flag_counters, this->kMaxProblemSize); \ } else { \ throw std::runtime_error( \ "INT3 quick all-reduce is only supported for world_size == 2 " \ @@ -94,7 +102,7 @@ struct DeviceComms { static int constexpr kMaxWorldSize = 8; bool initialized = false; - uint32_t flag_color = 1; + uint32_t* d_flag_counters = nullptr; int world_size; int rank; @@ -128,6 +136,16 @@ struct DeviceComms { // Clear the flags buffer. HIP_CHECK(hipMemset(dbuffer, 0, flags_buffer_size)); + // One flag-color counter per block, advanced by the kernel. Start at 1 + // to stay clear of the flags buffer we just zeroed. + HIP_CHECK(hipMalloc(&d_flag_counters, kMaxNumBlocks * sizeof(uint32_t))); + { + std::vector init_color(kMaxNumBlocks, 1u); + HIP_CHECK(hipMemcpy(d_flag_counters, init_color.data(), + kMaxNumBlocks * sizeof(uint32_t), + hipMemcpyHostToDevice)); + } + // Device-side list of IPC buffers. buffer_list.resize(world_size); HIP_CHECK(hipMalloc(&dbuffer_list, world_size * sizeof(uint8_t*))); @@ -144,6 +162,12 @@ struct DeviceComms { hipIpcMemHandle_t const get_handle() { return buffer_ipc_handle; } void destroy() { + // Allocated before `initialized` flips true, so free it on its own guard + // to avoid a leak if init fails partway through. + if (d_flag_counters) { + HIP_CHECK(hipFree(d_flag_counters)); + d_flag_counters = nullptr; + } if (initialized) { for (int i = 0; i < world_size; i++) { if (i != rank) { @@ -211,8 +235,6 @@ struct DeviceComms { break; } HIP_CHECK(cudaGetLastError()); - // Rotate the flag color. - flag_color += divceil(N, grid); } }; diff --git a/tests/distributed/test_rocm_quick_reduce.py b/tests/distributed/test_rocm_quick_reduce.py index a7e91f236a..e422b032e2 100644 --- a/tests/distributed/test_rocm_quick_reduce.py +++ b/tests/distributed/test_rocm_quick_reduce.py @@ -23,6 +23,8 @@ from vllm.platforms import current_platform from vllm.transformers_utils.repo_utils import hf_api from vllm.utils.network_utils import get_open_port +from ..utils import multi_gpu_test + pytestmark = pytest.mark.skipif( not current_platform.is_rocm(), reason="ROCm-only quick-reduce tests", @@ -151,6 +153,116 @@ def _run_two_gpu_quick_allreduce_test( ) +CUDAGRAPH_WORLD_SIZE = 2 +CUDAGRAPH_ROUNDS = 10 +CUDAGRAPH_NUM_ELEMENTS = 1 << 21 # 2M fp16 = 4 MB, above the quick-reduce thresholds + + +def _quick_allreduce_cudagraph_worker( + rank: int, + world_size: int, + port: int, + quant_level: str, +): + # FP keeps the all-reduce bit-exact for small fp16 integers, so every + # replayed round can be checked exactly. + os.environ["VLLM_ROCM_QUICK_REDUCE_QUANTIZATION"] = quant_level + os.environ["VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16"] = "0" + _log(f"cudagraph worker start: rank={rank} quant={quant_level}") + + device = torch.device(f"cuda:{rank}") + torch.accelerator.set_device_index(device) + dist.init_process_group( + backend="gloo", + init_method=f"tcp://127.0.0.1:{port}", + rank=rank, + world_size=world_size, + ) + + qar = None + try: + from vllm.distributed.device_communicators.quick_all_reduce import ( + QuickAllReduce, + ) + + qar = QuickAllReduce(group=dist.GroupMember.WORLD, device=rank) + assert not qar.disabled + + N = CUDAGRAPH_NUM_ELEMENTS + inp = torch.empty(N, dtype=torch.float16, device=device) + out = torch.empty(N, dtype=torch.float16, device=device) + assert qar.should_quick_allreduce(inp) + + # Every rank contributes the same value v in a round, so the true + # cross-rank all-reduce sum is simply world_size * v. + def expected(v): + return float(world_size * v) + + if rank == 0: + print( + f"[repro] world_size={world_size} elems={N} regime={quant_level} fp16", + flush=True, + ) + + # Warmup, then capture a graph with EXACTLY ONE quick-reduce (isolated + # qr, so it is the sole writer of its flag slot -- the condition that + # triggers the stale-flag bug). + inp.fill_(1.0) + qar.quick_all_reduce(inp, out=out) + torch.accelerator.synchronize() + dist.barrier() + + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + qar.quick_all_reduce(inp, out=out) + torch.accelerator.synchronize() + dist.barrier() + + for v in range(CUDAGRAPH_ROUNDS): + inp.fill_(float(v)) # in-place: same value on every rank + dist.barrier() + g.replay() + torch.accelerator.synchronize() + dist.barrier() + got = out.float() + expect = expected(v) + if rank == 0: + print(f"round {v}: got={got[:10]}, expected={expect}", flush=True) + mismatch = ~torch.isclose(got, torch.full_like(got, expect)) + num_mismatch = int(mismatch.sum().item()) + assert num_mismatch == 0, ( + f"rank={rank} round={v} expected={expect} " + f"mismatched {num_mismatch}/{got.numel()} elements; " + f"unique wrong values={torch.unique(got[mismatch])[:8].tolist()}" + ) + _log(f"cudagraph worker complete: rank={rank} rounds={CUDAGRAPH_ROUNDS}") + finally: + if qar is not None: + qar.close() + if dist.is_initialized(): + dist.destroy_process_group() + + +def _run_cudagraph_replay_test(*, world_size: int, quant_level: str): + _log(f"launch {world_size}-GPU cudagraph replay case: quant={quant_level}") + ctx = mp.get_context("spawn") + port = get_open_port() + procs = [] + + for rank in range(world_size): + proc = ctx.Process( + target=_quick_allreduce_cudagraph_worker, + args=(rank, world_size, port, quant_level), + ) + proc.start() + procs.append(proc) + + for proc in procs: + proc.join(timeout=120) + assert proc.exitcode == 0, f"worker exited with code {proc.exitcode}" + _log(f"finished {world_size}-GPU cudagraph replay case: quant={quant_level}") + + MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct" E2E_PREFILL_TOKENS = 1024 E2E_MAX_MODEL_LEN = 1536 @@ -729,6 +841,18 @@ def test_quick_allreduce_two_gpu_correctness(quant_level): ) +@multi_gpu_test(num_gpus=CUDAGRAPH_WORLD_SIZE) +def test_quick_allreduce_cudagraph_replay(): + # Regression test for the stale flag_color bug: a quick-reduce captured in a + # CUDA graph must return correct results on every replay, not the data from + # a previous round. + _log("cudagraph replay case") + _run_cudagraph_replay_test( + world_size=CUDAGRAPH_WORLD_SIZE, + quant_level="FP", + ) + + @pytest.mark.skipif( current_platform.device_count() < WORLD_SIZE, reason="requires 2 ROCm GPUs",