[communication] [bugfix] fix quickreduce acc error in cudagraph mode (#46913)

Signed-off-by: Haoyang Li <lihaoyang0109@gmail.com>
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
haoyangli0109
2026-07-27 01:38:45 -07:00
committed by GitHub
co-authored by tjtanaa mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Douglas Lehr Andreas Karatzas
parent eb290ab673
commit 7f599d7854
2 changed files with 154 additions and 8 deletions
+30 -8
View File
@@ -22,17 +22,25 @@ template <typename AllReduceKernel, typename T>
__global__ __quickreduce_launch_bounds_two_shot__ static void
allreduce_prototype_twoshot(T const* A, T* B, uint32_t N, uint32_t num_blocks,
int rank, uint8_t** dbuffer_list,
uint32_t data_offset, uint32_t flag_color,
uint32_t data_offset, uint32_t* d_flag_counters,
int64_t data_size_per_phase) {
int block = blockIdx.x;
int grid = gridDim.x;
// Load this block's counter from device memory and advance it on-device,
// so the color keeps changing across graph replays instead of being frozen.
uint32_t flag_color = d_flag_counters[blockIdx.x];
while (block < num_blocks) {
AllReduceKernel::run(A, B, N, block, rank, dbuffer_list, data_offset,
flag_color, data_size_per_phase);
block += grid;
flag_color++;
}
// All threads compute the same final value; one writer per block is enough.
if (threadIdx.x == 0 && threadIdx.y == 0) {
d_flag_counters[blockIdx.x] = flag_color;
}
}
#define TWOSHOT_DISPATCH(__codec) \
@@ -42,21 +50,21 @@ allreduce_prototype_twoshot(T const* A, T* B, uint32_t N, uint32_t num_blocks,
hipLaunchKernelGGL((allreduce_prototype_twoshot<AllReduceKernel, T>), \
dim3(grid), dim3(kBlockTwoShot), 0, stream, A, B, N, \
num_blocks, rank, dbuffer_list, data_offset, \
flag_color, this->kMaxProblemSize); \
d_flag_counters, this->kMaxProblemSize); \
} else if (world_size == 4) { \
using LineCodec = __codec<T, 4>; \
using AllReduceKernel = AllReduceTwoshot<T, LineCodec, cast_bf2half>; \
hipLaunchKernelGGL((allreduce_prototype_twoshot<AllReduceKernel, T>), \
dim3(grid), dim3(kBlockTwoShot), 0, stream, A, B, N, \
num_blocks, rank, dbuffer_list, data_offset, \
flag_color, this->kMaxProblemSize); \
d_flag_counters, this->kMaxProblemSize); \
} else if (world_size == 8) { \
using LineCodec = __codec<T, 8>; \
using AllReduceKernel = AllReduceTwoshot<T, LineCodec, cast_bf2half>; \
hipLaunchKernelGGL((allreduce_prototype_twoshot<AllReduceKernel, T>), \
dim3(grid), dim3(kBlockTwoShot), 0, stream, A, B, N, \
num_blocks, rank, dbuffer_list, data_offset, \
flag_color, this->kMaxProblemSize); \
d_flag_counters, this->kMaxProblemSize); \
}
// INT3 only retains good performance on TP2 (world_size == 2). On TP4/TP8
@@ -69,7 +77,7 @@ allreduce_prototype_twoshot(T const* A, T* B, uint32_t N, uint32_t num_blocks,
hipLaunchKernelGGL((allreduce_prototype_twoshot<AllReduceKernel, T>), \
dim3(grid), dim3(kBlockTwoShot), 0, stream, A, B, N, \
num_blocks, rank, dbuffer_list, data_offset, \
flag_color, this->kMaxProblemSize); \
d_flag_counters, this->kMaxProblemSize); \
} else { \
throw std::runtime_error( \
"INT3 quick all-reduce is only supported for world_size == 2 " \
@@ -94,7 +102,7 @@ struct DeviceComms {
static int constexpr kMaxWorldSize = 8;
bool initialized = false;
uint32_t flag_color = 1;
uint32_t* d_flag_counters = nullptr;
int world_size;
int rank;
@@ -128,6 +136,16 @@ struct DeviceComms {
// Clear the flags buffer.
HIP_CHECK(hipMemset(dbuffer, 0, flags_buffer_size));
// One flag-color counter per block, advanced by the kernel. Start at 1
// to stay clear of the flags buffer we just zeroed.
HIP_CHECK(hipMalloc(&d_flag_counters, kMaxNumBlocks * sizeof(uint32_t)));
{
std::vector<uint32_t> init_color(kMaxNumBlocks, 1u);
HIP_CHECK(hipMemcpy(d_flag_counters, init_color.data(),
kMaxNumBlocks * sizeof(uint32_t),
hipMemcpyHostToDevice));
}
// Device-side list of IPC buffers.
buffer_list.resize(world_size);
HIP_CHECK(hipMalloc(&dbuffer_list, world_size * sizeof(uint8_t*)));
@@ -144,6 +162,12 @@ struct DeviceComms {
hipIpcMemHandle_t const get_handle() { return buffer_ipc_handle; }
void destroy() {
// Allocated before `initialized` flips true, so free it on its own guard
// to avoid a leak if init fails partway through.
if (d_flag_counters) {
HIP_CHECK(hipFree(d_flag_counters));
d_flag_counters = nullptr;
}
if (initialized) {
for (int i = 0; i < world_size; i++) {
if (i != rank) {
@@ -211,8 +235,6 @@ struct DeviceComms {
break;
}
HIP_CHECK(cudaGetLastError());
// Rotate the flag color.
flag_color += divceil(N, grid);
}
};
+124
View File
@@ -23,6 +23,8 @@ from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
from vllm.utils.network_utils import get_open_port
from ..utils import multi_gpu_test
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm-only quick-reduce tests",
@@ -151,6 +153,116 @@ def _run_two_gpu_quick_allreduce_test(
)
CUDAGRAPH_WORLD_SIZE = 2
CUDAGRAPH_ROUNDS = 10
CUDAGRAPH_NUM_ELEMENTS = 1 << 21 # 2M fp16 = 4 MB, above the quick-reduce thresholds
def _quick_allreduce_cudagraph_worker(
rank: int,
world_size: int,
port: int,
quant_level: str,
):
# FP keeps the all-reduce bit-exact for small fp16 integers, so every
# replayed round can be checked exactly.
os.environ["VLLM_ROCM_QUICK_REDUCE_QUANTIZATION"] = quant_level
os.environ["VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16"] = "0"
_log(f"cudagraph worker start: rank={rank} quant={quant_level}")
device = torch.device(f"cuda:{rank}")
torch.accelerator.set_device_index(device)
dist.init_process_group(
backend="gloo",
init_method=f"tcp://127.0.0.1:{port}",
rank=rank,
world_size=world_size,
)
qar = None
try:
from vllm.distributed.device_communicators.quick_all_reduce import (
QuickAllReduce,
)
qar = QuickAllReduce(group=dist.GroupMember.WORLD, device=rank)
assert not qar.disabled
N = CUDAGRAPH_NUM_ELEMENTS
inp = torch.empty(N, dtype=torch.float16, device=device)
out = torch.empty(N, dtype=torch.float16, device=device)
assert qar.should_quick_allreduce(inp)
# Every rank contributes the same value v in a round, so the true
# cross-rank all-reduce sum is simply world_size * v.
def expected(v):
return float(world_size * v)
if rank == 0:
print(
f"[repro] world_size={world_size} elems={N} regime={quant_level} fp16",
flush=True,
)
# Warmup, then capture a graph with EXACTLY ONE quick-reduce (isolated
# qr, so it is the sole writer of its flag slot -- the condition that
# triggers the stale-flag bug).
inp.fill_(1.0)
qar.quick_all_reduce(inp, out=out)
torch.accelerator.synchronize()
dist.barrier()
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
qar.quick_all_reduce(inp, out=out)
torch.accelerator.synchronize()
dist.barrier()
for v in range(CUDAGRAPH_ROUNDS):
inp.fill_(float(v)) # in-place: same value on every rank
dist.barrier()
g.replay()
torch.accelerator.synchronize()
dist.barrier()
got = out.float()
expect = expected(v)
if rank == 0:
print(f"round {v}: got={got[:10]}, expected={expect}", flush=True)
mismatch = ~torch.isclose(got, torch.full_like(got, expect))
num_mismatch = int(mismatch.sum().item())
assert num_mismatch == 0, (
f"rank={rank} round={v} expected={expect} "
f"mismatched {num_mismatch}/{got.numel()} elements; "
f"unique wrong values={torch.unique(got[mismatch])[:8].tolist()}"
)
_log(f"cudagraph worker complete: rank={rank} rounds={CUDAGRAPH_ROUNDS}")
finally:
if qar is not None:
qar.close()
if dist.is_initialized():
dist.destroy_process_group()
def _run_cudagraph_replay_test(*, world_size: int, quant_level: str):
_log(f"launch {world_size}-GPU cudagraph replay case: quant={quant_level}")
ctx = mp.get_context("spawn")
port = get_open_port()
procs = []
for rank in range(world_size):
proc = ctx.Process(
target=_quick_allreduce_cudagraph_worker,
args=(rank, world_size, port, quant_level),
)
proc.start()
procs.append(proc)
for proc in procs:
proc.join(timeout=120)
assert proc.exitcode == 0, f"worker exited with code {proc.exitcode}"
_log(f"finished {world_size}-GPU cudagraph replay case: quant={quant_level}")
MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct"
E2E_PREFILL_TOKENS = 1024
E2E_MAX_MODEL_LEN = 1536
@@ -729,6 +841,18 @@ def test_quick_allreduce_two_gpu_correctness(quant_level):
)
@multi_gpu_test(num_gpus=CUDAGRAPH_WORLD_SIZE)
def test_quick_allreduce_cudagraph_replay():
# Regression test for the stale flag_color bug: a quick-reduce captured in a
# CUDA graph must return correct results on every replay, not the data from
# a previous round.
_log("cudagraph replay case")
_run_cudagraph_replay_test(
world_size=CUDAGRAPH_WORLD_SIZE,
quant_level="FP",
)
@pytest.mark.skipif(
current_platform.device_count() < WORLD_SIZE,
reason="requires 2 ROCm GPUs",