[communication] [bugfix] fix quickreduce acc error in cudagraph mode (#46913)
Signed-off-by: Haoyang Li <lihaoyang0109@gmail.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
tjtanaa
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Douglas Lehr
Andreas Karatzas
parent
eb290ab673
commit
7f599d7854
@@ -22,17 +22,25 @@ template <typename AllReduceKernel, typename T>
|
||||
__global__ __quickreduce_launch_bounds_two_shot__ static void
|
||||
allreduce_prototype_twoshot(T const* A, T* B, uint32_t N, uint32_t num_blocks,
|
||||
int rank, uint8_t** dbuffer_list,
|
||||
uint32_t data_offset, uint32_t flag_color,
|
||||
uint32_t data_offset, uint32_t* d_flag_counters,
|
||||
int64_t data_size_per_phase) {
|
||||
int block = blockIdx.x;
|
||||
int grid = gridDim.x;
|
||||
|
||||
// Load this block's counter from device memory and advance it on-device,
|
||||
// so the color keeps changing across graph replays instead of being frozen.
|
||||
uint32_t flag_color = d_flag_counters[blockIdx.x];
|
||||
|
||||
while (block < num_blocks) {
|
||||
AllReduceKernel::run(A, B, N, block, rank, dbuffer_list, data_offset,
|
||||
flag_color, data_size_per_phase);
|
||||
block += grid;
|
||||
flag_color++;
|
||||
}
|
||||
// All threads compute the same final value; one writer per block is enough.
|
||||
if (threadIdx.x == 0 && threadIdx.y == 0) {
|
||||
d_flag_counters[blockIdx.x] = flag_color;
|
||||
}
|
||||
}
|
||||
|
||||
#define TWOSHOT_DISPATCH(__codec) \
|
||||
@@ -42,21 +50,21 @@ allreduce_prototype_twoshot(T const* A, T* B, uint32_t N, uint32_t num_blocks,
|
||||
hipLaunchKernelGGL((allreduce_prototype_twoshot<AllReduceKernel, T>), \
|
||||
dim3(grid), dim3(kBlockTwoShot), 0, stream, A, B, N, \
|
||||
num_blocks, rank, dbuffer_list, data_offset, \
|
||||
flag_color, this->kMaxProblemSize); \
|
||||
d_flag_counters, this->kMaxProblemSize); \
|
||||
} else if (world_size == 4) { \
|
||||
using LineCodec = __codec<T, 4>; \
|
||||
using AllReduceKernel = AllReduceTwoshot<T, LineCodec, cast_bf2half>; \
|
||||
hipLaunchKernelGGL((allreduce_prototype_twoshot<AllReduceKernel, T>), \
|
||||
dim3(grid), dim3(kBlockTwoShot), 0, stream, A, B, N, \
|
||||
num_blocks, rank, dbuffer_list, data_offset, \
|
||||
flag_color, this->kMaxProblemSize); \
|
||||
d_flag_counters, this->kMaxProblemSize); \
|
||||
} else if (world_size == 8) { \
|
||||
using LineCodec = __codec<T, 8>; \
|
||||
using AllReduceKernel = AllReduceTwoshot<T, LineCodec, cast_bf2half>; \
|
||||
hipLaunchKernelGGL((allreduce_prototype_twoshot<AllReduceKernel, T>), \
|
||||
dim3(grid), dim3(kBlockTwoShot), 0, stream, A, B, N, \
|
||||
num_blocks, rank, dbuffer_list, data_offset, \
|
||||
flag_color, this->kMaxProblemSize); \
|
||||
d_flag_counters, this->kMaxProblemSize); \
|
||||
}
|
||||
|
||||
// INT3 only retains good performance on TP2 (world_size == 2). On TP4/TP8
|
||||
@@ -69,7 +77,7 @@ allreduce_prototype_twoshot(T const* A, T* B, uint32_t N, uint32_t num_blocks,
|
||||
hipLaunchKernelGGL((allreduce_prototype_twoshot<AllReduceKernel, T>), \
|
||||
dim3(grid), dim3(kBlockTwoShot), 0, stream, A, B, N, \
|
||||
num_blocks, rank, dbuffer_list, data_offset, \
|
||||
flag_color, this->kMaxProblemSize); \
|
||||
d_flag_counters, this->kMaxProblemSize); \
|
||||
} else { \
|
||||
throw std::runtime_error( \
|
||||
"INT3 quick all-reduce is only supported for world_size == 2 " \
|
||||
@@ -94,7 +102,7 @@ struct DeviceComms {
|
||||
static int constexpr kMaxWorldSize = 8;
|
||||
|
||||
bool initialized = false;
|
||||
uint32_t flag_color = 1;
|
||||
uint32_t* d_flag_counters = nullptr;
|
||||
int world_size;
|
||||
int rank;
|
||||
|
||||
@@ -128,6 +136,16 @@ struct DeviceComms {
|
||||
// Clear the flags buffer.
|
||||
HIP_CHECK(hipMemset(dbuffer, 0, flags_buffer_size));
|
||||
|
||||
// One flag-color counter per block, advanced by the kernel. Start at 1
|
||||
// to stay clear of the flags buffer we just zeroed.
|
||||
HIP_CHECK(hipMalloc(&d_flag_counters, kMaxNumBlocks * sizeof(uint32_t)));
|
||||
{
|
||||
std::vector<uint32_t> init_color(kMaxNumBlocks, 1u);
|
||||
HIP_CHECK(hipMemcpy(d_flag_counters, init_color.data(),
|
||||
kMaxNumBlocks * sizeof(uint32_t),
|
||||
hipMemcpyHostToDevice));
|
||||
}
|
||||
|
||||
// Device-side list of IPC buffers.
|
||||
buffer_list.resize(world_size);
|
||||
HIP_CHECK(hipMalloc(&dbuffer_list, world_size * sizeof(uint8_t*)));
|
||||
@@ -144,6 +162,12 @@ struct DeviceComms {
|
||||
hipIpcMemHandle_t const get_handle() { return buffer_ipc_handle; }
|
||||
|
||||
void destroy() {
|
||||
// Allocated before `initialized` flips true, so free it on its own guard
|
||||
// to avoid a leak if init fails partway through.
|
||||
if (d_flag_counters) {
|
||||
HIP_CHECK(hipFree(d_flag_counters));
|
||||
d_flag_counters = nullptr;
|
||||
}
|
||||
if (initialized) {
|
||||
for (int i = 0; i < world_size; i++) {
|
||||
if (i != rank) {
|
||||
@@ -211,8 +235,6 @@ struct DeviceComms {
|
||||
break;
|
||||
}
|
||||
HIP_CHECK(cudaGetLastError());
|
||||
// Rotate the flag color.
|
||||
flag_color += divceil(N, grid);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -23,6 +23,8 @@ from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
from ..utils import multi_gpu_test
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm-only quick-reduce tests",
|
||||
@@ -151,6 +153,116 @@ def _run_two_gpu_quick_allreduce_test(
|
||||
)
|
||||
|
||||
|
||||
CUDAGRAPH_WORLD_SIZE = 2
|
||||
CUDAGRAPH_ROUNDS = 10
|
||||
CUDAGRAPH_NUM_ELEMENTS = 1 << 21 # 2M fp16 = 4 MB, above the quick-reduce thresholds
|
||||
|
||||
|
||||
def _quick_allreduce_cudagraph_worker(
|
||||
rank: int,
|
||||
world_size: int,
|
||||
port: int,
|
||||
quant_level: str,
|
||||
):
|
||||
# FP keeps the all-reduce bit-exact for small fp16 integers, so every
|
||||
# replayed round can be checked exactly.
|
||||
os.environ["VLLM_ROCM_QUICK_REDUCE_QUANTIZATION"] = quant_level
|
||||
os.environ["VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16"] = "0"
|
||||
_log(f"cudagraph worker start: rank={rank} quant={quant_level}")
|
||||
|
||||
device = torch.device(f"cuda:{rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
dist.init_process_group(
|
||||
backend="gloo",
|
||||
init_method=f"tcp://127.0.0.1:{port}",
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
)
|
||||
|
||||
qar = None
|
||||
try:
|
||||
from vllm.distributed.device_communicators.quick_all_reduce import (
|
||||
QuickAllReduce,
|
||||
)
|
||||
|
||||
qar = QuickAllReduce(group=dist.GroupMember.WORLD, device=rank)
|
||||
assert not qar.disabled
|
||||
|
||||
N = CUDAGRAPH_NUM_ELEMENTS
|
||||
inp = torch.empty(N, dtype=torch.float16, device=device)
|
||||
out = torch.empty(N, dtype=torch.float16, device=device)
|
||||
assert qar.should_quick_allreduce(inp)
|
||||
|
||||
# Every rank contributes the same value v in a round, so the true
|
||||
# cross-rank all-reduce sum is simply world_size * v.
|
||||
def expected(v):
|
||||
return float(world_size * v)
|
||||
|
||||
if rank == 0:
|
||||
print(
|
||||
f"[repro] world_size={world_size} elems={N} regime={quant_level} fp16",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Warmup, then capture a graph with EXACTLY ONE quick-reduce (isolated
|
||||
# qr, so it is the sole writer of its flag slot -- the condition that
|
||||
# triggers the stale-flag bug).
|
||||
inp.fill_(1.0)
|
||||
qar.quick_all_reduce(inp, out=out)
|
||||
torch.accelerator.synchronize()
|
||||
dist.barrier()
|
||||
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
qar.quick_all_reduce(inp, out=out)
|
||||
torch.accelerator.synchronize()
|
||||
dist.barrier()
|
||||
|
||||
for v in range(CUDAGRAPH_ROUNDS):
|
||||
inp.fill_(float(v)) # in-place: same value on every rank
|
||||
dist.barrier()
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
dist.barrier()
|
||||
got = out.float()
|
||||
expect = expected(v)
|
||||
if rank == 0:
|
||||
print(f"round {v}: got={got[:10]}, expected={expect}", flush=True)
|
||||
mismatch = ~torch.isclose(got, torch.full_like(got, expect))
|
||||
num_mismatch = int(mismatch.sum().item())
|
||||
assert num_mismatch == 0, (
|
||||
f"rank={rank} round={v} expected={expect} "
|
||||
f"mismatched {num_mismatch}/{got.numel()} elements; "
|
||||
f"unique wrong values={torch.unique(got[mismatch])[:8].tolist()}"
|
||||
)
|
||||
_log(f"cudagraph worker complete: rank={rank} rounds={CUDAGRAPH_ROUNDS}")
|
||||
finally:
|
||||
if qar is not None:
|
||||
qar.close()
|
||||
if dist.is_initialized():
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
def _run_cudagraph_replay_test(*, world_size: int, quant_level: str):
|
||||
_log(f"launch {world_size}-GPU cudagraph replay case: quant={quant_level}")
|
||||
ctx = mp.get_context("spawn")
|
||||
port = get_open_port()
|
||||
procs = []
|
||||
|
||||
for rank in range(world_size):
|
||||
proc = ctx.Process(
|
||||
target=_quick_allreduce_cudagraph_worker,
|
||||
args=(rank, world_size, port, quant_level),
|
||||
)
|
||||
proc.start()
|
||||
procs.append(proc)
|
||||
|
||||
for proc in procs:
|
||||
proc.join(timeout=120)
|
||||
assert proc.exitcode == 0, f"worker exited with code {proc.exitcode}"
|
||||
_log(f"finished {world_size}-GPU cudagraph replay case: quant={quant_level}")
|
||||
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen2.5-0.5B-Instruct"
|
||||
E2E_PREFILL_TOKENS = 1024
|
||||
E2E_MAX_MODEL_LEN = 1536
|
||||
@@ -729,6 +841,18 @@ def test_quick_allreduce_two_gpu_correctness(quant_level):
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=CUDAGRAPH_WORLD_SIZE)
|
||||
def test_quick_allreduce_cudagraph_replay():
|
||||
# Regression test for the stale flag_color bug: a quick-reduce captured in a
|
||||
# CUDA graph must return correct results on every replay, not the data from
|
||||
# a previous round.
|
||||
_log("cudagraph replay case")
|
||||
_run_cudagraph_replay_test(
|
||||
world_size=CUDAGRAPH_WORLD_SIZE,
|
||||
quant_level="FP",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
current_platform.device_count() < WORLD_SIZE,
|
||||
reason="requires 2 ROCm GPUs",
|
||||
|
||||
Reference in New Issue
Block a user