[Perf][Distributed] Add low-SM multimem reduce-scatter for SM100/SM103 (#55072)

Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
Signed-off-by: Summer Yang <girasoleyang@gmail.com>
Co-authored-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
Yongye Zhu
2026-09-25 03:27:21 +00:00
committed by GitHub
co-authored by Summer Yang
parent 8b365ff949
commit 442bd00d9f
10 changed files with 1243 additions and 31 deletions
+125
View File
@@ -7,6 +7,10 @@ namespace vllm {
constexpr int kMnnvlLamportAgThreads = 128;
constexpr int kMnnvlLamportRsThreads = 256;
constexpr int kMnnvlLamportConcurrentPollMaxPacks = 8192;
constexpr int kMnnvlMultimemRsThreads = 1024;
constexpr int kMnnvlMultimemRsBlockLimit = 8;
constexpr int kMnnvlMultimemRsVectorBytes = 16;
constexpr int kMnnvlMultimemRsUnroll = 8;
using CopyPack = array_t<uint64_t, 2>;
@@ -48,6 +52,127 @@ __global__ void __launch_bounds__(512, 1)
barrier_at_end<ngpus, true>(sg, self_sg, rank);
}
// Multimem reduce-scatter reduces through an NVLS multicast mapping, which
// has no AMD equivalent. The whole path is compiled out on ROCm and the host
// entry point rejects the call instead of launching anything.
#if !defined(USE_ROCM)
template <typename T>
DINLINE void multimem_load_reduce_16(uint32_t (&result)[4], const T* address) {
#if CUDA_VERSION >= 12020 && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
if constexpr (std::is_same<T, nv_bfloat16>::value) {
asm volatile(
"multimem.ld_reduce.relaxed.sys.global.add.acc::f32.v4.bf16x2 "
"{%0,%1,%2,%3}, [%4];"
: "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3])
: "l"(address)
: "memory");
} else if constexpr (std::is_same<T, half>::value) {
asm volatile(
"multimem.ld_reduce.relaxed.sys.global.add.acc::f32.v4.f16x2 "
"{%0,%1,%2,%3}, [%4];"
: "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3])
: "l"(address)
: "memory");
} else {
static_assert(std::is_same<T, float>::value);
asm volatile(
"multimem.ld_reduce.relaxed.sys.global.add.v4.f32 "
"{%0,%1,%2,%3}, [%4];"
: "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3])
: "l"(address)
: "memory");
}
#else
asm volatile("trap;");
#endif
}
DINLINE void store_global_16(void* address, const uint32_t (&value)[4]) {
asm volatile("st.global.v4.u32 [%0], {%1,%2,%3,%4};"
:
: "l"(address), "r"(value[0]), "r"(value[1]), "r"(value[2]),
"r"(value[3])
: "memory");
}
DINLINE void mnnvl_multimem_publish_flag(FlagType* flag_addr, FlagType flag) {
#if CUDA_VERSION >= 12020 && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
asm volatile("multimem.st.release.sys.global.u32 [%0], %1;"
:
: "l"(flag_addr), "r"(flag)
: "memory");
// The flag is stored through the multicast alias and polled through the
// local unicast alias in the same grid.
asm volatile("fence.proxy.alias;" ::: "memory");
#else
asm volatile("trap;");
#endif
}
template <bool ready, int ngpus>
__global__ void mnnvl_multimem_barrier_kernel(Signal* local_signal,
Signal* multicast_signal,
int rank) {
FlagType flag = local_signal->_flag[0] + 1;
FlagType* multicast_counters =
ready ? multicast_signal->start[0] : multicast_signal->end[0];
FlagType* local_counters =
ready ? local_signal->start[0] : local_signal->end[0];
// Replicate this rank's counter into every backing allocation.
mnnvl_multimem_publish_flag(&multicast_counters[rank], flag);
#pragma unroll
for (int peer = 0; peer < ngpus; ++peer) {
while (ld_flag_acquire(&local_counters[peer]) != flag);
}
local_signal->_flag[0] = flag;
}
template <typename T, int ngpus>
__global__ void __launch_bounds__(kMnnvlMultimemRsThreads, 1)
mnnvl_multimem_reduce_scatter_kernel(const T* __restrict__ multicast_input,
T* __restrict__ result, int rank,
int packs_per_rank) {
constexpr int kThreadsPerWarp = 32;
constexpr int kWarpsPerCta = kMnnvlMultimemRsThreads / kThreadsPerWarp;
constexpr int kPacksPerWarpIteration =
kThreadsPerWarp * kMnnvlMultimemRsUnroll;
int lane;
asm volatile("mov.u32 %0, %%laneid;" : "=r"(lane));
int warp = blockIdx.x * kWarpsPerCta + threadIdx.x / kThreadsPerWarp;
int num_warps = gridDim.x * kWarpsPerCta;
int pack_offset = warp * kPacksPerWarpIteration + lane;
int pack_stride = num_warps * kPacksPerWarpIteration;
auto* rank_input = reinterpret_cast<const char*>(multicast_input) +
rank * packs_per_rank * kMnnvlMultimemRsVectorBytes;
auto* rank_output = reinterpret_cast<char*>(result);
while (pack_offset < packs_per_rank) {
uint32_t reduced[kMnnvlMultimemRsUnroll][4];
#pragma unroll
for (int u = 0; u < kMnnvlMultimemRsUnroll; ++u) {
int pack = pack_offset + u * kThreadsPerWarp;
if (pack < packs_per_rank) {
multimem_load_reduce_16<T>(
reduced[u], reinterpret_cast<const T*>(
rank_input + pack * kMnnvlMultimemRsVectorBytes));
}
}
#pragma unroll
for (int u = 0; u < kMnnvlMultimemRsUnroll; ++u) {
int pack = pack_offset + u * kThreadsPerWarp;
if (pack < packs_per_rank) {
store_global_16(rank_output + pack * kMnnvlMultimemRsVectorBytes,
reduced[u]);
}
}
pack_offset += pack_stride;
}
}
#endif // !defined(USE_ROCM)
template <typename P>
union LamportPack {
P packed;
+6
View File
@@ -339,6 +339,12 @@ class CustomAllreduce {
void mnnvl_lamport_reduce_scatter(cudaStream_t stream, T* input, T* output,
void* local_buffer, uint32_t* epochs,
int size, int stage_size_bytes);
template <typename T>
void mnnvl_multimem_reduce_scatter(cudaStream_t stream,
const T* multicast_input, T* output,
Signal* local_signal,
Signal* multicast_signal, int size,
int block_limit);
~CustomAllreduce() {
for (auto [_, ptr] : ipc_handles_) {
@@ -191,6 +191,55 @@ void CustomAllreduce::mnnvl_lamport_reduce_scatter(cudaStream_t stream,
#undef MNNVL_LAMPORT_RS_LAUNCH
}
template <typename T>
void CustomAllreduce::mnnvl_multimem_reduce_scatter(
cudaStream_t stream, const T* multicast_input, T* output,
Signal* local_signal, Signal* multicast_signal, int size, int block_limit) {
#if defined(USE_ROCM)
throw std::runtime_error(
"MNNVL multimem reduce-scatter requires NVLS multicast and is not "
"supported on ROCm");
#else
if (size <= 0)
throw std::runtime_error(
"MNNVL multimem reduce-scatter requires a non-empty input");
int size_bytes = size * sizeof(T);
if (size_bytes % (kMnnvlMultimemRsVectorBytes * world_size_) != 0)
throw std::runtime_error(
"MNNVL multimem reduce-scatter requires each output shard byte size "
"to be a multiple of 16");
if (block_limit <= 0 || block_limit > kMnnvlMultimemRsBlockLimit)
throw std::runtime_error(
"MNNVL multimem reduce-scatter block limit is out of range");
int packs_per_rank = size_bytes / kMnnvlMultimemRsVectorBytes / world_size_;
int packs_per_block = kMnnvlMultimemRsThreads * kMnnvlMultimemRsUnroll;
int blocks = std::min(
block_limit, (packs_per_rank + packs_per_block - 1) / packs_per_block);
#define MNNVL_MULTIMEM_RS_CASE(ngpus) \
case ngpus: \
mnnvl_multimem_barrier_kernel<true, ngpus> \
<<<1, 1, 0, stream>>>(local_signal, multicast_signal, rank_); \
mnnvl_multimem_reduce_scatter_kernel<T, ngpus> \
<<<blocks, kMnnvlMultimemRsThreads, 0, stream>>>( \
multicast_input, output, rank_, packs_per_rank); \
mnnvl_multimem_barrier_kernel<false, ngpus> \
<<<1, 1, 0, stream>>>(local_signal, multicast_signal, rank_); \
break;
switch (world_size_) {
MNNVL_MULTIMEM_RS_CASE(2)
MNNVL_MULTIMEM_RS_CASE(4)
MNNVL_MULTIMEM_RS_CASE(8)
default:
throw std::runtime_error(
"MNNVL multimem reduce-scatter only supports num gpus in (2,4,8)");
}
#undef MNNVL_MULTIMEM_RS_CASE
#endif
}
} // namespace vllm
using fptr_t = int64_t;
@@ -360,3 +409,62 @@ void mnnvl_lamport_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
"bfloat16");
}
}
void mnnvl_multimem_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
torch::stable::Tensor& out,
fptr_t _local_buffer,
fptr_t _multicast_buffer,
int64_t stage_sz_bytes,
int64_t block_limit) {
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
const torch::stable::accelerator::DeviceGuard device_guard(
inp.get_device_index());
const cudaStream_t stream = get_current_cuda_stream(inp.get_device_index());
STD_TORCH_CHECK((inp.scalar_type()) == (out.scalar_type()));
STD_TORCH_CHECK((out.numel() * fa->world_size_) == (inp.numel()));
STD_TORCH_CHECK(inp.numel() > 0);
STD_TORCH_CHECK(_is_weak_contiguous(out));
STD_TORCH_CHECK(_is_weak_contiguous(inp));
auto input_size = inp.numel() * inp.element_size();
STD_TORCH_CHECK(input_size <= stage_sz_bytes);
auto local_buffer = reinterpret_cast<void*>(_local_buffer);
auto multicast_buffer = reinterpret_cast<void*>(_multicast_buffer);
STD_TORCH_CHECK(local_buffer != nullptr);
STD_TORCH_CHECK(multicast_buffer != nullptr);
auto* local_signal = reinterpret_cast<vllm::Signal*>(
static_cast<char*>(local_buffer) - sizeof(vllm::Signal));
auto* multicast_signal = reinterpret_cast<vllm::Signal*>(
static_cast<char*>(multicast_buffer) - sizeof(vllm::Signal));
STD_CUDA_CHECK(cudaMemcpyAsync(local_buffer, inp.const_data_ptr(), input_size,
cudaMemcpyDeviceToDevice, stream));
switch (out.scalar_type()) {
case torch::headeronly::ScalarType::Float: {
fa->mnnvl_multimem_reduce_scatter<float>(
stream, reinterpret_cast<float*>(multicast_buffer),
reinterpret_cast<float*>(out.mutable_data_ptr()), local_signal,
multicast_signal, inp.numel(), block_limit);
break;
}
case torch::headeronly::ScalarType::Half: {
fa->mnnvl_multimem_reduce_scatter<half>(
stream, reinterpret_cast<half*>(multicast_buffer),
reinterpret_cast<half*>(out.mutable_data_ptr()), local_signal,
multicast_signal, inp.numel(), block_limit);
break;
}
#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
case torch::headeronly::ScalarType::BFloat16: {
fa->mnnvl_multimem_reduce_scatter<nv_bfloat16>(
stream, reinterpret_cast<nv_bfloat16*>(multicast_buffer),
reinterpret_cast<nv_bfloat16*>(out.mutable_data_ptr()), local_signal,
multicast_signal, inp.numel(), block_limit);
break;
}
#endif
default:
throw std::runtime_error(
"MNNVL multimem reduce-scatter only supports float32, float16 and "
"bfloat16");
}
}
@@ -17,6 +17,10 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_custom_ar, custom_ag_rs) {
custom_ag_rs.def(
"mnnvl_lamport_reduce_scatter(int fa, Tensor inp, Tensor! out, int "
"local_buffer, int epoch_buffer, int stage_sz_bytes) -> ()");
custom_ag_rs.def(
"mnnvl_multimem_reduce_scatter(int fa, Tensor inp, Tensor! out, int "
"local_buffer, int multicast_buffer, int stage_sz_bytes, int "
"block_limit) -> ()");
}
STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CUDA, custom_ag_rs) {
@@ -26,4 +30,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CUDA, custom_ag_rs) {
custom_ag_rs.impl("custom_reduce_scatter", TORCH_BOX(&custom_reduce_scatter));
custom_ag_rs.impl("mnnvl_lamport_reduce_scatter",
TORCH_BOX(&mnnvl_lamport_reduce_scatter));
custom_ag_rs.impl("mnnvl_multimem_reduce_scatter",
TORCH_BOX(&mnnvl_multimem_reduce_scatter));
}
+4
View File
@@ -527,6 +527,10 @@ void mnnvl_lamport_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
torch::stable::Tensor& out,
fptr_t local_buffer, fptr_t epoch_buffer,
int64_t stage_sz_bytes);
void mnnvl_multimem_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
torch::stable::Tensor& out,
fptr_t local_buffer, fptr_t multicast_buffer,
int64_t stage_sz_bytes, int64_t block_limit);
void dispose(fptr_t _fa);
int64_t meta_size();
void register_buffer(fptr_t _fa, const std::vector<int64_t>& fake_ipc_ptrs);