[Perf][Distributed] Add low-SM multimem reduce-scatter for SM100/SM103 (#55072)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Signed-off-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Summer Yang <girasoleyang@gmail.com>
This commit is contained in:
co-authored by
Summer Yang
parent
8b365ff949
commit
442bd00d9f
@@ -7,6 +7,10 @@ namespace vllm {
|
||||
constexpr int kMnnvlLamportAgThreads = 128;
|
||||
constexpr int kMnnvlLamportRsThreads = 256;
|
||||
constexpr int kMnnvlLamportConcurrentPollMaxPacks = 8192;
|
||||
constexpr int kMnnvlMultimemRsThreads = 1024;
|
||||
constexpr int kMnnvlMultimemRsBlockLimit = 8;
|
||||
constexpr int kMnnvlMultimemRsVectorBytes = 16;
|
||||
constexpr int kMnnvlMultimemRsUnroll = 8;
|
||||
|
||||
using CopyPack = array_t<uint64_t, 2>;
|
||||
|
||||
@@ -48,6 +52,127 @@ __global__ void __launch_bounds__(512, 1)
|
||||
barrier_at_end<ngpus, true>(sg, self_sg, rank);
|
||||
}
|
||||
|
||||
// Multimem reduce-scatter reduces through an NVLS multicast mapping, which
|
||||
// has no AMD equivalent. The whole path is compiled out on ROCm and the host
|
||||
// entry point rejects the call instead of launching anything.
|
||||
#if !defined(USE_ROCM)
|
||||
|
||||
template <typename T>
|
||||
DINLINE void multimem_load_reduce_16(uint32_t (&result)[4], const T* address) {
|
||||
#if CUDA_VERSION >= 12020 && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
if constexpr (std::is_same<T, nv_bfloat16>::value) {
|
||||
asm volatile(
|
||||
"multimem.ld_reduce.relaxed.sys.global.add.acc::f32.v4.bf16x2 "
|
||||
"{%0,%1,%2,%3}, [%4];"
|
||||
: "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3])
|
||||
: "l"(address)
|
||||
: "memory");
|
||||
} else if constexpr (std::is_same<T, half>::value) {
|
||||
asm volatile(
|
||||
"multimem.ld_reduce.relaxed.sys.global.add.acc::f32.v4.f16x2 "
|
||||
"{%0,%1,%2,%3}, [%4];"
|
||||
: "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3])
|
||||
: "l"(address)
|
||||
: "memory");
|
||||
} else {
|
||||
static_assert(std::is_same<T, float>::value);
|
||||
asm volatile(
|
||||
"multimem.ld_reduce.relaxed.sys.global.add.v4.f32 "
|
||||
"{%0,%1,%2,%3}, [%4];"
|
||||
: "=r"(result[0]), "=r"(result[1]), "=r"(result[2]), "=r"(result[3])
|
||||
: "l"(address)
|
||||
: "memory");
|
||||
}
|
||||
#else
|
||||
asm volatile("trap;");
|
||||
#endif
|
||||
}
|
||||
|
||||
DINLINE void store_global_16(void* address, const uint32_t (&value)[4]) {
|
||||
asm volatile("st.global.v4.u32 [%0], {%1,%2,%3,%4};"
|
||||
:
|
||||
: "l"(address), "r"(value[0]), "r"(value[1]), "r"(value[2]),
|
||||
"r"(value[3])
|
||||
: "memory");
|
||||
}
|
||||
|
||||
DINLINE void mnnvl_multimem_publish_flag(FlagType* flag_addr, FlagType flag) {
|
||||
#if CUDA_VERSION >= 12020 && defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
asm volatile("multimem.st.release.sys.global.u32 [%0], %1;"
|
||||
:
|
||||
: "l"(flag_addr), "r"(flag)
|
||||
: "memory");
|
||||
// The flag is stored through the multicast alias and polled through the
|
||||
// local unicast alias in the same grid.
|
||||
asm volatile("fence.proxy.alias;" ::: "memory");
|
||||
#else
|
||||
asm volatile("trap;");
|
||||
#endif
|
||||
}
|
||||
|
||||
template <bool ready, int ngpus>
|
||||
__global__ void mnnvl_multimem_barrier_kernel(Signal* local_signal,
|
||||
Signal* multicast_signal,
|
||||
int rank) {
|
||||
FlagType flag = local_signal->_flag[0] + 1;
|
||||
FlagType* multicast_counters =
|
||||
ready ? multicast_signal->start[0] : multicast_signal->end[0];
|
||||
FlagType* local_counters =
|
||||
ready ? local_signal->start[0] : local_signal->end[0];
|
||||
// Replicate this rank's counter into every backing allocation.
|
||||
mnnvl_multimem_publish_flag(&multicast_counters[rank], flag);
|
||||
#pragma unroll
|
||||
for (int peer = 0; peer < ngpus; ++peer) {
|
||||
while (ld_flag_acquire(&local_counters[peer]) != flag);
|
||||
}
|
||||
local_signal->_flag[0] = flag;
|
||||
}
|
||||
|
||||
template <typename T, int ngpus>
|
||||
__global__ void __launch_bounds__(kMnnvlMultimemRsThreads, 1)
|
||||
mnnvl_multimem_reduce_scatter_kernel(const T* __restrict__ multicast_input,
|
||||
T* __restrict__ result, int rank,
|
||||
int packs_per_rank) {
|
||||
constexpr int kThreadsPerWarp = 32;
|
||||
constexpr int kWarpsPerCta = kMnnvlMultimemRsThreads / kThreadsPerWarp;
|
||||
constexpr int kPacksPerWarpIteration =
|
||||
kThreadsPerWarp * kMnnvlMultimemRsUnroll;
|
||||
|
||||
int lane;
|
||||
asm volatile("mov.u32 %0, %%laneid;" : "=r"(lane));
|
||||
int warp = blockIdx.x * kWarpsPerCta + threadIdx.x / kThreadsPerWarp;
|
||||
int num_warps = gridDim.x * kWarpsPerCta;
|
||||
int pack_offset = warp * kPacksPerWarpIteration + lane;
|
||||
int pack_stride = num_warps * kPacksPerWarpIteration;
|
||||
auto* rank_input = reinterpret_cast<const char*>(multicast_input) +
|
||||
rank * packs_per_rank * kMnnvlMultimemRsVectorBytes;
|
||||
auto* rank_output = reinterpret_cast<char*>(result);
|
||||
|
||||
while (pack_offset < packs_per_rank) {
|
||||
uint32_t reduced[kMnnvlMultimemRsUnroll][4];
|
||||
#pragma unroll
|
||||
for (int u = 0; u < kMnnvlMultimemRsUnroll; ++u) {
|
||||
int pack = pack_offset + u * kThreadsPerWarp;
|
||||
if (pack < packs_per_rank) {
|
||||
multimem_load_reduce_16<T>(
|
||||
reduced[u], reinterpret_cast<const T*>(
|
||||
rank_input + pack * kMnnvlMultimemRsVectorBytes));
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int u = 0; u < kMnnvlMultimemRsUnroll; ++u) {
|
||||
int pack = pack_offset + u * kThreadsPerWarp;
|
||||
if (pack < packs_per_rank) {
|
||||
store_global_16(rank_output + pack * kMnnvlMultimemRsVectorBytes,
|
||||
reduced[u]);
|
||||
}
|
||||
}
|
||||
pack_offset += pack_stride;
|
||||
}
|
||||
}
|
||||
|
||||
#endif // !defined(USE_ROCM)
|
||||
|
||||
template <typename P>
|
||||
union LamportPack {
|
||||
P packed;
|
||||
|
||||
@@ -339,6 +339,12 @@ class CustomAllreduce {
|
||||
void mnnvl_lamport_reduce_scatter(cudaStream_t stream, T* input, T* output,
|
||||
void* local_buffer, uint32_t* epochs,
|
||||
int size, int stage_size_bytes);
|
||||
template <typename T>
|
||||
void mnnvl_multimem_reduce_scatter(cudaStream_t stream,
|
||||
const T* multicast_input, T* output,
|
||||
Signal* local_signal,
|
||||
Signal* multicast_signal, int size,
|
||||
int block_limit);
|
||||
|
||||
~CustomAllreduce() {
|
||||
for (auto [_, ptr] : ipc_handles_) {
|
||||
|
||||
@@ -191,6 +191,55 @@ void CustomAllreduce::mnnvl_lamport_reduce_scatter(cudaStream_t stream,
|
||||
#undef MNNVL_LAMPORT_RS_LAUNCH
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void CustomAllreduce::mnnvl_multimem_reduce_scatter(
|
||||
cudaStream_t stream, const T* multicast_input, T* output,
|
||||
Signal* local_signal, Signal* multicast_signal, int size, int block_limit) {
|
||||
#if defined(USE_ROCM)
|
||||
throw std::runtime_error(
|
||||
"MNNVL multimem reduce-scatter requires NVLS multicast and is not "
|
||||
"supported on ROCm");
|
||||
#else
|
||||
if (size <= 0)
|
||||
throw std::runtime_error(
|
||||
"MNNVL multimem reduce-scatter requires a non-empty input");
|
||||
int size_bytes = size * sizeof(T);
|
||||
if (size_bytes % (kMnnvlMultimemRsVectorBytes * world_size_) != 0)
|
||||
throw std::runtime_error(
|
||||
"MNNVL multimem reduce-scatter requires each output shard byte size "
|
||||
"to be a multiple of 16");
|
||||
if (block_limit <= 0 || block_limit > kMnnvlMultimemRsBlockLimit)
|
||||
throw std::runtime_error(
|
||||
"MNNVL multimem reduce-scatter block limit is out of range");
|
||||
|
||||
int packs_per_rank = size_bytes / kMnnvlMultimemRsVectorBytes / world_size_;
|
||||
int packs_per_block = kMnnvlMultimemRsThreads * kMnnvlMultimemRsUnroll;
|
||||
int blocks = std::min(
|
||||
block_limit, (packs_per_rank + packs_per_block - 1) / packs_per_block);
|
||||
|
||||
#define MNNVL_MULTIMEM_RS_CASE(ngpus) \
|
||||
case ngpus: \
|
||||
mnnvl_multimem_barrier_kernel<true, ngpus> \
|
||||
<<<1, 1, 0, stream>>>(local_signal, multicast_signal, rank_); \
|
||||
mnnvl_multimem_reduce_scatter_kernel<T, ngpus> \
|
||||
<<<blocks, kMnnvlMultimemRsThreads, 0, stream>>>( \
|
||||
multicast_input, output, rank_, packs_per_rank); \
|
||||
mnnvl_multimem_barrier_kernel<false, ngpus> \
|
||||
<<<1, 1, 0, stream>>>(local_signal, multicast_signal, rank_); \
|
||||
break;
|
||||
|
||||
switch (world_size_) {
|
||||
MNNVL_MULTIMEM_RS_CASE(2)
|
||||
MNNVL_MULTIMEM_RS_CASE(4)
|
||||
MNNVL_MULTIMEM_RS_CASE(8)
|
||||
default:
|
||||
throw std::runtime_error(
|
||||
"MNNVL multimem reduce-scatter only supports num gpus in (2,4,8)");
|
||||
}
|
||||
#undef MNNVL_MULTIMEM_RS_CASE
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
|
||||
using fptr_t = int64_t;
|
||||
@@ -360,3 +409,62 @@ void mnnvl_lamport_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
"bfloat16");
|
||||
}
|
||||
}
|
||||
|
||||
void mnnvl_multimem_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
torch::stable::Tensor& out,
|
||||
fptr_t _local_buffer,
|
||||
fptr_t _multicast_buffer,
|
||||
int64_t stage_sz_bytes,
|
||||
int64_t block_limit) {
|
||||
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
inp.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream(inp.get_device_index());
|
||||
|
||||
STD_TORCH_CHECK((inp.scalar_type()) == (out.scalar_type()));
|
||||
STD_TORCH_CHECK((out.numel() * fa->world_size_) == (inp.numel()));
|
||||
STD_TORCH_CHECK(inp.numel() > 0);
|
||||
STD_TORCH_CHECK(_is_weak_contiguous(out));
|
||||
STD_TORCH_CHECK(_is_weak_contiguous(inp));
|
||||
auto input_size = inp.numel() * inp.element_size();
|
||||
STD_TORCH_CHECK(input_size <= stage_sz_bytes);
|
||||
auto local_buffer = reinterpret_cast<void*>(_local_buffer);
|
||||
auto multicast_buffer = reinterpret_cast<void*>(_multicast_buffer);
|
||||
STD_TORCH_CHECK(local_buffer != nullptr);
|
||||
STD_TORCH_CHECK(multicast_buffer != nullptr);
|
||||
auto* local_signal = reinterpret_cast<vllm::Signal*>(
|
||||
static_cast<char*>(local_buffer) - sizeof(vllm::Signal));
|
||||
auto* multicast_signal = reinterpret_cast<vllm::Signal*>(
|
||||
static_cast<char*>(multicast_buffer) - sizeof(vllm::Signal));
|
||||
STD_CUDA_CHECK(cudaMemcpyAsync(local_buffer, inp.const_data_ptr(), input_size,
|
||||
cudaMemcpyDeviceToDevice, stream));
|
||||
switch (out.scalar_type()) {
|
||||
case torch::headeronly::ScalarType::Float: {
|
||||
fa->mnnvl_multimem_reduce_scatter<float>(
|
||||
stream, reinterpret_cast<float*>(multicast_buffer),
|
||||
reinterpret_cast<float*>(out.mutable_data_ptr()), local_signal,
|
||||
multicast_signal, inp.numel(), block_limit);
|
||||
break;
|
||||
}
|
||||
case torch::headeronly::ScalarType::Half: {
|
||||
fa->mnnvl_multimem_reduce_scatter<half>(
|
||||
stream, reinterpret_cast<half*>(multicast_buffer),
|
||||
reinterpret_cast<half*>(out.mutable_data_ptr()), local_signal,
|
||||
multicast_signal, inp.numel(), block_limit);
|
||||
break;
|
||||
}
|
||||
#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
|
||||
case torch::headeronly::ScalarType::BFloat16: {
|
||||
fa->mnnvl_multimem_reduce_scatter<nv_bfloat16>(
|
||||
stream, reinterpret_cast<nv_bfloat16*>(multicast_buffer),
|
||||
reinterpret_cast<nv_bfloat16*>(out.mutable_data_ptr()), local_signal,
|
||||
multicast_signal, inp.numel(), block_limit);
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
default:
|
||||
throw std::runtime_error(
|
||||
"MNNVL multimem reduce-scatter only supports float32, float16 and "
|
||||
"bfloat16");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,6 +17,10 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_custom_ar, custom_ag_rs) {
|
||||
custom_ag_rs.def(
|
||||
"mnnvl_lamport_reduce_scatter(int fa, Tensor inp, Tensor! out, int "
|
||||
"local_buffer, int epoch_buffer, int stage_sz_bytes) -> ()");
|
||||
custom_ag_rs.def(
|
||||
"mnnvl_multimem_reduce_scatter(int fa, Tensor inp, Tensor! out, int "
|
||||
"local_buffer, int multicast_buffer, int stage_sz_bytes, int "
|
||||
"block_limit) -> ()");
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CUDA, custom_ag_rs) {
|
||||
@@ -26,4 +30,6 @@ STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CUDA, custom_ag_rs) {
|
||||
custom_ag_rs.impl("custom_reduce_scatter", TORCH_BOX(&custom_reduce_scatter));
|
||||
custom_ag_rs.impl("mnnvl_lamport_reduce_scatter",
|
||||
TORCH_BOX(&mnnvl_lamport_reduce_scatter));
|
||||
custom_ag_rs.impl("mnnvl_multimem_reduce_scatter",
|
||||
TORCH_BOX(&mnnvl_multimem_reduce_scatter));
|
||||
}
|
||||
|
||||
@@ -527,6 +527,10 @@ void mnnvl_lamport_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
torch::stable::Tensor& out,
|
||||
fptr_t local_buffer, fptr_t epoch_buffer,
|
||||
int64_t stage_sz_bytes);
|
||||
void mnnvl_multimem_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
torch::stable::Tensor& out,
|
||||
fptr_t local_buffer, fptr_t multicast_buffer,
|
||||
int64_t stage_sz_bytes, int64_t block_limit);
|
||||
void dispose(fptr_t _fa);
|
||||
int64_t meta_size();
|
||||
void register_buffer(fptr_t _fa, const std::vector<int64_t>& fake_ipc_ptrs);
|
||||
|
||||
Reference in New Issue
Block a user