[Model] Add Kimi K3 support: model files and kernels [1/N] (#50089)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: gnovack <novackgm@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: xiaozhoupy <peiyuanzhou1994@gmail.com> Co-authored-by: Roy Wang <yasong.wang@inferact.ai> Co-authored-by: Jeff (Junze) Ma <93145857+majunze2001@users.noreply.github.com> Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
co-authored by
Isotr0py
Thien Tran
Bugen Zhao
Yongye Zhu
Ziming Huang
Roger Wang
Isotr0py
aoshen02
Woosuk Kwon
Jee Jee Li
aoshen02
Summer Yang
Kevin H. Luu
Bowen Wang
gnovack
Nick Hill
Yifan Qiao
xiaozhoupy
Roy Wang
Jeff Ma
Codex
parent
6f00a1ae3b
commit
7c6729b769
@@ -0,0 +1,327 @@
|
||||
#pragma once
|
||||
|
||||
#include "custom_collective_common.cuh"
|
||||
|
||||
namespace vllm {
|
||||
|
||||
constexpr int kMnnvlLamportAgThreads = 128;
|
||||
constexpr int kMnnvlLamportRsThreads = 256;
|
||||
constexpr int kMnnvlLamportConcurrentPollMaxPacks = 8192;
|
||||
|
||||
using CopyPack = array_t<uint64_t, 2>;
|
||||
|
||||
template <int ngpus>
|
||||
__global__ void __launch_bounds__(512, 1)
|
||||
cross_device_all_gather(RankData* _dp, RankSignals sg, Signal* self_sg,
|
||||
CopyPack* __restrict__ result, int rank,
|
||||
int size_per_rank) {
|
||||
auto dp = *_dp;
|
||||
int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int stride = gridDim.x * blockDim.x;
|
||||
barrier_at_start<ngpus>(sg, self_sg, rank);
|
||||
#pragma unroll
|
||||
for (int src_rank = 0; src_rank < ngpus; ++src_rank) {
|
||||
auto src = reinterpret_cast<const CopyPack*>(dp.ptrs[src_rank]);
|
||||
auto dst = result + src_rank * size_per_rank;
|
||||
for (int idx = tid; idx < size_per_rank; idx += stride) {
|
||||
dst[idx] = src[idx];
|
||||
}
|
||||
}
|
||||
barrier_at_end<ngpus, true>(sg, self_sg, rank);
|
||||
}
|
||||
|
||||
template <typename T, int ngpus>
|
||||
__global__ void __launch_bounds__(512, 1)
|
||||
cross_device_reduce_scatter(RankData* _dp, RankSignals sg, Signal* self_sg,
|
||||
T* __restrict__ result, int rank,
|
||||
int size_per_rank) {
|
||||
using P = typename packed_t<T>::P;
|
||||
using A = typename packed_t<T>::A;
|
||||
auto dp = *_dp;
|
||||
auto offset = rank * size_per_rank;
|
||||
barrier_at_start<ngpus>(sg, self_sg, rank);
|
||||
for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < size_per_rank;
|
||||
idx += gridDim.x * blockDim.x) {
|
||||
reinterpret_cast<P*>(result)[idx] =
|
||||
packed_reduce<P, ngpus, A>((const P**)&dp.ptrs[0], offset + idx);
|
||||
}
|
||||
barrier_at_end<ngpus, true>(sg, self_sg, rank);
|
||||
}
|
||||
|
||||
template <typename P>
|
||||
union LamportPack {
|
||||
P packed;
|
||||
uint32_t words[sizeof(P) / sizeof(uint32_t)];
|
||||
};
|
||||
|
||||
template <typename P>
|
||||
DINLINE LamportPack<P> load_lamport_pack(const P* ptr) {
|
||||
static_assert(sizeof(P) == 16);
|
||||
LamportPack<P> value;
|
||||
#if !defined(USE_ROCM)
|
||||
asm volatile("ld.volatile.global.v4.u32 {%0, %1, %2, %3}, [%4];"
|
||||
: "=r"(value.words[0]), "=r"(value.words[1]),
|
||||
"=r"(value.words[2]), "=r"(value.words[3])
|
||||
: "l"(ptr)
|
||||
: "memory");
|
||||
#else
|
||||
const volatile uint32_t* src =
|
||||
reinterpret_cast<const volatile uint32_t*>(ptr);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < sizeof(P) / sizeof(uint32_t); ++i) {
|
||||
value.words[i] = src[i];
|
||||
}
|
||||
#endif
|
||||
return value;
|
||||
}
|
||||
|
||||
template <typename P>
|
||||
DINLINE bool is_lamport_dirty(const LamportPack<P>& value) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < sizeof(P) / sizeof(uint32_t); ++i) {
|
||||
if (value.words[i] == 0x80000000U) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
template <typename P>
|
||||
DINLINE P lamport_sentinel() {
|
||||
LamportPack<P> value;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < sizeof(P) / sizeof(uint32_t); ++i) {
|
||||
value.words[i] = 0x80000000U;
|
||||
}
|
||||
return value.packed;
|
||||
}
|
||||
|
||||
template <typename P>
|
||||
DINLINE P sanitize_lamport_payload(P packed) {
|
||||
LamportPack<P> value{.packed = packed};
|
||||
#pragma unroll
|
||||
for (int i = 0; i < sizeof(P) / sizeof(uint32_t); ++i) {
|
||||
if (value.words[i] == 0x80000000U) value.words[i] = 0;
|
||||
}
|
||||
return value.packed;
|
||||
}
|
||||
|
||||
template <typename P>
|
||||
DINLINE P wait_lamport_payload(const P* ptr) {
|
||||
auto value = load_lamport_pack(ptr);
|
||||
while (is_lamport_dirty(value)) value = load_lamport_pack(ptr);
|
||||
return value.packed;
|
||||
}
|
||||
|
||||
template <typename P, int ngpus>
|
||||
DINLINE void wait_lamport_payloads(const P* base, int rank, int rank_stride,
|
||||
P local_value, P (&values)[ngpus]) {
|
||||
bool ready[ngpus];
|
||||
#pragma unroll
|
||||
for (int src_rank = 0; src_rank < ngpus; ++src_rank) {
|
||||
ready[src_rank] = src_rank == rank;
|
||||
if (src_rank == rank) values[src_rank] = local_value;
|
||||
}
|
||||
|
||||
int remaining = ngpus - 1;
|
||||
while (remaining != 0) {
|
||||
#pragma unroll
|
||||
for (int src_rank = 0; src_rank < ngpus; ++src_rank) {
|
||||
if (!ready[src_rank]) {
|
||||
auto value = load_lamport_pack(base + src_rank * rank_stride);
|
||||
if (!is_lamport_dirty(value)) {
|
||||
values[src_rank] = value.packed;
|
||||
ready[src_rank] = true;
|
||||
--remaining;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <typename P, typename A, int ngpus>
|
||||
DINLINE P reduce_lamport_payloads(const P* current_local, const P* packed_input,
|
||||
int rank, int size_per_rank, int idx) {
|
||||
P source_zero =
|
||||
rank == 0 ? packed_input[idx] : wait_lamport_payload(current_local + idx);
|
||||
A tmp = upcast(source_zero);
|
||||
#pragma unroll
|
||||
for (int src_rank = 1; src_rank < ngpus; ++src_rank) {
|
||||
P value = src_rank == rank
|
||||
? packed_input[rank * size_per_rank + idx]
|
||||
: wait_lamport_payload(current_local +
|
||||
src_rank * size_per_rank + idx);
|
||||
packed_assign_add(tmp, upcast(value));
|
||||
}
|
||||
return sanitize_lamport_payload(downcast<P>(tmp));
|
||||
}
|
||||
|
||||
DINLINE void lamport_cta_arrive(uint32_t* counter) {
|
||||
#if !defined(USE_ROCM)
|
||||
if (threadIdx.x < 32) {
|
||||
asm volatile("barrier.cta.sync 1, %0;" : : "r"(blockDim.x) : "memory");
|
||||
if (threadIdx.x == 0) {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 1000
|
||||
asm volatile("red.async.release.global.gpu.add.u32 [%0], 1;"
|
||||
:
|
||||
: "l"(counter)
|
||||
: "memory");
|
||||
#elif defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
|
||||
asm volatile("red.release.global.gpu.add.u32 [%0], 1;"
|
||||
:
|
||||
: "l"(counter)
|
||||
: "memory");
|
||||
#else
|
||||
atomicAdd(counter, 1);
|
||||
#endif
|
||||
}
|
||||
} else {
|
||||
asm volatile("barrier.cta.arrive 1, %0;" : : "r"(blockDim.x) : "memory");
|
||||
}
|
||||
#else
|
||||
__syncthreads();
|
||||
if (threadIdx.x == 0) atomicAdd(counter, 1);
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename T, int ngpus>
|
||||
__global__ void __launch_bounds__(kMnnvlLamportAgThreads, 1)
|
||||
mnnvl_lamport_all_gather(RankData* _dp, const T* __restrict__ input,
|
||||
T* __restrict__ result,
|
||||
T* __restrict__ multicast_buffer,
|
||||
uint32_t* __restrict__ epochs, int rank,
|
||||
int size_per_rank, int stage_size) {
|
||||
using P = typename packed_t<T>::P;
|
||||
#if !defined(USE_ROCM) && CUDA_VERSION >= 12000 && defined(__CUDA_ARCH__) && \
|
||||
(__CUDA_ARCH__ >= 900)
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
auto dp = *_dp;
|
||||
int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int stride = gridDim.x * blockDim.x;
|
||||
uint32_t epoch = epochs[0];
|
||||
int current_stage = epoch % 3;
|
||||
int dirty_stage = (epoch + 1) % 3;
|
||||
int dirty_size = epochs[2 + dirty_stage];
|
||||
auto local_buffer = reinterpret_cast<P*>(const_cast<void*>(dp.ptrs[rank]));
|
||||
auto current_local = local_buffer + current_stage * stage_size;
|
||||
auto dirty_local = local_buffer + dirty_stage * stage_size;
|
||||
auto current_multicast =
|
||||
reinterpret_cast<P*>(multicast_buffer) + current_stage * stage_size;
|
||||
auto packed_input = reinterpret_cast<const P*>(input);
|
||||
auto packed_result = reinterpret_cast<P*>(result);
|
||||
|
||||
int total_size = size_per_rank * ngpus;
|
||||
P local_value;
|
||||
if (tid < size_per_rank) {
|
||||
local_value = packed_input[tid];
|
||||
current_multicast[rank * size_per_rank + tid] =
|
||||
sanitize_lamport_payload(local_value);
|
||||
}
|
||||
#if !defined(USE_ROCM) && CUDA_VERSION >= 12000 && defined(__CUDA_ARCH__) && \
|
||||
(__CUDA_ARCH__ >= 900)
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
|
||||
lamport_cta_arrive(&epochs[1]);
|
||||
|
||||
for (int idx = tid; idx < dirty_size; idx += stride) {
|
||||
dirty_local[idx] = lamport_sentinel<P>();
|
||||
}
|
||||
|
||||
if (tid < size_per_rank) {
|
||||
#pragma unroll
|
||||
for (int src_rank = 0; src_rank < ngpus; ++src_rank) {
|
||||
int output_idx = src_rank * size_per_rank + tid;
|
||||
P value = src_rank == rank
|
||||
? local_value
|
||||
: wait_lamport_payload(current_local + output_idx);
|
||||
packed_result[output_idx] = value;
|
||||
}
|
||||
}
|
||||
|
||||
if (tid == 0) {
|
||||
while (*reinterpret_cast<volatile uint32_t*>(&epochs[1]) < gridDim.x);
|
||||
epochs[2 + current_stage] = total_size;
|
||||
epochs[0] = epoch + 1;
|
||||
epochs[1] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, int ngpus>
|
||||
__global__ void __launch_bounds__(kMnnvlLamportRsThreads, 1)
|
||||
mnnvl_lamport_reduce_scatter_kernel(RankData* _dp,
|
||||
const T* __restrict__ input,
|
||||
T* __restrict__ result,
|
||||
uint32_t* __restrict__ epochs, int rank,
|
||||
int size_per_rank, int stage_size) {
|
||||
using P = typename packed_t<T>::P;
|
||||
using A = typename packed_t<T>::A;
|
||||
#if !defined(USE_ROCM) && CUDA_VERSION >= 12000 && defined(__CUDA_ARCH__) && \
|
||||
(__CUDA_ARCH__ >= 900)
|
||||
cudaGridDependencySynchronize();
|
||||
#endif
|
||||
auto dp = *_dp;
|
||||
int dst_rank = blockIdx.x % ngpus;
|
||||
int tile = blockIdx.x / ngpus;
|
||||
int idx = tile * blockDim.x + threadIdx.x;
|
||||
int tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int stride = gridDim.x * blockDim.x;
|
||||
uint32_t epoch = epochs[0];
|
||||
int current_stage = epoch % 3;
|
||||
int dirty_stage = (epoch + 1) % 3;
|
||||
int dirty_size = epochs[2 + dirty_stage];
|
||||
auto local_buffer = reinterpret_cast<P*>(const_cast<void*>(dp.ptrs[rank]));
|
||||
auto current_local = local_buffer + current_stage * stage_size;
|
||||
auto dirty_local = local_buffer + dirty_stage * stage_size;
|
||||
auto packed_input = reinterpret_cast<const P*>(input);
|
||||
|
||||
if (idx < size_per_rank && dst_rank != rank) {
|
||||
auto dst = reinterpret_cast<P*>(const_cast<void*>(dp.ptrs[dst_rank])) +
|
||||
current_stage * stage_size + rank * size_per_rank;
|
||||
auto src = packed_input + dst_rank * size_per_rank;
|
||||
dst[idx] = sanitize_lamport_payload(src[idx]);
|
||||
}
|
||||
#if !defined(USE_ROCM) && CUDA_VERSION >= 12000 && defined(__CUDA_ARCH__) && \
|
||||
(__CUDA_ARCH__ >= 900)
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
#endif
|
||||
|
||||
lamport_cta_arrive(&epochs[1]);
|
||||
|
||||
for (int idx = tid; idx < dirty_size; idx += stride) {
|
||||
dirty_local[idx] = lamport_sentinel<P>();
|
||||
}
|
||||
|
||||
if (idx < size_per_rank && dst_rank == rank) {
|
||||
if constexpr (ngpus == 4) {
|
||||
if (size_per_rank > kMnnvlLamportConcurrentPollMaxPacks) {
|
||||
reinterpret_cast<P*>(result)[idx] =
|
||||
reduce_lamport_payloads<P, A, ngpus>(current_local, packed_input,
|
||||
rank, size_per_rank, idx);
|
||||
} else {
|
||||
P values[ngpus];
|
||||
wait_lamport_payloads<P, ngpus>(
|
||||
current_local + idx, rank, size_per_rank,
|
||||
packed_input[rank * size_per_rank + idx], values);
|
||||
A tmp = upcast(values[0]);
|
||||
#pragma unroll
|
||||
for (int src_rank = 1; src_rank < ngpus; ++src_rank) {
|
||||
packed_assign_add(tmp, upcast(values[src_rank]));
|
||||
}
|
||||
reinterpret_cast<P*>(result)[idx] =
|
||||
sanitize_lamport_payload(downcast<P>(tmp));
|
||||
}
|
||||
} else {
|
||||
reinterpret_cast<P*>(result)[idx] = reduce_lamport_payloads<P, A, ngpus>(
|
||||
current_local, packed_input, rank, size_per_rank, idx);
|
||||
}
|
||||
}
|
||||
|
||||
if (tid == 0) {
|
||||
while (*reinterpret_cast<volatile uint32_t*>(&epochs[1]) < gridDim.x);
|
||||
epochs[2 + current_stage] = size_per_rank * ngpus;
|
||||
epochs[0] = epoch + 1;
|
||||
epochs[1] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
+20
-296
@@ -1,299 +1,8 @@
|
||||
#pragma once
|
||||
|
||||
#include <cuda.h>
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
typedef __hip_bfloat16 nv_bfloat16;
|
||||
#endif
|
||||
|
||||
#include <iostream>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include "custom_collective_common.cuh"
|
||||
|
||||
namespace vllm {
|
||||
#define CUDACHECK(cmd) \
|
||||
do { \
|
||||
cudaError_t e = cmd; \
|
||||
if (e != cudaSuccess) { \
|
||||
printf("Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \
|
||||
cudaGetErrorString(e)); \
|
||||
exit(EXIT_FAILURE); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
// Maximal number of blocks in allreduce kernel.
|
||||
constexpr int kMaxBlocks = 36;
|
||||
|
||||
// Default number of blocks in allreduce kernel.
|
||||
#ifndef USE_ROCM
|
||||
const int defaultBlockLimit = 36;
|
||||
CUpointer_attribute rangeStartAddrAttr = CU_POINTER_ATTRIBUTE_RANGE_START_ADDR;
|
||||
#else
|
||||
const int defaultBlockLimit = 16;
|
||||
hipPointer_attribute rangeStartAddrAttr =
|
||||
HIP_POINTER_ATTRIBUTE_RANGE_START_ADDR;
|
||||
#endif
|
||||
|
||||
// Counter may overflow, but it's fine since unsigned int overflow is
|
||||
// well-defined behavior.
|
||||
using FlagType = uint32_t;
|
||||
|
||||
// Two sets of peer counters are needed for two syncs: starting and ending an
|
||||
// operation. The reason is that it's possible for peer GPU block to arrive at
|
||||
// the second sync point while the current GPU block haven't passed the first
|
||||
// sync point. Thus, peer GPU may write counter+1 while current GPU is busy
|
||||
// waiting for counter. We use alternating counter array to avoid this
|
||||
// possibility.
|
||||
struct Signal {
|
||||
alignas(128) FlagType start[kMaxBlocks][8];
|
||||
alignas(128) FlagType end[kMaxBlocks][8];
|
||||
alignas(128) FlagType _flag[kMaxBlocks]; // incremental flags for each rank
|
||||
};
|
||||
|
||||
struct __align__(16) RankData {
|
||||
const void* ptrs[8];
|
||||
};
|
||||
|
||||
struct __align__(16) RankSignals {
|
||||
Signal* signals[8];
|
||||
};
|
||||
|
||||
// like std::array, but aligned
|
||||
template <typename T, int sz>
|
||||
struct __align__(alignof(T) * sz) array_t {
|
||||
T data[sz];
|
||||
using type = T;
|
||||
static constexpr int size = sz;
|
||||
};
|
||||
|
||||
// use packed type to maximize memory efficiency
|
||||
// goal: generate ld.128 and st.128 instructions
|
||||
template <typename T>
|
||||
struct packed_t {
|
||||
// the (P)acked type for load/store
|
||||
using P = array_t<T, 16 / sizeof(T)>;
|
||||
// the (A)ccumulator type for reduction
|
||||
using A = array_t<float, 16 / sizeof(T)>;
|
||||
};
|
||||
|
||||
#define DINLINE __device__ __forceinline__
|
||||
|
||||
// scalar cast functions
|
||||
DINLINE float upcast_s(half val) { return __half2float(val); }
|
||||
|
||||
template <typename T>
|
||||
DINLINE T downcast_s(float val);
|
||||
template <>
|
||||
DINLINE half downcast_s(float val) {
|
||||
return __float2half(val);
|
||||
}
|
||||
|
||||
// scalar add functions
|
||||
// for some reason when compiling with Pytorch, the + operator for half and
|
||||
// bfloat is disabled so we call the intrinsics directly
|
||||
DINLINE half& assign_add(half& a, half b) {
|
||||
a = __hadd(a, b);
|
||||
return a;
|
||||
}
|
||||
DINLINE float& assign_add(float& a, float b) { return a += b; }
|
||||
|
||||
#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
|
||||
DINLINE float upcast_s(nv_bfloat16 val) { return __bfloat162float(val); }
|
||||
template <>
|
||||
DINLINE nv_bfloat16 downcast_s(float val) {
|
||||
return __float2bfloat16(val);
|
||||
}
|
||||
DINLINE nv_bfloat16& assign_add(nv_bfloat16& a, nv_bfloat16 b) {
|
||||
a = __hadd(a, b);
|
||||
return a;
|
||||
}
|
||||
#endif
|
||||
|
||||
template <typename T, int N>
|
||||
DINLINE array_t<T, N>& packed_assign_add(array_t<T, N>& a, array_t<T, N> b) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < N; i++) {
|
||||
assign_add(a.data[i], b.data[i]);
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
template <typename T, int N>
|
||||
DINLINE array_t<float, N> upcast(array_t<T, N> val) {
|
||||
if constexpr (std::is_same<T, float>::value) {
|
||||
return val;
|
||||
} else {
|
||||
array_t<float, N> out;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < N; i++) {
|
||||
out.data[i] = upcast_s(val.data[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename O>
|
||||
DINLINE O downcast(array_t<float, O::size> val) {
|
||||
if constexpr (std::is_same<typename O::type, float>::value) {
|
||||
return val;
|
||||
} else {
|
||||
O out;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < O::size; i++) {
|
||||
out.data[i] = downcast_s<typename O::type>(val.data[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
#if !defined(USE_ROCM)
|
||||
|
||||
static DINLINE void st_flag_release(FlagType* flag_addr, FlagType flag) {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
|
||||
asm volatile("st.release.sys.global.u32 [%1], %0;" ::"r"(flag),
|
||||
"l"(flag_addr));
|
||||
#else
|
||||
asm volatile("membar.sys; st.volatile.global.u32 [%1], %0;" ::"r"(flag),
|
||||
"l"(flag_addr));
|
||||
#endif
|
||||
}
|
||||
|
||||
static DINLINE FlagType ld_flag_acquire(FlagType* flag_addr) {
|
||||
FlagType flag;
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
|
||||
asm volatile("ld.acquire.sys.global.u32 %0, [%1];"
|
||||
: "=r"(flag)
|
||||
: "l"(flag_addr));
|
||||
#else
|
||||
asm volatile("ld.volatile.global.u32 %0, [%1]; membar.gl;"
|
||||
: "=r"(flag)
|
||||
: "l"(flag_addr));
|
||||
#endif
|
||||
return flag;
|
||||
}
|
||||
|
||||
static DINLINE void st_flag_volatile(FlagType* flag_addr, FlagType flag) {
|
||||
asm volatile("st.volatile.global.u32 [%1], %0;" ::"r"(flag), "l"(flag_addr));
|
||||
}
|
||||
|
||||
static DINLINE FlagType ld_flag_volatile(FlagType* flag_addr) {
|
||||
FlagType flag;
|
||||
asm volatile("ld.volatile.global.u32 %0, [%1];"
|
||||
: "=r"(flag)
|
||||
: "l"(flag_addr));
|
||||
return flag;
|
||||
}
|
||||
|
||||
// This function is meant to be used as the first synchronization in the all
|
||||
// reduce kernel. Thus, it doesn't need to make any visibility guarantees for
|
||||
// prior memory accesses. Note: volatile writes will not be reordered against
|
||||
// other volatile writes.
|
||||
template <int ngpus>
|
||||
DINLINE void barrier_at_start(const RankSignals& sg, Signal* self_sg,
|
||||
int rank) {
|
||||
uint32_t flag = self_sg->_flag[blockIdx.x] + 1;
|
||||
if (threadIdx.x < ngpus) {
|
||||
auto peer_counter_ptr = &sg.signals[threadIdx.x]->start[blockIdx.x][rank];
|
||||
auto self_counter_ptr = &self_sg->start[blockIdx.x][threadIdx.x];
|
||||
// Write the expected counter value to peer and wait for correct value
|
||||
// from peer.
|
||||
st_flag_volatile(peer_counter_ptr, flag);
|
||||
while (ld_flag_volatile(self_counter_ptr) != flag);
|
||||
}
|
||||
__syncthreads();
|
||||
// use one thread to update flag
|
||||
if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag;
|
||||
}
|
||||
|
||||
// This function is meant to be used as the second or the final
|
||||
// synchronization barrier in the all reduce kernel. If it's the final
|
||||
// synchronization barrier, we don't need to make any visibility guarantees
|
||||
// for prior memory accesses.
|
||||
template <int ngpus, bool final_sync = false>
|
||||
DINLINE void barrier_at_end(const RankSignals& sg, Signal* self_sg, int rank) {
|
||||
__syncthreads();
|
||||
uint32_t flag = self_sg->_flag[blockIdx.x] + 1;
|
||||
if (threadIdx.x < ngpus) {
|
||||
auto peer_counter_ptr = &sg.signals[threadIdx.x]->end[blockIdx.x][rank];
|
||||
auto self_counter_ptr = &self_sg->end[blockIdx.x][threadIdx.x];
|
||||
// Write the expected counter value to peer and wait for correct value from
|
||||
// peer.
|
||||
if constexpr (!final_sync) {
|
||||
st_flag_release(peer_counter_ptr, flag);
|
||||
while (ld_flag_acquire(self_counter_ptr) != flag);
|
||||
} else {
|
||||
st_flag_volatile(peer_counter_ptr, flag);
|
||||
while (ld_flag_volatile(self_counter_ptr) != flag);
|
||||
}
|
||||
}
|
||||
if constexpr (!final_sync) __syncthreads();
|
||||
|
||||
// use one thread to update flag
|
||||
if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
template <int ngpus>
|
||||
DINLINE void barrier_at_start(const RankSignals& sg, Signal* self_sg,
|
||||
int rank) {
|
||||
uint32_t flag = self_sg->_flag[blockIdx.x] + 1;
|
||||
if (threadIdx.x < ngpus) {
|
||||
// simultaneously write to the corresponding flag of all ranks.
|
||||
// Latency = 1 p2p write
|
||||
__scoped_atomic_store_n(&sg.signals[threadIdx.x]->start[blockIdx.x][rank],
|
||||
flag, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
|
||||
// wait until we got true from all ranks
|
||||
while (__scoped_atomic_load_n(&self_sg->start[blockIdx.x][threadIdx.x],
|
||||
__ATOMIC_RELAXED,
|
||||
__MEMORY_SCOPE_DEVICE) < flag);
|
||||
}
|
||||
__syncthreads();
|
||||
// use one thread to update flag
|
||||
if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag;
|
||||
}
|
||||
|
||||
template <int ngpus, bool final_sync = false>
|
||||
DINLINE void barrier_at_end(const RankSignals& sg, Signal* self_sg, int rank) {
|
||||
__syncthreads();
|
||||
uint32_t flag = self_sg->_flag[blockIdx.x] + 1;
|
||||
if (threadIdx.x < ngpus) {
|
||||
// simultaneously write to the corresponding flag of all ranks.
|
||||
// Latency = 1 p2p write
|
||||
__scoped_atomic_store_n(&sg.signals[threadIdx.x]->end[blockIdx.x][rank],
|
||||
flag,
|
||||
final_sync ? __ATOMIC_RELAXED : __ATOMIC_RELEASE,
|
||||
__MEMORY_SCOPE_SYSTEM);
|
||||
// wait until we got true from all ranks
|
||||
while (
|
||||
__scoped_atomic_load_n(&self_sg->end[blockIdx.x][threadIdx.x],
|
||||
final_sync ? __ATOMIC_RELAXED : __ATOMIC_ACQUIRE,
|
||||
__MEMORY_SCOPE_DEVICE) < flag);
|
||||
}
|
||||
if constexpr (!final_sync) __syncthreads();
|
||||
// use one thread to update flag
|
||||
if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
template <typename P, int ngpus, typename A>
|
||||
DINLINE P packed_reduce(const P* ptrs[], int idx) {
|
||||
A tmp = upcast(ptrs[0][idx]);
|
||||
#pragma unroll
|
||||
for (int i = 1; i < ngpus; i++) {
|
||||
packed_assign_add(tmp, upcast(ptrs[i][idx]));
|
||||
}
|
||||
return downcast<P>(tmp);
|
||||
}
|
||||
|
||||
template <typename T, int ngpus>
|
||||
__global__ void __launch_bounds__(512, 1)
|
||||
@@ -616,6 +325,21 @@ class CustomAllreduce {
|
||||
#undef KL
|
||||
}
|
||||
|
||||
void allgather(cudaStream_t stream, void* input, void* output, int size_bytes,
|
||||
int threads = 512, int block_limit = defaultBlockLimit);
|
||||
template <typename T>
|
||||
void mnnvl_lamport_allgather(cudaStream_t stream, T* input, T* output,
|
||||
void* local_buffer, void* multicast_buffer,
|
||||
uint32_t* epochs, int size_bytes,
|
||||
int stage_size_bytes);
|
||||
template <typename T>
|
||||
void reduce_scatter(cudaStream_t stream, T* input, T* output, int size,
|
||||
int threads = 512, int block_limit = defaultBlockLimit);
|
||||
template <typename T>
|
||||
void mnnvl_lamport_reduce_scatter(cudaStream_t stream, T* input, T* output,
|
||||
void* local_buffer, uint32_t* epochs,
|
||||
int size, int stage_size_bytes);
|
||||
|
||||
~CustomAllreduce() {
|
||||
for (auto [_, ptr] : ipc_handles_) {
|
||||
CUDACHECK(cudaIpcCloseMemHandle(ptr));
|
||||
@@ -625,8 +349,8 @@ class CustomAllreduce {
|
||||
|
||||
/**
|
||||
* To inspect PTX/SASS, copy paste this header file to compiler explorer and
|
||||
add a template instantiation:
|
||||
* add a template instantiation:
|
||||
* template void vllm::CustomAllreduce::allreduce<half>(cudaStream_t, half *,
|
||||
half *, int, int, int);
|
||||
*/
|
||||
} // namespace vllm
|
||||
* half *, int, int, int);
|
||||
*/
|
||||
} // namespace vllm
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
#pragma once
|
||||
|
||||
#include <cuda.h>
|
||||
#include <cuda_bf16.h>
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#if defined(USE_ROCM)
|
||||
typedef __hip_bfloat16 nv_bfloat16;
|
||||
#endif
|
||||
|
||||
#include <iostream>
|
||||
#include <array>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
|
||||
namespace vllm {
|
||||
constexpr int kMaxCustomCollectiveRanks = 16;
|
||||
|
||||
#define CUDACHECK(cmd) \
|
||||
do { \
|
||||
cudaError_t e = cmd; \
|
||||
if (e != cudaSuccess) { \
|
||||
printf("Failed: Cuda error %s:%d '%s'\n", __FILE__, __LINE__, \
|
||||
cudaGetErrorString(e)); \
|
||||
exit(EXIT_FAILURE); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
// Maximal number of blocks in allreduce kernel.
|
||||
constexpr int kMaxBlocks = 36;
|
||||
|
||||
// Default number of blocks in allreduce kernel.
|
||||
#ifndef USE_ROCM
|
||||
inline constexpr int defaultBlockLimit = 36;
|
||||
inline CUpointer_attribute rangeStartAddrAttr =
|
||||
CU_POINTER_ATTRIBUTE_RANGE_START_ADDR;
|
||||
#else
|
||||
inline constexpr int defaultBlockLimit = 16;
|
||||
inline hipPointer_attribute rangeStartAddrAttr =
|
||||
HIP_POINTER_ATTRIBUTE_RANGE_START_ADDR;
|
||||
#endif
|
||||
|
||||
// Counter may overflow, but it's fine since unsigned int overflow is
|
||||
// well-defined behavior.
|
||||
using FlagType = uint32_t;
|
||||
|
||||
// Two sets of peer counters are needed for two syncs: starting and ending an
|
||||
// operation. The reason is that it's possible for peer GPU block to arrive at
|
||||
// the second sync point while the current GPU block haven't passed the first
|
||||
// sync point. Thus, peer GPU may write counter+1 while current GPU is busy
|
||||
// waiting for counter. We use alternating counter array to avoid this
|
||||
// possibility.
|
||||
struct Signal {
|
||||
alignas(128) FlagType start[kMaxBlocks][kMaxCustomCollectiveRanks];
|
||||
alignas(128) FlagType end[kMaxBlocks][kMaxCustomCollectiveRanks];
|
||||
alignas(128) FlagType _flag[kMaxBlocks]; // incremental flags for each rank
|
||||
};
|
||||
|
||||
struct __align__(16) RankData {
|
||||
const void* ptrs[kMaxCustomCollectiveRanks];
|
||||
};
|
||||
|
||||
struct __align__(16) RankSignals {
|
||||
Signal* signals[kMaxCustomCollectiveRanks];
|
||||
};
|
||||
|
||||
// like std::array, but aligned
|
||||
template <typename T, int sz>
|
||||
struct __align__(alignof(T) * sz) array_t {
|
||||
T data[sz];
|
||||
using type = T;
|
||||
static constexpr int size = sz;
|
||||
};
|
||||
|
||||
// use packed type to maximize memory efficiency
|
||||
// goal: generate ld.128 and st.128 instructions
|
||||
template <typename T>
|
||||
struct packed_t {
|
||||
// the (P)acked type for load/store
|
||||
using P = array_t<T, 16 / sizeof(T)>;
|
||||
// the (A)ccumulator type for reduction
|
||||
using A = array_t<float, 16 / sizeof(T)>;
|
||||
};
|
||||
|
||||
#define DINLINE __device__ __forceinline__
|
||||
|
||||
// scalar cast functions
|
||||
DINLINE float upcast_s(half val) { return __half2float(val); }
|
||||
|
||||
template <typename T>
|
||||
DINLINE T downcast_s(float val);
|
||||
template <>
|
||||
DINLINE half downcast_s(float val) {
|
||||
return __float2half(val);
|
||||
}
|
||||
|
||||
// scalar add functions
|
||||
// for some reason when compiling with Pytorch, the + operator for half and
|
||||
// bfloat is disabled so we call the intrinsics directly
|
||||
DINLINE half& assign_add(half& a, half b) {
|
||||
a = __hadd(a, b);
|
||||
return a;
|
||||
}
|
||||
DINLINE float& assign_add(float& a, float b) { return a += b; }
|
||||
|
||||
#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
|
||||
DINLINE float upcast_s(nv_bfloat16 val) { return __bfloat162float(val); }
|
||||
template <>
|
||||
DINLINE nv_bfloat16 downcast_s(float val) {
|
||||
return __float2bfloat16(val);
|
||||
}
|
||||
DINLINE nv_bfloat16& assign_add(nv_bfloat16& a, nv_bfloat16 b) {
|
||||
a = __hadd(a, b);
|
||||
return a;
|
||||
}
|
||||
#endif
|
||||
|
||||
template <typename T, int N>
|
||||
DINLINE array_t<T, N>& packed_assign_add(array_t<T, N>& a, array_t<T, N> b) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < N; i++) {
|
||||
assign_add(a.data[i], b.data[i]);
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
template <typename T, int N>
|
||||
DINLINE array_t<float, N> upcast(array_t<T, N> val) {
|
||||
if constexpr (std::is_same<T, float>::value) {
|
||||
return val;
|
||||
} else {
|
||||
array_t<float, N> out;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < N; i++) {
|
||||
out.data[i] = upcast_s(val.data[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename O>
|
||||
DINLINE O downcast(array_t<float, O::size> val) {
|
||||
if constexpr (std::is_same<typename O::type, float>::value) {
|
||||
return val;
|
||||
} else {
|
||||
O out;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < O::size; i++) {
|
||||
out.data[i] = downcast_s<typename O::type>(val.data[i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
#if !defined(USE_ROCM)
|
||||
|
||||
static DINLINE void st_flag_release(FlagType* flag_addr, FlagType flag) {
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
|
||||
asm volatile("st.release.sys.global.u32 [%1], %0;" ::"r"(flag),
|
||||
"l"(flag_addr));
|
||||
#else
|
||||
asm volatile("membar.sys; st.volatile.global.u32 [%1], %0;" ::"r"(flag),
|
||||
"l"(flag_addr));
|
||||
#endif
|
||||
}
|
||||
|
||||
static DINLINE FlagType ld_flag_acquire(FlagType* flag_addr) {
|
||||
FlagType flag;
|
||||
#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 700
|
||||
asm volatile("ld.acquire.sys.global.u32 %0, [%1];"
|
||||
: "=r"(flag)
|
||||
: "l"(flag_addr));
|
||||
#else
|
||||
asm volatile("ld.volatile.global.u32 %0, [%1]; membar.gl;"
|
||||
: "=r"(flag)
|
||||
: "l"(flag_addr));
|
||||
#endif
|
||||
return flag;
|
||||
}
|
||||
|
||||
static DINLINE void st_flag_volatile(FlagType* flag_addr, FlagType flag) {
|
||||
asm volatile("st.volatile.global.u32 [%1], %0;" ::"r"(flag), "l"(flag_addr));
|
||||
}
|
||||
|
||||
static DINLINE FlagType ld_flag_volatile(FlagType* flag_addr) {
|
||||
FlagType flag;
|
||||
asm volatile("ld.volatile.global.u32 %0, [%1];"
|
||||
: "=r"(flag)
|
||||
: "l"(flag_addr));
|
||||
return flag;
|
||||
}
|
||||
|
||||
// This function is meant to be used as the first synchronization in the all
|
||||
// reduce kernel. Thus, it doesn't need to make any visibility guarantees for
|
||||
// prior memory accesses. Note: volatile writes will not be reordered against
|
||||
// other volatile writes.
|
||||
template <int ngpus>
|
||||
DINLINE void barrier_at_start(const RankSignals& sg, Signal* self_sg,
|
||||
int rank) {
|
||||
uint32_t flag = self_sg->_flag[blockIdx.x] + 1;
|
||||
if (threadIdx.x < ngpus) {
|
||||
auto peer_counter_ptr = &sg.signals[threadIdx.x]->start[blockIdx.x][rank];
|
||||
auto self_counter_ptr = &self_sg->start[blockIdx.x][threadIdx.x];
|
||||
// Write the expected counter value to peer and wait for correct value
|
||||
// from peer.
|
||||
st_flag_volatile(peer_counter_ptr, flag);
|
||||
while (ld_flag_volatile(self_counter_ptr) != flag);
|
||||
}
|
||||
__syncthreads();
|
||||
// use one thread to update flag
|
||||
if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag;
|
||||
}
|
||||
|
||||
template <int ngpus>
|
||||
DINLINE void barrier_at_start_release(const RankSignals& sg, Signal* self_sg,
|
||||
int rank) {
|
||||
__syncthreads();
|
||||
uint32_t flag = self_sg->_flag[blockIdx.x] + 1;
|
||||
if (threadIdx.x < ngpus) {
|
||||
auto peer_counter_ptr = &sg.signals[threadIdx.x]->start[blockIdx.x][rank];
|
||||
auto self_counter_ptr = &self_sg->start[blockIdx.x][threadIdx.x];
|
||||
st_flag_release(peer_counter_ptr, flag);
|
||||
while (ld_flag_acquire(self_counter_ptr) != flag);
|
||||
}
|
||||
__syncthreads();
|
||||
if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag;
|
||||
}
|
||||
|
||||
// This function is meant to be used as the second or the final
|
||||
// synchronization barrier in the all reduce kernel. If it's the final
|
||||
// synchronization barrier, we don't need to make any visibility guarantees
|
||||
// for prior memory accesses.
|
||||
template <int ngpus, bool final_sync = false>
|
||||
DINLINE void barrier_at_end(const RankSignals& sg, Signal* self_sg, int rank) {
|
||||
__syncthreads();
|
||||
uint32_t flag = self_sg->_flag[blockIdx.x] + 1;
|
||||
if (threadIdx.x < ngpus) {
|
||||
auto peer_counter_ptr = &sg.signals[threadIdx.x]->end[blockIdx.x][rank];
|
||||
auto self_counter_ptr = &self_sg->end[blockIdx.x][threadIdx.x];
|
||||
// Write the expected counter value to peer and wait for correct value from
|
||||
// peer.
|
||||
if constexpr (!final_sync) {
|
||||
st_flag_release(peer_counter_ptr, flag);
|
||||
while (ld_flag_acquire(self_counter_ptr) != flag);
|
||||
} else {
|
||||
st_flag_volatile(peer_counter_ptr, flag);
|
||||
while (ld_flag_volatile(self_counter_ptr) != flag);
|
||||
}
|
||||
}
|
||||
if constexpr (!final_sync) __syncthreads();
|
||||
|
||||
// use one thread to update flag
|
||||
if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag;
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
template <int ngpus>
|
||||
DINLINE void barrier_at_start(const RankSignals& sg, Signal* self_sg,
|
||||
int rank) {
|
||||
uint32_t flag = self_sg->_flag[blockIdx.x] + 1;
|
||||
if (threadIdx.x < ngpus) {
|
||||
// simultaneously write to the corresponding flag of all ranks.
|
||||
// Latency = 1 p2p write
|
||||
__scoped_atomic_store_n(&sg.signals[threadIdx.x]->start[blockIdx.x][rank],
|
||||
flag, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM);
|
||||
// wait until we got true from all ranks
|
||||
while (__scoped_atomic_load_n(&self_sg->start[blockIdx.x][threadIdx.x],
|
||||
__ATOMIC_RELAXED,
|
||||
__MEMORY_SCOPE_DEVICE) < flag);
|
||||
}
|
||||
__syncthreads();
|
||||
// use one thread to update flag
|
||||
if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag;
|
||||
}
|
||||
|
||||
template <int ngpus>
|
||||
DINLINE void barrier_at_start_release(const RankSignals& sg, Signal* self_sg,
|
||||
int rank) {
|
||||
__syncthreads();
|
||||
uint32_t flag = self_sg->_flag[blockIdx.x] + 1;
|
||||
if (threadIdx.x < ngpus) {
|
||||
__scoped_atomic_store_n(&sg.signals[threadIdx.x]->start[blockIdx.x][rank],
|
||||
flag, __ATOMIC_RELEASE, __MEMORY_SCOPE_SYSTEM);
|
||||
while (__scoped_atomic_load_n(&self_sg->start[blockIdx.x][threadIdx.x],
|
||||
__ATOMIC_ACQUIRE,
|
||||
__MEMORY_SCOPE_DEVICE) < flag);
|
||||
}
|
||||
__syncthreads();
|
||||
if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag;
|
||||
}
|
||||
|
||||
template <int ngpus, bool final_sync = false>
|
||||
DINLINE void barrier_at_end(const RankSignals& sg, Signal* self_sg, int rank) {
|
||||
__syncthreads();
|
||||
uint32_t flag = self_sg->_flag[blockIdx.x] + 1;
|
||||
if (threadIdx.x < ngpus) {
|
||||
// simultaneously write to the corresponding flag of all ranks.
|
||||
// Latency = 1 p2p write
|
||||
__scoped_atomic_store_n(&sg.signals[threadIdx.x]->end[blockIdx.x][rank],
|
||||
flag,
|
||||
final_sync ? __ATOMIC_RELAXED : __ATOMIC_RELEASE,
|
||||
__MEMORY_SCOPE_SYSTEM);
|
||||
// wait until we got true from all ranks
|
||||
while (
|
||||
__scoped_atomic_load_n(&self_sg->end[blockIdx.x][threadIdx.x],
|
||||
final_sync ? __ATOMIC_RELAXED : __ATOMIC_ACQUIRE,
|
||||
__MEMORY_SCOPE_DEVICE) < flag);
|
||||
}
|
||||
if constexpr (!final_sync) __syncthreads();
|
||||
// use one thread to update flag
|
||||
if (threadIdx.x == 0) self_sg->_flag[blockIdx.x] = flag;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
template <typename P, int ngpus, typename A>
|
||||
DINLINE P packed_reduce(const P* ptrs[], int idx) {
|
||||
A tmp = upcast(ptrs[0][idx]);
|
||||
#pragma unroll
|
||||
for (int i = 1; i < ngpus; i++) {
|
||||
packed_assign_add(tmp, upcast(ptrs[i][idx]));
|
||||
}
|
||||
return downcast<P>(tmp);
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
@@ -0,0 +1,17 @@
|
||||
#include "core/registration.h"
|
||||
#include "flash_kda.h"
|
||||
|
||||
TORCH_LIBRARY(_flashkda_C, m) {
|
||||
m.def("get_workspace_size(int T_total, int H, int N=1) -> int",
|
||||
&get_workspace_size);
|
||||
m.def(
|
||||
"fwd(Tensor q, Tensor k, Tensor v, Tensor g, Tensor beta, float scale, "
|
||||
"Tensor(a!) out, Tensor workspace, Tensor A_log, Tensor dt_bias, "
|
||||
"float lower_bound, "
|
||||
"Tensor? initial_state=None, Tensor(b!)? final_state=None, "
|
||||
"Tensor? cu_seqlens=None) -> ()");
|
||||
}
|
||||
|
||||
TORCH_LIBRARY_IMPL(_flashkda_C, CUDA, m) { m.impl("fwd", &fwd); }
|
||||
|
||||
REGISTER_EXTENSION(_flashkda_C)
|
||||
@@ -464,6 +464,66 @@ __global__ void swigluoai_and_mul_kernel(
|
||||
}
|
||||
}
|
||||
|
||||
// SITU (Kimi SituGLU) gated activation. Non-interleaved layout:
|
||||
// input = [gate(d), up(d)] per token.
|
||||
// gate_out = beta * tanh(gate / beta) * sigmoid(gate)
|
||||
// up_out = (linear_beta > 0) ? linear_beta * tanh(up / linear_beta) : up
|
||||
// out = gate_out * up_out
|
||||
// Compute is done in fp32 and written straight to `out` -- no intermediate
|
||||
// tensors and no full-tensor fp32 upcast (the pure-torch forward_native
|
||||
// allocated ~8 fp32 temporaries per call, which blows up MoE profiling).
|
||||
template <typename scalar_t>
|
||||
__global__ void situ_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, // [..., d]
|
||||
const scalar_t* __restrict__ input, // [..., 2, d]
|
||||
const int d, const float beta, const float linear_beta) {
|
||||
const int64_t row = blockIdx.x;
|
||||
const scalar_t* gate_ptr = input + row * 2 * d;
|
||||
const scalar_t* up_ptr = gate_ptr + d;
|
||||
scalar_t* out_ptr = out + row * d;
|
||||
const bool clamp_up = linear_beta > 0.0f;
|
||||
const float inv_beta = 1.0f / beta;
|
||||
const float inv_linear_beta = clamp_up ? 1.0f / linear_beta : 0.0f;
|
||||
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
|
||||
const float g = (float)VLLM_LDG(&gate_ptr[idx]);
|
||||
const float u = (float)VLLM_LDG(&up_ptr[idx]);
|
||||
const float gate_out = beta * tanhf(g * inv_beta) / (1.0f + expf(-g));
|
||||
const float up_out =
|
||||
clamp_up ? linear_beta * tanhf(u * inv_linear_beta) : u;
|
||||
out_ptr[idx] = (scalar_t)(gate_out * up_out);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename scalar_t>
|
||||
__global__ void masked_situ_and_mul_kernel(
|
||||
scalar_t* __restrict__ out, const scalar_t* __restrict__ input,
|
||||
const int* __restrict__ expert_num_tokens, const int max_num_tokens,
|
||||
const int d, const float beta, const float linear_beta) {
|
||||
const int expert = blockIdx.y;
|
||||
const int num_tokens = expert_num_tokens[expert];
|
||||
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (idx >= d || num_tokens == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const bool clamp_up = linear_beta > 0.0f;
|
||||
const float inv_beta = 1.0f / beta;
|
||||
const float inv_linear_beta = clamp_up ? 1.0f / linear_beta : 0.0f;
|
||||
const int64_t expert_row = static_cast<int64_t>(expert) * max_num_tokens;
|
||||
for (int token = 0; token < num_tokens; ++token) {
|
||||
const int64_t row = expert_row + token;
|
||||
const scalar_t* gate_ptr = input + row * 2 * d;
|
||||
const scalar_t* up_ptr = gate_ptr + d;
|
||||
scalar_t* out_ptr = out + row * d;
|
||||
const float g = (float)VLLM_LDG(&gate_ptr[idx]);
|
||||
const float u = (float)VLLM_LDG(&up_ptr[idx]);
|
||||
const float gate_out = beta * tanhf(g * inv_beta) / (1.0f + expf(-g));
|
||||
const float up_out =
|
||||
clamp_up ? linear_beta * tanhf(u * inv_linear_beta) : u;
|
||||
out_ptr[idx] = (scalar_t)(gate_out * up_out);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
|
||||
#define LAUNCH_ACTIVATION_GATE_KERNEL_WITH_PARAM(KERNEL, PACKED_KERNEL, PARAM) \
|
||||
@@ -553,6 +613,54 @@ void swigluoai_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
double alpha, double limit) {
|
||||
LAUNCH_SIGLUOAI_AND_MUL(vllm::swigluoai_and_mul, alpha, limit);
|
||||
}
|
||||
|
||||
// Kimi SITU gated activation. `linear_beta <= 0` means "unset" (up passed
|
||||
// through), matching SituAndMul(linear_beta=None) on the Python side.
|
||||
void situ_and_mul(torch::stable::Tensor& out, // [..., d]
|
||||
torch::stable::Tensor& input, // [..., 2 * d]
|
||||
double beta, double linear_beta) {
|
||||
int d = input.size(-1) / 2;
|
||||
int64_t num_tokens = input.numel() / input.size(-1);
|
||||
if (num_tokens == 0) {
|
||||
return;
|
||||
}
|
||||
dim3 grid(num_tokens);
|
||||
dim3 block(std::min(d, 1024));
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "situ_and_mul_kernel", [&] {
|
||||
vllm::situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),
|
||||
d, (float)beta, (float)linear_beta);
|
||||
});
|
||||
}
|
||||
|
||||
void masked_situ_and_mul(torch::stable::Tensor& out, // [E, T, d]
|
||||
torch::stable::Tensor& input, // [E, T, 2 * d]
|
||||
const torch::stable::Tensor& expert_num_tokens,
|
||||
double beta, double linear_beta) {
|
||||
int num_experts = input.size(0);
|
||||
int max_num_tokens = input.size(1);
|
||||
int d = input.size(2) / 2;
|
||||
if (num_experts == 0 || max_num_tokens == 0) {
|
||||
return;
|
||||
}
|
||||
constexpr int block_size = 256;
|
||||
dim3 grid((d + block_size - 1) / block_size, num_experts);
|
||||
dim3 block(block_size);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
input.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
input.scalar_type(), "masked_situ_and_mul_kernel", [&] {
|
||||
vllm::masked_situ_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
|
||||
out.mutable_data_ptr<scalar_t>(), input.const_data_ptr<scalar_t>(),
|
||||
expert_num_tokens.const_data_ptr<int>(), max_num_tokens, d,
|
||||
(float)beta, (float)linear_beta);
|
||||
});
|
||||
}
|
||||
namespace vllm {
|
||||
|
||||
// Element-wise activation kernel template.
|
||||
|
||||
@@ -21,7 +21,10 @@ __global__ void merge_attn_states_kernel(
|
||||
const float* prefix_lse, const scalar_t* suffix_output,
|
||||
const float* suffix_lse, const uint num_tokens, const uint num_heads,
|
||||
const uint head_size, const uint prefix_head_stride,
|
||||
const uint output_head_stride, const uint prefix_num_tokens,
|
||||
const uint output_head_stride, const uint prefix_lse_head_stride,
|
||||
const uint prefix_lse_token_stride, const uint suffix_lse_head_stride,
|
||||
const uint suffix_lse_token_stride, const uint output_lse_head_stride,
|
||||
const uint output_lse_token_stride, const uint prefix_num_tokens,
|
||||
const float* output_scale) {
|
||||
// Inputs always load 128-bit packs (pack_size elements of scalar_t).
|
||||
// Outputs store pack_size elements of output_t, which is smaller for FP8.
|
||||
@@ -84,15 +87,19 @@ __global__ void merge_attn_states_kernel(
|
||||
}
|
||||
}
|
||||
if (output_lse != nullptr && pack_idx == 0) {
|
||||
float s_lse = suffix_lse[head_idx * num_tokens + token_idx];
|
||||
output_lse[head_idx * num_tokens + token_idx] = s_lse;
|
||||
float s_lse = suffix_lse[head_idx * suffix_lse_head_stride +
|
||||
token_idx * suffix_lse_token_stride];
|
||||
output_lse[head_idx * output_lse_head_stride +
|
||||
token_idx * output_lse_token_stride] = s_lse;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// For tokens within prefix range, merge prefix and suffix
|
||||
float p_lse = prefix_lse[head_idx * num_tokens + token_idx];
|
||||
float s_lse = suffix_lse[head_idx * num_tokens + token_idx];
|
||||
float p_lse = prefix_lse[head_idx * prefix_lse_head_stride +
|
||||
token_idx * prefix_lse_token_stride];
|
||||
float s_lse = suffix_lse[head_idx * suffix_lse_head_stride +
|
||||
token_idx * suffix_lse_token_stride];
|
||||
p_lse = std::isinf(p_lse) ? -std::numeric_limits<float>::infinity() : p_lse;
|
||||
s_lse = std::isinf(s_lse) ? -std::numeric_limits<float>::infinity() : s_lse;
|
||||
|
||||
@@ -132,7 +139,8 @@ __global__ void merge_attn_states_kernel(
|
||||
}
|
||||
// We only need to write to output_lse once per head.
|
||||
if (output_lse != nullptr && pack_idx == 0) {
|
||||
output_lse[head_idx * num_tokens + token_idx] = max_lse;
|
||||
output_lse[head_idx * output_lse_head_stride +
|
||||
token_idx * output_lse_token_stride] = max_lse;
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -187,7 +195,8 @@ __global__ void merge_attn_states_kernel(
|
||||
// We only need to write to output_lse once per head.
|
||||
if (output_lse != nullptr && pack_idx == 0) {
|
||||
float out_lse = logf(out_se) + max_lse;
|
||||
output_lse[head_idx * num_tokens + token_idx] = out_lse;
|
||||
output_lse[head_idx * output_lse_head_stride +
|
||||
token_idx * output_lse_token_stride] = out_lse;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,6 +230,9 @@ __global__ void merge_attn_states_kernel(
|
||||
reinterpret_cast<scalar_t*>(suffix_output.data_ptr()), \
|
||||
reinterpret_cast<float*>(suffix_lse.data_ptr()), num_tokens, \
|
||||
num_heads, head_size, prefix_head_stride, output_head_stride, \
|
||||
prefix_lse_head_stride, prefix_lse_token_stride, \
|
||||
suffix_lse_head_stride, suffix_lse_token_stride, \
|
||||
output_lse_head_stride, output_lse_token_stride, \
|
||||
prefix_num_tokens, output_scale_ptr); \
|
||||
}
|
||||
|
||||
@@ -259,6 +271,19 @@ void merge_attn_states_launcher(
|
||||
const uint head_size = output.size(2);
|
||||
const uint prefix_head_stride = prefix_output.stride(1);
|
||||
const uint output_head_stride = output.stride(1);
|
||||
// lse tensors are [NUM_HEADS, NUM_TOKENS] but may be non-contiguous views
|
||||
// (e.g. a transpose of a backend's [NUM_TOKENS, NUM_HEADS] output), so index
|
||||
// them by their actual strides rather than assuming a contiguous layout.
|
||||
const uint prefix_lse_head_stride = prefix_lse.stride(0);
|
||||
const uint prefix_lse_token_stride = prefix_lse.stride(1);
|
||||
const uint suffix_lse_head_stride = suffix_lse.stride(0);
|
||||
const uint suffix_lse_token_stride = suffix_lse.stride(1);
|
||||
uint output_lse_head_stride = 0;
|
||||
uint output_lse_token_stride = 0;
|
||||
if (output_lse.has_value()) {
|
||||
output_lse_head_stride = output_lse.value().stride(0);
|
||||
output_lse_token_stride = output_lse.value().stride(1);
|
||||
}
|
||||
// Thread mapping is based on input BF16 pack_size
|
||||
const uint pack_size = 16 / sizeof(scalar_t);
|
||||
STD_TORCH_CHECK(head_size % pack_size == 0,
|
||||
|
||||
@@ -443,6 +443,55 @@ __global__ void concat_and_cache_mla_kernel(
|
||||
copy(k_pe, kv_cache, k_pe_stride, block_stride, pe_dim, kv_lora_rank);
|
||||
}
|
||||
|
||||
// Grouped variant of concat_and_cache_mla: inserts the context K/V for every
|
||||
// draft layer in a single launch. Grid is (num_tokens, num_layers); each layer
|
||||
// reads its own cache base pointer from kv_cache_ptrs (same pointer-array
|
||||
// pattern as copy_blocks_kernel). bf16 only, so it is a raw 16-bit copy with no
|
||||
// scaling or quantization; scalar_t is uint16_t for portability.
|
||||
template <typename scalar_t>
|
||||
__global__ void concat_and_cache_mla_grouped_kernel(
|
||||
const scalar_t* __restrict__ kv_c, // [num_layers, num_tokens,
|
||||
// kv_lora_rank]
|
||||
const scalar_t* __restrict__ k_pe, // [num_layers, num_tokens, pe_dim]
|
||||
const int64_t* __restrict__ kv_cache_ptrs, // [num_layers]
|
||||
const int64_t* __restrict__ slot_mapping, // [num_layers, num_tokens]
|
||||
const int64_t kv_c_layer_stride, const int64_t kv_c_token_stride,
|
||||
const int64_t k_pe_layer_stride, const int64_t k_pe_token_stride,
|
||||
const int64_t slot_layer_stride, const int64_t block_stride,
|
||||
const int64_t entry_stride, const int kv_lora_rank, const int pe_dim,
|
||||
const int block_size) {
|
||||
const int64_t token_idx = blockIdx.x;
|
||||
const int64_t layer_idx = blockIdx.y;
|
||||
const int64_t slot_idx =
|
||||
slot_mapping[layer_idx * slot_layer_stride + token_idx];
|
||||
// NOTE: slot_idx can be -1 if the token is padded
|
||||
if (slot_idx < 0) {
|
||||
return;
|
||||
}
|
||||
const int64_t block_idx = slot_idx / block_size;
|
||||
const int64_t block_offset = slot_idx % block_size;
|
||||
|
||||
scalar_t* __restrict__ kv_cache =
|
||||
reinterpret_cast<scalar_t*>(kv_cache_ptrs[layer_idx]);
|
||||
const scalar_t* __restrict__ kv_c_layer =
|
||||
kv_c + layer_idx * kv_c_layer_stride;
|
||||
const scalar_t* __restrict__ k_pe_layer =
|
||||
k_pe + layer_idx * k_pe_layer_stride;
|
||||
|
||||
auto copy = [&](const scalar_t* __restrict__ src, int64_t src_token_stride,
|
||||
int size, int offset) {
|
||||
for (int i = threadIdx.x; i < size; i += blockDim.x) {
|
||||
const int64_t src_idx = token_idx * src_token_stride + i;
|
||||
const int64_t dst_idx =
|
||||
block_idx * block_stride + block_offset * entry_stride + i + offset;
|
||||
kv_cache[dst_idx] = src[src_idx];
|
||||
}
|
||||
};
|
||||
|
||||
copy(kv_c_layer, kv_c_token_stride, kv_lora_rank, 0);
|
||||
copy(k_pe_layer, k_pe_token_stride, pe_dim, kv_lora_rank);
|
||||
}
|
||||
|
||||
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
|
||||
__global__ void concat_and_cache_ds_mla_kernel(
|
||||
const scalar_t* __restrict__ kv_c, // [num_tokens, kv_lora_rank]
|
||||
@@ -902,6 +951,53 @@ void concat_and_cache_mla(
|
||||
}
|
||||
}
|
||||
|
||||
void concat_and_cache_mla_grouped(
|
||||
torch::stable::Tensor& kv_c, // [num_layers, num_tokens, kv_lora_rank]
|
||||
torch::stable::Tensor& k_pe, // [num_layers, num_tokens, pe_dim]
|
||||
torch::stable::Tensor& kv_cache_ptrs, // [num_layers] int64, on device
|
||||
torch::stable::Tensor& slot_mapping, // [num_layers, num_tokens] int64
|
||||
int64_t block_size, int64_t block_stride, int64_t entry_stride) {
|
||||
int num_layers = kv_c.size(0);
|
||||
int num_tokens = kv_c.size(1);
|
||||
int kv_lora_rank = kv_c.size(2);
|
||||
int pe_dim = k_pe.size(2);
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
kv_c.scalar_type() == torch::headeronly::ScalarType::BFloat16 &&
|
||||
k_pe.scalar_type() == torch::headeronly::ScalarType::BFloat16,
|
||||
"concat_and_cache_mla_grouped only supports a bf16 KV cache; got kv_c=",
|
||||
kv_c.scalar_type(), ", k_pe=", k_pe.scalar_type());
|
||||
STD_TORCH_CHECK(
|
||||
kv_cache_ptrs.scalar_type() == torch::headeronly::ScalarType::Long,
|
||||
"kv_cache_ptrs must be int64");
|
||||
|
||||
if (num_tokens == 0 || num_layers == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const int64_t kv_c_layer_stride = kv_c.stride(0);
|
||||
const int64_t kv_c_token_stride = kv_c.stride(1);
|
||||
const int64_t k_pe_layer_stride = k_pe.stride(0);
|
||||
const int64_t k_pe_token_stride = k_pe.stride(1);
|
||||
const int64_t slot_layer_stride = slot_mapping.stride(0);
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
kv_c.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
|
||||
dim3 grid(num_tokens, num_layers);
|
||||
dim3 block(std::min(kv_lora_rank, 512));
|
||||
vllm::concat_and_cache_mla_grouped_kernel<uint16_t>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
reinterpret_cast<const uint16_t*>(kv_c.data_ptr()),
|
||||
reinterpret_cast<const uint16_t*>(k_pe.data_ptr()),
|
||||
kv_cache_ptrs.const_data_ptr<int64_t>(),
|
||||
slot_mapping.const_data_ptr<int64_t>(), kv_c_layer_stride,
|
||||
kv_c_token_stride, k_pe_layer_stride, k_pe_token_stride,
|
||||
slot_layer_stride, block_stride, entry_stride, kv_lora_rank, pe_dim,
|
||||
block_size);
|
||||
}
|
||||
|
||||
namespace vllm {
|
||||
|
||||
template <typename Tout, typename Tin, Fp8KVCacheDataType kv_dt>
|
||||
|
||||
@@ -0,0 +1,362 @@
|
||||
#include "torch_utils.h"
|
||||
|
||||
#include <torch/csrc/stable/macros.h>
|
||||
#include <torch/csrc/stable/accelerator.h>
|
||||
#include <torch/csrc/stable/tensor.h>
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include "custom_all_reduce.cuh"
|
||||
#include "custom_all_gather_reduce_scatter.cuh"
|
||||
|
||||
namespace vllm {
|
||||
|
||||
void CustomAllreduce::allgather(cudaStream_t stream, void* input, void* output,
|
||||
int size_bytes, int threads, int block_limit) {
|
||||
if (size_bytes % sizeof(CopyPack) != 0)
|
||||
throw std::runtime_error(
|
||||
"custom allgather requires input byte size to be a multiple of " +
|
||||
std::to_string(sizeof(CopyPack)));
|
||||
|
||||
auto ptrs = buffers_.at(input);
|
||||
int size_per_rank = size_bytes / sizeof(CopyPack);
|
||||
int total_size = size_per_rank * world_size_;
|
||||
int blocks = std::min(block_limit, (total_size + threads - 1) / threads);
|
||||
|
||||
#define AG_CASE(ngpus) \
|
||||
case ngpus: \
|
||||
cross_device_all_gather<ngpus><<<blocks, threads, 0, stream>>>( \
|
||||
ptrs, sg_, self_sg_, reinterpret_cast<CopyPack*>(output), rank_, \
|
||||
size_per_rank); \
|
||||
break;
|
||||
|
||||
switch (world_size_) {
|
||||
AG_CASE(2)
|
||||
AG_CASE(4)
|
||||
AG_CASE(6)
|
||||
AG_CASE(8)
|
||||
default:
|
||||
throw std::runtime_error(
|
||||
"custom allgather only supports num gpus in (2,4,6,8)");
|
||||
}
|
||||
#undef AG_CASE
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void CustomAllreduce::mnnvl_lamport_allgather(cudaStream_t stream, T* input,
|
||||
T* output, void* local_buffer,
|
||||
void* multicast_buffer,
|
||||
uint32_t* epochs, int size_bytes,
|
||||
int stage_size_bytes) {
|
||||
if (size_bytes % sizeof(typename packed_t<T>::P) != 0 ||
|
||||
stage_size_bytes % sizeof(typename packed_t<T>::P) != 0)
|
||||
throw std::runtime_error(
|
||||
"MNNVL Lamport allgather requires 16-byte aligned sizes");
|
||||
|
||||
auto ptrs = buffers_.at(local_buffer);
|
||||
int size_per_rank = size_bytes / sizeof(typename packed_t<T>::P);
|
||||
int stage_size = stage_size_bytes / sizeof(typename packed_t<T>::P);
|
||||
int blocks =
|
||||
(size_per_rank + kMnnvlLamportAgThreads - 1) / kMnnvlLamportAgThreads;
|
||||
|
||||
#if !defined(USE_ROCM) && CUDA_VERSION >= 12000
|
||||
cudaLaunchAttribute attributes[1]{};
|
||||
attributes[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attributes[0].val.programmaticStreamSerializationAllowed = 1;
|
||||
cudaLaunchConfig_t config{.gridDim = dim3(blocks),
|
||||
.blockDim = dim3(kMnnvlLamportAgThreads),
|
||||
.dynamicSmemBytes = 0,
|
||||
.stream = stream,
|
||||
.attrs = attributes,
|
||||
.numAttrs = 1};
|
||||
#define MNNVL_LAMPORT_AG_LAUNCH(ngpus) \
|
||||
CUDACHECK(cudaLaunchKernelEx(&config, &mnnvl_lamport_all_gather<T, ngpus>, \
|
||||
ptrs, input, output, \
|
||||
reinterpret_cast<T*>(multicast_buffer), \
|
||||
epochs, rank_, size_per_rank, stage_size))
|
||||
#else
|
||||
#define MNNVL_LAMPORT_AG_LAUNCH(ngpus) \
|
||||
mnnvl_lamport_all_gather<T, ngpus> \
|
||||
<<<blocks, kMnnvlLamportAgThreads, 0, stream>>>( \
|
||||
ptrs, input, output, reinterpret_cast<T*>(multicast_buffer), \
|
||||
epochs, rank_, size_per_rank, stage_size)
|
||||
#endif
|
||||
|
||||
#define MNNVL_LAMPORT_AG_CASE(ngpus) \
|
||||
case ngpus: \
|
||||
MNNVL_LAMPORT_AG_LAUNCH(ngpus); \
|
||||
break;
|
||||
|
||||
switch (world_size_) {
|
||||
MNNVL_LAMPORT_AG_CASE(2)
|
||||
MNNVL_LAMPORT_AG_CASE(4)
|
||||
MNNVL_LAMPORT_AG_CASE(6)
|
||||
MNNVL_LAMPORT_AG_CASE(8)
|
||||
MNNVL_LAMPORT_AG_CASE(16)
|
||||
default:
|
||||
throw std::runtime_error(
|
||||
"MNNVL Lamport allgather only supports num gpus in (2,4,6,8,16)");
|
||||
}
|
||||
#undef MNNVL_LAMPORT_AG_CASE
|
||||
#undef MNNVL_LAMPORT_AG_LAUNCH
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void CustomAllreduce::reduce_scatter(cudaStream_t stream, T* input, T* output,
|
||||
int size, int threads, int block_limit) {
|
||||
auto packed_size = packed_t<T>::P::size;
|
||||
if (size % (packed_size * world_size_) != 0)
|
||||
throw std::runtime_error(
|
||||
"custom reduce-scatter requires each output shard byte size to be "
|
||||
"a multiple of 16");
|
||||
|
||||
auto ptrs = buffers_.at(input);
|
||||
int size_per_rank = size / packed_size / world_size_;
|
||||
int blocks = std::min(block_limit, (size_per_rank + threads - 1) / threads);
|
||||
|
||||
#define RS_CASE(ngpus) \
|
||||
case ngpus: \
|
||||
cross_device_reduce_scatter<T, ngpus><<<blocks, threads, 0, stream>>>( \
|
||||
ptrs, sg_, self_sg_, output, rank_, size_per_rank); \
|
||||
break;
|
||||
|
||||
switch (world_size_) {
|
||||
RS_CASE(2)
|
||||
RS_CASE(4)
|
||||
RS_CASE(6)
|
||||
RS_CASE(8)
|
||||
default:
|
||||
throw std::runtime_error(
|
||||
"custom reduce-scatter only supports num gpus in (2,4,6,8)");
|
||||
}
|
||||
#undef RS_CASE
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void CustomAllreduce::mnnvl_lamport_reduce_scatter(cudaStream_t stream,
|
||||
T* input, T* output,
|
||||
void* local_buffer,
|
||||
uint32_t* epochs, int size,
|
||||
int stage_size_bytes) {
|
||||
auto packed_size = packed_t<T>::P::size;
|
||||
if (size % (packed_size * world_size_) != 0 ||
|
||||
stage_size_bytes % sizeof(typename packed_t<T>::P) != 0)
|
||||
throw std::runtime_error(
|
||||
"MNNVL Lamport reduce-scatter requires 16-byte aligned sizes");
|
||||
|
||||
auto ptrs = buffers_.at(local_buffer);
|
||||
int size_per_rank = size / packed_size / world_size_;
|
||||
int stage_size = stage_size_bytes / sizeof(typename packed_t<T>::P);
|
||||
int blocks_per_rank =
|
||||
(size_per_rank + kMnnvlLamportRsThreads - 1) / kMnnvlLamportRsThreads;
|
||||
int blocks = blocks_per_rank * world_size_;
|
||||
|
||||
#if !defined(USE_ROCM) && CUDA_VERSION >= 12000
|
||||
cudaLaunchAttribute attributes[1]{};
|
||||
attributes[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attributes[0].val.programmaticStreamSerializationAllowed = 1;
|
||||
cudaLaunchConfig_t config{.gridDim = dim3(blocks),
|
||||
.blockDim = dim3(kMnnvlLamportRsThreads),
|
||||
.dynamicSmemBytes = 0,
|
||||
.stream = stream,
|
||||
.attrs = attributes,
|
||||
.numAttrs = 1};
|
||||
#define MNNVL_LAMPORT_RS_LAUNCH(ngpus) \
|
||||
CUDACHECK(cudaLaunchKernelEx( \
|
||||
&config, &mnnvl_lamport_reduce_scatter_kernel<T, ngpus>, ptrs, input, \
|
||||
output, epochs, rank_, size_per_rank, stage_size))
|
||||
#else
|
||||
#define MNNVL_LAMPORT_RS_LAUNCH(ngpus) \
|
||||
mnnvl_lamport_reduce_scatter_kernel<T, ngpus> \
|
||||
<<<blocks, kMnnvlLamportRsThreads, 0, stream>>>( \
|
||||
ptrs, input, output, epochs, rank_, size_per_rank, stage_size)
|
||||
#endif
|
||||
|
||||
#define MNNVL_LAMPORT_RS_CASE(ngpus) \
|
||||
case ngpus: \
|
||||
MNNVL_LAMPORT_RS_LAUNCH(ngpus); \
|
||||
break;
|
||||
|
||||
switch (world_size_) {
|
||||
MNNVL_LAMPORT_RS_CASE(2)
|
||||
MNNVL_LAMPORT_RS_CASE(4)
|
||||
MNNVL_LAMPORT_RS_CASE(6)
|
||||
MNNVL_LAMPORT_RS_CASE(8)
|
||||
MNNVL_LAMPORT_RS_CASE(16)
|
||||
default:
|
||||
throw std::runtime_error(
|
||||
"MNNVL Lamport reduce-scatter only supports num gpus in "
|
||||
"(2,4,6,8,16)");
|
||||
}
|
||||
#undef MNNVL_LAMPORT_RS_CASE
|
||||
#undef MNNVL_LAMPORT_RS_LAUNCH
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
|
||||
using fptr_t = int64_t;
|
||||
static_assert(sizeof(void*) == sizeof(fptr_t));
|
||||
|
||||
bool _is_weak_contiguous(torch::stable::Tensor& t);
|
||||
|
||||
void custom_all_gather(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
torch::stable::Tensor& out, fptr_t _reg_buffer,
|
||||
int64_t reg_buffer_sz_bytes) {
|
||||
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
inp.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream(inp.get_device_index());
|
||||
|
||||
STD_TORCH_CHECK((inp.scalar_type()) == (out.scalar_type()));
|
||||
STD_TORCH_CHECK((inp.numel() * fa->world_size_) == (out.numel()));
|
||||
STD_TORCH_CHECK(_is_weak_contiguous(out));
|
||||
STD_TORCH_CHECK(_is_weak_contiguous(inp));
|
||||
auto input_size = inp.numel() * inp.element_size();
|
||||
auto reg_buffer = reinterpret_cast<void*>(_reg_buffer);
|
||||
STD_TORCH_CHECK(reg_buffer != nullptr);
|
||||
STD_TORCH_CHECK((input_size) <= (reg_buffer_sz_bytes));
|
||||
STD_CUDA_CHECK(cudaMemcpyAsync(reg_buffer, inp.const_data_ptr(), input_size,
|
||||
cudaMemcpyDeviceToDevice, stream));
|
||||
fa->allgather(stream, reg_buffer, out.mutable_data_ptr(), input_size);
|
||||
}
|
||||
|
||||
void mnnvl_lamport_all_gather(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
torch::stable::Tensor& out, fptr_t _local_buffer,
|
||||
fptr_t _multicast_buffer, fptr_t _epoch_buffer,
|
||||
int64_t stage_sz_bytes) {
|
||||
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
inp.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream(inp.get_device_index());
|
||||
|
||||
STD_TORCH_CHECK((inp.scalar_type()) == (out.scalar_type()));
|
||||
STD_TORCH_CHECK((inp.numel() * fa->world_size_) == (out.numel()));
|
||||
STD_TORCH_CHECK(_is_weak_contiguous(out));
|
||||
STD_TORCH_CHECK(_is_weak_contiguous(inp));
|
||||
auto input_size = inp.numel() * inp.element_size();
|
||||
STD_TORCH_CHECK((input_size * fa->world_size_) <= stage_sz_bytes);
|
||||
auto local_buffer = reinterpret_cast<void*>(_local_buffer);
|
||||
auto multicast_buffer = reinterpret_cast<void*>(_multicast_buffer);
|
||||
auto epochs = reinterpret_cast<uint32_t*>(_epoch_buffer);
|
||||
switch (out.scalar_type()) {
|
||||
case torch::headeronly::ScalarType::Float: {
|
||||
fa->mnnvl_lamport_allgather<float>(
|
||||
stream, reinterpret_cast<float*>(inp.mutable_data_ptr()),
|
||||
reinterpret_cast<float*>(out.mutable_data_ptr()), local_buffer,
|
||||
multicast_buffer, epochs, input_size, stage_sz_bytes);
|
||||
break;
|
||||
}
|
||||
case torch::headeronly::ScalarType::Half: {
|
||||
fa->mnnvl_lamport_allgather<half>(
|
||||
stream, reinterpret_cast<half*>(inp.mutable_data_ptr()),
|
||||
reinterpret_cast<half*>(out.mutable_data_ptr()), local_buffer,
|
||||
multicast_buffer, epochs, input_size, stage_sz_bytes);
|
||||
break;
|
||||
}
|
||||
#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
|
||||
case torch::headeronly::ScalarType::BFloat16: {
|
||||
fa->mnnvl_lamport_allgather<nv_bfloat16>(
|
||||
stream, reinterpret_cast<nv_bfloat16*>(inp.mutable_data_ptr()),
|
||||
reinterpret_cast<nv_bfloat16*>(out.mutable_data_ptr()), local_buffer,
|
||||
multicast_buffer, epochs, input_size, stage_sz_bytes);
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
default:
|
||||
throw std::runtime_error(
|
||||
"MNNVL Lamport allgather only supports float32, float16 and "
|
||||
"bfloat16");
|
||||
}
|
||||
}
|
||||
|
||||
void custom_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
torch::stable::Tensor& out, fptr_t _reg_buffer,
|
||||
int64_t reg_buffer_sz_bytes) {
|
||||
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
inp.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream(inp.get_device_index());
|
||||
|
||||
STD_TORCH_CHECK((inp.scalar_type()) == (out.scalar_type()));
|
||||
STD_TORCH_CHECK((out.numel() * fa->world_size_) == (inp.numel()));
|
||||
STD_TORCH_CHECK(_is_weak_contiguous(out));
|
||||
STD_TORCH_CHECK(_is_weak_contiguous(inp));
|
||||
auto input_size = inp.numel() * inp.element_size();
|
||||
auto reg_buffer = reinterpret_cast<void*>(_reg_buffer);
|
||||
STD_TORCH_CHECK(reg_buffer != nullptr);
|
||||
STD_TORCH_CHECK((input_size) <= (reg_buffer_sz_bytes));
|
||||
STD_CUDA_CHECK(cudaMemcpyAsync(reg_buffer, inp.const_data_ptr(), input_size,
|
||||
cudaMemcpyDeviceToDevice, stream));
|
||||
switch (out.scalar_type()) {
|
||||
case torch::headeronly::ScalarType::Float: {
|
||||
fa->reduce_scatter<float>(
|
||||
stream, reinterpret_cast<float*>(reg_buffer),
|
||||
reinterpret_cast<float*>(out.mutable_data_ptr()), inp.numel());
|
||||
break;
|
||||
}
|
||||
case torch::headeronly::ScalarType::Half: {
|
||||
fa->reduce_scatter<half>(stream, reinterpret_cast<half*>(reg_buffer),
|
||||
reinterpret_cast<half*>(out.mutable_data_ptr()),
|
||||
inp.numel());
|
||||
break;
|
||||
}
|
||||
#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
|
||||
case torch::headeronly::ScalarType::BFloat16: {
|
||||
fa->reduce_scatter<nv_bfloat16>(
|
||||
stream, reinterpret_cast<nv_bfloat16*>(reg_buffer),
|
||||
reinterpret_cast<nv_bfloat16*>(out.mutable_data_ptr()), inp.numel());
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
default:
|
||||
throw std::runtime_error(
|
||||
"custom reduce-scatter only supports float32, float16 and bfloat16");
|
||||
}
|
||||
}
|
||||
|
||||
void mnnvl_lamport_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
torch::stable::Tensor& out,
|
||||
fptr_t _local_buffer, fptr_t _epoch_buffer,
|
||||
int64_t stage_sz_bytes) {
|
||||
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
inp.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream(inp.get_device_index());
|
||||
|
||||
STD_TORCH_CHECK((inp.scalar_type()) == (out.scalar_type()));
|
||||
STD_TORCH_CHECK((out.numel() * fa->world_size_) == (inp.numel()));
|
||||
STD_TORCH_CHECK(_is_weak_contiguous(out));
|
||||
STD_TORCH_CHECK(_is_weak_contiguous(inp));
|
||||
auto input_size = inp.numel() * inp.element_size();
|
||||
STD_TORCH_CHECK(input_size <= stage_sz_bytes);
|
||||
auto local_buffer = reinterpret_cast<void*>(_local_buffer);
|
||||
auto epochs = reinterpret_cast<uint32_t*>(_epoch_buffer);
|
||||
switch (out.scalar_type()) {
|
||||
case torch::headeronly::ScalarType::Float: {
|
||||
fa->mnnvl_lamport_reduce_scatter<float>(
|
||||
stream, reinterpret_cast<float*>(inp.mutable_data_ptr()),
|
||||
reinterpret_cast<float*>(out.mutable_data_ptr()), local_buffer,
|
||||
epochs, inp.numel(), stage_sz_bytes);
|
||||
break;
|
||||
}
|
||||
case torch::headeronly::ScalarType::Half: {
|
||||
fa->mnnvl_lamport_reduce_scatter<half>(
|
||||
stream, reinterpret_cast<half*>(inp.mutable_data_ptr()),
|
||||
reinterpret_cast<half*>(out.mutable_data_ptr()), local_buffer, epochs,
|
||||
inp.numel(), stage_sz_bytes);
|
||||
break;
|
||||
}
|
||||
#if (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__))
|
||||
case torch::headeronly::ScalarType::BFloat16: {
|
||||
fa->mnnvl_lamport_reduce_scatter<nv_bfloat16>(
|
||||
stream, reinterpret_cast<nv_bfloat16*>(inp.mutable_data_ptr()),
|
||||
reinterpret_cast<nv_bfloat16*>(out.mutable_data_ptr()), local_buffer,
|
||||
epochs, inp.numel(), stage_sz_bytes);
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
default:
|
||||
throw std::runtime_error(
|
||||
"MNNVL Lamport reduce-scatter only supports float32, float16 and "
|
||||
"bfloat16");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
#include "ops.h"
|
||||
#include "core/registration.h"
|
||||
|
||||
#include <torch/csrc/stable/library.h>
|
||||
|
||||
STABLE_TORCH_LIBRARY_FRAGMENT(_C_custom_ar, custom_ag_rs) {
|
||||
custom_ag_rs.def(
|
||||
"custom_all_gather(int fa, Tensor inp, Tensor! out, int reg_buffer, "
|
||||
"int reg_buffer_sz_bytes) -> ()");
|
||||
custom_ag_rs.def(
|
||||
"mnnvl_lamport_all_gather(int fa, Tensor inp, Tensor! out, int "
|
||||
"local_buffer, int multicast_buffer, int epoch_buffer, int "
|
||||
"stage_sz_bytes) -> ()");
|
||||
custom_ag_rs.def(
|
||||
"custom_reduce_scatter(int fa, Tensor inp, Tensor! out, int reg_buffer, "
|
||||
"int reg_buffer_sz_bytes) -> ()");
|
||||
custom_ag_rs.def(
|
||||
"mnnvl_lamport_reduce_scatter(int fa, Tensor inp, Tensor! out, int "
|
||||
"local_buffer, int epoch_buffer, int stage_sz_bytes) -> ()");
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C_custom_ar, CUDA, custom_ag_rs) {
|
||||
custom_ag_rs.impl("custom_all_gather", TORCH_BOX(&custom_all_gather));
|
||||
custom_ag_rs.impl("mnnvl_lamport_all_gather",
|
||||
TORCH_BOX(&mnnvl_lamport_all_gather));
|
||||
custom_ag_rs.impl("custom_reduce_scatter", TORCH_BOX(&custom_reduce_scatter));
|
||||
custom_ag_rs.impl("mnnvl_lamport_reduce_scatter",
|
||||
TORCH_BOX(&mnnvl_lamport_reduce_scatter));
|
||||
}
|
||||
@@ -18,14 +18,14 @@ fptr_t init_custom_ar(const std::vector<fptr_t>& fake_ipc_ptrs,
|
||||
torch::stable::Tensor& rank_data, int64_t rank,
|
||||
bool fully_connected) {
|
||||
int world_size = fake_ipc_ptrs.size();
|
||||
if (world_size > 8)
|
||||
throw std::invalid_argument("world size > 8 is not supported");
|
||||
if (world_size > vllm::kMaxCustomCollectiveRanks)
|
||||
throw std::invalid_argument("world size > 16 is not supported");
|
||||
if (world_size % 2 != 0)
|
||||
throw std::invalid_argument("Odd num gpus is not supported for now");
|
||||
if (rank < 0 || rank >= world_size)
|
||||
throw std::invalid_argument("invalid rank passed in");
|
||||
|
||||
vllm::Signal* ipc_ptrs[8];
|
||||
vllm::Signal* ipc_ptrs[vllm::kMaxCustomCollectiveRanks];
|
||||
for (int i = 0; i < world_size; i++) {
|
||||
ipc_ptrs[i] = reinterpret_cast<vllm::Signal*>(fake_ipc_ptrs[i]);
|
||||
}
|
||||
@@ -124,7 +124,7 @@ int64_t meta_size() { return sizeof(vllm::Signal); }
|
||||
void register_buffer(fptr_t _fa, const std::vector<fptr_t>& fake_ipc_ptrs) {
|
||||
auto fa = reinterpret_cast<vllm::CustomAllreduce*>(_fa);
|
||||
STD_TORCH_CHECK(fake_ipc_ptrs.size() == fa->world_size_);
|
||||
void* ipc_ptrs[8];
|
||||
void* ipc_ptrs[vllm::kMaxCustomCollectiveRanks];
|
||||
for (int i = 0; i < fake_ipc_ptrs.size(); i++) {
|
||||
ipc_ptrs[i] = reinterpret_cast<void*>(fake_ipc_ptrs[i]);
|
||||
}
|
||||
|
||||
@@ -647,17 +647,17 @@ __global__ __launch_bounds__(256, 1) void fused_a_gemm_kernel(
|
||||
#endif
|
||||
}
|
||||
|
||||
template <typename T, int kHdIn, int kHdOut, int kTileN>
|
||||
template <typename T, int kHdIn, int kHdOut, int kTileN, int kTileK = 256>
|
||||
void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
|
||||
cudaStream_t const stream) {
|
||||
constexpr int gemm_m = kHdOut; // 2112
|
||||
int const gemm_n = num_tokens; // 1-16
|
||||
constexpr int gemm_k = kHdIn; // 7168
|
||||
cudaStream_t const stream, bool enable_pdl) {
|
||||
constexpr int gemm_m = kHdOut;
|
||||
int const gemm_n = num_tokens;
|
||||
constexpr int gemm_k = kHdIn;
|
||||
constexpr int batch_size = 1;
|
||||
std::swap(mat_a, mat_b);
|
||||
constexpr int tile_m = 16;
|
||||
constexpr int tile_n = kTileN; // 8 or 16
|
||||
constexpr int tile_k = std::max(256, 1024 / tile_n); // 256
|
||||
constexpr int tile_n = kTileN;
|
||||
constexpr int tile_k = kTileK;
|
||||
constexpr int max_stage_cnt =
|
||||
1024 * 192 / ((tile_m + tile_n) * tile_k * sizeof(bf16_t));
|
||||
constexpr int k_iter_cnt = gemm_k / tile_k;
|
||||
@@ -679,7 +679,8 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
|
||||
config.stream = stream;
|
||||
cudaLaunchAttribute attrs[1];
|
||||
attrs[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = getEnvEnablePDL();
|
||||
attrs[0].val.programmaticStreamSerializationAllowed =
|
||||
enable_pdl || getEnvEnablePDL();
|
||||
config.numAttrs = 1;
|
||||
config.attrs = attrs;
|
||||
if (smem_bytes >= (48 * 1024)) {
|
||||
@@ -694,36 +695,50 @@ void invokeFusedAGemm(T* output, T const* mat_a, T const* mat_b, int num_tokens,
|
||||
output, mat_a, mat_b, gemm_n);
|
||||
}
|
||||
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 8>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
|
||||
template void invokeFusedAGemm<__nv_bfloat16, 7168, 2112, 16>(
|
||||
__nv_bfloat16*, __nv_bfloat16 const*, __nv_bfloat16 const*, int num_tokens,
|
||||
cudaStream_t);
|
||||
template <typename T, int kHdIn, int kHdOut, int kTileK = 256>
|
||||
void invokeFusedAGemmForTokens(T* output, T const* mat_a, T const* mat_b,
|
||||
int num_tokens, cudaStream_t const stream,
|
||||
bool enable_pdl) {
|
||||
if (num_tokens <= 8) {
|
||||
invokeFusedAGemm<T, kHdIn, kHdOut, 8, kTileK>(
|
||||
output, mat_a, mat_b, num_tokens, stream, enable_pdl);
|
||||
} else {
|
||||
invokeFusedAGemm<T, kHdIn, kHdOut, 16, kTileK>(
|
||||
output, mat_a, mat_b, num_tokens, stream, enable_pdl);
|
||||
}
|
||||
}
|
||||
|
||||
void dsv3_fused_a_gemm(torch::stable::Tensor& output,
|
||||
torch::stable::Tensor const& mat_a,
|
||||
torch::stable::Tensor const& mat_b) {
|
||||
torch::stable::Tensor const& mat_b, bool enable_pdl) {
|
||||
STD_TORCH_CHECK(mat_a.dim() == 2 && mat_b.dim() == 2 && output.dim() == 2);
|
||||
int const num_tokens = mat_a.size(0);
|
||||
int const hd_in = mat_a.size(1);
|
||||
int const hd_out = mat_b.size(1);
|
||||
|
||||
constexpr int kHdIn = 7168;
|
||||
constexpr int kHdOut = 2112;
|
||||
STD_TORCH_CHECK(num_tokens >= 1 && num_tokens <= 16,
|
||||
"required 1 <= mat_a.shape[0] <= 16");
|
||||
STD_TORCH_CHECK(hd_in == kHdIn, "required mat_a.shape[1] == 7168");
|
||||
STD_TORCH_CHECK(hd_out == kHdOut, "required mat_b.shape[1] == 2112");
|
||||
STD_TORCH_CHECK(output.size(0) == num_tokens,
|
||||
"required output.shape[0] == mat_a.shape[0]");
|
||||
STD_TORCH_CHECK(output.size(1) == hd_out,
|
||||
"required output.shape[1] == mat_b.shape[1]");
|
||||
STD_TORCH_CHECK(mat_b.size(0) == hd_in,
|
||||
"required mat_b.shape[0] == mat_a.shape[1]");
|
||||
|
||||
STD_TORCH_CHECK(mat_a.stride(1) == 1, "mat_a must be a row major tensor");
|
||||
STD_TORCH_CHECK(output.stride(1) == 1, "output must be a row major tensor");
|
||||
STD_TORCH_CHECK(mat_b.stride(0) == 1, "mat_b must be a column major tensor");
|
||||
STD_TORCH_CHECK(mat_a.get_device_index() == mat_b.get_device_index() &&
|
||||
mat_a.get_device_index() == output.get_device_index(),
|
||||
"mat_a, mat_b, and output must be on the same device");
|
||||
|
||||
// The kernels index global memory with raw pointers and packed strides, so
|
||||
// reject any padded or transposed view rather than reading out of bounds.
|
||||
STD_TORCH_CHECK(
|
||||
mat_a.stride(0) == hd_in && mat_a.stride(1) == 1,
|
||||
"mat_a must be a packed row-major [num_tokens, hd_in] tensor");
|
||||
STD_TORCH_CHECK(
|
||||
output.stride(0) == hd_out && output.stride(1) == 1,
|
||||
"output must be a packed row-major [num_tokens, hd_out] tensor");
|
||||
STD_TORCH_CHECK(mat_b.stride(0) == 1 && mat_b.stride(1) == hd_in,
|
||||
"mat_b must be a packed column-major [hd_in, hd_out] tensor");
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
mat_a.scalar_type() == torch::headeronly::ScalarType::BFloat16 &&
|
||||
@@ -738,19 +753,85 @@ void dsv3_fused_a_gemm(torch::stable::Tensor& output,
|
||||
STD_TORCH_CHECK(getSMVersion() >= 90, "required CUDA ARCH >= SM_90");
|
||||
|
||||
auto stream = get_current_cuda_stream(mat_a.get_device_index());
|
||||
if (num_tokens <= 8) {
|
||||
invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 8>(
|
||||
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens,
|
||||
stream);
|
||||
} else {
|
||||
invokeFusedAGemm<__nv_bfloat16, kHdIn, kHdOut, 16>(
|
||||
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr()),
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr()), num_tokens,
|
||||
stream);
|
||||
auto* output_ptr =
|
||||
reinterpret_cast<__nv_bfloat16*>(output.mutable_data_ptr());
|
||||
auto const* mat_a_ptr =
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_a.data_ptr());
|
||||
auto const* mat_b_ptr =
|
||||
reinterpret_cast<__nv_bfloat16 const*>(mat_b.data_ptr());
|
||||
|
||||
#define DISPATCH_DSV3_SHAPE(HD_IN, HD_OUT) \
|
||||
if (hd_in == HD_IN && hd_out == HD_OUT) { \
|
||||
invokeFusedAGemmForTokens<__nv_bfloat16, HD_IN, HD_OUT>( \
|
||||
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl); \
|
||||
return; \
|
||||
}
|
||||
|
||||
// Shapes the Kimi-K3 selector routes to dsv3_fused_a (see the dsv3 winners
|
||||
// in KIMI_K3_PROJECTIONS) plus the DeepSeek V2/V3 QKV A-projection.
|
||||
DISPATCH_DSV3_SHAPE(7168, 1536)
|
||||
DISPATCH_DSV3_SHAPE(7168, 2112)
|
||||
DISPATCH_DSV3_SHAPE(1536, 2304)
|
||||
DISPATCH_DSV3_SHAPE(1536, 4608)
|
||||
DISPATCH_DSV3_SHAPE(7168, 3584)
|
||||
DISPATCH_DSV3_SHAPE(768, 7168)
|
||||
// TP16 dsv3 winners, as (hd_in=K, hd_out=N). TP16 dense down_proj is absent
|
||||
// because hd_in=2112 is not a multiple of any supported tile_k.
|
||||
DISPATCH_DSV3_SHAPE(1536, 1152)
|
||||
DISPATCH_DSV3_SHAPE(7168, 768)
|
||||
DISPATCH_DSV3_SHAPE(7168, 3216)
|
||||
DISPATCH_DSV3_SHAPE(7168, 4224)
|
||||
|
||||
#ifdef VLLM_K3_BENCH_SHAPES
|
||||
// The selector routes these shapes to CuTe or the default GEMM, so they are
|
||||
// never reached in production. They are compiled only for offline
|
||||
// DSV3-vs-CuTe benchmarking.
|
||||
DISPATCH_DSV3_SHAPE(7168, 6288)
|
||||
DISPATCH_DSV3_SHAPE(1536, 7168)
|
||||
DISPATCH_DSV3_SHAPE(3584, 7168)
|
||||
DISPATCH_DSV3_SHAPE(7168, 8448)
|
||||
DISPATCH_DSV3_SHAPE(7168, 20480)
|
||||
DISPATCH_DSV3_SHAPE(7168, 3072)
|
||||
DISPATCH_DSV3_SHAPE(7168, 12448)
|
||||
DISPATCH_DSV3_SHAPE(3072, 7168)
|
||||
DISPATCH_DSV3_SHAPE(8448, 7168)
|
||||
DISPATCH_DSV3_SHAPE(7168, 16896)
|
||||
DISPATCH_DSV3_SHAPE(7168, 40960)
|
||||
#endif
|
||||
|
||||
#undef DISPATCH_DSV3_SHAPE
|
||||
|
||||
if (hd_in == 128 && hd_out == 1536) {
|
||||
invokeFusedAGemmForTokens<__nv_bfloat16, 128, 1536, 128>(
|
||||
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
|
||||
return;
|
||||
}
|
||||
if (hd_in == 128 && hd_out == 3072) {
|
||||
invokeFusedAGemmForTokens<__nv_bfloat16, 128, 3072, 128>(
|
||||
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
|
||||
return;
|
||||
}
|
||||
// TP16 KDA f_b_proj and shared_expert down_proj. Neither hd_in is a multiple
|
||||
// of 256, so both need the 128 tile_k.
|
||||
if (hd_in == 128 && hd_out == 768) {
|
||||
invokeFusedAGemmForTokens<__nv_bfloat16, 128, 768, 128>(
|
||||
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
|
||||
return;
|
||||
}
|
||||
if (hd_in == 384 && hd_out == 7168) {
|
||||
invokeFusedAGemmForTokens<__nv_bfloat16, 384, 7168, 128>(
|
||||
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
|
||||
return;
|
||||
}
|
||||
#ifdef VLLM_K3_BENCH_SHAPES
|
||||
if (hd_in == 4224 && hd_out == 7168) {
|
||||
invokeFusedAGemmForTokens<__nv_bfloat16, 4224, 7168, 128>(
|
||||
output_ptr, mat_a_ptr, mat_b_ptr, num_tokens, stream, enable_pdl);
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
STD_TORCH_CHECK(false, "unsupported DSV3 fused-A GEMM shape");
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, m) {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -25,6 +25,7 @@
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <tuple>
|
||||
#include <cuda_fp16.h>
|
||||
#include <cuda_bf16.h>
|
||||
@@ -448,7 +449,8 @@ enum ScoringFunc {
|
||||
SCORING_SIGMOID = 1 // apply sigmoid
|
||||
};
|
||||
|
||||
// Efficient sigmoid approximation from TensorRT-LLM
|
||||
// Adapted from
|
||||
// https://github.com/NVIDIA/TensorRT-LLM/blob/v1.3.0rc2/cpp/tensorrt_llm/kernels/noAuxTcKernels.cu
|
||||
__device__ inline float sigmoid_accurate(float x) {
|
||||
return 0.5f * tanhf(0.5f * x) + 0.5f;
|
||||
}
|
||||
@@ -890,6 +892,434 @@ __global__ void grouped_topk_fused_small_expert_count_kernel(
|
||||
#endif
|
||||
}
|
||||
|
||||
// Adapted from
|
||||
// https://github.com/flashinfer-ai/flashinfer/blob/06400d062a2d51564bbe781f6f811d0b75ca593e/include/flashinfer/trtllm/fused_moe/RoutingKernelTopK.cuh
|
||||
namespace single_group_topk {
|
||||
namespace detail {
|
||||
|
||||
static constexpr int BlockDim = 256;
|
||||
static constexpr uint32_t FullWarpMask = 0xffffffffU;
|
||||
static constexpr float InvalidScore = -INFINITY;
|
||||
|
||||
// TopK-only tuning: use wider workers and keep these tiers on the block path.
|
||||
template <int MaxNumExperts, int MaxNumTopExperts>
|
||||
static constexpr bool UseTunedBlockPath =
|
||||
MaxNumTopExperts == 16 && (MaxNumExperts == 896 || MaxNumExperts == 1024);
|
||||
|
||||
template <typename T, typename BiasT, ScoringFunc SF>
|
||||
__device__ __forceinline__ void preprocess_score(T input, BiasT correction_bias,
|
||||
float& unbiased_score,
|
||||
float& selection_score) {
|
||||
unbiased_score = 0.0F;
|
||||
selection_score = InvalidScore;
|
||||
float const input_float = cuda_cast<float, T>(input);
|
||||
float const bias = cuda_cast<float, BiasT>(correction_bias);
|
||||
if (!is_finite(input_float) || !is_finite(bias)) {
|
||||
return;
|
||||
}
|
||||
|
||||
float const unbiased = apply_scoring<SF>(input_float);
|
||||
float const biased = unbiased + bias;
|
||||
if constexpr (SF == SCORING_NONE) {
|
||||
if (!is_finite(biased)) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
unbiased_score = unbiased;
|
||||
selection_score = biased == 0.0F ? 0.0F : biased;
|
||||
}
|
||||
|
||||
template <typename IdxT>
|
||||
__device__ __forceinline__ void write_outputs(
|
||||
cg::thread_block_tile<WARP_SIZE> const& warp, float lane_selection_score,
|
||||
float lane_unbiased, int32_t lane_expert, int32_t lane, int32_t token,
|
||||
int32_t topk, float* topk_values, IdxT* topk_indices, bool renormalize,
|
||||
float routed_scaling_factor) {
|
||||
bool const finite_selection =
|
||||
lane < topk && lane_selection_score != InvalidScore;
|
||||
lane_unbiased = finite_selection ? lane_unbiased : 0.0F;
|
||||
unsigned const finite_mask = __ballot_sync(FullWarpMask, finite_selection);
|
||||
float const sum = cg::reduce(warp, lane_unbiased, cg::plus<float>{});
|
||||
|
||||
if (lane < topk) {
|
||||
float output = 0.0F;
|
||||
if (finite_mask == 0) {
|
||||
if (renormalize) {
|
||||
output = 1.0F / static_cast<float>(topk);
|
||||
}
|
||||
} else if (finite_selection) {
|
||||
float scale = routed_scaling_factor;
|
||||
if (renormalize) {
|
||||
scale /= sum + 1e-20F;
|
||||
}
|
||||
output = lane_unbiased * scale;
|
||||
}
|
||||
|
||||
int64_t const output_index = int64_t{token} * topk + lane;
|
||||
topk_values[output_index] = output;
|
||||
topk_indices[output_index] = static_cast<IdxT>(lane_expert);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename BiasT, typename IdxT, ScoringFunc SF,
|
||||
int MaxNumExperts, int MaxNumTopExperts>
|
||||
__global__ void __launch_bounds__(BlockDim)
|
||||
single_group_topk_block_kernel(T const* scores, float* topk_values,
|
||||
IdxT* topk_indices, BiasT const* bias,
|
||||
int64_t num_experts, int64_t topk,
|
||||
bool renormalize,
|
||||
float routed_scaling_factor,
|
||||
bool enable_pdl) {
|
||||
static constexpr int NumChunks = (MaxNumExperts + WARP_SIZE - 1) / WARP_SIZE;
|
||||
static constexpr int WorkerValuesPerLane =
|
||||
UseTunedBlockPath<MaxNumExperts, MaxNumTopExperts> ? 8 : 4;
|
||||
static constexpr int ExpertsPerWorkerWarp = WorkerValuesPerLane * WARP_SIZE;
|
||||
using LaneOwnedRange =
|
||||
reduce_topk::HighExpertLaneOwnedTopKRange<MaxNumExperts,
|
||||
MaxNumTopExperts>;
|
||||
static constexpr int NumWorkerWarps =
|
||||
(MaxNumExperts + ExpertsPerWorkerWarp - 1) / ExpertsPerWorkerWarp;
|
||||
static constexpr int NumIntermediate = NumWorkerWarps * MaxNumTopExperts;
|
||||
static constexpr int MergeValuesPerLane =
|
||||
(NumIntermediate + WARP_SIZE - 1) / WARP_SIZE;
|
||||
static constexpr bool LaneOwnedResourcesFit =
|
||||
NumWorkerWarps <= BlockDim / WARP_SIZE && MergeValuesPerLane <= 64;
|
||||
static constexpr bool UseHierarchicalLaneTopK =
|
||||
LaneOwnedRange::kEnabled && LaneOwnedResourcesFit;
|
||||
|
||||
static_assert(NumChunks <= 64);
|
||||
static_assert(MaxNumTopExperts <= WARP_SIZE);
|
||||
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
if (enable_pdl) {
|
||||
cudaGridDependencySynchronize();
|
||||
}
|
||||
#endif
|
||||
|
||||
__shared__ float __attribute((aligned(128))) biased_scores[MaxNumExperts];
|
||||
__shared__ float __attribute((aligned(128))) unbiased_scores[MaxNumExperts];
|
||||
|
||||
int32_t const token = static_cast<int32_t>(blockIdx.x);
|
||||
int32_t const lane = static_cast<int32_t>(threadIdx.x) % WARP_SIZE;
|
||||
int32_t const warp_id = static_cast<int32_t>(threadIdx.x) / WARP_SIZE;
|
||||
int32_t const num_experts_i32 = static_cast<int32_t>(num_experts);
|
||||
int32_t const topk_i32 = static_cast<int32_t>(topk);
|
||||
T const* token_scores = scores + int64_t{token} * num_experts;
|
||||
|
||||
for (int32_t expert = static_cast<int32_t>(threadIdx.x);
|
||||
expert < num_experts_i32; expert += BlockDim) {
|
||||
preprocess_score<T, BiasT, SF>(token_scores[expert], bias[expert],
|
||||
unbiased_scores[expert],
|
||||
biased_scores[expert]);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
auto warp = cg::tiled_partition<WARP_SIZE>(cg::this_thread_block());
|
||||
|
||||
if constexpr (UseHierarchicalLaneTopK) {
|
||||
__shared__ float
|
||||
__attribute((aligned(128))) intermediate_scores[NumIntermediate];
|
||||
__shared__ int32_t
|
||||
__attribute((aligned(128))) intermediate_indices[NumIntermediate];
|
||||
|
||||
if (warp_id < NumWorkerWarps) {
|
||||
float local_scores[WorkerValuesPerLane];
|
||||
int32_t local_indices[WorkerValuesPerLane];
|
||||
#pragma unroll
|
||||
for (int index = 0; index < WorkerValuesPerLane; ++index) {
|
||||
int32_t const expert =
|
||||
warp_id * ExpertsPerWorkerWarp + index * WARP_SIZE + lane;
|
||||
local_scores[index] =
|
||||
expert < num_experts_i32 ? biased_scores[expert] : InvalidScore;
|
||||
local_indices[index] = expert;
|
||||
}
|
||||
|
||||
float lane_score;
|
||||
int32_t lane_expert;
|
||||
reduce_topk::reduceTopKForLane<MaxNumTopExperts>(
|
||||
warp, lane_score, lane_expert, local_scores, local_indices,
|
||||
InvalidScore, lane);
|
||||
if (lane < MaxNumTopExperts) {
|
||||
int32_t const intermediate = warp_id * MaxNumTopExperts + lane;
|
||||
bool const active = lane < topk_i32;
|
||||
intermediate_scores[intermediate] = active ? lane_score : InvalidScore;
|
||||
intermediate_indices[intermediate] =
|
||||
active ? lane_expert : MaxNumExperts;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
if (warp_id != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
float merge_scores[MergeValuesPerLane];
|
||||
int32_t merge_indices[MergeValuesPerLane];
|
||||
#pragma unroll
|
||||
for (int index = 0; index < MergeValuesPerLane; ++index) {
|
||||
int32_t const intermediate = index * WARP_SIZE + lane;
|
||||
bool const active = intermediate < NumIntermediate;
|
||||
merge_scores[index] =
|
||||
active ? intermediate_scores[intermediate] : InvalidScore;
|
||||
merge_indices[index] =
|
||||
active ? intermediate_indices[intermediate] : MaxNumExperts;
|
||||
}
|
||||
|
||||
float lane_score;
|
||||
int32_t lane_expert;
|
||||
reduce_topk::reduceTopKForLane<MaxNumTopExperts>(
|
||||
warp, lane_score, lane_expert, merge_scores, merge_indices,
|
||||
InvalidScore, lane);
|
||||
float const lane_unbiased =
|
||||
lane < topk_i32 && lane_expert >= 0 && lane_expert < num_experts_i32
|
||||
? unbiased_scores[lane_expert]
|
||||
: 0.0F;
|
||||
write_outputs(warp, lane_score, lane_unbiased, lane_expert, lane, token,
|
||||
topk_i32, topk_values, topk_indices, renormalize,
|
||||
routed_scaling_factor);
|
||||
} else {
|
||||
if (warp_id != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
float local_scores[NumChunks];
|
||||
int32_t local_indices[NumChunks];
|
||||
#pragma unroll
|
||||
for (int index = 0; index < NumChunks; ++index) {
|
||||
int32_t const expert = index * WARP_SIZE + lane;
|
||||
local_scores[index] =
|
||||
expert < num_experts_i32 ? biased_scores[expert] : InvalidScore;
|
||||
local_indices[index] = expert;
|
||||
}
|
||||
|
||||
float top_scores[MaxNumTopExperts];
|
||||
int32_t top_experts[MaxNumTopExperts];
|
||||
reduce_topk::reduceTopK(warp, top_scores, top_experts, local_scores,
|
||||
local_indices, InvalidScore, topk_i32);
|
||||
float const lane_score = lane < topk_i32 ? top_scores[lane] : InvalidScore;
|
||||
int32_t const lane_expert = lane < topk_i32 ? top_experts[lane] : -1;
|
||||
float const lane_unbiased =
|
||||
lane < topk_i32 && lane_expert >= 0 && lane_expert < num_experts_i32
|
||||
? unbiased_scores[lane_expert]
|
||||
: 0.0F;
|
||||
write_outputs(warp, lane_score, lane_unbiased, lane_expert, lane, token,
|
||||
topk_i32, topk_values, topk_indices, renormalize,
|
||||
routed_scaling_factor);
|
||||
}
|
||||
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
if (enable_pdl) {
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
template <int MaxNumExperts>
|
||||
struct WarpTopKLaunchConfig {
|
||||
static constexpr int DefaultBlockDim =
|
||||
MaxNumExperts <= 1024 ? MaxNumExperts : 1024;
|
||||
static constexpr int BlockDim = DefaultBlockDim > 256 ? 256 : DefaultBlockDim;
|
||||
static constexpr int NumWarps = BlockDim / WARP_SIZE;
|
||||
static constexpr int MaxBlockScale =
|
||||
(DefaultBlockDim + BlockDim - 1) / BlockDim;
|
||||
static constexpr int MaxBlocks = 1024 * MaxBlockScale;
|
||||
|
||||
static_assert(BlockDim % WARP_SIZE == 0);
|
||||
|
||||
static uint32_t grid_dim(int64_t num_tokens) {
|
||||
int64_t const token_blocks = (num_tokens + NumWarps - 1) / NumWarps;
|
||||
int64_t const selected =
|
||||
token_blocks < MaxBlocks ? token_blocks : MaxBlocks;
|
||||
return static_cast<uint32_t>(selected > 0 ? selected : 1);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename T, typename BiasT, typename IdxT, ScoringFunc SF,
|
||||
int MaxNumExperts, int MaxNumTopExperts>
|
||||
__global__ void __launch_bounds__(WarpTopKLaunchConfig<MaxNumExperts>::BlockDim)
|
||||
single_group_topk_warp_kernel(T const* scores, float* topk_values,
|
||||
IdxT* topk_indices, BiasT const* bias,
|
||||
int64_t num_tokens, int64_t num_experts,
|
||||
int64_t topk, bool renormalize,
|
||||
float routed_scaling_factor,
|
||||
bool enable_pdl) {
|
||||
static constexpr int NumChunks = (MaxNumExperts + WARP_SIZE - 1) / WARP_SIZE;
|
||||
static constexpr int WarpBlockDim =
|
||||
WarpTopKLaunchConfig<MaxNumExperts>::BlockDim;
|
||||
using LaneOwnedRange =
|
||||
reduce_topk::HighExpertLaneOwnedTopKRange<MaxNumExperts,
|
||||
MaxNumTopExperts>;
|
||||
|
||||
static_assert(NumChunks <= 64);
|
||||
static_assert(MaxNumTopExperts <= WARP_SIZE);
|
||||
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
if (enable_pdl) {
|
||||
cudaGridDependencySynchronize();
|
||||
}
|
||||
#endif
|
||||
|
||||
int32_t const lane = static_cast<int32_t>(threadIdx.x) % WARP_SIZE;
|
||||
int32_t const warp_id = static_cast<int32_t>(threadIdx.x) / WARP_SIZE;
|
||||
int32_t const global_warp =
|
||||
static_cast<int32_t>(blockIdx.x) * WarpBlockDim / WARP_SIZE + warp_id;
|
||||
int32_t const global_warp_stride =
|
||||
static_cast<int32_t>(gridDim.x) * WarpBlockDim / WARP_SIZE;
|
||||
int32_t const num_experts_i32 = static_cast<int32_t>(num_experts);
|
||||
int32_t const topk_i32 = static_cast<int32_t>(topk);
|
||||
auto warp = cg::tiled_partition<WARP_SIZE>(cg::this_thread_block());
|
||||
|
||||
for (int32_t token = global_warp; token < num_tokens;
|
||||
token += global_warp_stride) {
|
||||
T const* token_scores = scores + int64_t{token} * num_experts;
|
||||
float local_scores[NumChunks];
|
||||
int32_t local_indices[NumChunks];
|
||||
#pragma unroll
|
||||
for (int index = 0; index < NumChunks; ++index) {
|
||||
int32_t const expert = index * WARP_SIZE + lane;
|
||||
float unbiased;
|
||||
float selection;
|
||||
if (expert < num_experts_i32) {
|
||||
preprocess_score<T, BiasT, SF>(token_scores[expert], bias[expert],
|
||||
unbiased, selection);
|
||||
} else {
|
||||
selection = InvalidScore;
|
||||
}
|
||||
local_scores[index] = selection;
|
||||
local_indices[index] = expert;
|
||||
}
|
||||
|
||||
float lane_score;
|
||||
int32_t lane_expert;
|
||||
if constexpr (LaneOwnedRange::kEnabled) {
|
||||
reduce_topk::reduceTopKForLane<MaxNumTopExperts>(
|
||||
warp, lane_score, lane_expert, local_scores, local_indices,
|
||||
InvalidScore, lane);
|
||||
} else {
|
||||
float top_scores[MaxNumTopExperts];
|
||||
int32_t top_experts[MaxNumTopExperts];
|
||||
reduce_topk::reduceTopK(warp, top_scores, top_experts, local_scores,
|
||||
local_indices, InvalidScore, topk_i32);
|
||||
lane_score = lane < topk_i32 ? top_scores[lane] : InvalidScore;
|
||||
lane_expert = lane < topk_i32 ? top_experts[lane] : -1;
|
||||
}
|
||||
|
||||
float lane_unbiased = 0.0F;
|
||||
if (lane < topk_i32 && lane_expert >= 0 && lane_expert < num_experts_i32) {
|
||||
lane_unbiased = lane_score - cuda_cast<float, BiasT>(bias[lane_expert]);
|
||||
}
|
||||
write_outputs(warp, lane_score, lane_unbiased, lane_expert, lane, token,
|
||||
topk_i32, topk_values, topk_indices, renormalize,
|
||||
routed_scaling_factor);
|
||||
}
|
||||
|
||||
#if defined(__CUDA_ARCH__) && (__CUDA_ARCH__ >= 900)
|
||||
if (enable_pdl) {
|
||||
cudaTriggerProgrammaticLaunchCompletion();
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
template <int Experts, int TopK>
|
||||
struct Tier {
|
||||
static constexpr int kExperts = Experts;
|
||||
static constexpr int kTopK = TopK;
|
||||
};
|
||||
|
||||
template <typename... Tiers>
|
||||
struct TierList {};
|
||||
|
||||
using SigmoidBiasTiers =
|
||||
TierList<Tier<128, 8>, Tier<256, 8>, Tier<384, 8>, Tier<512, 8>,
|
||||
Tier<512, 22>, Tier<768, 16>, Tier<896, 16>, Tier<1024, 16>>;
|
||||
|
||||
using PrecomputedSoftmaxBiasTiers =
|
||||
TierList<Tier<128, 4>, Tier<128, 8>, Tier<160, 8>, Tier<256, 8>,
|
||||
Tier<256, 16>, Tier<512, 8>, Tier<512, 16>, Tier<512, 22>,
|
||||
Tier<512, 32>, Tier<576, 8>, Tier<768, 16>, Tier<896, 16>,
|
||||
Tier<1024, 16>>;
|
||||
|
||||
template <typename T, typename BiasT, typename IdxT, ScoringFunc SF,
|
||||
int MaxNumExperts, int MaxNumTopExperts>
|
||||
void launch(T* scores, float* topk_values, IdxT* topk_indices,
|
||||
BiasT const* bias, int64_t num_tokens, int64_t num_experts,
|
||||
int64_t topk, bool renormalize, double routed_scaling_factor,
|
||||
bool enable_pdl, cudaLaunchConfig_t& config) {
|
||||
config.dynamicSmemBytes = 0;
|
||||
bool const use_block_kernel =
|
||||
UseTunedBlockPath<MaxNumExperts, MaxNumTopExperts> ||
|
||||
MaxNumExperts > 1024 || num_experts >= 1024 ||
|
||||
(num_experts >= 256 && num_tokens <= 1024);
|
||||
if (use_block_kernel) {
|
||||
config.gridDim = static_cast<uint32_t>(num_tokens);
|
||||
config.blockDim = BlockDim;
|
||||
cudaLaunchKernelEx(
|
||||
&config,
|
||||
&single_group_topk_block_kernel<T, BiasT, IdxT, SF, MaxNumExperts,
|
||||
MaxNumTopExperts>,
|
||||
scores, topk_values, topk_indices, bias, num_experts, topk, renormalize,
|
||||
static_cast<float>(routed_scaling_factor), enable_pdl);
|
||||
} else {
|
||||
using WarpConfig = WarpTopKLaunchConfig<MaxNumExperts>;
|
||||
config.gridDim = WarpConfig::grid_dim(num_tokens);
|
||||
config.blockDim = WarpConfig::BlockDim;
|
||||
cudaLaunchKernelEx(
|
||||
&config,
|
||||
&single_group_topk_warp_kernel<T, BiasT, IdxT, SF, MaxNumExperts,
|
||||
MaxNumTopExperts>,
|
||||
scores, topk_values, topk_indices, bias, num_tokens, num_experts, topk,
|
||||
renormalize, static_cast<float>(routed_scaling_factor), enable_pdl);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename BiasT, typename IdxT, ScoringFunc SF>
|
||||
bool dispatch(TierList<>*, T*, float*, IdxT*, BiasT const*, int64_t, int64_t,
|
||||
int64_t, bool, double, bool, cudaLaunchConfig_t&) {
|
||||
return false;
|
||||
}
|
||||
|
||||
template <typename T, typename BiasT, typename IdxT, ScoringFunc SF,
|
||||
typename First, typename... Rest>
|
||||
bool dispatch(TierList<First, Rest...>*, T* scores, float* topk_values,
|
||||
IdxT* topk_indices, BiasT const* bias, int64_t num_tokens,
|
||||
int64_t num_experts, int64_t topk, bool renormalize,
|
||||
double routed_scaling_factor, bool enable_pdl,
|
||||
cudaLaunchConfig_t& config) {
|
||||
if (num_experts <= First::kExperts && topk <= First::kTopK) {
|
||||
launch<T, BiasT, IdxT, SF, First::kExperts, First::kTopK>(
|
||||
scores, topk_values, topk_indices, bias, num_tokens, num_experts, topk,
|
||||
renormalize, routed_scaling_factor, enable_pdl, config);
|
||||
return true;
|
||||
}
|
||||
return dispatch<T, BiasT, IdxT, SF>(
|
||||
static_cast<TierList<Rest...>*>(nullptr), scores, topk_values,
|
||||
topk_indices, bias, num_tokens, num_experts, topk, renormalize,
|
||||
routed_scaling_factor, enable_pdl, config);
|
||||
}
|
||||
|
||||
} // namespace detail
|
||||
|
||||
template <typename T, typename BiasT, typename IdxT, ScoringFunc SF>
|
||||
bool invoke(T* scores, float* topk_values, IdxT* topk_indices,
|
||||
BiasT const* bias, int64_t num_tokens, int64_t num_experts,
|
||||
int64_t topk, bool renormalize, double routed_scaling_factor,
|
||||
bool enable_pdl, cudaLaunchConfig_t& config) {
|
||||
static_assert(SF == SCORING_NONE || SF == SCORING_SIGMOID);
|
||||
if constexpr (SF == SCORING_SIGMOID) {
|
||||
return detail::dispatch<T, BiasT, IdxT, SF>(
|
||||
static_cast<detail::SigmoidBiasTiers*>(nullptr), scores, topk_values,
|
||||
topk_indices, bias, num_tokens, num_experts, topk, renormalize,
|
||||
routed_scaling_factor, enable_pdl, config);
|
||||
} else {
|
||||
return detail::dispatch<T, BiasT, IdxT, SF>(
|
||||
static_cast<detail::PrecomputedSoftmaxBiasTiers*>(nullptr), scores,
|
||||
topk_values, topk_indices, bias, num_tokens, num_experts, topk,
|
||||
renormalize, routed_scaling_factor, enable_pdl, config);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace single_group_topk
|
||||
|
||||
template <typename T, typename BiasT, typename IdxT, ScoringFunc SF>
|
||||
void invokeNoAuxTc(T* scores, float* topk_values, IdxT* topk_indices,
|
||||
BiasT const* bias, int64_t const num_tokens,
|
||||
@@ -905,6 +1335,12 @@ void invokeNoAuxTc(T* scores, float* topk_values, IdxT* topk_indices,
|
||||
attrs[0].val.programmaticStreamSerializationAllowed = enable_pdl;
|
||||
config.numAttrs = 1;
|
||||
config.attrs = attrs;
|
||||
if (n_group == 1 && topk_group == 1 &&
|
||||
single_group_topk::invoke<T, BiasT, IdxT, SF>(
|
||||
scores, topk_values, topk_indices, bias, num_tokens, num_experts,
|
||||
topk, renormalize, routed_scaling_factor, enable_pdl, config)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if we can use the optimized
|
||||
// grouped_topk_fused_small_expert_count_kernel
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
/*
|
||||
* Adapted from
|
||||
* https://github.com/NVIDIA/TensorRT-LLM/blob/v1.3.0rc2/cpp/tensorrt_llm/kernels/moeTopKFuncs.cuh
|
||||
* https://github.com/flashinfer-ai/flashinfer/blob/06400d062a2d51564bbe781f6f811d0b75ca593e/include/flashinfer/trtllm/fused_moe/RoutingKernelTopK.cuh
|
||||
* Copyright (c) 2026, The vLLM team.
|
||||
* SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION. All rights
|
||||
* reserved. SPDX-License-Identifier: Apache-2.0
|
||||
@@ -23,6 +24,9 @@
|
||||
#include <cooperative_groups/reduce.h>
|
||||
#include <cub/cub.cuh>
|
||||
|
||||
#include <cstdint>
|
||||
#include <type_traits>
|
||||
|
||||
namespace vllm {
|
||||
namespace moe {
|
||||
namespace reduce_topk {
|
||||
@@ -38,11 +42,10 @@ struct TopKRedType {
|
||||
"Top K reduction only implemented for int, float, float16 and bfloat16");
|
||||
|
||||
using TypeCmp = std::conditional_t<sizeof(T) == 4, uint64_t, uint32_t>;
|
||||
using IdxT = std::conditional_t<sizeof(T) == 4, int32_t, int16_t>;
|
||||
|
||||
static constexpr int kMoveBits = (sizeof(T) == 4) ? 32 : 16;
|
||||
static constexpr int kMaxIdx = 65535;
|
||||
TypeCmp compValIdx;
|
||||
TypeCmp compVal;
|
||||
|
||||
static __host__ __device__ inline TypeCmp makeCmpVal(T val, int32_t idx = 0) {
|
||||
auto valueBits = cub::Traits<T>::TwiddleIn(
|
||||
@@ -69,69 +72,175 @@ struct TopKRedType {
|
||||
__host__ __device__ TopKRedType() = default;
|
||||
|
||||
__host__ __device__ TopKRedType(T val, int32_t idx)
|
||||
: compValIdx(makeCmpVal(val, idx)) {}
|
||||
: compVal(makeCmpVal(val, idx)) {}
|
||||
|
||||
__host__ __device__ operator TypeCmp() const noexcept { return compValIdx; }
|
||||
__host__ __device__ operator TypeCmp() const noexcept { return compVal; }
|
||||
|
||||
__device__ inline TypeCmp reduce(
|
||||
cg::thread_block_tile<kWARP_SIZE> const& warp) {
|
||||
return cg::reduce(warp, compValIdx, cg::greater<TypeCmp>{});
|
||||
#ifdef __CUDA_ARCH__
|
||||
static constexpr bool kHAS_FAST_REDUX = (__CUDA_ARCH__ / 100) >= 10;
|
||||
#else
|
||||
static constexpr bool kHAS_FAST_REDUX = false;
|
||||
#endif
|
||||
if constexpr (!kHAS_FAST_REDUX) {
|
||||
return cg::reduce(warp, compVal, cg::greater<TypeCmp>{});
|
||||
} else if constexpr (sizeof(TypeCmp) == 8) {
|
||||
uint32_t hi = static_cast<uint32_t>(compVal >> 32);
|
||||
uint32_t lo = static_cast<uint32_t>(compVal & 0xffffffffu);
|
||||
uint32_t maxHi;
|
||||
asm volatile("redux.sync.max.u32 %0, %1, 0xffffffff;\n"
|
||||
: "=r"(maxHi)
|
||||
: "r"(hi));
|
||||
uint32_t loContrib = hi == maxHi ? lo : 0u;
|
||||
uint32_t maxLo;
|
||||
asm volatile("redux.sync.max.u32 %0, %1, 0xffffffff;\n"
|
||||
: "=r"(maxLo)
|
||||
: "r"(loContrib));
|
||||
return (static_cast<TypeCmp>(maxHi) << 32) | static_cast<TypeCmp>(maxLo);
|
||||
} else {
|
||||
TypeCmp result;
|
||||
asm volatile("redux.sync.max.u32 %0, %1, 0xffffffff;\n"
|
||||
: "=r"(result)
|
||||
: "r"(compVal));
|
||||
return result;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <int K_, bool Enable_>
|
||||
struct TopKIdx {
|
||||
// by default, empty
|
||||
template <int N>
|
||||
struct IsPowerOf2 {
|
||||
static constexpr bool value = N > 0 && (N & (N - 1)) == 0;
|
||||
};
|
||||
|
||||
template <int K_>
|
||||
struct TopKIdx<K_, true> {
|
||||
static constexpr int K = K_;
|
||||
int32_t val[K];
|
||||
template <int N>
|
||||
struct NextPow2 {
|
||||
private:
|
||||
static constexpr unsigned u = static_cast<unsigned>(N - 1);
|
||||
static constexpr unsigned s1 = u | (u >> 1);
|
||||
static constexpr unsigned s2 = s1 | (s1 >> 2);
|
||||
static constexpr unsigned s3 = s2 | (s2 >> 4);
|
||||
static constexpr unsigned s4 = s3 | (s3 >> 8);
|
||||
static constexpr unsigned s5 = s4 | (s4 >> 16);
|
||||
|
||||
public:
|
||||
static constexpr int value = N <= 1 ? 1 : static_cast<int>(s5 + 1);
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
#define TOPK_SWAP(I, J) \
|
||||
{ \
|
||||
auto pairMin = min(topK[I].compValIdx, topK[J].compValIdx); \
|
||||
auto pairMax = max(topK[I].compValIdx, topK[J].compValIdx); \
|
||||
topK[I].compValIdx = pairMax; \
|
||||
topK[J].compValIdx = pairMin; \
|
||||
template <int A, int B, int Size, typename T>
|
||||
__device__ __forceinline__ void topkCompareSwap(T* a) {
|
||||
if constexpr (A < Size && B < Size) {
|
||||
if (a[A] < a[B]) {
|
||||
T tmp = a[A];
|
||||
a[A] = a[B];
|
||||
a[B] = tmp;
|
||||
}
|
||||
} else {
|
||||
(void)a;
|
||||
}
|
||||
}
|
||||
|
||||
template <int I, int End, int Step, int PairStride, int Size, typename T>
|
||||
__device__ __forceinline__ void topkMergePairs(T* a) {
|
||||
if constexpr (I + Step < End) {
|
||||
topkCompareSwap<I, I + Step, Size, T>(a);
|
||||
topkMergePairs<I + PairStride, End, Step, PairStride, Size, T>(a);
|
||||
} else {
|
||||
(void)a;
|
||||
}
|
||||
}
|
||||
|
||||
template <int Lo, int N, int R, int Size, typename T>
|
||||
__device__ __forceinline__ void topkOEM(T* a) {
|
||||
constexpr int M = R * 2;
|
||||
if constexpr (M < N) {
|
||||
topkOEM<Lo, N, M, Size, T>(a);
|
||||
topkOEM<Lo + R, N - R, M, Size, T>(a);
|
||||
topkMergePairs<Lo + R, Lo + N, R, M, Size, T>(a);
|
||||
} else if constexpr (R < N) {
|
||||
topkCompareSwap<Lo, Lo + R, Size, T>(a);
|
||||
} else {
|
||||
(void)a;
|
||||
}
|
||||
}
|
||||
|
||||
template <int Lo, int N, int Size, typename T>
|
||||
__device__ __forceinline__ void topkSortBatcher(T* a) {
|
||||
if constexpr (N > 1) {
|
||||
constexpr int Half = N / 2;
|
||||
topkSortBatcher<Lo, Half, Size, T>(a);
|
||||
topkSortBatcher<Lo + Half, N - Half, Size, T>(a);
|
||||
topkOEM<Lo, N, 1, Size, T>(a);
|
||||
} else {
|
||||
(void)a;
|
||||
}
|
||||
}
|
||||
|
||||
template <int N, typename RedType>
|
||||
struct Sort;
|
||||
struct Sort {
|
||||
static_assert(N > 0 && N <= 64, "Sort only supports N in range [1, 64]");
|
||||
|
||||
static __device__ void run(RedType* topK) {
|
||||
if constexpr (IsPowerOf2<N>::value) {
|
||||
#pragma unroll
|
||||
for (int k = 2; k <= N; k *= 2) {
|
||||
#pragma unroll
|
||||
for (int j = k / 2; j > 0; j /= 2) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < N; ++i) {
|
||||
int ixj = i ^ j;
|
||||
if (ixj > i) {
|
||||
if ((i & k) == 0) {
|
||||
if (topK[i].compVal < topK[ixj].compVal) {
|
||||
auto tmp = topK[i].compVal;
|
||||
topK[i].compVal = topK[ixj].compVal;
|
||||
topK[ixj].compVal = tmp;
|
||||
}
|
||||
} else {
|
||||
if (topK[i].compVal > topK[ixj].compVal) {
|
||||
auto tmp = topK[i].compVal;
|
||||
topK[i].compVal = topK[ixj].compVal;
|
||||
topK[ixj].compVal = tmp;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
constexpr int P = NextPow2<N>::value;
|
||||
topkSortBatcher<0, P, N, RedType>(topK);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename RedType>
|
||||
struct Sort<1, RedType> {
|
||||
static __device__ void run(RedType* topK) {}
|
||||
static __device__ void run(RedType*) {}
|
||||
};
|
||||
|
||||
template <typename RedType>
|
||||
struct Sort<2, RedType> {
|
||||
static __device__ void run(RedType* topK) { TOPK_SWAP(0, 1); }
|
||||
static __device__ void run(RedType* topK) { topkCompareSwap<0, 1, 2>(topK); }
|
||||
};
|
||||
|
||||
template <typename RedType>
|
||||
struct Sort<3, RedType> {
|
||||
static __device__ void run(RedType* topK) {
|
||||
TOPK_SWAP(0, 1);
|
||||
TOPK_SWAP(1, 2);
|
||||
TOPK_SWAP(0, 1);
|
||||
topkCompareSwap<0, 1, 3>(topK);
|
||||
topkCompareSwap<1, 2, 3>(topK);
|
||||
topkCompareSwap<0, 1, 3>(topK);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename RedType>
|
||||
struct Sort<4, RedType> {
|
||||
static __device__ void run(RedType* topK) {
|
||||
TOPK_SWAP(0, 2);
|
||||
TOPK_SWAP(1, 3);
|
||||
TOPK_SWAP(0, 1);
|
||||
TOPK_SWAP(2, 3);
|
||||
TOPK_SWAP(1, 2);
|
||||
topkCompareSwap<0, 2, 4>(topK);
|
||||
topkCompareSwap<1, 3, 4>(topK);
|
||||
topkCompareSwap<0, 1, 4>(topK);
|
||||
topkCompareSwap<2, 3, 4>(topK);
|
||||
topkCompareSwap<1, 2, 4>(topK);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -147,110 +256,112 @@ __forceinline__ __device__ void reduceTopK(
|
||||
typename RedType::TypeCmp packedMax{};
|
||||
#pragma unroll
|
||||
for (int kk = 0; kk < actualK; ++kk) {
|
||||
topK =
|
||||
kk > 0 && packedMax == topK.compValIdx ? RedType{minValue, idx} : topK;
|
||||
// get the next largest value
|
||||
topK = kk > 0 && packedMax == topK.compVal ? RedType{minValue, idx} : topK;
|
||||
packedMax = topK.reduce(warp);
|
||||
RedType::unpack(out[kk], outIdx[kk], packedMax);
|
||||
}
|
||||
};
|
||||
|
||||
template <int K, typename Type, int N, bool IsSorted = false>
|
||||
__device__ void reduceTopKFunc(cg::thread_block_tile<kWARP_SIZE> const& warp,
|
||||
Type (&out)[K], int32_t (&outIdx)[K],
|
||||
Type (&value)[N], int32_t (&idx)[N],
|
||||
Type minValue, int actualK = K) {
|
||||
static_assert(K > 0, "Top K must have K > 0");
|
||||
static_assert(K < kWARP_SIZE, "Top K must have K < kWARP_SIZE");
|
||||
static_assert(N > 0, "Top K must have N > 0");
|
||||
static_assert(N < 5,
|
||||
"Only support candidates number less than or equal to 128");
|
||||
using RedType = TopKRedType<Type>;
|
||||
RedType topK[N];
|
||||
#pragma unroll
|
||||
for (int nn = 0; nn < N; ++nn) {
|
||||
topK[nn] = RedType{value[nn], idx[nn]};
|
||||
}
|
||||
|
||||
if constexpr (!IsSorted) {
|
||||
Sort<N, RedType>::run(topK);
|
||||
}
|
||||
typename RedType::TypeCmp packedMax{};
|
||||
#pragma unroll
|
||||
for (int kk = 0; kk < actualK; ++kk) {
|
||||
bool update = kk > 0 && packedMax == topK[0].compValIdx;
|
||||
#pragma unroll
|
||||
for (int nn = 0; nn < N; ++nn) {
|
||||
topK[nn] = update && nn == N - 1 ? RedType{minValue, idx[nn]}
|
||||
: update ? topK[nn + 1]
|
||||
: topK[nn];
|
||||
}
|
||||
// get the next largest value
|
||||
packedMax = topK[0].reduce(warp);
|
||||
RedType::unpack(out[kk], outIdx[kk], packedMax);
|
||||
}
|
||||
};
|
||||
|
||||
template <int K, typename Type, int N>
|
||||
__forceinline__ __device__ void reduceTopK(
|
||||
cg::thread_block_tile<kWARP_SIZE> const& warp, Type (&out)[K],
|
||||
int32_t (&outIdx)[K], Type (&value)[N], int32_t (&idx)[N],
|
||||
Type const minValue, int actualK = K) {
|
||||
static_assert(K > 0, "Top K must have K > 0");
|
||||
static_assert(K < kWARP_SIZE, "Top K must have K < kWARP_SIZE");
|
||||
static_assert(K <= kWARP_SIZE, "Top K must have K <= kWARP_SIZE");
|
||||
static_assert(N > 0, "Top K must have N > 0");
|
||||
static_assert(
|
||||
N <= 16,
|
||||
"Only support candidates number less than or equal to 16*32=512");
|
||||
static_assert(N <= 4 || N % 4 == 0,
|
||||
"Only support candidates number is a multiple of 4*32=128 or "
|
||||
"less than or equal to 4");
|
||||
static_assert(N <= 64,
|
||||
"Only support candidates number less than or equal to "
|
||||
"64*32=2048");
|
||||
using RedType = TopKRedType<Type>;
|
||||
RedType topK[N];
|
||||
#pragma unroll
|
||||
for (int nn = 0; nn < N; ++nn) {
|
||||
topK[nn] = RedType{value[nn], idx[nn]};
|
||||
}
|
||||
|
||||
if constexpr (N <= 4) {
|
||||
reduceTopKFunc<K, Type, N>(warp, out, outIdx, value, idx, minValue,
|
||||
actualK);
|
||||
} else {
|
||||
constexpr int numLoops = N / 4;
|
||||
constexpr int numResults = (numLoops * K - 1) / kWARP_SIZE + 1;
|
||||
Sort<N, RedType>::run(topK);
|
||||
|
||||
Type topKBufferValue[numResults];
|
||||
int32_t topKBufferIdx[numResults];
|
||||
int32_t laneIdx = threadIdx.x % kWARP_SIZE;
|
||||
|
||||
for (int ii = 0; ii < numResults; ++ii) {
|
||||
topKBufferValue[ii] = minValue;
|
||||
topKBufferIdx[ii] = ii * kWARP_SIZE - 1;
|
||||
typename RedType::TypeCmp packedMax{};
|
||||
for (int kk = 0; kk < actualK; ++kk) {
|
||||
bool update = kk > 0 && packedMax == topK[0].compVal;
|
||||
#pragma unroll
|
||||
for (int nn = 0; nn < N; ++nn) {
|
||||
topK[nn] = update && nn == N - 1 ? RedType{minValue, idx[nn]}
|
||||
: update ? topK[nn + 1]
|
||||
: topK[nn];
|
||||
}
|
||||
for (int loop = 0; loop < numLoops; ++loop) {
|
||||
int start = loop * 4;
|
||||
Type topKValue[K];
|
||||
int32_t topKIdx[K];
|
||||
Type inValue[4];
|
||||
int32_t inIdx[4];
|
||||
for (int i = 0; i < 4; ++i) {
|
||||
inValue[i] = value[start + i];
|
||||
inIdx[i] = idx[start + i];
|
||||
}
|
||||
reduceTopKFunc<K, Type, 4>(warp, topKValue, topKIdx, inValue, inIdx,
|
||||
minValue, actualK);
|
||||
int inOffset = laneIdx % K;
|
||||
if (laneIdx >= loop * K && laneIdx < (loop + 1) * K) {
|
||||
topKBufferValue[0] = topKValue[inOffset];
|
||||
topKBufferIdx[0] = topKIdx[inOffset];
|
||||
}
|
||||
if (loop == numLoops - 1 && (laneIdx < (numLoops * K - kWARP_SIZE))) {
|
||||
topKBufferValue[1] = topKValue[inOffset];
|
||||
topKBufferIdx[1] = topKIdx[inOffset];
|
||||
}
|
||||
}
|
||||
|
||||
reduceTopKFunc<K, Type, numResults>(warp, out, outIdx, topKBufferValue,
|
||||
topKBufferIdx, minValue, actualK);
|
||||
packedMax = topK[0].reduce(warp);
|
||||
RedType::unpack(out[kk], outIdx[kk], packedMax);
|
||||
}
|
||||
};
|
||||
|
||||
#undef TOPK_SWAP
|
||||
template <int NumExperts, int NumTopExperts, int MinExperts, int MaxExperts,
|
||||
int MinTopExperts, int MaxTopExperts>
|
||||
struct LaneOwnedTopKRange {
|
||||
static_assert(MinExperts > 0 && MinExperts <= MaxExperts);
|
||||
static_assert(MinTopExperts > 0 && MinTopExperts <= MaxTopExperts);
|
||||
static constexpr bool kEnabled =
|
||||
NumExperts >= MinExperts && NumExperts <= MaxExperts &&
|
||||
NumTopExperts >= MinTopExperts && NumTopExperts <= MaxTopExperts;
|
||||
};
|
||||
|
||||
static constexpr int kHIGH_EXPERT_LANE_OWNED_TOPK_MIN_EXPERTS = 512;
|
||||
static constexpr int kHIGH_EXPERT_LANE_OWNED_TOPK_MAX_EXPERTS = 1024;
|
||||
static constexpr int kHIGH_EXPERT_LANE_OWNED_TOPK_MIN_TOP_EXPERTS = 9;
|
||||
static constexpr int kHIGH_EXPERT_LANE_OWNED_TOPK_MAX_TOP_EXPERTS = 16;
|
||||
|
||||
template <int NumExperts, int NumTopExperts>
|
||||
using HighExpertLaneOwnedTopKRange =
|
||||
LaneOwnedTopKRange<NumExperts, NumTopExperts,
|
||||
kHIGH_EXPERT_LANE_OWNED_TOPK_MIN_EXPERTS,
|
||||
kHIGH_EXPERT_LANE_OWNED_TOPK_MAX_EXPERTS,
|
||||
kHIGH_EXPERT_LANE_OWNED_TOPK_MIN_TOP_EXPERTS,
|
||||
kHIGH_EXPERT_LANE_OWNED_TOPK_MAX_TOP_EXPERTS>;
|
||||
|
||||
template <int K, typename Type, int N>
|
||||
__forceinline__ __device__ void reduceTopKForLane(
|
||||
cg::thread_block_tile<kWARP_SIZE> const& warp, Type& out, int32_t& outIdx,
|
||||
Type (&value)[N], int32_t (&idx)[N], Type const minValue, int32_t laneIdx) {
|
||||
static_assert(K > 0, "Top K must have K > 0");
|
||||
static_assert(K <= kWARP_SIZE, "Top K must have K <= kWARP_SIZE");
|
||||
static_assert(N > 0, "Top K must have N > 0");
|
||||
static_assert(N <= 64,
|
||||
"Only support candidates number less than or equal to "
|
||||
"64*32=2048");
|
||||
using RedType = TopKRedType<Type>;
|
||||
RedType topK[N];
|
||||
#pragma unroll
|
||||
for (int nn = 0; nn < N; ++nn) {
|
||||
topK[nn] = RedType{value[nn], idx[nn]};
|
||||
}
|
||||
|
||||
Sort<N, RedType>::run(topK);
|
||||
|
||||
typename RedType::TypeCmp packedMax{};
|
||||
typename RedType::TypeCmp lanePacked{};
|
||||
#pragma unroll
|
||||
for (int kk = 0; kk < K; ++kk) {
|
||||
bool update = kk > 0 && packedMax == topK[0].compVal;
|
||||
#pragma unroll
|
||||
for (int nn = 0; nn < N; ++nn) {
|
||||
topK[nn] = update && nn == N - 1 ? RedType{minValue, idx[nn]}
|
||||
: update ? topK[nn + 1]
|
||||
: topK[nn];
|
||||
}
|
||||
packedMax = topK[0].reduce(warp);
|
||||
if (laneIdx == kk) {
|
||||
lanePacked = packedMax;
|
||||
}
|
||||
}
|
||||
|
||||
if (laneIdx < K) {
|
||||
RedType::unpack(out, outIdx, lanePacked);
|
||||
} else {
|
||||
out = minValue;
|
||||
outIdx = -1;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace reduce_topk
|
||||
} // namespace moe
|
||||
|
||||
@@ -1086,4 +1086,4 @@ void moe_lora_align_block_size(
|
||||
has_expert_map);
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -276,6 +276,61 @@ void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_bf16_insert(
|
||||
torch::stable::Tensor const& cos_sin_cache, double eps,
|
||||
int64_t cache_block_size);
|
||||
|
||||
void fused_kimi_k3_mla_key_concat_kv_cache_insert(
|
||||
torch::stable::Tensor& q, torch::stable::Tensor const& k_nope,
|
||||
torch::stable::Tensor const& k_pe, torch::stable::Tensor const& kv_c_normed,
|
||||
torch::stable::Tensor& k_out, torch::stable::Tensor& k_cache,
|
||||
torch::stable::Tensor const& slot_mapping, int64_t cache_block_size,
|
||||
std::optional<torch::stable::Tensor> position_ids,
|
||||
std::optional<torch::stable::Tensor> cos_sin_cache);
|
||||
|
||||
void fused_kimi_k3_mla_key_concat_ds_mla_insert(
|
||||
torch::stable::Tensor& q, torch::stable::Tensor const& k_nope,
|
||||
torch::stable::Tensor const& k_pe, torch::stable::Tensor const& kv_c_normed,
|
||||
torch::stable::Tensor& k_out, torch::stable::Tensor& k_cache,
|
||||
torch::stable::Tensor const& slot_mapping, int64_t cache_block_size,
|
||||
std::optional<torch::stable::Tensor> position_ids,
|
||||
std::optional<torch::stable::Tensor> cos_sin_cache);
|
||||
|
||||
void fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert(
|
||||
torch::stable::Tensor const& q, torch::stable::Tensor const& k_nope,
|
||||
torch::stable::Tensor const& k_pe, torch::stable::Tensor const& kv_c_normed,
|
||||
torch::stable::Tensor const& v, torch::stable::Tensor& q_fp8,
|
||||
torch::stable::Tensor& k_fp8, torch::stable::Tensor& v_fp8,
|
||||
torch::stable::Tensor& k_cache, torch::stable::Tensor const& slot_mapping,
|
||||
torch::stable::Tensor const& q_scale_inv,
|
||||
torch::stable::Tensor const& k_scale_inv,
|
||||
torch::stable::Tensor const& v_scale_inv,
|
||||
torch::stable::Tensor const& cache_scale_inv, int64_t cache_block_size,
|
||||
std::optional<torch::stable::Tensor> position_ids,
|
||||
std::optional<torch::stable::Tensor> cos_sin_cache);
|
||||
|
||||
void fused_kimi_k3_mla_decode_q_concat_kv_cache_insert(
|
||||
torch::stable::Tensor const& ql_nope, torch::stable::Tensor const& q_pe,
|
||||
torch::stable::Tensor const& kv_c_normed, torch::stable::Tensor const& k_pe,
|
||||
torch::stable::Tensor& mqa_q, torch::stable::Tensor& k_cache,
|
||||
torch::stable::Tensor const& slot_mapping, int64_t cache_block_size,
|
||||
std::optional<torch::stable::Tensor> position_ids,
|
||||
std::optional<torch::stable::Tensor> cos_sin_cache);
|
||||
|
||||
void fused_kimi_k3_mla_decode_q_concat_kv_cache_fp8_insert(
|
||||
torch::stable::Tensor const& ql_nope, torch::stable::Tensor const& q_pe,
|
||||
torch::stable::Tensor const& kv_c_normed, torch::stable::Tensor const& k_pe,
|
||||
torch::stable::Tensor& mqa_q, torch::stable::Tensor& k_cache,
|
||||
torch::stable::Tensor const& slot_mapping,
|
||||
torch::stable::Tensor const& q_scale_inv,
|
||||
torch::stable::Tensor const& cache_scale_inv, int64_t cache_block_size,
|
||||
std::optional<torch::stable::Tensor> position_ids,
|
||||
std::optional<torch::stable::Tensor> cos_sin_cache);
|
||||
|
||||
void fused_kimi_k3_mla_decode_q_concat_ds_mla_insert(
|
||||
torch::stable::Tensor const& ql_nope, torch::stable::Tensor const& q_pe,
|
||||
torch::stable::Tensor const& kv_c_normed, torch::stable::Tensor const& k_pe,
|
||||
torch::stable::Tensor& mqa_q, torch::stable::Tensor& k_cache,
|
||||
torch::stable::Tensor const& slot_mapping, int64_t cache_block_size,
|
||||
std::optional<torch::stable::Tensor> position_ids,
|
||||
std::optional<torch::stable::Tensor> cos_sin_cache);
|
||||
|
||||
void fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert(
|
||||
torch::stable::Tensor const& q, torch::stable::Tensor const& kv,
|
||||
torch::stable::Tensor& q_fp8, torch::stable::Tensor& k_cache,
|
||||
@@ -315,6 +370,19 @@ void fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
std::optional<torch::stable::Tensor> index_q_out,
|
||||
const std::string& kv_cache_dtype, bool skip_index_branch);
|
||||
|
||||
#ifdef VLLM_ENABLE_FUSED_KDA_DECODE
|
||||
void fused_kda_decode(
|
||||
torch::stable::Tensor const& x, torch::stable::Tensor const& weight,
|
||||
std::optional<torch::stable::Tensor> bias,
|
||||
torch::stable::Tensor& conv_state, torch::stable::Tensor const& raw_g,
|
||||
torch::stable::Tensor const& raw_beta, torch::stable::Tensor const& a_log,
|
||||
torch::stable::Tensor const& dt_bias,
|
||||
torch::stable::Tensor const& state_indices, torch::stable::Tensor& state,
|
||||
torch::stable::Tensor& out, std::optional<double> lower_bound,
|
||||
std::optional<torch::stable::Tensor> output_gate,
|
||||
std::optional<torch::stable::Tensor> norm_weight, double norm_eps);
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
void kimi_k3_attn_res(torch::stable::Tensor& prefix,
|
||||
torch::stable::Tensor const& delta,
|
||||
@@ -383,6 +451,20 @@ fptr_t init_custom_ar(const std::vector<int64_t>& fake_ipc_ptrs,
|
||||
void all_reduce(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
torch::stable::Tensor& out, fptr_t reg_buffer,
|
||||
int64_t reg_buffer_sz_bytes);
|
||||
void custom_all_gather(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
torch::stable::Tensor& out, fptr_t reg_buffer,
|
||||
int64_t reg_buffer_sz_bytes);
|
||||
void mnnvl_lamport_all_gather(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
torch::stable::Tensor& out, fptr_t local_buffer,
|
||||
fptr_t multicast_buffer, fptr_t epoch_buffer,
|
||||
int64_t stage_sz_bytes);
|
||||
void custom_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
torch::stable::Tensor& out, fptr_t reg_buffer,
|
||||
int64_t reg_buffer_sz_bytes);
|
||||
void mnnvl_lamport_reduce_scatter(fptr_t _fa, torch::stable::Tensor& inp,
|
||||
torch::stable::Tensor& out,
|
||||
fptr_t local_buffer, fptr_t epoch_buffer,
|
||||
int64_t stage_sz_bytes);
|
||||
void dispose(fptr_t _fa);
|
||||
int64_t meta_size();
|
||||
void register_buffer(fptr_t _fa, const std::vector<int64_t>& fake_ipc_ptrs);
|
||||
@@ -421,6 +503,12 @@ void fatrelu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
||||
double threshold);
|
||||
void swigluoai_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
||||
double alpha = 1.702, double limit = 7.0);
|
||||
void situ_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input,
|
||||
double beta = 1.0, double linear_beta = -1.0);
|
||||
void masked_situ_and_mul(torch::stable::Tensor& out,
|
||||
torch::stable::Tensor& input,
|
||||
const torch::stable::Tensor& expert_num_tokens,
|
||||
double beta = 1.0, double linear_beta = -1.0);
|
||||
void gelu_new(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void gelu_fast(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void gelu_quick(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
@@ -497,6 +585,13 @@ void concat_and_cache_mla(torch::stable::Tensor& kv_c,
|
||||
const std::string& kv_cache_dtype,
|
||||
torch::stable::Tensor& scale);
|
||||
|
||||
void concat_and_cache_mla_grouped(torch::stable::Tensor& kv_c,
|
||||
torch::stable::Tensor& k_pe,
|
||||
torch::stable::Tensor& kv_cache_ptrs,
|
||||
torch::stable::Tensor& slot_mapping,
|
||||
int64_t block_size, int64_t block_stride,
|
||||
int64_t entry_stride);
|
||||
|
||||
// NOTE: k_pe and kv_c order is flipped compared to concat_and_cache_mla
|
||||
void concat_and_cache_mla_rope_fused(
|
||||
torch::stable::Tensor& positions, torch::stable::Tensor& q_pe,
|
||||
|
||||
@@ -324,7 +324,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
// DeepSeek V3 fused A GEMM (SM 9.0+, bf16 only, 1-16 tokens).
|
||||
// conditionally compiled so impl registration is in source file
|
||||
ops.def(
|
||||
"dsv3_fused_a_gemm(Tensor! output, Tensor mat_a, Tensor mat_b) -> ()");
|
||||
"dsv3_fused_a_gemm(Tensor! output, Tensor mat_a, Tensor mat_b, "
|
||||
"bool enable_pdl=False) -> ()");
|
||||
|
||||
// BF16/FP32 x FP32 -> FP32 router GEMM for H=3072, E=256, M<=32 (SM90+).
|
||||
// conditionally compiled so impl registration is in source file
|
||||
@@ -447,6 +448,48 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"Tensor fp8_scale, Tensor q_fp8_scale_inv, float eps, "
|
||||
"int cache_block_size) -> ()");
|
||||
|
||||
// Kimi-K3 MLA epilogues: optional RoPE followed by concat/cache insertion.
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_key_concat_kv_cache_insert("
|
||||
"Tensor! q, Tensor k_nope, Tensor k_pe, Tensor kv_c_normed, "
|
||||
"Tensor! k_out, Tensor! k_cache, Tensor slot_mapping, "
|
||||
"int cache_block_size, Tensor? position_ids=None, "
|
||||
"Tensor? cos_sin_cache=None) -> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_key_concat_ds_mla_insert("
|
||||
"Tensor! q, Tensor k_nope, Tensor k_pe, Tensor kv_c_normed, "
|
||||
"Tensor! k_out, Tensor! k_cache, Tensor slot_mapping, "
|
||||
"int cache_block_size, Tensor? position_ids=None, "
|
||||
"Tensor? cos_sin_cache=None) -> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert("
|
||||
"Tensor q, Tensor k_nope, Tensor k_pe, Tensor kv_c_normed, Tensor v, "
|
||||
"Tensor! q_fp8, Tensor! k_fp8, Tensor! v_fp8, Tensor! k_cache, "
|
||||
"Tensor slot_mapping, Tensor q_scale_inv, Tensor k_scale_inv, "
|
||||
"Tensor v_scale_inv, Tensor cache_scale_inv, int cache_block_size, "
|
||||
"Tensor? position_ids=None, Tensor? cos_sin_cache=None) -> ()");
|
||||
|
||||
// Kimi-K3 MLA decode epilogue: concat mqa_q = [ql_nope | q_pe] and insert the
|
||||
// latent [kv_c_normed | k_pe] into the paged cache (bf16 / fp8 / fp8_ds_mla).
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_decode_q_concat_kv_cache_insert("
|
||||
"Tensor ql_nope, Tensor q_pe, Tensor kv_c_normed, Tensor k_pe, "
|
||||
"Tensor! mqa_q, Tensor! k_cache, Tensor slot_mapping, "
|
||||
"int cache_block_size, Tensor? position_ids=None, "
|
||||
"Tensor? cos_sin_cache=None) -> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_decode_q_concat_kv_cache_fp8_insert("
|
||||
"Tensor ql_nope, Tensor q_pe, Tensor kv_c_normed, Tensor k_pe, "
|
||||
"Tensor! mqa_q, Tensor! k_cache, Tensor slot_mapping, "
|
||||
"Tensor q_scale_inv, Tensor cache_scale_inv, int cache_block_size, "
|
||||
"Tensor? position_ids=None, Tensor? cos_sin_cache=None) -> ()");
|
||||
ops.def(
|
||||
"fused_kimi_k3_mla_decode_q_concat_ds_mla_insert("
|
||||
"Tensor ql_nope, Tensor q_pe, Tensor kv_c_normed, Tensor k_pe, "
|
||||
"Tensor! mqa_q, Tensor! k_cache, Tensor slot_mapping, "
|
||||
"int cache_block_size, Tensor? position_ids=None, "
|
||||
"Tensor? cos_sin_cache=None) -> ()");
|
||||
|
||||
#ifndef USE_ROCM
|
||||
ops.def(
|
||||
"minimax_allreduce_rms_qk("
|
||||
@@ -468,6 +511,16 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"int block_size, Tensor!? q_out, Tensor!? index_q_out, "
|
||||
"str kv_cache_dtype, bool skip_index_branch=False) -> ()");
|
||||
|
||||
#ifdef VLLM_ENABLE_FUSED_KDA_DECODE
|
||||
ops.def(
|
||||
"fused_kda_decode("
|
||||
"Tensor x, Tensor weight, Tensor? bias, Tensor! conv_state, "
|
||||
"Tensor raw_g, Tensor raw_beta, Tensor A_log, Tensor dt_bias, "
|
||||
"Tensor state_indices, Tensor! state, Tensor! out, "
|
||||
"float? lower_bound=None, Tensor? output_gate=None, "
|
||||
"Tensor? norm_weight=None, float norm_eps=1e-5) -> ()");
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
ops.def(
|
||||
"kimi_k3_attn_res("
|
||||
@@ -538,6 +591,14 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"limit=7.0) "
|
||||
"-> ()");
|
||||
|
||||
// SituGLU implementation used in Kimi models.
|
||||
ops.def(
|
||||
"situ_and_mul(Tensor! out, Tensor input, float beta=1.0, float "
|
||||
"linear_beta=-1.0) -> ()");
|
||||
ops.def(
|
||||
"masked_situ_and_mul(Tensor! out, Tensor input, Tensor "
|
||||
"expert_num_tokens, float beta=1.0, float linear_beta=-1.0) -> ()");
|
||||
|
||||
// GELU implementation used in GPT-2.
|
||||
ops.def("gelu_new(Tensor! out, Tensor input) -> ()");
|
||||
|
||||
@@ -696,11 +757,27 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl(
|
||||
"fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert",
|
||||
TORCH_BOX(&fused_deepseek_v4_qnorm_rope_kv_rope_full_cache_fp8_insert));
|
||||
ops.impl("fused_kimi_k3_mla_key_concat_kv_cache_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_key_concat_kv_cache_insert));
|
||||
ops.impl("fused_kimi_k3_mla_key_concat_ds_mla_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_key_concat_ds_mla_insert));
|
||||
ops.impl("fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_qkv_quant_kv_cache_fp8_insert));
|
||||
ops.impl("fused_kimi_k3_mla_decode_q_concat_kv_cache_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_decode_q_concat_kv_cache_insert));
|
||||
ops.impl("fused_kimi_k3_mla_decode_q_concat_kv_cache_fp8_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_decode_q_concat_kv_cache_fp8_insert));
|
||||
ops.impl("fused_kimi_k3_mla_decode_q_concat_ds_mla_insert",
|
||||
TORCH_BOX(&fused_kimi_k3_mla_decode_q_concat_ds_mla_insert));
|
||||
#ifndef USE_ROCM
|
||||
ops.impl("minimax_allreduce_rms_qk", TORCH_BOX(&minimax_allreduce_rms_qk));
|
||||
#endif
|
||||
ops.impl("fused_minimax_m3_qknorm_rope_kv_insert",
|
||||
TORCH_BOX(&fused_minimax_m3_qknorm_rope_kv_insert));
|
||||
#ifdef VLLM_ENABLE_FUSED_KDA_DECODE
|
||||
ops.impl("fused_kda_decode", TORCH_BOX(&fused_kda_decode));
|
||||
#endif
|
||||
|
||||
#ifdef VLLM_ENABLE_KIMI_K3_ATTN_RES
|
||||
ops.impl("kimi_k3_attn_res", TORCH_BOX(&kimi_k3_attn_res));
|
||||
#endif
|
||||
@@ -726,6 +803,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("gelu_tanh_and_mul", TORCH_BOX(&gelu_tanh_and_mul));
|
||||
ops.impl("fatrelu_and_mul", TORCH_BOX(&fatrelu_and_mul));
|
||||
ops.impl("swigluoai_and_mul", TORCH_BOX(&swigluoai_and_mul));
|
||||
ops.impl("situ_and_mul", TORCH_BOX(&situ_and_mul));
|
||||
ops.impl("masked_situ_and_mul", TORCH_BOX(&masked_situ_and_mul));
|
||||
ops.impl("gelu_new", TORCH_BOX(&gelu_new));
|
||||
ops.impl("gelu_fast", TORCH_BOX(&gelu_fast));
|
||||
ops.impl("gelu_quick", TORCH_BOX(&gelu_quick));
|
||||
@@ -823,6 +902,15 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
|
||||
" str kv_cache_dtype,"
|
||||
" Tensor scale) -> ()");
|
||||
|
||||
// Grouped concat_and_cache_mla across all layers (bf16 only). Each
|
||||
// layer's cache base pointer is read from kv_cache_ptrs.
|
||||
ops.def(
|
||||
"concat_and_cache_mla_grouped(Tensor kv_c, Tensor k_pe,"
|
||||
" Tensor kv_cache_ptrs,"
|
||||
" Tensor slot_mapping,"
|
||||
" int block_size, int block_stride,"
|
||||
" int entry_stride) -> ()");
|
||||
|
||||
// Rotate Q and K, then write to kv cache for MLA
|
||||
ops.def(
|
||||
"concat_and_cache_mla_rope_fused("
|
||||
@@ -921,6 +1009,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C_cache_ops, CUDA, ops) {
|
||||
ops.impl("reshape_and_cache", TORCH_BOX(&reshape_and_cache));
|
||||
ops.impl("reshape_and_cache_flash", TORCH_BOX(&reshape_and_cache_flash));
|
||||
ops.impl("concat_and_cache_mla", TORCH_BOX(&concat_and_cache_mla));
|
||||
ops.impl("concat_and_cache_mla_grouped",
|
||||
TORCH_BOX(&concat_and_cache_mla_grouped));
|
||||
ops.impl("concat_and_cache_mla_rope_fused",
|
||||
TORCH_BOX(&concat_and_cache_mla_rope_fused));
|
||||
ops.impl("convert_fp8", TORCH_BOX(&convert_fp8));
|
||||
|
||||
@@ -13,10 +13,18 @@ namespace vllm {
|
||||
namespace fp8 {
|
||||
#ifdef ENABLE_FP8
|
||||
|
||||
// Unspecialized conversions are a compile error: the old passthrough
|
||||
// (`return x;`) silently skipped fp8 encoding for any (Tout, Tin) pair
|
||||
// without a specialization below (e.g. the torch stable-ABI scalar types),
|
||||
// corrupting quantized data with no runtime signal.
|
||||
template <typename>
|
||||
inline constexpr bool _no_conversion_specialization = false;
|
||||
|
||||
template <typename Tout, typename Tin>
|
||||
__inline__ __device__ Tout vec_conversion(
|
||||
const Tin& x, const __nv_fp8_interpretation_t fp8_type = __NV_E4M3) {
|
||||
return x;
|
||||
static_assert(_no_conversion_specialization<Tin>,
|
||||
"no vec_conversion specialization for this (Tout, Tin) pair");
|
||||
}
|
||||
|
||||
// float -> c10::Float8_e4m3fn
|
||||
@@ -301,7 +309,9 @@ __inline__ __device__ bf16_8_t vec_conversion<bf16_8_t, Float8_>(
|
||||
template <typename Tout, typename Tin>
|
||||
__inline__ __device__ Tout scaled_vec_conversion(
|
||||
const Tin& x, const float scale, const __nv_fp8_interpretation_t fp8_type) {
|
||||
return x;
|
||||
static_assert(
|
||||
_no_conversion_specialization<Tin>,
|
||||
"no scaled_vec_conversion specialization for this (Tout, Tin) pair");
|
||||
}
|
||||
|
||||
// fp8 -> half
|
||||
@@ -492,6 +502,25 @@ __inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, __nv_bfloat16>(
|
||||
__builtin_unreachable(); // Suppress missing return statement warning
|
||||
}
|
||||
|
||||
// torch stable-ABI (headeronly) scalar types delegate to the CUDA-native
|
||||
// conversions, so libtorch_stable kernels dispatched on c10::BFloat16 /
|
||||
// c10::Half quantize correctly without manual casts.
|
||||
template <>
|
||||
__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, c10::BFloat16>(
|
||||
const c10::BFloat16& a, const float scale,
|
||||
const __nv_fp8_interpretation_t fp8_type) {
|
||||
return scaled_vec_conversion<uint8_t, __nv_bfloat16>(
|
||||
reinterpret_cast<const __nv_bfloat16&>(a), scale, fp8_type);
|
||||
}
|
||||
|
||||
template <>
|
||||
__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, c10::Half>(
|
||||
const c10::Half& a, const float scale,
|
||||
const __nv_fp8_interpretation_t fp8_type) {
|
||||
return scaled_vec_conversion<uint8_t, uint16_t>(
|
||||
reinterpret_cast<const uint16_t&>(a), scale, fp8_type);
|
||||
}
|
||||
|
||||
// float -> fp8
|
||||
template <>
|
||||
__inline__ __device__ uint8_t scaled_vec_conversion<uint8_t, float>(
|
||||
|
||||
Reference in New Issue
Block a user