[8/n] Migrate merge_attn_states, mamba, sampler to torch stable ABI (continued) (#43361)
Signed-off-by: Mikayla Gawarecki <mikaylagawarecki@gmail.com> Signed-off-by: Chris Leonard <chleonar@redhat.com> Co-authored-by: Mikayla Gawarecki <mikaylagawarecki@gmail.com> Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
co-authored by
Mikayla Gawarecki
Shengqi Chen
parent
05c50c721e
commit
284e6f543d
@@ -0,0 +1,337 @@
|
||||
#include <optional>
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
|
||||
#include "../torch_utils.h"
|
||||
#include "../dispatch_utils.h"
|
||||
#include <torch/headeronly/core/ScalarType.h>
|
||||
|
||||
#include "../../attention/attention_dtypes.h"
|
||||
#include "../../attention/attention_utils.cuh"
|
||||
#include "../../quantization/w8a8/fp8/common.cuh"
|
||||
|
||||
namespace vllm {
|
||||
|
||||
// Implements section 2.2 of https://www.arxiv.org/pdf/2501.01005
|
||||
// can be used to combine partial attention results (in the split-KV case)
|
||||
template <typename scalar_t, typename output_t, const uint NUM_THREADS,
|
||||
bool USE_FP8_OUTPUT>
|
||||
__global__ void merge_attn_states_kernel(
|
||||
output_t* output, float* output_lse, const scalar_t* prefix_output,
|
||||
const float* prefix_lse, const scalar_t* suffix_output,
|
||||
const float* suffix_lse, const uint num_tokens, const uint num_heads,
|
||||
const uint head_size, const uint prefix_head_stride,
|
||||
const uint output_head_stride, const uint prefix_num_tokens,
|
||||
const float* output_scale) {
|
||||
// Inputs always load 128-bit packs (pack_size elements of scalar_t).
|
||||
// Outputs store pack_size elements of output_t, which is smaller for FP8.
|
||||
using input_pack_t = uint4;
|
||||
using output_pack_t =
|
||||
std::conditional_t<USE_FP8_OUTPUT,
|
||||
std::conditional_t<sizeof(scalar_t) == 4, uint, uint2>,
|
||||
uint4>;
|
||||
const uint pack_size = 16 / sizeof(scalar_t);
|
||||
const uint threads_per_head = head_size / pack_size;
|
||||
|
||||
const uint global_idx = blockIdx.x * NUM_THREADS + threadIdx.x;
|
||||
const uint token_head_threads = num_tokens * num_heads * threads_per_head;
|
||||
|
||||
if (global_idx >= token_head_threads) return;
|
||||
|
||||
// global_idx -> token_idx + head_idx + pack_idx
|
||||
const uint token_head_idx = global_idx / threads_per_head;
|
||||
const uint pack_idx = global_idx % threads_per_head;
|
||||
|
||||
const uint token_idx = token_head_idx / num_heads;
|
||||
const uint head_idx = token_head_idx % num_heads;
|
||||
|
||||
const uint pack_offset = pack_idx * pack_size; // (0~15)*8, etc.
|
||||
const uint src_head_offset = token_idx * num_heads * prefix_head_stride +
|
||||
head_idx * prefix_head_stride;
|
||||
const uint dst_head_offset = token_idx * num_heads * output_head_stride +
|
||||
head_idx * output_head_stride;
|
||||
const scalar_t* prefix_head_ptr = prefix_output + src_head_offset;
|
||||
const scalar_t* suffix_head_ptr = suffix_output + src_head_offset;
|
||||
output_t* output_head_ptr = output + dst_head_offset;
|
||||
|
||||
// Pre-invert scale: multiplication is faster than division
|
||||
float fp8_scale_inv = 1.0f;
|
||||
if constexpr (USE_FP8_OUTPUT) {
|
||||
fp8_scale_inv = 1.0f / *output_scale;
|
||||
}
|
||||
|
||||
// If token_idx >= prefix_num_tokens, just copy from suffix
|
||||
if (token_idx >= prefix_num_tokens) {
|
||||
if (pack_offset < head_size) {
|
||||
input_pack_t s_out_pack = reinterpret_cast<const input_pack_t*>(
|
||||
suffix_head_ptr)[pack_offset / pack_size];
|
||||
|
||||
if constexpr (USE_FP8_OUTPUT) {
|
||||
output_t o_out_pack[pack_size];
|
||||
#pragma unroll
|
||||
for (uint i = 0; i < pack_size; ++i) {
|
||||
const float val =
|
||||
vllm::to_float(reinterpret_cast<const scalar_t*>(&s_out_pack)[i]);
|
||||
o_out_pack[i] =
|
||||
vllm::scaled_fp8_conversion<true, output_t>(val, fp8_scale_inv);
|
||||
}
|
||||
reinterpret_cast<output_pack_t*>(
|
||||
output_head_ptr)[pack_offset / pack_size] =
|
||||
*reinterpret_cast<output_pack_t*>(o_out_pack);
|
||||
} else {
|
||||
reinterpret_cast<output_pack_t*>(
|
||||
output_head_ptr)[pack_offset / pack_size] = s_out_pack;
|
||||
}
|
||||
}
|
||||
if (output_lse != nullptr && pack_idx == 0) {
|
||||
float s_lse = suffix_lse[head_idx * num_tokens + token_idx];
|
||||
output_lse[head_idx * num_tokens + token_idx] = s_lse;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// For tokens within prefix range, merge prefix and suffix
|
||||
float p_lse = prefix_lse[head_idx * num_tokens + token_idx];
|
||||
float s_lse = suffix_lse[head_idx * num_tokens + token_idx];
|
||||
p_lse = std::isinf(p_lse) ? -std::numeric_limits<float>::infinity() : p_lse;
|
||||
s_lse = std::isinf(s_lse) ? -std::numeric_limits<float>::infinity() : s_lse;
|
||||
|
||||
const float max_lse = fmaxf(p_lse, s_lse);
|
||||
|
||||
/* In certain edge cases, MLA can produce p_lse = s_lse = -inf;
|
||||
continuing the pipeline then yields NaN. Root cause: with chunked prefill
|
||||
a batch may be split into two chunks; if a request in that batch has no
|
||||
prefix hit, every LSE entry for that request's position is -inf, and at
|
||||
this moment we merge cross-attention at first. For now we simply emit
|
||||
prefix_output (expected to be all zeros) and prefix_lse (-inf) to fix
|
||||
this problem.
|
||||
*/
|
||||
if (std::isinf(max_lse)) {
|
||||
if (pack_offset < head_size) {
|
||||
input_pack_t p_out_pack = reinterpret_cast<const input_pack_t*>(
|
||||
prefix_head_ptr)[pack_offset / pack_size];
|
||||
|
||||
if constexpr (USE_FP8_OUTPUT) {
|
||||
// Convert prefix values to FP8 (since -inf means no data,
|
||||
// prefix_output is expected to be zeros)
|
||||
output_t o_out_pack[pack_size];
|
||||
#pragma unroll
|
||||
for (uint i = 0; i < pack_size; ++i) {
|
||||
const float val =
|
||||
vllm::to_float(reinterpret_cast<const scalar_t*>(&p_out_pack)[i]);
|
||||
o_out_pack[i] =
|
||||
vllm::scaled_fp8_conversion<true, output_t>(val, fp8_scale_inv);
|
||||
}
|
||||
reinterpret_cast<output_pack_t*>(
|
||||
output_head_ptr)[pack_offset / pack_size] =
|
||||
*reinterpret_cast<output_pack_t*>(o_out_pack);
|
||||
} else {
|
||||
reinterpret_cast<output_pack_t*>(
|
||||
output_head_ptr)[pack_offset / pack_size] = p_out_pack;
|
||||
}
|
||||
}
|
||||
// We only need to write to output_lse once per head.
|
||||
if (output_lse != nullptr && pack_idx == 0) {
|
||||
output_lse[head_idx * num_tokens + token_idx] = max_lse;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
p_lse = p_lse - max_lse;
|
||||
s_lse = s_lse - max_lse;
|
||||
const float p_se = expf(p_lse);
|
||||
const float s_se = expf(s_lse);
|
||||
const float out_se = p_se + s_se;
|
||||
const float p_scale = p_se / out_se;
|
||||
const float s_scale = s_se / out_se;
|
||||
|
||||
if (pack_offset < head_size) {
|
||||
input_pack_t p_out_pack = reinterpret_cast<const input_pack_t*>(
|
||||
prefix_head_ptr)[pack_offset / pack_size];
|
||||
input_pack_t s_out_pack = reinterpret_cast<const input_pack_t*>(
|
||||
suffix_head_ptr)[pack_offset / pack_size];
|
||||
|
||||
// Compute merged values in float32
|
||||
float o_out_f[pack_size];
|
||||
#pragma unroll
|
||||
for (uint i = 0; i < pack_size; ++i) {
|
||||
const float p_out_f =
|
||||
vllm::to_float(reinterpret_cast<const scalar_t*>(&p_out_pack)[i]);
|
||||
const float s_out_f =
|
||||
vllm::to_float(reinterpret_cast<const scalar_t*>(&s_out_pack)[i]);
|
||||
o_out_f[i] = p_out_f * p_scale + (s_out_f * s_scale);
|
||||
}
|
||||
|
||||
// Convert and store
|
||||
if constexpr (USE_FP8_OUTPUT) {
|
||||
output_t o_out_pack[pack_size];
|
||||
#pragma unroll
|
||||
for (uint i = 0; i < pack_size; ++i) {
|
||||
o_out_pack[i] = vllm::scaled_fp8_conversion<true, output_t>(
|
||||
o_out_f[i], fp8_scale_inv);
|
||||
}
|
||||
reinterpret_cast<output_pack_t*>(
|
||||
output_head_ptr)[pack_offset / pack_size] =
|
||||
*reinterpret_cast<output_pack_t*>(o_out_pack);
|
||||
} else {
|
||||
output_pack_t o_out_pack;
|
||||
#pragma unroll
|
||||
for (uint i = 0; i < pack_size; ++i) {
|
||||
vllm::from_float(reinterpret_cast<scalar_t*>(&o_out_pack)[i],
|
||||
o_out_f[i]);
|
||||
}
|
||||
reinterpret_cast<output_pack_t*>(
|
||||
output_head_ptr)[pack_offset / pack_size] = o_out_pack;
|
||||
}
|
||||
}
|
||||
// We only need to write to output_lse once per head.
|
||||
if (output_lse != nullptr && pack_idx == 0) {
|
||||
float out_lse = logf(out_se) + max_lse;
|
||||
output_lse[head_idx * num_tokens + token_idx] = out_lse;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
|
||||
// The following macro is used to dispatch the conversion function based on
|
||||
// the output data type. The FN is a macro that calls a function with
|
||||
// template<typename scalar_t>.
|
||||
#define DISPATCH_BY_SCALAR_DTYPE(scalar_dtype, fn) \
|
||||
{ \
|
||||
if (scalar_dtype == torch::headeronly::ScalarType::Float) { \
|
||||
fn(float); \
|
||||
} else if (scalar_dtype == torch::headeronly::ScalarType::Half) { \
|
||||
fn(uint16_t); \
|
||||
} else if (scalar_dtype == torch::headeronly::ScalarType::BFloat16) { \
|
||||
fn(__nv_bfloat16); \
|
||||
} else { \
|
||||
STD_TORCH_CHECK(false, "Unsupported data type of O: ", scalar_dtype); \
|
||||
} \
|
||||
}
|
||||
|
||||
#define LAUNCH_MERGE_ATTN_STATES(scalar_t, output_t, NUM_THREADS, \
|
||||
USE_FP8_OUTPUT) \
|
||||
{ \
|
||||
vllm::merge_attn_states_kernel<scalar_t, output_t, NUM_THREADS, \
|
||||
USE_FP8_OUTPUT> \
|
||||
<<<grid, block, 0, stream>>>( \
|
||||
reinterpret_cast<output_t*>(output.data_ptr()), output_lse_ptr, \
|
||||
reinterpret_cast<scalar_t*>(prefix_output.data_ptr()), \
|
||||
reinterpret_cast<float*>(prefix_lse.data_ptr()), \
|
||||
reinterpret_cast<scalar_t*>(suffix_output.data_ptr()), \
|
||||
reinterpret_cast<float*>(suffix_lse.data_ptr()), num_tokens, \
|
||||
num_heads, head_size, prefix_head_stride, output_head_stride, \
|
||||
prefix_num_tokens, output_scale_ptr); \
|
||||
}
|
||||
|
||||
/*@brief Merges the attention states from prefix and suffix
|
||||
* into the output tensor. NUM_TOKENS: n, NUM_HEADS: h, HEAD_SIZE: d
|
||||
*
|
||||
* @param output [n,h,d] The output tensor to store the merged attention states.
|
||||
* @param output_lse [h,n] Optional tensor to store the log-sum-exp values.
|
||||
* @param prefix_output [n,h,d] The prefix attention states.
|
||||
* @param prefix_lse [h,n] The log-sum-exp values for the prefix attention
|
||||
* states.
|
||||
* @param suffix_output [n,h,d] The suffix attention states.
|
||||
* @param suffix_lse [h,n] The log-sum-exp values for the suffix attention
|
||||
* states.
|
||||
* @param prefill_tokens_with_context Number of prefill tokens with context
|
||||
* For the first p tokens (0 <= token_idx < prefill_tokens_with_context), output
|
||||
* is computed by merging prefix_output and suffix_output. For remaining tokens
|
||||
* (prefill_tokens_with_context <= token_idx < n), output is copied directly
|
||||
* from suffix_output.
|
||||
* @param output_scale Optional scalar tensor for FP8 static quantization.
|
||||
* When provided, output must be FP8 dtype.
|
||||
*/
|
||||
template <typename scalar_t>
|
||||
void merge_attn_states_launcher(
|
||||
torch::stable::Tensor& output,
|
||||
std::optional<torch::stable::Tensor> output_lse,
|
||||
const torch::stable::Tensor& prefix_output,
|
||||
const torch::stable::Tensor& prefix_lse,
|
||||
const torch::stable::Tensor& suffix_output,
|
||||
const torch::stable::Tensor& suffix_lse,
|
||||
const std::optional<int64_t> prefill_tokens_with_context,
|
||||
const std::optional<torch::stable::Tensor>& output_scale) {
|
||||
constexpr uint NUM_THREADS = 128;
|
||||
const uint num_tokens = output.size(0);
|
||||
const uint num_heads = output.size(1);
|
||||
const uint head_size = output.size(2);
|
||||
const uint prefix_head_stride = prefix_output.stride(1);
|
||||
const uint output_head_stride = output.stride(1);
|
||||
// Thread mapping is based on input BF16 pack_size
|
||||
const uint pack_size = 16 / sizeof(scalar_t);
|
||||
STD_TORCH_CHECK(head_size % pack_size == 0,
|
||||
"headsize must be multiple of pack_size:", pack_size);
|
||||
|
||||
const uint prefix_num_tokens =
|
||||
prefill_tokens_with_context.has_value()
|
||||
? static_cast<uint>(prefill_tokens_with_context.value())
|
||||
: num_tokens;
|
||||
STD_TORCH_CHECK(prefix_num_tokens <= num_tokens,
|
||||
"prefix_num_tokens must be <= num_tokens");
|
||||
|
||||
float* output_lse_ptr = nullptr;
|
||||
if (output_lse.has_value()) {
|
||||
output_lse_ptr = output_lse.value().mutable_data_ptr<float>();
|
||||
}
|
||||
float* output_scale_ptr = nullptr;
|
||||
if (output_scale.has_value()) {
|
||||
output_scale_ptr = output_scale.value().mutable_data_ptr<float>();
|
||||
}
|
||||
// Process one pack elements per thread. for float, the
|
||||
// pack_size is 4 for half/bf16, the pack_size is 8.
|
||||
const uint threads_per_head = head_size / pack_size;
|
||||
const uint total_threads = num_tokens * num_heads * threads_per_head;
|
||||
|
||||
dim3 block(NUM_THREADS);
|
||||
dim3 grid((total_threads + NUM_THREADS - 1) / NUM_THREADS);
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
prefix_output.get_device_index());
|
||||
auto stream = get_current_cuda_stream();
|
||||
|
||||
if (output_scale.has_value()) {
|
||||
// FP8 output path - dispatch on output FP8 type
|
||||
VLLM_STABLE_DISPATCH_FP8_TYPES(
|
||||
output.scalar_type(), "merge_attn_states_fp8",
|
||||
[&] { LAUNCH_MERGE_ATTN_STATES(scalar_t, fp8_t, NUM_THREADS, true); });
|
||||
} else {
|
||||
// Original BF16/FP16/FP32 output path
|
||||
LAUNCH_MERGE_ATTN_STATES(scalar_t, scalar_t, NUM_THREADS, false);
|
||||
}
|
||||
}
|
||||
|
||||
#define CALL_MERGE_ATTN_STATES_LAUNCHER(scalar_t) \
|
||||
{ \
|
||||
merge_attn_states_launcher<scalar_t>( \
|
||||
output, output_lse, prefix_output, prefix_lse, suffix_output, \
|
||||
suffix_lse, prefill_tokens_with_context, output_scale); \
|
||||
}
|
||||
|
||||
void merge_attn_states(
|
||||
torch::stable::Tensor& output,
|
||||
std::optional<torch::stable::Tensor> output_lse,
|
||||
const torch::stable::Tensor& prefix_output,
|
||||
const torch::stable::Tensor& prefix_lse,
|
||||
const torch::stable::Tensor& suffix_output,
|
||||
const torch::stable::Tensor& suffix_lse,
|
||||
const std::optional<int64_t> prefill_tokens_with_context,
|
||||
const std::optional<torch::stable::Tensor>& output_scale) {
|
||||
if (output_scale.has_value()) {
|
||||
STD_TORCH_CHECK(
|
||||
output.scalar_type() == torch::headeronly::ScalarType::Float8_e4m3fn ||
|
||||
output.scalar_type() ==
|
||||
torch::headeronly::ScalarType::Float8_e4m3fnuz,
|
||||
"output must be FP8 when output_scale is provided, got: ",
|
||||
output.scalar_type());
|
||||
} else {
|
||||
STD_TORCH_CHECK(
|
||||
output.scalar_type() == prefix_output.scalar_type(), "output dtype (",
|
||||
output.scalar_type(), ") must match prefix_output dtype (",
|
||||
prefix_output.scalar_type(), ") when output_scale is not set");
|
||||
}
|
||||
// Always dispatch on prefix_output (input) dtype
|
||||
DISPATCH_BY_SCALAR_DTYPE(prefix_output.scalar_type(),
|
||||
CALL_MERGE_ATTN_STATES_LAUNCHER);
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
/******************************************************************************
|
||||
* Copyright (c) 2023, Tri Dao.
|
||||
******************************************************************************/
|
||||
// clang-format off
|
||||
// adapted from https://github.com/state-spaces/mamba/blob/main/csrc/selective_scan/selective_scan.h
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include <cuda_bf16.h>
|
||||
#else
|
||||
#include <hip/hip_bf16.h>
|
||||
#endif
|
||||
#include <cuda_fp16.h>
|
||||
|
||||
#include <torch/headeronly/util/Half.h>
|
||||
#include <torch/headeronly/util/BFloat16.h>
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
struct SSMParamsBase {
|
||||
using index_t = size_t;
|
||||
|
||||
int batch, dim, seqlen, dstate, n_groups;
|
||||
int dim_ngroups_ratio;
|
||||
bool is_variable_B;
|
||||
bool is_variable_C;
|
||||
int64_t null_block_id;
|
||||
|
||||
bool delta_softplus;
|
||||
bool cache_enabled;
|
||||
int block_size;
|
||||
|
||||
index_t A_d_stride;
|
||||
index_t A_dstate_stride;
|
||||
index_t B_batch_stride;
|
||||
index_t B_d_stride;
|
||||
index_t B_dstate_stride;
|
||||
index_t B_group_stride;
|
||||
index_t C_batch_stride;
|
||||
index_t C_d_stride;
|
||||
index_t C_dstate_stride;
|
||||
index_t C_group_stride;
|
||||
index_t u_batch_stride;
|
||||
index_t u_d_stride;
|
||||
index_t delta_batch_stride;
|
||||
index_t delta_d_stride;
|
||||
index_t z_batch_stride;
|
||||
index_t z_d_stride;
|
||||
index_t out_batch_stride;
|
||||
index_t out_d_stride;
|
||||
index_t out_z_batch_stride;
|
||||
index_t out_z_d_stride;
|
||||
index_t ssm_states_batch_stride;
|
||||
index_t ssm_states_dim_stride;
|
||||
index_t ssm_states_dstate_stride;
|
||||
index_t cache_indices_stride;
|
||||
|
||||
// Common data pointers.
|
||||
void *__restrict__ A_ptr;
|
||||
void *__restrict__ B_ptr;
|
||||
void *__restrict__ C_ptr;
|
||||
void *__restrict__ D_ptr;
|
||||
void *__restrict__ u_ptr;
|
||||
void *__restrict__ delta_ptr;
|
||||
void *__restrict__ delta_bias_ptr;
|
||||
void *__restrict__ out_ptr;
|
||||
void *__restrict__ ssm_states_ptr;
|
||||
void *__restrict__ z_ptr;
|
||||
void *__restrict__ out_z_ptr;
|
||||
|
||||
void *__restrict__ query_start_loc_ptr;
|
||||
void *__restrict__ cache_indices_ptr;
|
||||
void *__restrict__ has_initial_state_ptr;
|
||||
|
||||
void *__restrict__ block_idx_first_scheduled_token_ptr; // (batch,) - first block to write
|
||||
void *__restrict__ block_idx_last_scheduled_token_ptr; // (batch,) - last block to write
|
||||
void *__restrict__ initial_state_idx_ptr; // (batch,) - index of the initial state to use
|
||||
void *__restrict__ cu_chunk_seqlen_ptr; // (nchunks+1,) - cumulative chunk token offsets
|
||||
void *__restrict__ last_chunk_indices_ptr; // (batch,) - index of last chunk per sequence
|
||||
};
|
||||
|
||||
|
||||
|
||||
|
||||
#ifndef USE_ROCM
|
||||
|
||||
constexpr size_t custom_max(std::initializer_list<size_t> ilist)
|
||||
{
|
||||
return std::max(ilist);
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
constexpr T constexpr_min(T a, T b) {
|
||||
return std::min(a, b);
|
||||
}
|
||||
|
||||
#else
|
||||
constexpr size_t custom_max(std::initializer_list<size_t> ilist)
|
||||
{
|
||||
return *std::max_element(ilist.begin(), ilist.end());
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
constexpr T constexpr_min(T a, T b) {
|
||||
return a < b ? a : b;
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#define MAX_DSTATE 256
|
||||
|
||||
|
||||
inline __device__ float2 operator+(const float2 & a, const float2 & b){
|
||||
return {a.x + b.x, a.y + b.y};
|
||||
}
|
||||
|
||||
inline __device__ float3 operator+(const float3 &a, const float3 &b) {
|
||||
return {a.x + b.x, a.y + b.y, a.z + b.z};
|
||||
}
|
||||
|
||||
inline __device__ float4 operator+(const float4 & a, const float4 & b){
|
||||
return {a.x + b.x, a.y + b.y, a.z + b.z, a.w + b.w};
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template<int BYTES> struct BytesToType {};
|
||||
|
||||
template<> struct BytesToType<16> {
|
||||
using Type = uint4;
|
||||
static_assert(sizeof(Type) == 16);
|
||||
};
|
||||
|
||||
template<> struct BytesToType<8> {
|
||||
using Type = uint64_t;
|
||||
static_assert(sizeof(Type) == 8);
|
||||
};
|
||||
|
||||
template<> struct BytesToType<4> {
|
||||
using Type = uint32_t;
|
||||
static_assert(sizeof(Type) == 4);
|
||||
};
|
||||
|
||||
template<> struct BytesToType<2> {
|
||||
using Type = uint16_t;
|
||||
static_assert(sizeof(Type) == 2);
|
||||
};
|
||||
|
||||
template<> struct BytesToType<1> {
|
||||
using Type = uint8_t;
|
||||
static_assert(sizeof(Type) == 1);
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template<typename scalar_t, int N>
|
||||
struct Converter{
|
||||
static inline __device__ void to_float(const scalar_t (&src)[N], float (&dst)[N]) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < N; ++i) { dst[i] = src[i]; }
|
||||
}
|
||||
};
|
||||
|
||||
template<int N>
|
||||
struct Converter<torch::headeronly::Half, N>{
|
||||
static inline __device__ void to_float(const torch::headeronly::Half (&src)[N], float (&dst)[N]) {
|
||||
static_assert(N % 2 == 0);
|
||||
auto &src2 = reinterpret_cast<const half2 (&)[N / 2]>(src);
|
||||
auto &dst2 = reinterpret_cast<float2 (&)[N / 2]>(dst);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < N / 2; ++i) { dst2[i] = __half22float2(src2[i]); }
|
||||
}
|
||||
};
|
||||
|
||||
#if __CUDA_ARCH__ >= 800
|
||||
template<int N>
|
||||
struct Converter<torch::headeronly::BFloat16, N>{
|
||||
static inline __device__ void to_float(const torch::headeronly::BFloat16 (&src)[N], float (&dst)[N]) {
|
||||
static_assert(N % 2 == 0);
|
||||
auto &src2 = reinterpret_cast<const nv_bfloat162 (&)[N / 2]>(src);
|
||||
auto &dst2 = reinterpret_cast<float2 (&)[N / 2]>(dst);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < N / 2; ++i) { dst2[i] = __bfloat1622float2(src2[i]); }
|
||||
}
|
||||
};
|
||||
#endif
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
template<typename scalar_t> struct SSMScanOp;
|
||||
|
||||
template<>
|
||||
struct SSMScanOp<float> {
|
||||
__device__ __forceinline__ float2 operator()(const float2 &ab0, const float2 &ab1) const {
|
||||
return make_float2(ab1.x * ab0.x, ab1.x * ab0.y + ab1.y);
|
||||
}
|
||||
};
|
||||
|
||||
// A stateful callback functor that maintains a running prefix to be applied
|
||||
// during consecutive scan operations.
|
||||
template <typename scalar_t> struct SSMScanPrefixCallbackOp {
|
||||
using scan_t = std::conditional_t<std::is_same_v<scalar_t, float>, float2, float4>;
|
||||
scan_t running_prefix;
|
||||
// Constructor
|
||||
__device__ SSMScanPrefixCallbackOp(scan_t running_prefix_) : running_prefix(running_prefix_) {}
|
||||
// Callback operator to be entered by the first warp of threads in the block.
|
||||
// Thread-0 is responsible for returning a value for seeding the block-wide scan.
|
||||
__device__ scan_t operator()(scan_t block_aggregate) {
|
||||
scan_t old_prefix = running_prefix;
|
||||
running_prefix = SSMScanOp<scalar_t>()(running_prefix, block_aggregate);
|
||||
return old_prefix;
|
||||
}
|
||||
};
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template<typename Ktraits>
|
||||
inline __device__ void load_input(typename Ktraits::input_t *u,
|
||||
typename Ktraits::input_t (&u_vals)[Ktraits::kNItems],
|
||||
typename Ktraits::BlockLoadT::TempStorage &smem_load,
|
||||
int seqlen) {
|
||||
if constexpr (Ktraits::kIsEvenLen && !Ktraits::kVarlen) {
|
||||
auto& smem_load_vec = reinterpret_cast<typename Ktraits::BlockLoadVecT::TempStorage&>(smem_load);
|
||||
using vec_t = typename Ktraits::vec_t;
|
||||
typename Ktraits::BlockLoadVecT(smem_load_vec).Load(
|
||||
reinterpret_cast<vec_t*>(u),
|
||||
reinterpret_cast<vec_t(&)[Ktraits::kNLoads]>(u_vals)
|
||||
#ifdef USE_ROCM
|
||||
, Ktraits::kNThreads * Ktraits::kNLoads
|
||||
#endif
|
||||
|
||||
);
|
||||
} else {
|
||||
typename Ktraits::BlockLoadT(smem_load).Load(u, u_vals, seqlen, 0.f);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template<typename Ktraits>
|
||||
inline __device__ void load_weight(typename Ktraits::input_t *Bvar,
|
||||
typename Ktraits::weight_t (&B_vals)[Ktraits::kNItems],
|
||||
typename Ktraits::BlockLoadWeightT::TempStorage &smem_load_weight,
|
||||
int seqlen) {
|
||||
constexpr int kNItems = Ktraits::kNItems;
|
||||
typename Ktraits::input_t B_vals_load[kNItems];
|
||||
if constexpr (Ktraits::kIsEvenLen && !Ktraits::kVarlen) {
|
||||
auto& smem_load_weight_vec = reinterpret_cast<typename Ktraits::BlockLoadWeightVecT::TempStorage&>(smem_load_weight);
|
||||
using vec_t = typename Ktraits::vec_t;
|
||||
typename Ktraits::BlockLoadWeightVecT(smem_load_weight_vec).Load(
|
||||
reinterpret_cast<vec_t*>(Bvar),
|
||||
reinterpret_cast<vec_t(&)[Ktraits::kNLoads]>(B_vals_load)
|
||||
);
|
||||
} else {
|
||||
typename Ktraits::BlockLoadWeightT(smem_load_weight).Load(Bvar, B_vals_load, seqlen, 0.f);
|
||||
}
|
||||
// #pragma unroll
|
||||
// for (int i = 0; i < kNItems; ++i) { B_vals[i] = B_vals_load[i]; }
|
||||
Converter<typename Ktraits::input_t, kNItems>::to_float(B_vals_load, B_vals);
|
||||
}
|
||||
|
||||
template<typename Ktraits>
|
||||
inline __device__ void store_output(typename Ktraits::input_t *out,
|
||||
const float (&out_vals)[Ktraits::kNItems],
|
||||
typename Ktraits::BlockStoreT::TempStorage &smem_store,
|
||||
int seqlen) {
|
||||
typename Ktraits::input_t write_vals[Ktraits::kNItems];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < Ktraits::kNItems; ++i) { write_vals[i] = out_vals[i]; }
|
||||
if constexpr (Ktraits::kIsEvenLen && !Ktraits::kVarlen) {
|
||||
auto& smem_store_vec = reinterpret_cast<typename Ktraits::BlockStoreVecT::TempStorage&>(smem_store);
|
||||
using vec_t = typename Ktraits::vec_t;
|
||||
typename Ktraits::BlockStoreVecT(smem_store_vec).Store(
|
||||
reinterpret_cast<vec_t*>(out),
|
||||
reinterpret_cast<vec_t(&)[Ktraits::kNLoads]>(write_vals)
|
||||
);
|
||||
} else {
|
||||
typename Ktraits::BlockStoreT(smem_store).Store(out, write_vals, seqlen);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,822 @@
|
||||
// clang-format off
|
||||
// adapted from https://github.com/state-spaces/mamba/blob/main/csrc/selective_scan/selective_scan_fwd_kernel.cuh
|
||||
#include "../torch_utils.h"
|
||||
#include <torch/csrc/stable/macros.h>
|
||||
#include "selective_scan.h"
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include <cub/block/block_load.cuh>
|
||||
#include <cub/block/block_store.cuh>
|
||||
#include <cub/block/block_scan.cuh>
|
||||
#else
|
||||
#include <hipcub/hipcub.hpp>
|
||||
namespace cub = hipcub;
|
||||
#endif
|
||||
|
||||
#include "selective_scan.h"
|
||||
#include "static_switch.h"
|
||||
|
||||
template<int kNThreads_, int kNItems_, int kNRows_, bool kIsEvenLen_,
|
||||
bool kIsVariableB_, bool kIsVariableC_,
|
||||
bool kHasZ_, bool kVarlen_, typename input_t_, typename weight_t_, typename state_t_>
|
||||
struct Selective_Scan_fwd_kernel_traits {
|
||||
static_assert(kNItems_ % 4 == 0);
|
||||
using input_t = input_t_;
|
||||
using weight_t = weight_t_;
|
||||
using state_t = state_t_;
|
||||
static constexpr int kNThreads = kNThreads_;
|
||||
// Setting MinBlocksPerMP to be 3 (instead of 2) for 128 threads improves occupancy.
|
||||
static constexpr int kMinBlocks = kNThreads < 128 ? 5 : 3;
|
||||
static constexpr int kNItems = kNItems_;
|
||||
static constexpr int kNRows = kNRows_;
|
||||
static constexpr int kNBytes = sizeof(input_t);
|
||||
static_assert(kNBytes == 2 || kNBytes == 4);
|
||||
static constexpr int kNElts = kNBytes == 4 ? 4 : constexpr_min(8, kNItems);
|
||||
static_assert(kNItems % kNElts == 0);
|
||||
static constexpr int kNLoads = kNItems / kNElts;
|
||||
static constexpr bool kIsEvenLen = kVarlen_ ? false : kIsEvenLen_;
|
||||
static constexpr bool kIsVariableB = kIsVariableB_;
|
||||
static constexpr bool kIsVariableC = kIsVariableC_;
|
||||
static constexpr bool kHasZ = kHasZ_;
|
||||
static constexpr bool kVarlen = kVarlen_;
|
||||
|
||||
static constexpr bool kDirectIO = kVarlen_ ? false : kIsEvenLen && kNLoads == 1;
|
||||
static constexpr int kNLoadsIndex = kNItems / 4;
|
||||
using vec_t = typename BytesToType<kNBytes * kNElts>::Type;
|
||||
using scan_t = float2;
|
||||
using BlockLoadT = cub::BlockLoad<input_t, kNThreads, kNItems, cub::BLOCK_LOAD_WARP_TRANSPOSE>;
|
||||
using BlockLoadVecT = cub::BlockLoad<vec_t, kNThreads, kNLoads,
|
||||
!kDirectIO ? cub::BLOCK_LOAD_WARP_TRANSPOSE : cub::BLOCK_LOAD_DIRECT>;
|
||||
using BlockLoadWeightT = cub::BlockLoad<input_t, kNThreads, kNItems , cub::BLOCK_LOAD_WARP_TRANSPOSE>;
|
||||
using BlockLoadWeightVecT = cub::BlockLoad<vec_t, kNThreads, kNLoads ,
|
||||
!kDirectIO ? cub::BLOCK_LOAD_WARP_TRANSPOSE : cub::BLOCK_LOAD_DIRECT>;
|
||||
using BlockStoreT = cub::BlockStore<input_t, kNThreads, kNItems, cub::BLOCK_STORE_WARP_TRANSPOSE>;
|
||||
using BlockStoreVecT = cub::BlockStore<vec_t, kNThreads, kNLoads,
|
||||
!kDirectIO ? cub::BLOCK_STORE_WARP_TRANSPOSE : cub::BLOCK_STORE_DIRECT>;
|
||||
// using BlockScanT = cub::BlockScan<scan_t, kNThreads, cub::BLOCK_SCAN_RAKING_MEMOIZE>;
|
||||
// using BlockScanT = cub::BlockScan<scan_t, kNThreads, cub::BLOCK_SCAN_RAKING>;
|
||||
using BlockScanT = cub::BlockScan<scan_t, kNThreads, cub::BLOCK_SCAN_WARP_SCANS>;
|
||||
static constexpr int kSmemIOSize = custom_max({sizeof(typename BlockLoadT::TempStorage),
|
||||
sizeof(typename BlockLoadVecT::TempStorage),
|
||||
(int(kIsVariableB) + int(kIsVariableC)) * sizeof(typename BlockLoadWeightT::TempStorage),
|
||||
(int(kIsVariableB) + int(kIsVariableC)) * sizeof(typename BlockLoadWeightVecT::TempStorage),
|
||||
sizeof(typename BlockStoreT::TempStorage),
|
||||
sizeof(typename BlockStoreVecT::TempStorage)});
|
||||
static constexpr int kSmemSize = kSmemIOSize + sizeof(typename BlockScanT::TempStorage);
|
||||
};
|
||||
|
||||
template<typename Ktraits>
|
||||
__global__ __launch_bounds__(Ktraits::kNThreads, Ktraits::kMinBlocks)
|
||||
void selective_scan_fwd_kernel(SSMParamsBase params) {
|
||||
constexpr bool kIsVariableB = Ktraits::kIsVariableB;
|
||||
constexpr bool kIsVariableC = Ktraits::kIsVariableC;
|
||||
constexpr bool kHasZ = Ktraits::kHasZ;
|
||||
constexpr bool kVarlen = Ktraits::kVarlen;
|
||||
constexpr int kNItems = Ktraits::kNItems;
|
||||
constexpr int kNRows = Ktraits::kNRows;
|
||||
constexpr bool kDirectIO = Ktraits::kDirectIO;
|
||||
using input_t = typename Ktraits::input_t;
|
||||
using weight_t = typename Ktraits::weight_t;
|
||||
using scan_t = typename Ktraits::scan_t;
|
||||
|
||||
// Shared memory.
|
||||
extern __shared__ char smem_[];
|
||||
// cast to lvalue reference of expected type
|
||||
// char *smem_loadstorescan = smem_ + 2 * MAX_DSTATE * sizeof(weight_t);
|
||||
// auto& smem_load = reinterpret_cast<typename BlockLoadT::TempStorage&>(smem_ + 2 * MAX_DSTATE * sizeof(weight_t));
|
||||
// auto& smem_load = reinterpret_cast<typename BlockLoadT::TempStorage&>(smem_loadstorescan);
|
||||
auto& smem_load = reinterpret_cast<typename Ktraits::BlockLoadT::TempStorage&>(smem_);
|
||||
auto& smem_load_weight = reinterpret_cast<typename Ktraits::BlockLoadWeightT::TempStorage&>(smem_);
|
||||
auto& smem_load_weight1 = *reinterpret_cast<typename Ktraits::BlockLoadWeightT::TempStorage*>(smem_ + sizeof(typename Ktraits::BlockLoadWeightT::TempStorage));
|
||||
auto& smem_store = reinterpret_cast<typename Ktraits::BlockStoreT::TempStorage&>(smem_);
|
||||
auto& smem_scan = *reinterpret_cast<typename Ktraits::BlockScanT::TempStorage*>(smem_ + Ktraits::kSmemIOSize);
|
||||
// weight_t *smem_a = reinterpret_cast<weight_t *>(smem_ + smem_loadstorescan_size);
|
||||
// weight_t *smem_bc = reinterpret_cast<weight_t *>(smem_a + MAX_DSTATE);
|
||||
scan_t *smem_running_prefix = reinterpret_cast<scan_t *>(smem_ + Ktraits::kSmemSize);
|
||||
|
||||
const int batch_id = blockIdx.x;
|
||||
const int dim_id = blockIdx.y;
|
||||
const int group_id = dim_id / (params.dim_ngroups_ratio);
|
||||
int seqlen = params.seqlen;
|
||||
int sequence_start_index = batch_id;
|
||||
if constexpr (kVarlen){
|
||||
int *query_start_loc = reinterpret_cast<int *>(params.query_start_loc_ptr);
|
||||
sequence_start_index = query_start_loc[batch_id];
|
||||
seqlen = query_start_loc[batch_id + 1] - sequence_start_index;
|
||||
}
|
||||
const bool has_initial_state = params.has_initial_state_ptr == nullptr ? false
|
||||
: reinterpret_cast<bool *>(params.has_initial_state_ptr)[batch_id];
|
||||
|
||||
const int* cache_indices = params.cache_indices_ptr == nullptr ? nullptr
|
||||
: reinterpret_cast<int *>(params.cache_indices_ptr);
|
||||
int cache_index;
|
||||
if (cache_indices == nullptr) {
|
||||
cache_index = batch_id;
|
||||
} else if (params.cache_enabled) {
|
||||
const int* initial_state_idx = reinterpret_cast<const int*>(params.initial_state_idx_ptr);
|
||||
cache_index = cache_indices[batch_id * params.cache_indices_stride + initial_state_idx[batch_id]];
|
||||
} else {
|
||||
cache_index = cache_indices[batch_id];
|
||||
}
|
||||
// Skip batch entries whose cache index maps to the null block (padding).
|
||||
if (cache_indices != nullptr && cache_index == params.null_block_id){
|
||||
return;
|
||||
}
|
||||
input_t *u = reinterpret_cast<input_t *>(params.u_ptr) + sequence_start_index * params.u_batch_stride
|
||||
+ dim_id * kNRows * params.u_d_stride;
|
||||
input_t *delta = reinterpret_cast<input_t *>(params.delta_ptr) + sequence_start_index * params.delta_batch_stride
|
||||
+ dim_id * kNRows * params.delta_d_stride;
|
||||
weight_t *A = reinterpret_cast<weight_t *>(params.A_ptr) + dim_id * kNRows * params.A_d_stride;
|
||||
weight_t *B = reinterpret_cast<weight_t *>(params.B_ptr) + dim_id * kNRows * params.B_d_stride;
|
||||
input_t *Bvar = reinterpret_cast<input_t *>(params.B_ptr) + sequence_start_index * params.B_batch_stride + group_id * params.B_group_stride;
|
||||
weight_t *C = reinterpret_cast<weight_t *>(params.C_ptr) + dim_id * kNRows * params.C_d_stride;
|
||||
input_t *Cvar = reinterpret_cast<input_t *>(params.C_ptr) + sequence_start_index * params.C_batch_stride + group_id * params.C_group_stride;
|
||||
|
||||
typename Ktraits::state_t *ssm_states;
|
||||
if (params.cache_enabled) {
|
||||
// APC mode: ssm_states points to the base, we'll use absolute cache slots later
|
||||
ssm_states = reinterpret_cast<typename Ktraits::state_t *>(params.ssm_states_ptr) +
|
||||
dim_id * kNRows * params.ssm_states_dim_stride;
|
||||
} else {
|
||||
// Non-APC mode: offset by cache_index as before
|
||||
ssm_states = reinterpret_cast<typename Ktraits::state_t *>(params.ssm_states_ptr) +
|
||||
cache_index * params.ssm_states_batch_stride +
|
||||
dim_id * kNRows * params.ssm_states_dim_stride;
|
||||
}
|
||||
|
||||
float D_val[kNRows] = {0};
|
||||
if (params.D_ptr != nullptr) {
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kNRows; ++r) {
|
||||
D_val[r] = reinterpret_cast<float *>(params.D_ptr)[dim_id * kNRows + r];
|
||||
}
|
||||
}
|
||||
float delta_bias[kNRows] = {0};
|
||||
if (params.delta_bias_ptr != nullptr) {
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kNRows; ++r) {
|
||||
delta_bias[r] = reinterpret_cast<float *>(params.delta_bias_ptr)[dim_id * kNRows + r];
|
||||
}
|
||||
}
|
||||
|
||||
// Use block_size for chunking when APC is enabled, otherwise use 2048 for backwards compatibility
|
||||
const int block_size = params.cache_enabled ? params.block_size : 2048;
|
||||
|
||||
const int* batch_cache_indices = cache_indices != nullptr ?
|
||||
cache_indices + batch_id * params.cache_indices_stride : nullptr;
|
||||
const int* block_idx_first_scheduled = params.block_idx_first_scheduled_token_ptr != nullptr ?
|
||||
reinterpret_cast<const int*>(params.block_idx_first_scheduled_token_ptr) : nullptr;
|
||||
const int* block_idx_last_scheduled = params.block_idx_last_scheduled_token_ptr != nullptr ?
|
||||
reinterpret_cast<const int*>(params.block_idx_last_scheduled_token_ptr) : nullptr;
|
||||
const int* initial_state_idx = params.initial_state_idx_ptr != nullptr ?
|
||||
reinterpret_cast<const int*>(params.initial_state_idx_ptr) : nullptr;
|
||||
const int* cu_chunk_seqlen = params.cu_chunk_seqlen_ptr != nullptr ?
|
||||
reinterpret_cast<const int*>(params.cu_chunk_seqlen_ptr) : nullptr;
|
||||
const int* last_chunk_indices = params.last_chunk_indices_ptr != nullptr ?
|
||||
reinterpret_cast<const int*>(params.last_chunk_indices_ptr) : nullptr;
|
||||
|
||||
const size_t load_cache_slot = params.cache_enabled && batch_cache_indices != nullptr ? batch_cache_indices[initial_state_idx[batch_id]] : cache_index;
|
||||
|
||||
const int block_idx_first = (params.cache_enabled && block_idx_first_scheduled != nullptr) ?
|
||||
block_idx_first_scheduled[batch_id] : 0;
|
||||
|
||||
// Determine chunk boundaries from pre-computed metadata (APC mode)
|
||||
// or fall back to simple block_size chunking.
|
||||
int first_chunk_idx, n_chunks;
|
||||
int current_position;
|
||||
|
||||
if (cu_chunk_seqlen != nullptr && last_chunk_indices != nullptr) {
|
||||
const int last_chunk_idx = last_chunk_indices[batch_id];
|
||||
first_chunk_idx = (batch_id == 0) ? 0 : last_chunk_indices[batch_id - 1] + 1;
|
||||
n_chunks = last_chunk_idx - first_chunk_idx + 1;
|
||||
// Derive current_position: if the first chunk is partial (fills remainder
|
||||
// of a started block), offset into the block accordingly.
|
||||
const int first_chunk_tokens = cu_chunk_seqlen[first_chunk_idx + 1] - cu_chunk_seqlen[first_chunk_idx];
|
||||
const int chunk_start_offset = (n_chunks > 1 && first_chunk_tokens < block_size)
|
||||
? (block_size - first_chunk_tokens) : 0;
|
||||
current_position = block_idx_first * block_size + chunk_start_offset;
|
||||
} else {
|
||||
first_chunk_idx = 0;
|
||||
n_chunks = (seqlen + block_size - 1) / block_size;
|
||||
current_position = 0;
|
||||
}
|
||||
|
||||
int tokens_processed = 0;
|
||||
|
||||
for (int chunk = 0; chunk < n_chunks; ++chunk) {
|
||||
const int chunk_tokens = (cu_chunk_seqlen != nullptr)
|
||||
? cu_chunk_seqlen[first_chunk_idx + chunk + 1] - cu_chunk_seqlen[first_chunk_idx + chunk]
|
||||
: min(block_size, seqlen - tokens_processed);
|
||||
if (chunk_tokens <= 0) break;
|
||||
input_t u_vals[kNRows][kNItems], delta_vals_load[kNRows][kNItems];
|
||||
|
||||
__syncthreads();
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kNRows; ++r) {
|
||||
if constexpr (!kDirectIO) {
|
||||
if (r > 0) { __syncthreads(); }
|
||||
}
|
||||
load_input<Ktraits>(u + r * params.u_d_stride, u_vals[r], smem_load, chunk_tokens);
|
||||
if constexpr (!kDirectIO) { __syncthreads(); }
|
||||
load_input<Ktraits>(delta + r * params.delta_d_stride, delta_vals_load[r], smem_load, chunk_tokens);
|
||||
}
|
||||
u += chunk_tokens;
|
||||
delta += chunk_tokens;
|
||||
|
||||
float delta_vals[kNRows][kNItems], delta_u_vals[kNRows][kNItems], out_vals[kNRows][kNItems];
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kNRows; ++r) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kNItems; ++i) {
|
||||
float u_val = float(u_vals[r][i]);
|
||||
delta_vals[r][i] = float(delta_vals_load[r][i]) + delta_bias[r];
|
||||
if (params.delta_softplus) {
|
||||
delta_vals[r][i] = delta_vals[r][i] <= 20.f ? log1pf(expf(delta_vals[r][i])) : delta_vals[r][i];
|
||||
}
|
||||
delta_u_vals[r][i] = delta_vals[r][i] * u_val;
|
||||
out_vals[r][i] = D_val[r] * u_val;
|
||||
}
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
for (int state_idx = 0; state_idx < params.dstate; ++state_idx) {
|
||||
weight_t A_val[kNRows];
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kNRows; ++r) {
|
||||
A_val[r] = A[state_idx * params.A_dstate_stride + r * params.A_d_stride];
|
||||
// Multiply the real part of A with LOG2E so we can use exp2f instead of expf.
|
||||
constexpr float kLog2e = M_LOG2E;
|
||||
A_val[r] *= kLog2e;
|
||||
}
|
||||
// This variable holds B * C if both B and C are constant across seqlen. If only B varies
|
||||
// across seqlen, this holds C. If only C varies across seqlen, this holds B.
|
||||
// If both B and C vary, this is unused.
|
||||
weight_t BC_val[kNRows];
|
||||
weight_t B_vals[kNItems], C_vals[kNItems];
|
||||
if constexpr (kIsVariableB) {
|
||||
load_weight<Ktraits>(Bvar + state_idx * params.B_dstate_stride, B_vals,
|
||||
smem_load_weight, chunk_tokens);
|
||||
if constexpr (!kIsVariableC) {
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kNRows; ++r) {
|
||||
BC_val[r] = C[state_idx * params.C_dstate_stride + r * params.C_d_stride];
|
||||
}
|
||||
}
|
||||
}
|
||||
if constexpr (kIsVariableC) {
|
||||
auto &smem_load_weight_C = !kIsVariableB ? smem_load_weight : smem_load_weight1;
|
||||
load_weight<Ktraits>(Cvar + state_idx * params.C_dstate_stride, C_vals,
|
||||
smem_load_weight_C, chunk_tokens);
|
||||
if constexpr (!kIsVariableB) {
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kNRows; ++r) {
|
||||
BC_val[r] = B[state_idx * params.B_dstate_stride + r * params.B_d_stride];
|
||||
}
|
||||
}
|
||||
}
|
||||
if constexpr (!kIsVariableB && !kIsVariableC) {
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kNRows; ++r) {
|
||||
BC_val[r] = B[state_idx * params.B_dstate_stride + r * params.B_d_stride] * C[state_idx * params.C_dstate_stride + r * params.C_d_stride];
|
||||
}
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kNRows; ++r) {
|
||||
if (r > 0) { __syncthreads(); } // Scan could be using the same smem
|
||||
scan_t thread_data[kNItems];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kNItems; ++i) {
|
||||
thread_data[i] = make_float2(exp2f(delta_vals[r][i] * A_val[r]),
|
||||
!kIsVariableB ? delta_u_vals[r][i] : B_vals[i] * delta_u_vals[r][i]);
|
||||
if (threadIdx.x * kNItems + i >= chunk_tokens) {
|
||||
thread_data[i] = make_float2(1.f, 0.f);
|
||||
}
|
||||
}
|
||||
// Initialize running total
|
||||
scan_t running_prefix;
|
||||
if (chunk > 0) {
|
||||
running_prefix = smem_running_prefix[state_idx + r * MAX_DSTATE];
|
||||
} else {
|
||||
// Load initial state
|
||||
if (params.cache_enabled && has_initial_state && batch_cache_indices != nullptr) {
|
||||
size_t state_offset = load_cache_slot * params.ssm_states_batch_stride +
|
||||
r * params.ssm_states_dim_stride +
|
||||
state_idx * params.ssm_states_dstate_stride;
|
||||
running_prefix = make_float2(1.0, float(ssm_states[state_offset]));
|
||||
} else if (has_initial_state) {
|
||||
// Non-APC mode: load from current batch position
|
||||
running_prefix = make_float2(1.0, float(ssm_states[state_idx * params.ssm_states_dstate_stride]));
|
||||
} else {
|
||||
// No initial state
|
||||
running_prefix = make_float2(1.0, 0.0);
|
||||
}
|
||||
}
|
||||
|
||||
SSMScanPrefixCallbackOp<weight_t> prefix_op(running_prefix);
|
||||
typename Ktraits::BlockScanT(smem_scan).InclusiveScan(
|
||||
thread_data, thread_data, SSMScanOp<weight_t>(), prefix_op
|
||||
);
|
||||
// There's a syncthreads in the scan op, so we don't need to sync here.
|
||||
// Unless there's only 1 warp, but then it's the same thread (0) reading and writing.
|
||||
if (threadIdx.x == 0) {
|
||||
smem_running_prefix[state_idx + r * MAX_DSTATE] = prefix_op.running_prefix;
|
||||
|
||||
// Store state at the end of each aligned chunk when cache is enabled
|
||||
if (params.cache_enabled && batch_cache_indices != nullptr) {
|
||||
size_t cache_slot;
|
||||
if (chunk == n_chunks - 1) {
|
||||
cache_slot = batch_cache_indices[block_idx_last_scheduled[batch_id]];
|
||||
} else {
|
||||
const int block_idx_completed = (current_position + chunk_tokens - 1) / block_size;
|
||||
cache_slot = batch_cache_indices[block_idx_completed];
|
||||
}
|
||||
|
||||
size_t state_offset = cache_slot * params.ssm_states_batch_stride +
|
||||
r * params.ssm_states_dim_stride +
|
||||
state_idx * params.ssm_states_dstate_stride;
|
||||
|
||||
ssm_states[state_offset] = typename Ktraits::state_t(prefix_op.running_prefix.y);
|
||||
} else if (!params.cache_enabled && chunk == n_chunks - 1) {
|
||||
// Non-APC mode: store only final state at current batch position
|
||||
ssm_states[state_idx * params.ssm_states_dstate_stride] = typename Ktraits::state_t(prefix_op.running_prefix.y);
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kNItems; ++i) {
|
||||
const weight_t C_val = !kIsVariableC
|
||||
? BC_val[r]
|
||||
: (!kIsVariableB ? BC_val[r] * C_vals[i] : C_vals[i]);
|
||||
out_vals[r][i] += thread_data[i].y * C_val;
|
||||
}
|
||||
}
|
||||
}
|
||||
input_t *out = reinterpret_cast<input_t *>(params.out_ptr) + sequence_start_index * params.out_batch_stride
|
||||
+ dim_id * kNRows * params.out_d_stride + tokens_processed;
|
||||
__syncthreads();
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kNRows; ++r) {
|
||||
if constexpr (!kDirectIO) {
|
||||
if (r > 0) { __syncthreads(); }
|
||||
}
|
||||
store_output<Ktraits>(out + r * params.out_d_stride, out_vals[r], smem_store, chunk_tokens);
|
||||
}
|
||||
|
||||
if constexpr (kHasZ) {
|
||||
input_t *z = reinterpret_cast<input_t *>(params.z_ptr) + sequence_start_index * params.z_batch_stride
|
||||
+ dim_id * kNRows * params.z_d_stride + tokens_processed;
|
||||
input_t *out_z = reinterpret_cast<input_t *>(params.out_z_ptr) + sequence_start_index * params.out_z_batch_stride
|
||||
+ dim_id * kNRows * params.out_z_d_stride + tokens_processed;
|
||||
#pragma unroll
|
||||
for (int r = 0; r < kNRows; ++r) {
|
||||
input_t z_vals[kNItems];
|
||||
__syncthreads();
|
||||
load_input<Ktraits>(z + r * params.z_d_stride, z_vals, smem_load, chunk_tokens);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < kNItems; ++i) {
|
||||
float z_val = z_vals[i];
|
||||
out_vals[r][i] *= z_val / (1 + expf(-z_val));
|
||||
}
|
||||
__syncthreads();
|
||||
store_output<Ktraits>(out_z + r * params.out_z_d_stride, out_vals[r], smem_store, chunk_tokens);
|
||||
}
|
||||
}
|
||||
|
||||
Bvar += chunk_tokens;
|
||||
Cvar += chunk_tokens;
|
||||
|
||||
tokens_processed += chunk_tokens;
|
||||
current_position += chunk_tokens;
|
||||
}
|
||||
}
|
||||
|
||||
template<int kNThreads, int kNItems, typename input_t, typename weight_t, typename state_t>
|
||||
void selective_scan_fwd_launch(SSMParamsBase ¶ms, cudaStream_t stream) {
|
||||
// Only kNRows == 1 is tested for now, which ofc doesn't differ from previously when we had each block
|
||||
// processing 1 row.
|
||||
constexpr int kNRows = 1;
|
||||
// kIsVariableB, kIsVariableC and kHasZ are all set to True to reduce binary size
|
||||
constexpr bool kIsVariableB = true;
|
||||
constexpr bool kIsVariableC = true;
|
||||
BOOL_SWITCH(params.seqlen % (kNThreads * kNItems) == 0, kIsEvenLen, [&] {
|
||||
BOOL_SWITCH(params.z_ptr != nullptr , kHasZ, [&] {
|
||||
BOOL_SWITCH(params.query_start_loc_ptr != nullptr , kVarlen, [&] {
|
||||
using Ktraits = Selective_Scan_fwd_kernel_traits<kNThreads, kNItems, kNRows, kIsEvenLen, kIsVariableB, kIsVariableC, kHasZ, kVarlen, input_t, weight_t, state_t>;
|
||||
constexpr int kSmemSize = Ktraits::kSmemSize + kNRows * MAX_DSTATE * sizeof(typename Ktraits::scan_t);
|
||||
dim3 grid(params.batch, params.dim / kNRows);
|
||||
auto kernel = &selective_scan_fwd_kernel<Ktraits>;
|
||||
if (kSmemSize >= 48 * 1024) {
|
||||
#ifdef USE_ROCM
|
||||
STD_CUDA_CHECK(hipFuncSetAttribute(
|
||||
reinterpret_cast<const void*>(kernel), hipFuncAttributeMaxDynamicSharedMemorySize, kSmemSize));
|
||||
#else
|
||||
STD_CUDA_CHECK(cudaFuncSetAttribute(
|
||||
kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, kSmemSize));
|
||||
#endif
|
||||
}
|
||||
kernel<<<grid, Ktraits::kNThreads, kSmemSize, stream>>>(params);
|
||||
STD_CUDA_KERNEL_LAUNCH_CHECK();
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
template<typename input_t, typename weight_t, typename state_t>
|
||||
void selective_scan_fwd_cuda(SSMParamsBase ¶ms, cudaStream_t stream) {
|
||||
|
||||
#ifndef USE_ROCM
|
||||
if (params.cache_enabled && params.block_size == 1024) {
|
||||
selective_scan_fwd_launch<64, 16, input_t, weight_t, state_t>(params, stream);
|
||||
} else if (params.seqlen <= 128) {
|
||||
selective_scan_fwd_launch<32, 4, input_t, weight_t, state_t>(params, stream);
|
||||
} else if (params.seqlen <= 256) {
|
||||
selective_scan_fwd_launch<32, 8, input_t, weight_t, state_t>(params, stream);
|
||||
} else if (params.seqlen <= 512) {
|
||||
selective_scan_fwd_launch<32, 16, input_t, weight_t, state_t>(params, stream);
|
||||
} else if (params.seqlen <= 1024) {
|
||||
selective_scan_fwd_launch<64, 16, input_t, weight_t, state_t>(params, stream);
|
||||
} else {
|
||||
selective_scan_fwd_launch<128, 16, input_t, weight_t, state_t>(params, stream);
|
||||
}
|
||||
#else
|
||||
if (params.cache_enabled && params.block_size == 1024) {
|
||||
selective_scan_fwd_launch<64, 16, input_t, weight_t, state_t>(params, stream);
|
||||
} else if (params.seqlen <= 256) {
|
||||
selective_scan_fwd_launch<64, 4, input_t, weight_t, state_t>(params, stream);
|
||||
} else if (params.seqlen <= 512) {
|
||||
selective_scan_fwd_launch<64, 8, input_t, weight_t, state_t>(params, stream);
|
||||
} else if (params.seqlen <= 1024) {
|
||||
selective_scan_fwd_launch<64, 16, input_t, weight_t, state_t>(params, stream);
|
||||
} else {
|
||||
selective_scan_fwd_launch<128, 16, input_t, weight_t, state_t>(params, stream);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
template void selective_scan_fwd_cuda<torch::headeronly::BFloat16, float, torch::headeronly::BFloat16>(SSMParamsBase ¶ms, cudaStream_t stream);
|
||||
template void selective_scan_fwd_cuda<torch::headeronly::BFloat16, float, float>(SSMParamsBase ¶ms, cudaStream_t stream);
|
||||
template void selective_scan_fwd_cuda<torch::headeronly::Half, float, torch::headeronly::Half>(SSMParamsBase ¶ms, cudaStream_t stream);
|
||||
template void selective_scan_fwd_cuda<torch::headeronly::Half, float, float>(SSMParamsBase ¶ms, cudaStream_t stream);
|
||||
template void selective_scan_fwd_cuda<float, float, float>(SSMParamsBase ¶ms, cudaStream_t stream);
|
||||
|
||||
#define CHECK_SHAPE(x, ...) STD_TORCH_CHECK(x.sizes().equals(torch::headeronly::IntHeaderOnlyArrayRef({__VA_ARGS__})), #x " must have shape (" #__VA_ARGS__ ")")
|
||||
|
||||
#define DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(ITYPE, STYPE, NAME, ...) \
|
||||
if (ITYPE == torch::headeronly::ScalarType::Half) { \
|
||||
using input_t = torch::headeronly::Half; \
|
||||
using weight_t = float; \
|
||||
if (STYPE == torch::headeronly::ScalarType::Half) { \
|
||||
using state_t = torch::headeronly::Half; \
|
||||
__VA_ARGS__(); \
|
||||
} else if (STYPE == torch::headeronly::ScalarType::Float) { \
|
||||
using state_t = float; \
|
||||
__VA_ARGS__(); \
|
||||
} else { \
|
||||
STD_TORCH_CHECK(false, #NAME " not implemented for state type '", STYPE, "'"); \
|
||||
} \
|
||||
} else if (ITYPE == torch::headeronly::ScalarType::BFloat16) { \
|
||||
using input_t = torch::headeronly::BFloat16; \
|
||||
using weight_t = float; \
|
||||
if (STYPE == torch::headeronly::ScalarType::BFloat16) { \
|
||||
using state_t = torch::headeronly::BFloat16; \
|
||||
__VA_ARGS__(); \
|
||||
} else if (STYPE == torch::headeronly::ScalarType::Float) { \
|
||||
using state_t = float; \
|
||||
__VA_ARGS__(); \
|
||||
} else { \
|
||||
STD_TORCH_CHECK(false, #NAME " not implemented for state type '", STYPE, "'"); \
|
||||
} \
|
||||
} else if (ITYPE == torch::headeronly::ScalarType::Float) { \
|
||||
using input_t = float; \
|
||||
using weight_t = float; \
|
||||
using state_t = float; \
|
||||
__VA_ARGS__(); \
|
||||
} else { \
|
||||
STD_TORCH_CHECK(false, #NAME " not implemented for input type '", ITYPE, "'"); \
|
||||
}
|
||||
|
||||
|
||||
template<typename input_t, typename weight_t, typename state_t>
|
||||
void selective_scan_fwd_cuda(SSMParamsBase ¶ms, cudaStream_t stream);
|
||||
|
||||
void set_ssm_params_fwd(SSMParamsBase ¶ms,
|
||||
// sizes
|
||||
const size_t batch,
|
||||
const size_t dim,
|
||||
const size_t seqlen,
|
||||
const size_t dstate,
|
||||
const size_t n_groups,
|
||||
const bool is_variable_B,
|
||||
const bool is_variable_C,
|
||||
// device pointers
|
||||
const torch::stable::Tensor u,
|
||||
const torch::stable::Tensor delta,
|
||||
const torch::stable::Tensor A,
|
||||
const torch::stable::Tensor B,
|
||||
const torch::stable::Tensor C,
|
||||
const torch::stable::Tensor out,
|
||||
const torch::stable::Tensor z,
|
||||
const torch::stable::Tensor out_z,
|
||||
const std::optional<torch::stable::Tensor>& D,
|
||||
const std::optional<torch::stable::Tensor>& delta_bias,
|
||||
const torch::stable::Tensor ssm_states,
|
||||
bool has_z,
|
||||
bool delta_softplus,
|
||||
const std::optional<torch::stable::Tensor>& query_start_loc,
|
||||
const std::optional<torch::stable::Tensor>& cache_indices,
|
||||
const std::optional<torch::stable::Tensor>& has_initial_state,
|
||||
bool varlen,
|
||||
int64_t null_block_id,
|
||||
int64_t block_size,
|
||||
const std::optional<torch::stable::Tensor> &block_idx_first_scheduled_token,
|
||||
const std::optional<torch::stable::Tensor> &block_idx_last_scheduled_token,
|
||||
const std::optional<torch::stable::Tensor> &initial_state_idx,
|
||||
const std::optional<torch::stable::Tensor> &cu_chunk_seqlen,
|
||||
const std::optional<torch::stable::Tensor> &last_chunk_indices) {
|
||||
|
||||
// Reset the parameters
|
||||
memset(¶ms, 0, sizeof(params));
|
||||
|
||||
params.batch = batch;
|
||||
params.dim = dim;
|
||||
params.seqlen = seqlen;
|
||||
params.dstate = dstate;
|
||||
params.n_groups = n_groups;
|
||||
params.dim_ngroups_ratio = dim / n_groups;
|
||||
params.null_block_id = null_block_id;
|
||||
|
||||
params.delta_softplus = delta_softplus;
|
||||
|
||||
params.is_variable_B = is_variable_B;
|
||||
params.is_variable_C = is_variable_C;
|
||||
|
||||
// Set the pointers and strides.
|
||||
params.u_ptr = u.data_ptr();
|
||||
params.delta_ptr = delta.data_ptr();
|
||||
params.A_ptr = A.data_ptr();
|
||||
params.B_ptr = B.data_ptr();
|
||||
params.C_ptr = C.data_ptr();
|
||||
params.D_ptr = D.has_value() ? D.value().data_ptr() : nullptr;
|
||||
params.delta_bias_ptr = delta_bias.has_value() ? delta_bias.value().data_ptr() : nullptr;
|
||||
params.out_ptr = out.data_ptr();
|
||||
params.ssm_states_ptr = ssm_states.data_ptr();
|
||||
params.z_ptr = has_z ? z.data_ptr() : nullptr;
|
||||
params.out_z_ptr = has_z ? out_z.data_ptr() : nullptr;
|
||||
params.query_start_loc_ptr = query_start_loc.has_value() ? query_start_loc.value().data_ptr() : nullptr;
|
||||
params.cache_indices_ptr = cache_indices.has_value() ? cache_indices.value().data_ptr() : nullptr;
|
||||
params.has_initial_state_ptr = has_initial_state.has_value() ? has_initial_state.value().data_ptr() : nullptr;
|
||||
|
||||
// Set cache parameters - cache is enabled if we have direct cache writing params
|
||||
params.cache_enabled = block_idx_first_scheduled_token.has_value();
|
||||
params.block_size = static_cast<int>(block_size);
|
||||
|
||||
// Set direct cache writing pointers
|
||||
params.block_idx_first_scheduled_token_ptr = block_idx_first_scheduled_token.has_value() ? block_idx_first_scheduled_token.value().data_ptr() : nullptr;
|
||||
params.block_idx_last_scheduled_token_ptr = block_idx_last_scheduled_token.has_value() ? block_idx_last_scheduled_token.value().data_ptr() : nullptr;
|
||||
params.initial_state_idx_ptr = initial_state_idx.has_value() ? initial_state_idx.value().data_ptr() : nullptr;
|
||||
params.cu_chunk_seqlen_ptr = cu_chunk_seqlen.has_value() ? cu_chunk_seqlen.value().data_ptr() : nullptr;
|
||||
params.last_chunk_indices_ptr = last_chunk_indices.has_value() ? last_chunk_indices.value().data_ptr() : nullptr;
|
||||
|
||||
// All stride are in elements, not bytes.
|
||||
params.A_d_stride = A.stride(0);
|
||||
params.A_dstate_stride = A.stride(1);
|
||||
|
||||
if (varlen){
|
||||
params.B_batch_stride = B.stride(2);
|
||||
params.B_group_stride = B.stride(0);
|
||||
params.B_dstate_stride = B.stride(1);
|
||||
params.C_batch_stride = C.stride(2);
|
||||
params.C_group_stride = C.stride(0);
|
||||
params.C_dstate_stride = C.stride(1);
|
||||
|
||||
params.u_batch_stride = u.stride(1);
|
||||
params.u_d_stride = u.stride(0);
|
||||
params.delta_batch_stride = delta.stride(1);
|
||||
params.delta_d_stride = delta.stride(0);
|
||||
if (has_z) {
|
||||
params.z_batch_stride = z.stride(1);
|
||||
params.z_d_stride = z.stride(0);
|
||||
params.out_z_batch_stride = out_z.stride(1);
|
||||
params.out_z_d_stride = out_z.stride(0);
|
||||
}
|
||||
params.out_batch_stride = out.stride(1);
|
||||
params.out_d_stride = out.stride(0);
|
||||
|
||||
params.ssm_states_batch_stride = ssm_states.stride(0);
|
||||
params.ssm_states_dim_stride = ssm_states.stride(1);
|
||||
params.ssm_states_dstate_stride = ssm_states.stride(2);
|
||||
|
||||
params.cache_indices_stride = cache_indices.has_value() ? cache_indices.value().stride(0) : 0;
|
||||
|
||||
}
|
||||
else{
|
||||
if (!is_variable_B) {
|
||||
params.B_d_stride = B.stride(0);
|
||||
} else {
|
||||
params.B_batch_stride = B.stride(0);
|
||||
params.B_group_stride = B.stride(1);
|
||||
}
|
||||
params.B_dstate_stride = !is_variable_B ? B.stride(1) : B.stride(2);
|
||||
if (!is_variable_C) {
|
||||
params.C_d_stride = C.stride(0);
|
||||
} else {
|
||||
params.C_batch_stride = C.stride(0);
|
||||
params.C_group_stride = C.stride(1);
|
||||
}
|
||||
params.C_dstate_stride = !is_variable_C ? C.stride(1) : C.stride(2);
|
||||
params.u_batch_stride = u.stride(0);
|
||||
params.u_d_stride = u.stride(1);
|
||||
params.delta_batch_stride = delta.stride(0);
|
||||
params.delta_d_stride = delta.stride(1);
|
||||
if (has_z) {
|
||||
params.z_batch_stride = z.stride(0);
|
||||
params.z_d_stride = z.stride(1);
|
||||
params.out_z_batch_stride = out_z.stride(0);
|
||||
params.out_z_d_stride = out_z.stride(1);
|
||||
}
|
||||
params.out_batch_stride = out.stride(0);
|
||||
params.out_d_stride = out.stride(1);
|
||||
|
||||
params.ssm_states_batch_stride = ssm_states.stride(0);
|
||||
params.ssm_states_dim_stride = ssm_states.stride(1);
|
||||
params.ssm_states_dstate_stride = ssm_states.stride(2);
|
||||
|
||||
params.cache_indices_stride = cache_indices.has_value() ? cache_indices.value().stride(0) : 0;
|
||||
}
|
||||
}
|
||||
|
||||
void selective_scan_fwd(const torch::stable::Tensor &u, const torch::stable::Tensor &delta,
|
||||
const torch::stable::Tensor &A, const torch::stable::Tensor &B, const torch::stable::Tensor &C,
|
||||
const std::optional<torch::stable::Tensor> &D_,
|
||||
const std::optional<torch::stable::Tensor> &z_,
|
||||
const std::optional<torch::stable::Tensor> &delta_bias_,
|
||||
bool delta_softplus,
|
||||
const std::optional<torch::stable::Tensor> &query_start_loc,
|
||||
const std::optional<torch::stable::Tensor> &cache_indices,
|
||||
const std::optional<torch::stable::Tensor> &has_initial_state,
|
||||
const torch::stable::Tensor &ssm_states,
|
||||
// used to identify padding entries if cache_indices provided
|
||||
// in case of padding, the kernel will return early
|
||||
int64_t null_block_id,
|
||||
int64_t block_size,
|
||||
const std::optional<torch::stable::Tensor> &block_idx_first_scheduled_token,
|
||||
const std::optional<torch::stable::Tensor> &block_idx_last_scheduled_token,
|
||||
const std::optional<torch::stable::Tensor> &initial_state_idx,
|
||||
const std::optional<torch::stable::Tensor> &cu_chunk_seqlen,
|
||||
const std::optional<torch::stable::Tensor> &last_chunk_indices) {
|
||||
auto input_type = u.scalar_type();
|
||||
auto weight_type = A.scalar_type();
|
||||
STD_TORCH_CHECK(input_type == torch::headeronly::ScalarType::Float || input_type == torch::headeronly::ScalarType::Half || input_type == torch::headeronly::ScalarType::BFloat16);
|
||||
STD_TORCH_CHECK(weight_type == torch::headeronly::ScalarType::Float);
|
||||
|
||||
const bool is_variable_B = B.dim() >= 3;
|
||||
const bool is_variable_C = C.dim() >= 3;
|
||||
|
||||
STD_TORCH_CHECK(delta.scalar_type() == input_type);
|
||||
STD_TORCH_CHECK(B.scalar_type() == (!is_variable_B ? weight_type : input_type));
|
||||
STD_TORCH_CHECK(C.scalar_type() == (!is_variable_C ? weight_type : input_type));
|
||||
|
||||
STD_TORCH_CHECK(u.is_cuda());
|
||||
STD_TORCH_CHECK(delta.is_cuda());
|
||||
STD_TORCH_CHECK(A.is_cuda());
|
||||
STD_TORCH_CHECK(B.is_cuda());
|
||||
STD_TORCH_CHECK(C.is_cuda());
|
||||
|
||||
STD_TORCH_CHECK(u.stride(-1) == 1 || u.size(-1) == 1);
|
||||
STD_TORCH_CHECK(delta.stride(-1) == 1 || delta.size(-1) == 1);
|
||||
|
||||
const auto sizes = u.sizes();
|
||||
const bool varlen = query_start_loc.has_value();
|
||||
const int batch_size = varlen ? query_start_loc.value().sizes()[0] - 1 : sizes[0];
|
||||
const int dim = varlen ? sizes[0] : sizes[1];
|
||||
const int seqlen = varlen ? sizes[1] : sizes[2];
|
||||
const int dstate = A.size(1);
|
||||
const int n_groups = varlen ? B.size(0) : B.size(1);
|
||||
|
||||
STD_TORCH_CHECK(dstate <= 256, "selective_scan only supports state dimension <= 256");
|
||||
|
||||
if (varlen) {
|
||||
CHECK_SHAPE(u, dim, seqlen);
|
||||
CHECK_SHAPE(delta, dim, seqlen);
|
||||
} else {
|
||||
CHECK_SHAPE(u, batch_size, dim, seqlen);
|
||||
CHECK_SHAPE(delta, batch_size, dim, seqlen);
|
||||
}
|
||||
CHECK_SHAPE(A, dim, dstate);
|
||||
STD_TORCH_CHECK(is_variable_B, "is_variable_B = False is disabled in favor of reduced binary size");
|
||||
if (varlen) {
|
||||
CHECK_SHAPE(B, n_groups, dstate, seqlen);
|
||||
} else {
|
||||
CHECK_SHAPE(B, batch_size, n_groups, dstate, seqlen);
|
||||
}
|
||||
STD_TORCH_CHECK(B.stride(-1) == 1 || B.size(-1) == 1);
|
||||
|
||||
STD_TORCH_CHECK(is_variable_C, "is_variable_C = False is disabled in favor of reduced binary size");
|
||||
if (varlen) {
|
||||
CHECK_SHAPE(C, n_groups, dstate, seqlen);
|
||||
} else {
|
||||
CHECK_SHAPE(C, batch_size, n_groups, dstate, seqlen);
|
||||
}
|
||||
STD_TORCH_CHECK(C.stride(-1) == 1 || C.size(-1) == 1);
|
||||
|
||||
if (D_.has_value()) {
|
||||
auto D = D_.value();
|
||||
STD_TORCH_CHECK(D.scalar_type() == torch::headeronly::ScalarType::Float);
|
||||
STD_TORCH_CHECK(D.is_cuda());
|
||||
STD_TORCH_CHECK(D.stride(-1) == 1 || D.size(-1) == 1);
|
||||
CHECK_SHAPE(D, dim);
|
||||
}
|
||||
|
||||
if (delta_bias_.has_value()) {
|
||||
auto delta_bias = delta_bias_.value();
|
||||
STD_TORCH_CHECK(delta_bias.scalar_type() == torch::headeronly::ScalarType::Float);
|
||||
STD_TORCH_CHECK(delta_bias.is_cuda());
|
||||
STD_TORCH_CHECK(delta_bias.stride(-1) == 1 || delta_bias.size(-1) == 1);
|
||||
CHECK_SHAPE(delta_bias, dim);
|
||||
}
|
||||
|
||||
|
||||
if (has_initial_state.has_value()) {
|
||||
auto has_initial_state_ = has_initial_state.value();
|
||||
STD_TORCH_CHECK(has_initial_state_.scalar_type() == torch::headeronly::ScalarType::Bool);
|
||||
STD_TORCH_CHECK(has_initial_state_.is_cuda());
|
||||
CHECK_SHAPE(has_initial_state_, batch_size);
|
||||
}
|
||||
|
||||
|
||||
if (query_start_loc.has_value()) {
|
||||
auto query_start_loc_ = query_start_loc.value();
|
||||
STD_TORCH_CHECK(query_start_loc_.scalar_type() == torch::headeronly::ScalarType::Int);
|
||||
STD_TORCH_CHECK(query_start_loc_.is_cuda());
|
||||
}
|
||||
|
||||
|
||||
if (cache_indices.has_value()) {
|
||||
auto cache_indices_ = cache_indices.value();
|
||||
STD_TORCH_CHECK(cache_indices_.scalar_type() == torch::headeronly::ScalarType::Int);
|
||||
STD_TORCH_CHECK(cache_indices_.is_cuda());
|
||||
|
||||
// cache_indices can be either 1D (batch_size,) for non-APC mode
|
||||
// or 2D (batch_size, max_positions) for APC mode
|
||||
const bool is_apc_mode = block_idx_first_scheduled_token.has_value();
|
||||
if (is_apc_mode) {
|
||||
STD_TORCH_CHECK(cache_indices_.dim() == 2, "cache_indices must be 2D for APC mode");
|
||||
STD_TORCH_CHECK(cache_indices_.size(0) == batch_size, "cache_indices first dimension must match batch_size");
|
||||
} else {
|
||||
CHECK_SHAPE(cache_indices_, batch_size);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
torch::stable::Tensor z, out_z;
|
||||
const bool has_z = z_.has_value();
|
||||
if (has_z) {
|
||||
z = z_.value();
|
||||
STD_TORCH_CHECK(z.scalar_type() == input_type);
|
||||
STD_TORCH_CHECK(z.is_cuda());
|
||||
STD_TORCH_CHECK(z.stride(-1) == 1 || z.size(-1) == 1);
|
||||
if (varlen){
|
||||
CHECK_SHAPE(z, dim, seqlen);
|
||||
} else {
|
||||
CHECK_SHAPE(z, batch_size, dim, seqlen);
|
||||
}
|
||||
|
||||
out_z = z;
|
||||
}
|
||||
|
||||
// Right now u has BHL layout and delta has HBL layout, and we want out to have HBL layout
|
||||
torch::stable::Tensor out = delta;
|
||||
// ssm_states can now be either the same as input_type or float32
|
||||
auto state_type = ssm_states.scalar_type();
|
||||
STD_TORCH_CHECK(state_type == input_type || state_type == torch::headeronly::ScalarType::Float);
|
||||
STD_TORCH_CHECK(ssm_states.is_cuda());
|
||||
STD_TORCH_CHECK(ssm_states.stride(-1) == 1);
|
||||
|
||||
SSMParamsBase params;
|
||||
set_ssm_params_fwd(params, batch_size, dim, seqlen, dstate, n_groups, is_variable_B, is_variable_C,
|
||||
u, delta, A, B, C, out, z, out_z,
|
||||
D_,
|
||||
delta_bias_,
|
||||
ssm_states,
|
||||
has_z,
|
||||
delta_softplus,
|
||||
query_start_loc,
|
||||
cache_indices,
|
||||
has_initial_state,
|
||||
varlen,
|
||||
null_block_id,
|
||||
block_size,
|
||||
block_idx_first_scheduled_token,
|
||||
block_idx_last_scheduled_token,
|
||||
initial_state_idx,
|
||||
cu_chunk_seqlen,
|
||||
last_chunk_indices
|
||||
);
|
||||
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(u.get_device_index());
|
||||
auto stream = get_current_cuda_stream();
|
||||
DISPATCH_WTYPE_ITYPE_FLOAT_AND_HALF_AND_BF16(u.scalar_type(), ssm_states.scalar_type(), "selective_scan_fwd", [&] {
|
||||
selective_scan_fwd_cuda<input_t, weight_t, state_t>(params, stream);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// Inspired by
|
||||
// https://github.com/NVIDIA/DALI/blob/main/include/dali/core/static_switch.h
|
||||
// and https://github.com/pytorch/pytorch/blob/master/aten/src/ATen/Dispatch.h
|
||||
|
||||
// clang-format off
|
||||
// adapted from https://github.com/state-spaces/mamba/blob/main/csrc/selective_scan/static_switch.h
|
||||
#pragma once
|
||||
|
||||
/// @param COND - a boolean expression to switch by
|
||||
/// @param CONST_NAME - a name given for the constexpr bool variable.
|
||||
/// @param ... - code to execute for true and false
|
||||
///
|
||||
/// Usage:
|
||||
/// ```
|
||||
/// BOOL_SWITCH(flag, BoolConst, [&] {
|
||||
/// some_function<BoolConst>(...);
|
||||
/// });
|
||||
/// ```
|
||||
#define BOOL_SWITCH(COND, CONST_NAME, ...) \
|
||||
[&] { \
|
||||
if (COND) { \
|
||||
constexpr bool CONST_NAME = true; \
|
||||
return __VA_ARGS__(); \
|
||||
} else { \
|
||||
constexpr bool CONST_NAME = false; \
|
||||
return __VA_ARGS__(); \
|
||||
} \
|
||||
}()
|
||||
@@ -164,6 +164,17 @@ torch::stable::Tensor awq_dequantize(torch::stable::Tensor _kernel,
|
||||
|
||||
#endif
|
||||
|
||||
// Attention kernels (shared CUDA/ROCm)
|
||||
void merge_attn_states(
|
||||
torch::stable::Tensor& output,
|
||||
std::optional<torch::stable::Tensor> output_lse,
|
||||
const torch::stable::Tensor& prefix_output,
|
||||
const torch::stable::Tensor& prefix_lse,
|
||||
const torch::stable::Tensor& suffix_output,
|
||||
const torch::stable::Tensor& suffix_lse,
|
||||
const std::optional<int64_t> prefill_tokens_with_context,
|
||||
const std::optional<torch::stable::Tensor>& output_scale = std::nullopt);
|
||||
|
||||
torch::stable::Tensor hadacore_transform(torch::stable::Tensor& x,
|
||||
bool inplace);
|
||||
|
||||
@@ -220,6 +231,48 @@ void fused_qk_norm_rope(torch::stable::Tensor& qkv, int64_t num_heads_q,
|
||||
torch::stable::Tensor& position_ids,
|
||||
int64_t forced_token_heads_per_warp);
|
||||
|
||||
// Sampler kernels (shared CUDA/ROCm)
|
||||
void apply_repetition_penalties_(
|
||||
torch::stable::Tensor& logits, const torch::stable::Tensor& prompt_mask,
|
||||
const torch::stable::Tensor& output_mask,
|
||||
const torch::stable::Tensor& repetition_penalties);
|
||||
|
||||
void top_k_per_row_prefill(const torch::stable::Tensor& logits,
|
||||
const torch::stable::Tensor& rowStarts,
|
||||
const torch::stable::Tensor& rowEnds,
|
||||
torch::stable::Tensor& indices, int64_t numRows,
|
||||
int64_t stride0, int64_t stride1, int64_t topK);
|
||||
|
||||
void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
|
||||
const torch::stable::Tensor& seqLens,
|
||||
torch::stable::Tensor& indices, int64_t numRows,
|
||||
int64_t stride0, int64_t stride1, int64_t topK);
|
||||
|
||||
void persistent_topk(const torch::stable::Tensor& logits,
|
||||
const torch::stable::Tensor& lengths,
|
||||
torch::stable::Tensor& output,
|
||||
torch::stable::Tensor& workspace, int64_t k,
|
||||
int64_t max_seq_len);
|
||||
|
||||
void selective_scan_fwd(
|
||||
const torch::stable::Tensor& u, const torch::stable::Tensor& delta,
|
||||
const torch::stable::Tensor& A, const torch::stable::Tensor& B,
|
||||
const torch::stable::Tensor& C,
|
||||
const std::optional<torch::stable::Tensor>& D_,
|
||||
const std::optional<torch::stable::Tensor>& z_,
|
||||
const std::optional<torch::stable::Tensor>& delta_bias_,
|
||||
bool delta_softplus,
|
||||
const std::optional<torch::stable::Tensor>& query_start_loc,
|
||||
const std::optional<torch::stable::Tensor>& cache_indices,
|
||||
const std::optional<torch::stable::Tensor>& has_initial_state,
|
||||
const torch::stable::Tensor& ssm_states, int64_t null_block_id,
|
||||
int64_t block_size,
|
||||
const std::optional<torch::stable::Tensor>& block_idx_first_scheduled_token,
|
||||
const std::optional<torch::stable::Tensor>& block_idx_last_scheduled_token,
|
||||
const std::optional<torch::stable::Tensor>& initial_state_idx,
|
||||
const std::optional<torch::stable::Tensor>& cu_chunk_seqlen,
|
||||
const std::optional<torch::stable::Tensor>& last_chunk_indices);
|
||||
|
||||
// Activation kernels (shared CUDA/ROCm)
|
||||
void silu_and_mul(torch::stable::Tensor& out, torch::stable::Tensor& input);
|
||||
void silu_and_mul_clamp(torch::stable::Tensor& out,
|
||||
|
||||
@@ -0,0 +1,752 @@
|
||||
#include "../cuda_compat.h"
|
||||
#include "dispatch_utils.h"
|
||||
#include "torch_utils.h"
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include <cub/cub.cuh>
|
||||
#else
|
||||
#include <hipcub/hipcub.hpp>
|
||||
#endif
|
||||
|
||||
namespace vllm {
|
||||
|
||||
template <typename scalar_t>
|
||||
__global__ void apply_repetition_penalties_kernel(
|
||||
scalar_t* __restrict__ logits, // [num_seqs, vocab_size]
|
||||
const bool* __restrict__ prompt_mask, // [num_seqs, vocab_size]
|
||||
const bool* __restrict__ output_mask, // [num_seqs, vocab_size]
|
||||
const scalar_t* __restrict__ repetition_penalties, // [num_seqs]
|
||||
const int num_seqs, const int vocab_size, const int tile_size) {
|
||||
// Each block handles one sequence and a tile of vocab
|
||||
const int seq_idx = blockIdx.x;
|
||||
if (seq_idx >= num_seqs) return;
|
||||
|
||||
const int tile_start = blockIdx.y * tile_size;
|
||||
const int tile_end = min(tile_start + tile_size, vocab_size);
|
||||
|
||||
// Load repetition penalty for this sequence
|
||||
const scalar_t penalty = repetition_penalties[seq_idx];
|
||||
|
||||
// Each thread processes multiple vocab items within the tile
|
||||
for (int vocab_idx = tile_start + threadIdx.x; vocab_idx < tile_end;
|
||||
vocab_idx += blockDim.x) {
|
||||
const int64_t idx = static_cast<int64_t>(seq_idx) * vocab_size + vocab_idx;
|
||||
const bool is_repeated = prompt_mask[idx] || output_mask[idx];
|
||||
if (is_repeated) {
|
||||
scalar_t logit = logits[idx];
|
||||
if (logit > 0) {
|
||||
logits[idx] = logit / penalty;
|
||||
} else {
|
||||
logits[idx] = logit * penalty;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ auto convert_to_uint32(float x) -> uint32_t {
|
||||
uint32_t bits = __float_as_uint(x);
|
||||
return (bits & 0x80000000) ? bits : ~bits & 0x7fffffff;
|
||||
}
|
||||
|
||||
template <int step>
|
||||
static inline __device__ uint32_t extractBinIdx(float x) {
|
||||
if constexpr (step == 0) {
|
||||
__half hx = __float2half(x);
|
||||
uint16_t bits = __half_as_ushort(hx);
|
||||
bits = (bits & 0x8000) ? bits : ~bits & 0x7fff;
|
||||
return bits >> 5;
|
||||
} else {
|
||||
uint32_t bits = __float_as_uint(x);
|
||||
bits = (bits & 0x80000000) ? bits : ~bits & 0x7fffffff;
|
||||
|
||||
if constexpr (step == 1) {
|
||||
return bits >> 21;
|
||||
} else if constexpr (step == 2) {
|
||||
return (bits >> 10) & 0x7ff;
|
||||
} else if constexpr (step == 3) {
|
||||
return bits & 0x3ff;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <int shift>
|
||||
static inline __device__ bool isPartialMatch(float x, uint32_t pattern) {
|
||||
if constexpr (shift == 0) {
|
||||
return true;
|
||||
}
|
||||
uint32_t bits = __float_as_uint(x);
|
||||
bits = (bits & 0x80000000) ? bits : ~bits & 0x7fffffff;
|
||||
return (bits ^ pattern) >> shift == 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a Func over the input data, using vectorized load instructions if
|
||||
* possible.
|
||||
*
|
||||
* @tparam T element type
|
||||
* @tparam IdxT indexing type
|
||||
* @tparam Func void (T x, IdxT idx)
|
||||
*
|
||||
* @param thread_rank rank of the calling thread among all participating threads
|
||||
* @param num_threads number of the threads that participate in processing
|
||||
* @param in the input data
|
||||
* @param len the number of elements to read
|
||||
* @param f the lambda taking two arguments (T x, IdxT idx)
|
||||
*/
|
||||
template <typename T, typename idxT, typename Func>
|
||||
__device__ void vectorized_process(size_t thread_rank, size_t num_threads,
|
||||
const T* in, idxT len, Func f) {
|
||||
// Use dynamic WARP_SIZE from cuda_compat.h to support both
|
||||
// Wave64 (MI300X/gfx942) and Wave32 (Strix Halo/gfx1151) architectures
|
||||
constexpr int kWarpSize = WARP_SIZE;
|
||||
using WideT = float4;
|
||||
if constexpr (sizeof(T) >= sizeof(WideT)) {
|
||||
for (idxT i = thread_rank; i < len; i += num_threads) {
|
||||
f(in[i], i);
|
||||
}
|
||||
} else {
|
||||
static_assert(sizeof(WideT) % sizeof(T) == 0);
|
||||
constexpr int items_per_scalar = sizeof(WideT) / sizeof(T);
|
||||
// TODO: it's UB
|
||||
union {
|
||||
WideT scalar;
|
||||
T array[items_per_scalar];
|
||||
} wide;
|
||||
|
||||
int skip_cnt =
|
||||
(reinterpret_cast<size_t>(in) % sizeof(WideT))
|
||||
? ((sizeof(WideT) - reinterpret_cast<size_t>(in) % sizeof(WideT)) /
|
||||
sizeof(T))
|
||||
: 0;
|
||||
if (skip_cnt > len) {
|
||||
skip_cnt = len;
|
||||
}
|
||||
const WideT* in_cast = reinterpret_cast<decltype(in_cast)>(in + skip_cnt);
|
||||
const idxT len_cast = (len - skip_cnt) / items_per_scalar;
|
||||
|
||||
for (idxT i = thread_rank; i < len_cast; i += num_threads) {
|
||||
wide.scalar = in_cast[i];
|
||||
const idxT real_i = skip_cnt + i * items_per_scalar;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < items_per_scalar; ++j) {
|
||||
f(wide.array[j], real_i + j);
|
||||
}
|
||||
}
|
||||
|
||||
static_assert(kWarpSize >= items_per_scalar);
|
||||
// and because items_per_scalar > skip_cnt, kWarpSize > skip_cnt
|
||||
// no need to use loop
|
||||
if (thread_rank < skip_cnt) {
|
||||
f(in[thread_rank], thread_rank);
|
||||
}
|
||||
// because len_cast = (len - skip_cnt) / items_per_scalar,
|
||||
// len_cast * items_per_scalar + items_per_scalar > len - skip_cnt;
|
||||
// and so
|
||||
// len - (skip_cnt + len_cast * items_per_scalar) < items_per_scalar <=
|
||||
// kWarpSize no need to use loop
|
||||
const idxT remain_i = skip_cnt + len_cast * items_per_scalar + thread_rank;
|
||||
if (remain_i < len) {
|
||||
f(in[remain_i], remain_i);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <int step, int kNumThreadsPerBlock, int kNumBins, int kNumFinalItems,
|
||||
bool multipleBlocksPerRow, bool mergeBlocks, typename SmemFinalType,
|
||||
typename SmemOutputType>
|
||||
__device__ bool processHistogramStep(
|
||||
const int* indices, const float* logits, int rowEnd, uint32_t& logitPattern,
|
||||
int& thresholdBinIdx, SmemOutputType& smemOutput, int* smemThresholdBinIdx,
|
||||
int* smemFinalDstIdx, int* smemFinalBinSize, int* smemFoundTopKValues,
|
||||
SmemFinalType& smemFinal, int stride1, int rowStart, int topK) {
|
||||
// Clear the histogram.
|
||||
#pragma unroll
|
||||
for (int idx = threadIdx.x; idx < kNumBins; idx += kNumThreadsPerBlock) {
|
||||
smemFinal.histo.data[idx] = 0;
|
||||
}
|
||||
|
||||
// Make sure the histogram is ready.
|
||||
__syncthreads();
|
||||
|
||||
// Update pattern
|
||||
constexpr auto patternShift = step < 2 ? 0 : step == 2 ? 21 : 10;
|
||||
if constexpr (step == 2) {
|
||||
logitPattern = static_cast<uint32_t>(thresholdBinIdx & 0x7ff)
|
||||
<< patternShift;
|
||||
} else if constexpr (step == 3) {
|
||||
logitPattern |= static_cast<uint32_t>(thresholdBinIdx & 0x7ff)
|
||||
<< patternShift;
|
||||
}
|
||||
|
||||
auto distributeToBins = [&](float logit, int /* idx */ = 0) {
|
||||
if (isPartialMatch<patternShift>(logit, logitPattern)) {
|
||||
uint32_t binIdx = extractBinIdx<step>(logit);
|
||||
atomicAdd(&smemFinal.histo.data[binIdx], 1);
|
||||
}
|
||||
};
|
||||
|
||||
// Distribute the elements to the histogram bins.
|
||||
if (stride1 == 1) {
|
||||
vectorized_process(threadIdx.x, kNumThreadsPerBlock, logits + rowStart,
|
||||
rowEnd - rowStart, distributeToBins);
|
||||
} else {
|
||||
for (int idx = rowStart + threadIdx.x; idx < rowEnd;
|
||||
idx += kNumThreadsPerBlock) {
|
||||
float logit = logits[idx * stride1];
|
||||
distributeToBins(logit, idx);
|
||||
}
|
||||
}
|
||||
// Make sure the histogram is ready.
|
||||
__syncthreads();
|
||||
|
||||
// Reads the value of the starting position in the smemOutput array
|
||||
int lastValue = smemFoundTopKValues[0];
|
||||
|
||||
for (int round = 0; round < kNumBins / kNumThreadsPerBlock; round++) {
|
||||
// Read the values from SMEM.
|
||||
int idx = threadIdx.x + kNumThreadsPerBlock * round;
|
||||
int binCount{0};
|
||||
binCount = smemFinal.histo.data[idx];
|
||||
|
||||
// Make sure each thread has read its value.
|
||||
__syncthreads();
|
||||
|
||||
// Compute the prefix sum.
|
||||
int prefixSum{0}, totalSum{0};
|
||||
using Scan = cub::BlockScan<int, kNumThreadsPerBlock>;
|
||||
Scan(smemFinal.histo.scan).ExclusiveSum(binCount, prefixSum, totalSum);
|
||||
|
||||
// Update the histogram with the prefix sums.
|
||||
prefixSum += lastValue;
|
||||
totalSum += lastValue;
|
||||
smemFinal.histo.data[idx] = prefixSum;
|
||||
|
||||
// Make sure the data is in shared memory.
|
||||
__syncthreads();
|
||||
|
||||
// Find the last valid bin.
|
||||
bool foundThreshold = false;
|
||||
if (prefixSum < topK) {
|
||||
int nextPrefixSum = threadIdx.x == kNumThreadsPerBlock - 1
|
||||
? totalSum
|
||||
: smemFinal.histo.data[idx + 1];
|
||||
|
||||
if (nextPrefixSum >= topK) {
|
||||
smemThresholdBinIdx[0] = idx;
|
||||
smemFinalBinSize[0] = nextPrefixSum - prefixSum;
|
||||
foundThreshold = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Early exit: if any thread found the threshold, we can skip remaining
|
||||
// rounds
|
||||
if (__syncthreads_or(foundThreshold)) {
|
||||
break;
|
||||
}
|
||||
|
||||
lastValue = totalSum;
|
||||
}
|
||||
|
||||
// Make sure the data is in shared memory.
|
||||
__syncthreads();
|
||||
|
||||
// The threshold bin.
|
||||
thresholdBinIdx = smemThresholdBinIdx[0];
|
||||
|
||||
auto processBins = [&](float logit, int idx) {
|
||||
if (isPartialMatch<patternShift>(logit, logitPattern)) {
|
||||
uint32_t binIdx = extractBinIdx<step>(logit);
|
||||
// Only write elements with binIdx < thresholdBinIdx when:
|
||||
// 1. This is step 0 and the threshold bin is small enough (no step 1)
|
||||
// 2. This is step >= 1 (where pattern matching filters correctly)
|
||||
// This prevents duplicates when step 0 and step 1 both run.
|
||||
bool shouldWriteDirectly =
|
||||
(step == 0 && smemFinalBinSize[0] <= kNumFinalItems) || (step >= 1);
|
||||
if (binIdx < thresholdBinIdx && shouldWriteDirectly) {
|
||||
// The element is part of the top-k selection
|
||||
int dstIdx = atomicAdd(&smemFoundTopKValues[0], 1);
|
||||
|
||||
if constexpr (mergeBlocks) {
|
||||
smemOutput[dstIdx] = indices[idx];
|
||||
} else if constexpr (multipleBlocksPerRow) {
|
||||
smemOutput[dstIdx] = idx + rowStart;
|
||||
reinterpret_cast<float*>(smemOutput + topK)[dstIdx] = logit;
|
||||
} else {
|
||||
smemOutput[dstIdx] = idx;
|
||||
}
|
||||
}
|
||||
if constexpr (step < 3) {
|
||||
// Only fill the final items for sorting if the threshold bin fits
|
||||
if (binIdx == thresholdBinIdx &&
|
||||
smemFinalBinSize[0] <= kNumFinalItems) {
|
||||
int dstIdx = atomicAdd(&smemFinalDstIdx[0], 1);
|
||||
smemFinal.items.logits[dstIdx] = logit;
|
||||
if constexpr (mergeBlocks) {
|
||||
smemFinal.items.indices[dstIdx] = indices[idx];
|
||||
} else if constexpr (multipleBlocksPerRow) {
|
||||
smemFinal.items.indices[dstIdx] = idx + rowStart;
|
||||
} else {
|
||||
smemFinal.items.indices[dstIdx] = idx;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (binIdx == thresholdBinIdx) {
|
||||
// The elements in the threshold bin share the same 32 bits at step 3
|
||||
int dstIdx = atomicAdd(&smemFinal.histo.data[binIdx], 1);
|
||||
if (dstIdx < topK) {
|
||||
if constexpr (mergeBlocks) {
|
||||
smemOutput[dstIdx] = indices[idx];
|
||||
} else if constexpr (multipleBlocksPerRow) {
|
||||
smemOutput[dstIdx] = idx + rowStart;
|
||||
reinterpret_cast<float*>(smemOutput + topK)[dstIdx] = logit;
|
||||
} else {
|
||||
smemOutput[dstIdx] = idx;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (stride1 == 1) {
|
||||
vectorized_process(threadIdx.x, kNumThreadsPerBlock, logits + rowStart,
|
||||
rowEnd - rowStart, processBins);
|
||||
} else {
|
||||
for (int idx = rowStart + threadIdx.x; idx < rowEnd;
|
||||
idx += kNumThreadsPerBlock) {
|
||||
float logit = logits[idx * stride1];
|
||||
processBins(logit, idx);
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure the elements are in shared memory.
|
||||
__syncthreads();
|
||||
|
||||
// Check if we should continue to next step
|
||||
return smemFinalBinSize[0] > kNumFinalItems;
|
||||
}
|
||||
|
||||
// Follows half - 11 - 11 - 10 bit iterations
|
||||
template <int kNumThreadsPerBlock, int kNumBins, bool useRadixSort,
|
||||
bool multipleBlocksPerRow = false, bool mergeBlocks = false>
|
||||
static __device__ void topKPerRowJob(const int* indices, const float* logits,
|
||||
int rowStart, int rowEnd, int* outIndices,
|
||||
float* outLogits, int stride1, int topK) {
|
||||
// The number of slots for the final pass.
|
||||
static constexpr int kNumFinalItems = 2048;
|
||||
// The number of elements per thread for the final sort.
|
||||
static constexpr int kNumFinalItemsPerThread =
|
||||
kNumFinalItems / kNumThreadsPerBlock;
|
||||
// The class to sort the elements during the final pass.
|
||||
using FinalSort = cub::BlockRadixSort<float, kNumThreadsPerBlock,
|
||||
kNumFinalItemsPerThread, int>;
|
||||
using FinalSortTempStorage =
|
||||
std::conditional_t<useRadixSort, typename FinalSort::TempStorage, int>;
|
||||
// The class to compute the inclusive prefix-sum over the histogram.
|
||||
using Scan = cub::BlockScan<int, kNumThreadsPerBlock>;
|
||||
|
||||
// The structure to store the final items (for the final pass).
|
||||
struct FinalItems {
|
||||
// Shared memory to store the indices for the final pass.
|
||||
int indices[kNumFinalItems];
|
||||
// Shared memory to store the logits for the final pass.
|
||||
float logits[kNumFinalItems];
|
||||
};
|
||||
|
||||
struct Histogram {
|
||||
typename Scan::TempStorage scan;
|
||||
int data[kNumBins];
|
||||
};
|
||||
|
||||
// Shared memory to compute the block sort.
|
||||
__shared__ union {
|
||||
FinalItems items;
|
||||
FinalSortTempStorage finalSort;
|
||||
Histogram histo;
|
||||
} smemFinal;
|
||||
|
||||
// Shared memory to store the selected indices.
|
||||
// If we are processing using multiple blocks, we need to store the logits and
|
||||
// indices.
|
||||
extern __shared__ int32_t smemOutput[];
|
||||
|
||||
// Shared memory to store the threshold bin.
|
||||
__shared__ int smemThresholdBinIdx[1];
|
||||
// Shared memory counter to register the candidates for the final phase.
|
||||
__shared__ int smemFinalDstIdx[1];
|
||||
// Shared memory to determine if the threshold bin fits in the final items.
|
||||
__shared__ int smemFinalBinSize[1];
|
||||
// Shared memory to keep track of the top-k values found so far by the
|
||||
// previous iterations
|
||||
__shared__ int smemFoundTopKValues[1];
|
||||
|
||||
// The length of the row.
|
||||
int rowLen = rowEnd - rowStart;
|
||||
|
||||
// Shortcut if the length of the row is smaller than Top-K. Indices are not
|
||||
// sorted by their corresponding logit.
|
||||
if (rowLen <= topK) {
|
||||
for (int rowIt = threadIdx.x; rowIt < rowLen;
|
||||
rowIt += kNumThreadsPerBlock) {
|
||||
if constexpr (multipleBlocksPerRow) {
|
||||
outIndices[rowIt] = rowIt + rowStart;
|
||||
outLogits[rowIt] = logits[rowIt + rowStart];
|
||||
} else {
|
||||
outIndices[rowIt] = rowIt;
|
||||
}
|
||||
}
|
||||
for (int rowIt = rowLen + threadIdx.x; rowIt < topK;
|
||||
rowIt += kNumThreadsPerBlock) {
|
||||
outIndices[rowIt] = -1;
|
||||
if constexpr (multipleBlocksPerRow) {
|
||||
outLogits[rowIt] = -FLT_MAX;
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
// Initialize values
|
||||
if (threadIdx.x == 0) {
|
||||
smemFinalDstIdx[0] = 0;
|
||||
smemFoundTopKValues[0] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
int thresholdBinIdx = -1;
|
||||
uint32_t logitPattern = 0;
|
||||
|
||||
// Step 0: Process first 11 bits of half representation
|
||||
bool continueToNextStep =
|
||||
processHistogramStep<0, kNumThreadsPerBlock, kNumBins, kNumFinalItems,
|
||||
multipleBlocksPerRow, mergeBlocks>(
|
||||
indices, logits, rowEnd, logitPattern, thresholdBinIdx, smemOutput,
|
||||
smemThresholdBinIdx, smemFinalDstIdx, smemFinalBinSize,
|
||||
smemFoundTopKValues, smemFinal, stride1, rowStart, topK);
|
||||
|
||||
if (continueToNextStep) {
|
||||
// Step 1: Process next 11 bits
|
||||
continueToNextStep =
|
||||
processHistogramStep<1, kNumThreadsPerBlock, kNumBins, kNumFinalItems,
|
||||
multipleBlocksPerRow, mergeBlocks>(
|
||||
indices, logits, rowEnd, logitPattern, thresholdBinIdx, smemOutput,
|
||||
smemThresholdBinIdx, smemFinalDstIdx, smemFinalBinSize,
|
||||
smemFoundTopKValues, smemFinal, stride1, rowStart, topK);
|
||||
}
|
||||
|
||||
if (continueToNextStep) {
|
||||
// Step 2: Process next 11 bits
|
||||
continueToNextStep =
|
||||
processHistogramStep<2, kNumThreadsPerBlock, kNumBins, kNumFinalItems,
|
||||
multipleBlocksPerRow, mergeBlocks>(
|
||||
indices, logits, rowEnd, logitPattern, thresholdBinIdx, smemOutput,
|
||||
smemThresholdBinIdx, smemFinalDstIdx, smemFinalBinSize,
|
||||
smemFoundTopKValues, smemFinal, stride1, rowStart, topK);
|
||||
}
|
||||
|
||||
if (continueToNextStep) {
|
||||
// Step 3: Process last 10 bits
|
||||
processHistogramStep<3, kNumThreadsPerBlock, kNumBins, kNumFinalItems,
|
||||
multipleBlocksPerRow, mergeBlocks>(
|
||||
indices, logits, rowEnd, logitPattern, thresholdBinIdx, smemOutput,
|
||||
smemThresholdBinIdx, smemFinalDstIdx, smemFinalBinSize,
|
||||
smemFoundTopKValues, smemFinal, stride1, rowStart, topK);
|
||||
}
|
||||
|
||||
if (!continueToNextStep) {
|
||||
// The histogram did not proceed to the final 10 bits, therefore we need to
|
||||
// sort the final items The logits of the elements to be sorted in the final
|
||||
// pass.
|
||||
if constexpr (useRadixSort) {
|
||||
// Sorting with radix sort
|
||||
float finalLogits[kNumFinalItemsPerThread];
|
||||
// The indices of the elements to be sorted in the final pass.
|
||||
int finalIndices[kNumFinalItemsPerThread];
|
||||
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
|
||||
finalLogits[ii] = -FLT_MAX;
|
||||
}
|
||||
|
||||
// Read the elements from SMEM.
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
|
||||
int srcIdx = ii * kNumThreadsPerBlock + threadIdx.x;
|
||||
if (srcIdx < smemFinalDstIdx[0]) {
|
||||
finalLogits[ii] = smemFinal.items.logits[srcIdx];
|
||||
finalIndices[ii] = smemFinal.items.indices[srcIdx];
|
||||
}
|
||||
}
|
||||
// Make sure the shared memory has been read.
|
||||
__syncthreads();
|
||||
|
||||
// Sort the elements.
|
||||
FinalSort(smemFinal.finalSort)
|
||||
.SortDescendingBlockedToStriped(finalLogits, finalIndices);
|
||||
|
||||
// Copy the data back to the shared memory storage.
|
||||
int baseIdx = smemFoundTopKValues[0];
|
||||
|
||||
#pragma unroll
|
||||
for (int ii = 0; ii < kNumFinalItemsPerThread; ++ii) {
|
||||
int srcIdx = ii * kNumThreadsPerBlock + threadIdx.x;
|
||||
int dstIdx = baseIdx + srcIdx;
|
||||
|
||||
if (dstIdx < topK) {
|
||||
smemOutput[dstIdx] = finalIndices[ii];
|
||||
if constexpr (multipleBlocksPerRow) {
|
||||
reinterpret_cast<float*>(smemOutput + topK)[dstIdx] =
|
||||
finalLogits[ii];
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Sorting with insertion sort
|
||||
auto baseIdx = smemFoundTopKValues[0];
|
||||
for (int i = threadIdx.x; i < smemFinalDstIdx[0];
|
||||
i += kNumThreadsPerBlock) {
|
||||
int outIndex = 0;
|
||||
auto logit = smemFinal.items.logits[i];
|
||||
for (int j = 0; j < smemFinalDstIdx[0]; j++) {
|
||||
auto otherLogit = smemFinal.items.logits[j];
|
||||
if (logit < otherLogit || (logit == otherLogit && i < j)) {
|
||||
outIndex++;
|
||||
}
|
||||
}
|
||||
// Store if outIndex is in bounds
|
||||
if (outIndex + baseIdx < topK) {
|
||||
smemOutput[outIndex + baseIdx] = smemFinal.items.indices[i];
|
||||
if constexpr (multipleBlocksPerRow) {
|
||||
reinterpret_cast<float*>(smemOutput + topK)[outIndex + baseIdx] =
|
||||
smemFinal.items.logits[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
// Store to global memory.
|
||||
for (int i = threadIdx.x; i < topK; i += kNumThreadsPerBlock) {
|
||||
if constexpr (multipleBlocksPerRow) {
|
||||
outIndices[i] = smemOutput[i];
|
||||
outLogits[i] = reinterpret_cast<float*>(smemOutput + topK)[i];
|
||||
} else {
|
||||
if (stride1 == 1) {
|
||||
// stride1 == 1 will use vectorized_process, which indexes already skip
|
||||
// the rowStart.
|
||||
outIndices[i] = smemOutput[i];
|
||||
} else {
|
||||
outIndices[i] = smemOutput[i] - rowStart;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <int kNumThreadsPerBlock, bool useRadixSort>
|
||||
static __global__ __launch_bounds__(kNumThreadsPerBlock) void topKPerRowPrefill(
|
||||
const float* logits, const int* rowStarts, const int* rowEnds,
|
||||
int* outIndices, int stride0, int stride1, const int topK,
|
||||
const int offsetIndex) {
|
||||
// The number of bins in the histogram.
|
||||
static constexpr int kNumBins = 2048;
|
||||
|
||||
// The row computed by this block.
|
||||
int rowIdx = blockIdx.x + offsetIndex;
|
||||
|
||||
// The range of logits within the row.
|
||||
int rowStart = rowStarts[rowIdx];
|
||||
int rowEnd = rowEnds[rowIdx];
|
||||
|
||||
// Local pointers to this block
|
||||
outIndices += static_cast<int64_t>(rowIdx) * topK;
|
||||
logits += static_cast<int64_t>(rowIdx) * stride0;
|
||||
|
||||
topKPerRowJob<kNumThreadsPerBlock, kNumBins, useRadixSort>(
|
||||
nullptr, logits, rowStart, rowEnd, outIndices, nullptr, stride1, topK);
|
||||
}
|
||||
|
||||
template <int kNumThreadsPerBlock, bool useRadixSort,
|
||||
bool multipleBlocksPerRow = false, bool mergeBlocks = false>
|
||||
static __global__ __launch_bounds__(kNumThreadsPerBlock) void topKPerRowDecode(
|
||||
const float* logits, const int* seqLens, int* outIndices, int stride0,
|
||||
int stride1, const int topK, int next_n, int seqLensIs2D = 0,
|
||||
float* outLogits = nullptr, const int numBlocksToMerge = 0,
|
||||
const int* indices = nullptr) {
|
||||
// The number of bins in the histogram.
|
||||
static constexpr int kNumBins = 2048;
|
||||
|
||||
// The row computed by this block.
|
||||
int rowIdx = blockIdx.x;
|
||||
|
||||
// The range of logits within the row.
|
||||
int rowStart = 0;
|
||||
int batch_idx = rowIdx / next_n;
|
||||
int next_n_idx = rowIdx % next_n;
|
||||
// seqLensIs2D=0: 1D seqLens — all rows in a batch share the same seq_len;
|
||||
// kernel computes per-row effective length via offset.
|
||||
// seqLensIs2D=1: 2D seqLens — each logit row has its own pre-computed
|
||||
// effective length (flat index rowIdx = b*next_n + j maps
|
||||
// directly to seqLens[b, j] in C-contiguous layout).
|
||||
int seq_len = seqLensIs2D ? seqLens[rowIdx] : seqLens[batch_idx];
|
||||
int rowEnd =
|
||||
seqLensIs2D ? max(0, seq_len) : max(0, seq_len - next_n + next_n_idx + 1);
|
||||
|
||||
// Local pointers to this block
|
||||
if constexpr (!multipleBlocksPerRow && !mergeBlocks) {
|
||||
outIndices += static_cast<int64_t>(rowIdx) * topK;
|
||||
} else if constexpr (multipleBlocksPerRow) {
|
||||
const auto blockSize = rowEnd / gridDim.y; // 16384 / 2 = 8192
|
||||
rowStart = blockSize * blockIdx.y; // 8192 * 1 = 8192
|
||||
rowEnd = gridDim.y == blockIdx.y + 1 ? rowEnd : rowStart + blockSize;
|
||||
outIndices +=
|
||||
static_cast<int64_t>(rowIdx) * gridDim.y * topK + blockIdx.y * topK;
|
||||
outLogits +=
|
||||
static_cast<int64_t>(rowIdx) * gridDim.y * topK + blockIdx.y * topK;
|
||||
} else if constexpr (mergeBlocks) {
|
||||
rowEnd = numBlocksToMerge * topK;
|
||||
indices += static_cast<int64_t>(rowIdx) * numBlocksToMerge * topK;
|
||||
outIndices += static_cast<int64_t>(rowIdx) * topK;
|
||||
}
|
||||
logits += static_cast<int64_t>(rowIdx) * stride0;
|
||||
|
||||
topKPerRowJob<kNumThreadsPerBlock, kNumBins, useRadixSort,
|
||||
multipleBlocksPerRow, mergeBlocks>(
|
||||
indices, logits, rowStart, rowEnd, outIndices, outLogits, stride1, topK);
|
||||
}
|
||||
|
||||
} // namespace vllm
|
||||
|
||||
void apply_repetition_penalties_(
|
||||
torch::stable::Tensor& logits, // [num_seqs, vocab_size], in-place
|
||||
const torch::stable::Tensor& prompt_mask, // [num_seqs, vocab_size]
|
||||
const torch::stable::Tensor& output_mask, // [num_seqs, vocab_size]
|
||||
const torch::stable::Tensor& repetition_penalties) { // [num_seqs]
|
||||
STD_TORCH_CHECK(logits.is_contiguous());
|
||||
STD_TORCH_CHECK(prompt_mask.is_contiguous());
|
||||
STD_TORCH_CHECK(output_mask.is_contiguous());
|
||||
STD_TORCH_CHECK(repetition_penalties.is_contiguous());
|
||||
|
||||
int vocab_size = logits.size(-1);
|
||||
int num_seqs = logits.size(0);
|
||||
|
||||
if (num_seqs == 0) return;
|
||||
|
||||
// Get number of SMs on the current device
|
||||
int sms = 0;
|
||||
cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount,
|
||||
logits.get_device_index());
|
||||
|
||||
// Compute tile_num and tile_size
|
||||
int tile_num =
|
||||
std::min(vocab_size, std::max(1, (sms + num_seqs - 1) / num_seqs));
|
||||
int tile_size = (vocab_size + tile_num - 1) / tile_num;
|
||||
|
||||
// Each block handles one sequence and a tile of vocab
|
||||
dim3 grid(num_seqs, tile_num);
|
||||
dim3 block(std::min(tile_size, 1024));
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
logits.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
VLLM_STABLE_DISPATCH_FLOATING_TYPES(
|
||||
logits.scalar_type(), "apply_repetition_penalties_kernel", [&] {
|
||||
vllm::apply_repetition_penalties_kernel<scalar_t>
|
||||
<<<grid, block, 0, stream>>>(
|
||||
logits.mutable_data_ptr<scalar_t>(),
|
||||
prompt_mask.const_data_ptr<bool>(),
|
||||
output_mask.const_data_ptr<bool>(),
|
||||
repetition_penalties.const_data_ptr<scalar_t>(), num_seqs,
|
||||
vocab_size, tile_size);
|
||||
});
|
||||
}
|
||||
|
||||
void top_k_per_row_decode(const torch::stable::Tensor& logits, int64_t next_n,
|
||||
const torch::stable::Tensor& seqLens,
|
||||
torch::stable::Tensor& indices, int64_t numRows,
|
||||
int64_t stride0, int64_t stride1, int64_t topK) {
|
||||
constexpr int kSortingAlgorithmThreshold = 12288;
|
||||
constexpr int kSplitWorkThreshold = 200 * 1000;
|
||||
constexpr int kNumThreadsPerBlock = 512;
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
const auto numColumns = logits.size(1);
|
||||
|
||||
// True if seqLens is 2D (B, next_n): each logit row has its own pre-computed
|
||||
// effective seq_len. False if seqLens is 1D (B,): all rows in a batch share
|
||||
// the same seq_len and the kernel computes the per-row offset itself.
|
||||
int seqLensIs2D = seqLens.dim() == 2 ? 1 : 0;
|
||||
|
||||
if (numColumns < kSortingAlgorithmThreshold) {
|
||||
// Use insertion sort
|
||||
vllm::topKPerRowDecode<kNumThreadsPerBlock, false>
|
||||
<<<numRows, kNumThreadsPerBlock, topK * sizeof(int32_t), stream>>>(
|
||||
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
indices.mutable_data_ptr<int>(), static_cast<int>(stride0),
|
||||
static_cast<int>(stride1), static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D);
|
||||
} else if (numColumns < kSplitWorkThreshold) {
|
||||
// From this threshold, use radix sort instead
|
||||
vllm::topKPerRowDecode<kNumThreadsPerBlock, true>
|
||||
<<<numRows, kNumThreadsPerBlock, topK * sizeof(int32_t), stream>>>(
|
||||
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
indices.mutable_data_ptr<int>(), static_cast<int>(stride0),
|
||||
static_cast<int>(stride1), static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D);
|
||||
} else {
|
||||
// Long sequences are run in two steps
|
||||
constexpr auto multipleBlocksPerRowConfig = 10;
|
||||
|
||||
const auto outIndicesAux = torch::stable::empty(
|
||||
{numRows, multipleBlocksPerRowConfig, topK},
|
||||
torch::headeronly::ScalarType::Int, std::nullopt, logits.device());
|
||||
const auto outLogitsAux = torch::stable::empty(
|
||||
{numRows, multipleBlocksPerRowConfig, topK},
|
||||
torch::headeronly::ScalarType::Float, std::nullopt, logits.device());
|
||||
|
||||
vllm::topKPerRowDecode<kNumThreadsPerBlock, true, true>
|
||||
<<<dim3(numRows, multipleBlocksPerRowConfig), kNumThreadsPerBlock,
|
||||
2 * topK * sizeof(int32_t), stream>>>(
|
||||
logits.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
outIndicesAux.mutable_data_ptr<int>(), static_cast<int>(stride0),
|
||||
static_cast<int>(stride1), static_cast<int>(topK),
|
||||
static_cast<int>(next_n), seqLensIs2D,
|
||||
outLogitsAux.mutable_data_ptr<float>());
|
||||
|
||||
constexpr int kNumThreadsPerBlockMerge = 1024;
|
||||
vllm::topKPerRowDecode<kNumThreadsPerBlockMerge, true, false, true>
|
||||
<<<numRows, kNumThreadsPerBlockMerge, topK * sizeof(int32_t), stream>>>(
|
||||
outLogitsAux.const_data_ptr<float>(), seqLens.const_data_ptr<int>(),
|
||||
indices.mutable_data_ptr<int>(), multipleBlocksPerRowConfig * topK,
|
||||
1, static_cast<int>(topK), static_cast<int>(next_n), seqLensIs2D,
|
||||
nullptr, multipleBlocksPerRowConfig,
|
||||
outIndicesAux.const_data_ptr<int>());
|
||||
}
|
||||
}
|
||||
|
||||
void top_k_per_row_prefill(const torch::stable::Tensor& logits,
|
||||
const torch::stable::Tensor& rowStarts,
|
||||
const torch::stable::Tensor& rowEnds,
|
||||
torch::stable::Tensor& indices, int64_t numRows,
|
||||
int64_t stride0, int64_t stride1, int64_t topK) {
|
||||
constexpr int kSortingAlgorithmThreshold = 12288;
|
||||
constexpr int kNumThreadsPerBlock = 512;
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
|
||||
int numInsertionBlocks =
|
||||
std::min(static_cast<int>(numRows), kSortingAlgorithmThreshold);
|
||||
vllm::topKPerRowPrefill<kNumThreadsPerBlock, false>
|
||||
<<<numInsertionBlocks, kNumThreadsPerBlock, topK * sizeof(int32_t),
|
||||
stream>>>(logits.const_data_ptr<float>(),
|
||||
rowStarts.const_data_ptr<int>(),
|
||||
rowEnds.const_data_ptr<int>(),
|
||||
indices.mutable_data_ptr<int>(), static_cast<int>(stride0),
|
||||
static_cast<int>(stride1), static_cast<int>(topK), 0);
|
||||
|
||||
if (numRows > kSortingAlgorithmThreshold) {
|
||||
int numRadixBlocks = numRows - kSortingAlgorithmThreshold;
|
||||
vllm::topKPerRowPrefill<kNumThreadsPerBlock, true>
|
||||
<<<numRadixBlocks, kNumThreadsPerBlock, topK * sizeof(int32_t),
|
||||
stream>>>(
|
||||
logits.const_data_ptr<float>(), rowStarts.const_data_ptr<int>(),
|
||||
rowEnds.const_data_ptr<int>(), indices.mutable_data_ptr<int>(),
|
||||
static_cast<int>(stride0), static_cast<int>(stride1),
|
||||
static_cast<int>(topK), kSortingAlgorithmThreshold);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
// Persistent TopK kernel for DeepSeek V3 sparse attention indexer.
|
||||
// See persistent_topk.cuh for kernel implementation.
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <algorithm>
|
||||
|
||||
#include "torch_utils.h"
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include "../persistent_topk.cuh"
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
|
||||
#ifndef USE_ROCM
|
||||
template <int TopK>
|
||||
void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
const torch::stable::Tensor& lengths,
|
||||
torch::stable::Tensor& output,
|
||||
torch::stable::Tensor& workspace,
|
||||
int64_t max_seq_len) {
|
||||
namespace P = vllm::persistent;
|
||||
|
||||
const int64_t num_rows = logits.size(0);
|
||||
const int64_t stride = logits.stride(0);
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
|
||||
static int num_sms = 0;
|
||||
static int max_smem_per_block = 0;
|
||||
if (num_sms == 0) {
|
||||
const cudaDeviceProp* device_prop = get_device_prop();
|
||||
num_sms = device_prop->multiProcessorCount;
|
||||
max_smem_per_block = device_prop->sharedMemPerBlockOptin;
|
||||
}
|
||||
|
||||
if (num_rows > 32 && max_smem_per_block >= 128 * 1024) {
|
||||
cudaError_t status =
|
||||
vllm::FilteredTopKRaggedTransform<float, int32_t, TopK>(
|
||||
logits.const_data_ptr<float>(), output.mutable_data_ptr<int32_t>(),
|
||||
lengths.const_data_ptr<int32_t>(), static_cast<uint32_t>(num_rows),
|
||||
static_cast<uint32_t>(TopK), static_cast<uint32_t>(stride), stream);
|
||||
STD_TORCH_CHECK(status == cudaSuccess,
|
||||
"FilteredTopK failed: ", cudaGetErrorString(status));
|
||||
} else {
|
||||
STD_TORCH_CHECK(workspace.is_cuda(), "workspace must be CUDA tensor");
|
||||
STD_TORCH_CHECK(
|
||||
workspace.scalar_type() == torch::headeronly::ScalarType::Byte,
|
||||
"workspace must be uint8");
|
||||
|
||||
int effective_max_smem;
|
||||
if (num_rows <= 4) {
|
||||
effective_max_smem =
|
||||
std::min(max_smem_per_block, static_cast<int>(P::kSmemMedium));
|
||||
} else if (num_rows <= 8) {
|
||||
constexpr int kSmemCapMedium = 48 * 1024;
|
||||
effective_max_smem = std::min(max_smem_per_block, kSmemCapMedium);
|
||||
} else {
|
||||
effective_max_smem = max_smem_per_block;
|
||||
}
|
||||
|
||||
size_t available_for_ordered =
|
||||
static_cast<size_t>(effective_max_smem) - P::kFixedSmemLarge;
|
||||
uint32_t max_chunk_elements =
|
||||
static_cast<uint32_t>(available_for_ordered / sizeof(uint32_t));
|
||||
|
||||
uint32_t vec_size = 1;
|
||||
if (stride % 4 == 0)
|
||||
vec_size = 4;
|
||||
else if (stride % 2 == 0)
|
||||
vec_size = 2;
|
||||
|
||||
max_chunk_elements = (max_chunk_elements / vec_size) * vec_size;
|
||||
uint32_t min_chunk = vec_size * P::kThreadsPerBlock;
|
||||
if (max_chunk_elements < min_chunk) max_chunk_elements = min_chunk;
|
||||
|
||||
uint32_t ctas_per_group =
|
||||
(static_cast<uint32_t>(stride) + max_chunk_elements - 1) /
|
||||
max_chunk_elements;
|
||||
uint32_t chunk_size =
|
||||
(static_cast<uint32_t>(stride) + ctas_per_group - 1) / ctas_per_group;
|
||||
chunk_size = ((chunk_size + vec_size - 1) / vec_size) * vec_size;
|
||||
if (chunk_size > max_chunk_elements) chunk_size = max_chunk_elements;
|
||||
|
||||
size_t smem_size = P::kFixedSmemLarge + chunk_size * sizeof(uint32_t);
|
||||
if (smem_size < P::kSmemMedium) smem_size = P::kSmemMedium;
|
||||
|
||||
// Query occupancy for the instantiation that will actually launch;
|
||||
// overestimating it deadlocks the cooperative barrier.
|
||||
int occupancy = 1;
|
||||
cudaError_t occ_err = cudaSuccess;
|
||||
if (vec_size == 4) {
|
||||
occ_err = cudaOccupancyMaxActiveBlocksPerMultiprocessor(
|
||||
&occupancy, P::persistent_topk_kernel<TopK, 4>, P::kThreadsPerBlock,
|
||||
smem_size);
|
||||
} else if (vec_size == 2) {
|
||||
occ_err = cudaOccupancyMaxActiveBlocksPerMultiprocessor(
|
||||
&occupancy, P::persistent_topk_kernel<TopK, 2>, P::kThreadsPerBlock,
|
||||
smem_size);
|
||||
} else {
|
||||
occ_err = cudaOccupancyMaxActiveBlocksPerMultiprocessor(
|
||||
&occupancy, P::persistent_topk_kernel<TopK, 1>, P::kThreadsPerBlock,
|
||||
smem_size);
|
||||
}
|
||||
STD_TORCH_CHECK(occ_err == cudaSuccess,
|
||||
"persistent_topk occupancy query failed: ",
|
||||
cudaGetErrorString(occ_err));
|
||||
if (occupancy < 1) occupancy = 1;
|
||||
|
||||
// The cooperative spin-wait barrier only runs when at least one row hits
|
||||
// the radix path (seq_len > RADIX_THRESHOLD). Below that, non-CTA-0 CTAs
|
||||
// early-exit, so oversubscription can't deadlock and headroom is wasted.
|
||||
const bool needs_cooperative =
|
||||
static_cast<uint32_t>(max_seq_len) > P::RADIX_THRESHOLD;
|
||||
|
||||
const uint32_t hw_resident_cap =
|
||||
static_cast<uint32_t>(num_sms) * static_cast<uint32_t>(occupancy);
|
||||
uint32_t max_resident_ctas = hw_resident_cap;
|
||||
if (needs_cooperative) {
|
||||
// Reserve one CTA per SM when occupancy allows; fall back to a single
|
||||
// CTA when occupancy == 1 (the most deadlock-prone case — any straggler
|
||||
// kernel that takes the only slot on one SM hangs the barrier). Never
|
||||
// drop below one full group's worth.
|
||||
uint32_t headroom = (occupancy > 1) ? static_cast<uint32_t>(num_sms) : 1u;
|
||||
if (max_resident_ctas >= headroom + ctas_per_group) {
|
||||
max_resident_ctas -= headroom;
|
||||
}
|
||||
}
|
||||
uint32_t num_groups = std::min(max_resident_ctas / ctas_per_group,
|
||||
static_cast<uint32_t>(num_rows));
|
||||
if (num_groups == 0) num_groups = 1;
|
||||
uint32_t total_ctas = num_groups * ctas_per_group;
|
||||
|
||||
// If the cooperative launch wouldn't fit, fall back to FilteredTopK
|
||||
// instead of deadlocking. Only relevant when needs_cooperative.
|
||||
if (needs_cooperative && total_ctas > hw_resident_cap) {
|
||||
STD_TORCH_CHECK(
|
||||
max_smem_per_block >= 128 * 1024,
|
||||
"persistent_topk would oversubscribe and the FilteredTopK "
|
||||
"fallback requires >=128KB smem per block (have ",
|
||||
max_smem_per_block, "). total_ctas=", total_ctas,
|
||||
" > num_sms*occupancy=", hw_resident_cap, " (TopK=", TopK,
|
||||
", vec_size=", vec_size, ", ctas_per_group=", ctas_per_group,
|
||||
", smem=", smem_size, ").");
|
||||
cudaError_t status =
|
||||
vllm::FilteredTopKRaggedTransform<float, int32_t, TopK>(
|
||||
logits.const_data_ptr<float>(),
|
||||
output.mutable_data_ptr<int32_t>(),
|
||||
lengths.const_data_ptr<int32_t>(),
|
||||
static_cast<uint32_t>(num_rows), static_cast<uint32_t>(TopK),
|
||||
static_cast<uint32_t>(stride), stream);
|
||||
STD_TORCH_CHECK(status == cudaSuccess, "FilteredTopK fallback failed: ",
|
||||
cudaGetErrorString(status));
|
||||
return;
|
||||
}
|
||||
|
||||
size_t state_bytes = num_groups * sizeof(P::RadixRowState);
|
||||
STD_TORCH_CHECK(workspace.size(0) >= static_cast<int64_t>(state_bytes),
|
||||
"workspace too small, need ", state_bytes, " bytes");
|
||||
|
||||
// Zero the per-group RadixRowState region before launch.
|
||||
//
|
||||
// Issued UNCONDITIONALLY so the memset is captured as its own node in
|
||||
// the cudagraph (a separate cudaMemsetAsync node, sequenced before the
|
||||
// persistent_topk_kernel launch on the same stream). The previous
|
||||
// host-side guard `if (needs_cooperative)` was evaluated at capture time;
|
||||
// when capture-time max_seq_len <= RADIX_THRESHOLD (always true under
|
||||
// FULL_DECODE_ONLY with max_model_len < 32 K) the memset would NOT be
|
||||
// captured, leaving the workspace state to accumulate across replays.
|
||||
// That's a latent correctness bug if the runtime data ever takes the
|
||||
// radix path, and removes one variable while debugging hangs in the
|
||||
// decode/medium paths.
|
||||
//
|
||||
// Cost is sub-microsecond: state_bytes = num_groups * sizeof(RadixRowState)
|
||||
// is ~3 KB per group, ~100 KB for the largest grids on this hardware.
|
||||
//
|
||||
// Why the memset is required (regardless of which path the kernel takes):
|
||||
// 1. arrival_counter accumulates within a launch and is never reset,
|
||||
// so a prior call leaves it at a large positive value. Without this
|
||||
// reset, the very first wait_ge in the next call sees counter >>
|
||||
// target and returns instantly, breaking the barrier.
|
||||
// 2. The previous in-kernel init only ran in CTA-0 with intra-CTA
|
||||
// __syncthreads(), so it had no happens-before edge to CTA-1+'s
|
||||
// first red_release. cudaMemsetAsync is stream-ordered: the zero
|
||||
// is globally visible before any CTA runs.
|
||||
{
|
||||
cudaError_t mz_err = cudaMemsetAsync(
|
||||
workspace.mutable_data_ptr<uint8_t>(), 0, state_bytes, stream);
|
||||
STD_TORCH_CHECK(mz_err == cudaSuccess,
|
||||
"row_states memset failed: ", cudaGetErrorString(mz_err));
|
||||
}
|
||||
|
||||
P::PersistentTopKParams params;
|
||||
params.input = logits.const_data_ptr<float>();
|
||||
params.output = output.mutable_data_ptr<int32_t>();
|
||||
params.lengths = lengths.const_data_ptr<int32_t>();
|
||||
params.num_rows = static_cast<uint32_t>(num_rows);
|
||||
params.stride = static_cast<uint32_t>(stride);
|
||||
params.top_k = static_cast<uint32_t>(TopK);
|
||||
params.chunk_size = chunk_size;
|
||||
params.row_states = reinterpret_cast<P::RadixRowState*>(
|
||||
workspace.mutable_data_ptr<uint8_t>());
|
||||
params.ctas_per_group = ctas_per_group;
|
||||
params.max_seq_len = static_cast<uint32_t>(max_seq_len);
|
||||
|
||||
#define LAUNCH_PERSISTENT(TOPK_VAL, VS) \
|
||||
do { \
|
||||
auto kernel = &P::persistent_topk_kernel<TOPK_VAL, VS>; \
|
||||
cudaError_t err = cudaFuncSetAttribute( \
|
||||
kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size); \
|
||||
STD_TORCH_CHECK(err == cudaSuccess, \
|
||||
"Failed to set smem: ", cudaGetErrorString(err)); \
|
||||
kernel<<<total_ctas, P::kThreadsPerBlock, smem_size, stream>>>(params); \
|
||||
} while (0)
|
||||
|
||||
if (vec_size == 4) {
|
||||
LAUNCH_PERSISTENT(TopK, 4);
|
||||
} else if (vec_size == 2) {
|
||||
LAUNCH_PERSISTENT(TopK, 2);
|
||||
} else {
|
||||
LAUNCH_PERSISTENT(TopK, 1);
|
||||
}
|
||||
#undef LAUNCH_PERSISTENT
|
||||
}
|
||||
|
||||
cudaError_t err = cudaGetLastError();
|
||||
STD_TORCH_CHECK(err == cudaSuccess,
|
||||
"persistent_topk failed: ", cudaGetErrorString(err));
|
||||
}
|
||||
#endif
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
void persistent_topk(const torch::stable::Tensor& logits,
|
||||
const torch::stable::Tensor& lengths,
|
||||
torch::stable::Tensor& output,
|
||||
torch::stable::Tensor& workspace, int64_t k,
|
||||
int64_t max_seq_len) {
|
||||
#ifndef USE_ROCM
|
||||
STD_TORCH_CHECK(logits.is_cuda(), "logits must be CUDA tensor");
|
||||
STD_TORCH_CHECK(lengths.is_cuda(), "lengths must be CUDA tensor");
|
||||
STD_TORCH_CHECK(output.is_cuda(), "output must be CUDA tensor");
|
||||
STD_TORCH_CHECK(logits.scalar_type() == torch::headeronly::ScalarType::Float,
|
||||
"Only float32 supported");
|
||||
STD_TORCH_CHECK(lengths.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"lengths must be int32");
|
||||
STD_TORCH_CHECK(output.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"output must be int32");
|
||||
STD_TORCH_CHECK(logits.dim() == 2, "logits must be 2D");
|
||||
STD_TORCH_CHECK(lengths.dim() == 1 || lengths.dim() == 2,
|
||||
"lengths must be 1D or 2D");
|
||||
STD_TORCH_CHECK(lengths.is_contiguous(), "lengths must be contiguous");
|
||||
STD_TORCH_CHECK(output.dim() == 2, "output must be 2D");
|
||||
|
||||
const int64_t num_rows = logits.size(0);
|
||||
|
||||
STD_TORCH_CHECK(lengths.numel() == num_rows, "lengths size mismatch");
|
||||
STD_TORCH_CHECK(output.size(0) == num_rows && output.size(1) == k,
|
||||
"output size mismatch");
|
||||
STD_TORCH_CHECK(
|
||||
k == 512 || k == 1024 || k == 2048,
|
||||
"persistent_topk supports k=512, k=1024, or k=2048, got k=", k);
|
||||
|
||||
if (k == 512) {
|
||||
launch_persistent_topk<512>(logits, lengths, output, workspace,
|
||||
max_seq_len);
|
||||
} else if (k == 1024) {
|
||||
launch_persistent_topk<1024>(logits, lengths, output, workspace,
|
||||
max_seq_len);
|
||||
} else {
|
||||
launch_persistent_topk<2048>(logits, lengths, output, workspace,
|
||||
max_seq_len);
|
||||
}
|
||||
#else
|
||||
STD_TORCH_CHECK(false, "persistent_topk is not supported on ROCm");
|
||||
#endif
|
||||
}
|
||||
@@ -263,6 +263,20 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"CUBLAS_M_THRESHOLD, bool has_zp, bool n32k16_reorder) -> Tensor");
|
||||
#endif
|
||||
|
||||
// Merge attn states
|
||||
// Implements section 2.2 of https://www.arxiv.org/pdf/2501.01005
|
||||
// can be used to combine partial attention results (in the split-KV case)
|
||||
ops.def(
|
||||
"merge_attn_states("
|
||||
" Tensor! output,"
|
||||
" Tensor!? output_lse,"
|
||||
" Tensor prefix_output,"
|
||||
" Tensor prefix_lse,"
|
||||
" Tensor suffix_output,"
|
||||
" Tensor suffix_lse,"
|
||||
" int!? prefill_tokens_with_context,"
|
||||
" Tensor? output_scale=None) -> ()");
|
||||
|
||||
// Hadamard transforms
|
||||
// conditionally compiled so impl registration is in source file
|
||||
ops.def("hadacore_transform(Tensor! x, bool inplace) -> Tensor");
|
||||
@@ -319,6 +333,26 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"bool is_neox, Tensor position_ids, "
|
||||
"int forced_token_heads_per_warp=-1) -> ()");
|
||||
|
||||
// Apply repetition penalties to logits in-place.
|
||||
ops.def(
|
||||
"apply_repetition_penalties_(Tensor! logits, Tensor prompt_mask, "
|
||||
"Tensor output_mask, Tensor repetition_penalties) -> ()");
|
||||
|
||||
// Optimized top-k per row operations.
|
||||
ops.def(
|
||||
"top_k_per_row_prefill(Tensor logits, Tensor rowStarts, Tensor rowEnds, "
|
||||
"Tensor! indices, int numRows, int stride0, "
|
||||
"int stride1, int topK) -> ()");
|
||||
|
||||
ops.def(
|
||||
"top_k_per_row_decode(Tensor logits, int next_n, "
|
||||
"Tensor seq_lens, Tensor! indices, "
|
||||
"int numRows, int stride0, int stride1, int topK) -> ()");
|
||||
|
||||
ops.def(
|
||||
"persistent_topk(Tensor logits, Tensor lengths, Tensor! output, "
|
||||
"Tensor workspace, int k, int max_seq_len) -> ()");
|
||||
|
||||
// Activation ops
|
||||
// Activation function used in SwiGLU.
|
||||
ops.def("silu_and_mul(Tensor! result, Tensor input) -> ()");
|
||||
@@ -422,6 +456,24 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"int type, SymInt row, SymInt tokens) -> Tensor");
|
||||
|
||||
ops.def("ggml_moe_get_block_size(int type) -> int");
|
||||
|
||||
// Mamba selective scan kernel
|
||||
ops.def(
|
||||
"selective_scan_fwd(Tensor! u, Tensor! delta,"
|
||||
"Tensor! A, Tensor! B, Tensor! C,"
|
||||
"Tensor? D_, Tensor!? z_, Tensor? delta_bias_,"
|
||||
"bool delta_softplus,"
|
||||
"Tensor? query_start_loc,"
|
||||
"Tensor? cache_indices,"
|
||||
"Tensor? has_initial_state,"
|
||||
"Tensor! ssm_states,"
|
||||
"int null_block_id,"
|
||||
"int block_size,"
|
||||
"Tensor? block_idx_first_scheduled_token,"
|
||||
"Tensor? block_idx_last_scheduled_token,"
|
||||
"Tensor? initial_state_idx,"
|
||||
"Tensor? cu_chunk_seqlen,"
|
||||
"Tensor? last_chunk_indices) -> ()");
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
@@ -469,6 +521,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
// files (allspark_repack.cu and allspark_qgemm_w8a16.cu)
|
||||
#endif
|
||||
|
||||
ops.impl("merge_attn_states", TORCH_BOX(&merge_attn_states));
|
||||
|
||||
// Layernorm kernels (shared CUDA/ROCm)
|
||||
ops.impl("rms_norm", TORCH_BOX(&rms_norm));
|
||||
ops.impl("fused_add_rms_norm", TORCH_BOX(&fused_add_rms_norm));
|
||||
@@ -487,6 +541,13 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("rotary_embedding", TORCH_BOX(&rotary_embedding));
|
||||
ops.impl("fused_qk_norm_rope", TORCH_BOX(&fused_qk_norm_rope));
|
||||
|
||||
// Sampler kernels (shared CUDA/ROCm)
|
||||
ops.impl("apply_repetition_penalties_",
|
||||
TORCH_BOX(&apply_repetition_penalties_));
|
||||
ops.impl("top_k_per_row_prefill", TORCH_BOX(&top_k_per_row_prefill));
|
||||
ops.impl("top_k_per_row_decode", TORCH_BOX(&top_k_per_row_decode));
|
||||
ops.impl("persistent_topk", TORCH_BOX(&persistent_topk));
|
||||
|
||||
// Activation kernels (shared CUDA/ROCm)
|
||||
ops.impl("silu_and_mul", TORCH_BOX(&silu_and_mul));
|
||||
ops.impl("mul_and_silu", TORCH_BOX(&mul_and_silu));
|
||||
@@ -519,6 +580,7 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("ggml_mul_mat_a8", TORCH_BOX(&ggml_mul_mat_a8));
|
||||
ops.impl("ggml_moe_a8", TORCH_BOX(&ggml_moe_a8));
|
||||
ops.impl("ggml_moe_a8_vec", TORCH_BOX(&ggml_moe_a8_vec));
|
||||
ops.impl("selective_scan_fwd", TORCH_BOX(&selective_scan_fwd));
|
||||
}
|
||||
|
||||
// These capability-check functions take only primitive args (no tensors), so
|
||||
|
||||
Reference in New Issue
Block a user