[Kimi K3][Kernel] Support DS conv-state layout in fused KDA decode kernel (#53396)
Signed-off-by: Canlin Guo <canlinguosdu@gmail.com>
This commit is contained in:
@@ -261,15 +261,16 @@ __device__ __forceinline__ Sum2 block_reduce_sum2_active_for(float x, float y,
|
||||
return {scratch[0], scratch[1]};
|
||||
}
|
||||
|
||||
template <bool kApplyOnorm, bool kUseStaticDecodeLayout = false,
|
||||
int kFixedHeads = 0, int kFixedValueHeads = 0,
|
||||
bool kUseHeadGrid = false, bool kAccumulateOnormSumsq = false,
|
||||
bool kUseActiveQkReduction = false, bool kUseCacheGlobalStore = false,
|
||||
bool kComputeOutputBeforeStore = false, bool kSkipWarpSync = false,
|
||||
bool kPreloadOnormParams = false,
|
||||
bool kPrefetchNextStateChunk = false,
|
||||
bool kUseActiveOnormReduction = false, bool kUpdateConvState = false,
|
||||
bool kUseLowerBound = false, bool kApplyBetaSigmoid = true>
|
||||
template <
|
||||
bool kApplyOnorm, bool kUseStaticDecodeLayout = false, int kFixedHeads = 0,
|
||||
int kFixedValueHeads = 0, bool kUseHeadGrid = false,
|
||||
bool kAccumulateOnormSumsq = false, bool kUseActiveQkReduction = false,
|
||||
bool kUseCacheGlobalStore = false, bool kComputeOutputBeforeStore = false,
|
||||
bool kSkipWarpSync = false, bool kPreloadOnormParams = false,
|
||||
bool kPrefetchNextStateChunk = false, bool kUseActiveOnormReduction = false,
|
||||
bool kUpdateConvState = false, bool kUseLowerBound = false,
|
||||
bool kApplyBetaSigmoid = true, int kConvStateChannelStride = 1,
|
||||
int kConvStateTapStride = 3 * kFixedHeads * kDimK>
|
||||
__global__
|
||||
__launch_bounds__(kThreads, 2) void kda_decode_fusion_many_heads_kernel(
|
||||
const __nv_bfloat16* __restrict__ x_q,
|
||||
@@ -329,7 +330,6 @@ __launch_bounds__(kThreads, 2) void kda_decode_fusion_many_heads_kernel(
|
||||
}
|
||||
|
||||
constexpr int kLocalDim = kFixedHeads * kDimK;
|
||||
constexpr int kPackedDim = 3 * kLocalDim;
|
||||
const int hk_off = i_h * kDimK;
|
||||
const int hv_off = i_hv * kDimV;
|
||||
constexpr int hv_count = kFixedValueHeads;
|
||||
@@ -368,8 +368,10 @@ __launch_bounds__(kThreads, 2) void kda_decode_fusion_many_heads_kernel(
|
||||
__nv_bfloat16 k_shift1 = __float2bfloat16(0.0f);
|
||||
#pragma unroll
|
||||
for (int w = 0; w < kConvStateWidth; ++w) {
|
||||
const __nv_bfloat16 q_state = cs_q_for_slot[hk + w * kPackedDim];
|
||||
const __nv_bfloat16 k_state = cs_k_for_slot[hk + w * kPackedDim];
|
||||
const int cs_idx =
|
||||
hk * kConvStateChannelStride + w * kConvStateTapStride;
|
||||
const __nv_bfloat16 q_state = cs_q_for_slot[cs_idx];
|
||||
const __nv_bfloat16 k_state = cs_k_for_slot[cs_idx];
|
||||
q_acc += __bfloat162float(q_state) *
|
||||
conv_weight_load<kLocalDim>(w_q_t, hk, w);
|
||||
k_acc += __bfloat162float(k_state) *
|
||||
@@ -389,12 +391,16 @@ __launch_bounds__(kThreads, 2) void kda_decode_fusion_many_heads_kernel(
|
||||
k_acc += __bfloat162float(k_new) *
|
||||
conv_weight_load<kLocalDim>(w_k_t, hk, kKernelWidth - 1);
|
||||
|
||||
cs_q_for_slot[hk] = q_shift0;
|
||||
cs_q_for_slot[hk + kPackedDim] = q_shift1;
|
||||
cs_q_for_slot[hk + 2 * kPackedDim] = q_new;
|
||||
cs_k_for_slot[hk] = k_shift0;
|
||||
cs_k_for_slot[hk + kPackedDim] = k_shift1;
|
||||
cs_k_for_slot[hk + 2 * kPackedDim] = k_new;
|
||||
cs_q_for_slot[hk * kConvStateChannelStride] = q_shift0;
|
||||
cs_q_for_slot[hk * kConvStateChannelStride + kConvStateTapStride] =
|
||||
q_shift1;
|
||||
cs_q_for_slot[hk * kConvStateChannelStride + 2 * kConvStateTapStride] =
|
||||
q_new;
|
||||
cs_k_for_slot[hk * kConvStateChannelStride] = k_shift0;
|
||||
cs_k_for_slot[hk * kConvStateChannelStride + kConvStateTapStride] =
|
||||
k_shift1;
|
||||
cs_k_for_slot[hk * kConvStateChannelStride + 2 * kConvStateTapStride] =
|
||||
k_new;
|
||||
|
||||
s_q[k] = silu_fast(q_acc);
|
||||
s_k[k] = silu_fast(k_acc);
|
||||
@@ -418,7 +424,8 @@ __launch_bounds__(kThreads, 2) void kda_decode_fusion_many_heads_kernel(
|
||||
float k_acc = bias_k == nullptr ? 0.0f : bf16_load(bias_k, hk);
|
||||
#pragma unroll
|
||||
for (int w = 0; w < kConvStateWidth; ++w) {
|
||||
const int cs_idx = hk + w * kPackedDim;
|
||||
const int cs_idx =
|
||||
hk * kConvStateChannelStride + w * kConvStateTapStride;
|
||||
q_acc += bf16_load(cs_q_for_slot, cs_idx) *
|
||||
conv_weight_load<kLocalDim>(w_q_t, hk, w);
|
||||
k_acc += bf16_load(cs_k_for_slot, cs_idx) *
|
||||
@@ -453,7 +460,9 @@ __launch_bounds__(kThreads, 2) void kda_decode_fusion_many_heads_kernel(
|
||||
__nv_bfloat16 v_shift1 = __float2bfloat16(0.0f);
|
||||
#pragma unroll
|
||||
for (int w = 0; w < kConvStateWidth; ++w) {
|
||||
const __nv_bfloat16 v_state = cs_v_for_slot[hvv + w * kPackedDim];
|
||||
const int cs_idx =
|
||||
hvv * kConvStateChannelStride + w * kConvStateTapStride;
|
||||
const __nv_bfloat16 v_state = cs_v_for_slot[cs_idx];
|
||||
v_acc += __bfloat162float(v_state) *
|
||||
conv_weight_load<kLocalDim>(w_v_t, hvv, w);
|
||||
if (w == 1) {
|
||||
@@ -465,9 +474,11 @@ __launch_bounds__(kThreads, 2) void kda_decode_fusion_many_heads_kernel(
|
||||
const __nv_bfloat16 v_new = x_v[xv_idx];
|
||||
v_acc += __bfloat162float(v_new) *
|
||||
conv_weight_load<kLocalDim>(w_v_t, hvv, kKernelWidth - 1);
|
||||
cs_v_for_slot[hvv] = v_shift0;
|
||||
cs_v_for_slot[hvv + kPackedDim] = v_shift1;
|
||||
cs_v_for_slot[hvv + 2 * kPackedDim] = v_new;
|
||||
cs_v_for_slot[hvv * kConvStateChannelStride] = v_shift0;
|
||||
cs_v_for_slot[hvv * kConvStateChannelStride + kConvStateTapStride] =
|
||||
v_shift1;
|
||||
cs_v_for_slot[hvv * kConvStateChannelStride + 2 * kConvStateTapStride] =
|
||||
v_new;
|
||||
s_v[v] = silu_fast(v_acc);
|
||||
|
||||
if constexpr (kApplyOnorm && kPreloadOnormParams) {
|
||||
@@ -484,7 +495,8 @@ __launch_bounds__(kThreads, 2) void kda_decode_fusion_many_heads_kernel(
|
||||
float v_acc = bias_v == nullptr ? 0.0f : bf16_load(bias_v, hvv);
|
||||
#pragma unroll
|
||||
for (int w = 0; w < kConvStateWidth; ++w) {
|
||||
const int cs_idx = hvv + w * kPackedDim;
|
||||
const int cs_idx =
|
||||
hvv * kConvStateChannelStride + w * kConvStateTapStride;
|
||||
v_acc += bf16_load(cs_v_for_slot, cs_idx) *
|
||||
conv_weight_load<kLocalDim>(w_v_t, hvv, w);
|
||||
}
|
||||
@@ -723,7 +735,8 @@ __launch_bounds__(kThreads, 2) void kda_decode_fusion_many_heads_kernel(
|
||||
}
|
||||
|
||||
template <int kHeads, bool kApplyOnorm, bool kUpdateConvState,
|
||||
bool kUseLowerBound, bool kApplyBetaSigmoid>
|
||||
bool kUseLowerBound, bool kApplyBetaSigmoid,
|
||||
int kConvStateChannelStride, int kConvStateTapStride>
|
||||
void launch_kda_decode_many_heads_raw(
|
||||
const void* x_q, const void* x_k, const void* x_v, const void* w_q_t,
|
||||
const void* w_k_t, const void* w_v_t, const void* bias_q,
|
||||
@@ -736,7 +749,7 @@ void launch_kda_decode_many_heads_raw(
|
||||
auto kernel = &kda_decode_fusion_many_heads_kernel<
|
||||
kApplyOnorm, true, kHeads, kHeads, true, false, false, false, false,
|
||||
false, true, true, true, kUpdateConvState, kUseLowerBound,
|
||||
kApplyBetaSigmoid>;
|
||||
kApplyBetaSigmoid, kConvStateChannelStride, kConvStateTapStride>;
|
||||
cudaLaunchConfig_t config{};
|
||||
config.gridDim = dim3(B, kHeads);
|
||||
config.blockDim = dim3(kThreads);
|
||||
@@ -776,26 +789,39 @@ void launch_kda_decode_many_heads_selected(
|
||||
const float* a_log, const void* g, const float* dt_bias, const void* beta,
|
||||
const void* onorm_g, const float* onorm_weight,
|
||||
const int* ssm_state_indices, const int* cu_seqlens, float* state,
|
||||
void* out, int B, int H, int HV, bool update_conv_cache, float lower_bound,
|
||||
float scale, float onorm_eps, KdaDecodeStrides strides,
|
||||
cudaStream_t stream) {
|
||||
#define LAUNCH_KDA_DECODE(NUM_HEADS) \
|
||||
do { \
|
||||
if (update_conv_cache) { \
|
||||
launch_kda_decode_many_heads_raw<NUM_HEADS, kApplyOnorm, true, \
|
||||
kUseLowerBound, kApplyBetaSigmoid>( \
|
||||
x_q, x_k, x_v, w_q_t, w_k_t, w_v_t, bias_q, bias_k, bias_v, cs_q, \
|
||||
cs_k, cs_v, a_log, g, dt_bias, beta, onorm_g, onorm_weight, \
|
||||
ssm_state_indices, cu_seqlens, state, out, B, H, HV, lower_bound, \
|
||||
scale, onorm_eps, strides, stream); \
|
||||
} else { \
|
||||
launch_kda_decode_many_heads_raw<NUM_HEADS, kApplyOnorm, false, \
|
||||
kUseLowerBound, kApplyBetaSigmoid>( \
|
||||
x_q, x_k, x_v, w_q_t, w_k_t, w_v_t, bias_q, bias_k, bias_v, cs_q, \
|
||||
cs_k, cs_v, a_log, g, dt_bias, beta, onorm_g, onorm_weight, \
|
||||
ssm_state_indices, cu_seqlens, state, out, B, H, HV, lower_bound, \
|
||||
scale, onorm_eps, strides, stream); \
|
||||
} \
|
||||
void* out, int B, int H, int HV, bool update_conv_cache,
|
||||
bool conv_state_is_ds, float lower_bound, float scale, float onorm_eps,
|
||||
KdaDecodeStrides strides, cudaStream_t stream) {
|
||||
#define LAUNCH_KDA_DECODE_LAYOUT(NUM_HEADS, CS_CHANNEL_STRIDE, CS_TAP_STRIDE) \
|
||||
do { \
|
||||
if (update_conv_cache) { \
|
||||
launch_kda_decode_many_heads_raw<NUM_HEADS, kApplyOnorm, true, \
|
||||
kUseLowerBound, kApplyBetaSigmoid, \
|
||||
CS_CHANNEL_STRIDE, CS_TAP_STRIDE>( \
|
||||
x_q, x_k, x_v, w_q_t, w_k_t, w_v_t, bias_q, bias_k, bias_v, cs_q, \
|
||||
cs_k, cs_v, a_log, g, dt_bias, beta, onorm_g, onorm_weight, \
|
||||
ssm_state_indices, cu_seqlens, state, out, B, H, HV, lower_bound, \
|
||||
scale, onorm_eps, strides, stream); \
|
||||
} else { \
|
||||
launch_kda_decode_many_heads_raw<NUM_HEADS, kApplyOnorm, false, \
|
||||
kUseLowerBound, kApplyBetaSigmoid, \
|
||||
CS_CHANNEL_STRIDE, CS_TAP_STRIDE>( \
|
||||
x_q, x_k, x_v, w_q_t, w_k_t, w_v_t, bias_q, bias_k, bias_v, cs_q, \
|
||||
cs_k, cs_v, a_log, g, dt_bias, beta, onorm_g, onorm_weight, \
|
||||
ssm_state_indices, cu_seqlens, state, out, B, H, HV, lower_bound, \
|
||||
scale, onorm_eps, strides, stream); \
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
// SD packs (tap, channel) per slot while DS packs (channel, tap); both are
|
||||
// baked into the kernel as compile-time strides.
|
||||
#define LAUNCH_KDA_DECODE(NUM_HEADS) \
|
||||
do { \
|
||||
if (conv_state_is_ds) { \
|
||||
LAUNCH_KDA_DECODE_LAYOUT(NUM_HEADS, kConvStateWidth, 1); \
|
||||
} else { \
|
||||
LAUNCH_KDA_DECODE_LAYOUT(NUM_HEADS, 1, 3 * (NUM_HEADS) * kDimK); \
|
||||
} \
|
||||
} while (false)
|
||||
|
||||
switch (H) {
|
||||
@@ -816,6 +842,7 @@ void launch_kda_decode_many_heads_selected(
|
||||
}
|
||||
|
||||
#undef LAUNCH_KDA_DECODE
|
||||
#undef LAUNCH_KDA_DECODE_LAYOUT
|
||||
}
|
||||
|
||||
struct KdaDecodeLaunchParams {
|
||||
@@ -845,6 +872,7 @@ struct KdaDecodeLaunchParams {
|
||||
int H;
|
||||
int HV;
|
||||
bool update_conv_cache;
|
||||
bool conv_state_is_ds;
|
||||
float lower_bound;
|
||||
float scale;
|
||||
float onorm_eps;
|
||||
@@ -859,8 +887,8 @@ void launch_kda_decode_selected_backend(const KdaDecodeLaunchParams& p) {
|
||||
p.x_q, p.x_k, p.x_v, p.w_q_t, p.w_k_t, p.w_v_t, p.bias_q, p.bias_k,
|
||||
p.bias_v, p.cs_q, p.cs_k, p.cs_v, p.a_log, p.g, p.dt_bias, p.beta,
|
||||
p.onorm_g, p.onorm_weight, p.ssm_state_indices, p.cu_seqlens, p.state,
|
||||
p.out, p.B, p.H, p.HV, p.update_conv_cache, p.lower_bound, p.scale,
|
||||
p.onorm_eps, p.strides, p.stream);
|
||||
p.out, p.B, p.H, p.HV, p.update_conv_cache, p.conv_state_is_ds,
|
||||
p.lower_bound, p.scale, p.onorm_eps, p.strides, p.stream);
|
||||
}
|
||||
|
||||
template <bool kApplyOnorm, bool kUseLowerBound>
|
||||
@@ -908,6 +936,8 @@ extern "C" void launch_kda_decode_many_heads_cuda(
|
||||
cudaStream_t stream) {
|
||||
const KdaDecodeStrides strides{raw_strides[0], raw_strides[1], raw_strides[2],
|
||||
raw_strides[3], raw_strides[4]};
|
||||
const bool conv_state_is_ds =
|
||||
raw_strides[5] == kConvStateWidth && raw_strides[6] == 1;
|
||||
const KdaDecodeLaunchParams params{x_q,
|
||||
x_k,
|
||||
x_v,
|
||||
@@ -934,6 +964,7 @@ extern "C" void launch_kda_decode_many_heads_cuda(
|
||||
H,
|
||||
HV,
|
||||
update_conv_cache,
|
||||
conv_state_is_ds,
|
||||
lower_bound,
|
||||
scale,
|
||||
onorm_eps,
|
||||
@@ -1027,10 +1058,15 @@ void fused_kda_decode(
|
||||
"out must have shape [1, B, H, 128]");
|
||||
STD_TORCH_CHECK(x.stride(1) == 1,
|
||||
"x must be contiguous in its channel dimension");
|
||||
STD_TORCH_CHECK(conv_state.stride(0) >= 3 * dim * (kConvWidth - 1) &&
|
||||
conv_state.stride(1) == 1 &&
|
||||
conv_state.stride(2) == 3 * dim,
|
||||
"conv_state must use the SD cache layout");
|
||||
// Both conv-state cache layouts are supported: SD stores (state_len, dim)
|
||||
// per block while DS stores (dim, state_len). The kernel only needs the
|
||||
// per-plane strides.
|
||||
STD_TORCH_CHECK(
|
||||
conv_state.stride(0) >= 3 * dim * (kConvWidth - 1) &&
|
||||
((conv_state.stride(1) == 1 && conv_state.stride(2) == 3 * dim) ||
|
||||
(conv_state.stride(1) == kConvWidth - 1 &&
|
||||
conv_state.stride(2) == 1)),
|
||||
"conv_state must use the SD or DS cache layout");
|
||||
STD_TORCH_CHECK(state.stride(0) >= num_heads * kHeadDim * kHeadDim &&
|
||||
state.stride(1) == kHeadDim * kHeadDim &&
|
||||
state.stride(2) == kHeadDim && state.stride(3) == 1,
|
||||
@@ -1096,9 +1132,10 @@ void fused_kda_decode(
|
||||
int64_t const conv_segment_bytes =
|
||||
dim * conv_state.stride(1) * sizeof(__nv_bfloat16);
|
||||
int64_t const bias_segment_bytes = dim * sizeof(float);
|
||||
std::array<int64_t, 5> const strides{
|
||||
std::array<int64_t, 7> const strides{
|
||||
x.stride(0), raw_beta.stride(1), output_gate_row_stride,
|
||||
conv_state.stride(0), state.stride(0),
|
||||
conv_state.stride(0), state.stride(0), conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
};
|
||||
bool const use_lower_bound = lower_bound.has_value();
|
||||
float const lower_bound_value =
|
||||
|
||||
@@ -147,7 +147,8 @@ __device__ __forceinline__ Sum3 block_reduce_sum3(float x, float y, float z,
|
||||
scratch[3 * kGroups + 2]};
|
||||
}
|
||||
|
||||
template <bool kApplyOnorm, int kFixedHeads, bool kUpdateConvState,
|
||||
template <bool kApplyOnorm, int kConvStateChannelStride,
|
||||
int kConvStateTapStride, int kFixedHeads, bool kUpdateConvState,
|
||||
bool kUseLowerBound, bool kApplyBetaSigmoid>
|
||||
__global__ __launch_bounds__(kThreads) void kda_decode_fusion_kernel(
|
||||
const bf16_t* __restrict__ x_q, const bf16_t* __restrict__ x_k,
|
||||
@@ -186,7 +187,6 @@ __global__ __launch_bounds__(kThreads) void kda_decode_fusion_kernel(
|
||||
}
|
||||
return;
|
||||
}
|
||||
constexpr int kPackedDim = 3 * kLocalDim;
|
||||
const int hk_off = i_h * kDimK;
|
||||
const int hv_off = i_hv * kDimV;
|
||||
float* const state_for_slot = state + slot * strides.state_slot;
|
||||
@@ -242,8 +242,10 @@ __global__ __launch_bounds__(kThreads) void kda_decode_fusion_kernel(
|
||||
bf16_t k_shift1 = __float2bfloat16(0.0f);
|
||||
#pragma unroll
|
||||
for (int w = 0; w < kConvStateWidth; ++w) {
|
||||
const bf16_t q_state = cs_q_for_slot[hk + w * kPackedDim];
|
||||
const bf16_t k_state = cs_k_for_slot[hk + w * kPackedDim];
|
||||
const int cs_idx =
|
||||
hk * kConvStateChannelStride + w * kConvStateTapStride;
|
||||
const bf16_t q_state = cs_q_for_slot[cs_idx];
|
||||
const bf16_t k_state = cs_k_for_slot[cs_idx];
|
||||
q_acc += __bfloat162float(q_state) *
|
||||
conv_weight_load<kLocalDim>(w_q_t, hk, w);
|
||||
k_acc += __bfloat162float(k_state) *
|
||||
@@ -264,16 +266,21 @@ __global__ __launch_bounds__(kThreads) void kda_decode_fusion_kernel(
|
||||
k_acc += __bfloat162float(k_new) *
|
||||
conv_weight_load<kLocalDim>(w_k_t, hk, kKernelWidth - 1);
|
||||
|
||||
cs_q_for_slot[hk] = q_shift0;
|
||||
cs_q_for_slot[hk + kPackedDim] = q_shift1;
|
||||
cs_q_for_slot[hk + 2 * kPackedDim] = q_new;
|
||||
cs_k_for_slot[hk] = k_shift0;
|
||||
cs_k_for_slot[hk + kPackedDim] = k_shift1;
|
||||
cs_k_for_slot[hk + 2 * kPackedDim] = k_new;
|
||||
cs_q_for_slot[hk * kConvStateChannelStride] = q_shift0;
|
||||
cs_q_for_slot[hk * kConvStateChannelStride + kConvStateTapStride] =
|
||||
q_shift1;
|
||||
cs_q_for_slot[hk * kConvStateChannelStride + 2 * kConvStateTapStride] =
|
||||
q_new;
|
||||
cs_k_for_slot[hk * kConvStateChannelStride] = k_shift0;
|
||||
cs_k_for_slot[hk * kConvStateChannelStride + kConvStateTapStride] =
|
||||
k_shift1;
|
||||
cs_k_for_slot[hk * kConvStateChannelStride + 2 * kConvStateTapStride] =
|
||||
k_new;
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int w = 0; w < kConvStateWidth; ++w) {
|
||||
const int cs_idx = hk + w * kPackedDim;
|
||||
const int cs_idx =
|
||||
hk * kConvStateChannelStride + w * kConvStateTapStride;
|
||||
q_acc += bf16_load(cs_q_for_slot, cs_idx) *
|
||||
conv_weight_load<kLocalDim>(w_q_t, hk, w);
|
||||
k_acc += bf16_load(cs_k_for_slot, cs_idx) *
|
||||
@@ -307,7 +314,9 @@ __global__ __launch_bounds__(kThreads) void kda_decode_fusion_kernel(
|
||||
bf16_t v_shift1 = __float2bfloat16(0.0f);
|
||||
#pragma unroll
|
||||
for (int w = 0; w < kConvStateWidth; ++w) {
|
||||
const bf16_t v_state = cs_v_for_slot[hvv + w * kPackedDim];
|
||||
const int cs_idx =
|
||||
hvv * kConvStateChannelStride + w * kConvStateTapStride;
|
||||
const bf16_t v_state = cs_v_for_slot[cs_idx];
|
||||
v_acc += __bfloat162float(v_state) *
|
||||
conv_weight_load<kLocalDim>(w_v_t, hvv, w);
|
||||
if (w == 1) {
|
||||
@@ -320,13 +329,16 @@ __global__ __launch_bounds__(kThreads) void kda_decode_fusion_kernel(
|
||||
const bf16_t v_new = x_v[xv_idx];
|
||||
v_acc += __bfloat162float(v_new) *
|
||||
conv_weight_load<kLocalDim>(w_v_t, hvv, kKernelWidth - 1);
|
||||
cs_v_for_slot[hvv] = v_shift0;
|
||||
cs_v_for_slot[hvv + kPackedDim] = v_shift1;
|
||||
cs_v_for_slot[hvv + 2 * kPackedDim] = v_new;
|
||||
cs_v_for_slot[hvv * kConvStateChannelStride] = v_shift0;
|
||||
cs_v_for_slot[hvv * kConvStateChannelStride + kConvStateTapStride] =
|
||||
v_shift1;
|
||||
cs_v_for_slot[hvv * kConvStateChannelStride + 2 * kConvStateTapStride] =
|
||||
v_new;
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int w = 0; w < kConvStateWidth; ++w) {
|
||||
const int cs_idx = hvv + w * kPackedDim;
|
||||
const int cs_idx =
|
||||
hvv * kConvStateChannelStride + w * kConvStateTapStride;
|
||||
v_acc += bf16_load(cs_v_for_slot, cs_idx) *
|
||||
conv_weight_load<kLocalDim>(w_v_t, hvv, w);
|
||||
}
|
||||
@@ -461,7 +473,8 @@ __global__ __launch_bounds__(kThreads) void kda_decode_fusion_kernel(
|
||||
}
|
||||
|
||||
template <int kHeads, bool kApplyOnorm, bool kUpdateConvState,
|
||||
bool kUseLowerBound, bool kApplyBetaSigmoid>
|
||||
bool kUseLowerBound, bool kApplyBetaSigmoid,
|
||||
int kConvStateChannelStride, int kConvStateTapStride>
|
||||
void launch_kda_decode_raw(
|
||||
const void* x_q, const void* x_k, const void* x_v, const void* w_q_t,
|
||||
const void* w_k_t, const void* w_v_t, const void* bias_q,
|
||||
@@ -471,7 +484,8 @@ void launch_kda_decode_raw(
|
||||
const int* ssm_state_indices, float* state, void* out, int B,
|
||||
float lower_bound, float scale, float onorm_eps, KdaDecodeStrides strides,
|
||||
hipStream_t stream) {
|
||||
kda_decode_fusion_kernel<kApplyOnorm, kHeads, kUpdateConvState,
|
||||
kda_decode_fusion_kernel<kApplyOnorm, kConvStateChannelStride,
|
||||
kConvStateTapStride, kHeads, kUpdateConvState,
|
||||
kUseLowerBound, kApplyBetaSigmoid>
|
||||
<<<dim3(kHeads, B), dim3(kThreads), 0, stream>>>(
|
||||
reinterpret_cast<const bf16_t*>(x_q),
|
||||
@@ -517,6 +531,7 @@ struct KdaDecodeLaunchParams {
|
||||
int B;
|
||||
int H;
|
||||
bool update_conv_cache;
|
||||
bool conv_state_is_ds;
|
||||
float lower_bound;
|
||||
float scale;
|
||||
float onorm_eps;
|
||||
@@ -527,19 +542,29 @@ struct KdaDecodeLaunchParams {
|
||||
template <int kHeads, bool kApplyOnorm, bool kUseLowerBound,
|
||||
bool kApplyBetaSigmoid>
|
||||
void dispatch_kda_decode_conv(const KdaDecodeLaunchParams& p) {
|
||||
#define LAUNCH_KDA_DECODE(UPDATE_CONV) \
|
||||
launch_kda_decode_raw<kHeads, kApplyOnorm, UPDATE_CONV, kUseLowerBound, \
|
||||
kApplyBetaSigmoid>( \
|
||||
p.x_q, p.x_k, p.x_v, p.w_q_t, p.w_k_t, p.w_v_t, p.bias_q, p.bias_k, \
|
||||
p.bias_v, p.cs_q, p.cs_k, p.cs_v, p.a_log, p.g, p.dt_bias, p.beta, \
|
||||
p.onorm_g, p.onorm_weight, p.ssm_state_indices, p.state, p.out, p.B, \
|
||||
#define LAUNCH_KDA_DECODE_LAYOUT(UPDATE_CONV, CS_CHANNEL_STRIDE, \
|
||||
CS_TAP_STRIDE) \
|
||||
launch_kda_decode_raw<kHeads, kApplyOnorm, UPDATE_CONV, kUseLowerBound, \
|
||||
kApplyBetaSigmoid, CS_CHANNEL_STRIDE, CS_TAP_STRIDE>( \
|
||||
p.x_q, p.x_k, p.x_v, p.w_q_t, p.w_k_t, p.w_v_t, p.bias_q, p.bias_k, \
|
||||
p.bias_v, p.cs_q, p.cs_k, p.cs_v, p.a_log, p.g, p.dt_bias, p.beta, \
|
||||
p.onorm_g, p.onorm_weight, p.ssm_state_indices, p.state, p.out, p.B, \
|
||||
p.lower_bound, p.scale, p.onorm_eps, p.strides, p.stream)
|
||||
// SD packs (tap, channel) per slot while DS packs (channel, tap); both are
|
||||
// baked into the kernel as compile-time strides.
|
||||
#define LAUNCH_KDA_DECODE(UPDATE_CONV) \
|
||||
if (p.conv_state_is_ds) { \
|
||||
LAUNCH_KDA_DECODE_LAYOUT(UPDATE_CONV, kConvStateWidth, 1); \
|
||||
} else { \
|
||||
LAUNCH_KDA_DECODE_LAYOUT(UPDATE_CONV, 1, 3 * kHeads * kDimK); \
|
||||
}
|
||||
if (p.update_conv_cache) {
|
||||
LAUNCH_KDA_DECODE(true);
|
||||
} else {
|
||||
LAUNCH_KDA_DECODE(false);
|
||||
}
|
||||
#undef LAUNCH_KDA_DECODE
|
||||
#undef LAUNCH_KDA_DECODE_LAYOUT
|
||||
}
|
||||
|
||||
template <bool kApplyOnorm, bool kUseLowerBound, bool kApplyBetaSigmoid>
|
||||
@@ -686,10 +711,12 @@ void fused_kda_decode(
|
||||
"out must have shape [1, B, H, 128]");
|
||||
STD_TORCH_CHECK(x.stride(1) == 1,
|
||||
"x must be contiguous in its channel dimension");
|
||||
STD_TORCH_CHECK(conv_state.stride(0) >= 3 * dim * (kConvWidth - 1) &&
|
||||
conv_state.stride(1) == 1 &&
|
||||
conv_state.stride(2) == 3 * dim,
|
||||
"conv_state must use the SD cache layout");
|
||||
STD_TORCH_CHECK(
|
||||
conv_state.stride(0) >= 3 * dim * (kConvWidth - 1) &&
|
||||
((conv_state.stride(1) == 1 && conv_state.stride(2) == 3 * dim) ||
|
||||
(conv_state.stride(1) == kConvWidth - 1 &&
|
||||
conv_state.stride(2) == 1)),
|
||||
"conv_state must use the SD or DS cache layout");
|
||||
STD_TORCH_CHECK(state.stride(0) >= num_heads * kHeadDim * kHeadDim &&
|
||||
state.stride(1) == kHeadDim * kHeadDim &&
|
||||
state.stride(2) == kHeadDim && state.stride(3) == 1,
|
||||
@@ -761,6 +788,8 @@ void fused_kda_decode(
|
||||
|
||||
torch::stable::accelerator::DeviceGuard const device_guard(
|
||||
x.get_device_index());
|
||||
bool const conv_state_is_ds =
|
||||
conv_state.stride(1) == kConvStateWidth && conv_state.stride(2) == 1;
|
||||
KdaDecodeLaunchParams const params{
|
||||
x_ptr,
|
||||
x_ptr + segment_bytes,
|
||||
@@ -786,6 +815,7 @@ void fused_kda_decode(
|
||||
batch_size,
|
||||
static_cast<int>(num_heads),
|
||||
true,
|
||||
conv_state_is_ds,
|
||||
lower_bound_value,
|
||||
0.08838834764831845f,
|
||||
static_cast<float>(norm_eps),
|
||||
|
||||
Reference in New Issue
Block a user