[Bugfix] Make DSV4 sparse MLA work end-to-end for plain decode, MTP, and DSpark (#51538)
This commit is contained in:
@@ -484,7 +484,10 @@ __device__ void large_topk(const float* __restrict__ row_input,
|
||||
template <uint32_t TopK, uint32_t CS>
|
||||
__device__ void cooperative_topk_body(CooperativeTopKParams<TopK> params) {
|
||||
const auto rank = blockIdx.y, row = blockIdx.x, tx = threadIdx.x;
|
||||
const auto sl = params.lengths[row];
|
||||
// Clamp at 0: `sl` is compared signed here but cast to uint32 below, so a
|
||||
// negative length would otherwise emit indices 0..TopK-1 as valid instead
|
||||
// of the -1 padding.
|
||||
const int32_t sl = params.lengths[row] > 0 ? params.lengths[row] : 0;
|
||||
int32_t* out = params.output + row * TopK;
|
||||
const float* in = params.input + row * params.stride;
|
||||
|
||||
|
||||
@@ -906,7 +906,26 @@ __global__ void __launch_bounds__(kThreadsPerBlock, 2)
|
||||
uint32_t row_idx = group_id + iter * num_groups;
|
||||
if (row_idx >= params.num_rows) break;
|
||||
|
||||
const uint32_t seq_len = params.lengths[row_idx];
|
||||
// Clamp the row length before any decision is made on it.
|
||||
//
|
||||
// `lengths` is int32 and is consumed here as uint32, so a negative value
|
||||
// (e.g. a padded decode slot whose per-token context length underflowed)
|
||||
// would reinterpret as ~4e9 and sail past every threshold below. Any
|
||||
// value beyond the row width would also read into the next row.
|
||||
//
|
||||
// Clamping to max_seq_len additionally keeps this per-row decision
|
||||
// consistent with the `cta_in_group != 0` early exit above, which is
|
||||
// taken from the host-side scalar: when max_seq_len <= RADIX_THRESHOLD
|
||||
// the non-leader CTAs return immediately, so a leader that reached the
|
||||
// cooperative radix path would wait on the inter-CTA barrier for peers
|
||||
// that no longer exist and spin until the kernel is killed.
|
||||
const int32_t raw_len = params.lengths[row_idx];
|
||||
const uint32_t row_bound =
|
||||
params.stride < params.max_seq_len ? params.stride : params.max_seq_len;
|
||||
const uint32_t non_negative_len =
|
||||
raw_len > 0 ? static_cast<uint32_t>(raw_len) : 0u;
|
||||
const uint32_t seq_len =
|
||||
non_negative_len < row_bound ? non_negative_len : row_bound;
|
||||
int32_t* row_output = params.output + row_idx * params.top_k;
|
||||
const float* row_input = params.input + row_idx * params.stride;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user