[Bugfix] Make DSV4 sparse MLA work end-to-end for plain decode, MTP, and DSpark (#51538)

This commit is contained in:
Gabriel Wu
2026-08-15 07:01:32 -07:00
committed by GitHub
parent ac2ae8798c
commit 97388c44f9
20 changed files with 797 additions and 120 deletions
+4 -1
View File
@@ -484,7 +484,10 @@ __device__ void large_topk(const float* __restrict__ row_input,
template <uint32_t TopK, uint32_t CS>
__device__ void cooperative_topk_body(CooperativeTopKParams<TopK> params) {
const auto rank = blockIdx.y, row = blockIdx.x, tx = threadIdx.x;
const auto sl = params.lengths[row];
// Clamp at 0: `sl` is compared signed here but cast to uint32 below, so a
// negative length would otherwise emit indices 0..TopK-1 as valid instead
// of the -1 padding.
const int32_t sl = params.lengths[row] > 0 ? params.lengths[row] : 0;
int32_t* out = params.output + row * TopK;
const float* in = params.input + row * params.stride;
+20 -1
View File
@@ -906,7 +906,26 @@ __global__ void __launch_bounds__(kThreadsPerBlock, 2)
uint32_t row_idx = group_id + iter * num_groups;
if (row_idx >= params.num_rows) break;
const uint32_t seq_len = params.lengths[row_idx];
// Clamp the row length before any decision is made on it.
//
// `lengths` is int32 and is consumed here as uint32, so a negative value
// (e.g. a padded decode slot whose per-token context length underflowed)
// would reinterpret as ~4e9 and sail past every threshold below. Any
// value beyond the row width would also read into the next row.
//
// Clamping to max_seq_len additionally keeps this per-row decision
// consistent with the `cta_in_group != 0` early exit above, which is
// taken from the host-side scalar: when max_seq_len <= RADIX_THRESHOLD
// the non-leader CTAs return immediately, so a leader that reached the
// cooperative radix path would wait on the inter-CTA barrier for peers
// that no longer exist and spin until the kernel is killed.
const int32_t raw_len = params.lengths[row_idx];
const uint32_t row_bound =
params.stride < params.max_seq_len ? params.stride : params.max_seq_len;
const uint32_t non_negative_len =
raw_len > 0 ? static_cast<uint32_t>(raw_len) : 0u;
const uint32_t seq_len =
non_negative_len < row_bound ? non_negative_len : row_bound;
int32_t* row_output = params.output + row_idx * params.top_k;
const float* row_input = params.input + row_idx * params.stride;