[Kernel] Support DS Mamba tail copy for MTP align mode (#45473)
Signed-off-by: Sungsoo Ha <sungsooh@nvidia.com> Co-authored-by: Thomas Parnell <tom.parnell@gmail.com>
This commit is contained in:
co-authored by
Thomas Parnell
parent
520828789c
commit
4bf699d310
@@ -303,18 +303,14 @@ def get_conv_copy_spec(
|
||||
src_block_id = block_ids[cur_block_idx]
|
||||
offset = num_accepted_tokens - 1
|
||||
if is_conv_state_dim_first():
|
||||
# DS layout: (num_blocks, dim, state_len) — state_len is last.
|
||||
if offset > 0:
|
||||
# Slicing along the last dim yields a non-contiguous view
|
||||
# because features (dim) are strided by state_len.
|
||||
raise NotImplementedError(
|
||||
"DS conv state layout does not yet support speculative "
|
||||
"decoding with mamba_cache_mode='align' "
|
||||
"(num_accepted_tokens > 1)."
|
||||
)
|
||||
# DS offset > 0 is handled by the fused postprocess kernel.
|
||||
assert offset == 0, (
|
||||
"DS conv state with num_accepted_tokens > 1 must be handled by "
|
||||
"the fused postprocess kernel, not get_conv_copy_spec"
|
||||
)
|
||||
src_state = state[src_block_id]
|
||||
else:
|
||||
# SD layout: (num_blocks, state_len, dim) — dim contiguous.
|
||||
# SD layout: (num_blocks, state_len, dim), with dim contiguous.
|
||||
src_state = state[src_block_id, offset:]
|
||||
return MambaCopySpec(
|
||||
start_addr=src_state.data_ptr(), num_elements=src_state.numel()
|
||||
|
||||
Reference in New Issue
Block a user