[Model] Reduce redundant computations in mamba2 blocks for Bamba-9B (#15423)
Signed-off-by: Chih-Chieh-Yang <7364402+cyang49@users.noreply.github.com> Co-authored-by: Yu Chin Fabian Lim <flim@sg.ibm.com>
This commit is contained in:
co-authored by
Yu Chin Fabian Lim
parent
5fbab20e02
commit
daefed052c
@@ -5,8 +5,6 @@
|
||||
|
||||
# ruff: noqa: E501,SIM102
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
import triton
|
||||
import triton.language as tl
|
||||
@@ -442,40 +440,6 @@ def _chunk_scan_fwd_kernel(
|
||||
(offs_out_n[None, :] < hdim))
|
||||
|
||||
|
||||
def _seq_idx_to_chunk_indices_offsets(seq_idx, chunk_size: int):
|
||||
|
||||
# convert seq_idx to chunk indices and offsets
|
||||
# - derive the cu_seqlens
|
||||
_, cu_seqlens = torch.where(seq_idx.diff())
|
||||
cu_seqlens += 1
|
||||
|
||||
# outputs will have length expansion of chunks that do not divide
|
||||
# chunk_size
|
||||
N = math.ceil(seq_idx.shape[-1] / chunk_size) + (cu_seqlens % chunk_size
|
||||
> 0).sum()
|
||||
chunk_indices = torch.arange(N, dtype=torch.int, device=seq_idx.device)
|
||||
chunk_offsets = torch.zeros((N, ), dtype=torch.int, device=seq_idx.device)
|
||||
|
||||
cu_seqlens = cu_seqlens.tolist() + [seq_idx.shape[-1]]
|
||||
p = 0 # num of insertions
|
||||
for s, e in zip(cu_seqlens[:-1], cu_seqlens[1:]):
|
||||
|
||||
# if does not divide chunk_size, then there is one chunk insertion
|
||||
p += (s % chunk_size > 0)
|
||||
|
||||
# get the dimensions
|
||||
# - the + 1 for _e is to shift the boundary by one chunk
|
||||
# - this shifting is not needed if chunk_size divides e
|
||||
_s, _e = s // chunk_size + p, e // chunk_size + p + (e % chunk_size
|
||||
> 0)
|
||||
|
||||
# adjust inidces and offsets
|
||||
chunk_indices[_s:_e] -= p
|
||||
chunk_offsets[_s] = s % chunk_size
|
||||
|
||||
return chunk_indices, chunk_offsets
|
||||
|
||||
|
||||
def _chunk_scan_fwd(
|
||||
cb,
|
||||
x,
|
||||
@@ -486,6 +450,8 @@ def _chunk_scan_fwd(
|
||||
D=None,
|
||||
z=None,
|
||||
seq_idx=None,
|
||||
chunk_indices=None,
|
||||
chunk_offsets=None,
|
||||
initial_states=None,
|
||||
):
|
||||
batch, seqlen, nheads, headdim = x.shape
|
||||
@@ -502,7 +468,6 @@ def _chunk_scan_fwd(
|
||||
assert dA_cumsum.shape == (batch, nheads, nchunks, chunk_size)
|
||||
assert states.shape == (batch, nchunks, nheads, headdim, dstate)
|
||||
|
||||
chunk_indices, chunk_offsets = None, None
|
||||
if seq_idx is not None:
|
||||
assert seq_idx.shape == (batch, seqlen)
|
||||
|
||||
@@ -510,15 +475,19 @@ def _chunk_scan_fwd(
|
||||
# with initial states, we need to take care of how
|
||||
# seq_idx crosses the boundaries
|
||||
assert batch == 1, "chunk scan only supports initial states with batch 1"
|
||||
assert initial_states.shape == (seq_idx[0].max() + 1, nheads,
|
||||
headdim, dstate)
|
||||
|
||||
if initial_states.shape[0] == 1:
|
||||
# no in this case no point to use initial states
|
||||
initial_states = None
|
||||
else:
|
||||
chunk_indices, chunk_offsets = _seq_idx_to_chunk_indices_offsets(
|
||||
seq_idx, chunk_size)
|
||||
assert chunk_indices is not None and chunk_offsets is not None, \
|
||||
(
|
||||
"chunk_indices and chunk_offsets should have been set"
|
||||
)
|
||||
else:
|
||||
chunk_indices, chunk_offsets = None, None
|
||||
else:
|
||||
chunk_indices, chunk_offsets = None, None
|
||||
|
||||
# Allocates output.
|
||||
out = torch.empty(batch,
|
||||
|
||||
@@ -30,6 +30,8 @@ def _mamba_chunk_scan_combined_fwd(x,
|
||||
dt_bias=None,
|
||||
initial_states=None,
|
||||
seq_idx=None,
|
||||
chunk_indices=None,
|
||||
chunk_offsets=None,
|
||||
cu_seqlens=None,
|
||||
dt_softplus=False,
|
||||
dt_limit=(0.0, float("inf"))):
|
||||
@@ -96,7 +98,7 @@ def _mamba_chunk_scan_combined_fwd(x,
|
||||
# 3. Compute the inter-chunk SSM recurrence; produces correct SSM states at chunk boundaries
|
||||
# (middle term of factorization of off-diag blocks; A terms)
|
||||
# - for handling chunked prefill, this requires i) initial_states
|
||||
# ii) seq_idx and iii) has_cu_seqlens to be all specified.
|
||||
# ii) seq_idx and iii) is_cont_batched to be all specified.
|
||||
# - When a new seq_idx is detected, we will stop passing the prev_state
|
||||
# and switch accordingly to the init_state corresponding to the new seq_idx.
|
||||
# - this will ensure that states will be updated with the rightmost flushed seq_idx
|
||||
@@ -141,6 +143,8 @@ def _mamba_chunk_scan_combined_fwd(x,
|
||||
D=D,
|
||||
z=z,
|
||||
seq_idx=seq_idx,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
initial_states=initial_states,
|
||||
)
|
||||
if cu_seqlens is None:
|
||||
@@ -170,6 +174,8 @@ def mamba_chunk_scan_combined(x,
|
||||
dt_bias=None,
|
||||
initial_states=None,
|
||||
seq_idx=None,
|
||||
chunk_indices=None,
|
||||
chunk_offsets=None,
|
||||
cu_seqlens=None,
|
||||
dt_softplus=False,
|
||||
dt_limit=(0.0, float("inf")),
|
||||
@@ -210,6 +216,8 @@ def mamba_chunk_scan_combined(x,
|
||||
dt_bias=dt_bias,
|
||||
initial_states=initial_states,
|
||||
seq_idx=seq_idx,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets,
|
||||
cu_seqlens=cu_seqlens,
|
||||
dt_softplus=dt_softplus,
|
||||
dt_limit=dt_limit)
|
||||
|
||||
@@ -150,8 +150,6 @@ def _state_passing_fwd(
|
||||
# are used for continuous batching. In which case we
|
||||
# require seq_idx to be provided
|
||||
assert seq_idx is not None, ""
|
||||
assert initial_states.shape == (seq_idx.max().item() + 1, nheads,
|
||||
dim)
|
||||
else:
|
||||
# - this is the regular batching case, where initial
|
||||
# states are used are for each example of the batch.
|
||||
|
||||
Reference in New Issue
Block a user