[Kernel] Triton implementation of causal-conv1d for Mamba-based models (#18218)
Signed-off-by: Tuan M. Hoang-Trong <tmhoangt@us.ibm.com> Co-authored-by: Tuan M. Hoang-Trong <tmhoangt@us.ibm.com> Co-authored-by: Tyler Michael Smith <tysmith@redhat.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
co-authored by
Tuan M. Hoang-Trong
Tyler Michael Smith
Tyler Michael Smith
parent
31b96d1c64
commit
47043eb678
@@ -1,14 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.attention.backends.abstract import AttentionMetadata
|
||||
from vllm.attention.backends.placeholder_attn import (
|
||||
PlaceholderAttentionMetadata)
|
||||
from vllm.attention.backends.utils import PAD_SLOT_ID
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mamba_attn import (
|
||||
Mamba2AttentionMetadata, _query_start_loc_to_chunk_indices_offsets)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -21,6 +25,29 @@ class Mamba2Metadata:
|
||||
seq_idx: torch.Tensor
|
||||
chunk_indices: torch.Tensor
|
||||
chunk_offsets: torch.Tensor
|
||||
"""
|
||||
With continuous batching layout of `x` in vLLM, to enable a Triton program
|
||||
to handle a request in parallel, two supporting tensors are used
|
||||
(batch_ptr, token_chunk_offset_ptr)
|
||||
BLOCK_M = the # tokens to be handled by a Triton program
|
||||
(can be customized for different hardware)
|
||||
|
||||
nums_dict:
|
||||
tracks the data associated with a given value of BLOCK_M
|
||||
BLOCK_M = #tokens handled by a Triton program
|
||||
cu_seqlen: total tokens per batch
|
||||
(used as flag to update other data at each new input)
|
||||
batch_ptr: tracks batch-id handled by the Triton program
|
||||
token_chunk_offset_ptr: tracks token group_idx handled by the Triton program
|
||||
(Triton implementation of causal_conv1d handles parallelism in 3-axes
|
||||
- feature-axis
|
||||
- batch-axis
|
||||
- sequence-axis)
|
||||
"""
|
||||
nums_dict: Optional[dict] = None
|
||||
cu_seqlen: Optional[int] = None
|
||||
batch_ptr: Optional[torch.tensor] = None
|
||||
token_chunk_offset_ptr: Optional[torch.tensor] = None
|
||||
|
||||
|
||||
def get_platform_metadata_classes() -> tuple[type[AttentionMetadata], ...]:
|
||||
@@ -38,45 +65,10 @@ def get_platform_metadata_classes() -> tuple[type[AttentionMetadata], ...]:
|
||||
f"Unsupported platform for Mamba2: {current_platform.device_type}")
|
||||
|
||||
|
||||
def _query_start_loc_to_chunk_indices_offsets(query_start_loc: torch.Tensor,
|
||||
chunk_size: int,
|
||||
total_seqlens: int):
|
||||
|
||||
cu_seqlens = query_start_loc[1:] # remove prepended 0
|
||||
|
||||
# outputs will have length expansion of chunks that do not divide
|
||||
# chunk_size
|
||||
N = math.ceil(total_seqlens / chunk_size) + (cu_seqlens[:-1] % chunk_size
|
||||
> 0).sum()
|
||||
chunk_indices = torch.arange(N,
|
||||
dtype=torch.int,
|
||||
device=query_start_loc.device)
|
||||
chunk_offsets = torch.zeros((N, ),
|
||||
dtype=torch.int,
|
||||
device=query_start_loc.device)
|
||||
|
||||
p = 0 # num of insertions
|
||||
for s, e in zip(cu_seqlens[:-1], cu_seqlens[1:]):
|
||||
|
||||
# if does not divide chunk_size, then there is one chunk insertion
|
||||
p += (s % chunk_size > 0)
|
||||
|
||||
# get the dimensions
|
||||
# - the + 1 for _e is to shift the boundary by one chunk
|
||||
# - this shifting is not needed if chunk_size divides e
|
||||
_s, _e = s // chunk_size + p, e // chunk_size + p + (e % chunk_size
|
||||
> 0)
|
||||
|
||||
# adjust inidces and offsets
|
||||
chunk_indices[_s:_e] -= p
|
||||
chunk_offsets[_s] = s % chunk_size
|
||||
|
||||
return chunk_indices, chunk_offsets
|
||||
|
||||
|
||||
def prepare_mamba2_metadata(
|
||||
chunk_size: int,
|
||||
attn_metadata: AttentionMetadata,
|
||||
mamba2_metadata=None,
|
||||
) -> Mamba2Metadata:
|
||||
|
||||
# compute number of prefill and decode requests
|
||||
@@ -96,12 +88,12 @@ def prepare_mamba2_metadata(
|
||||
attn_metadata_instances = get_platform_metadata_classes()
|
||||
if (isinstance(attn_metadata, attn_metadata_instances)
|
||||
and attn_metadata.context_lens_tensor is not None):
|
||||
has_initial_states = \
|
||||
attn_metadata.context_lens_tensor[:num_prefills] > 0 #[batch,]
|
||||
# precompute flag to avoid device syncs in mamba2 layer forwards
|
||||
# precompute flag to avoid device syncs later in mamba2 layer
|
||||
# forwards
|
||||
# prep is only needed for mamba2 ssd prefill processing
|
||||
prep_initial_states = torch.any(has_initial_states).item()
|
||||
|
||||
has_initial_states = attn_metadata.context_lens_tensor > 0
|
||||
prep_initial_states = torch.any(
|
||||
has_initial_states[:num_prefills]).item()
|
||||
query_start_loc = attn_metadata.query_start_loc[:num_prefills + 1]
|
||||
seq_idx = torch.repeat_interleave(torch.arange(
|
||||
num_prefills, dtype=torch.int32, device=query_start_loc.device),
|
||||
@@ -117,9 +109,78 @@ def prepare_mamba2_metadata(
|
||||
_query_start_loc_to_chunk_indices_offsets(
|
||||
query_start_loc, chunk_size, num_prefill_tokens)
|
||||
|
||||
if mamba2_metadata is not None:
|
||||
mamba2_metadata.has_initial_states = has_initial_states
|
||||
mamba2_metadata.prep_initial_states = prep_initial_states
|
||||
mamba2_metadata.chunk_size = chunk_size
|
||||
mamba2_metadata.seq_idx = seq_idx
|
||||
mamba2_metadata.chunk_indices = chunk_indices
|
||||
mamba2_metadata.chunk_offsets = chunk_offsets
|
||||
# We use 1 reset flag:
|
||||
# * mamba2_metadata.cu_seqlen is None
|
||||
# update config specific to (each input)
|
||||
# (become available at first layer, e.g. conv_weights)
|
||||
mamba2_metadata.cu_seqlen = None # suppose to be updated at each input
|
||||
|
||||
return mamba2_metadata
|
||||
return Mamba2Metadata(has_initial_states=has_initial_states,
|
||||
prep_initial_states=prep_initial_states,
|
||||
chunk_size=chunk_size,
|
||||
seq_idx=seq_idx,
|
||||
chunk_indices=chunk_indices,
|
||||
chunk_offsets=chunk_offsets)
|
||||
|
||||
|
||||
def update_metadata(x: torch.Tensor, query_start_loc: torch.Tensor,
|
||||
mamba2_metadata: Union[Mamba2Metadata,
|
||||
Mamba2AttentionMetadata]):
|
||||
"""
|
||||
this is triggered upon handling a new input at the first layer
|
||||
"""
|
||||
dim, cu_seqlen = x.shape
|
||||
mamba2_metadata.cu_seqlen = cu_seqlen
|
||||
seqlens = np.diff(query_start_loc.to('cpu'))
|
||||
nums_dict = {} # type: ignore
|
||||
for BLOCK_M in [8]: # cover all BLOCK_M values
|
||||
nums = -(-seqlens // BLOCK_M)
|
||||
nums_dict[BLOCK_M] = {}
|
||||
nums_dict[BLOCK_M]['nums'] = nums
|
||||
nums_dict[BLOCK_M]['tot'] = nums.sum().item()
|
||||
mlist = torch.from_numpy(np.repeat(np.arange(len(nums)), nums))
|
||||
nums_dict[BLOCK_M]['mlist'] = mlist
|
||||
mlist_len = len(nums_dict[BLOCK_M]['mlist'])
|
||||
nums_dict[BLOCK_M]['mlist_len'] = mlist_len
|
||||
MAX_NUM_PROGRAMS = max(1024, mlist_len) * 2
|
||||
offsetlist = [] # type: ignore
|
||||
for idx, num in enumerate(nums):
|
||||
offsetlist.extend(range(num))
|
||||
offsetlist = torch.tensor(offsetlist, dtype=torch.int32)
|
||||
nums_dict[BLOCK_M]['offsetlist'] = offsetlist
|
||||
|
||||
if mamba2_metadata.batch_ptr is None:
|
||||
# Update default value after class definition
|
||||
#mamba2_metadata.MAX_NUM_PROGRAMS *= 2
|
||||
mamba2_metadata.batch_ptr = torch.full((MAX_NUM_PROGRAMS, ),
|
||||
PAD_SLOT_ID,
|
||||
dtype=torch.int32,
|
||||
device='cuda')
|
||||
mamba2_metadata.token_chunk_offset_ptr = torch.full(
|
||||
(MAX_NUM_PROGRAMS, ),
|
||||
PAD_SLOT_ID,
|
||||
dtype=torch.int32,
|
||||
device='cuda')
|
||||
else:
|
||||
if mamba2_metadata.batch_ptr.nelement() < MAX_NUM_PROGRAMS:
|
||||
mamba2_metadata.batch_ptr.resize_(MAX_NUM_PROGRAMS).fill_(
|
||||
PAD_SLOT_ID)
|
||||
mamba2_metadata.token_chunk_offset_ptr.resize_( # type: ignore
|
||||
MAX_NUM_PROGRAMS).fill_(PAD_SLOT_ID)
|
||||
|
||||
mamba2_metadata.batch_ptr[0:mlist_len].copy_(mlist)
|
||||
mamba2_metadata.token_chunk_offset_ptr[ # type: ignore
|
||||
0:mlist_len].copy_(offsetlist)
|
||||
nums_dict[BLOCK_M]['batch_ptr'] = mamba2_metadata.batch_ptr
|
||||
nums_dict[BLOCK_M]['token_chunk_offset_ptr'] = (
|
||||
mamba2_metadata.token_chunk_offset_ptr) # type: ignore
|
||||
mamba2_metadata.nums_dict = nums_dict
|
||||
return mamba2_metadata
|
||||
|
||||
@@ -159,7 +159,7 @@ class MambaMixer(CustomOp):
|
||||
hidden_states = causal_conv1d_fn(
|
||||
hidden_states,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
bias=self.conv1d.bias,
|
||||
activation=self.activation,
|
||||
conv_states=mamba_cache_params.conv_state,
|
||||
has_initial_state=attn_metadata.context_lens_tensor > 0,
|
||||
|
||||
@@ -17,7 +17,8 @@ from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.custom_op import CustomOp
|
||||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
RowParallelLinear)
|
||||
from vllm.model_executor.layers.mamba.mamba2_metadata import Mamba2Metadata
|
||||
from vllm.model_executor.layers.mamba.mamba2_metadata import (Mamba2Metadata,
|
||||
update_metadata)
|
||||
from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
|
||||
causal_conv1d_fn, causal_conv1d_update)
|
||||
from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
|
||||
@@ -161,9 +162,9 @@ def mamba_v2_sharded_weight_loader(
|
||||
tp_size: int,
|
||||
tp_rank: int,
|
||||
) -> LoaderFunction:
|
||||
"""Create a weight loader for mamba v2. This ensures that the projections
|
||||
are correctly sharded so that they can be split into x, B, C. It also
|
||||
ensures that all the groups corresponding to a head shard is placed
|
||||
"""Create a weight loader for mamba v2. This ensures that the projections
|
||||
are correctly sharded so that they can be split into x, B, C. It also
|
||||
ensures that all the groups corresponding to a head shard is placed
|
||||
together with it.
|
||||
"""
|
||||
|
||||
@@ -458,9 +459,11 @@ class MambaMixer2(CustomOp):
|
||||
if attn_metadata is not None:
|
||||
assert isinstance(attn_metadata, dict)
|
||||
attn_metadata = attn_metadata[self.prefix]
|
||||
mamba2_metadata = attn_metadata
|
||||
assert isinstance(attn_metadata, Mamba2AttentionMetadata)
|
||||
self_kv_cache = self.kv_cache[forward_context.virtual_engine]
|
||||
conv_state = self_kv_cache[0]
|
||||
# conv_state = (..., dim, width-1) yet contiguous along 'dim'
|
||||
conv_state = self_kv_cache[0].transpose(-1, -2)
|
||||
ssm_state = self_kv_cache[1]
|
||||
state_indices_tensor = attn_metadata.state_indices_tensor
|
||||
has_initial_states_p = attn_metadata.has_initial_states
|
||||
@@ -531,6 +534,7 @@ class MambaMixer2(CustomOp):
|
||||
# NOTE: V0 put prefill before decode, v1 puts decode before prefill
|
||||
# Separate prefill and decode by splitting varlen input
|
||||
# Split along token dimension
|
||||
# NOTE: V0 put prefill before decode, v1 puts decode before prefill
|
||||
if envs.VLLM_USE_V1:
|
||||
hidden_states_B_C_d, hidden_states_B_C_p = torch.split(
|
||||
hidden_states_B_C,
|
||||
@@ -579,8 +583,13 @@ class MambaMixer2(CustomOp):
|
||||
# 2. Convolution sequence transformation
|
||||
# - "cache_indices" updates the conv_state cache in positions
|
||||
# pointed to by "state_indices_tensor"
|
||||
x = hidden_states_B_C_p.transpose(
|
||||
0, 1) # this is the form that causal-conv see
|
||||
if mamba2_metadata.cu_seqlen is None:
|
||||
mamba2_metadata = update_metadata(
|
||||
x, attn_metadata.query_start_loc, mamba2_metadata)
|
||||
hidden_states_B_C_p = causal_conv1d_fn(
|
||||
hidden_states_B_C_p.transpose(0, 1),
|
||||
x,
|
||||
conv_weights,
|
||||
self.conv1d.bias,
|
||||
activation=self.activation,
|
||||
@@ -590,8 +599,6 @@ class MambaMixer2(CustomOp):
|
||||
query_start_loc=query_start_loc_p).transpose(
|
||||
0, 1)[:num_prefill_tokens]
|
||||
|
||||
# TODO: Why is this needed?
|
||||
hidden_states_B_C_p = hidden_states_B_C_p.contiguous()
|
||||
hidden_states_p, B_p, C_p = split_hidden_states_B_C_fn(
|
||||
hidden_states_B_C_p)
|
||||
|
||||
@@ -715,9 +722,10 @@ class MambaMixer2(CustomOp):
|
||||
# - heads and n_groups are TP-ed
|
||||
conv_dim = (self.intermediate_size +
|
||||
2 * n_groups * self.ssm_state_size)
|
||||
# contiguous along 'dim' axis
|
||||
conv_state_shape = (
|
||||
divide(conv_dim, world_size),
|
||||
self.conv_kernel_size - 1,
|
||||
divide(conv_dim, world_size),
|
||||
)
|
||||
|
||||
# These are not TP-ed as they depend on A, dt_bias, D
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -36,10 +36,12 @@ class MambaCacheManager(ConstantSizeCache):
|
||||
# Initialize parent class
|
||||
super().__init__(max_batch_size)
|
||||
|
||||
# assume conv_state = (dim, state_len)
|
||||
assert conv_state_shape[0] > conv_state_shape[1]
|
||||
conv_state = torch.empty(size=(num_mamba_layers, max_batch_size) +
|
||||
conv_state_shape,
|
||||
(conv_state_shape[1], conv_state_shape[0]),
|
||||
dtype=dtype,
|
||||
device="cuda")
|
||||
device="cuda").transpose(-1, -2)
|
||||
temporal_state = torch.empty(size=(num_mamba_layers, max_batch_size) +
|
||||
temporal_state_shape,
|
||||
dtype=dtype,
|
||||
|
||||
Reference in New Issue
Block a user