[V1] [Hybrid] Support using float32 for state in Hybrid Models (Mamba2, Mamba1, Minimax) (#22928)
Signed-off-by: Daniel Afrimi <danielafrimi8@gmail.com> Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com> Signed-off-by: Chen Zhang <zhangch99@outlook.com> Co-authored-by: Daniel Afrimi <danielafrimi8@gmail.com> Co-authored-by: Burkhard Ringlein <ngl@zurich.ibm.com> Co-authored-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
co-authored by
Daniel Afrimi
Burkhard Ringlein
Chen Zhang
parent
22341b996e
commit
75531a6c13
@@ -24,9 +24,14 @@ class MambaCacheParams:
|
||||
|
||||
class MambaCacheManager(ConstantSizeCache):
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, dtype: torch.dtype,
|
||||
num_mamba_layers: int, conv_state_shape: tuple[int, int],
|
||||
temporal_state_shape: tuple[int, int]):
|
||||
def __init__(self, vllm_config: VllmConfig, num_mamba_layers: int,
|
||||
conv_state_shape: tuple[int, int],
|
||||
temporal_state_shape: tuple[int, int],
|
||||
conv_state_dtype: torch.dtype,
|
||||
temporal_state_dtype: torch.dtype):
|
||||
|
||||
self.conv_state_dtype = conv_state_dtype
|
||||
self.temporal_state_dtype = temporal_state_dtype
|
||||
|
||||
# Determine max batch size to set size of MambaCache
|
||||
max_batch_size = vllm_config.scheduler_config.max_num_seqs
|
||||
@@ -40,11 +45,11 @@ class MambaCacheManager(ConstantSizeCache):
|
||||
assert conv_state_shape[0] > conv_state_shape[1]
|
||||
conv_state = torch.empty(size=(num_mamba_layers, max_batch_size) +
|
||||
(conv_state_shape[1], conv_state_shape[0]),
|
||||
dtype=dtype,
|
||||
dtype=self.conv_state_dtype,
|
||||
device="cuda").transpose(-1, -2)
|
||||
temporal_state = torch.empty(size=(num_mamba_layers, max_batch_size) +
|
||||
temporal_state_shape,
|
||||
dtype=dtype,
|
||||
dtype=self.temporal_state_dtype,
|
||||
device="cuda")
|
||||
|
||||
self._mamba_cache = (conv_state, temporal_state)
|
||||
|
||||
Reference in New Issue
Block a user