[Model] Reduce redundant computations in mamba2 blocks for Bamba-9B (#15423)
Signed-off-by: Chih-Chieh-Yang <7364402+cyang49@users.noreply.github.com> Co-authored-by: Yu Chin Fabian Lim <flim@sg.ibm.com>
This commit is contained in:
co-authored by
Yu Chin Fabian Lim
parent
5fbab20e02
commit
daefed052c
@@ -18,6 +18,8 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
RowParallelLinear)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.mamba2_metadata import (
|
||||
Mamba2Metadata, prepare_mamba2_metadata)
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import (
|
||||
MambaMixer2, extra_groups_for_head_shards)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -94,7 +96,6 @@ class BambaMixerDecoderLayer(nn.Module):
|
||||
head_dim=config.mamba_d_head,
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
activation=config.hidden_act,
|
||||
chunk_size=config.mamba_chunk_size,
|
||||
quant_config=quant_config)
|
||||
|
||||
self.feed_forward = BambaMLP(config, quant_config=quant_config)
|
||||
@@ -108,7 +109,7 @@ class BambaMixerDecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
mamba_cache_params: MambaCacheParams,
|
||||
sequence_idx: Optional[torch.Tensor] = None,
|
||||
mamba2_metadata: Mamba2Metadata,
|
||||
**kwargs,
|
||||
):
|
||||
if residual is None:
|
||||
@@ -119,7 +120,7 @@ class BambaMixerDecoderLayer(nn.Module):
|
||||
hidden_states, residual)
|
||||
|
||||
hidden_states = self.mamba(hidden_states, mamba_cache_params,
|
||||
sequence_idx)
|
||||
mamba2_metadata)
|
||||
# Fully Connected
|
||||
hidden_states, residual = self.pre_ff_layernorm(
|
||||
hidden_states, residual)
|
||||
@@ -259,7 +260,7 @@ class BambaModel(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
config: BambaConfig = vllm_config.model_config.hf_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
lora_config = vllm_config.lora_config
|
||||
@@ -309,20 +310,13 @@ class BambaModel(nn.Module):
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
|
||||
# pass a sequence index tensor, that is required for
|
||||
# proper continuous batching computation including
|
||||
# chunked prefill
|
||||
seq_idx = None
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if attn_metadata.num_prefills > 0:
|
||||
seq_idx = torch.zeros_like(input_ids, dtype=torch.int32)
|
||||
for i, (srt, end) in enumerate(
|
||||
zip(
|
||||
attn_metadata.query_start_loc,
|
||||
attn_metadata.query_start_loc[1:],
|
||||
)):
|
||||
seq_idx[srt:end] = i
|
||||
seq_idx.unsqueeze_(0)
|
||||
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.mamba_chunk_size,
|
||||
input_ids=input_ids,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
@@ -352,7 +346,7 @@ class BambaModel(nn.Module):
|
||||
hidden_states=hidden_states,
|
||||
residual=residual,
|
||||
mamba_cache_params=layer_mamba_cache_params,
|
||||
sequence_idx=seq_idx,
|
||||
mamba2_metadata=mamba2_metadata,
|
||||
)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
@@ -555,4 +549,4 @@ class BambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
|
||||
def load_weights(self, weights: Iterable[Tuple[str,
|
||||
torch.Tensor]]) -> Set[str]:
|
||||
loader = AutoWeightsLoader(self)
|
||||
return loader.load_weights(weights)
|
||||
return loader.load_weights(weights)
|
||||
|
||||
@@ -13,6 +13,8 @@ from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.forward_context import get_forward_context
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.mamba2_metadata import (
|
||||
Mamba2Metadata, prepare_mamba2_metadata)
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import (
|
||||
MambaMixer2, extra_groups_for_head_shards)
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
@@ -57,7 +59,6 @@ class Mamba2DecoderLayer(nn.Module):
|
||||
head_dim=config.head_dim,
|
||||
rms_norm_eps=config.layer_norm_epsilon,
|
||||
activation=config.hidden_act,
|
||||
chunk_size=config.chunk_size,
|
||||
quant_config=quant_config)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
|
||||
@@ -67,7 +68,7 @@ class Mamba2DecoderLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
residual: Optional[torch.Tensor],
|
||||
mamba_cache_params: MambaCacheParams,
|
||||
sequence_idx: Optional[torch.Tensor],
|
||||
mamba2_metadata: Mamba2Metadata,
|
||||
**kwargs,
|
||||
):
|
||||
if residual is None:
|
||||
@@ -77,7 +78,7 @@ class Mamba2DecoderLayer(nn.Module):
|
||||
hidden_states, residual = self.norm(hidden_states, residual)
|
||||
|
||||
hidden_states = self.mixer(hidden_states, mamba_cache_params,
|
||||
sequence_idx)
|
||||
mamba2_metadata)
|
||||
return hidden_states, residual
|
||||
|
||||
|
||||
@@ -138,20 +139,13 @@ class Mamba2Model(nn.Module):
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
|
||||
# pass a sequence index tensor, that is required for
|
||||
# proper continuous batching computation including
|
||||
# chunked prefill
|
||||
seq_idx = None
|
||||
attn_metadata: AttentionMetadata = get_forward_context().attn_metadata
|
||||
if attn_metadata.num_prefills > 0:
|
||||
seq_idx = torch.zeros_like(input_ids, dtype=torch.int32)
|
||||
for i, (srt, end) in enumerate(
|
||||
zip(
|
||||
attn_metadata.query_start_loc,
|
||||
attn_metadata.query_start_loc[1:],
|
||||
)):
|
||||
seq_idx[srt:end] = i
|
||||
seq_idx.unsqueeze_(0)
|
||||
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.chunk_size,
|
||||
input_ids=input_ids,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
|
||||
for i in range(len(self.layers)):
|
||||
layer = self.layers[i]
|
||||
@@ -162,7 +156,7 @@ class Mamba2Model(nn.Module):
|
||||
residual=residual,
|
||||
mamba_cache_params=mamba_cache_params.at_layer_idx(
|
||||
i - self.start_layer),
|
||||
sequence_idx=seq_idx)
|
||||
mamba2_metadata=mamba2_metadata)
|
||||
|
||||
if not get_pp_group().is_last_rank:
|
||||
return IntermediateTensors({
|
||||
|
||||
@@ -25,6 +25,8 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.mamba2_metadata import (
|
||||
Mamba2Metadata, prepare_mamba2_metadata)
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import (
|
||||
MambaMixer2, extra_groups_for_head_shards)
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -495,7 +497,6 @@ class Zamba2MambaDecoderLayer(nn.Module):
|
||||
head_dim=intermediate_size // config.n_mamba_heads,
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
activation="silu",
|
||||
chunk_size=config.chunk_size,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
@@ -507,7 +508,7 @@ class Zamba2MambaDecoderLayer(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
mamba_cache_params: MambaCacheParams,
|
||||
sequence_idx: Optional[torch.Tensor] = None,
|
||||
mamba2_metadata: Mamba2Metadata,
|
||||
transformer_hidden_states: Optional[torch.Tensor] = None,
|
||||
positions: Optional[torch.Tensor] = None,
|
||||
original_hidden_states: Optional[torch.Tensor] = None,
|
||||
@@ -547,7 +548,7 @@ class Zamba2MambaDecoderLayer(nn.Module):
|
||||
hidden_states = self.mamba(
|
||||
hidden_states,
|
||||
mamba_cache_params=mamba_cache_params,
|
||||
sequence_idx=sequence_idx,
|
||||
mamba2_metadata=mamba2_metadata,
|
||||
)
|
||||
|
||||
# residual connection after mamba
|
||||
@@ -594,8 +595,8 @@ class Zamba2HybridLayer(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
original_hidden_states: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
mamba_cache_params: Optional[MambaCacheParams] = None,
|
||||
sequence_idx: Optional[torch.Tensor] = None,
|
||||
mamba_cache_params: MambaCacheParams,
|
||||
mamba2_metadata: Mamba2Metadata,
|
||||
) -> torch.Tensor:
|
||||
"""Forward pass through the hybrid layer.
|
||||
|
||||
@@ -634,7 +635,7 @@ class Zamba2HybridLayer(nn.Module):
|
||||
hidden_states,
|
||||
transformer_hidden_states=transformer_hidden_states,
|
||||
mamba_cache_params=mamba_cache_params,
|
||||
sequence_idx=sequence_idx,
|
||||
mamba2_metadata=mamba2_metadata,
|
||||
)
|
||||
|
||||
return layer_outputs
|
||||
@@ -747,20 +748,13 @@ class Zamba2Model(nn.Module):
|
||||
inputs_embeds = self.get_input_embeddings(input_ids)
|
||||
hidden_states = inputs_embeds
|
||||
|
||||
# pass a sequence index tensor, that is required for
|
||||
# proper continuous batching computation including
|
||||
# chunked prefill
|
||||
seq_idx = None
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
if attn_metadata.num_prefills > 0:
|
||||
seq_idx = torch.zeros_like(input_ids, dtype=torch.int32)
|
||||
for i, (srt, end) in enumerate(
|
||||
zip(
|
||||
attn_metadata.query_start_loc,
|
||||
attn_metadata.query_start_loc[1:],
|
||||
)):
|
||||
seq_idx[srt:end] = i
|
||||
seq_idx.unsqueeze_(0)
|
||||
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.chunk_size,
|
||||
input_ids=input_ids,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
|
||||
# Process through layers
|
||||
original_hidden_states = torch.clone(hidden_states)
|
||||
@@ -770,7 +764,7 @@ class Zamba2Model(nn.Module):
|
||||
original_hidden_states=original_hidden_states,
|
||||
positions=positions,
|
||||
mamba_cache_params=mamba_cache_params.at_layer_idx(layer_idx),
|
||||
sequence_idx=seq_idx,
|
||||
mamba2_metadata=mamba2_metadata,
|
||||
)
|
||||
hidden_states = layer_outputs
|
||||
|
||||
|
||||
Reference in New Issue
Block a user