Enable V1 for Hybrid SSM/Attention Models (#20016)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Stanislaw Wozniak <stw@zurich.ibm.com> Co-authored-by: Tyler Michael Smith <tysmith@redhat.com> Co-authored-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
co-authored by
Stanislaw Wozniak
Tyler Michael Smith
Chen Zhang
parent
ffe00ef77a
commit
2f35a022e6
@@ -9,6 +9,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import BambaConfig
|
||||
|
||||
from vllm import envs
|
||||
from vllm.attention.layer import Attention
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_world_size
|
||||
@@ -36,7 +37,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils import LayerBlockType
|
||||
|
||||
from .interfaces import (HasInnerState, IsHybrid, SupportsLoRA, SupportsPP,
|
||||
SupportsQuant, SupportsV0Only)
|
||||
SupportsQuant)
|
||||
from .utils import (AutoWeightsLoader, is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory, make_layers,
|
||||
maybe_prefix)
|
||||
@@ -97,7 +98,9 @@ class BambaMixerDecoderLayer(nn.Module):
|
||||
head_dim=config.mamba_d_head,
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
activation=config.hidden_act,
|
||||
quant_config=quant_config)
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mixer",
|
||||
chunk_size=config.mamba_chunk_size)
|
||||
|
||||
self.feed_forward = BambaMLP(config, quant_config=quant_config)
|
||||
self.input_layernorm = RMSNorm(config.hidden_size,
|
||||
@@ -313,10 +316,14 @@ class BambaModel(nn.Module):
|
||||
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.mamba_chunk_size,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
if not envs.VLLM_USE_V1:
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.mamba_chunk_size,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
else:
|
||||
# v1 get mamba2_metadata from forward_context
|
||||
mamba2_metadata = None
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
@@ -337,7 +344,8 @@ class BambaModel(nn.Module):
|
||||
num_attn += 1
|
||||
|
||||
layer_mamba_cache_params = None
|
||||
if isinstance(layer, BambaMixerDecoderLayer):
|
||||
if isinstance(layer,
|
||||
BambaMixerDecoderLayer) and mamba_cache_params:
|
||||
layer_mamba_cache_params = mamba_cache_params.at_layer_idx(
|
||||
i - num_attn)
|
||||
|
||||
@@ -411,7 +419,7 @@ class BambaModel(nn.Module):
|
||||
|
||||
|
||||
class BambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
|
||||
IsHybrid, SupportsV0Only, SupportsQuant):
|
||||
IsHybrid, SupportsQuant):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -475,15 +483,22 @@ class BambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs):
|
||||
if self.mamba_cache is None:
|
||||
|
||||
num_mamba_layers = self.model_config.get_num_layers_by_block_type(
|
||||
self.vllm_config.parallel_config, LayerBlockType.mamba)
|
||||
mamba_cache_params = None
|
||||
if not envs.VLLM_USE_V1:
|
||||
if self.mamba_cache is None:
|
||||
num_mamba_layers = \
|
||||
self.model_config.get_num_layers_by_block_type(
|
||||
self.vllm_config.parallel_config,
|
||||
LayerBlockType.mamba
|
||||
)
|
||||
|
||||
self.mamba_cache = MambaCacheManager(
|
||||
self.vllm_config, self.lm_head.weight.dtype,
|
||||
num_mamba_layers, *self._get_mamba_cache_shape())
|
||||
|
||||
mamba_cache_params = self.mamba_cache.current_run_tensors(**kwargs)
|
||||
|
||||
self.mamba_cache = MambaCacheManager(
|
||||
self.vllm_config, self.lm_head.weight.dtype, num_mamba_layers,
|
||||
*self._get_mamba_cache_shape())
|
||||
mamba_cache_params = self.mamba_cache.current_run_tensors(**kwargs)
|
||||
hidden_states = self.model(input_ids, positions, mamba_cache_params,
|
||||
intermediate_tensors, inputs_embeds)
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import FalconH1Config
|
||||
|
||||
from vllm import envs
|
||||
from vllm.attention.layer import Attention
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_world_size
|
||||
@@ -33,8 +34,7 @@ from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import (HasInnerState, IsHybrid, SupportsLoRA, SupportsPP,
|
||||
SupportsV0Only)
|
||||
from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP
|
||||
from .utils import (PPMissingLayer, is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory, make_layers,
|
||||
maybe_prefix)
|
||||
@@ -85,6 +85,7 @@ class FalconH1SSMDecoderLayer(nn.Module):
|
||||
config: FalconH1Config,
|
||||
cache_config: Optional[CacheConfig] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = config
|
||||
@@ -107,6 +108,8 @@ class FalconH1SSMDecoderLayer(nn.Module):
|
||||
activation=config.hidden_act,
|
||||
quant_config=quant_config,
|
||||
use_rms_norm=config.mamba_rms_norm,
|
||||
prefix=f"{prefix}.mixer",
|
||||
chunk_size=config.mamba_chunk_size,
|
||||
)
|
||||
# n_groups is overridden later by `MambaMixer2`
|
||||
self.groups_time_state_size = self.mamba.n_groups * config.mamba_d_state
|
||||
@@ -316,6 +319,7 @@ class FalconH1ParallelHybrid(nn.Module):
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
# Instantiate the attention branch
|
||||
self.self_attn = FalconH1AttentionDecoderLayer(
|
||||
config=config,
|
||||
@@ -323,11 +327,18 @@ class FalconH1ParallelHybrid(nn.Module):
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
# In V1 all attention/ssm layers must have
|
||||
# different index in prefix
|
||||
ssm_layer_idx = config.num_hidden_layers + layer_idx
|
||||
ssm_prefix = prefix.split(".")[0] + f".{ssm_layer_idx}"
|
||||
|
||||
# Instantiate the SSM branch
|
||||
self.mamba = FalconH1SSMDecoderLayer(
|
||||
config=config,
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=ssm_prefix,
|
||||
)
|
||||
self.ssm_out_multiplier = config.ssm_out_multiplier
|
||||
self.ssm_in_multiplier = config.ssm_in_multiplier
|
||||
@@ -452,10 +463,16 @@ class FalconH1Model(nn.Module):
|
||||
# proper continuous batching computation including
|
||||
# chunked prefill
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.mamba_chunk_size,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
|
||||
if not envs.VLLM_USE_V1:
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.mamba_chunk_size,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
else:
|
||||
# v1 get mamba2_metadata from forward_context
|
||||
mamba2_metadata = None
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
hidden_states = inputs_embeds * self.embedding_multiplier
|
||||
@@ -468,7 +485,9 @@ class FalconH1Model(nn.Module):
|
||||
|
||||
for i in range(self.start_layer, self.end_layer):
|
||||
layer = self.layers[i]
|
||||
layer_mamba_cache_params = mamba_cache_params.at_layer_idx(i)
|
||||
layer_mamba_cache_params = None
|
||||
if mamba_cache_params:
|
||||
layer_mamba_cache_params = mamba_cache_params.at_layer_idx(i)
|
||||
hidden_states = layer(
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
@@ -484,7 +503,7 @@ class FalconH1Model(nn.Module):
|
||||
|
||||
|
||||
class FalconH1ForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
|
||||
IsHybrid, SupportsV0Only):
|
||||
IsHybrid):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
@@ -558,15 +577,19 @@ class FalconH1ForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs,
|
||||
):
|
||||
if self.mamba_cache is None:
|
||||
self.mamba_cache = MambaCacheManager(
|
||||
self.vllm_config,
|
||||
self.lm_head.weight.dtype
|
||||
if hasattr(self.lm_head, 'weight') else torch.bfloat16,
|
||||
self.config.num_hidden_layers,
|
||||
*self._get_mamba_cache_shape(),
|
||||
)
|
||||
mamba_cache_params = self.mamba_cache.current_run_tensors(**kwargs)
|
||||
|
||||
mamba_cache_params = None
|
||||
if not envs.VLLM_USE_V1:
|
||||
if self.mamba_cache is None:
|
||||
self.mamba_cache = MambaCacheManager(
|
||||
self.vllm_config,
|
||||
self.lm_head.weight.dtype if hasattr(
|
||||
self.lm_head, 'weight') else torch.bfloat16,
|
||||
self.config.num_hidden_layers,
|
||||
*self._get_mamba_cache_shape(),
|
||||
)
|
||||
mamba_cache_params = self.mamba_cache.current_run_tensors(**kwargs)
|
||||
|
||||
hidden_states = self.model(
|
||||
input_ids,
|
||||
positions,
|
||||
|
||||
@@ -9,6 +9,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import GraniteMoeHybridConfig
|
||||
|
||||
from vllm import envs
|
||||
from vllm.attention.layer import Attention
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_world_size
|
||||
@@ -36,7 +37,7 @@ from vllm.utils import LayerBlockType
|
||||
from .granitemoe import GraniteMoeMoE
|
||||
from .granitemoeshared import GraniteMoeSharedMLP
|
||||
from .interfaces import (HasInnerState, IsHybrid, SupportsLoRA, SupportsPP,
|
||||
SupportsQuant, SupportsV0Only)
|
||||
SupportsQuant)
|
||||
from .utils import (AutoWeightsLoader, is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory, make_layers,
|
||||
maybe_prefix)
|
||||
@@ -67,7 +68,9 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
|
||||
head_dim=config.mamba_d_head,
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
activation=config.hidden_act,
|
||||
quant_config=quant_config)
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mixer",
|
||||
chunk_size=config.mamba_chunk_size)
|
||||
|
||||
self.block_sparse_moe = None
|
||||
if getattr(config, "num_local_experts", 0) > 0:
|
||||
@@ -361,10 +364,15 @@ class GraniteMoeHybridModel(nn.Module):
|
||||
) -> torch.Tensor:
|
||||
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.mamba_chunk_size,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
|
||||
if not envs.VLLM_USE_V1:
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.mamba_chunk_size,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
else:
|
||||
# v1 get mamba2_metadata from forward_context
|
||||
mamba2_metadata = None
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
@@ -386,7 +394,9 @@ class GraniteMoeHybridModel(nn.Module):
|
||||
num_attn += 1
|
||||
|
||||
layer_mamba_cache_params = None
|
||||
if isinstance(layer, GraniteMoeHybridMambaDecoderLayer):
|
||||
if isinstance(
|
||||
layer,
|
||||
GraniteMoeHybridMambaDecoderLayer) and mamba_cache_params:
|
||||
layer_mamba_cache_params = mamba_cache_params.at_layer_idx(
|
||||
i - num_attn)
|
||||
|
||||
@@ -501,8 +511,7 @@ class GraniteMoeHybridModel(nn.Module):
|
||||
|
||||
|
||||
class GraniteMoeHybridForCausalLM(nn.Module, HasInnerState, SupportsLoRA,
|
||||
SupportsPP, IsHybrid, SupportsV0Only,
|
||||
SupportsQuant):
|
||||
SupportsPP, IsHybrid, SupportsQuant):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -571,14 +580,20 @@ class GraniteMoeHybridForCausalLM(nn.Module, HasInnerState, SupportsLoRA,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs):
|
||||
if self.mamba_cache is None:
|
||||
num_mamba_layers = self.model_config.get_num_layers_by_block_type(
|
||||
self.vllm_config.parallel_config, LayerBlockType.mamba)
|
||||
self.mamba_cache = MambaCacheManager(
|
||||
self.vllm_config, self.model_config.dtype, num_mamba_layers,
|
||||
*self._get_mamba_cache_shape())
|
||||
|
||||
mamba_cache_params = self.mamba_cache.current_run_tensors(**kwargs)
|
||||
mamba_cache_params = None
|
||||
if not envs.VLLM_USE_V1:
|
||||
if self.mamba_cache is None:
|
||||
num_mamba_layers = (
|
||||
self.model_config.get_num_layers_by_block_type(
|
||||
self.vllm_config.parallel_config,
|
||||
LayerBlockType.mamba))
|
||||
self.mamba_cache = MambaCacheManager(
|
||||
self.vllm_config, self.model_config.dtype,
|
||||
num_mamba_layers, *self._get_mamba_cache_shape())
|
||||
|
||||
mamba_cache_params = self.mamba_cache.current_run_tensors(**kwargs)
|
||||
|
||||
hidden_states = self.model(input_ids, positions, mamba_cache_params,
|
||||
intermediate_tensors, inputs_embeds)
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ from typing import Optional
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm import envs
|
||||
from vllm.attention.layer import Attention
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_world_size
|
||||
@@ -44,8 +45,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.models.interfaces import (HasInnerState, IsHybrid,
|
||||
SupportsLoRA, SupportsPP,
|
||||
SupportsQuant,
|
||||
SupportsV0Only)
|
||||
SupportsQuant)
|
||||
from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
|
||||
MambaCacheParams)
|
||||
from vllm.model_executor.models.utils import (
|
||||
@@ -153,6 +153,8 @@ class NemotronHMambaDecoderLayer(nn.Module):
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
activation=config.mamba_hidden_act,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mixer",
|
||||
chunk_size=config.chunk_size,
|
||||
)
|
||||
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
@@ -348,10 +350,14 @@ class NemotronHModel(nn.Module):
|
||||
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.chunk_size,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
if not envs.VLLM_USE_V1:
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.chunk_size,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
else:
|
||||
# v1 get mamba2_metadata from forward_context
|
||||
mamba2_metadata = None
|
||||
|
||||
if get_pp_group().is_first_rank:
|
||||
if inputs_embeds is not None:
|
||||
@@ -369,7 +375,8 @@ class NemotronHModel(nn.Module):
|
||||
for i in range(len(self.layers)):
|
||||
layer = self.layers[i]
|
||||
layer_mamba_cache_params = None
|
||||
if isinstance(layer, NemotronHMambaDecoderLayer):
|
||||
if isinstance(layer,
|
||||
NemotronHMambaDecoderLayer) and mamba_cache_params:
|
||||
layer_mamba_cache_params = mamba_cache_params.at_layer_idx(
|
||||
i - num_non_mamba_layers)
|
||||
else:
|
||||
@@ -437,7 +444,7 @@ class NemotronHModel(nn.Module):
|
||||
|
||||
|
||||
class NemotronHForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
|
||||
IsHybrid, SupportsV0Only, SupportsQuant):
|
||||
IsHybrid, SupportsQuant):
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -499,15 +506,23 @@ class NemotronHForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
inputs_embeds: Optional[torch.Tensor] = None,
|
||||
**kwargs):
|
||||
if self.mamba_cache is None:
|
||||
|
||||
num_mamba_layers = self.model_config.get_num_layers_by_block_type(
|
||||
self.vllm_config.parallel_config, LayerBlockType.mamba)
|
||||
mamba_cache_params = None
|
||||
if not envs.VLLM_USE_V1:
|
||||
if self.mamba_cache is None:
|
||||
|
||||
num_mamba_layers = \
|
||||
self.model_config.get_num_layers_by_block_type(
|
||||
self.vllm_config.parallel_config,
|
||||
LayerBlockType.mamba
|
||||
)
|
||||
|
||||
self.mamba_cache = MambaCacheManager(
|
||||
self.vllm_config, self.lm_head.weight.dtype,
|
||||
num_mamba_layers, *self._get_mamba_cache_shape())
|
||||
|
||||
mamba_cache_params = self.mamba_cache.current_run_tensors(**kwargs)
|
||||
|
||||
self.mamba_cache = MambaCacheManager(
|
||||
self.vllm_config, self.lm_head.weight.dtype, num_mamba_layers,
|
||||
*self._get_mamba_cache_shape())
|
||||
mamba_cache_params = self.mamba_cache.current_run_tensors(**kwargs)
|
||||
hidden_states = self.model(input_ids, positions, mamba_cache_params,
|
||||
intermediate_tensors, inputs_embeds)
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ import torch
|
||||
from torch import nn
|
||||
from transformers import Zamba2Config
|
||||
|
||||
from vllm import envs
|
||||
from vllm.attention.layer import Attention
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import divide, get_tensor_model_parallel_world_size
|
||||
@@ -41,7 +42,7 @@ from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import HasInnerState, IsHybrid, SupportsV0Only
|
||||
from .interfaces import HasInnerState, IsHybrid
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
|
||||
|
||||
@@ -58,6 +59,7 @@ class Zamba2LoRA(nn.Module):
|
||||
rank: int,
|
||||
output_dim: Union[int, list[int]],
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
"""Initialize the attention layer.
|
||||
|
||||
@@ -283,6 +285,7 @@ class Zamba2MLP(nn.Module):
|
||||
bare_block_idx: int,
|
||||
num_hybrid_layers: dict[int, int],
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
"""Initialize the MLP layer.
|
||||
|
||||
@@ -471,11 +474,10 @@ class Zamba2MambaDecoderLayer(nn.Module):
|
||||
computation depending on configuration.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Zamba2Config,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
) -> None:
|
||||
def __init__(self,
|
||||
config: Zamba2Config,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "") -> None:
|
||||
"""Initialize the Mamba decoder layer.
|
||||
|
||||
Args:
|
||||
@@ -486,20 +488,21 @@ class Zamba2MambaDecoderLayer(nn.Module):
|
||||
|
||||
# Initialize Mamba mixer with expanded intermediate size
|
||||
intermediate_size = config.mamba_expand * config.hidden_size
|
||||
self.mamba = MambaMixer2(
|
||||
hidden_size=config.hidden_size,
|
||||
ssm_state_size=config.mamba_d_state,
|
||||
conv_kernel_size=config.mamba_d_conv,
|
||||
intermediate_size=intermediate_size,
|
||||
use_conv_bias=config.use_conv_bias,
|
||||
use_bias=config.add_bias_linear,
|
||||
n_groups=config.mamba_ngroups,
|
||||
num_heads=config.n_mamba_heads,
|
||||
head_dim=intermediate_size // config.n_mamba_heads,
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
activation="silu",
|
||||
quant_config=quant_config,
|
||||
)
|
||||
self.mamba = MambaMixer2(hidden_size=config.hidden_size,
|
||||
ssm_state_size=config.mamba_d_state,
|
||||
conv_kernel_size=config.mamba_d_conv,
|
||||
intermediate_size=intermediate_size,
|
||||
use_conv_bias=config.use_conv_bias,
|
||||
use_bias=config.add_bias_linear,
|
||||
n_groups=config.mamba_ngroups,
|
||||
num_heads=config.n_mamba_heads,
|
||||
head_dim=intermediate_size //
|
||||
config.n_mamba_heads,
|
||||
rms_norm_eps=config.rms_norm_eps,
|
||||
activation="silu",
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.mixer",
|
||||
chunk_size=config.chunk_size)
|
||||
|
||||
# Input normalization
|
||||
self.input_layernorm = RMSNorm(config.hidden_size,
|
||||
@@ -573,6 +576,7 @@ class Zamba2HybridLayer(nn.Module):
|
||||
config: Zamba2Config,
|
||||
block_idx: int,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
"""Initialize the hybrid layer.
|
||||
|
||||
@@ -589,7 +593,8 @@ class Zamba2HybridLayer(nn.Module):
|
||||
bias=False,
|
||||
quant_config=quant_config)
|
||||
self.mamba_decoder = Zamba2MambaDecoderLayer(config,
|
||||
quant_config=quant_config)
|
||||
quant_config=quant_config,
|
||||
prefix=prefix)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -699,14 +704,23 @@ class Zamba2Model(nn.Module):
|
||||
# Initialize layers according to block type configuration
|
||||
layers = []
|
||||
for layer_idx, layer_type in enumerate(config.layers_block_type):
|
||||
# tdoublep: avoid layers getting same index
|
||||
# somewhat hacky but correct (I think)
|
||||
prefix = str(len(layer2block_map) + layer_idx)
|
||||
if layer_type == "hybrid":
|
||||
block = next(blocks)
|
||||
block_idx = layer2block_map[layer_idx]
|
||||
layers.append(
|
||||
Zamba2HybridLayer(block, config, block_idx, quant_config))
|
||||
Zamba2HybridLayer(block,
|
||||
config,
|
||||
block_idx,
|
||||
quant_config,
|
||||
prefix=prefix))
|
||||
else:
|
||||
layers.append(
|
||||
Zamba2MambaDecoderLayer(config, quant_config=quant_config))
|
||||
Zamba2MambaDecoderLayer(config,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix))
|
||||
self.layers = nn.ModuleList(layers)
|
||||
|
||||
# Final layer normalization
|
||||
@@ -751,19 +765,30 @@ class Zamba2Model(nn.Module):
|
||||
|
||||
attn_metadata = get_forward_context().attn_metadata
|
||||
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.chunk_size,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
if not envs.VLLM_USE_V1:
|
||||
mamba2_metadata = prepare_mamba2_metadata(
|
||||
chunk_size=self.config.chunk_size,
|
||||
attn_metadata=attn_metadata,
|
||||
)
|
||||
else:
|
||||
# v1 get mamba2_metadata from forward_context
|
||||
mamba2_metadata = None
|
||||
|
||||
# Process through layers
|
||||
original_hidden_states = torch.clone(hidden_states)
|
||||
for layer_idx, layer in enumerate(self.layers):
|
||||
|
||||
layer_mamba_cache_params = None
|
||||
if (isinstance(layer, (Zamba2HybridLayer, Zamba2MambaDecoderLayer))
|
||||
and mamba_cache_params):
|
||||
layer_mamba_cache_params = mamba_cache_params.at_layer_idx(
|
||||
layer_idx)
|
||||
|
||||
layer_outputs = layer(
|
||||
hidden_states,
|
||||
original_hidden_states=original_hidden_states,
|
||||
positions=positions,
|
||||
mamba_cache_params=mamba_cache_params.at_layer_idx(layer_idx),
|
||||
mamba_cache_params=layer_mamba_cache_params,
|
||||
mamba2_metadata=mamba2_metadata,
|
||||
)
|
||||
hidden_states = layer_outputs
|
||||
@@ -803,7 +828,7 @@ class Zamba2Model(nn.Module):
|
||||
return loaded_params
|
||||
|
||||
|
||||
class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsV0Only):
|
||||
class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid):
|
||||
"""Zamba2 model with causal language modeling head.
|
||||
|
||||
This class wraps the core Zamba2 model and adds:
|
||||
@@ -897,14 +922,16 @@ class Zamba2ForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsV0Only):
|
||||
Output hidden states
|
||||
"""
|
||||
# Initialize Mamba cache if needed
|
||||
if self.mamba_cache is None:
|
||||
num_mamba_layers = self.config.num_hidden_layers
|
||||
self.mamba_cache = MambaCacheManager(
|
||||
self.vllm_config, self.lm_head.weight.dtype, num_mamba_layers,
|
||||
*self._get_mamba_cache_shape())
|
||||
mamba_cache_params = None
|
||||
if not envs.VLLM_USE_V1:
|
||||
if self.mamba_cache is None:
|
||||
num_mamba_layers = self.config.num_hidden_layers
|
||||
self.mamba_cache = MambaCacheManager(
|
||||
self.vllm_config, self.lm_head.weight.dtype,
|
||||
num_mamba_layers, *self._get_mamba_cache_shape())
|
||||
|
||||
# Get cache parameters for current run
|
||||
mamba_cache_params = self.mamba_cache.current_run_tensors(**kwargs)
|
||||
# Get cache parameters for current run
|
||||
mamba_cache_params = self.mamba_cache.current_run_tensors(**kwargs)
|
||||
|
||||
# Forward pass through model
|
||||
hidden_states = self.model(
|
||||
|
||||
Reference in New Issue
Block a user