[Attention] Mamba attention module refactor (#41126)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
This commit is contained in:
@@ -7,7 +7,7 @@ import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CacheConfig, ModelConfig, ParallelConfig, VllmConfig
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
from vllm.distributed import (
|
||||
get_pp_group,
|
||||
get_tensor_model_parallel_world_size,
|
||||
@@ -18,7 +18,6 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoE,
|
||||
fused_moe_make_expert_params_mapping,
|
||||
)
|
||||
from vllm.model_executor.layers.kda import KimiDeltaAttention
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
@@ -27,6 +26,9 @@ from vllm.model_executor.layers.linear import (
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.mamba.gdn.kimi_gdn_linear_attn import (
|
||||
KimiGatedDeltaNetAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import (
|
||||
MambaStateCopyFunc,
|
||||
MambaStateCopyFuncCalculator,
|
||||
@@ -286,26 +288,22 @@ class KimiDecoderLayer(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
config: KimiLinearConfig,
|
||||
layer_idx: int,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
parallel_config: ParallelConfig | None = None,
|
||||
model_config: ModelConfig | None = None,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
**kwargs,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = config.hidden_size
|
||||
|
||||
self.is_moe = config.is_moe
|
||||
layer_idx = int(prefix.rsplit(".", 1)[1])
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
if config.is_kda_layer(layer_idx):
|
||||
self.self_attn = KimiDeltaAttention(
|
||||
layer_idx=layer_idx,
|
||||
hidden_size=config.hidden_size,
|
||||
quant_config=quant_config,
|
||||
cache_config=cache_config,
|
||||
model_config=config,
|
||||
self.self_attn = KimiGatedDeltaNetAttention(
|
||||
config,
|
||||
vllm_config,
|
||||
prefix=f"{prefix}.self_attn",
|
||||
)
|
||||
else:
|
||||
@@ -385,10 +383,6 @@ class KimiLinearModel(nn.Module):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
model_config = vllm_config.model_config
|
||||
cache_config = vllm_config.cache_config
|
||||
quant_config = vllm_config.quant_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
self.config = config
|
||||
|
||||
self.vocab_size = config.vocab_size
|
||||
@@ -402,19 +396,11 @@ class KimiLinearModel(nn.Module):
|
||||
else:
|
||||
self.embed_tokens = PPMissingLayer()
|
||||
|
||||
extra_kwargs = {}
|
||||
|
||||
def get_layer(prefix: str):
|
||||
layer_idx = int(prefix.rsplit(".", 1)[1])
|
||||
return KimiDecoderLayer(
|
||||
config,
|
||||
layer_idx,
|
||||
cache_config,
|
||||
quant_config,
|
||||
parallel_config,
|
||||
model_config,
|
||||
vllm_config,
|
||||
prefix,
|
||||
**extra_kwargs,
|
||||
)
|
||||
|
||||
self.start_layer, self.end_layer, self.layers = make_layers(
|
||||
|
||||
Reference in New Issue
Block a user