[Model] Support bidirectional (encoder-only) attention for DeepSeek e… (#52948)
Signed-off-by: Egor <erop001@gmail.com> Signed-off-by: Egor <47443236+Lossfull@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -593,6 +593,11 @@ _EMBEDDING_EXAMPLE_MODELS = {
|
||||
hf_overrides={"architectures": ["BertSpladeSparseEmbeddingModel"]},
|
||||
),
|
||||
"BgeM3EmbeddingModel": _HfExamplesInfo("BAAI/bge-m3"),
|
||||
"DeepseekV3BidirectionalModel": _HfExamplesInfo(
|
||||
"ai-sage/Giga-Embeddings-instruct-10B-A1.8B-0826",
|
||||
trust_remote_code=True,
|
||||
hf_overrides={"model_type": "deepseek_v3", "auto_map": None},
|
||||
),
|
||||
"Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"),
|
||||
"Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"),
|
||||
"GritLM": _HfExamplesInfo("parasail-ai/GritLM-7B-vllm"),
|
||||
|
||||
@@ -1877,6 +1877,12 @@ class ModelConfig:
|
||||
# kv_lora_rank indicates that a Transformers model implementation uses MLA
|
||||
return getattr(self.hf_text_config, "kv_lora_rank", None) is not None
|
||||
# Manually maintained list of model types for vLLM model implementations
|
||||
|
||||
# Bidirectional DeepSeek variants (is_causal=False, used by some
|
||||
# embedding models) must use the non-MLA attention path, since the
|
||||
# MLA kernels only support causal attention.
|
||||
if not getattr(self.hf_text_config, "is_causal", True):
|
||||
return False
|
||||
return self.is_deepseek_mla
|
||||
|
||||
@property
|
||||
|
||||
@@ -47,7 +47,11 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.attention import Attention, RSWAAttention
|
||||
from vllm.model_executor.layers.attention import (
|
||||
Attention,
|
||||
EncoderOnlyAttention,
|
||||
RSWAAttention,
|
||||
)
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
@@ -98,7 +102,7 @@ from vllm.model_executor.models.utils import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.attention.backend import AttentionBackend, AttentionType
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerBackend,
|
||||
)
|
||||
@@ -553,7 +557,22 @@ class DeepseekV2Attention(nn.Module):
|
||||
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
|
||||
self.scaling = self.scaling * mscale * mscale
|
||||
|
||||
self.attn = Attention(
|
||||
# DeepSeek is causal by default (decoder-only). Bidirectional
|
||||
# (encoder-only) attention is used by embedding models derived from
|
||||
# this architecture, which set `is_causal=False` on the HF config.
|
||||
# This only applies on the non-MLA path, since the MLA kernels are
|
||||
# causal-only; `ModelConfig.use_mla` already disables MLA for these
|
||||
# models.
|
||||
if getattr(config, "is_causal", True):
|
||||
attn_type = AttentionType.DECODER
|
||||
else:
|
||||
attn_type = AttentionType.ENCODER_ONLY
|
||||
attn_cls = (
|
||||
EncoderOnlyAttention
|
||||
if attn_type == AttentionType.ENCODER_ONLY
|
||||
else Attention
|
||||
)
|
||||
self.attn = attn_cls(
|
||||
self.num_local_heads,
|
||||
self.qk_head_dim,
|
||||
self.scaling,
|
||||
@@ -561,6 +580,7 @@ class DeepseekV2Attention(nn.Module):
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
attn_type=attn_type,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -230,6 +230,7 @@ _EMBEDDING_MODELS = {
|
||||
"BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"),
|
||||
"BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"),
|
||||
"DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"),
|
||||
"DeepseekV3BidirectionalModel": ("deepseek_v2", "DeepseekV3ForCausalLM"),
|
||||
"Gemma2Model": ("gemma2", "Gemma2ForCausalLM"),
|
||||
"Gemma3TextModel": ("gemma3", "Gemma3Model"),
|
||||
"GlmForCausalLM": ("glm", "GlmForCausalLM"),
|
||||
|
||||
Reference in New Issue
Block a user