From e9e1630e93b13b241c4cdb52c47e35fd38c6eb46 Mon Sep 17 00:00:00 2001 From: Egor <47443236+Lossfull@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:36:27 +0300 Subject: [PATCH] =?UTF-8?q?[Model]=20Support=20bidirectional=20(encoder-on?= =?UTF-8?q?ly)=20attention=20for=20DeepSeek=20e=E2=80=A6=20(#52948)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Egor Signed-off-by: Egor <47443236+Lossfull@users.noreply.github.com> Co-authored-by: Claude --- tests/models/registry.py | 5 +++++ vllm/config/model.py | 6 ++++++ vllm/model_executor/models/deepseek_v2.py | 26 ++++++++++++++++++++--- vllm/model_executor/models/registry.py | 1 + 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/tests/models/registry.py b/tests/models/registry.py index c862e767cd..bc3ea0733d 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -593,6 +593,11 @@ _EMBEDDING_EXAMPLE_MODELS = { hf_overrides={"architectures": ["BertSpladeSparseEmbeddingModel"]}, ), "BgeM3EmbeddingModel": _HfExamplesInfo("BAAI/bge-m3"), + "DeepseekV3BidirectionalModel": _HfExamplesInfo( + "ai-sage/Giga-Embeddings-instruct-10B-A1.8B-0826", + trust_remote_code=True, + hf_overrides={"model_type": "deepseek_v3", "auto_map": None}, + ), "Gemma2Model": _HfExamplesInfo("BAAI/bge-multilingual-gemma2"), "Gemma3TextModel": _HfExamplesInfo("google/embeddinggemma-300m"), "GritLM": _HfExamplesInfo("parasail-ai/GritLM-7B-vllm"), diff --git a/vllm/config/model.py b/vllm/config/model.py index a5f98d8c5c..16ff966c2d 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -1877,6 +1877,12 @@ class ModelConfig: # kv_lora_rank indicates that a Transformers model implementation uses MLA return getattr(self.hf_text_config, "kv_lora_rank", None) is not None # Manually maintained list of model types for vLLM model implementations + + # Bidirectional DeepSeek variants (is_causal=False, used by some + # embedding models) must use the non-MLA attention path, since the + # MLA kernels only support causal attention. + if not getattr(self.hf_text_config, "is_causal", True): + return False return self.is_deepseek_mla @property diff --git a/vllm/model_executor/models/deepseek_v2.py b/vllm/model_executor/models/deepseek_v2.py index d5521a1960..d8bc3fb64b 100644 --- a/vllm/model_executor/models/deepseek_v2.py +++ b/vllm/model_executor/models/deepseek_v2.py @@ -47,7 +47,11 @@ from vllm.distributed import ( ) from vllm.logger import init_logger from vllm.model_executor.layers.activation import SiluAndMul -from vllm.model_executor.layers.attention import Attention, RSWAAttention +from vllm.model_executor.layers.attention import ( + Attention, + EncoderOnlyAttention, + RSWAAttention, +) from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.fused_moe import ( FusedMoEFactory, @@ -98,7 +102,7 @@ from vllm.model_executor.models.utils import ( from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors from vllm.utils.torch_utils import direct_register_custom_op -from vllm.v1.attention.backend import AttentionBackend +from vllm.v1.attention.backend import AttentionBackend, AttentionType from vllm.v1.attention.backends.mla.indexer import ( DeepseekV32IndexerBackend, ) @@ -553,7 +557,22 @@ class DeepseekV2Attention(nn.Module): mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim)) self.scaling = self.scaling * mscale * mscale - self.attn = Attention( + # DeepSeek is causal by default (decoder-only). Bidirectional + # (encoder-only) attention is used by embedding models derived from + # this architecture, which set `is_causal=False` on the HF config. + # This only applies on the non-MLA path, since the MLA kernels are + # causal-only; `ModelConfig.use_mla` already disables MLA for these + # models. + if getattr(config, "is_causal", True): + attn_type = AttentionType.DECODER + else: + attn_type = AttentionType.ENCODER_ONLY + attn_cls = ( + EncoderOnlyAttention + if attn_type == AttentionType.ENCODER_ONLY + else Attention + ) + self.attn = attn_cls( self.num_local_heads, self.qk_head_dim, self.scaling, @@ -561,6 +580,7 @@ class DeepseekV2Attention(nn.Module): cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.attn", + attn_type=attn_type, ) def forward( diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 75ef6756dd..8912827d0e 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -230,6 +230,7 @@ _EMBEDDING_MODELS = { "BertSpladeSparseEmbeddingModel": ("bert", "BertSpladeSparseEmbeddingModel"), "BgeM3EmbeddingModel": ("roberta", "BgeM3EmbeddingModel"), "DeciLMForCausalLM": ("nemotron_nas", "DeciLMForCausalLM"), + "DeepseekV3BidirectionalModel": ("deepseek_v2", "DeepseekV3ForCausalLM"), "Gemma2Model": ("gemma2", "Gemma2ForCausalLM"), "Gemma3TextModel": ("gemma3", "Gemma3Model"), "GlmForCausalLM": ("glm", "GlmForCausalLM"),