[ModelRunner v2] Support Transformers pooling model (#52425)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-08-16 15:25:49 -04:00
committed by GitHub
parent ef43e3101b
commit fdab2b10bc
2 changed files with 18 additions and 5 deletions
+8 -4
View File
@@ -3,8 +3,9 @@
import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.attention import CrossAttention, EncoderOnlyAttention
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.model_executor.layers.attention import Attention, CrossAttention
from vllm.v1.attention.backend import AttentionType
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
@@ -27,8 +28,11 @@ def init_model_state(
return EncoderDecoderModelState(vllm_config, model, encoder_cache, device)
# Encoder-only models (BERT/RoBERTa): non-causal self-attention, no KV cache.
if any(isinstance(m, EncoderOnlyAttention) for m in model.modules()):
# Encoder-only attention is non-causal and needs no KV cache.
if any(
layer.attn_type == AttentionType.ENCODER_ONLY
for layer in get_layers_from_vllm_config(vllm_config, Attention).values()
):
from vllm.v1.worker.gpu.model_states.encoder_only import EncoderOnlyModelState
return EncoderOnlyModelState(vllm_config, model, encoder_cache, device)