[ModelRunner V2] Support encoder-only attention (#49331)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Nick Hill
2026-07-27 00:38:44 +00:00
committed by GitHub
co-authored by Taneem Ibrahim
parent 0934b26790
commit fdaa0d9e59
8 changed files with 276 additions and 4 deletions
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from transformers import AutoModel
from vllm.config import PoolerConfig
@@ -87,3 +89,55 @@ def test_models(
name_1="vllm",
tol=1e-2,
)
@pytest.mark.parametrize(
"model",
[
"BAAI/bge-base-en-v1.5",
"intfloat/multilingual-e5-small",
],
)
@torch.inference_mode()
def test_encoder_only_model_runner_v2_attention(
hf_runner,
vllm_runner,
monkeypatch,
model: str,
) -> None:
prompts = [
"short input",
"a longer input that exercises mixed sequence lengths",
]
with hf_runner(model, dtype="float", auto_cls=AutoModel) as hf_model:
hf_outputs = []
for prompt in prompts:
inputs = hf_model.tokenizer(prompt, return_tensors="pt")
output = hf_model.model(**hf_model.wrap_device(inputs))
embedding = torch.nn.functional.normalize(
output.last_hidden_state[0, -1].float(), dim=0
)
hf_outputs.append(embedding.cpu().tolist())
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
with vllm_runner(
model,
runner="pooling",
dtype="float",
max_model_len=64,
max_num_seqs=2,
gpu_memory_utilization=0.25,
pooler_config=PoolerConfig(
task="embed", seq_pooling_type="LAST", use_activation=True
),
) as vllm_model:
vllm_outputs = vllm_model.embed(prompts)
check_embeddings_close(
embeddings_0_lst=hf_outputs,
embeddings_1_lst=vllm_outputs,
name_0="hf",
name_1="vllm",
tol=1e-2,
)
+12
View File
@@ -46,6 +46,18 @@ class AttentionCGSupportInfo:
min_cg_support: AttentionCGSupport = AttentionCGSupport.ALWAYS
min_cg_attn_backend: str | None = None
def narrow(
self, support: AttentionCGSupport, backend: str | None
) -> "AttentionCGSupportInfo":
"""Return an info tightened by ``support`` if it is more restrictive.
Lets attention groups built outside ``init_attn_backend`` (e.g.
encoder-only layers) contribute to the runner's cudagraph decision.
"""
if support.value < self.min_cg_support.value:
return AttentionCGSupportInfo(support, backend)
return self
def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
kv_cache_spec: dict[str, KVCacheSpec] = {}
+1 -1
View File
@@ -123,7 +123,7 @@ class BlockTables:
if self.num_kv_cache_groups == 1:
# Single group: write directly, skipping the per-write group lookup.
self.block_tables[0].apply_write()
else:
elif self.num_kv_cache_groups > 1:
# Multiple groups: apply all block tables with one fused kernel.
assert self.fused_writer is not None
self.fused_writer.apply(
+3
View File
@@ -459,6 +459,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
self.attn_groups, attn_cg_support, self.kernel_block_sizes = init_attn_backend(
self.kv_cache_config, self.vllm_config, self.device
)
attn_cg_support = attn_cg_support.narrow(
*self.model_state.get_additional_cg_support()
)
self.block_tables = BlockTables(
block_sizes=block_sizes,
max_num_reqs=self.max_num_reqs,
+7 -1
View File
@@ -4,7 +4,7 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.model_executor.layers.attention import CrossAttention
from vllm.model_executor.layers.attention import CrossAttention, EncoderOnlyAttention
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
@@ -27,6 +27,12 @@ def init_model_state(
return EncoderDecoderModelState(vllm_config, model, encoder_cache, device)
# Encoder-only models (BERT/RoBERTa): non-causal self-attention, no KV cache.
if any(isinstance(m, EncoderOnlyAttention) for m in model.modules()):
from vllm.v1.worker.gpu.model_states.encoder_only import EncoderOnlyModelState
return EncoderOnlyModelState(vllm_config, model, encoder_cache, device)
if vllm_config.model_config.is_hybrid:
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
@@ -0,0 +1,182 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any, cast
import torch
import torch.nn as nn
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.config.compilation import CUDAGraphMode
from vllm.model_executor.layers.attention import Attention
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
from vllm.v1.attention.backend import (
AttentionCGSupport,
AttentionType,
CommonAttentionMetadata,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
EncoderOnlyAttentionSpec,
KVCacheConfig,
)
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
from vllm.v1.worker.utils import AttentionGroup
class EncoderOnlyModelState(DefaultModelState):
"""ModelState for encoder-only (BERT/RoBERTa) models.
Encoder attention needs no KV cache: it runs full, bidirectional
self-attention over each request's tokens in a single forward. Such layers
return no ``get_kv_cache_spec`` and therefore never enter
``kv_cache_config.kv_cache_groups``, so the KV-backed attention path never
builds their metadata. We build their (non-causal) metadata here, keeping
the normal KV-backed path untouched.
"""
def __init__(
self,
vllm_config: VllmConfig,
model: nn.Module,
encoder_cache: EncoderCache | None,
device: torch.device,
):
super().__init__(vllm_config, model, encoder_cache, device)
cache_config = vllm_config.cache_config
if cache_config.cache_dtype == "auto":
kv_cache_dtype = self.dtype
else:
kv_cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
# Build an attention group (and its non-causal metadata builder) for the
# encoder-only layers, grouped by backend + query/KV head config. Models
# are typically uniform, yielding a single group.
attn_layers = get_layers_from_vllm_config(vllm_config, Attention)
groups: dict[tuple[tuple[str, str], int, int, int], AttentionGroup] = {}
for name, layer in attn_layers.items():
if layer.attn_type != AttentionType.ENCODER_ONLY:
continue
# No KV cache is bound for these layers; give them an (empty)
# device tensor so the attention forward context is well-formed.
layer.kv_cache = torch.empty(0, dtype=kv_cache_dtype, device=device)
backend = layer.get_attn_backend()
# Metadata builders require a uniform query-head count per group.
key = (
backend.full_cls_name(),
layer.num_heads,
layer.num_kv_heads,
layer.head_size,
)
group = groups.get(key)
if group is None:
spec = EncoderOnlyAttentionSpec(
block_size=cache_config.block_size,
num_kv_heads=layer.num_kv_heads,
head_size=layer.head_size,
dtype=kv_cache_dtype,
)
group = AttentionGroup(backend, [], spec, kv_cache_group_id=len(groups))
groups[key] = group
group.layer_names.append(name)
self.encoder_attn_groups = list(groups.values())
for group in self.encoder_attn_groups:
group.create_metadata_builders(vllm_config, device)
# Encoder attention reads neither the block table nor the slot mapping
# (full varlen self-attention over the batch's q/k/v), but the metadata
# builder expects both tensors to be present.
self._dummy_block_table = torch.zeros(
self.max_num_reqs, 1, dtype=torch.int32, device=device
)
self._dummy_slot_mapping = torch.zeros(
self.max_num_tokens, dtype=torch.int64, device=device
)
def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]:
# Encoder groups are built here rather than in init_attn_backend, so
# their cudagraph support must be surfaced to the runner separately.
support = AttentionCGSupport.ALWAYS
backend: str | None = None
for group in self.encoder_attn_groups:
builder = group.get_metadata_builder(0)
cg_support = builder.get_cudagraph_support(
self.vllm_config, cast(AttentionSpec, group.kv_cache_spec)
)
if cg_support.value < support.value:
support = cg_support
backend = group.backend.__name__
return support, backend
def prepare_attn(
self,
input_batch: InputBatch,
cudagraph_mode: CUDAGraphMode,
block_tables: tuple[torch.Tensor, ...],
slot_mappings: torch.Tensor,
attn_groups: list[list[AttentionGroup]],
kv_cache_config: KVCacheConfig,
for_capture: bool = False,
) -> dict[str, Any]:
attn_metadata = super().prepare_attn(
input_batch,
cudagraph_mode,
block_tables,
slot_mappings,
attn_groups,
kv_cache_config,
for_capture,
)
attn_metadata.update(
self._build_encoder_attn_metadata(input_batch, cudagraph_mode, for_capture)
)
return attn_metadata
def _build_encoder_attn_metadata(
self,
input_batch: InputBatch,
cudagraph_mode: CUDAGraphMode,
for_capture: bool,
) -> dict[str, Any]:
if cudagraph_mode == CUDAGraphMode.FULL:
num_reqs = input_batch.num_reqs_after_padding
num_tokens = input_batch.num_tokens_after_padding
else:
num_reqs = input_batch.num_reqs
num_tokens = input_batch.num_tokens
max_query_len = int(input_batch.num_scheduled_tokens.max())
if for_capture:
max_seq_len = self.max_model_len
else:
max_seq_len = int(input_batch.seq_lens_cpu_upper_bound[:num_reqs].max())
# The encoder builder forces ``causal=False`` regardless of this value.
common_attn_metadata = CommonAttentionMetadata(
query_start_loc=input_batch.query_start_loc,
query_start_loc_cpu=torch.from_numpy(input_batch.query_start_loc_np),
seq_lens=input_batch.seq_lens[:num_reqs],
num_reqs=num_reqs,
num_actual_tokens=num_tokens,
max_query_len=max_query_len,
max_seq_len=max_seq_len,
block_table_tensor=self._dummy_block_table[:num_reqs],
slot_mapping=self._dummy_slot_mapping[:num_tokens],
seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound[:num_reqs],
positions=input_batch.positions,
)
attn_metadata: dict[str, Any] = {}
for group in self.encoder_attn_groups:
builder = group.get_metadata_builder(0)
if for_capture:
metadata = builder.build_for_cudagraph_capture(common_attn_metadata)
else:
metadata = builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
for name in group.layer_names:
attn_metadata[name] = metadata
return attn_metadata
@@ -9,6 +9,7 @@ import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.tasks import GenerationTask
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.input_batch import InputBatch
@@ -95,6 +96,15 @@ class ModelState(ABC):
def apply_staged_writes(self) -> None:
return None
def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]:
"""Cudagraph support of attention groups this ModelState builds outside
``init_attn_backend`` (e.g. encoder-only layers).
Returns the minimum support level and its backend name. The default of
``ALWAYS`` imposes no extra constraint on the runner's cudagraph mode.
"""
return AttentionCGSupport.ALWAYS, None
def preprocess_state(
self,
input_batch: InputBatch,
+7 -2
View File
@@ -217,9 +217,14 @@ def warmup_kernels(
model_runner.scheduler_config.max_num_seqs,
model_runner.scheduler_config.max_num_batched_tokens
// max(prompt_len, decode_query_len),
# Reserve block 0 (null block) and ensure we have enough blocks.
max(1, (model_runner.kv_cache_config.num_blocks - 1) // max_blocks_per_req),
)
if max_blocks_per_req > 0:
# Reserve block 0 (null block) and ensure we have enough blocks.
# Encoder-only models allocate no KV blocks, so this cap doesn't apply.
num_reqs = min(
num_reqs,
max(1, (model_runner.kv_cache_config.num_blocks - 1) // max_blocks_per_req),
)
req_ids = [f"_warmup_{i}_" for i in range(num_reqs)]