diff --git a/tests/models/language/pooling/test_embedding.py b/tests/models/language/pooling/test_embedding.py index e105195afe..798aa89001 100644 --- a/tests/models/language/pooling/test_embedding.py +++ b/tests/models/language/pooling/test_embedding.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import pytest +import torch +from transformers import AutoModel from vllm.config import PoolerConfig @@ -87,3 +89,55 @@ def test_models( name_1="vllm", tol=1e-2, ) + + +@pytest.mark.parametrize( + "model", + [ + "BAAI/bge-base-en-v1.5", + "intfloat/multilingual-e5-small", + ], +) +@torch.inference_mode() +def test_encoder_only_model_runner_v2_attention( + hf_runner, + vllm_runner, + monkeypatch, + model: str, +) -> None: + prompts = [ + "short input", + "a longer input that exercises mixed sequence lengths", + ] + + with hf_runner(model, dtype="float", auto_cls=AutoModel) as hf_model: + hf_outputs = [] + for prompt in prompts: + inputs = hf_model.tokenizer(prompt, return_tensors="pt") + output = hf_model.model(**hf_model.wrap_device(inputs)) + embedding = torch.nn.functional.normalize( + output.last_hidden_state[0, -1].float(), dim=0 + ) + hf_outputs.append(embedding.cpu().tolist()) + + monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1") + with vllm_runner( + model, + runner="pooling", + dtype="float", + max_model_len=64, + max_num_seqs=2, + gpu_memory_utilization=0.25, + pooler_config=PoolerConfig( + task="embed", seq_pooling_type="LAST", use_activation=True + ), + ) as vllm_model: + vllm_outputs = vllm_model.embed(prompts) + + check_embeddings_close( + embeddings_0_lst=hf_outputs, + embeddings_1_lst=vllm_outputs, + name_0="hf", + name_1="vllm", + tol=1e-2, + ) diff --git a/vllm/v1/worker/gpu/attn_utils.py b/vllm/v1/worker/gpu/attn_utils.py index b6bbe28e5f..7dff4cbe4c 100644 --- a/vllm/v1/worker/gpu/attn_utils.py +++ b/vllm/v1/worker/gpu/attn_utils.py @@ -46,6 +46,18 @@ class AttentionCGSupportInfo: min_cg_support: AttentionCGSupport = AttentionCGSupport.ALWAYS min_cg_attn_backend: str | None = None + def narrow( + self, support: AttentionCGSupport, backend: str | None + ) -> "AttentionCGSupportInfo": + """Return an info tightened by ``support`` if it is more restrictive. + + Lets attention groups built outside ``init_attn_backend`` (e.g. + encoder-only layers) contribute to the runner's cudagraph decision. + """ + if support.value < self.min_cg_support.value: + return AttentionCGSupportInfo(support, backend) + return self + def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]: kv_cache_spec: dict[str, KVCacheSpec] = {} diff --git a/vllm/v1/worker/gpu/block_table.py b/vllm/v1/worker/gpu/block_table.py index 22c4afc11b..df5d2be629 100644 --- a/vllm/v1/worker/gpu/block_table.py +++ b/vllm/v1/worker/gpu/block_table.py @@ -123,7 +123,7 @@ class BlockTables: if self.num_kv_cache_groups == 1: # Single group: write directly, skipping the per-write group lookup. self.block_tables[0].apply_write() - else: + elif self.num_kv_cache_groups > 1: # Multiple groups: apply all block tables with one fused kernel. assert self.fused_writer is not None self.fused_writer.apply( diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index fdedbfb86d..64a4f55351 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -459,6 +459,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): self.attn_groups, attn_cg_support, self.kernel_block_sizes = init_attn_backend( self.kv_cache_config, self.vllm_config, self.device ) + attn_cg_support = attn_cg_support.narrow( + *self.model_state.get_additional_cg_support() + ) self.block_tables = BlockTables( block_sizes=block_sizes, max_num_reqs=self.max_num_reqs, diff --git a/vllm/v1/worker/gpu/model_states/__init__.py b/vllm/v1/worker/gpu/model_states/__init__.py index dc52dc4ee5..2372a01dd3 100644 --- a/vllm/v1/worker/gpu/model_states/__init__.py +++ b/vllm/v1/worker/gpu/model_states/__init__.py @@ -4,7 +4,7 @@ import torch import torch.nn as nn from vllm.config import VllmConfig -from vllm.model_executor.layers.attention import CrossAttention +from vllm.model_executor.layers.attention import CrossAttention, EncoderOnlyAttention from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache @@ -27,6 +27,12 @@ def init_model_state( return EncoderDecoderModelState(vllm_config, model, encoder_cache, device) + # Encoder-only models (BERT/RoBERTa): non-causal self-attention, no KV cache. + if any(isinstance(m, EncoderOnlyAttention) for m in model.modules()): + from vllm.v1.worker.gpu.model_states.encoder_only import EncoderOnlyModelState + + return EncoderOnlyModelState(vllm_config, model, encoder_cache, device) + if vllm_config.model_config.is_hybrid: from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState diff --git a/vllm/v1/worker/gpu/model_states/encoder_only.py b/vllm/v1/worker/gpu/model_states/encoder_only.py new file mode 100644 index 0000000000..6595b9e434 --- /dev/null +++ b/vllm/v1/worker/gpu/model_states/encoder_only.py @@ -0,0 +1,182 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any, cast + +import torch +import torch.nn as nn + +from vllm.config import VllmConfig, get_layers_from_vllm_config +from vllm.config.compilation import CUDAGraphMode +from vllm.model_executor.layers.attention import Attention +from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE +from vllm.v1.attention.backend import ( + AttentionCGSupport, + AttentionType, + CommonAttentionMetadata, +) +from vllm.v1.kv_cache_interface import ( + AttentionSpec, + EncoderOnlyAttentionSpec, + KVCacheConfig, +) +from vllm.v1.worker.gpu.input_batch import InputBatch +from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache +from vllm.v1.worker.gpu.model_states.default import DefaultModelState +from vllm.v1.worker.utils import AttentionGroup + + +class EncoderOnlyModelState(DefaultModelState): + """ModelState for encoder-only (BERT/RoBERTa) models. + + Encoder attention needs no KV cache: it runs full, bidirectional + self-attention over each request's tokens in a single forward. Such layers + return no ``get_kv_cache_spec`` and therefore never enter + ``kv_cache_config.kv_cache_groups``, so the KV-backed attention path never + builds their metadata. We build their (non-causal) metadata here, keeping + the normal KV-backed path untouched. + """ + + def __init__( + self, + vllm_config: VllmConfig, + model: nn.Module, + encoder_cache: EncoderCache | None, + device: torch.device, + ): + super().__init__(vllm_config, model, encoder_cache, device) + + cache_config = vllm_config.cache_config + if cache_config.cache_dtype == "auto": + kv_cache_dtype = self.dtype + else: + kv_cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype] + + # Build an attention group (and its non-causal metadata builder) for the + # encoder-only layers, grouped by backend + query/KV head config. Models + # are typically uniform, yielding a single group. + attn_layers = get_layers_from_vllm_config(vllm_config, Attention) + groups: dict[tuple[tuple[str, str], int, int, int], AttentionGroup] = {} + for name, layer in attn_layers.items(): + if layer.attn_type != AttentionType.ENCODER_ONLY: + continue + # No KV cache is bound for these layers; give them an (empty) + # device tensor so the attention forward context is well-formed. + layer.kv_cache = torch.empty(0, dtype=kv_cache_dtype, device=device) + backend = layer.get_attn_backend() + # Metadata builders require a uniform query-head count per group. + key = ( + backend.full_cls_name(), + layer.num_heads, + layer.num_kv_heads, + layer.head_size, + ) + group = groups.get(key) + if group is None: + spec = EncoderOnlyAttentionSpec( + block_size=cache_config.block_size, + num_kv_heads=layer.num_kv_heads, + head_size=layer.head_size, + dtype=kv_cache_dtype, + ) + group = AttentionGroup(backend, [], spec, kv_cache_group_id=len(groups)) + groups[key] = group + group.layer_names.append(name) + + self.encoder_attn_groups = list(groups.values()) + for group in self.encoder_attn_groups: + group.create_metadata_builders(vllm_config, device) + + # Encoder attention reads neither the block table nor the slot mapping + # (full varlen self-attention over the batch's q/k/v), but the metadata + # builder expects both tensors to be present. + self._dummy_block_table = torch.zeros( + self.max_num_reqs, 1, dtype=torch.int32, device=device + ) + self._dummy_slot_mapping = torch.zeros( + self.max_num_tokens, dtype=torch.int64, device=device + ) + + def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]: + # Encoder groups are built here rather than in init_attn_backend, so + # their cudagraph support must be surfaced to the runner separately. + support = AttentionCGSupport.ALWAYS + backend: str | None = None + for group in self.encoder_attn_groups: + builder = group.get_metadata_builder(0) + cg_support = builder.get_cudagraph_support( + self.vllm_config, cast(AttentionSpec, group.kv_cache_spec) + ) + if cg_support.value < support.value: + support = cg_support + backend = group.backend.__name__ + return support, backend + + def prepare_attn( + self, + input_batch: InputBatch, + cudagraph_mode: CUDAGraphMode, + block_tables: tuple[torch.Tensor, ...], + slot_mappings: torch.Tensor, + attn_groups: list[list[AttentionGroup]], + kv_cache_config: KVCacheConfig, + for_capture: bool = False, + ) -> dict[str, Any]: + attn_metadata = super().prepare_attn( + input_batch, + cudagraph_mode, + block_tables, + slot_mappings, + attn_groups, + kv_cache_config, + for_capture, + ) + attn_metadata.update( + self._build_encoder_attn_metadata(input_batch, cudagraph_mode, for_capture) + ) + return attn_metadata + + def _build_encoder_attn_metadata( + self, + input_batch: InputBatch, + cudagraph_mode: CUDAGraphMode, + for_capture: bool, + ) -> dict[str, Any]: + if cudagraph_mode == CUDAGraphMode.FULL: + num_reqs = input_batch.num_reqs_after_padding + num_tokens = input_batch.num_tokens_after_padding + else: + num_reqs = input_batch.num_reqs + num_tokens = input_batch.num_tokens + max_query_len = int(input_batch.num_scheduled_tokens.max()) + if for_capture: + max_seq_len = self.max_model_len + else: + max_seq_len = int(input_batch.seq_lens_cpu_upper_bound[:num_reqs].max()) + + # The encoder builder forces ``causal=False`` regardless of this value. + common_attn_metadata = CommonAttentionMetadata( + query_start_loc=input_batch.query_start_loc, + query_start_loc_cpu=torch.from_numpy(input_batch.query_start_loc_np), + seq_lens=input_batch.seq_lens[:num_reqs], + num_reqs=num_reqs, + num_actual_tokens=num_tokens, + max_query_len=max_query_len, + max_seq_len=max_seq_len, + block_table_tensor=self._dummy_block_table[:num_reqs], + slot_mapping=self._dummy_slot_mapping[:num_tokens], + seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound[:num_reqs], + positions=input_batch.positions, + ) + + attn_metadata: dict[str, Any] = {} + for group in self.encoder_attn_groups: + builder = group.get_metadata_builder(0) + if for_capture: + metadata = builder.build_for_cudagraph_capture(common_attn_metadata) + else: + metadata = builder.build( + common_prefix_len=0, common_attn_metadata=common_attn_metadata + ) + for name in group.layer_names: + attn_metadata[name] = metadata + return attn_metadata diff --git a/vllm/v1/worker/gpu/model_states/interface.py b/vllm/v1/worker/gpu/model_states/interface.py index df86efa4a7..666d460c20 100644 --- a/vllm/v1/worker/gpu/model_states/interface.py +++ b/vllm/v1/worker/gpu/model_states/interface.py @@ -9,6 +9,7 @@ import torch.nn as nn from vllm.config import VllmConfig from vllm.config.compilation import CUDAGraphMode from vllm.tasks import GenerationTask +from vllm.v1.attention.backend import AttentionCGSupport from vllm.v1.core.sched.output import NewRequestData from vllm.v1.kv_cache_interface import KVCacheConfig from vllm.v1.worker.gpu.input_batch import InputBatch @@ -95,6 +96,15 @@ class ModelState(ABC): def apply_staged_writes(self) -> None: return None + def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]: + """Cudagraph support of attention groups this ModelState builds outside + ``init_attn_backend`` (e.g. encoder-only layers). + + Returns the minimum support level and its backend name. The default of + ``ALWAYS`` imposes no extra constraint on the runner's cudagraph mode. + """ + return AttentionCGSupport.ALWAYS, None + def preprocess_state( self, input_batch: InputBatch, diff --git a/vllm/v1/worker/gpu/warmup.py b/vllm/v1/worker/gpu/warmup.py index f4047a0be8..785188dc3c 100644 --- a/vllm/v1/worker/gpu/warmup.py +++ b/vllm/v1/worker/gpu/warmup.py @@ -217,9 +217,14 @@ def warmup_kernels( model_runner.scheduler_config.max_num_seqs, model_runner.scheduler_config.max_num_batched_tokens // max(prompt_len, decode_query_len), - # Reserve block 0 (null block) and ensure we have enough blocks. - max(1, (model_runner.kv_cache_config.num_blocks - 1) // max_blocks_per_req), ) + if max_blocks_per_req > 0: + # Reserve block 0 (null block) and ensure we have enough blocks. + # Encoder-only models allocate no KV blocks, so this cap doesn't apply. + num_reqs = min( + num_reqs, + max(1, (model_runner.kv_cache_config.num_blocks - 1) // max_blocks_per_req), + ) req_ids = [f"_warmup_{i}_" for i in range(num_reqs)]