[ModelRunner V2] Support encoder-only attention (#49331)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
co-authored by
Taneem Ibrahim
parent
0934b26790
commit
fdaa0d9e59
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import AutoModel
|
||||
|
||||
from vllm.config import PoolerConfig
|
||||
|
||||
@@ -87,3 +89,55 @@ def test_models(
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model",
|
||||
[
|
||||
"BAAI/bge-base-en-v1.5",
|
||||
"intfloat/multilingual-e5-small",
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_encoder_only_model_runner_v2_attention(
|
||||
hf_runner,
|
||||
vllm_runner,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
) -> None:
|
||||
prompts = [
|
||||
"short input",
|
||||
"a longer input that exercises mixed sequence lengths",
|
||||
]
|
||||
|
||||
with hf_runner(model, dtype="float", auto_cls=AutoModel) as hf_model:
|
||||
hf_outputs = []
|
||||
for prompt in prompts:
|
||||
inputs = hf_model.tokenizer(prompt, return_tensors="pt")
|
||||
output = hf_model.model(**hf_model.wrap_device(inputs))
|
||||
embedding = torch.nn.functional.normalize(
|
||||
output.last_hidden_state[0, -1].float(), dim=0
|
||||
)
|
||||
hf_outputs.append(embedding.cpu().tolist())
|
||||
|
||||
monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", "1")
|
||||
with vllm_runner(
|
||||
model,
|
||||
runner="pooling",
|
||||
dtype="float",
|
||||
max_model_len=64,
|
||||
max_num_seqs=2,
|
||||
gpu_memory_utilization=0.25,
|
||||
pooler_config=PoolerConfig(
|
||||
task="embed", seq_pooling_type="LAST", use_activation=True
|
||||
),
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.embed(prompts)
|
||||
|
||||
check_embeddings_close(
|
||||
embeddings_0_lst=hf_outputs,
|
||||
embeddings_1_lst=vllm_outputs,
|
||||
name_0="hf",
|
||||
name_1="vllm",
|
||||
tol=1e-2,
|
||||
)
|
||||
|
||||
@@ -46,6 +46,18 @@ class AttentionCGSupportInfo:
|
||||
min_cg_support: AttentionCGSupport = AttentionCGSupport.ALWAYS
|
||||
min_cg_attn_backend: str | None = None
|
||||
|
||||
def narrow(
|
||||
self, support: AttentionCGSupport, backend: str | None
|
||||
) -> "AttentionCGSupportInfo":
|
||||
"""Return an info tightened by ``support`` if it is more restrictive.
|
||||
|
||||
Lets attention groups built outside ``init_attn_backend`` (e.g.
|
||||
encoder-only layers) contribute to the runner's cudagraph decision.
|
||||
"""
|
||||
if support.value < self.min_cg_support.value:
|
||||
return AttentionCGSupportInfo(support, backend)
|
||||
return self
|
||||
|
||||
|
||||
def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
|
||||
@@ -123,7 +123,7 @@ class BlockTables:
|
||||
if self.num_kv_cache_groups == 1:
|
||||
# Single group: write directly, skipping the per-write group lookup.
|
||||
self.block_tables[0].apply_write()
|
||||
else:
|
||||
elif self.num_kv_cache_groups > 1:
|
||||
# Multiple groups: apply all block tables with one fused kernel.
|
||||
assert self.fused_writer is not None
|
||||
self.fused_writer.apply(
|
||||
|
||||
@@ -459,6 +459,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.attn_groups, attn_cg_support, self.kernel_block_sizes = init_attn_backend(
|
||||
self.kv_cache_config, self.vllm_config, self.device
|
||||
)
|
||||
attn_cg_support = attn_cg_support.narrow(
|
||||
*self.model_state.get_additional_cg_support()
|
||||
)
|
||||
self.block_tables = BlockTables(
|
||||
block_sizes=block_sizes,
|
||||
max_num_reqs=self.max_num_reqs,
|
||||
|
||||
@@ -4,7 +4,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.attention import CrossAttention
|
||||
from vllm.model_executor.layers.attention import CrossAttention, EncoderOnlyAttention
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
|
||||
|
||||
@@ -27,6 +27,12 @@ def init_model_state(
|
||||
|
||||
return EncoderDecoderModelState(vllm_config, model, encoder_cache, device)
|
||||
|
||||
# Encoder-only models (BERT/RoBERTa): non-causal self-attention, no KV cache.
|
||||
if any(isinstance(m, EncoderOnlyAttention) for m in model.modules()):
|
||||
from vllm.v1.worker.gpu.model_states.encoder_only import EncoderOnlyModelState
|
||||
|
||||
return EncoderOnlyModelState(vllm_config, model, encoder_cache, device)
|
||||
|
||||
if vllm_config.model_config.is_hybrid:
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any, cast
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionCGSupport,
|
||||
AttentionType,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
EncoderOnlyAttentionSpec,
|
||||
KVCacheConfig,
|
||||
)
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
from vllm.v1.worker.gpu.model_states.default import DefaultModelState
|
||||
from vllm.v1.worker.utils import AttentionGroup
|
||||
|
||||
|
||||
class EncoderOnlyModelState(DefaultModelState):
|
||||
"""ModelState for encoder-only (BERT/RoBERTa) models.
|
||||
|
||||
Encoder attention needs no KV cache: it runs full, bidirectional
|
||||
self-attention over each request's tokens in a single forward. Such layers
|
||||
return no ``get_kv_cache_spec`` and therefore never enter
|
||||
``kv_cache_config.kv_cache_groups``, so the KV-backed attention path never
|
||||
builds their metadata. We build their (non-causal) metadata here, keeping
|
||||
the normal KV-backed path untouched.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
model: nn.Module,
|
||||
encoder_cache: EncoderCache | None,
|
||||
device: torch.device,
|
||||
):
|
||||
super().__init__(vllm_config, model, encoder_cache, device)
|
||||
|
||||
cache_config = vllm_config.cache_config
|
||||
if cache_config.cache_dtype == "auto":
|
||||
kv_cache_dtype = self.dtype
|
||||
else:
|
||||
kv_cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
|
||||
|
||||
# Build an attention group (and its non-causal metadata builder) for the
|
||||
# encoder-only layers, grouped by backend + query/KV head config. Models
|
||||
# are typically uniform, yielding a single group.
|
||||
attn_layers = get_layers_from_vllm_config(vllm_config, Attention)
|
||||
groups: dict[tuple[tuple[str, str], int, int, int], AttentionGroup] = {}
|
||||
for name, layer in attn_layers.items():
|
||||
if layer.attn_type != AttentionType.ENCODER_ONLY:
|
||||
continue
|
||||
# No KV cache is bound for these layers; give them an (empty)
|
||||
# device tensor so the attention forward context is well-formed.
|
||||
layer.kv_cache = torch.empty(0, dtype=kv_cache_dtype, device=device)
|
||||
backend = layer.get_attn_backend()
|
||||
# Metadata builders require a uniform query-head count per group.
|
||||
key = (
|
||||
backend.full_cls_name(),
|
||||
layer.num_heads,
|
||||
layer.num_kv_heads,
|
||||
layer.head_size,
|
||||
)
|
||||
group = groups.get(key)
|
||||
if group is None:
|
||||
spec = EncoderOnlyAttentionSpec(
|
||||
block_size=cache_config.block_size,
|
||||
num_kv_heads=layer.num_kv_heads,
|
||||
head_size=layer.head_size,
|
||||
dtype=kv_cache_dtype,
|
||||
)
|
||||
group = AttentionGroup(backend, [], spec, kv_cache_group_id=len(groups))
|
||||
groups[key] = group
|
||||
group.layer_names.append(name)
|
||||
|
||||
self.encoder_attn_groups = list(groups.values())
|
||||
for group in self.encoder_attn_groups:
|
||||
group.create_metadata_builders(vllm_config, device)
|
||||
|
||||
# Encoder attention reads neither the block table nor the slot mapping
|
||||
# (full varlen self-attention over the batch's q/k/v), but the metadata
|
||||
# builder expects both tensors to be present.
|
||||
self._dummy_block_table = torch.zeros(
|
||||
self.max_num_reqs, 1, dtype=torch.int32, device=device
|
||||
)
|
||||
self._dummy_slot_mapping = torch.zeros(
|
||||
self.max_num_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]:
|
||||
# Encoder groups are built here rather than in init_attn_backend, so
|
||||
# their cudagraph support must be surfaced to the runner separately.
|
||||
support = AttentionCGSupport.ALWAYS
|
||||
backend: str | None = None
|
||||
for group in self.encoder_attn_groups:
|
||||
builder = group.get_metadata_builder(0)
|
||||
cg_support = builder.get_cudagraph_support(
|
||||
self.vllm_config, cast(AttentionSpec, group.kv_cache_spec)
|
||||
)
|
||||
if cg_support.value < support.value:
|
||||
support = cg_support
|
||||
backend = group.backend.__name__
|
||||
return support, backend
|
||||
|
||||
def prepare_attn(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
cudagraph_mode: CUDAGraphMode,
|
||||
block_tables: tuple[torch.Tensor, ...],
|
||||
slot_mappings: torch.Tensor,
|
||||
attn_groups: list[list[AttentionGroup]],
|
||||
kv_cache_config: KVCacheConfig,
|
||||
for_capture: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
attn_metadata = super().prepare_attn(
|
||||
input_batch,
|
||||
cudagraph_mode,
|
||||
block_tables,
|
||||
slot_mappings,
|
||||
attn_groups,
|
||||
kv_cache_config,
|
||||
for_capture,
|
||||
)
|
||||
attn_metadata.update(
|
||||
self._build_encoder_attn_metadata(input_batch, cudagraph_mode, for_capture)
|
||||
)
|
||||
return attn_metadata
|
||||
|
||||
def _build_encoder_attn_metadata(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
cudagraph_mode: CUDAGraphMode,
|
||||
for_capture: bool,
|
||||
) -> dict[str, Any]:
|
||||
if cudagraph_mode == CUDAGraphMode.FULL:
|
||||
num_reqs = input_batch.num_reqs_after_padding
|
||||
num_tokens = input_batch.num_tokens_after_padding
|
||||
else:
|
||||
num_reqs = input_batch.num_reqs
|
||||
num_tokens = input_batch.num_tokens
|
||||
max_query_len = int(input_batch.num_scheduled_tokens.max())
|
||||
if for_capture:
|
||||
max_seq_len = self.max_model_len
|
||||
else:
|
||||
max_seq_len = int(input_batch.seq_lens_cpu_upper_bound[:num_reqs].max())
|
||||
|
||||
# The encoder builder forces ``causal=False`` regardless of this value.
|
||||
common_attn_metadata = CommonAttentionMetadata(
|
||||
query_start_loc=input_batch.query_start_loc,
|
||||
query_start_loc_cpu=torch.from_numpy(input_batch.query_start_loc_np),
|
||||
seq_lens=input_batch.seq_lens[:num_reqs],
|
||||
num_reqs=num_reqs,
|
||||
num_actual_tokens=num_tokens,
|
||||
max_query_len=max_query_len,
|
||||
max_seq_len=max_seq_len,
|
||||
block_table_tensor=self._dummy_block_table[:num_reqs],
|
||||
slot_mapping=self._dummy_slot_mapping[:num_tokens],
|
||||
seq_lens_cpu_upper_bound=input_batch.seq_lens_cpu_upper_bound[:num_reqs],
|
||||
positions=input_batch.positions,
|
||||
)
|
||||
|
||||
attn_metadata: dict[str, Any] = {}
|
||||
for group in self.encoder_attn_groups:
|
||||
builder = group.get_metadata_builder(0)
|
||||
if for_capture:
|
||||
metadata = builder.build_for_cudagraph_capture(common_attn_metadata)
|
||||
else:
|
||||
metadata = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
for name in group.layer_names:
|
||||
attn_metadata[name] = metadata
|
||||
return attn_metadata
|
||||
@@ -9,6 +9,7 @@ import torch.nn as nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.tasks import GenerationTask
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
@@ -95,6 +96,15 @@ class ModelState(ABC):
|
||||
def apply_staged_writes(self) -> None:
|
||||
return None
|
||||
|
||||
def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]:
|
||||
"""Cudagraph support of attention groups this ModelState builds outside
|
||||
``init_attn_backend`` (e.g. encoder-only layers).
|
||||
|
||||
Returns the minimum support level and its backend name. The default of
|
||||
``ALWAYS`` imposes no extra constraint on the runner's cudagraph mode.
|
||||
"""
|
||||
return AttentionCGSupport.ALWAYS, None
|
||||
|
||||
def preprocess_state(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
|
||||
@@ -217,9 +217,14 @@ def warmup_kernels(
|
||||
model_runner.scheduler_config.max_num_seqs,
|
||||
model_runner.scheduler_config.max_num_batched_tokens
|
||||
// max(prompt_len, decode_query_len),
|
||||
# Reserve block 0 (null block) and ensure we have enough blocks.
|
||||
max(1, (model_runner.kv_cache_config.num_blocks - 1) // max_blocks_per_req),
|
||||
)
|
||||
if max_blocks_per_req > 0:
|
||||
# Reserve block 0 (null block) and ensure we have enough blocks.
|
||||
# Encoder-only models allocate no KV blocks, so this cap doesn't apply.
|
||||
num_reqs = min(
|
||||
num_reqs,
|
||||
max(1, (model_runner.kv_cache_config.num_blocks - 1) // max_blocks_per_req),
|
||||
)
|
||||
|
||||
req_ids = [f"_warmup_{i}_" for i in range(num_reqs)]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user