[ModelRunner V2] Support encoder-only attention (#49331)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Nick Hill
2026-07-27 00:38:44 +00:00
committed by GitHub
co-authored by Taneem Ibrahim
parent 0934b26790
commit fdaa0d9e59
8 changed files with 276 additions and 4 deletions
@@ -9,6 +9,7 @@ import torch.nn as nn
from vllm.config import VllmConfig
from vllm.config.compilation import CUDAGraphMode
from vllm.tasks import GenerationTask
from vllm.v1.attention.backend import AttentionCGSupport
from vllm.v1.core.sched.output import NewRequestData
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.worker.gpu.input_batch import InputBatch
@@ -95,6 +96,15 @@ class ModelState(ABC):
def apply_staged_writes(self) -> None:
return None
def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]:
"""Cudagraph support of attention groups this ModelState builds outside
``init_attn_backend`` (e.g. encoder-only layers).
Returns the minimum support level and its backend name. The default of
``ALWAYS`` imposes no extra constraint on the runner's cudagraph mode.
"""
return AttentionCGSupport.ALWAYS, None
def preprocess_state(
self,
input_batch: InputBatch,