[ModelRunner V2] Support encoder-only attention (#49331)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
co-authored by
Taneem Ibrahim
parent
0934b26790
commit
fdaa0d9e59
@@ -9,6 +9,7 @@ import torch.nn as nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.tasks import GenerationTask
|
||||
from vllm.v1.attention.backend import AttentionCGSupport
|
||||
from vllm.v1.core.sched.output import NewRequestData
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
@@ -95,6 +96,15 @@ class ModelState(ABC):
|
||||
def apply_staged_writes(self) -> None:
|
||||
return None
|
||||
|
||||
def get_additional_cg_support(self) -> tuple[AttentionCGSupport, str | None]:
|
||||
"""Cudagraph support of attention groups this ModelState builds outside
|
||||
``init_attn_backend`` (e.g. encoder-only layers).
|
||||
|
||||
Returns the minimum support level and its backend name. The default of
|
||||
``ALWAYS`` imposes no extra constraint on the runner's cudagraph mode.
|
||||
"""
|
||||
return AttentionCGSupport.ALWAYS, None
|
||||
|
||||
def preprocess_state(
|
||||
self,
|
||||
input_batch: InputBatch,
|
||||
|
||||
Reference in New Issue
Block a user