[Core] Refactor _prepare_model_input_tensors - take 2 (#6164)
This commit is contained in:
@@ -1,11 +1,15 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass, fields
|
||||
from enum import Enum, auto
|
||||
from typing import (Any, Dict, Generic, List, Optional, Set, Tuple, Type,
|
||||
TypeVar)
|
||||
from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
|
||||
Tuple, Type, TypeVar)
|
||||
|
||||
import torch
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.sequence import SequenceGroupMetadata
|
||||
from vllm.worker.model_runner_base import ModelRunnerInputBuilderBase
|
||||
|
||||
|
||||
class AttentionType(Enum):
|
||||
DECODER = auto() # Decoder attention between previous layer Q/K/V
|
||||
@@ -35,6 +39,16 @@ class AttentionBackend(ABC):
|
||||
def make_metadata(cls, *args, **kwargs) -> "AttentionMetadata":
|
||||
return cls.get_metadata_cls()(*args, **kwargs)
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_builder_cls() -> Type["AttentionMetadataBuilder"]:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def make_metadata_builder(cls, *args,
|
||||
**kwargs) -> "AttentionMetadataBuilder":
|
||||
return cls.get_builder_cls()(*args, **kwargs)
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_kv_cache_shape(
|
||||
@@ -110,6 +124,33 @@ class AttentionMetadata:
|
||||
T = TypeVar("T", bound=AttentionMetadata)
|
||||
|
||||
|
||||
class AttentionMetadataBuilder(ABC, Generic[T]):
|
||||
"""Abstract class for attention metadata builders."""
|
||||
|
||||
@abstractmethod
|
||||
def __init__(self, input_builder) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def add_seq_group(self, seq_group_metadata: "SequenceGroupMetadata",
|
||||
token_lens: List[int], seq_lens: List[int],
|
||||
curr_seq_lens: List[int], query_lens: List[int],
|
||||
context_lens: List[int],
|
||||
curr_sliding_window_blocks: List[int],
|
||||
prefix_cache_hit: bool, chunked_prefill_enabled: bool):
|
||||
"""Add a sequence group to the metadata and update
|
||||
corresponding fields (in Python objects).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def build(self, runner: "ModelRunnerInputBuilderBase", seq_lens: List[int],
|
||||
query_lens: List[int], cuda_graph_pad_size: int,
|
||||
batch_size: int) -> T:
|
||||
"""Build attention metadata with on-device tensors."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class AttentionImpl(ABC, Generic[T]):
|
||||
|
||||
@abstractmethod
|
||||
|
||||
Reference in New Issue
Block a user