[Core] Cross-attention KV caching and memory-management (towards eventual encoder/decoder model support) (#4837)
This commit is contained in:
@@ -430,6 +430,8 @@ class SequenceGroup:
|
||||
for an embedding model.
|
||||
pooling_params: The pooling parameters used to generate the pooling
|
||||
for an embedding model.
|
||||
encoder_seq: Optional, the single encoder sequence. Should be None
|
||||
unless you are working with an encoder/decoder model.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -441,6 +443,7 @@ class SequenceGroup:
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
embeddings: Optional[List[float]] = None,
|
||||
pooling_params: Optional[PoolingParams] = None,
|
||||
encoder_seq: Optional[Sequence] = None,
|
||||
) -> None:
|
||||
self.request_id = request_id
|
||||
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
|
||||
@@ -455,6 +458,7 @@ class SequenceGroup:
|
||||
self.state = SequenceGroupState()
|
||||
self.embeddings = embeddings
|
||||
self.pooling_params = pooling_params
|
||||
self.encoder_seq = encoder_seq
|
||||
|
||||
@property
|
||||
def prompt(self) -> Optional[str]:
|
||||
@@ -538,6 +542,12 @@ class SequenceGroup:
|
||||
seq for seq in self.seqs_dict.values() if seq.status == status
|
||||
]
|
||||
|
||||
def is_encoder_decoder(self) -> bool:
|
||||
return self.encoder_seq is not None
|
||||
|
||||
def get_encoder_seq(self) -> Optional[Sequence]:
|
||||
return self.encoder_seq
|
||||
|
||||
def get_unfinished_seqs(self) -> List[Sequence]:
|
||||
return [
|
||||
seq for seq in self.seqs_dict.values() if not seq.is_finished()
|
||||
@@ -621,6 +631,15 @@ class SequenceGroupMetadata:
|
||||
used in prefix caching.
|
||||
state: Internal state tied to this sequence group.
|
||||
multi_modal_data: Multi modal data.
|
||||
encoder_seq_data: Optional sequence data for encoder prompt
|
||||
(SequenceGroup.encoder_seq). Should be None
|
||||
unless you are working with an encoder/decoder
|
||||
model.
|
||||
cross_block_table: Optional cross-attention block table associated
|
||||
with the encoder prompt
|
||||
(SequenceGroup.encoder_seq). Should be None
|
||||
unless you are working with an encoder/decoder
|
||||
model.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -637,6 +656,8 @@ class SequenceGroupMetadata:
|
||||
computed_block_nums: Optional[List[int]] = None,
|
||||
state: Optional[SequenceGroupState] = None,
|
||||
multi_modal_data: Optional[MultiModalData] = None,
|
||||
encoder_seq_data: Optional[SequenceData] = None,
|
||||
cross_block_table: Optional[List[int]] = None,
|
||||
) -> None:
|
||||
self.request_id = request_id
|
||||
self.is_prompt = is_prompt
|
||||
@@ -648,6 +669,8 @@ class SequenceGroupMetadata:
|
||||
self.computed_block_nums = computed_block_nums
|
||||
self.multi_modal_data = multi_modal_data
|
||||
self.state = SequenceGroupState() if state is None else state
|
||||
self.encoder_seq_data = encoder_seq_data
|
||||
self.cross_block_table = cross_block_table
|
||||
self._token_chunk_size = token_chunk_size
|
||||
self.do_sample = do_sample
|
||||
|
||||
|
||||
Reference in New Issue
Block a user