[Core] Cross-attention KV caching and memory-management (towards eventual encoder/decoder model support) (#4837)

This commit is contained in:
afeldman-nm
2024-05-29 16:09:13 +00:00
committed by GitHub
parent 594392d27a
commit 4238bc82f2
7 changed files with 731 additions and 65 deletions
+23
View File
@@ -430,6 +430,8 @@ class SequenceGroup:
for an embedding model.
pooling_params: The pooling parameters used to generate the pooling
for an embedding model.
encoder_seq: Optional, the single encoder sequence. Should be None
unless you are working with an encoder/decoder model.
"""
def __init__(
@@ -441,6 +443,7 @@ class SequenceGroup:
lora_request: Optional[LoRARequest] = None,
embeddings: Optional[List[float]] = None,
pooling_params: Optional[PoolingParams] = None,
encoder_seq: Optional[Sequence] = None,
) -> None:
self.request_id = request_id
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -455,6 +458,7 @@ class SequenceGroup:
self.state = SequenceGroupState()
self.embeddings = embeddings
self.pooling_params = pooling_params
self.encoder_seq = encoder_seq
@property
def prompt(self) -> Optional[str]:
@@ -538,6 +542,12 @@ class SequenceGroup:
seq for seq in self.seqs_dict.values() if seq.status == status
]
def is_encoder_decoder(self) -> bool:
return self.encoder_seq is not None
def get_encoder_seq(self) -> Optional[Sequence]:
return self.encoder_seq
def get_unfinished_seqs(self) -> List[Sequence]:
return [
seq for seq in self.seqs_dict.values() if not seq.is_finished()
@@ -621,6 +631,15 @@ class SequenceGroupMetadata:
used in prefix caching.
state: Internal state tied to this sequence group.
multi_modal_data: Multi modal data.
encoder_seq_data: Optional sequence data for encoder prompt
(SequenceGroup.encoder_seq). Should be None
unless you are working with an encoder/decoder
model.
cross_block_table: Optional cross-attention block table associated
with the encoder prompt
(SequenceGroup.encoder_seq). Should be None
unless you are working with an encoder/decoder
model.
"""
def __init__(
@@ -637,6 +656,8 @@ class SequenceGroupMetadata:
computed_block_nums: Optional[List[int]] = None,
state: Optional[SequenceGroupState] = None,
multi_modal_data: Optional[MultiModalData] = None,
encoder_seq_data: Optional[SequenceData] = None,
cross_block_table: Optional[List[int]] = None,
) -> None:
self.request_id = request_id
self.is_prompt = is_prompt
@@ -648,6 +669,8 @@ class SequenceGroupMetadata:
self.computed_block_nums = computed_block_nums
self.multi_modal_data = multi_modal_data
self.state = SequenceGroupState() if state is None else state
self.encoder_seq_data = encoder_seq_data
self.cross_block_table = cross_block_table
self._token_chunk_size = token_chunk_size
self.do_sample = do_sample