[Feature] Add vision language model support. (#3042)
This commit is contained in:
@@ -303,6 +303,25 @@ class SequenceGroupState:
|
||||
generator: Optional = None
|
||||
|
||||
|
||||
class MultiModalData:
|
||||
"""Multi modal request.
|
||||
|
||||
Args:
|
||||
type: The data type.
|
||||
data: The actual data.
|
||||
The required shape and semantic meaning of it depends on the vision
|
||||
language config of the hosted model.
|
||||
See `VisionLanguageConfig` in `config.py`.
|
||||
"""
|
||||
|
||||
class Type(enum.Enum):
|
||||
IMAGE = enum.auto()
|
||||
|
||||
def __init__(self, type: Type, data: "torch.Tensor"):
|
||||
self.type = type
|
||||
self.data = data
|
||||
|
||||
|
||||
class SequenceGroup:
|
||||
"""A group of sequences that are generated from the same prompt.
|
||||
|
||||
@@ -312,6 +331,7 @@ class SequenceGroup:
|
||||
sampling_params: The sampling parameters used to generate the outputs.
|
||||
arrival_time: The arrival time of the request.
|
||||
lora_request: LoRA request.
|
||||
multi_modal_data: Multi modal data associated with the request.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -321,6 +341,7 @@ class SequenceGroup:
|
||||
sampling_params: SamplingParams,
|
||||
arrival_time: float,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
multi_modal_data: Optional[MultiModalData] = None,
|
||||
) -> None:
|
||||
self.request_id = request_id
|
||||
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
|
||||
@@ -333,6 +354,7 @@ class SequenceGroup:
|
||||
self.lora_request = lora_request
|
||||
self.prompt_logprobs: Optional[PromptLogprobs] = None
|
||||
self.state = SequenceGroupState()
|
||||
self.multi_modal_data = multi_modal_data
|
||||
|
||||
@property
|
||||
def prompt(self) -> str:
|
||||
@@ -450,6 +472,7 @@ class SequenceGroupMetadata:
|
||||
numbers)
|
||||
state: Internal state tied to this sequence group.
|
||||
lora_request: LoRA request.
|
||||
multi_modal_data: Multi modal data.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -462,6 +485,7 @@ class SequenceGroupMetadata:
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
computed_block_nums: Optional[List[int]] = None,
|
||||
state: Optional[SequenceGroupState] = None,
|
||||
multi_modal_data: Optional[MultiModalData] = None,
|
||||
) -> None:
|
||||
self.request_id = request_id
|
||||
self.is_prompt = is_prompt
|
||||
@@ -470,6 +494,7 @@ class SequenceGroupMetadata:
|
||||
self.block_tables = block_tables
|
||||
self.lora_request = lora_request
|
||||
self.computed_block_nums = computed_block_nums
|
||||
self.multi_modal_data = multi_modal_data
|
||||
self.state = SequenceGroupState() if state is None else state
|
||||
|
||||
@property
|
||||
|
||||
Reference in New Issue
Block a user