[Core] Subclass ModelRunner to support cross-attention & encoder sequences (towards eventual encoder/decoder model support) (#4942)
Co-authored-by: Andrew Feldman <afeld2012@gmail.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com>
This commit is contained in:
co-authored by
Andrew Feldman
Nick Hill
parent
660470e5a3
commit
fd95e026e0
+19
-1
@@ -70,12 +70,20 @@ class RequestOutput:
|
||||
Args:
|
||||
request_id: The unique ID of the request.
|
||||
prompt: The prompt string of the request.
|
||||
For encoder/decoder models, this is the
|
||||
decoder input prompt.
|
||||
prompt_token_ids: The token IDs of the prompt.
|
||||
For encoder/decoder models, this is the
|
||||
decoder input prompt token ids.
|
||||
prompt_logprobs: The log probabilities to return per prompt token.
|
||||
outputs: The output sequences of the request.
|
||||
finished: Whether the whole request is finished.
|
||||
metrics: Metrics associated with the request.
|
||||
lora_request: The LoRA request that was used to generate the output.
|
||||
encoder_prompt: The encoder prompt string of the request;
|
||||
None if decoder-only
|
||||
encoder_prompt_token_ids: The token IDs of the encoder prompt;
|
||||
None if decoder-only
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -88,6 +96,8 @@ class RequestOutput:
|
||||
finished: bool,
|
||||
metrics: Optional[RequestMetrics] = None,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
encoder_prompt: Optional[str] = None,
|
||||
encoder_prompt_token_ids: Optional[List[int]] = None,
|
||||
) -> None:
|
||||
self.request_id = request_id
|
||||
self.prompt = prompt
|
||||
@@ -97,6 +107,8 @@ class RequestOutput:
|
||||
self.finished = finished
|
||||
self.metrics = metrics
|
||||
self.lora_request = lora_request
|
||||
self.encoder_prompt = encoder_prompt
|
||||
self.encoder_prompt_token_ids = encoder_prompt_token_ids
|
||||
|
||||
@classmethod
|
||||
def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
|
||||
@@ -137,6 +149,8 @@ class RequestOutput:
|
||||
# Every sequence in the sequence group should have the same prompt.
|
||||
prompt = seq_group.prompt
|
||||
prompt_token_ids = seq_group.prompt_token_ids
|
||||
encoder_prompt = seq_group.encoder_prompt
|
||||
encoder_prompt_token_ids = seq_group.encoder_prompt_token_ids
|
||||
prompt_logprobs = seq_group.prompt_logprobs
|
||||
finished = seq_group.is_finished()
|
||||
finished_time = time.time() if finished else None
|
||||
@@ -148,12 +162,16 @@ class RequestOutput:
|
||||
outputs,
|
||||
finished,
|
||||
seq_group.metrics,
|
||||
lora_request=seq_group.lora_request)
|
||||
lora_request=seq_group.lora_request,
|
||||
encoder_prompt=encoder_prompt,
|
||||
encoder_prompt_token_ids=encoder_prompt_token_ids)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return (f"RequestOutput(request_id={self.request_id}, "
|
||||
f"prompt={self.prompt!r}, "
|
||||
f"prompt_token_ids={self.prompt_token_ids}, "
|
||||
f"encoder_prompt={self.encoder_prompt!r}, "
|
||||
f"encoder_prompt_token_ids={self.encoder_prompt_token_ids}, "
|
||||
f"prompt_logprobs={self.prompt_logprobs}, "
|
||||
f"outputs={self.outputs}, "
|
||||
f"finished={self.finished}, "
|
||||
|
||||
Reference in New Issue
Block a user