[Frontend] Add Support for MM Encoder/Decoder Beam Search (Offline) (#36153)
Signed-off-by: Alex Brooks <albrooks@redhat.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
parent
5b3ba94ab4
commit
10f4db4dbe
+53
-5
@@ -3,7 +3,8 @@
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from vllm.inputs import TokenInputs, token_inputs
|
||||
from vllm.inputs import EncoderDecoderInputs, TokenInputs, token_inputs
|
||||
from vllm.inputs.data import DecoderInputs
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import MultiModalInputs, mm_inputs
|
||||
@@ -17,9 +18,9 @@ class BeamSearchSequence:
|
||||
about to be returned to the user.
|
||||
"""
|
||||
|
||||
orig_prompt: TokenInputs | MultiModalInputs
|
||||
orig_prompt: TokenInputs | MultiModalInputs | EncoderDecoderInputs
|
||||
|
||||
# The tokens include the prompt.
|
||||
# NOTE: Tokens represents decoder tokens in the encoder / decoder case
|
||||
tokens: list[int]
|
||||
logprobs: list[dict[int, Logprob]]
|
||||
lora_request: LoRARequest | None = None
|
||||
@@ -31,6 +32,10 @@ class BeamSearchSequence:
|
||||
def get_prompt(self):
|
||||
prompt = self.orig_prompt
|
||||
|
||||
if prompt["type"] == "enc_dec":
|
||||
return self._build_encoder_decoder_inputs(prompt)
|
||||
|
||||
# Handle decoder-only inputs
|
||||
prompt_text = prompt.get("prompt")
|
||||
cache_salt = prompt.get("cache_salt")
|
||||
|
||||
@@ -50,6 +55,44 @@ class BeamSearchSequence:
|
||||
cache_salt=cache_salt,
|
||||
)
|
||||
|
||||
def _build_encoder_decoder_inputs(
|
||||
self, prompt: EncoderDecoderInputs
|
||||
) -> EncoderDecoderInputs:
|
||||
"""Rebuild the encoder-decoder inputs with the current beam search
|
||||
sequence's tokens.
|
||||
|
||||
FIXME (alex) - the encoder multimodal cache is not properly wired up
|
||||
yet, which means that currently we are running the encoder on every
|
||||
new beam because num_computed_tokens is 0 on each new request. This
|
||||
will be fixed once the cache is correctly implemented.
|
||||
"""
|
||||
dec_prompt = prompt["decoder_prompt"]
|
||||
|
||||
# Rebuild decoder prompt with updated tokens,
|
||||
# but keep everything else the same.
|
||||
new_dec_prompt: DecoderInputs
|
||||
if dec_prompt["type"] == "multimodal":
|
||||
new_dec_prompt = mm_inputs(
|
||||
self.tokens,
|
||||
mm_kwargs=dec_prompt["mm_kwargs"],
|
||||
mm_hashes=dec_prompt["mm_hashes"],
|
||||
mm_placeholders=dec_prompt["mm_placeholders"],
|
||||
prompt=dec_prompt.get("prompt"),
|
||||
cache_salt=dec_prompt.get("cache_salt"),
|
||||
)
|
||||
else:
|
||||
new_dec_prompt = token_inputs(
|
||||
self.tokens,
|
||||
prompt=dec_prompt.get("prompt"),
|
||||
cache_salt=dec_prompt.get("cache_salt"),
|
||||
)
|
||||
|
||||
return EncoderDecoderInputs(
|
||||
type="enc_dec",
|
||||
encoder_prompt=prompt["encoder_prompt"],
|
||||
decoder_prompt=new_dec_prompt,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BeamSearchOutput:
|
||||
@@ -64,15 +107,20 @@ class BeamSearchOutput:
|
||||
class BeamSearchInstance:
|
||||
def __init__(
|
||||
self,
|
||||
prompt: TokenInputs | MultiModalInputs,
|
||||
prompt: TokenInputs | MultiModalInputs | EncoderDecoderInputs,
|
||||
lora_request: LoRARequest | None = None,
|
||||
logprobs: list[dict[int, Logprob]] | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
decoder_prompt = (
|
||||
prompt if prompt["type"] != "enc_dec" else prompt["decoder_prompt"]
|
||||
)
|
||||
initial_tokens = decoder_prompt["prompt_token_ids"]
|
||||
|
||||
self.beams: list[BeamSearchSequence] = [
|
||||
BeamSearchSequence(
|
||||
orig_prompt=prompt,
|
||||
tokens=prompt["prompt_token_ids"],
|
||||
tokens=initial_tokens,
|
||||
logprobs=[] if logprobs is None else list(logprobs),
|
||||
lora_request=lora_request,
|
||||
**kwargs,
|
||||
|
||||
Reference in New Issue
Block a user