[Core][Bugfix] Fix Online MM Beam Search (#19688)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>
This commit is contained in:
@@ -15,7 +15,8 @@ from tqdm.auto import tqdm
|
||||
from typing_extensions import TypeVar, deprecated
|
||||
|
||||
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
|
||||
BeamSearchSequence, get_beam_search_score)
|
||||
BeamSearchSequence,
|
||||
create_sort_beams_key_function)
|
||||
from vllm.config import (CompilationConfig, ModelDType, TokenizerMode,
|
||||
is_init_field)
|
||||
from vllm.engine.arg_utils import (EngineArgs, HfOverrides, PoolerConfig,
|
||||
@@ -575,10 +576,11 @@ class LLM:
|
||||
lora_requests = self._get_beam_search_lora_requests(
|
||||
lora_request, prompts)
|
||||
|
||||
def sort_beams_key(x: BeamSearchSequence) -> float:
|
||||
return get_beam_search_score(x.tokens, x.cum_logprob,
|
||||
tokenizer.eos_token_id,
|
||||
length_penalty)
|
||||
tokenizer = self.get_tokenizer()
|
||||
sort_beams_key = create_sort_beams_key_function(
|
||||
tokenizer.eos_token_id,
|
||||
length_penalty,
|
||||
)
|
||||
|
||||
def create_tokens_prompt_from_beam(
|
||||
beam: BeamSearchSequence) -> TokensPrompt:
|
||||
@@ -593,7 +595,6 @@ class LLM:
|
||||
"mm_processor_kwargs"] = beam.mm_processor_kwargs
|
||||
return TokensPrompt(**token_prompt_kwargs)
|
||||
|
||||
tokenizer = self.get_tokenizer()
|
||||
# generate 2 * beam_width candidates at each step
|
||||
# following the huggingface transformers implementation
|
||||
# at https://github.com/huggingface/transformers/blob/e15687fffe5c9d20598a19aeab721ae0a7580f8a/src/transformers/generation/beam_search.py#L534 # noqa
|
||||
|
||||
Reference in New Issue
Block a user