[Core] Consolidate prompt arguments to LLM engines (#4328)
Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
co-authored by
Roger Wang
parent
290f4ada2b
commit
5ae5ed1e60
+17
-6
@@ -12,6 +12,7 @@ from transformers import (AutoModelForCausalLM, AutoProcessor, AutoTokenizer,
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
|
||||
from vllm.distributed import destroy_model_parallel
|
||||
from vllm.inputs import PromptInputs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sequence import MultiModalData
|
||||
|
||||
@@ -402,12 +403,22 @@ class VllmRunner:
|
||||
) -> List[Tuple[List[int], str]]:
|
||||
if images is not None:
|
||||
assert len(prompts) == images.shape[0]
|
||||
req_outputs = self.model.generate(
|
||||
prompts,
|
||||
sampling_params=sampling_params,
|
||||
multi_modal_data=MultiModalData(type=MultiModalData.Type.IMAGE,
|
||||
data=images)
|
||||
if images is not None else None)
|
||||
|
||||
prompt_inputs: List[PromptInputs] = []
|
||||
for i, prompt in enumerate(prompts):
|
||||
image = None if images is None else images[i:i + 1]
|
||||
mm_data = None if image is None else MultiModalData(
|
||||
type=MultiModalData.Type.IMAGE,
|
||||
data=image,
|
||||
)
|
||||
|
||||
prompt_inputs.append({
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": mm_data,
|
||||
})
|
||||
|
||||
req_outputs = self.model.generate(prompt_inputs,
|
||||
sampling_params=sampling_params)
|
||||
outputs = []
|
||||
for req_output in req_outputs:
|
||||
prompt_str = req_output.prompt
|
||||
|
||||
Reference in New Issue
Block a user