[Renderer] Move InputPreprocessor into Renderer (2/2) (#34560)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -96,15 +96,19 @@ from vllm.entrypoints.serve.tokenize.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.utils import get_max_tokens, sanitize_message
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs.data import PromptType, SingletonPrompt, TokensPrompt
|
||||
from vllm.inputs.data import (
|
||||
ProcessorInputs,
|
||||
PromptType,
|
||||
SingletonPrompt,
|
||||
TokensPrompt,
|
||||
token_inputs,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob, PromptLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal import MultiModalDataDict
|
||||
from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.renderers.inputs import TokPrompt
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_prompt_components,
|
||||
extract_prompt_len,
|
||||
@@ -206,7 +210,7 @@ class ServeContext(Generic[RequestT]):
|
||||
request_id: str
|
||||
created_time: int = field(default_factory=lambda: int(time.time()))
|
||||
lora_request: LoRARequest | None = None
|
||||
engine_prompts: list[TokPrompt] | None = None
|
||||
engine_prompts: list[ProcessorInputs] | None = None
|
||||
|
||||
result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (
|
||||
None
|
||||
@@ -249,7 +253,7 @@ class OpenAIServing:
|
||||
|
||||
async def beam_search(
|
||||
self,
|
||||
prompt: TokPrompt,
|
||||
prompt: ProcessorInputs,
|
||||
request_id: str,
|
||||
params: BeamSearchParams,
|
||||
lora_request: LoRARequest | None = None,
|
||||
@@ -262,86 +266,53 @@ class OpenAIServing:
|
||||
length_penalty = params.length_penalty
|
||||
include_stop_str_in_output = params.include_stop_str_in_output
|
||||
|
||||
input_processor = self.input_processor
|
||||
tokenizer = input_processor.tokenizer
|
||||
if tokenizer is None:
|
||||
raise VLLMValidationError(
|
||||
"You cannot use beam search when `skip_tokenizer_init=True`",
|
||||
parameter="skip_tokenizer_init",
|
||||
value=True,
|
||||
)
|
||||
|
||||
eos_token_id: int = tokenizer.eos_token_id # type: ignore
|
||||
|
||||
if isinstance(prompt, dict) and "encoder_prompt" in prompt:
|
||||
raise NotImplementedError("Encoder-decoder prompt not supported")
|
||||
|
||||
prompt_text: str | None = prompt.get("prompt") # type: ignore
|
||||
prompt_token_ids: list[int] = prompt.get("prompt_token_ids", []) # type: ignore
|
||||
multi_modal_data: MultiModalDataDict | None = prompt.get("multi_modal_data") # type: ignore
|
||||
|
||||
mm_processor_kwargs: dict[str, Any] | None = None
|
||||
|
||||
# This is a workaround to fix multimodal beam search; this is a
|
||||
# bandaid fix for 2 small problems:
|
||||
# 1. Multi_modal_data on the processed_inputs currently resolves to
|
||||
# `None`.
|
||||
# 2. preprocessing above expands the multimodal placeholders. However,
|
||||
# this happens again in generation, so the double expansion causes
|
||||
# a mismatch.
|
||||
# TODO - would be ideal to handle this more gracefully.
|
||||
|
||||
tokenized_length = len(prompt_token_ids)
|
||||
|
||||
tokenizer = self.renderer.get_tokenizer()
|
||||
eos_token_id = tokenizer.eos_token_id
|
||||
sort_beams_key = create_sort_beams_key_function(eos_token_id, length_penalty)
|
||||
|
||||
if prompt["type"] == "embeds":
|
||||
raise NotImplementedError("Embedding prompt not supported for beam search")
|
||||
if prompt["type"] == "enc_dec":
|
||||
raise NotImplementedError(
|
||||
"Encoder-decoder prompt not supported for beam search"
|
||||
)
|
||||
|
||||
prompt_text = prompt.get("prompt")
|
||||
prompt_token_ids = prompt["prompt_token_ids"]
|
||||
tokenized_length = len(prompt_token_ids)
|
||||
|
||||
logprobs_num = 2 * beam_width
|
||||
beam_search_params = SamplingParams(
|
||||
sampling_params = SamplingParams(
|
||||
logprobs=logprobs_num,
|
||||
max_tokens=1,
|
||||
temperature=temperature,
|
||||
)
|
||||
all_beams = [
|
||||
BeamSearchSequence(
|
||||
orig_prompt=prompt,
|
||||
tokens=prompt_token_ids,
|
||||
cum_logprob=0,
|
||||
logprobs=[],
|
||||
multi_modal_data=multi_modal_data,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
]
|
||||
completed = []
|
||||
|
||||
for _ in range(max_tokens):
|
||||
prompts_batch, lora_req_batch = zip(
|
||||
*[
|
||||
(
|
||||
TokensPrompt(
|
||||
prompt_token_ids=beam.tokens,
|
||||
multi_modal_data=beam.multi_modal_data,
|
||||
mm_processor_kwargs=beam.mm_processor_kwargs,
|
||||
),
|
||||
beam.lora_request,
|
||||
)
|
||||
for beam in all_beams
|
||||
]
|
||||
)
|
||||
|
||||
tasks = []
|
||||
request_id_batch = f"{request_id}-{random_uuid()}"
|
||||
|
||||
for i, (individual_prompt, lora_req) in enumerate(
|
||||
zip(prompts_batch, lora_req_batch)
|
||||
):
|
||||
for i, beam in enumerate(all_beams):
|
||||
prompt_item = beam.get_prompt()
|
||||
lora_request_item = beam.lora_request
|
||||
request_id_item = f"{request_id_batch}-beam-{i}"
|
||||
task = asyncio.create_task(
|
||||
collect_from_async_generator(
|
||||
self.engine_client.generate(
|
||||
individual_prompt,
|
||||
beam_search_params,
|
||||
prompt_item,
|
||||
sampling_params,
|
||||
request_id_item,
|
||||
lora_request=lora_req,
|
||||
lora_request=lora_request_item,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
)
|
||||
@@ -406,6 +377,7 @@ class OpenAIServing:
|
||||
logprobs_entry = result.outputs[0].logprobs[0]
|
||||
completed.append(
|
||||
BeamSearchSequence(
|
||||
orig_prompt=prompt,
|
||||
tokens=current_beam.tokens + [eos_token_id]
|
||||
if include_stop_str_in_output
|
||||
else current_beam.tokens,
|
||||
@@ -433,12 +405,11 @@ class OpenAIServing:
|
||||
logprobs_entry = result.outputs[0].logprobs[0]
|
||||
new_beams.append(
|
||||
BeamSearchSequence(
|
||||
orig_prompt=prompt,
|
||||
tokens=current_beam.tokens + [token_id],
|
||||
logprobs=current_beam.logprobs + [logprobs_entry],
|
||||
lora_request=current_beam.lora_request,
|
||||
cum_logprob=float(all_beams_logprob[idx]),
|
||||
multi_modal_data=current_beam.multi_modal_data,
|
||||
mm_processor_kwargs=current_beam.mm_processor_kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -958,7 +929,7 @@ class OpenAIServing:
|
||||
request: RendererRequest,
|
||||
prompt_input: str | list[str] | list[int] | list[list[int]] | None,
|
||||
prompt_embeds: bytes | list[bytes] | None,
|
||||
) -> list[TokPrompt]:
|
||||
) -> list[ProcessorInputs]:
|
||||
prompts = list[SingletonPrompt | bytes]()
|
||||
if prompt_embeds is not None: # embeds take higher priority
|
||||
prompts.extend(prompt_to_seq(prompt_embeds))
|
||||
@@ -971,7 +942,7 @@ class OpenAIServing:
|
||||
self,
|
||||
request: RendererRequest,
|
||||
prompts: Sequence[PromptType | bytes],
|
||||
) -> list[TokPrompt]:
|
||||
) -> list[ProcessorInputs]:
|
||||
renderer = self.renderer
|
||||
model_config = self.model_config
|
||||
|
||||
@@ -1004,7 +975,7 @@ class OpenAIServing:
|
||||
default_template_kwargs: dict[str, Any] | None,
|
||||
tool_dicts: list[dict[str, Any]] | None = None,
|
||||
tool_parser: Callable[[TokenizerLike], ToolParser] | None = None,
|
||||
) -> tuple[list[ConversationMessage], list[TokPrompt]]:
|
||||
) -> tuple[list[ConversationMessage], list[ProcessorInputs]]:
|
||||
from vllm.tokenizers.mistral import MistralTokenizer
|
||||
|
||||
renderer = self.renderer
|
||||
@@ -1052,13 +1023,13 @@ class OpenAIServing:
|
||||
|
||||
return conversation, [engine_prompt]
|
||||
|
||||
def _extract_prompt_components(self, prompt: object):
|
||||
def _extract_prompt_components(self, prompt: PromptType | ProcessorInputs):
|
||||
return extract_prompt_components(self.model_config, prompt)
|
||||
|
||||
def _extract_prompt_text(self, prompt: object):
|
||||
def _extract_prompt_text(self, prompt: ProcessorInputs):
|
||||
return self._extract_prompt_components(prompt).text
|
||||
|
||||
def _extract_prompt_len(self, prompt: object):
|
||||
def _extract_prompt_len(self, prompt: ProcessorInputs):
|
||||
return extract_prompt_len(self.model_config, prompt)
|
||||
|
||||
async def _render_next_turn(
|
||||
@@ -1088,16 +1059,14 @@ class OpenAIServing:
|
||||
async def _generate_with_builtin_tools(
|
||||
self,
|
||||
request_id: str,
|
||||
engine_prompt: TokPrompt,
|
||||
engine_prompt: ProcessorInputs,
|
||||
sampling_params: SamplingParams,
|
||||
tok_params: TokenizeParams,
|
||||
context: ConversationContext,
|
||||
lora_request: LoRARequest | None = None,
|
||||
priority: int = 0,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
):
|
||||
max_model_len = self.model_config.max_model_len
|
||||
prompt_text = self._extract_prompt_text(engine_prompt)
|
||||
|
||||
orig_priority = priority
|
||||
sub_request = 0
|
||||
@@ -1112,26 +1081,13 @@ class OpenAIServing:
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
tokenization_kwargs = tok_params.get_encode_kwargs()
|
||||
engine_request = self.input_processor.process_inputs(
|
||||
sub_request_id,
|
||||
generator = self.engine_client.generate(
|
||||
engine_prompt,
|
||||
sampling_params,
|
||||
lora_request=lora_request,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
)
|
||||
|
||||
generator = self.engine_client.generate(
|
||||
engine_request,
|
||||
sampling_params,
|
||||
sub_request_id,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
prompt_text=prompt_text,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
async for res in generator:
|
||||
@@ -1154,11 +1110,11 @@ class OpenAIServing:
|
||||
# Render the next prompt token ids and update sampling_params.
|
||||
if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
|
||||
token_ids = context.render_for_completion()
|
||||
engine_prompt = TokensPrompt(prompt_token_ids=token_ids)
|
||||
engine_prompt = token_inputs(token_ids)
|
||||
|
||||
sampling_params.max_tokens = max_model_len - len(token_ids)
|
||||
elif isinstance(context, ParsableContext):
|
||||
engine_prompts = await self._render_next_turn(
|
||||
(engine_prompt,) = await self._render_next_turn(
|
||||
context.request,
|
||||
context.parser.response_messages,
|
||||
context.tool_dicts,
|
||||
@@ -1166,8 +1122,6 @@ class OpenAIServing:
|
||||
context.chat_template,
|
||||
context.chat_template_content_format,
|
||||
)
|
||||
engine_prompt = engine_prompts[0]
|
||||
prompt_text = self._extract_prompt_text(engine_prompt)
|
||||
|
||||
sampling_params.max_tokens = get_max_tokens(
|
||||
max_model_len,
|
||||
@@ -1184,7 +1138,7 @@ class OpenAIServing:
|
||||
def _log_inputs(
|
||||
self,
|
||||
request_id: str,
|
||||
inputs: PromptType | TokPrompt,
|
||||
inputs: PromptType | ProcessorInputs,
|
||||
params: SamplingParams | PoolingParams | BeamSearchParams | None,
|
||||
lora_request: LoRARequest | None,
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user