[Frontend] Parallelize preprocessing within the same request for pooling models online serving. (#49153)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -17,7 +17,7 @@ from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
CustomChatCompletionMessageParam,
|
||||
)
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
from vllm.renderers import merge_kwargs
|
||||
@@ -28,11 +28,14 @@ from vllm.utils.mistral import is_mistral_tokenizer
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..scoring.io_processor import JinaRankingIOProcessorMixin
|
||||
from ..typing import (
|
||||
ALLOfflineInputsContext,
|
||||
AnyOfflineInputsContext,
|
||||
AnyRenderParam,
|
||||
ChunkedEmbeddingMetadata,
|
||||
EncodeChatRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
PoolingChatLikeRequest,
|
||||
PoolingCompletionLikeRequest,
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
)
|
||||
@@ -76,9 +79,11 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
list(self.task_instructions.keys()),
|
||||
)
|
||||
|
||||
def pre_process_online(self, ctx: PoolingServeContext):
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
if isinstance(ctx.request, CohereEmbedRequest):
|
||||
self._pre_process_cohere_online(ctx)
|
||||
requests = self._get_request_factory_cohere_online(ctx)
|
||||
elif isinstance(
|
||||
ctx.request,
|
||||
(
|
||||
@@ -88,12 +93,11 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
EmbeddingBatchChatInputRequest,
|
||||
),
|
||||
):
|
||||
self._pre_process_openai_chat_online(ctx)
|
||||
requests = self._get_request_factory_chat_input_online(ctx)
|
||||
else:
|
||||
super().pre_process_online(ctx)
|
||||
requests = super().get_request_factory_online(ctx)
|
||||
|
||||
if self.enable_chunked_processing:
|
||||
self._pre_process_chunked(ctx)
|
||||
return requests
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
@@ -114,18 +118,21 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
# PTAL: examples/pooling/embed/openai_embedding_long_text
|
||||
#################################################################
|
||||
|
||||
def _pre_process_chunked(self, ctx: PoolingServeContext) -> None:
|
||||
def maybe_pre_process_chunked(self, ctx: PoolingServeContext) -> None:
|
||||
if not self.enable_chunked_processing:
|
||||
return None
|
||||
|
||||
if ctx.engine_inputs is None:
|
||||
raise ValueError("Engine prompts not available")
|
||||
|
||||
ctx.original_engine_inputs = ctx.engine_inputs
|
||||
request_id = ctx.request_id
|
||||
max_model_len = self.model_config.max_model_len
|
||||
chunked_engine_inputs: list[EngineInput] = []
|
||||
chunked_engine_inputs: list[PoolingEngineInput] = []
|
||||
prompt_request_ids: list[str] = []
|
||||
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] = []
|
||||
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
|
||||
token_ids = engine_input.get("prompt_token_ids", None)
|
||||
token_ids = engine_input["prompts"].get("prompt_token_ids", None)
|
||||
if token_ids is None:
|
||||
raise NotImplementedError(
|
||||
"Long Text Embedding with Chunked Processing does "
|
||||
@@ -138,7 +145,12 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
chunk_list(prompt_token_ids, max_model_len)
|
||||
):
|
||||
chunked_engine_inputs.append(
|
||||
tokens_input(prompt_token_ids=chunk_tokens)
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(prompt_token_ids=chunk_tokens),
|
||||
params=engine_input["params"],
|
||||
lora_requests=engine_input["lora_requests"],
|
||||
priorities=engine_input["priorities"],
|
||||
)
|
||||
)
|
||||
prompt_request_ids.append(
|
||||
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
|
||||
@@ -234,7 +246,7 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
|
||||
# Get original prompt token IDs for this prompt
|
||||
original_prompt = original_engine_inputs[prompt_idx]
|
||||
token_ids = original_prompt.get("prompt_token_ids", None)
|
||||
token_ids = original_prompt["prompts"].get("prompt_token_ids", None)
|
||||
if token_ids is None:
|
||||
raise NotImplementedError(
|
||||
"Long Text Embedding with Chunked Processing does "
|
||||
@@ -262,6 +274,68 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
|
||||
return None
|
||||
|
||||
#################################################################
|
||||
# Chat input Request Preprocessing & Postprocessing
|
||||
#################################################################
|
||||
|
||||
def _get_request_factory_chat_input_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
renderer = self.renderer
|
||||
|
||||
self._validate_chat_template(
|
||||
request_chat_template=request.chat_template,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
|
||||
if isinstance(
|
||||
request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest)
|
||||
):
|
||||
all_messages = request.messages
|
||||
else:
|
||||
all_messages = [request.messages]
|
||||
num_requests = len(all_messages)
|
||||
|
||||
default_template_kwargs = merge_kwargs(
|
||||
self.template_kwargs,
|
||||
dict(
|
||||
tools=self.tool_dicts,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
)
|
||||
|
||||
mm_config = self.model_config.multimodal_config
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
self.chat_template, self.chat_template_content_format
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeChatRenderParams(
|
||||
conversations=all_messages[i],
|
||||
chat_params=chat_params,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
return requests
|
||||
|
||||
#################################################################
|
||||
# Cohere Request Preprocessing & Postprocessing
|
||||
#################################################################
|
||||
@@ -378,71 +452,9 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
)
|
||||
return super().create_pooling_params(request)
|
||||
|
||||
def _pre_process_openai_chat_online(
|
||||
self,
|
||||
ctx: PoolingServeContext[
|
||||
EmbeddingChatRequest
|
||||
| EmbeddingBatchChatRequest
|
||||
| EmbeddingChatInputRequest
|
||||
| EmbeddingBatchChatInputRequest
|
||||
],
|
||||
) -> None:
|
||||
request = ctx.request
|
||||
self._validate_chat_template(
|
||||
request_chat_template=request.chat_template,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
|
||||
if isinstance(
|
||||
request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest)
|
||||
):
|
||||
all_messages = request.messages
|
||||
else:
|
||||
all_messages = [request.messages]
|
||||
ctx.engine_inputs = self._batch_render_openai_chat(request, all_messages)
|
||||
|
||||
def _batch_render_openai_chat(
|
||||
self,
|
||||
request: (
|
||||
EmbeddingChatRequest
|
||||
| EmbeddingBatchChatRequest
|
||||
| EmbeddingChatInputRequest
|
||||
| EmbeddingBatchChatInputRequest
|
||||
),
|
||||
all_messages: Sequence[list[ChatCompletionMessageParam]],
|
||||
) -> list[EngineInput]:
|
||||
renderer = self.renderer
|
||||
mm_config = self.model_config.multimodal_config
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
self.chat_template,
|
||||
self.chat_template_content_format,
|
||||
).with_defaults(
|
||||
merge_kwargs(
|
||||
None,
|
||||
dict(
|
||||
tools=None,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
),
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
_, engine_inputs = renderer.render_chat(
|
||||
all_messages,
|
||||
chat_params,
|
||||
tok_params,
|
||||
prompt_extras={
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
},
|
||||
)
|
||||
return engine_inputs
|
||||
|
||||
def _pre_process_cohere_online(self, ctx: PoolingServeContext) -> None:
|
||||
def _get_request_factory_cohere_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
"""Convert a ``CohereEmbedRequest`` into engine prompts.
|
||||
|
||||
If a model has a chat template the task instruction are rendered
|
||||
@@ -479,13 +491,12 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
task_prefix = self._get_task_instruction_prefix(input_type)
|
||||
|
||||
if task_prefix is None:
|
||||
ctx.engine_inputs = self._preprocess_cohere_text_completion(
|
||||
request,
|
||||
return self._get_request_factory_cohere_text_completion(
|
||||
ctx,
|
||||
texts,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
return
|
||||
|
||||
all_messages = [
|
||||
self._mixed_input_to_messages(
|
||||
@@ -497,27 +508,30 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
for text in texts
|
||||
]
|
||||
if self._has_chat_template():
|
||||
ctx.engine_inputs = self._batch_render_chat(
|
||||
request,
|
||||
return self._batch_render_chat(
|
||||
ctx,
|
||||
all_messages,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
else:
|
||||
ctx.engine_inputs = self._preprocess_cohere_text_completion(
|
||||
request,
|
||||
return self._get_request_factory_cohere_text_completion(
|
||||
ctx,
|
||||
self._apply_task_instruction(texts, input_type),
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
return
|
||||
|
||||
task_prefix = self._get_task_instruction_prefix(input_type)
|
||||
all_messages = [
|
||||
self._mixed_input_to_messages(inp, task_prefix=task_prefix) for inp in input
|
||||
]
|
||||
ctx.engine_inputs = self._batch_render_chat(
|
||||
request, all_messages, truncate_prompt_tokens, truncation_side
|
||||
|
||||
return self._batch_render_chat(
|
||||
ctx,
|
||||
all_messages,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
|
||||
def _has_chat_template(self) -> bool:
|
||||
@@ -531,13 +545,14 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
is not None
|
||||
)
|
||||
|
||||
def _preprocess_cohere_text_completion(
|
||||
def _get_request_factory_cohere_text_completion(
|
||||
self,
|
||||
request: CohereEmbedRequest,
|
||||
ctx: PoolingServeContext,
|
||||
texts: list[str],
|
||||
truncate_prompt_tokens: int | None,
|
||||
truncation_side: Literal["left", "right"] | None,
|
||||
) -> list[EngineInput]:
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
proxy = EmbeddingCompletionRequest(
|
||||
model=request.model,
|
||||
input=texts,
|
||||
@@ -546,50 +561,32 @@ class EmbedIOProcessor(PoolingIOProcessor):
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
truncation_side=truncation_side,
|
||||
)
|
||||
return self._preprocess_cmpl_online(
|
||||
proxy, prompt_input=proxy.input, prompt_embeds=None
|
||||
)
|
||||
ctx.request = proxy
|
||||
requests = super().get_request_factory_online(ctx)
|
||||
ctx.request = request
|
||||
return requests
|
||||
|
||||
def _batch_render_chat(
|
||||
self,
|
||||
request: CohereEmbedRequest,
|
||||
ctx: PoolingServeContext,
|
||||
all_messages: Sequence[list[ChatCompletionMessageParam]],
|
||||
truncate_prompt_tokens: int | None,
|
||||
truncation_side: Literal["left", "right"] | None,
|
||||
) -> list[EngineInput]:
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
"""Batch-render multiple conversations through the chat template."""
|
||||
if not all_messages:
|
||||
return []
|
||||
|
||||
proxy = EmbeddingChatRequest(
|
||||
request = ctx.request
|
||||
proxy = EmbeddingBatchChatRequest(
|
||||
model=request.model,
|
||||
messages=list(all_messages[0]),
|
||||
messages=all_messages,
|
||||
dimensions=request.output_dimension,
|
||||
encoding_format="float",
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
truncation_side=truncation_side,
|
||||
)
|
||||
|
||||
renderer = self.renderer
|
||||
mm_config = self.model_config.multimodal_config
|
||||
|
||||
tok_params = proxy.build_tok_params(self.model_config)
|
||||
chat_params = proxy.build_chat_params(
|
||||
self.chat_template,
|
||||
self.chat_template_content_format,
|
||||
).with_defaults(
|
||||
merge_kwargs(
|
||||
None,
|
||||
dict(
|
||||
tools=None,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
),
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
_, engine_inputs = renderer.render_chat(all_messages, chat_params, tok_params)
|
||||
return engine_inputs
|
||||
ctx.request = proxy
|
||||
requests = self._get_request_factory_chat_input_online(ctx)
|
||||
ctx.request = request
|
||||
return requests
|
||||
|
||||
def _validate_input_type(self, input_type: str | None) -> None:
|
||||
"""Raise if *input_type* is not supported by this model."""
|
||||
@@ -640,7 +637,9 @@ class TokenEmbedIOProcessor(PoolingIOProcessor):
|
||||
class JinaRankingTokenEmbedIOProcessor(
|
||||
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
|
||||
):
|
||||
def pre_process_online(self, ctx: PoolingServeContext):
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
if isinstance(request, PoolingCompletionLikeRequest):
|
||||
prompts = request.input
|
||||
@@ -655,20 +654,15 @@ class JinaRankingTokenEmbedIOProcessor(
|
||||
query=text_prompts[-1], docs=text_prompts[:-1]
|
||||
)
|
||||
|
||||
engine_inputs = self._preprocess_cmpl_online(
|
||||
request,
|
||||
prompt_input=prompt_input,
|
||||
prompt_embeds=None,
|
||||
)
|
||||
request.input = prompt_input
|
||||
return super().get_request_factory_online(ctx)
|
||||
elif isinstance(request, PoolingChatLikeRequest):
|
||||
raise ValueError("The JinaForRanking does not support chat Request.")
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.name} request type")
|
||||
|
||||
ctx.engine_inputs = engine_inputs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
|
||||
|
||||
Reference in New Issue
Block a user