[Frontend] Overlap preprocessing and computation for pooling models offline inference (#47699)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -103,10 +103,10 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s models/language/generation_ppl_test
|
||||
|
||||
- label: Language Models Test (Extended Pooling) # 36min
|
||||
- label: Language Models Test (Extended Pooling)
|
||||
device: h200_35gb
|
||||
key: language-models-test-extended-pooling
|
||||
timeout_in_minutes: 70
|
||||
timeout_in_minutes: 120
|
||||
optional: true
|
||||
source_file_dependencies:
|
||||
- vllm/
|
||||
@@ -116,7 +116,7 @@ steps:
|
||||
mirror:
|
||||
amd:
|
||||
device: mi325_1
|
||||
timeout_in_minutes: 100
|
||||
timeout_in_minutes: 120
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import weakref
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm import LLM, PoolingParams
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
MODEL_NAME = "intfloat/multilingual-e5-small"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def llm():
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
max_num_seqs=2, # small to trigger tiling
|
||||
tensor_parallel_size=1,
|
||||
gpu_memory_utilization=0.75,
|
||||
enforce_eager=True,
|
||||
seed=0,
|
||||
)
|
||||
|
||||
yield weakref.proxy(llm)
|
||||
|
||||
del llm
|
||||
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_tiling_engine_basic(llm):
|
||||
"""
|
||||
Basic test with a small number of prompts (less than max_num_seqs).
|
||||
No tiling should be triggered, but the engine still processes correctly.
|
||||
"""
|
||||
prompts = ["Hello", "World"]
|
||||
outputs = llm.encode(prompts, pooling_task="embed")
|
||||
assert len(outputs) == len(prompts)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_tiling_engine_many_requests(llm):
|
||||
"""
|
||||
Test with a large number of prompts that exceeds max_num_seqs.
|
||||
This verifies that _run_tiling_engine correctly chunks requests,
|
||||
processes all of them, and returns outputs in the correct order.
|
||||
"""
|
||||
num_prompts = 10
|
||||
prompts = [f"Prompt {i}" for i in range(num_prompts)]
|
||||
outputs = llm.encode(prompts, pooling_task="embed")
|
||||
assert len(outputs) == num_prompts
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_tiling_engine_with_pooling_params(llm):
|
||||
"""
|
||||
Test the tiling engine when different PoolingParams are provided.
|
||||
The engine must handle a list of params that matches the number of prompts.
|
||||
"""
|
||||
num_prompts = 10
|
||||
prompts = [f"Prompt {i}" for i in range(num_prompts)]
|
||||
pooling_params = [PoolingParams() for _ in range(num_prompts)]
|
||||
|
||||
outputs = llm.encode(prompts, pooling_params=pooling_params, pooling_task="embed")
|
||||
assert len(outputs) == num_prompts
|
||||
|
||||
# Single PoolingParams shared across all prompts
|
||||
single_param = PoolingParams()
|
||||
outputs = llm.encode(prompts, pooling_params=single_param, pooling_task="embed")
|
||||
assert len(outputs) == num_prompts
|
||||
|
||||
# None PoolingParams should fall back to default
|
||||
outputs = llm.encode(prompts, pooling_params=None, pooling_task="embed")
|
||||
assert len(outputs) == num_prompts
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_tiling_engine_abort_on_exception(llm):
|
||||
"""
|
||||
Test that abort_request IS called with the correct arguments when an
|
||||
exception occurs inside the engine's step() loop.
|
||||
"""
|
||||
prompts = ["Prompt 0", "Prompt 1", "Prompt 2"]
|
||||
|
||||
# Mock the step method to throw an exception on the second call
|
||||
original_step = llm.llm_engine.step
|
||||
call_count = 0
|
||||
|
||||
def mocked_step():
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 2:
|
||||
raise RuntimeError("Simulated engine error")
|
||||
return original_step()
|
||||
|
||||
with mock.patch.object(llm.llm_engine, "step", side_effect=mocked_step):
|
||||
# We expect an exception to be raised from encode
|
||||
with mock.patch.object(llm.llm_engine, "abort_request") as mock_abort: # noqa: SIM117
|
||||
with pytest.raises(RuntimeError, match="Simulated engine error"):
|
||||
llm.encode(prompts, pooling_task="embed")
|
||||
|
||||
args, kwargs = mock_abort.call_args
|
||||
request_ids = args[0]
|
||||
assert isinstance(request_ids, list)
|
||||
assert len(request_ids) > 0
|
||||
@@ -12,7 +12,7 @@ from vllm.config import PoolerConfig
|
||||
"model",
|
||||
["BAAI/bge-m3"],
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", ["half"])
|
||||
@pytest.mark.parametrize("dtype", ["float"])
|
||||
@torch.inference_mode
|
||||
def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype: str):
|
||||
with vllm_runner(
|
||||
@@ -20,13 +20,11 @@ def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype
|
||||
runner="pooling",
|
||||
pooler_config=PoolerConfig(task="token_embed"),
|
||||
max_model_len=None,
|
||||
dtype=dtype,
|
||||
) as vllm_model:
|
||||
vllm_outputs = vllm_model.token_embed(example_prompts)
|
||||
|
||||
with hf_runner(
|
||||
model,
|
||||
auto_cls=AutoModel,
|
||||
) as hf_model:
|
||||
with hf_runner(model, auto_cls=AutoModel, dtype=dtype) as hf_model:
|
||||
tokenizer = hf_model.tokenizer
|
||||
hf_outputs = []
|
||||
for prompt in example_prompts:
|
||||
@@ -34,7 +32,6 @@ def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype
|
||||
inputs = hf_model.wrap_device(inputs)
|
||||
output = hf_model.model(**inputs)
|
||||
embedding = output.last_hidden_state[0].float()
|
||||
# normal
|
||||
hf_outputs.append(embedding.cpu())
|
||||
|
||||
for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
|
||||
|
||||
@@ -356,7 +356,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin):
|
||||
# path; the synchronous `LLM` entrypoint runs multimodal
|
||||
# preprocessing serially. Warn so the setting is not a silent
|
||||
# no-op. See vllm-project/vllm#42901.
|
||||
if self.model_config.renderer_num_workers > 1:
|
||||
if self.model_config.renderer_num_workers > 1 and self.runner_type != "pooling":
|
||||
logger.warning_once(
|
||||
"`renderer_num_workers=%d` was set, but the offline `LLM` "
|
||||
"entrypoint uses the synchronous renderer path and runs "
|
||||
|
||||
@@ -2,9 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Final
|
||||
from typing import Any, Final, cast
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput, PromptType
|
||||
from vllm import (
|
||||
PoolingParams,
|
||||
PoolingRequestOutput,
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
@@ -14,18 +17,25 @@ from vllm.entrypoints.chat_utils import (
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.typing import RendererChatRequest, RendererRequest
|
||||
from vllm.inputs import EngineInput, SingletonPrompt
|
||||
from vllm.renderers import BaseRenderer, TokenizeParams, merge_kwargs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||
from vllm.tool_parsers import ToolParser
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ..scoring.typing import ScoringData
|
||||
from ..typing import (
|
||||
OfflineInputsContext,
|
||||
ALLOfflineInputsContext,
|
||||
EncodeChatRenderParams,
|
||||
EncodeCMPLRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
PoolingChatLikeRequest,
|
||||
PoolingCompletionLikeRequest,
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
RequestGenerator,
|
||||
ScoringRenderParams,
|
||||
)
|
||||
|
||||
|
||||
@@ -98,16 +108,62 @@ class PoolingIOProcessor:
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
|
||||
assert not isinstance(ctx.prompts, ScoringData) and not (
|
||||
isinstance(ctx.prompts, dict) and "data" in ctx.prompts
|
||||
)
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
|
||||
prompts_seq = prompt_to_seq(ctx.prompts)
|
||||
num_requests = len(prompts_seq)
|
||||
pooling_task = ctx.pooling_task
|
||||
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(self.model_config, prompt)
|
||||
)
|
||||
for prompt in prompts_seq
|
||||
]
|
||||
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(ctx.tokenization_kwargs or {})
|
||||
)
|
||||
return self._preprocess_cmpl_offline(prompts=prompts_seq, tok_params=tok_params)
|
||||
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams]
|
||||
if ctx.pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
else:
|
||||
pooling_params = ctx.pooling_params
|
||||
|
||||
params_seq = self._params_to_seq(pooling_params, num_requests)
|
||||
|
||||
for param in params_seq:
|
||||
if param.task is None:
|
||||
param.task = pooling_task
|
||||
elif pooling_task == "plugin":
|
||||
# `plugin` task uses io_processor.parse_request to verify inputs.
|
||||
# We actually allow plugin to overwrite pooling_task.
|
||||
pass
|
||||
elif param.task != pooling_task:
|
||||
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
|
||||
raise ValueError(msg)
|
||||
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
def request_factory() -> RequestGenerator:
|
||||
for i in range(num_requests):
|
||||
yield EncodeCMPLRenderParams(
|
||||
prompts=parsed_prompts[i],
|
||||
tok_params=tok_params,
|
||||
prompt_extras=None,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
|
||||
return request_factory, num_requests
|
||||
|
||||
def post_process_offline(
|
||||
self,
|
||||
@@ -118,6 +174,41 @@ class PoolingIOProcessor:
|
||||
#######################################
|
||||
# helpers
|
||||
|
||||
def render(
|
||||
self,
|
||||
render_params: EncodeCMPLRenderParams
|
||||
| EncodeChatRenderParams
|
||||
| ScoringRenderParams,
|
||||
) -> PoolingEngineInput:
|
||||
if "conversations" in render_params:
|
||||
render_params = cast(EncodeChatRenderParams, render_params)
|
||||
(_,), engine_input = self.renderer.render_chat(
|
||||
conversations=[render_params["conversations"]],
|
||||
chat_params=render_params["chat_params"],
|
||||
tok_params=render_params["tok_params"],
|
||||
prompt_extras=render_params["prompt_extras"],
|
||||
skip_mm_cache=render_params["skip_mm_cache"],
|
||||
)
|
||||
elif "prompts" in render_params:
|
||||
render_params = cast(EncodeCMPLRenderParams, render_params)
|
||||
engine_input = self.renderer.render_cmpl(
|
||||
prompts=[render_params["prompts"]],
|
||||
tok_params=render_params["tok_params"],
|
||||
prompt_extras=render_params["prompt_extras"],
|
||||
skip_mm_cache=render_params["skip_mm_cache"],
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported render_params type {render_params.__class__.__name__}"
|
||||
)
|
||||
|
||||
return PoolingEngineInput(
|
||||
prompts=engine_input[0],
|
||||
params=render_params["params"],
|
||||
lora_requests=render_params["lora_requests"],
|
||||
priorities=render_params["priorities"],
|
||||
)
|
||||
|
||||
def _preprocess_cmpl_online(
|
||||
self,
|
||||
request: RendererRequest,
|
||||
@@ -196,26 +287,6 @@ class PoolingIOProcessor:
|
||||
|
||||
return conversation, [engine_input]
|
||||
|
||||
def _preprocess_cmpl_offline(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
tok_params: TokenizeParams,
|
||||
prompt_extras: dict[str, Any] | None = None,
|
||||
) -> Sequence[EngineInput]:
|
||||
prompts = prompt_to_seq(prompts)
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(self.model_config, prompt)
|
||||
)
|
||||
for prompt in prompts
|
||||
]
|
||||
|
||||
return self.renderer.render_cmpl(
|
||||
parsed_prompts, tok_params, prompt_extras=prompt_extras
|
||||
)
|
||||
|
||||
def _validate_chat_template(
|
||||
self,
|
||||
request_chat_template: str | None,
|
||||
@@ -251,3 +322,35 @@ class PoolingIOProcessor:
|
||||
return params
|
||||
|
||||
return [params] * num_requests
|
||||
|
||||
def _lora_request_to_seq(
|
||||
self,
|
||||
lora_request: LoRARequest | None | Sequence[LoRARequest | None],
|
||||
num_requests: int,
|
||||
) -> Sequence[LoRARequest | None]:
|
||||
if isinstance(lora_request, Sequence):
|
||||
if len(lora_request) != num_requests:
|
||||
raise ValueError(
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
f"and lora_request ({len(lora_request)}) must be the same."
|
||||
)
|
||||
|
||||
return lora_request
|
||||
|
||||
return [lora_request] * num_requests
|
||||
|
||||
def _priority_to_seq(
|
||||
self,
|
||||
priority: Sequence[int] | None,
|
||||
num_requests: int,
|
||||
) -> Sequence[int]:
|
||||
if priority is not None:
|
||||
if len(priority) != num_requests:
|
||||
raise ValueError(
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
f"and priority ({len(priority)}) must be the same."
|
||||
)
|
||||
|
||||
return priority
|
||||
|
||||
return [0] * num_requests
|
||||
|
||||
@@ -28,11 +28,13 @@ from vllm.utils.mistral import is_mistral_tokenizer
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..scoring.io_processor import JinaRankingIOProcessorMixin
|
||||
from ..typing import (
|
||||
ALLOfflineInputsContext,
|
||||
ChunkedEmbeddingMetadata,
|
||||
OfflineInputsContext,
|
||||
OfflineEncodeInputsContext,
|
||||
PoolingChatLikeRequest,
|
||||
PoolingCompletionLikeRequest,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
)
|
||||
from .protocol import (
|
||||
CohereEmbedContent,
|
||||
@@ -665,7 +667,10 @@ class JinaRankingTokenEmbedIOProcessor(
|
||||
|
||||
ctx.engine_inputs = engine_inputs
|
||||
|
||||
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
|
||||
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
|
||||
|
||||
@@ -677,4 +682,4 @@ class JinaRankingTokenEmbedIOProcessor(
|
||||
query=text_prompts[-1], docs=text_prompts[:-1]
|
||||
)
|
||||
|
||||
return super().pre_process_offline(ctx)
|
||||
return super().get_request_factory_offline(ctx)
|
||||
|
||||
@@ -20,10 +20,18 @@ from vllm.outputs import (
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.tasks import SCORE_TYPE_MAP, PoolingTask, SupportedTask
|
||||
|
||||
from .base.io_processor import PoolingIOProcessor
|
||||
from .factories import init_pooling_io_processors
|
||||
from .scoring.io_processor import ScoringIOProcessor
|
||||
from .scoring.typing import ScoreInput
|
||||
from .typing import OfflineInputsContext, OfflineOutputsContext
|
||||
from .typing import (
|
||||
ALLOfflineInputsContext,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
OfflinePluginInputsContext,
|
||||
OfflineScoringInputsContext,
|
||||
RequestFactory,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -48,6 +56,9 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
chat_template_config=self.chat_template_config,
|
||||
)
|
||||
|
||||
# Use thread pool executor to accelerate preprocessing.
|
||||
self._executor = self.renderer._executor
|
||||
|
||||
def encode(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType] | DataPrompt,
|
||||
@@ -93,43 +104,30 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
|
||||
io_processor = self.pooling_io_processors[pooling_task]
|
||||
|
||||
if pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
ctx: ALLOfflineInputsContext
|
||||
if isinstance(prompts, dict) and "data" in prompts:
|
||||
ctx = OfflinePluginInputsContext(
|
||||
pooling_task=pooling_task,
|
||||
prompts=prompts, # type: ignore[arg-type]
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
pooling_params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
priorities=None,
|
||||
)
|
||||
else:
|
||||
ctx = OfflineEncodeInputsContext(
|
||||
pooling_task=pooling_task,
|
||||
prompts=prompts,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
pooling_params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
priorities=None,
|
||||
)
|
||||
|
||||
ctx = OfflineInputsContext(
|
||||
prompts=prompts,
|
||||
pooling_params=pooling_params,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
|
||||
outputs = self._run_tiling_engine(
|
||||
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
|
||||
)
|
||||
|
||||
engine_inputs = io_processor.pre_process_offline(ctx)
|
||||
n_inputs = len(engine_inputs)
|
||||
assert ctx.pooling_params is not None
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, n_inputs)
|
||||
|
||||
for param in params_seq:
|
||||
if param.task is None:
|
||||
param.task = pooling_task
|
||||
elif pooling_task == "plugin":
|
||||
# `plugin` task uses io_processor.parse_request to verify inputs.
|
||||
# We actually allow plugin to overwrite pooling_task.
|
||||
pass
|
||||
elif param.task != pooling_task:
|
||||
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
|
||||
raise ValueError(msg)
|
||||
|
||||
seq_lora_requests = self._lora_request_to_seq(lora_request, n_inputs)
|
||||
seq_priority = self._priority_to_seq(None, n_inputs)
|
||||
|
||||
self._render_and_add_requests(
|
||||
prompts=engine_inputs,
|
||||
params=params_seq,
|
||||
lora_requests=seq_lora_requests,
|
||||
priorities=seq_priority,
|
||||
)
|
||||
|
||||
outputs = self._run_engine(use_tqdm=use_tqdm, output_type=PoolingRequestOutput)
|
||||
outputs = io_processor.post_process_offline(
|
||||
ctx=OfflineOutputsContext(outputs=outputs)
|
||||
)
|
||||
@@ -357,46 +355,121 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
io_processor = self.pooling_io_processors[score_type]
|
||||
assert isinstance(io_processor, ScoringIOProcessor)
|
||||
|
||||
pooling_task = io_processor.pooling_task
|
||||
scoring_data = io_processor.valid_inputs(data_1, data_2)
|
||||
n_queries = len(scoring_data.data_1)
|
||||
|
||||
if pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
|
||||
ctx = OfflineInputsContext(
|
||||
prompts=scoring_data,
|
||||
assert isinstance(pooling_params, PoolingParams)
|
||||
pooling_task = io_processor.pooling_task
|
||||
if pooling_params.task is None:
|
||||
pooling_params.task = pooling_task
|
||||
elif pooling_params.task != pooling_task:
|
||||
msg = (
|
||||
f"You cannot overwrite {pooling_params.task=!r} with {pooling_task=!r}!"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
ctx = OfflineScoringInputsContext(
|
||||
pooling_task=pooling_task,
|
||||
scoring_data=scoring_data,
|
||||
pooling_params=pooling_params,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
lora_request=lora_request,
|
||||
chat_template=chat_template,
|
||||
n_queries=n_queries,
|
||||
priorities=None,
|
||||
)
|
||||
|
||||
engine_inputs = io_processor.pre_process_offline(ctx)
|
||||
n_inputs = len(engine_inputs)
|
||||
|
||||
seq_lora_requests = self._lora_request_to_seq(lora_request, n_inputs)
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, n_inputs)
|
||||
|
||||
for param in params_seq:
|
||||
if param.task is None:
|
||||
param.task = pooling_task
|
||||
elif param.task != pooling_task:
|
||||
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
|
||||
raise ValueError(msg)
|
||||
|
||||
seq_priority = self._priority_to_seq(None, n_inputs)
|
||||
|
||||
self._render_and_add_requests(
|
||||
prompts=engine_inputs,
|
||||
params=params_seq,
|
||||
lora_requests=seq_lora_requests,
|
||||
priorities=seq_priority,
|
||||
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
|
||||
outputs = self._run_tiling_engine(
|
||||
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
|
||||
)
|
||||
|
||||
outputs = self._run_engine(use_tqdm=use_tqdm, output_type=PoolingRequestOutput)
|
||||
outputs = io_processor.post_process_offline(
|
||||
ctx=OfflineOutputsContext(outputs=outputs, n_queries=n_queries),
|
||||
)
|
||||
|
||||
return [ScoringRequestOutput.from_base(item) for item in outputs]
|
||||
|
||||
def _run_tiling_engine(
|
||||
self,
|
||||
io_processor: PoolingIOProcessor,
|
||||
request_factory: RequestFactory,
|
||||
num_requests: int,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
):
|
||||
# Keeping max_num_seqs * 2 requests in the core can already saturate the core.
|
||||
# Therefore, keep most requests waiting outside the core.
|
||||
max_requests_in_core = (
|
||||
self.llm_engine.vllm_config.scheduler_config.max_num_seqs * 2
|
||||
)
|
||||
num_requests_in_core = 0
|
||||
num_waited_requests = num_requests
|
||||
|
||||
if use_tqdm:
|
||||
tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
|
||||
pbar = tqdm_func(
|
||||
total=num_requests,
|
||||
desc="Processed prompts",
|
||||
dynamic_ncols=True,
|
||||
postfix=f"est. speed input: {0:.2f} toks/s, output: {0:.2f} toks/s",
|
||||
)
|
||||
|
||||
outputs: list[PoolingRequestOutput] = []
|
||||
added_request_ids: set[str] = set()
|
||||
|
||||
it = self._executor.map(io_processor.render, request_factory())
|
||||
|
||||
try:
|
||||
while num_waited_requests or self.llm_engine.has_unfinished_requests():
|
||||
requests = []
|
||||
for _ in range(max_requests_in_core - num_requests_in_core):
|
||||
if num_waited_requests == 0:
|
||||
break
|
||||
try:
|
||||
request = next(it)
|
||||
requests.append(request)
|
||||
except StopIteration:
|
||||
num_waited_requests = 0
|
||||
break
|
||||
|
||||
num_waited_requests -= 1
|
||||
num_requests_in_core += 1
|
||||
|
||||
if requests:
|
||||
request_ids = self._render_and_add_requests(
|
||||
prompts=[x["prompts"] for x in requests],
|
||||
params=[x["params"] for x in requests],
|
||||
lora_requests=[x["lora_requests"] for x in requests],
|
||||
priorities=[x["priorities"] for x in requests],
|
||||
)
|
||||
|
||||
for request_id in request_ids:
|
||||
# undo assign_request_id
|
||||
request_id = request_id.split("-", 1)[0]
|
||||
added_request_ids.add(request_id)
|
||||
|
||||
step_outputs = self.llm_engine.step()
|
||||
for output in step_outputs:
|
||||
assert isinstance(output, PoolingRequestOutput)
|
||||
assert output.finished
|
||||
outputs.append(output)
|
||||
added_request_ids.discard(output.request_id)
|
||||
num_requests_in_core -= 1
|
||||
|
||||
if use_tqdm:
|
||||
pbar.update(1)
|
||||
if pbar.n == num_requests:
|
||||
pbar.refresh()
|
||||
|
||||
except Exception:
|
||||
if added_request_ids:
|
||||
self.llm_engine.abort_request(list(added_request_ids))
|
||||
raise
|
||||
|
||||
finally:
|
||||
if use_tqdm:
|
||||
pbar.close()
|
||||
|
||||
return sorted(outputs, key=lambda x: int(x.request_id))
|
||||
|
||||
@@ -4,13 +4,19 @@ from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.logger import init_logger
|
||||
from vllm.plugins.io_processors import get_io_processor
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..typing import OfflineInputsContext, OfflineOutputsContext, PoolingServeContext
|
||||
from ..typing import (
|
||||
ALLOfflineInputsContext,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
OfflinePluginInputsContext,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
)
|
||||
from .protocol import IOProcessorRequest, IOProcessorResponse
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -107,9 +113,11 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflinePluginInputsContext)
|
||||
assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts
|
||||
assert ctx.pooling_params is not None
|
||||
|
||||
# Validate the request data is valid for the loaded plugin
|
||||
prompt_data = ctx.prompts.get("data")
|
||||
@@ -126,20 +134,35 @@ class PluginWithIOProcessorPlugins(PoolingIOProcessor):
|
||||
prompts = self.io_processor.pre_process(prompt=validated_prompt)
|
||||
prompts_seq = prompt_to_seq(prompts)
|
||||
|
||||
num_requests = len(prompts_seq)
|
||||
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams]
|
||||
if ctx.pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
else:
|
||||
pooling_params = ctx.pooling_params
|
||||
|
||||
params_seq: list[PoolingParams] = [
|
||||
self.io_processor.merge_pooling_params(param)
|
||||
for param in self._params_to_seq(
|
||||
ctx.pooling_params,
|
||||
len(prompts_seq),
|
||||
pooling_params,
|
||||
num_requests,
|
||||
)
|
||||
]
|
||||
for p in params_seq:
|
||||
if p.task is None:
|
||||
p.task = "plugin"
|
||||
|
||||
ctx.pooling_params = params_seq
|
||||
ctx.prompts = prompts_seq
|
||||
return super().pre_process_offline(ctx)
|
||||
return super().get_request_factory_offline(
|
||||
OfflineEncodeInputsContext(
|
||||
prompts=prompts_seq,
|
||||
pooling_params=params_seq,
|
||||
pooling_task="plugin",
|
||||
tokenization_kwargs=ctx.tokenization_kwargs,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
)
|
||||
)
|
||||
|
||||
def post_process_offline(
|
||||
self,
|
||||
|
||||
@@ -2,24 +2,36 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, TypeAlias
|
||||
from typing import Any, TypeAlias, cast
|
||||
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
|
||||
from vllm import PoolingParams, PoolingRequestOutput, PromptType, TokensPrompt
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.renderers.hf import safe_apply_chat_template
|
||||
from vllm.renderers.inputs.preprocess import extract_target_prompt
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_target_prompt,
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ...chat_utils import ChatTemplateResolutionError
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..typing import (
|
||||
OfflineInputsContext,
|
||||
ALLOfflineInputsContext,
|
||||
EncodeChatRenderParams,
|
||||
EncodeCMPLRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
OfflineScoringInputsContext,
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
RequestGenerator,
|
||||
ScoringRenderParams,
|
||||
)
|
||||
from .protocol import RerankRequest, ScoreRequest, ScoringRequest
|
||||
from .typing import ScoreData, ScoreInput, ScoringData
|
||||
@@ -213,25 +225,37 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
|
||||
assert isinstance(ctx.prompts, ScoringData)
|
||||
assert not isinstance(ctx.pooling_params, Sequence)
|
||||
|
||||
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(ctx.tokenization_kwargs or {})
|
||||
)
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineScoringInputsContext)
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
pooling_params=ctx.pooling_params
|
||||
)
|
||||
|
||||
scoring_data = ctx.prompts
|
||||
scoring_data = ctx.scoring_data
|
||||
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
|
||||
scoring_data = self._truncate_scoring_data(
|
||||
scoring_data, max_tokens_per_query, max_tokens_per_doc
|
||||
)
|
||||
|
||||
return self._pre_process(scoring_data, tok_params)
|
||||
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
|
||||
data_2 = score_data_to_prompts(
|
||||
scoring_data.data_2, "document", self.model_config
|
||||
)
|
||||
prompts = data_1 + data_2
|
||||
|
||||
return super().get_request_factory_offline(
|
||||
OfflineEncodeInputsContext(
|
||||
pooling_task=self.pooling_task,
|
||||
prompts=prompts,
|
||||
tokenization_kwargs=ctx.tokenization_kwargs,
|
||||
pooling_params=ctx.pooling_params,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
)
|
||||
)
|
||||
|
||||
def post_process_offline(
|
||||
self,
|
||||
@@ -258,6 +282,26 @@ class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
prompts=data_1 + data_2, tok_params=tok_params, prompt_extras=prompt_extras
|
||||
)
|
||||
|
||||
def _preprocess_cmpl_offline(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
tok_params: TokenizeParams,
|
||||
prompt_extras: dict[str, Any] | None = None,
|
||||
) -> Sequence[EngineInput]:
|
||||
prompts = prompt_to_seq(prompts)
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(self.model_config, prompt)
|
||||
)
|
||||
for prompt in prompts
|
||||
]
|
||||
|
||||
return self.renderer.render_cmpl(
|
||||
parsed_prompts, tok_params, prompt_extras=prompt_extras
|
||||
)
|
||||
|
||||
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
|
||||
emb_data_1 = outputs[:n_queries]
|
||||
emb_data_2 = outputs[n_queries:]
|
||||
@@ -429,34 +473,96 @@ class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def pre_process_offline(self, ctx: OfflineInputsContext) -> Sequence[EngineInput]:
|
||||
assert isinstance(ctx.prompts, ScoringData)
|
||||
assert not isinstance(ctx.pooling_params, Sequence)
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineScoringInputsContext)
|
||||
|
||||
data_1 = ctx.scoring_data.data_1
|
||||
data_2 = ctx.scoring_data.data_2
|
||||
num_requests = len(data_2)
|
||||
|
||||
if len(data_1) == 1:
|
||||
data_1 = data_1 * num_requests
|
||||
|
||||
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(ctx.tokenization_kwargs or {})
|
||||
)
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
pooling_params=ctx.pooling_params
|
||||
)
|
||||
prompt_extras = ctx.pooling_params.extra_kwargs
|
||||
|
||||
prompt_extras = ctx.pooling_params.extra_kwargs if ctx.pooling_params else None
|
||||
engine_inputs, pooling_params_list = self._pre_process(
|
||||
ctx.prompts,
|
||||
tok_params,
|
||||
ctx.pooling_params,
|
||||
ctx.chat_template,
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
prompt_extras=prompt_extras,
|
||||
)
|
||||
ctx.pooling_params = pooling_params_list
|
||||
return engine_inputs
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
def request_factory() -> RequestGenerator:
|
||||
for i in range(num_requests):
|
||||
yield ScoringRenderParams(
|
||||
data_1=data_1[i],
|
||||
data_2=data_2[i],
|
||||
chat_template=ctx.chat_template,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=ctx.pooling_params,
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
|
||||
return request_factory, num_requests
|
||||
|
||||
#######################################
|
||||
# helpers
|
||||
|
||||
def render(
|
||||
self,
|
||||
render_params: EncodeCMPLRenderParams
|
||||
| EncodeChatRenderParams
|
||||
| ScoringRenderParams,
|
||||
) -> PoolingEngineInput:
|
||||
if "data_1" not in render_params:
|
||||
raise ValueError(
|
||||
f"Unsupported render_params type {render_params.__class__.__name__}"
|
||||
)
|
||||
render_params = cast(ScoringRenderParams, render_params)
|
||||
|
||||
arrival_time = time.time()
|
||||
|
||||
tok_params = render_params["tok_params"]
|
||||
params = render_params["params"]
|
||||
prompt_extras = render_params["prompt_extras"]
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
pooling_params=params
|
||||
)
|
||||
|
||||
_, engine_prompt = self.get_score_prompt(
|
||||
data_1=render_params["data_1"],
|
||||
data_2=render_params["data_2"],
|
||||
encode_kwargs=tok_params.get_encode_kwargs(),
|
||||
chat_template=render_params["chat_template"],
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
|
||||
if prompt_extras
|
||||
else None,
|
||||
)
|
||||
|
||||
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
|
||||
|
||||
if token_type_ids := engine_prompt.pop("token_type_ids", None):
|
||||
params = params.clone()
|
||||
compressed = compress_token_type_ids(token_type_ids)
|
||||
params.extra_kwargs = {"compressed_token_type_ids": compressed}
|
||||
|
||||
engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time)
|
||||
|
||||
return PoolingEngineInput(
|
||||
prompts=engine_input,
|
||||
params=params,
|
||||
lora_requests=render_params["lora_requests"],
|
||||
priorities=render_params["priorities"],
|
||||
)
|
||||
|
||||
def _pre_process(
|
||||
self,
|
||||
scoring_data: ScoringData,
|
||||
@@ -731,6 +837,49 @@ class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorM
|
||||
name = "jina-reranking-scoring"
|
||||
pooling_task: PoolingTask = "token_embed"
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: ALLOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineScoringInputsContext)
|
||||
|
||||
scoring_data = ctx.scoring_data
|
||||
prompt_extras = ctx.pooling_params.extra_kwargs
|
||||
|
||||
queries = self.ensure_str(scoring_data.data_1)
|
||||
docs = self.ensure_str(scoring_data.data_2)
|
||||
chat_template_kwargs = (
|
||||
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
|
||||
)
|
||||
instruction = (
|
||||
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
|
||||
)
|
||||
|
||||
if len(queries) == 1:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=queries[0], docs=docs, instruction=instruction
|
||||
)
|
||||
]
|
||||
else:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=q, docs=[d], instruction=instruction
|
||||
)
|
||||
for q, d in zip(queries, docs)
|
||||
]
|
||||
|
||||
return PoolingIOProcessor.get_request_factory_offline(
|
||||
self,
|
||||
OfflineEncodeInputsContext(
|
||||
pooling_task=self.pooling_task,
|
||||
prompts=prompts,
|
||||
tokenization_kwargs=ctx.tokenization_kwargs,
|
||||
pooling_params=ctx.pooling_params,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
),
|
||||
)
|
||||
|
||||
def _pre_process(
|
||||
self,
|
||||
scoring_data: ScoringData,
|
||||
|
||||
@@ -1,17 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Sequence
|
||||
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Generic, TypeAlias, TypeVar
|
||||
from typing import Any, Generic, TypeAlias, TypedDict, TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import ConfigDict
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput, PromptType
|
||||
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.inputs import DataPrompt, EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers import ChatParams, TokenizeParams
|
||||
from vllm.renderers.inputs import DictPrompt
|
||||
|
||||
from ...tasks import PoolingTask
|
||||
from .classify.protocol import (
|
||||
ClassificationChatRequest,
|
||||
ClassificationCompletionRequest,
|
||||
@@ -34,7 +38,7 @@ from .pooling.protocol import (
|
||||
PoolingResponse,
|
||||
)
|
||||
from .scoring.protocol import ScoringRequest, ScoringResponse
|
||||
from .scoring.typing import ScoringData
|
||||
from .scoring.typing import ScoreData, ScoringData
|
||||
|
||||
PoolingCompletionLikeRequest: TypeAlias = (
|
||||
EmbeddingCompletionRequest
|
||||
@@ -111,13 +115,36 @@ class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
|
||||
@dataclass
|
||||
class OfflineInputsContext:
|
||||
prompts: PromptType | Sequence[PromptType] | DataPrompt | ScoringData
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams]
|
||||
tokenization_kwargs: dict[str, Any] | None = None
|
||||
chat_template: str | None = None
|
||||
pooling_task: PoolingTask
|
||||
tokenization_kwargs: dict[str, Any] | None
|
||||
lora_request: Sequence[LoRARequest | None] | None
|
||||
priorities: Sequence[int] | None
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
n_queries: int | None = None
|
||||
|
||||
@dataclass
|
||||
class OfflineEncodeInputsContext(OfflineInputsContext):
|
||||
prompts: PromptType | Sequence[PromptType]
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflineScoringInputsContext(OfflineInputsContext):
|
||||
scoring_data: ScoringData
|
||||
chat_template: str | None
|
||||
pooling_params: PoolingParams
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflinePluginInputsContext(OfflineInputsContext):
|
||||
prompts: DataPrompt
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None
|
||||
|
||||
|
||||
ALLOfflineInputsContext: TypeAlias = (
|
||||
OfflineEncodeInputsContext
|
||||
| OfflineScoringInputsContext
|
||||
| OfflinePluginInputsContext
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -126,3 +153,41 @@ class OfflineOutputsContext:
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
n_queries: int | None = None
|
||||
|
||||
|
||||
class RenderParams(TypedDict):
|
||||
tok_params: TokenizeParams
|
||||
prompt_extras: dict[str, Any] | None
|
||||
skip_mm_cache: bool
|
||||
|
||||
params: PoolingParams
|
||||
lora_requests: LoRARequest | None
|
||||
priorities: int
|
||||
|
||||
|
||||
class EncodeCMPLRenderParams(RenderParams):
|
||||
prompts: DictPrompt
|
||||
|
||||
|
||||
class EncodeChatRenderParams(RenderParams):
|
||||
conversations: list["ChatCompletionMessageParam"]
|
||||
chat_params: ChatParams
|
||||
|
||||
|
||||
class ScoringRenderParams(RenderParams):
|
||||
data_1: ScoreData
|
||||
data_2: ScoreData
|
||||
chat_template: str | None
|
||||
|
||||
|
||||
class PoolingEngineInput(TypedDict):
|
||||
prompts: EngineInput
|
||||
params: PoolingParams
|
||||
lora_requests: LoRARequest | None
|
||||
priorities: int
|
||||
|
||||
|
||||
RequestGenerator: TypeAlias = Generator[
|
||||
EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams
|
||||
]
|
||||
RequestFactory: TypeAlias = Callable[[], RequestGenerator]
|
||||
|
||||
Reference in New Issue
Block a user