[Mini-vLLM] SM86-only cut: models, backends, Rust, tests

- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries
- Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels
- Remove inc/quark quant loaders; keep compressed-tensors and Marlin
- Drop the Rust frontend, pooling entrypoints, and idle test suites
- Rebuild kernels for sm_80;sm_86 and strip the extensions
- Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-18 19:19:56 +07:00
co-authored by Hinata
parent c5ab42bc0f
commit e61dcb9143
2728 changed files with 159 additions and 950401 deletions
-27
View File
@@ -1,27 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkSubcommandBase(CLISubcommand):
"""The base class of subcommands for `vllm bench`."""
help: str
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
"""Add the CLI arguments to the parser."""
raise NotImplementedError
@staticmethod
def cmd(args: argparse.Namespace) -> None:
"""Run the benchmark.
Args:
args: The arguments to the command.
"""
raise NotImplementedError
-22
View File
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.benchmarks.forward.main import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkForwardSubcommand(BenchmarkSubcommandBase):
"""The `forward` subcommand for `vllm bench`."""
name = "forward"
help = "Benchmark the model forward of exact batches as CUDA graphs."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
-22
View File
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.benchmarks.latency import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
"""The `latency` subcommand for `vllm bench`."""
name = "latency"
help = "Benchmark the latency of a single batch of requests."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
-100
View File
@@ -1,100 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import os
import sys
import typing
from vllm import envs
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
if typing.TYPE_CHECKING:
from vllm.utils.argparse_utils import FlexibleArgumentParser
else:
FlexibleArgumentParser = argparse.ArgumentParser
logger = init_logger(__name__)
def maybe_exec_rust_bench() -> None:
if sys.argv[1:3] != ["bench", "serve"] or not envs.VLLM_USE_RUST_BENCH:
return
rust_cli = envs.VLLM_RUST_FRONTEND_PATH
if rust_cli is None:
raise RuntimeError(
"VLLM_USE_RUST_BENCH=1 requires VLLM_RUST_FRONTEND_PATH "
"to resolve to the vllm-rs binary."
)
logger.info("Delegating `vllm bench serve` to Rust binary at %s.", rust_cli)
os.execv(rust_cli, [rust_cli, "bench", "serve", *sys.argv[3:]])
def _import_bench_subcommand_modules() -> None:
# Imported lazily so `BenchmarkSubcommandBase` subclasses register only
# when `vllm bench` is actually invoked.
import vllm.entrypoints.cli.benchmark.forward # noqa: F401
import vllm.entrypoints.cli.benchmark.latency # noqa: F401
import vllm.entrypoints.cli.benchmark.mm_processor # noqa: F401
import vllm.entrypoints.cli.benchmark.serve # noqa: F401
import vllm.entrypoints.cli.benchmark.startup # noqa: F401
import vllm.entrypoints.cli.benchmark.sweep # noqa: F401
import vllm.entrypoints.cli.benchmark.throughput # noqa: F401
class BenchmarkSubcommand(CLISubcommand):
"""The `bench` subcommand for the vLLM CLI."""
name = "bench"
help = "vLLM bench subcommand."
@staticmethod
def cmd(args: argparse.Namespace) -> None:
args.dispatch_function(args)
def validate(self, args: argparse.Namespace) -> None:
pass
def subparser_init(
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
bench_parser = subparsers.add_parser(
self.name,
help=self.help,
description=self.help,
usage=f"vllm {self.name} <bench_type> [options]",
)
bench_subparsers = bench_parser.add_subparsers(required=True, dest="bench_type")
# Only build the nested bench subparsers when the user is actually
# invoking `bench`; otherwise we'd drag in imports
# unnecessarily on every `vllm --help` and `vllm serve`.
# Scan for the first positional arg so global flags (e.g. `-v`)
# before the subcommand don't break detection.
first_positional = next(
(arg for arg in sys.argv[1:] if not arg.startswith("-")), None
)
if first_positional == self.name:
_import_bench_subcommand_modules()
for cmd_cls in BenchmarkSubcommandBase.__subclasses__():
cmd_subparser = bench_subparsers.add_parser(
cmd_cls.name,
help=cmd_cls.help,
description=cmd_cls.help,
usage=f"vllm {self.name} {cmd_cls.name} [options]",
)
cmd_subparser.set_defaults(dispatch_function=cmd_cls.cmd)
cmd_cls.add_cli_args(cmd_subparser)
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
subcmd=f"{self.name} {cmd_cls.name}"
)
return bench_parser
def cmd_init() -> list[CLISubcommand]:
return [BenchmarkSubcommand()]
@@ -1,69 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import inspect
from vllm.benchmarks.mm_processor import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
r"""`vllm bench mm-processor` profiles the multimodal input processor pipeline of
vision-language models. It measures per-stage latency from the HuggingFace
processor through to the encoder forward pass, helping you identify
preprocessing bottlenecks and understand how different image resolutions or
item counts affect end-to-end request time.
The benchmark supports two data sources: synthetic random multimodal inputs
(`random-mm`) and HuggingFace datasets (`hf`). Warmup requests are run before
measurement to ensure stable results.
## Quick Start
```bash
vllm bench mm-processor \
--model Qwen/Qwen2-VL-7B-Instruct \
--dataset-name random-mm \
--num-prompts 50 \
--random-input-len 300 \
--random-output-len 40 \
--random-mm-base-items-per-request 2 \
--random-mm-limit-mm-per-prompt '{"image": 3, "video": 0}' \
--random-mm-bucket-config '{(256, 256, 1): 0.7, (720, 1280, 1): 0.3}'
```
## Measured Stages
| Stage | Description |
| ----- | ----------- |
| `get_mm_hashes_secs` | Time spent hashing multimodal inputs |
| `get_cache_missing_items_secs` | Time spent looking up the processor cache |
| `apply_hf_processor_secs` | Time spent in the HuggingFace processor |
| `merge_mm_kwargs_secs` | Time spent merging multimodal kwargs |
| `apply_prompt_updates_secs` | Time spent updating prompt tokens |
| `preprocessor_total_secs` | Total preprocessing time |
| `encoder_forward_secs` | Time spent in the encoder model forward pass |
| `num_encoder_calls` | Number of encoder invocations per request |
The benchmark also reports end-to-end latency (TTFT + decode time) per
request. Use `--metric-percentiles` to select which percentiles to report
(default: p99) and `--output-json` to save results.
For more examples (HF datasets, warmup, JSON output), see the
[multimodal processor benchmark guide](https://docs.vllm.ai/en/latest/benchmarking/cli/#multimodal-processor-benchmark).
"""
name = "mm-processor"
help = "Benchmark multimodal processor latency across different configurations."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
# The class docstring is the page overview / `--help` description.
if cls.__doc__:
parser.description = inspect.cleandoc(cls.__doc__)
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
-22
View File
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.benchmarks.serve import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
"""The `serve` subcommand for `vllm bench`."""
name = "serve"
help = "Benchmark the online serving throughput."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
-22
View File
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.benchmarks.startup import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkStartupSubcommand(BenchmarkSubcommandBase):
"""The `startup` subcommand for `vllm bench`."""
name = "startup"
help = "Benchmark the startup time of vLLM models."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
-22
View File
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.benchmarks.sweep.cli import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
"""The `sweep` subcommand for `vllm bench`."""
name = "sweep"
help = "Benchmark for a parameter sweep."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
@@ -1,22 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
from vllm.benchmarks.throughput import add_cli_args, main
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
from vllm.utils.argparse_utils import FlexibleArgumentParser
class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
"""The `throughput` subcommand for `vllm bench`."""
name = "throughput"
help = "Benchmark offline inference throughput."
@classmethod
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
add_cli_args(parser)
@staticmethod
def cmd(args: argparse.Namespace) -> None:
main(args)
-20
View File
@@ -40,7 +40,6 @@ def main():
if maybe_run_omni():
return
import vllm.entrypoints.cli.benchmark.main
import vllm.entrypoints.cli.collect_env
import vllm.entrypoints.cli.download_kernels
import vllm.entrypoints.cli.launch
@@ -59,7 +58,6 @@ def main():
vllm.entrypoints.cli.openai,
vllm.entrypoints.cli.serve,
vllm.entrypoints.cli.launch,
vllm.entrypoints.cli.benchmark.main,
vllm.entrypoints.cli.collect_env,
vllm.entrypoints.cli.download_kernels,
vllm.entrypoints.cli.preload,
@@ -70,24 +68,6 @@ def main():
if sys.argv[1:2] != ["snapshot"]:
cli_env_setup()
vllm.entrypoints.cli.benchmark.main.maybe_exec_rust_bench()
# For 'vllm bench *': use CPU instead of UnspecifiedPlatform by default
if len(sys.argv) > 1 and sys.argv[1] == "bench":
logger.debug(
"Bench command detected, must ensure current platform is not "
"UnspecifiedPlatform to avoid device type inference error"
)
from vllm import platforms
if platforms.current_platform.is_unspecified():
from vllm.platforms.cpu import CpuPlatform
platforms.current_platform = CpuPlatform()
logger.info(
"Unspecified platform detected, switching to CPU Platform instead."
)
parser = FlexibleArgumentParser(
description="vLLM CLI",
epilog=VLLM_SUBCMD_PARSER_EPILOG.format(subcmd="[subcommand]"),
+13 -65
View File
@@ -17,7 +17,6 @@ from vllm.entrypoints.launchers.cli_args import (
validate_parsed_serve_args,
)
from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor
from vllm.entrypoints.launchers.launcher import create_server_socket
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
from vllm.reasoning import ReasoningParserManager
@@ -30,7 +29,6 @@ from vllm.v1.executor.multiproc_executor import MultiprocExecutor
from vllm.v1.metrics.prometheus import setup_multiprocess_prometheus
from vllm.v1.utils import (
APIServerProcessManager,
RustFrontendProcessManager,
wait_for_completion_or_failure,
)
@@ -62,10 +60,6 @@ class ServeSubcommand(CLISubcommand):
uvloop.run(serve_grpc(args))
return
rust_frontend_path = (
envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None
)
if args.headless:
if args.api_server_count is not None and args.api_server_count > 0:
raise ValueError(
@@ -106,12 +100,11 @@ class ServeSubcommand(CLISubcommand):
# Default api_server_count if not explicitly set.
# - Multi-port: 1 (supervisor spawns one server per local DP rank)
# - Rust frontend: 1 (not applicable as it's multithreaded)
# - External LB: 1 (external LB handles distribution)
# - Hybrid LB: Use local DP size (internal LB for local ranks only)
# - Internal LB: Use full DP size
if args.api_server_count is None:
if is_multi_port or is_external_lb or rust_frontend_path:
if is_multi_port or is_external_lb:
args.api_server_count = 1
elif is_hybrid_lb:
args.api_server_count = args.data_parallel_size_local or 1
@@ -128,12 +121,6 @@ class ServeSubcommand(CLISubcommand):
"Defaulting api_server_count to data_parallel_size (%d).",
args.api_server_count,
)
elif rust_frontend_path and args.api_server_count > 1:
logger.warning(
"Ignoring --api-server-count=%d when using rust front-end process",
args.api_server_count,
)
args.api_server_count = 1
# Elastic EP currently only supports running with at most one API server.
if getattr(args, "enable_elastic_ep", False) and args.api_server_count > 1:
@@ -148,7 +135,7 @@ class ServeSubcommand(CLISubcommand):
run_dp_supervisor(args)
elif args.api_server_count < 1:
run_headless(args)
elif args.api_server_count > 1 or rust_frontend_path:
elif args.api_server_count > 1:
run_multi_api_server(args)
else:
# Single API server (this process).
@@ -268,17 +255,9 @@ def run_headless(args: argparse.Namespace):
def run_multi_api_server(args: argparse.Namespace):
assert not args.headless
rust_frontend_path = (
envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None
)
num_api_servers: int = args.api_server_count
assert num_api_servers > 0
if rust_frontend_path and num_api_servers > 1:
raise ValueError(
"VLLM_RUST_FRONTEND_PATH does not support api_server_count > 1"
)
if num_api_servers > 1:
setup_multiprocess_prometheus()
@@ -296,13 +275,6 @@ def run_multi_api_server(args: argparse.Namespace):
signal.signal(signal.SIGINT, signal_handler)
listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1)
# `--grpc-port` is only accepted for the Rust frontend, which inherits this
# listener like the HTTP one. gRPC follows the HTTP TCP host, or IPv4
# loopback when HTTP uses a Unix socket.
grpc_sock = None
if args.grpc_port is not None:
grpc_host = "127.0.0.1" if args.uds else (args.host or "")
grpc_sock = create_server_socket((grpc_host, args.grpc_port), reuse_port=False)
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
propagate_flash_late_interaction(args, engine_args)
@@ -324,9 +296,7 @@ def run_multi_api_server(args: argparse.Namespace):
dp_rank = parallel_config.data_parallel_rank
assert parallel_config.local_engines_only or dp_rank == 0
api_server_manager: APIServerProcessManager | RustFrontendProcessManager | None = (
None
)
api_server_manager: APIServerProcessManager | None = None
from vllm.v1.engine.utils import bind_engine_zmq_listeners
@@ -343,38 +313,16 @@ def run_multi_api_server(args: argparse.Namespace):
coordinator.get_stats_publish_address() if coordinator else None
)
if rust_frontend_path:
if parallel_config.local_engines_only:
expected_engine_start_index = parallel_config.data_parallel_rank
expected_engine_count = parallel_config.data_parallel_size_local
else:
expected_engine_start_index = 0
expected_engine_count = parallel_config.data_parallel_size
# Start rust front-end process.
api_server_manager = RustFrontendProcessManager(
binary_path=rust_frontend_path,
sock=sock,
grpc_sock=grpc_sock,
args=args,
input_listener=zmq_listeners.inputs[0],
output_listener=zmq_listeners.outputs[0],
engine_start_index=expected_engine_start_index,
engine_count=expected_engine_count,
data_parallel_size=parallel_config.data_parallel_size,
stats_update_address=stats_update_address,
)
else:
# Start API server(s).
api_server_manager = APIServerProcessManager(
listen_address=listen_address,
sock=sock,
args=args,
num_servers=num_api_servers,
input_listeners=zmq_listeners.inputs,
output_listeners=zmq_listeners.outputs,
stats_update_address=stats_update_address,
tensor_queue=engine_launch.tensor_queue,
)
api_server_manager = APIServerProcessManager(
listen_address=listen_address,
sock=sock,
args=args,
num_servers=num_api_servers,
input_listeners=zmq_listeners.inputs,
output_listeners=zmq_listeners.outputs,
stats_update_address=stats_update_address,
tensor_queue=engine_launch.tensor_queue,
)
# Set frontend processes to watch during engine startup.
# If any of these processes exit before the engines are up, the engine startup
@@ -18,7 +18,7 @@ from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
from vllm.renderers.chat_utils import load_chat_template
from vllm.renderers.online_derenderer import OnlineDerenderer
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, POOLING_TASKS, SupportedTask
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
async def init_app_state(
@@ -152,11 +152,6 @@ async def init_app_state(
engine_client, state, args, request_logger, supported_tasks
)
if any(task in POOLING_TASKS for task in supported_tasks):
from vllm.entrypoints.pooling.factories import init_pooling_state
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
from vllm.entrypoints.generate.structured_decisions.api_router import (
init_structured_decisions_state,
)
@@ -6,7 +6,7 @@ from fastapi import FastAPI
from vllm import envs
from vllm.config import ModelConfig
from vllm.tasks import POOLING_TASKS, SupportedTask
from vllm.tasks import SupportedTask
def register_api_routers(
@@ -77,11 +77,6 @@ def register_api_routers(
register_speech_to_text_api_routers(app, supported_tasks)
if any(task in POOLING_TASKS for task in supported_tasks):
from vllm.entrypoints.pooling.factories import register_pooling_api_routers
register_pooling_api_routers(app, supported_tasks, model_config)
if getattr(args, "enable_fault_tolerance", False):
from vllm.entrypoints.serve.fault_tolerance.api_router import (
register_fault_tolerance_api_router,
-39
View File
@@ -285,11 +285,6 @@ class FrontendArgs(BaseFrontendArgs):
"""Host name."""
port: int = 8000
"""Port number."""
grpc_port: int | None = None
"""Enable the Rust frontend's additional gRPC Inference and Control services
on this port. Requires `VLLM_USE_RUST_FRONTEND=1 vllm serve`; HTTP remains on
`--port`. Binds on `--host`, or 127.0.0.1 with `--uds`. Cannot be combined
with the Python gRPC server's `--grpc` flag."""
data_parallel_supervisor_port: int = 9256
"""HTTP port for aggregated health endpoints in multi-port external LB
mode."""
@@ -468,38 +463,6 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
return parser
def validate_grpc_port_arg(args: argparse.Namespace) -> None:
"""Validate the Rust frontend's optional gRPC listener."""
if getattr(args, "grpc_port", None) is None:
return
if args.grpc:
raise ValueError(
"--grpc and --grpc-port are mutually exclusive. Use --grpc "
"--port for the Python gRPC server, or VLLM_USE_RUST_FRONTEND=1 "
"with --grpc-port for the Rust frontend."
)
if not envs.VLLM_USE_RUST_FRONTEND or getattr(args, "subparser", None) != "serve":
raise ValueError("--grpc-port requires VLLM_USE_RUST_FRONTEND=1 vllm serve")
if args.headless or (
args.api_server_count is not None and args.api_server_count <= 0
):
raise ValueError(
"--grpc-port requires a Rust frontend; remove --headless "
"and non-positive --api-server-count"
)
if args.data_parallel_multi_port_external_lb:
raise ValueError(
"--grpc-port is incompatible with "
"--data-parallel-multi-port-external-lb: "
"its frontends would share the same gRPC port"
)
if not 0 <= args.grpc_port <= 65535:
raise ValueError("--grpc-port must be between 0 and 65535")
if args.grpc_port != 0 and args.grpc_port == args.port and not args.uds:
raise ValueError("--grpc-port must differ from --port when HTTP uses TCP")
def validate_parsed_serve_args(args: argparse.Namespace):
"""Quick checks for model serve args that raise prior to loading."""
# `vllm launch <component>` builds its parser with make_arg_parser too (see
@@ -518,8 +481,6 @@ def validate_parsed_serve_args(args: argparse.Namespace):
"the checkpoint's output format."
)
validate_grpc_port_arg(args)
# Enable auto tool needs a tool call parser to be valid
if args.enable_auto_tool_choice and not args.tool_call_parser:
raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser")
+1 -11
View File
@@ -22,7 +22,6 @@ import uvicorn
import uvloop
from fastapi import FastAPI, Response
import vllm.envs as envs
from vllm.logger import configure_logging_from_args, init_logger
from vllm.utils.system_utils import (
decorate_logs,
@@ -243,12 +242,6 @@ def _run_python_vllm_dp_server(child_args: argparse.Namespace) -> None:
uvloop.run(run_server(child_args))
def _run_rust_vllm_dp_server(child_args: argparse.Namespace) -> None:
from vllm.entrypoints.cli.serve import run_multi_api_server
run_multi_api_server(child_args)
def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
"""Entrypoint function for the vLLM DP Server."""
configure_logging_from_args(child_args)
@@ -260,10 +253,7 @@ def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
name = f"APIServer_DP{child_args.data_parallel_rank}"
set_process_title(name)
decorate_logs(name)
if envs.VLLM_USE_RUST_FRONTEND and envs.VLLM_RUST_FRONTEND_PATH:
_run_rust_vllm_dp_server(child_args)
else:
_run_python_vllm_dp_server(child_args)
_run_python_vllm_dp_server(child_args)
class DPSupervisor:
+3 -51
View File
@@ -35,16 +35,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
)
from vllm.entrypoints.pooling.embed.protocol import (
EmbeddingRequest,
EmbeddingResponse,
)
from vllm.entrypoints.pooling.scoring.protocol import (
RerankRequest,
RerankResponse,
ScoreRequest,
ScoreResponse,
)
from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.engine.protocol import (
ErrorInfo,
@@ -147,12 +137,7 @@ class BatchTranslationRequest(TranslationRequest):
BatchRequestInputBody: TypeAlias = (
ChatCompletionRequest
| EmbeddingRequest
| ScoreRequest
| RerankRequest
| BatchTranscriptionRequest
| BatchTranslationRequest
ChatCompletionRequest | BatchTranscriptionRequest | BatchTranslationRequest
)
@@ -184,12 +169,6 @@ class BatchRequestInput(OpenAIBaseModel):
url: str = info.data["url"]
if url == "/v1/chat/completions":
return ChatCompletionRequest.model_validate(value)
if url == "/v1/embeddings":
return TypeAdapter(EmbeddingRequest).validate_python(value)
if url.endswith("/score"):
return TypeAdapter(ScoreRequest).validate_python(value)
if url.endswith("/rerank"):
return RerankRequest.model_validate(value)
if url == "/v1/audio/transcriptions":
return BatchTranscriptionRequest.model_validate(value)
if url == "/v1/audio/translations":
@@ -199,9 +178,6 @@ class BatchRequestInput(OpenAIBaseModel):
AllResponse: TypeAlias = (
ChatCompletionResponse
| EmbeddingResponse
| ScoreResponse
| RerankResponse
| TranscriptionResponseVariant
| TranslationResponse
| TranslationResponseVerbose
@@ -738,8 +714,6 @@ async def build_endpoint_registry(
openai_serving_chat = getattr(state, "openai_serving_chat", None)
openai_serving_transcription = getattr(state, "openai_serving_transcription", None)
openai_serving_translation = getattr(state, "openai_serving_translation", None)
serving_embedding = getattr(state, "serving_embedding", None)
serving_scores = getattr(state, "serving_scores", None)
allowed_media_domains = getattr(args, "allowed_media_domains", None)
@@ -754,27 +728,6 @@ async def build_endpoint_registry(
),
"wrapper_fn": None,
},
"embeddings": {
"url_matcher": lambda url: url == "/v1/embeddings",
"handler_getter": lambda: (
serving_embedding if serving_embedding is not None else None
),
"wrapper_fn": None,
},
"score": {
"url_matcher": lambda url: url.endswith("/score"),
"handler_getter": lambda: (
serving_scores if serving_scores is not None else None
),
"wrapper_fn": None,
},
"rerank": {
"url_matcher": lambda url: url.endswith("/rerank"),
"handler_getter": lambda: (
serving_scores if serving_scores is not None else None
),
"wrapper_fn": None,
},
"transcriptions": {
"url_matcher": lambda url: url == "/v1/audio/transcriptions",
"handler_getter": lambda: (
@@ -859,9 +812,8 @@ async def run_batch(
make_async_error_request_output(
request,
error_msg=f"URL {request.url} was used. "
"Supported endpoints: /v1/chat/completions, /v1/embeddings,"
" /v1/audio/transcriptions, /v1/audio/translations, /score, "
" /rerank.",
"Supported endpoints: /v1/chat/completions,"
" /v1/audio/transcriptions, /v1/audio/translations.",
)
)
+1 -6
View File
@@ -30,7 +30,6 @@ from vllm.config.quantization import QuantizationConfigArgs
from vllm.engine.arg_utils import EngineArgs
from vllm.entrypoints.common.offline import _O, _R, OfflineInferenceMixin
from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
from vllm.entrypoints.pooling.offline import PoolingOfflineMixin
from vllm.entrypoints.rl.offline import RLOfflineMixin
from vllm.entrypoints.serve.utils.api_utils import log_non_default_args
from vllm.inputs import PromptType
@@ -60,9 +59,7 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
class LLM(
BeamSearchOfflineMixin, PoolingOfflineMixin, RLOfflineMixin, OfflineInferenceMixin
):
class LLM(BeamSearchOfflineMixin, RLOfflineMixin, OfflineInferenceMixin):
"""An LLM for generating texts from given prompts and sampling parameters.
This class includes a tokenizer, a language model (possibly distributed
@@ -373,8 +370,6 @@ class LLM(
self.model_config.renderer_num_workers,
)
PoolingOfflineMixin.__init__(self)
# Cache for __repr__ to avoid repeated collective_rpc calls
self._cached_repr: str | None = None
@@ -1,356 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from concurrent.futures import Executor
from typing import Any, Final, cast
from vllm import (
PoolingParams,
PoolingRequestOutput,
)
from vllm.config import VllmConfig
from vllm.exceptions import VLLMValidationError
from vllm.lora.request import LoRARequest
from vllm.renderers import BaseRenderer, merge_kwargs
from vllm.renderers.chat_utils import (
ChatTemplateConfig,
)
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from vllm.utils.async_utils import make_async
from vllm.utils.mistral import is_mistral_tokenizer
from ..typing import (
AnyOfflineInputsContext,
AnyRenderParam,
EncodeChatRenderParams,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
OfflineOutputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
ScoringRenderParams,
)
class PoolingIOProcessor:
"""Processor for handling preprocessing & postprocessing ops for pooling requests.
This class manages both online (serving) and offline (batch) processing of pooling
requests, handling chat and completion formats.
"""
name: str
def __init__(
self,
vllm_config: VllmConfig,
renderer: BaseRenderer,
chat_template_config: ChatTemplateConfig,
):
self.vllm_config = vllm_config
self.model_config = vllm_config.model_config
self.renderer = renderer
self.chat_template = chat_template_config.chat_template
self.chat_template_content_format: Final = (
chat_template_config.chat_template_content_format
)
self.trust_request_chat_template = (
chat_template_config.trust_request_chat_template
)
self.template_kwargs = None
self.tool_dicts = None
# Shared thread pool executor for preprocessing
self._executor: Executor = self.renderer._executor
self.render_async = make_async(self.render, executor=self._executor)
#######################################
# online APIs
def create_pooling_params(self, request):
return request.to_pooling_params()
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
renderer = self.renderer
requests: Sequence[AnyRenderParam]
if isinstance(request, PoolingChatLikeRequest):
self._validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
num_requests = 1
default_template_kwargs = merge_kwargs(
self.template_kwargs,
dict(
tools=self.tool_dicts,
tokenize=is_mistral_tokenizer(renderer.tokenizer),
),
)
mm_config = self.model_config.multimodal_config
tok_params = request.build_tok_params(self.model_config)
chat_params = request.build_chat_params(
self.chat_template, self.chat_template_content_format
).with_defaults(
default_template_kwargs,
default_media_io_kwargs=(
mm_config.media_io_kwargs if mm_config else None
),
)
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(
ctx.lora_request, num_requests
)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeChatRenderParams(
conversations=request.messages,
chat_params=chat_params,
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
elif isinstance(request, PoolingCompletionLikeRequest):
model_config = self.model_config
prompts_seq = prompt_to_seq(request.input)
num_requests = len(prompts_seq)
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(model_config, prompt)
)
for prompt in prompts_seq
]
tok_params = request.build_tok_params(model_config)
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(
ctx.lora_request, num_requests
)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
else:
raise ValueError(f"Invalid {self.name} request type")
def post_process_online(
self,
ctx: PoolingServeContext,
):
pass
#######################################
# offline APIs
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineEncodeInputsContext)
prompts_seq = prompt_to_seq(ctx.prompts)
num_requests = len(prompts_seq)
pooling_task = ctx.pooling_task
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(self.model_config, prompt)
)
for prompt in prompts_seq
]
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
pooling_params: PoolingParams | Sequence[PoolingParams]
if ctx.pooling_params is None:
pooling_params = PoolingParams()
else:
pooling_params = ctx.pooling_params
# Clone to avoid modifying the caller's pooling parameters.
params_seq = [
param.clone() for param in self._params_to_seq(pooling_params, num_requests)
]
for param in params_seq:
if param.task is None:
param.task = pooling_task
elif pooling_task == "plugin":
# `plugin` task uses io_processor.parse_data to verify inputs.
# We actually allow plugin to overwrite pooling_task.
pass
elif param.task != pooling_task:
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
raise VLLMValidationError(msg)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
def request_factory() -> RequestGenerator:
for i in range(num_requests):
yield EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=None,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
return request_factory, num_requests
def post_process_offline(
self,
ctx: OfflineOutputsContext,
) -> list[PoolingRequestOutput]:
return ctx.outputs
#######################################
# helpers
def render(
self,
render_params: EncodeCMPLRenderParams
| EncodeChatRenderParams
| ScoringRenderParams,
) -> PoolingEngineInput:
if "conversations" in render_params:
render_params = cast(EncodeChatRenderParams, render_params)
(_,), engine_input = self.renderer.render_chat(
conversations=[render_params["conversations"]],
chat_params=render_params["chat_params"],
tok_params=render_params["tok_params"],
prompt_extras=render_params["prompt_extras"],
skip_mm_cache=render_params["skip_mm_cache"],
)
elif "prompts" in render_params:
render_params = cast(EncodeCMPLRenderParams, render_params)
engine_input = self.renderer.render_cmpl(
prompts=[render_params["prompts"]],
tok_params=render_params["tok_params"],
prompt_extras=render_params["prompt_extras"],
skip_mm_cache=render_params["skip_mm_cache"],
)
else:
raise ValueError(
f"Unsupported render_params type {render_params.__class__.__name__}"
)
return PoolingEngineInput(
prompts=engine_input[0],
params=render_params["params"],
lora_requests=render_params["lora_requests"],
priorities=render_params["priorities"],
)
def _validate_chat_template(
self,
request_chat_template: str | None,
chat_template_kwargs: dict[str, Any] | None,
trust_request_chat_template: bool,
):
if not trust_request_chat_template and (
request_chat_template is not None
or (
chat_template_kwargs
and chat_template_kwargs.get("chat_template") is not None
)
):
raise VLLMValidationError(
"Chat template is passed with request, but "
"--trust-request-chat-template is not set. "
"Refused request with untrusted chat template."
)
return None
def _params_to_seq(
self,
params: PoolingParams | Sequence[PoolingParams],
num_requests: int,
) -> Sequence[PoolingParams]:
if isinstance(params, Sequence):
if len(params) != num_requests:
raise VLLMValidationError(
f"The lengths of prompts ({num_requests}) "
f"and params ({len(params)}) must be the same."
)
return params
return [params] * num_requests
def _lora_request_to_seq(
self,
lora_request: LoRARequest | None | Sequence[LoRARequest | None],
num_requests: int,
) -> Sequence[LoRARequest | None]:
if isinstance(lora_request, Sequence):
if len(lora_request) != num_requests:
raise VLLMValidationError(
f"The lengths of prompts ({num_requests}) "
f"and lora_request ({len(lora_request)}) must be the same."
)
return lora_request
return [lora_request] * num_requests
def _priority_to_seq(
self,
priority: int | Sequence[int] | None,
num_requests: int,
) -> Sequence[int]:
if priority is not None:
if isinstance(priority, int):
return [priority] * num_requests
if len(priority) != num_requests:
raise ValueError(
f"The lengths of prompts ({num_requests}) "
f"and priority ({len(priority)}) must be the same."
)
return priority
return [0] * num_requests
-366
View File
@@ -1,366 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Annotated, Any, Literal
from pydantic import Field, model_validator
from vllm.config import ModelConfig
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
from vllm.exceptions import VLLMValidationError
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
from vllm.renderers.chat_utils import (
ChatCompletionMessageParam,
ChatTemplateContentFormatOption,
)
from vllm.tasks import check_removed_pooling_task
from vllm.utils import random_uuid
from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness
def reject_removed_pooling_parameters(data):
if not isinstance(data, dict):
return data
if "normalize" in data:
raise VLLMValidationError(
"Parameter `normalize` was removed; use `use_activation` instead.",
parameter="normalize",
)
check_removed_pooling_task(data.get("task"))
return data
class PoolingBasicRequestMixin(OpenAIBaseModel):
# --8<-- [start:pooling-common-params]
model: str | None = None
user: str | None = None
# --8<-- [end:pooling-common-params]
# --8<-- [start:pooling-common-extra-params]
truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
padding: Literal["max_length", "do_not_pad"] | None = Field(
default=None,
description=(
"Whether to pad the prompt, using the same names as the "
"Transformers tokenizer. 'max_length' pads to the maximum input "
"length, 'do_not_pad' leaves the prompt as is. Models trained "
"with a fixed sequence length and no attention mask, such as "
"SigLIP, need 'max_length' to match training, otherwise their "
"embeddings are not comparable."
),
)
truncation_side: Literal["left", "right"] | None = Field(
default=None,
description=(
"Which side to truncate from when truncate_prompt_tokens is active. "
"'right' keeps the first N tokens. "
"'left' keeps the last N tokens."
),
)
request_id: str = Field(
default_factory=random_uuid,
description=(
"The request_id related to this request. If the caller does "
"not set it, a random_uuid will be generated. This id is used "
"through out the inference process and return in response."
),
)
priority: int = Field(
default=0,
ge=-(2**63),
le=2**63 - 1,
description=(
"The priority of the request (lower means earlier handling; "
"default: 0). Any priority other than 0 will raise an error "
"if the served model does not use priority scheduling."
),
)
mm_processor_kwargs: dict[str, Any] | None = Field(
default=None,
description="Additional kwargs to pass to the HF processor.",
)
cache_salt: str | None = Field(
default=None,
min_length=1,
max_length=1024,
description=(
"If specified, the prefix cache will be salted with the provided "
"string to prevent an attacker to guess prompts in multi-user "
"environments. The salt should be random, protected from "
"access by 3rd parties, and long enough to be "
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
"to 256 bit)."
),
)
# --8<-- [end:pooling-common-extra-params]
@model_validator(mode="before")
@classmethod
def check_cache_salt_support(cls, data: Any) -> Any:
if not isinstance(data, dict):
return data
validate_cache_salt(data.get("cache_salt"))
return data
def _build_pooling_tok_params(
self,
model_config: ModelConfig,
*,
add_special_tokens: bool,
max_total_tokens: int | None,
max_output_tokens: int,
max_total_tokens_param: str = "max_model_len",
max_output_tokens_param: str | None = None,
) -> TokenizeParams:
encoder_config = model_config.encoder_config or {}
if max_output_tokens_param is None:
tok_params = TokenizeParams(
max_total_tokens=max_total_tokens,
max_output_tokens=max_output_tokens,
truncate_prompt_tokens=self.truncate_prompt_tokens,
truncation_side=self.truncation_side,
do_lower_case=encoder_config.get("do_lower_case", False),
add_special_tokens=add_special_tokens,
max_total_tokens_param=max_total_tokens_param,
)
else:
tok_params = TokenizeParams(
max_total_tokens=max_total_tokens,
max_output_tokens=max_output_tokens,
truncate_prompt_tokens=self.truncate_prompt_tokens,
truncation_side=self.truncation_side,
do_lower_case=encoder_config.get("do_lower_case", False),
add_special_tokens=add_special_tokens,
max_total_tokens_param=max_total_tokens_param,
max_output_tokens_param=max_output_tokens_param,
)
if self.padding is None:
return tok_params
return tok_params.with_kwargs(padding=self.padding)
class PoolingTokenizeParamsMixin:
add_special_tokens: bool
def _build_pooling_tok_params(
self,
model_config: ModelConfig,
*,
add_special_tokens: bool,
max_total_tokens: int | None,
max_output_tokens: int,
max_total_tokens_param: str = "max_model_len",
max_output_tokens_param: str | None = None,
) -> TokenizeParams:
raise NotImplementedError
class FixedMaxLenTokenizeParamsMixin(PoolingTokenizeParamsMixin):
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
return self._build_pooling_tok_params(
model_config,
add_special_tokens=self.add_special_tokens,
max_total_tokens=model_config.max_model_len,
max_output_tokens=0,
)
class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
padding: Literal["max_length", "do_not_pad"] | None
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
default_max_total_tokens = model_config.max_model_len
max_total_tokens: int | None = default_max_total_tokens
max_output_tokens = 0
pooler_config = model_config.pooler_config
if pooler_config is not None:
if pooler_config.enable_chunked_processing:
max_total_tokens = pooler_config.max_embed_len
else:
max_total_tokens = (
pooler_config.max_embed_len or default_max_total_tokens
)
tok_params = self._build_pooling_tok_params(
model_config,
add_special_tokens=self.add_special_tokens,
max_total_tokens=max_total_tokens,
max_output_tokens=max_output_tokens,
)
if (
self.padding == "max_length"
and pooler_config is not None
and pooler_config.enable_chunked_processing
):
max_padding_len = pooler_config.max_embed_len
if max_padding_len is None:
max_padding_len = default_max_total_tokens
return tok_params.with_kwargs(
pad_prompt_tokens=min(default_max_total_tokens, max_padding_len)
)
return tok_params
class CompletionRequestMixin(OpenAIBaseModel):
# --8<-- [start:completion-params]
input: list[int] | list[list[int]] | str | list[str]
# --8<-- [end:completion-params]
# --8<-- [start:completion-extra-params]
add_special_tokens: bool = Field(
default=True,
description=(
"If true (the default), special tokens (e.g. BOS) will be added to "
"the prompt."
),
)
# --8<-- [end:completion-extra-params]
class ChatRequestOptionsMixin(OpenAIBaseModel):
# --8<-- [start:chat-extra-params]
add_generation_prompt: bool = Field(
default=False,
description=(
"If true, the generation prompt will be added to the chat template. "
"This is a parameter used by chat template in tokenizer config of the "
"model."
),
)
continue_final_message: bool = Field(
default=False,
description=(
"If this is set, the chat will be formatted so that the final "
"message in the chat is open-ended, without any EOS tokens. The "
"model will continue this message rather than starting a new one. "
'This allows you to "prefill" part of the model\'s response for it. '
"Cannot be used at the same time as `add_generation_prompt`."
),
)
add_special_tokens: bool = Field(
default=False,
description=(
"If true, special tokens (e.g. BOS) will be added to the prompt "
"on top of what is added by the chat template. "
"For most models, the chat template takes care of adding the "
"special tokens so this should be set to false (as is the "
"default)."
),
)
chat_template: str | None = Field(
default=None,
description=(
"A Jinja template to use for this conversion. "
"As of transformers v4.44, default chat template is no longer "
"allowed, so you must provide a chat template if the tokenizer "
"does not define one."
),
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the template renderer. "
"Will be accessible by the chat template."
),
)
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
default=None,
description=(
"Additional kwargs to pass to the media IO connectors, "
"keyed by modality. Merged with engine-level media_io_kwargs."
),
)
# --8<-- [end:chat-extra-params]
@model_validator(mode="before")
@classmethod
def check_generation_prompt(cls, data):
if not isinstance(data, dict):
return data
if data.get("continue_final_message") and data.get("add_generation_prompt"):
raise VLLMValidationError(
"Cannot set both `continue_final_message` and "
"`add_generation_prompt` to True.",
)
return data
def build_chat_params(
self,
default_template: str | None,
default_template_content_format: ChatTemplateContentFormatOption,
) -> ChatParams:
return ChatParams(
chat_template=self.chat_template or default_template,
chat_template_content_format=default_template_content_format,
chat_template_kwargs=merge_kwargs(
self.chat_template_kwargs,
dict(
add_generation_prompt=self.add_generation_prompt,
continue_final_message=self.continue_final_message,
),
),
media_io_kwargs=self.media_io_kwargs,
)
class ChatRequestMixin(ChatRequestOptionsMixin):
# --8<-- [start:chat-params]
messages: list[ChatCompletionMessageParam]
# --8<-- [end:chat-params]
class EncodingRequestMixin(OpenAIBaseModel):
# --8<-- [start:encoding-params]
encoding_format: EncodingFormat = "float"
# --8<-- [end:encoding-params]
# --8<-- [start:encoding-extra-params]
embed_dtype: EmbedDType = Field(
default="float32",
description=(
"What dtype to use for encoding. Default to using float32 for base64 "
"encoding to match the OpenAI python client behavior. "
"This parameter will affect base64 and binary_response."
),
)
endianness: Endianness = Field(
default="native",
description=(
"What endianness to use for encoding. Default to using native for "
"base64 encoding to match the OpenAI python client behavior."
"This parameter will affect base64 and binary_response."
),
)
# --8<-- [end:encoding-extra-params]
class EmbedRequestMixin(EncodingRequestMixin):
# --8<-- [start:embed-params]
dimensions: int | None = None
# --8<-- [end:embed-params]
# --8<-- [start:embed-extra-params]
use_activation: bool | None = Field(
default=None,
description="Whether to use activation for the pooler outputs. "
"`None` uses the pooler's default, which is `True` in most cases.",
)
# --8<-- [end:embed-extra-params]
class ClassifyRequestMixin(OpenAIBaseModel):
# --8<-- [start:classify-extra-params]
use_activation: bool | None = Field(
default=None,
description="Whether to use activation for the pooler outputs. "
"`None` uses the pooler's default, which is `True` in most cases.",
)
# --8<-- [end:classify-extra-params]
-280
View File
@@ -1,280 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from abc import ABC, abstractmethod
from collections.abc import AsyncGenerator
from concurrent.futures import Executor
from http import HTTPStatus
from typing import ClassVar
import torch
from fastapi import Request
from fastapi.responses import Response
from vllm import PoolingRequestOutput, envs
from vllm.config import VllmConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.engine.typing import AnyRequest
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.exceptions import GenerationError, RetryableRequestError
from vllm.lora.request import LoRARequest
from vllm.renderers.base import BaseRenderer
from vllm.renderers.chat_utils import ChatTemplateConfig
from vllm.utils.async_utils import make_async, merge_async_iterators
from ...serve.engine.protocol import ErrorResponse
from ..typing import AnyPoolingRequest, PoolingServeContext
from .io_processor import PoolingIOProcessor
class PoolingBaseServing(ABC, BaseServing):
request_id_prefix: ClassVar[str]
def __init__(
self,
engine_client: EngineClient,
models: OpenAIServingModels,
*,
request_logger: RequestLogger | None,
chat_template_config: ChatTemplateConfig,
return_tokens_as_token_ids: bool = False,
log_error_stack: bool = False,
):
super().__init__(
models=models,
model_config=models.model_config,
request_logger=request_logger,
)
self.engine_client = engine_client
self.renderer = engine_client.renderer
self.vllm_config = engine_client.vllm_config
self.max_model_len = self.model_config.max_model_len
self.return_tokens_as_token_ids = return_tokens_as_token_ids
self.log_error_stack = log_error_stack
self.chat_template_config = chat_template_config
self.trust_request_mm_kwargs = chat_template_config.trust_request_mm_kwargs
# Shared thread pool executor for preprocessing and postprocessing.
self._executor: Executor = self.renderer._executor
self._postprocessing_async = make_async(
self._postprocessing, executor=self._executor
)
async def __call__(
self,
request: AnyPoolingRequest,
raw_request: Request | None = None,
) -> Response:
io_processor = self.get_io_processor(request)
ctx = await self._init_ctx(io_processor, request, raw_request)
await self._preprocessing(io_processor, ctx)
await self._prepare_generators(ctx)
await self._collect_batch(ctx)
return await self._postprocessing_async(io_processor, ctx)
@abstractmethod
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
raise NotImplementedError
async def _preprocessing(
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
):
requests = io_processor.get_request_factory_online(ctx)
if len(requests) == 0:
raise ValueError("You must pass at least one prompt")
ctx.engine_inputs = await asyncio.gather(
*[io_processor.render_async(request) for request in requests]
)
@torch.inference_mode()
def _postprocessing(
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
):
io_processor.post_process_online(ctx)
return self._build_response(ctx)
async def _init_ctx(
self,
io_processor: PoolingIOProcessor,
request: AnyPoolingRequest,
raw_request: Request | None = None,
):
validate_request_mm_kwargs(
mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
media_io_kwargs=getattr(request, "media_io_kwargs", None),
trust_request_mm_kwargs=self.trust_request_mm_kwargs,
)
base_request_id = self._base_request_id(
raw_request, getattr(request, "request_id", None)
)
request_id = f"{self.request_id_prefix}-{base_request_id}"
await self._check_model(request)
pooling_params = io_processor.create_pooling_params(request)
lora_request = self._maybe_get_adapters(request)
model_name = self.models.model_name(lora_request)
priorities = getattr(request, "priority", 0)
prompt_extras = {
k: v
for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
if (v := getattr(request, k, None)) is not None
}
ctx = PoolingServeContext(
request=request,
raw_request=raw_request,
model_name=model_name,
pooling_params=pooling_params,
request_id=request_id,
lora_request=lora_request,
priorities=priorities,
prompt_extras=prompt_extras,
)
self._validate_request(ctx)
return ctx
async def _prepare_generators(
self,
ctx: PoolingServeContext,
):
if ctx.engine_inputs is None:
raise ValueError("Engine inputs not available")
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
trace_headers = (
None
if ctx.raw_request is None
else await self._get_trace_headers(ctx.raw_request.headers)
)
assert ctx.pooling_params is not None
pooling_params = ctx.pooling_params
pooling_params.verify(self.model_config)
for i, engine_input in enumerate(ctx.engine_inputs):
prompt_request_id = (
f"{ctx.request_id}-{i}"
if ctx.prompt_request_ids is None
else ctx.prompt_request_ids[i]
)
self._log_inputs(
prompt_request_id,
engine_input["prompts"],
params=engine_input["params"],
lora_request=ctx.lora_request,
)
generator = self.engine_client.encode(
request_id=prompt_request_id,
prompt=engine_input["prompts"],
pooling_params=engine_input["params"],
lora_request=engine_input["lora_requests"],
priority=engine_input["priorities"],
trace_headers=trace_headers,
)
generators.append(generator)
ctx.result_generator = merge_async_iterators(*generators)
async def _collect_batch(
self,
ctx: PoolingServeContext,
):
if ctx.engine_inputs is None:
raise ValueError("Engine prompts not available")
if ctx.result_generator is None:
raise ValueError("Result generator not available")
num_inputs = len(ctx.engine_inputs)
final_res_batch: list[PoolingRequestOutput | None]
final_res_batch = [None] * num_inputs
async for i, res in ctx.result_generator:
if res.error is not None:
if res.error.retryable:
raise RetryableRequestError(res.error.message)
raise GenerationError(res.error.message)
final_res_batch[i] = res
if None in final_res_batch:
raise ValueError("Failed to generate results for all prompts")
ctx.final_res_batch = [res for res in final_res_batch if res is not None]
@abstractmethod
def _build_response(
self,
ctx: PoolingServeContext,
) -> Response:
raise NotImplementedError
async def _check_model(
self,
request: AnyRequest | AnyPoolingRequest,
) -> ErrorResponse | None:
if self._is_model_supported(request.model):
return None
if request.model in self.models.lora_requests:
return None
if (
envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
and request.model
and (load_result := await self.models.resolve_lora(request.model))
):
if isinstance(load_result, LoRARequest):
return None
if (
isinstance(load_result, ErrorResponse)
and load_result.error.code == HTTPStatus.BAD_REQUEST.value
):
raise ValueError(load_result.error.message)
return None
def _validate_request(self, ctx: PoolingServeContext) -> None:
truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens", None)
if (
truncate_prompt_tokens is not None
and truncate_prompt_tokens > self.max_model_len
):
raise ValueError(
"truncate_prompt_tokens value is "
"greater than max_model_len."
" Please request a smaller truncation size."
)
return None
class PoolingServing(PoolingBaseServing, ABC):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.io_processor = self.init_io_processor(
vllm_config=self.vllm_config,
renderer=self.renderer,
chat_template_config=self.chat_template_config,
)
@abstractmethod
def init_io_processor(
self,
vllm_config: VllmConfig,
renderer: BaseRenderer,
chat_template_config: ChatTemplateConfig,
) -> PoolingIOProcessor:
raise NotImplementedError
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
return self.io_processor
@@ -1,33 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fastapi import APIRouter, Depends, Request
from fastapi.responses import Response
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from .protocol import ClassificationRequest
from .serving import ServingClassification
router = APIRouter()
def classify(request: Request) -> ServingClassification:
handler = getattr(request.app.state, "serving_classification", None)
if handler is None:
raise NotImplementedError("The model does not support Classification API")
return handler
@router.post("/classify", dependencies=[Depends(validate_json_request)])
@with_cancellation
@load_aware_call
async def create_classify(
request: ClassificationRequest, raw_request: Request
) -> Response:
handler = classify(raw_request)
return await handler(request, raw_request)
@@ -1,12 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from ..base.io_processor import PoolingIOProcessor
class ClassifyIOProcessor(PoolingIOProcessor):
name = "classify"
class TokenClassifyIOProcessor(PoolingIOProcessor):
name = "token_classify"
@@ -1,71 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import Annotated, TypeAlias
from pydantic import BeforeValidator, Field
from vllm import PoolingParams
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.logger import init_logger
from vllm.utils import random_uuid
from ..base.protocol import (
ChatRequestMixin,
ClassifyRequestMixin,
CompletionRequestMixin,
FixedMaxLenTokenizeParamsMixin,
PoolingBasicRequestMixin,
reject_removed_pooling_parameters,
)
logger = init_logger(__name__)
class ClassificationCompletionRequest(
PoolingBasicRequestMixin,
CompletionRequestMixin,
ClassifyRequestMixin,
FixedMaxLenTokenizeParamsMixin,
):
def to_pooling_params(self):
return PoolingParams(
task="classify",
use_activation=self.use_activation,
)
class ClassificationChatRequest(
PoolingBasicRequestMixin,
ChatRequestMixin,
ClassifyRequestMixin,
FixedMaxLenTokenizeParamsMixin,
):
def to_pooling_params(self):
return PoolingParams(
task="classify",
use_activation=self.use_activation,
)
ClassificationRequest: TypeAlias = Annotated[
ClassificationCompletionRequest | ClassificationChatRequest,
BeforeValidator(reject_removed_pooling_parameters),
]
class ClassificationData(OpenAIBaseModel):
index: int
label: str | None
probs: list[float]
num_classes: int
class ClassificationResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"classify-{random_uuid()}")
object: str = "list"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
data: list[ClassificationData]
usage: UsageInfo
@@ -1,72 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TypeAlias
import numpy as np
from fastapi.responses import JSONResponse
from vllm.entrypoints.serve.engine.protocol import UsageInfo
from vllm.logger import init_logger
from vllm.outputs import ClassificationOutput
from ..base.serving import PoolingServing
from ..typing import PoolingServeContext
from .io_processor import ClassifyIOProcessor
from .protocol import (
ClassificationData,
ClassificationRequest,
ClassificationResponse,
)
logger = init_logger(__name__)
ClassificationServeContext: TypeAlias = PoolingServeContext[ClassificationRequest]
class ServingClassification(PoolingServing):
request_id_prefix = "classify"
def init_io_processor(self, *args, **kwargs) -> ClassifyIOProcessor:
return ClassifyIOProcessor(*args, **kwargs)
def _build_response(
self,
ctx: ClassificationServeContext,
) -> JSONResponse:
id2label = getattr(self.model_config.hf_config, "id2label", {})
num_prompt_tokens = 0
items: list[ClassificationData] = []
for idx, final_res in enumerate(ctx.final_res_batch):
classify_res = ClassificationOutput.from_base(final_res.outputs)
probs = classify_res.probs
predicted_index = int(np.argmax(probs))
label = id2label.get(predicted_index)
item = ClassificationData(
index=idx,
label=label,
probs=probs,
num_classes=len(probs),
)
items.append(item)
prompt_token_ids = final_res.prompt_token_ids
num_prompt_tokens += len(prompt_token_ids)
usage = UsageInfo(
prompt_tokens=num_prompt_tokens,
total_tokens=num_prompt_tokens,
)
response = ClassificationResponse(
id=ctx.request_id,
created=ctx.created_time,
model=ctx.model_name,
data=items,
usage=usage,
)
return JSONResponse(content=response.model_dump())
@@ -1,61 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from .protocol import CohereEmbedRequest, EmbeddingRequest
from .serving import ServingEmbedding
router = APIRouter()
def embedding(request: Request) -> ServingEmbedding:
handler = getattr(request.app.state, "serving_embedding", None)
if handler is None:
raise NotImplementedError("The model does not support Embeddings API")
return handler
@router.post(
"/v1/embeddings",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_embedding(
request: EmbeddingRequest,
raw_request: Request,
):
handler = embedding(raw_request)
return await handler(request, raw_request)
@router.post(
"/v2/embed",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_cohere_embedding(
request: CohereEmbedRequest,
raw_request: Request,
):
handler = embedding(raw_request)
return await handler(request, raw_request)
@@ -1,777 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any, Literal, cast
import torch
from openai.types.chat import (
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
)
from vllm import PoolingParams
from vllm.exceptions import VLLMValidationError
from vllm.inputs import tokens_input
from vllm.logger import init_logger
from vllm.outputs import PoolingOutput, PoolingRequestOutput
from vllm.renderers import merge_kwargs
from vllm.renderers.chat_utils import (
ChatCompletionContentPartParam,
ChatCompletionMessageParam,
CustomChatCompletionMessageParam,
ImageURL,
)
from vllm.renderers.hf import resolve_chat_template
from vllm.utils.collection_utils import chunk_list
from vllm.utils.mistral import is_mistral_tokenizer
from ..base.io_processor import PoolingIOProcessor
from ..late_chunking import (
attach_late_chunking_metadata,
build_late_chunking_metadata,
prepare_late_chunking_input,
)
from ..scoring.io_processor import JinaRankingIOProcessorMixin
from ..typing import (
AnyOfflineInputsContext,
AnyRenderParam,
ChunkedEmbeddingMetadata,
EncodeChatRenderParams,
LateChunkingSlot,
OfflineEncodeInputsContext,
OfflineOutputsContext,
PoolingChatLikeRequest,
PoolingCompletionLikeRequest,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
)
from .protocol import (
CohereEmbedContent,
CohereEmbedInput,
CohereEmbedRequest,
EmbeddingBatchChatInputRequest,
EmbeddingBatchChatRequest,
EmbeddingChatInputRequest,
EmbeddingChatRequest,
EmbeddingCompletionRequest,
)
logger = init_logger(__name__)
@dataclass
class _ChunkedPromptAggregator:
weighted_sum: torch.Tensor | None = None
total_weight: int = 0
num_chunks: int = 0
all_chunks_l2_normalized: bool = True
class EmbedIOProcessor(PoolingIOProcessor):
name = "embed"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
assert self.model_config.pooler_config is not None
self.pooler_config = self.model_config.pooler_config
self.enable_chunked_processing = self.pooler_config.enable_chunked_processing
# Load task instructions from HF config or sentence-transformers config
self.task_instructions: dict[str, str] | None = self._load_task_instructions(
self.model_config.hf_config
) or self._load_st_prompts(self.model_config.model, self.model_config.revision)
if self.task_instructions:
logger.info(
"Loaded prompt prefixes for input_type: %s",
list(self.task_instructions.keys()),
)
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
if isinstance(ctx.request, CohereEmbedRequest):
requests = self._get_request_factory_cohere_online(ctx)
elif isinstance(
ctx.request,
(
EmbeddingChatRequest,
EmbeddingBatchChatRequest,
EmbeddingChatInputRequest,
EmbeddingBatchChatInputRequest,
),
):
requests = self._get_request_factory_chat_input_online(ctx)
else:
requests = super().get_request_factory_online(ctx)
return requests
def post_process_online(
self,
ctx: PoolingServeContext,
):
if ctx.final_res_batch is None:
raise ValueError("Final response batch not available")
if not self.enable_chunked_processing:
self._enforce_cohere_max_tokens(ctx)
return super().post_process_online(ctx)
self._post_process_chunked(ctx)
self._enforce_cohere_max_tokens(ctx)
#################################################################
# Long Text Embedding with Chunked Processing
# PTAL: examples/pooling/embed/openai_embedding_long_text
#################################################################
def maybe_pre_process_chunked(self, ctx: PoolingServeContext) -> None:
if not self.enable_chunked_processing:
return None
if ctx.engine_inputs is None:
raise ValueError("Engine prompts not available")
ctx.original_engine_inputs = ctx.engine_inputs
request_id = ctx.request_id
max_model_len = self.model_config.max_model_len
chunked_engine_inputs: list[PoolingEngineInput] = []
prompt_request_ids: list[str] = []
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] = []
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
token_ids = engine_input["prompts"].get("prompt_token_ids", None)
if token_ids is None:
raise NotImplementedError(
"Long Text Embedding with Chunked Processing does "
"not support EmbedsPrompt and EncoderDecoderInput."
)
prompt_token_ids = cast(list[int], token_ids)
cache_salt = cast(str | None, engine_input["prompts"].get("cache_salt"))
for chunk_idx, chunk_tokens in enumerate(
chunk_list(prompt_token_ids, max_model_len)
):
chunked_engine_inputs.append(
PoolingEngineInput(
prompts=tokens_input(
prompt_token_ids=chunk_tokens, cache_salt=cache_salt
),
params=engine_input["params"],
lora_requests=engine_input["lora_requests"],
priorities=engine_input["priorities"],
)
)
prompt_request_ids.append(
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
)
chunked_embedding_metadata.append(
ChunkedEmbeddingMetadata(
prompt_index=prompt_idx,
chunk_index=chunk_idx,
)
)
ctx.engine_inputs = chunked_engine_inputs
ctx.prompt_request_ids = prompt_request_ids
ctx.chunked_embedding_metadata = chunked_embedding_metadata
return None
def _post_process_chunked(self, ctx: PoolingServeContext) -> None:
# Online aggregation for chunked requests to
# minimize memory usage
# Track aggregation state for each prompt
if ctx.chunked_embedding_metadata is None:
raise ValueError("Chunked embedding metadata not available")
if len(ctx.chunked_embedding_metadata) != len(ctx.final_res_batch):
raise ValueError(
"Chunked embedding metadata count does not match result count"
)
prompt_aggregators: dict[int, _ChunkedPromptAggregator] = {}
for result, chunk_metadata in zip(
ctx.final_res_batch, ctx.chunked_embedding_metadata
):
prompt_idx = chunk_metadata.prompt_index
aggregator = prompt_aggregators.setdefault(
prompt_idx, _ChunkedPromptAggregator()
)
# MEAN pooling with online weighted averaging
# Ensure result is PoolingRequestOutput
# for embedding processing
if not isinstance(result, PoolingRequestOutput):
raise ValueError(
f"Expected PoolingRequestOutput for "
f"chunked embedding, got "
f"{type(result).__name__}"
)
if result.prompt_token_ids is None:
raise ValueError(
"prompt_token_ids cannot be None for chunked processing"
)
weight = len(result.prompt_token_ids)
embedding_data = result.outputs.data
float_embedding = embedding_data.to(dtype=torch.float32)
if ctx.pooling_params.use_activation:
# Custom activations need not L2-normalize; allow BF16 rounding.
embedding_norm = torch.linalg.vector_norm(float_embedding)
aggregator.all_chunks_l2_normalized &= torch.allclose(
embedding_norm,
torch.ones_like(embedding_norm),
rtol=1e-2,
atol=1e-5,
)
weighted_embedding = float_embedding * weight
if aggregator.weighted_sum is None:
# First chunk
aggregator.weighted_sum = weighted_embedding
else:
# Accumulate
aggregator.weighted_sum += weighted_embedding
aggregator.total_weight += weight
aggregator.num_chunks += 1
if ctx.original_engine_inputs is None:
raise ValueError("Original engine inputs not available")
original_engine_inputs = ctx.original_engine_inputs
num_prompts = len(original_engine_inputs)
# Finalize aggregated results
final_res_batch: list[PoolingRequestOutput] = []
for prompt_idx in range(num_prompts):
if prompt_idx in prompt_aggregators:
# Finalize MEAN aggregation for this chunked prompt
aggregator = prompt_aggregators[prompt_idx]
weighted_sum = aggregator.weighted_sum
total_weight = aggregator.total_weight
if (
weighted_sum is not None
and isinstance(weighted_sum, torch.Tensor)
and isinstance(total_weight, (int, float))
and total_weight > 0
):
# Compute final mean embedding
final_embedding = weighted_sum / total_weight
if (
ctx.pooling_params.use_activation
and aggregator.num_chunks > 1
and aggregator.all_chunks_l2_normalized
):
final_embedding = torch.nn.functional.normalize(
final_embedding, dim=-1
)
# Create a PoolingRequestOutput
# for the aggregated result
pooling_output_data = PoolingOutput(data=final_embedding)
# Get original prompt token IDs for this prompt
original_prompt = original_engine_inputs[prompt_idx]
token_ids = original_prompt["prompts"].get("prompt_token_ids", None)
if token_ids is None:
raise NotImplementedError(
"Long Text Embedding with Chunked Processing does "
"not support EmbedsPrompt and EncoderDecoderInput."
)
original_token_ids = cast(list[int], token_ids)
pooling_request_output = PoolingRequestOutput(
request_id=f"{ctx.request_id}-prompt-{prompt_idx}",
prompt_token_ids=original_token_ids,
outputs=pooling_output_data,
num_cached_tokens=0,
finished=True,
)
final_res_batch.append(pooling_request_output)
else:
raise ValueError(
f"Failed to aggregate chunks for prompt {prompt_idx}"
)
else:
raise ValueError(f"Result not found for prompt {prompt_idx}")
ctx.final_res_batch = final_res_batch
return None
#################################################################
# Chat input Request Preprocessing & Postprocessing
#################################################################
def _get_request_factory_chat_input_online(
self,
ctx: PoolingServeContext,
) -> Sequence[AnyRenderParam]:
request = ctx.request
renderer = self.renderer
self._validate_chat_template(
request_chat_template=request.chat_template,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=self.trust_request_chat_template,
)
if isinstance(
request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest)
):
all_messages = request.messages
else:
all_messages = [request.messages]
num_requests = len(all_messages)
default_template_kwargs = merge_kwargs(
self.template_kwargs,
dict(
tools=self.tool_dicts,
tokenize=is_mistral_tokenizer(renderer.tokenizer),
),
)
mm_config = self.model_config.multimodal_config
tok_params = request.build_tok_params(self.model_config)
chat_params = request.build_chat_params(
self.chat_template, self.chat_template_content_format
).with_defaults(
default_template_kwargs,
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
)
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeChatRenderParams(
conversations=all_messages[i],
chat_params=chat_params,
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
#################################################################
# Cohere Request Preprocessing & Postprocessing
#################################################################
@staticmethod
def _load_task_instructions(hf_config: Any) -> dict[str, str] | None:
"""Extract ``task_instructions`` from the HF model config."""
ti = getattr(hf_config, "task_instructions", None)
if not isinstance(ti, dict) or not ti:
return None
return {k: v for k, v in ti.items() if isinstance(v, str)}
@staticmethod
def _load_st_prompts(
model: str | Any,
revision: str | None,
) -> dict[str, str] | None:
"""Load ``task_instructions`` from ``config_sentence_transformers.json``."""
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
try:
cfg = get_hf_file_to_dict(
"config_sentence_transformers.json", str(model), revision
)
except (ValueError, OSError):
return None
if cfg is None:
return None
prompts = cfg.get("prompts")
if not isinstance(prompts, dict) or not prompts:
return None
return {k: v for k, v in prompts.items() if isinstance(v, str)}
@staticmethod
def _mixed_input_to_messages(
inp: CohereEmbedInput,
*,
task_prefix: str | None = None,
) -> list[ChatCompletionMessageParam]:
"""Build chat messages from a mixed text+image input.
When *task_prefix* is given, it is used as the system prompt.
"""
messages: list[ChatCompletionMessageParam] = []
if task_prefix is not None:
messages.append(
CustomChatCompletionMessageParam(
role="system",
content=[
ChatCompletionContentPartTextParam(
type="text", text=task_prefix
)
],
)
)
parts: list[ChatCompletionContentPartParam] = []
for item in inp.content:
if item.type == "text" and item.text is not None:
parts.append(
ChatCompletionContentPartTextParam(type="text", text=item.text)
)
elif item.type == "image_url" and item.image_url is not None:
parts.append(
ChatCompletionContentPartImageParam(
type="image_url",
image_url=ImageURL(url=item.image_url["url"]),
)
)
messages.append(CustomChatCompletionMessageParam(role="user", content=parts))
return messages
@staticmethod
def _check_cohere_max_tokens(
outputs: list[PoolingRequestOutput],
max_tokens_check: int | None,
) -> None:
"""Raise if any output exceeds *max_tokens_check* tokens.
Used to enforce ``truncate=NONE`` with an explicit ``max_tokens``:
the pipeline runs without truncation and we reject afterwards.
"""
if max_tokens_check is None:
return
for out in outputs:
n = len(out.prompt_token_ids)
if n > max_tokens_check:
raise ValueError(
f"Input of {n} tokens exceeds max_tokens={max_tokens_check} "
"with truncate=NONE. Set truncate to END or START to "
"allow truncation."
)
@staticmethod
def _resolve_cohere_truncation(
request: CohereEmbedRequest,
) -> tuple[int | None, Literal["left", "right"] | None]:
"""Return ``(truncate_prompt_tokens, truncation_side)``."""
if request.truncate == "NONE":
return None, None
if request.truncate == "START":
tokens = request.max_tokens if request.max_tokens is not None else -1
return tokens, "left"
if request.max_tokens is not None:
return request.max_tokens, None
return -1, None
def create_pooling_params(self, request):
if isinstance(request, CohereEmbedRequest):
return PoolingParams(
task="embed",
dimensions=request.output_dimension,
)
return super().create_pooling_params(request)
def _get_request_factory_cohere_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
"""Convert a ``CohereEmbedRequest`` into engine prompts.
If a model has a chat template the task instruction are rendered
as a system prompt. Otherwise they are just prepended to the input text.
Images and mixed inputs are always batch-rendered through the chat
template in one ``render_chat`` call.
"""
request = ctx.request
assert isinstance(request, CohereEmbedRequest)
if request.texts is None and request.images is None and request.inputs is None:
raise ValueError("One of texts, images, or inputs must be provided")
truncate_prompt_tokens, truncation_side = self._resolve_cohere_truncation(
request
)
input_type = request.input_type
self._validate_input_type(input_type)
if request.images is not None:
input: list[CohereEmbedInput] = [
CohereEmbedInput(
content=[
CohereEmbedContent(type="image_url", image_url={"url": uri})
]
)
for uri in request.images
]
elif request.inputs is not None:
input = request.inputs
else:
texts = request.texts or []
task_prefix = self._get_task_instruction_prefix(input_type)
if task_prefix is None:
return self._get_request_factory_cohere_text_completion(
ctx,
texts,
truncate_prompt_tokens,
truncation_side,
)
all_messages = [
self._mixed_input_to_messages(
CohereEmbedInput(
content=[CohereEmbedContent(type="text", text=text)]
),
task_prefix=task_prefix,
)
for text in texts
]
if self._has_chat_template():
return self._batch_render_chat(
ctx,
all_messages,
truncate_prompt_tokens,
truncation_side,
)
else:
return self._get_request_factory_cohere_text_completion(
ctx,
self._apply_task_instruction(texts, input_type),
truncate_prompt_tokens,
truncation_side,
)
task_prefix = self._get_task_instruction_prefix(input_type)
all_messages = [
self._mixed_input_to_messages(inp, task_prefix=task_prefix) for inp in input
]
return self._batch_render_chat(
ctx,
all_messages,
truncate_prompt_tokens,
truncation_side,
)
def _has_chat_template(self) -> bool:
return (
resolve_chat_template(
self.renderer.tokenizer,
chat_template=self.chat_template,
tools=None,
model_config=self.model_config,
)
is not None
)
def _get_request_factory_cohere_text_completion(
self,
ctx: PoolingServeContext,
texts: list[str],
truncate_prompt_tokens: int | None,
truncation_side: Literal["left", "right"] | None,
) -> Sequence[AnyRenderParam]:
request = ctx.request
proxy = EmbeddingCompletionRequest(
model=request.model,
input=texts,
dimensions=request.output_dimension,
encoding_format="float",
truncate_prompt_tokens=truncate_prompt_tokens,
truncation_side=truncation_side,
)
ctx.request = proxy
requests = super().get_request_factory_online(ctx)
ctx.request = request
return requests
def _batch_render_chat(
self,
ctx: PoolingServeContext,
all_messages: Sequence[list[ChatCompletionMessageParam]],
truncate_prompt_tokens: int | None,
truncation_side: Literal["left", "right"] | None,
) -> Sequence[AnyRenderParam]:
"""Batch-render multiple conversations through the chat template."""
request = ctx.request
proxy = EmbeddingBatchChatRequest(
model=request.model,
messages=all_messages,
dimensions=request.output_dimension,
encoding_format="float",
truncate_prompt_tokens=truncate_prompt_tokens,
truncation_side=truncation_side,
)
ctx.request = proxy
requests = self._get_request_factory_chat_input_online(ctx)
ctx.request = request
return requests
def _validate_input_type(self, input_type: str | None) -> None:
"""Raise if *input_type* is not supported by this model."""
if input_type is None:
return
if self.task_instructions is None:
raise ValueError(
f"Unsupported input_type {input_type!r}. "
"This model does not define any input_type task instructions."
)
if input_type not in self.task_instructions:
supported = ", ".join(sorted(self.task_instructions))
raise ValueError(
f"Unsupported input_type {input_type!r}. Supported values: {supported}"
)
def _apply_task_instruction(
self,
texts: list[str],
input_type: str | None,
) -> list[str]:
"""Prepend the task-instruction prefix for *input_type*.
Returns *texts* unchanged when no matching prefix is configured.
"""
prefix = self._get_task_instruction_prefix(input_type)
if not prefix:
return texts
return [prefix + t for t in texts]
def _get_task_instruction_prefix(self, input_type: str | None) -> str | None:
"""Return the task-instruction prefix for *input_type*, or ``None``."""
if not self.task_instructions or input_type is None:
return None
return self.task_instructions.get(input_type) or None
def _enforce_cohere_max_tokens(self, ctx: PoolingServeContext) -> None:
if isinstance(ctx.request, CohereEmbedRequest):
request = ctx.request
if request.truncate == "NONE" and request.max_tokens is not None:
self._check_cohere_max_tokens(ctx.final_res_batch, request.max_tokens)
class TokenEmbedIOProcessor(PoolingIOProcessor):
name = "token_embed"
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
factory, num_requests = super().get_request_factory_offline(ctx)
assert isinstance(ctx, OfflineEncodeInputsContext)
if ctx.pooling_params is None or not any(
p.late_chunking_params is not None
for p in self._params_to_seq(ctx.pooling_params, num_requests)
):
return factory, num_requests
if (ctx.tokenization_kwargs or {}).get("padding") not in (
None,
False,
"do_not_pad",
):
raise VLLMValidationError("Late chunking does not support input padding")
def request_factory() -> RequestGenerator:
for request in factory():
slot = None
if request["params"].late_chunking_params is not None:
slot = LateChunkingSlot()
request["late_chunking"] = slot
ctx.late_chunking.append(slot)
yield request
return request_factory, num_requests
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
late_chunking = render_params["params"].late_chunking_params
if late_chunking is None:
return super().render(render_params)
text, tok_params = prepare_late_chunking_input(self.vllm_config, render_params)
render_params = render_params.copy()
render_params["tok_params"] = tok_params
result = super().render(render_params)
prompt = result["prompts"]
if prompt["type"] != "token":
raise VLLMValidationError("Late chunking requires tokenized text")
render_params["late_chunking"].metadata = build_late_chunking_metadata(
text,
len(prompt["prompt_token_ids"]),
prompt.pop("prompt_token_offsets", None),
late_chunking.chunk_size,
)
return result
def post_process_offline(self, ctx: OfflineOutputsContext):
if ctx.late_chunking:
for output, slot in zip(ctx.outputs, ctx.late_chunking, strict=True):
if slot is not None:
assert slot.metadata is not None
attach_late_chunking_metadata(output, slot.metadata)
return ctx.outputs
class JinaRankingTokenEmbedIOProcessor(
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
):
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
if render_params["params"].late_chunking_params is not None:
raise VLLMValidationError(
"JinaForRanking returns document/query vectors and does not support "
"late chunking"
)
return super().render(render_params)
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
if isinstance(request, PoolingCompletionLikeRequest):
prompts = request.input
if not isinstance(prompts, Sequence) or len(prompts) < 2:
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
text_prompts = self.ensure_str(prompts)
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
prompt_input = self.format_docs_prompts_func(
query=text_prompts[-1], docs=text_prompts[:-1]
)
request.input = prompt_input
return super().get_request_factory_online(ctx)
elif isinstance(request, PoolingChatLikeRequest):
raise ValueError("The JinaForRanking does not support chat Request.")
else:
raise ValueError(f"Invalid {self.name} request type")
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineEncodeInputsContext)
if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
text_prompts = self.ensure_str(ctx.prompts)
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
ctx.prompts = self.format_docs_prompts_func(
query=text_prompts[-1], docs=text_prompts[:-1]
)
return super().get_request_factory_offline(ctx)
-354
View File
@@ -1,354 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Embedding API protocol models for OpenAI and Cohere formats.
OpenAI: https://platform.openai.com/docs/api-reference/embeddings
Cohere: https://docs.cohere.com/reference/embed
"""
import builtins
import struct
import time
from collections.abc import Sequence
from typing import Annotated, Any, Literal, TypeAlias
import numpy as np
import pybase64 as base64
from pydantic import BaseModel, BeforeValidator, Field, model_validator
from vllm import PoolingParams
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.renderers.chat_utils import ChatCompletionMessageParam
from vllm.utils import random_uuid
from ..base.protocol import (
ChatRequestMixin,
ChatRequestOptionsMixin,
CompletionRequestMixin,
EmbeddingTokenizeParamsMixin,
EmbedRequestMixin,
PoolingBasicRequestMixin,
reject_removed_pooling_parameters,
)
class EmbeddingCompletionRequest(
PoolingBasicRequestMixin,
CompletionRequestMixin,
EmbedRequestMixin,
EmbeddingTokenizeParamsMixin,
):
def to_pooling_params(self):
return PoolingParams(
task="embed",
dimensions=self.dimensions,
use_activation=self.use_activation,
)
def _is_chat_message(value: Any) -> bool:
return isinstance(value, dict) and isinstance(value.get("role"), str)
def _is_chat_messages(value: Any) -> bool:
return (
isinstance(value, list)
and bool(value)
and all(_is_chat_message(item) for item in value)
)
def _is_batched_chat_messages(value: Any) -> bool:
return (
isinstance(value, list)
and bool(value)
and all(_is_chat_messages(item) for item in value)
)
class EmbeddingChatRequest(
PoolingBasicRequestMixin,
ChatRequestMixin,
EmbedRequestMixin,
EmbeddingTokenizeParamsMixin,
):
"""OpenAI embeddings request with one top-level chat conversation."""
def to_pooling_params(self):
return PoolingParams(
task="embed",
dimensions=self.dimensions,
use_activation=self.use_activation,
)
class EmbeddingBatchChatRequest(
PoolingBasicRequestMixin,
ChatRequestOptionsMixin,
EmbedRequestMixin,
EmbeddingTokenizeParamsMixin,
):
"""OpenAI embeddings request with batched top-level chat conversations.
Mirrors ``BatchChatCompletionRequest`` by keeping batched conversations in
``messages`` instead of introducing a separate batch-specific field.
"""
messages: Sequence[
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
] = Field(..., min_length=1)
def to_pooling_params(self):
return PoolingParams(
task="embed",
dimensions=self.dimensions,
use_activation=self.use_activation,
)
class EmbeddingChatInputRequest(
EmbeddingChatRequest,
):
"""OpenAI embeddings request with one chat conversation in ``input``."""
input: list[ChatCompletionMessageParam]
@model_validator(mode="before")
@classmethod
def normalize_input_messages(cls, data):
if not isinstance(data, dict):
return data
if "messages" in data or "input" not in data:
return data
input_data = data["input"]
if not _is_chat_messages(input_data):
return data
normalized = dict(data)
normalized["messages"] = input_data
return normalized
class EmbeddingBatchChatInputRequest(EmbeddingBatchChatRequest):
"""OpenAI embeddings request with batched chat conversations in ``input``."""
input: Sequence[
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
] = Field(..., min_length=1)
@model_validator(mode="before")
@classmethod
def normalize_input_messages(cls, data):
if not isinstance(data, dict):
return data
if "messages" in data or "input" not in data:
return data
input_data = data["input"]
if not _is_batched_chat_messages(input_data):
return data
normalized = dict(data)
normalized["messages"] = input_data
return normalized
EmbeddingRequest: TypeAlias = Annotated[
EmbeddingCompletionRequest
| EmbeddingChatRequest
| EmbeddingBatchChatRequest
| EmbeddingChatInputRequest
| EmbeddingBatchChatInputRequest,
BeforeValidator(reject_removed_pooling_parameters),
]
# ---------------------------------------------------------------------------
# OpenAI /v1/embeddings — response models
# ---------------------------------------------------------------------------
class EmbeddingResponseData(OpenAIBaseModel):
index: int
object: str = "embedding"
embedding: list[float] | str
class EmbeddingResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
object: str = "list"
created: int = Field(default_factory=lambda: int(time.time()))
model: str | None = None
data: list[EmbeddingResponseData]
usage: UsageInfo
class EmbeddingBytesResponse(OpenAIBaseModel):
content: list[bytes]
headers: dict[str, str] | None = None
media_type: str = "application/octet-stream"
# ---------------------------------------------------------------------------
# Cohere /v2/embed — request models
# ---------------------------------------------------------------------------
CohereEmbeddingType = Literal[
"float",
"binary",
"ubinary",
"base64",
]
CohereTruncate = Literal["NONE", "START", "END"]
class CohereEmbedContent(BaseModel):
type: Literal["text", "image_url"]
text: str | None = None
image_url: dict[str, str] | None = None
@model_validator(mode="after")
def validate_content_payload(self):
if self.type == "text":
if self.text is None:
raise ValueError("CohereEmbedContent with type='text' requires text")
elif not self.image_url or not self.image_url.get("url"):
raise ValueError(
"CohereEmbedContent with type='image_url' requires image_url.url"
)
return self
class CohereEmbedInput(BaseModel):
content: list[CohereEmbedContent]
class CohereEmbedRequest(BaseModel):
model: str | None = None
input_type: str | None = None
texts: list[str] | None = None
images: list[str] | None = None
inputs: list[CohereEmbedInput] | None = None
output_dimension: int | None = None
embedding_types: list[CohereEmbeddingType] | None = None
truncate: CohereTruncate = "END"
max_tokens: int | None = None
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
@model_validator(mode="after")
def validate_input_fields(self):
input_fields = (self.texts, self.images, self.inputs)
provided_fields = [field for field in input_fields if field is not None]
if len(provided_fields) != 1 or not provided_fields[0]:
raise ValueError(
"Exactly one of texts, images, or inputs must be provided, "
"and it must be non-empty"
)
return self
# ---------------------------------------------------------------------------
# Cohere /v2/embed — response models
# ---------------------------------------------------------------------------
class CohereApiVersion(BaseModel):
version: str = "2"
class CohereBilledUnits(BaseModel):
input_tokens: int | None = None
image_tokens: int | None = None
class CohereMeta(BaseModel):
api_version: CohereApiVersion = Field(default_factory=CohereApiVersion)
billed_units: CohereBilledUnits | None = None
class CohereEmbedByTypeEmbeddings(BaseModel):
# The field name ``float`` shadows the builtin type, so the annotation
# must use ``builtins.float`` to avoid a self-referential type error.
float: list[list[builtins.float]] | None = None
binary: list[list[int]] | None = None
ubinary: list[list[int]] | None = None
base64: list[str] | None = None
class CohereEmbedResponse(BaseModel):
id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
embeddings: CohereEmbedByTypeEmbeddings
texts: list[str] | None = None
meta: CohereMeta | None = None
response_type: Literal["embeddings_by_type"] = "embeddings_by_type"
# ---------------------------------------------------------------------------
# Cohere embedding type conversion helpers
# ---------------------------------------------------------------------------
_UNSIGNED_TO_SIGNED_DIFF = 1 << 7 # 128
def _pack_binary_embeddings(
float_embeddings: list[list[float]],
signed: bool,
) -> list[list[int]]:
"""Bit-pack float embeddings: positive -> 1, negative -> 0.
Bits are packed MSB-first, eight per byte.
"""
if not float_embeddings:
return []
array = np.asarray(float_embeddings, dtype=np.float64)
if array.ndim != 2:
raise ValueError(
f"Expected a 2D batch of embeddings, but got {array.ndim}D input."
)
dim = array.shape[1]
if dim % 8 != 0:
raise ValueError(
"Embedding dimension must be a multiple of 8 for binary "
f"embedding types, but got {dim}."
)
packed = np.packbits(array >= 0, axis=-1)
if signed:
packed = packed.astype(np.int16) - _UNSIGNED_TO_SIGNED_DIFF
return packed.tolist()
def _encode_base64_embeddings(
float_embeddings: list[list[float]],
) -> list[str]:
"""Encode float embeddings as base64 (little-endian float32)."""
result: list[str] = []
for embedding in float_embeddings:
buf = struct.pack(f"<{len(embedding)}f", *embedding)
result.append(base64.b64encode(buf).decode("utf-8"))
return result
def build_typed_embeddings(
float_embeddings: list[list[float]],
embedding_types: Sequence[str],
) -> CohereEmbedByTypeEmbeddings:
"""Convert float embeddings to all requested Cohere embedding types."""
result = CohereEmbedByTypeEmbeddings()
for emb_type in embedding_types:
if emb_type == "float":
result.float = float_embeddings
elif emb_type == "binary":
result.binary = _pack_binary_embeddings(float_embeddings, signed=True)
elif emb_type == "ubinary":
result.ubinary = _pack_binary_embeddings(float_embeddings, signed=False)
elif emb_type == "base64":
result.base64 = _encode_base64_embeddings(float_embeddings)
return result
-220
View File
@@ -1,220 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TypeAlias, assert_never, cast
from fastapi.responses import JSONResponse, Response, StreamingResponse
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput
from vllm.utils.serial_utils import EmbedDType, Endianness
from ..base.io_processor import PoolingIOProcessor
from ..base.serving import PoolingServing
from ..typing import PoolingServeContext
from ..utils import (
BytesEncodingFormat,
JsonEncodingFormat,
build_pooling_bytes_streaming_response,
encode_pooling_output_float,
encode_pooling_output_float_or_ndarray,
get_json_response_cls,
get_pooling_output_encoder,
get_pooling_usage,
)
from .io_processor import EmbedIOProcessor
from .protocol import (
CohereBilledUnits,
CohereEmbedRequest,
CohereEmbedResponse,
CohereMeta,
EmbeddingRequest,
EmbeddingResponse,
EmbeddingResponseData,
build_typed_embeddings,
)
logger = init_logger(__name__)
EmbeddingServeContext: TypeAlias = PoolingServeContext[EmbeddingRequest]
class ServingEmbedding(PoolingServing):
"""Embedding API supporting both OpenAI and Cohere formats."""
request_id_prefix = "embd"
io_processor: EmbedIOProcessor
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.json_response_cls = get_json_response_cls()
def init_io_processor(self, *args, **kwargs) -> EmbedIOProcessor:
return EmbedIOProcessor(*args, **kwargs)
async def _preprocessing(
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
):
await super()._preprocessing(io_processor, ctx)
self.io_processor.maybe_pre_process_chunked(ctx)
def _build_response(
self,
ctx: PoolingServeContext,
) -> Response:
if isinstance(ctx.request, CohereEmbedRequest):
return self._build_cohere_response_from_ctx(ctx)
return self._build_openai_response(ctx)
def _build_openai_response(
self,
ctx: EmbeddingServeContext,
) -> JSONResponse | StreamingResponse:
encoding_format = ctx.request.encoding_format
embed_dtype = ctx.request.embed_dtype
endianness = ctx.request.endianness
if encoding_format == "float" or encoding_format == "base64":
return self._openai_json_response(
ctx.final_res_batch,
ctx.request_id,
ctx.created_time,
ctx.model_name,
encoding_format,
embed_dtype,
endianness,
)
if encoding_format == "bytes" or encoding_format == "bytes_only":
return self._openai_bytes_response(
ctx.final_res_batch,
ctx.request_id,
ctx.created_time,
ctx.model_name,
encoding_format,
embed_dtype,
endianness,
)
assert_never(encoding_format)
def _openai_json_response(
self,
final_res_batch: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
encoding_format: JsonEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> JSONResponse:
use_ndarray_response = (
encoding_format == "float"
and self.json_response_cls.__name__ == "ORJSONResponse"
)
if use_ndarray_response:
ndarray_items: list[dict[str, object]] = []
for idx, final_res in enumerate(final_res_batch):
item_dict = EmbeddingResponseData(
index=idx,
embedding=[],
).model_dump()
item_dict["embedding"] = encode_pooling_output_float_or_ndarray(
final_res
)
ndarray_items.append(item_dict)
ndarray_response = EmbeddingResponse(
id=request_id,
created=created_time,
model=model_name,
data=[], # type: ignore[arg-type]
usage=get_pooling_usage(final_res_batch),
).model_dump()
ndarray_response["data"] = ndarray_items
return self.json_response_cls(content=ndarray_response)
encode_fn = get_pooling_output_encoder(
encoding_format=encoding_format,
embed_dtype=embed_dtype,
endianness=endianness,
)
items: list[EmbeddingResponseData] = []
for idx, final_res in enumerate(final_res_batch):
item = EmbeddingResponseData(
index=idx,
embedding=cast(list[float] | str, encode_fn(final_res)),
)
items.append(item)
response = EmbeddingResponse(
id=request_id,
created=created_time,
model=model_name,
data=items,
usage=get_pooling_usage(final_res_batch),
)
return self.json_response_cls(content=response.model_dump())
def _openai_bytes_response(
self,
final_res_batch: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
encoding_format: BytesEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> StreamingResponse:
return build_pooling_bytes_streaming_response(
pooling_outputs=final_res_batch,
request_id=request_id,
created_time=created_time,
model_name=model_name,
encoding_format=encoding_format,
embed_dtype=embed_dtype,
endianness=endianness,
)
def _build_cohere_response_from_ctx(
self,
ctx: PoolingServeContext,
) -> JSONResponse:
request = ctx.request
assert isinstance(request, CohereEmbedRequest)
all_floats = [
cast(list[float], encode_pooling_output_float(out))
for out in ctx.final_res_batch
]
total_tokens = get_pooling_usage(ctx.final_res_batch).prompt_tokens
has_image_input = request.images is not None or any(
content.type == "image_url"
for input_item in request.inputs or []
for content in input_item.content
)
image_tokens = total_tokens if has_image_input else 0
texts_echo = request.texts
embedding_types = request.embedding_types or ["float"]
embeddings_obj = build_typed_embeddings(all_floats, embedding_types)
input_tokens = total_tokens - image_tokens
response = CohereEmbedResponse(
id=ctx.request_id,
embeddings=embeddings_obj,
texts=texts_echo,
meta=CohereMeta(
billed_units=CohereBilledUnits(
input_tokens=input_tokens,
image_tokens=image_tokens,
),
),
)
return self.json_response_cls(content=response.model_dump(exclude_none=True))
-272
View File
@@ -1,272 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import TYPE_CHECKING
from fastapi import FastAPI
from vllm.config import ModelConfig, VllmConfig
from vllm.logger import init_logger
from vllm.plugins.io_processors import has_io_processor
from vllm.renderers import BaseRenderer
from vllm.renderers.chat_utils import ChatTemplateConfig
from vllm.tasks import POOLING_TASKS, SCORE_TYPE_MAP, SupportedTask
from .base.io_processor import PoolingIOProcessor
from .utils import enable_scoring_api
if TYPE_CHECKING:
from argparse import Namespace
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.sagemaker.api_router import (
EndpointFn,
GetHandlerFn,
RequestType,
)
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
else:
RequestLogger = object
logger = init_logger(__name__)
def init_pooling_io_processors(
supported_tasks: tuple[SupportedTask, ...],
vllm_config: VllmConfig,
renderer: BaseRenderer,
chat_template_config: ChatTemplateConfig,
) -> dict[str, PoolingIOProcessor]:
model_config = vllm_config.model_config
processors: dict[str, type[PoolingIOProcessor]] = {}
pooling_task = model_config.get_pooling_task(supported_tasks)
if pooling_task == "classify":
from .classify.io_processor import ClassifyIOProcessor
processors["classify"] = ClassifyIOProcessor
if pooling_task == "token_classify":
from .classify.io_processor import TokenClassifyIOProcessor
processors["token_classify"] = TokenClassifyIOProcessor
if pooling_task == "embed":
from .embed.io_processor import EmbedIOProcessor
processors["embed"] = EmbedIOProcessor
if pooling_task == "token_embed":
from .embed.io_processor import TokenEmbedIOProcessor
processors["token_embed"] = TokenEmbedIOProcessor
if pooling_task == "embed&token_classify":
from .pooling.io_processor import UnsupportedCombinedTaskIOProcessor
processors[pooling_task] = UnsupportedCombinedTaskIOProcessor
has_plugin = has_io_processor(
vllm_config,
model_config.io_processor_plugin,
)
if has_plugin:
from .pooling.io_processor import PluginWithIOProcessorPlugins
processors["plugin"] = PluginWithIOProcessorPlugins
elif pooling_task == "plugin":
from .pooling.io_processor import PluginWithoutIOProcessorPlugins
processors["plugin"] = PluginWithoutIOProcessorPlugins
if enable_scoring_api(supported_tasks, model_config):
from .scoring.io_processor import ScoringIOProcessors
score_type: str | None = SCORE_TYPE_MAP.get(pooling_task, None) # type: ignore[arg-type]
if score_type is not None and score_type in ScoringIOProcessors:
processors[score_type] = ScoringIOProcessors[score_type]
if model_config.architecture == "JinaForRanking":
from .embed.io_processor import JinaRankingTokenEmbedIOProcessor
from .scoring.io_processor import ScoringIOProcessors
processors["token_embed"] = JinaRankingTokenEmbedIOProcessor
processors["late-interaction"] = ScoringIOProcessors["jina-reranking-scoring"]
return {
task: processor_cls(
vllm_config=vllm_config,
renderer=renderer,
chat_template_config=chat_template_config,
)
for task, processor_cls in processors.items()
}
def register_pooling_api_routers(
app: FastAPI,
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
if model_config is None:
return
pooling_task = model_config.get_pooling_task(supported_tasks)
if pooling_task is not None:
from .pooling.api_router import router as pooling_router
app.include_router(pooling_router)
if "classify" in supported_tasks:
from .classify.api_router import (
router as classify_router,
)
app.include_router(classify_router)
if "embed" in supported_tasks:
from .embed.api_router import router as embed_router
app.include_router(embed_router)
if enable_scoring_api(supported_tasks, model_config):
from .scoring.api_router import router as score_router
app.include_router(score_router)
def init_pooling_state(
engine_client: "EngineClient",
state: "State",
args: "Namespace",
request_logger: RequestLogger | None,
supported_tasks: tuple["SupportedTask", ...],
):
model_config = engine_client.model_config
if model_config is None:
return
from vllm.renderers.chat_utils import load_chat_template
from vllm.tasks import POOLING_TASKS
from .classify.serving import ServingClassification
from .embed.serving import ServingEmbedding
from .pooling.serving import ServingPooling
from .scoring.serving import ServingScores
resolved_chat_template = load_chat_template(args.chat_template)
pooling_task = model_config.get_pooling_task(supported_tasks)
chat_template_config = ChatTemplateConfig(
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
trust_request_chat_template=args.trust_request_chat_template,
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
)
state.serving_pooling = (
(
ServingPooling(
engine_client,
state.openai_serving_models,
supported_tasks=supported_tasks,
request_logger=request_logger,
chat_template_config=chat_template_config,
)
)
if any(t in supported_tasks for t in POOLING_TASKS)
else None
)
state.serving_embedding = (
ServingEmbedding(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
chat_template_config=chat_template_config,
)
if pooling_task == "embed"
else None
)
state.serving_classification = (
ServingClassification(
engine_client,
state.openai_serving_models,
request_logger=request_logger,
chat_template_config=chat_template_config,
)
if pooling_task == "classify"
else None
)
state.serving_scores = (
ServingScores(
engine_client,
state.openai_serving_models,
supported_tasks=supported_tasks,
request_logger=request_logger,
chat_template_config=chat_template_config,
enable_flash_late_interaction=getattr(
args, "enable_flash_late_interaction", True
),
)
if enable_scoring_api(supported_tasks, model_config)
else None
)
def get_pooling_invocation_types(
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
):
# NOTE: Items defined earlier take higher priority
invocation_types: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
if model_config is None:
return invocation_types
pooling_task = model_config.get_pooling_task(supported_tasks)
if pooling_task == "embed":
from .embed.api_router import create_embedding, embedding
from .embed.protocol import EmbeddingRequest
invocation_types += [
(EmbeddingRequest, (embedding, create_embedding)),
]
if pooling_task == "classify":
from .classify.api_router import classify, create_classify
from .classify.protocol import ClassificationRequest
invocation_types += [
(ClassificationRequest, (classify, create_classify)),
]
if enable_scoring_api(supported_tasks, model_config):
from .scoring.api_router import do_rerank, rerank
from .scoring.protocol import RerankRequest
invocation_types += [
(RerankRequest, (rerank, do_rerank)),
]
from .scoring.api_router import create_score, score
from .scoring.protocol import ScoreRequest
invocation_types += [
(ScoreRequest, (score, create_score)),
]
if any(task in POOLING_TASKS for task in supported_tasks):
from .pooling.api_router import create_pooling, pooling
from .pooling.protocol import PoolingRequest
invocation_types += [
(PoolingRequest, (pooling, create_pooling)),
]
return invocation_types
-92
View File
@@ -1,92 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from dataclasses import replace
from typing import cast
from vllm.config import VllmConfig
from vllm.exceptions import VLLMValidationError
from vllm.outputs import LateChunk, LateChunkingMetadata, PoolingRequestOutput
from vllm.renderers import TokenizeParams
from .typing import AnyRenderParam, EncodeCMPLRenderParams
def prepare_late_chunking_input(
config: VllmConfig, render_params: AnyRenderParam
) -> tuple[str, TokenizeParams]:
"""Validate full-document text input before requesting tokenizer offsets."""
params = render_params["params"]
params.verify(config.model_config)
if "prompts" not in render_params:
raise VLLMValidationError("Late chunking requires plain-text input")
prompt = cast(EncodeCMPLRenderParams, render_params)["prompts"]
if (
not isinstance(prompt, dict)
or set(prompt) - {"prompt", "cache_salt"}
or not isinstance(text := prompt.get("prompt"), str)
or not text
):
raise VLLMValidationError("Late chunking requires nonempty plain-text input")
tok_params = render_params["tok_params"]
if (
tok_params.pad_prompt_tokens is not None
or tok_params.truncate_prompt_tokens is not None
or tok_params.do_lower_case
):
raise VLLMValidationError(
"Late chunking does not support input padding, truncation or "
"renderer text normalization"
)
return text, replace(tok_params, return_token_offsets=True)
def build_late_chunking_metadata(
text: str,
num_tokens: int,
offsets: Sequence[tuple[int, int]] | None,
chunk_size: int,
) -> LateChunkingMetadata:
"""Map chunks to original text, keeping special-token-only chunks."""
if num_tokens <= 0 or offsets is None or len(offsets) != num_tokens:
raise VLLMValidationError(
"Late chunking requires a fast tokenizer with offsets aligned to "
"all input tokens"
)
previous_start = previous_end = 0
for start, end in offsets:
if not 0 <= start <= end <= len(text):
raise VLLMValidationError("Token offsets are not aligned to the input text")
if start < end:
if start < previous_start or end < previous_end:
raise VLLMValidationError("Token offsets are not in source-text order")
previous_start, previous_end = start, end
chunks = []
for start in range(0, num_tokens, chunk_size):
end = min(start + chunk_size, num_tokens)
source = [(a, b) for a, b in offsets[start:end] if a < b]
char_range = (
(min(a for a, _ in source), max(b for _, b in source)) if source else None
)
chunks.append(LateChunk(token_range=(start, end), char_range=char_range))
return LateChunkingMetadata(
chunk_size=chunk_size, input_tokens=num_tokens, chunks=chunks
)
def attach_late_chunking_metadata(
output: PoolingRequestOutput, metadata: LateChunkingMetadata
) -> None:
"""Attach ranges only to successful, complete chunk-vector results."""
if not output.finished or output.error is not None:
return
data = output.outputs.data
if (
data.ndim != 2
or data.shape[0] != len(metadata.chunks)
or len(output.prompt_token_ids) != metadata.input_tokens
):
raise ValueError("Late-chunking output does not match its token/source ranges")
output.late_chunking = metadata
-479
View File
@@ -1,479 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable, Sequence
from typing import Any
from tqdm.auto import tqdm
from vllm.entrypoints.common.offline import OfflineInferenceMixin
from vllm.inputs import DataPrompt, PromptType
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.outputs import (
ClassificationRequestOutput,
EmbeddingRequestOutput,
PoolingRequestOutput,
ScoringRequestOutput,
)
from vllm.pooling_params import PoolingParams
from vllm.renderers.chat_utils import ChatTemplateConfig
from vllm.tasks import SCORE_TYPE_MAP, PoolingTask, SupportedTask
from .base.io_processor import PoolingIOProcessor
from .factories import init_pooling_io_processors
from .scoring.io_processor import ScoringIOProcessor
from .scoring.typing import ScoreInput
from .typing import (
AnyOfflineInputsContext,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflinePluginInputsContext,
OfflineScoringInputsContext,
RequestFactory,
)
logger = init_logger(__name__)
class PoolingOfflineMixin(OfflineInferenceMixin):
"""Offline inference for pooling models."""
runner_type: str
chat_template: str | None
supported_tasks: tuple[SupportedTask, ...]
def __init__(self):
self.pooling_task = self.model_config.get_pooling_task(self.supported_tasks)
if self.pooling_task is not None:
logger.info("Supported pooling task: %s", self.pooling_task)
self.chat_template_config = ChatTemplateConfig(chat_template=self.chat_template)
self.pooling_io_processors = init_pooling_io_processors(
supported_tasks=self.supported_tasks,
vllm_config=self.llm_engine.vllm_config,
renderer=self.renderer,
chat_template_config=self.chat_template_config,
)
# Use thread pool executor to accelerate preprocessing.
self._executor = self.renderer._executor
def encode(
self,
prompts: PromptType | Sequence[PromptType] | DataPrompt,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
*,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: list[LoRARequest] | LoRARequest | None = None,
pooling_task: PoolingTask | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[PoolingRequestOutput]:
"""Apply pooling to the hidden states corresponding to the input
prompts.
This class automatically batches the given prompts, considering
the memory constraint. For the best performance, put all of your prompts
into a single list and pass it to this method.
Args:
prompts: The prompts to the LLM. You may pass a sequence of prompts
for batch inference. See [PromptType][vllm.inputs.PromptType]
for more details about the format of each prompt.
pooling_params: The pooling parameters for pooling. If None, we
use the default pooling parameters.
use_tqdm: If `True`, shows a tqdm progress bar.
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
pooling_task: Override the pooling task to use.
tokenization_kwargs: Overrides for `tokenizer.encode`.
Returns:
A list of `PoolingRequestOutput` objects containing the
pooled hidden states in the same order as the input prompts.
"""
if isinstance(prompts, dict) and "data" in prompts and pooling_task != "plugin":
raise ValueError(
"The 'data' field is only supported for the 'plugin' pooling task."
)
self._verify_pooling_task(pooling_task)
assert pooling_task is not None and pooling_task in self.pooling_io_processors
io_processor = self.pooling_io_processors[pooling_task]
ctx: AnyOfflineInputsContext
if isinstance(prompts, dict) and "data" in prompts:
ctx = OfflinePluginInputsContext(
pooling_task=pooling_task,
prompts=prompts, # type: ignore[arg-type]
tokenization_kwargs=tokenization_kwargs,
pooling_params=pooling_params,
lora_request=lora_request,
priorities=None,
)
else:
ctx = OfflineEncodeInputsContext(
pooling_task=pooling_task,
prompts=prompts,
tokenization_kwargs=tokenization_kwargs,
pooling_params=pooling_params,
lora_request=lora_request,
priorities=None,
)
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
outputs = self._run_tiling_engine(
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
)
outputs = io_processor.post_process_offline(
ctx=OfflineOutputsContext(outputs=outputs, late_chunking=ctx.late_chunking)
)
return outputs
def _verify_pooling_task(self, pooling_task: PoolingTask | None):
if self.runner_type != "pooling":
raise ValueError(
"LLM.encode() is only supported for pooling models. "
"Try passing `--runner pooling` to use the model as a "
"pooling model."
)
if pooling_task is None:
raise ValueError(
"""
pooling_task required for `LLM.encode`.
Please use one of the more specific methods or set the pooling_task when using `LLM.encode`:
- For embeddings, use `LLM.embed(...)` or `pooling_task="embed"`.
- For classification logits, use `LLM.classify(...)` or `pooling_task="classify"`.
- For similarity scores, use `LLM.score(...)`.
- For rewards, `pooling_task="classify"` or `pooling_task="token_classify"`.
- For token classification, use `pooling_task="token_classify"`.
- For multi-vector retrieval, use `pooling_task="token_embed"`.
""" # noqa: E501
)
if (
pooling_task in ("embed", "token_embed")
and pooling_task not in self.supported_tasks
):
raise ValueError(
"Embedding API is not supported by this model. "
"Try converting the model using `--convert embed`."
)
if (
pooling_task in ("classify", "token_classify")
and pooling_task not in self.supported_tasks
):
raise ValueError(
"Classification API is not supported by this model. "
"Try converting the model using `--convert classify`."
)
# plugin task uses io_processor.parse_data to verify inputs
if pooling_task != "plugin" and pooling_task != self.pooling_task:
if pooling_task not in self.supported_tasks:
raise ValueError(
f"Unsupported task: {pooling_task!r} "
f"Supported tasks: {self.supported_tasks}"
)
else:
raise ValueError(
f"Try switching the model's pooling_task "
f'via `PoolerConfig(task="{pooling_task}")`'
)
if pooling_task == "plugin" and "plugin" not in self.pooling_io_processors:
raise ValueError(
"No IOProcessor plugin installed. Please refer "
"to the documentation and to the "
"'prithvi_geospatial_mae_io_processor' "
"offline inference example for more details."
)
def embed(
self,
prompts: PromptType | Sequence[PromptType],
*,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[EmbeddingRequestOutput]:
"""Generate an embedding vector for each prompt.
This class automatically batches the given prompts, considering
the memory constraint. For the best performance, put all of your prompts
into a single list and pass it to this method.
Args:
prompts: The prompts to the LLM. You may pass a sequence of prompts
for batch inference. See [PromptType][vllm.inputs.PromptType]
for more details about the format of each prompt.
pooling_params: The pooling parameters for pooling. If None, we
use the default pooling parameters.
use_tqdm: If `True`, shows a tqdm progress bar.
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
tokenization_kwargs: Overrides for `tokenizer.encode`.
Returns:
A list of `EmbeddingRequestOutput` objects containing the
embedding vectors in the same order as the input prompts.
"""
items = self.encode(
prompts,
use_tqdm=use_tqdm,
pooling_params=pooling_params,
lora_request=lora_request,
pooling_task="embed",
tokenization_kwargs=tokenization_kwargs,
)
return [EmbeddingRequestOutput.from_base(item) for item in items]
def classify(
self,
prompts: PromptType | Sequence[PromptType],
*,
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
use_tqdm: bool | Callable[..., tqdm] = True,
lora_request: list[LoRARequest] | LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
) -> list[ClassificationRequestOutput]:
"""Generate class logits for each prompt.
This class automatically batches the given prompts, considering
the memory constraint. For the best performance, put all of your prompts
into a single list and pass it to this method.
Args:
prompts: The prompts to the LLM. You may pass a sequence of prompts
for batch inference. See [PromptType][vllm.inputs.PromptType]
for more details about the format of each prompt.
pooling_params: The pooling parameters for pooling. If None, we
use the default pooling parameters.
use_tqdm: If `True`, shows a tqdm progress bar.
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
tokenization_kwargs: Overrides for `tokenizer.encode`.
Returns:
A list of `ClassificationRequestOutput` objects containing the
embedding vectors in the same order as the input prompts.
"""
items = self.encode(
prompts,
use_tqdm=use_tqdm,
pooling_params=pooling_params,
lora_request=lora_request,
pooling_task="classify",
tokenization_kwargs=tokenization_kwargs,
)
return [ClassificationRequestOutput.from_base(item) for item in items]
def score(
self,
data_1: ScoreInput | list[ScoreInput],
data_2: ScoreInput | list[ScoreInput],
/,
*,
use_tqdm: bool | Callable[..., tqdm] = True,
pooling_params: PoolingParams | None = None,
lora_request: list[LoRARequest] | LoRARequest | None = None,
tokenization_kwargs: dict[str, Any] | None = None,
chat_template: str | None = None,
) -> list[ScoringRequestOutput]:
"""Generate similarity scores for all pairs `<text,text_pair>` or
`<multi-modal data, multi-modal data pair>`.
The inputs can be `1 -> 1`, `1 -> N` or `N -> N`.
In the `1 - N` case the `data_1` input will be replicated `N`
times to pair with the `data_2` inputs.
The input pairs are used to build a list of prompts for the
cross encoder model. This class automatically batches the prompts,
considering the memory constraint. For the best performance, put all
of your inputs into a single list and pass it to this method.
Supports both text and multi-modal data (images, etc.) when used with
appropriate multi-modal models. For multi-modal inputs, ensure the
prompt structure matches the model's expected input format.
Args:
data_1: Can be a single prompt, a list of prompts or
`ScoreMultiModalParam`, which can contain either text or
multi-modal data. When a list, it must have the same length as
the `data_2` list.
data_2: The data to pair with the query to form the input to
the LLM. Can be text or multi-modal data. See [PromptType]
[vllm.inputs.PromptType] for more details about the format of
each prompt.
pooling_params: The pooling parameters for pooling. If None, we
use the default pooling parameters.
use_tqdm: If `True`, shows a tqdm progress bar.
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
it is used to create the progress bar.
If `False`, no progress bar is created.
lora_request: LoRA request to use for generation, if any.
chat_template: The chat template to use for scoring. If None, use
the chat template passed to the LLM constructor.
tokenization_kwargs: Overrides for `tokenizer.encode`.
Returns:
A list of `ScoringRequestOutput` objects containing the
generated scores in the same order as the input prompts.
"""
if self.runner_type != "pooling":
raise ValueError(
"LLM.score() is only supported for pooling models. "
"Try passing `--runner pooling` to use the model as a "
"pooling model."
)
score_type: str | None = SCORE_TYPE_MAP.get(self.pooling_task, None) # type: ignore[arg-type]
if (
score_type == "cross-encoder"
and getattr(self.model_config.hf_config, "num_labels", 0) != 1
):
raise ValueError("Scoring API is only enabled for num_labels == 1.")
if score_type is None or score_type not in self.pooling_io_processors:
raise ValueError("This model does not support the Scoring API.")
io_processor = self.pooling_io_processors[score_type]
assert isinstance(io_processor, ScoringIOProcessor)
scoring_data = io_processor.valid_inputs(data_1, data_2)
n_queries = len(scoring_data.data_1)
if pooling_params is None:
pooling_params = PoolingParams()
assert isinstance(pooling_params, PoolingParams)
pooling_task = io_processor.pooling_task
# Clone to avoid modifying the caller's pooling parameters.
pooling_params = pooling_params.clone()
if pooling_params.task is None:
pooling_params.task = pooling_task
elif pooling_params.task != pooling_task:
msg = (
f"You cannot overwrite {pooling_params.task=!r} with {pooling_task=!r}!"
)
raise ValueError(msg)
ctx = OfflineScoringInputsContext(
pooling_task=pooling_task,
scoring_data=scoring_data,
pooling_params=pooling_params,
tokenization_kwargs=tokenization_kwargs,
lora_request=lora_request,
chat_template=chat_template,
priorities=None,
)
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
outputs = self._run_tiling_engine(
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
)
outputs = io_processor.post_process_offline(
ctx=OfflineOutputsContext(outputs=outputs, n_queries=n_queries),
)
return [ScoringRequestOutput.from_base(item) for item in outputs]
def _run_tiling_engine(
self,
io_processor: PoolingIOProcessor,
request_factory: RequestFactory,
num_requests: int,
use_tqdm: bool | Callable[..., tqdm] = True,
):
if num_requests == 0:
raise ValueError("You must pass at least one prompt")
# Keeping max_num_seqs * 2 requests in the core can already saturate the core.
# Therefore, keep most requests waiting outside the core.
max_requests_in_core = (
self.llm_engine.vllm_config.scheduler_config.max_num_seqs * 2
)
num_requests_in_core = 0
num_waited_requests = num_requests
if use_tqdm:
tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
pbar = tqdm_func(
total=num_requests,
desc="Processed prompts",
dynamic_ncols=True,
postfix=f"est. speed input: {0:.2f} toks/s, output: {0:.2f} toks/s",
)
outputs: list[PoolingRequestOutput] = []
added_request_ids: set[str] = set()
it = self._executor.map(io_processor.render, request_factory())
try:
while num_waited_requests or self.llm_engine.has_unfinished_requests():
requests = []
for _ in range(max_requests_in_core - num_requests_in_core):
if num_waited_requests == 0:
break
try:
request = next(it)
requests.append(request)
except StopIteration:
num_waited_requests = 0
break
num_waited_requests -= 1
num_requests_in_core += 1
if requests:
request_ids = self._render_and_add_requests(
prompts=[x["prompts"] for x in requests],
params=[x["params"] for x in requests],
lora_requests=[x["lora_requests"] for x in requests],
priorities=[x["priorities"] for x in requests],
)
for request_id in request_ids:
# undo assign_request_id
request_id = request_id.split("-", 1)[0]
added_request_ids.add(request_id)
step_outputs = self.llm_engine.step()
for output in step_outputs:
assert isinstance(output, PoolingRequestOutput)
assert output.finished
outputs.append(output)
added_request_ids.discard(output.request_id)
num_requests_in_core -= 1
if use_tqdm:
pbar.update(1)
if pbar.n == num_requests:
pbar.refresh()
except Exception:
if added_request_ids:
self.llm_engine.abort_request(list(added_request_ids))
raise
finally:
if use_tqdm:
pbar.close()
return sorted(outputs, key=lambda x: int(x.request_id))
@@ -1,39 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from .protocol import PoolingRequest
from .serving import ServingPooling
router = APIRouter()
def pooling(request: Request) -> ServingPooling:
handler = getattr(request.app.state, "serving_pooling", None)
if handler is None:
raise NotImplementedError("The model does not support Pooling API")
return handler
@router.post(
"/pooling",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_pooling(request: PoolingRequest, raw_request: Request):
handler = pooling(raw_request)
return await handler(request, raw_request)
@@ -1,187 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Sequence
from typing import Any
from vllm import PoolingParams, PoolingRequestOutput
from vllm.plugins.io_processors import get_io_processor
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
from ..base.io_processor import PoolingIOProcessor
from ..typing import (
AnyOfflineInputsContext,
AnyRenderParam,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflinePluginInputsContext,
PoolingServeContext,
RequestFactory,
)
from .protocol import IOProcessorRequest, IOProcessorResponse
class UnsupportedCombinedTaskIOProcessor(PoolingIOProcessor):
name = "embed&token_classify"
def create_pooling_params(self, request):
raise ValueError(
"The 'embed&token_classify' pooling task is only available "
"through an IO processor plugin. Send a plugin request with "
"a 'data' field, "
"or select a concrete task with --pooler-config.task."
)
class PluginWithoutIOProcessorPlugins(PoolingIOProcessor):
# Some models, such as Terratorch (tests/models/test_terratorch.py),
# use plugin tasks in the pooler but do not use IO Processor plugins.
name = "plugin"
class PluginWithIOProcessorPlugins(PoolingIOProcessor):
"""IO Processor plugins are a feature that allows pre- and post-processing
of the model input and output for pooling models."""
name = "plugin"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
io_processor = get_io_processor(
self.vllm_config,
self.renderer,
self.model_config.io_processor_plugin,
)
assert io_processor is not None
self.io_processor = io_processor
#######################################
# online APIs
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
assert isinstance(ctx.request, IOProcessorRequest)
validated_prompt = self.io_processor.parse_data(ctx.request.data)
raw_prompts = self.io_processor.pre_process(
prompt=validated_prompt, request_id=ctx.request_id
)
parsed_prompts = [
(
prompt
if isinstance(prompt, bytes)
else parse_model_prompt(self.model_config, prompt)
)
for prompt in prompt_to_seq(raw_prompts)
]
num_requests = len(parsed_prompts)
tok_params = ctx.request.build_tok_params(self.model_config)
pooling_params = self.io_processor.merge_pooling_params()
if pooling_params.task is None:
pooling_params.task = "plugin"
ctx.pooling_params = pooling_params
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
def post_process_online(
self,
ctx: PoolingServeContext,
):
output = self.io_processor.post_process(
ctx.final_res_batch,
request_id=ctx.request_id,
)
ctx.response = IOProcessorResponse(request_id=ctx.request_id, data=output)
#######################################
# offline APIs
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflinePluginInputsContext)
assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts
# Validate the request data is valid for the loaded plugin
prompt_data = ctx.prompts.get("data")
if prompt_data is None:
raise ValueError(
"The 'data' field of the prompt is expected to contain "
"the prompt data and it cannot be None. "
"Refer to the documentation of the IOProcessor "
"in use for more details."
)
validated_prompt = self.io_processor.parse_data(prompt_data)
# obtain the actual model prompts from the pre-processor
prompts = self.io_processor.pre_process(prompt=validated_prompt)
prompts_seq = prompt_to_seq(prompts)
num_requests = len(prompts_seq)
pooling_params: PoolingParams | Sequence[PoolingParams]
if ctx.pooling_params is None:
pooling_params = PoolingParams()
else:
pooling_params = ctx.pooling_params
params_seq: list[PoolingParams] = [
self.io_processor.merge_pooling_params(param.clone())
for param in self._params_to_seq(
pooling_params,
num_requests,
)
]
for p in params_seq:
if p.task is None:
p.task = "plugin"
return super().get_request_factory_offline(
OfflineEncodeInputsContext(
prompts=prompts_seq,
pooling_params=params_seq,
pooling_task="plugin",
tokenization_kwargs=ctx.tokenization_kwargs,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
)
)
def post_process_offline(
self,
ctx: OfflineOutputsContext,
) -> list[PoolingRequestOutput]:
processed_outputs = self.io_processor.post_process(ctx.outputs)
return [
PoolingRequestOutput[Any](
request_id="",
outputs=processed_outputs,
num_cached_tokens=getattr(processed_outputs, "num_cached_tokens", 0),
prompt_token_ids=[],
finished=True,
)
]
@@ -1,120 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import Annotated, Generic, TypeAlias, TypeVar
from pydantic import BeforeValidator, Field
from vllm import PoolingParams
from vllm.config import ModelConfig
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.renderers import TokenizeParams
from vllm.tasks import PoolingTask
from vllm.utils import random_uuid
from ..base.protocol import (
ChatRequestMixin,
ClassifyRequestMixin,
CompletionRequestMixin,
EmbedRequestMixin,
EncodingRequestMixin,
FixedMaxLenTokenizeParamsMixin,
PoolingBasicRequestMixin,
reject_removed_pooling_parameters,
)
class PoolingCompletionRequest(
PoolingBasicRequestMixin,
CompletionRequestMixin,
EmbedRequestMixin,
ClassifyRequestMixin,
FixedMaxLenTokenizeParamsMixin,
):
task: PoolingTask | None = None
def to_pooling_params(self):
return PoolingParams(
task=self.task,
use_activation=self.use_activation,
dimensions=self.dimensions,
)
class PoolingChatRequest(
PoolingBasicRequestMixin,
ChatRequestMixin,
EmbedRequestMixin,
ClassifyRequestMixin,
FixedMaxLenTokenizeParamsMixin,
):
task: PoolingTask | None = None
def to_pooling_params(self):
return PoolingParams(
task=self.task,
use_activation=self.use_activation,
dimensions=self.dimensions,
)
T = TypeVar("T")
class IOProcessorRequest(PoolingBasicRequestMixin, EncodingRequestMixin, Generic[T]):
data: T
task: PoolingTask = "plugin"
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
return self._build_pooling_tok_params(
model_config,
add_special_tokens=not model_config.is_encoder_decoder,
max_total_tokens=model_config.max_model_len,
max_output_tokens=0,
)
def to_pooling_params(self):
return PoolingParams(
task=self.task,
)
class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
request_id: str | None = None
"""
The request_id associated with this response
"""
created_at: int = Field(default_factory=lambda: int(time.time()))
data: T
"""
When using plugins IOProcessor plugins, the actual output is generated
by the plugin itself. Hence, we use a generic type for the response data
"""
PoolingRequest: TypeAlias = Annotated[
PoolingCompletionRequest | PoolingChatRequest | IOProcessorRequest,
BeforeValidator(reject_removed_pooling_parameters),
]
class PoolingResponseData(OpenAIBaseModel):
index: int
object: str = "pooling"
data: list[list[float]] | list[float] | str
class PoolingResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"pool-{random_uuid()}")
object: str = "list"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
data: list[PoolingResponseData]
usage: UsageInfo
class PoolingBytesResponse(OpenAIBaseModel):
content: list[bytes]
headers: dict[str, str] | None = None
media_type: str = "application/octet-stream"
-191
View File
@@ -1,191 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import assert_never, cast
from fastapi.responses import JSONResponse, Response, StreamingResponse
from vllm.exceptions import VLLMValidationError
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput
from vllm.tasks import SupportedTask
from vllm.utils.serial_utils import EmbedDType, Endianness
from ..base.io_processor import PoolingIOProcessor
from ..base.serving import PoolingBaseServing
from ..factories import init_pooling_io_processors
from ..typing import AnyPoolingRequest, PoolingServeContext
from ..utils import (
BytesEncodingFormat,
JsonEncodingFormat,
build_pooling_bytes_streaming_response,
get_json_response_cls,
get_pooling_output_encoder,
get_pooling_usage,
)
from .protocol import (
IOProcessorRequest,
PoolingRequest,
PoolingResponse,
PoolingResponseData,
)
logger = init_logger(__name__)
class ServingPooling(PoolingBaseServing):
request_id_prefix = "pooling"
def __init__(
self,
*args,
supported_tasks: tuple[SupportedTask, ...],
**kwargs,
):
super().__init__(*args, **kwargs)
self.supported_tasks = supported_tasks
self.pooling_task = self.model_config.get_pooling_task(supported_tasks)
self.io_processors = init_pooling_io_processors(
supported_tasks=supported_tasks,
vllm_config=self.vllm_config,
renderer=self.renderer,
chat_template_config=self.chat_template_config,
)
self.json_response_cls = get_json_response_cls()
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
request = cast(PoolingRequest, request)
pooling_task = self._verify_pooling_task(request)
return self.io_processors[pooling_task]
def _verify_pooling_task(self, request: PoolingRequest) -> str:
if getattr(request, "dimensions", None) is not None:
raise VLLMValidationError(
"dimensions is currently not supported", parameter="dimensions"
)
if request.task is None:
request.task = self.pooling_task
if isinstance(request, IOProcessorRequest):
request.task = "plugin"
assert request.task is not None
pooling_task = request.task
# plugin task uses io_processor.parse_data to verify inputs
if pooling_task != "plugin" and pooling_task != self.pooling_task:
if pooling_task not in self.supported_tasks:
raise VLLMValidationError(
f"Unsupported task: {pooling_task!r} "
f"Supported tasks: {self.supported_tasks}",
parameter="task",
)
else:
raise VLLMValidationError(
"Try switching the model's pooling_task "
f"via --pooler-config.task {request.task}.",
parameter="task",
)
if pooling_task == "plugin" and "plugin" not in self.io_processors:
raise VLLMValidationError(
"No IOProcessor plugin installed. Please refer "
"to the documentation and to the "
"'prithvi_geospatial_mae_io_processor' "
"offline inference example for more details.",
parameter="task",
)
return pooling_task
def _build_response(
self,
ctx: PoolingServeContext,
) -> Response:
if ctx.response is not None:
# for IOProcessorResponse
return self.json_response_cls(content=ctx.response.model_dump())
encoding_format = ctx.request.encoding_format
embed_dtype = ctx.request.embed_dtype
endianness = ctx.request.endianness
if encoding_format == "float" or encoding_format == "base64":
return self.request_output_to_pooling_json_response(
ctx.final_res_batch,
ctx.request_id,
ctx.created_time,
ctx.model_name,
encoding_format,
embed_dtype,
endianness,
)
if encoding_format == "bytes" or encoding_format == "bytes_only":
return self.request_output_to_pooling_bytes_response(
ctx.final_res_batch,
ctx.request_id,
ctx.created_time,
ctx.model_name,
encoding_format,
embed_dtype,
endianness,
)
assert_never(encoding_format)
def request_output_to_pooling_json_response(
self,
final_res_batch: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
encoding_format: JsonEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> JSONResponse:
encode_fn = get_pooling_output_encoder(
encoding_format=encoding_format,
embed_dtype=embed_dtype,
endianness=endianness,
)
items: list[PoolingResponseData] = []
for idx, final_res in enumerate(final_res_batch):
item = PoolingResponseData(
index=idx,
data=encode_fn(final_res),
)
items.append(item)
response = PoolingResponse(
id=request_id,
created=created_time,
model=model_name,
data=items,
usage=get_pooling_usage(final_res_batch),
)
return self.json_response_cls(content=response.model_dump())
def request_output_to_pooling_bytes_response(
self,
final_res_batch: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
encoding_format: BytesEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> StreamingResponse:
return build_pooling_bytes_streaming_response(
pooling_outputs=final_res_batch,
request_id=request_id,
created_time=created_time,
model_name=model_name,
encoding_format=encoding_format,
embed_dtype=embed_dtype,
endianness=endianness,
)
@@ -1,111 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, Request
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from vllm.logger import init_logger
from .protocol import RerankRequest, ScoreRequest
from .serving import ServingScores
router = APIRouter()
logger = init_logger(__name__)
def score(request: Request) -> ServingScores:
handler = getattr(request.app.state, "serving_scores", None)
if handler is None:
raise NotImplementedError("The model does not support Score API")
return handler
def rerank(request: Request) -> ServingScores:
handler = getattr(request.app.state, "serving_scores", None)
if handler is None:
raise NotImplementedError("The model does not support Rerank (Score) API")
return handler
@router.post(
"/score",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_score(request: ScoreRequest, raw_request: Request):
handler = score(raw_request)
return await handler(request, raw_request)
@router.post(
"/v1/score",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def create_score_v1(request: ScoreRequest, raw_request: Request):
logger.warning(
"To indicate that Score API is not part of standard OpenAI API, we "
"have moved it to `/score`. Please update your client accordingly."
)
return await create_score(request, raw_request)
@router.post(
"/rerank",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def do_rerank(request: RerankRequest, raw_request: Request):
handler = rerank(raw_request)
return await handler(request, raw_request)
@router.post(
"/v1/rerank",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def do_rerank_v1(request: RerankRequest, raw_request: Request):
logger.warning_once(
"To indicate that the rerank API is not part of the standard OpenAI"
" API, we have located it at `/rerank`. Please update your client "
"accordingly. (Note: Conforms to JinaAI rerank API)"
)
return await do_rerank(request, raw_request)
@router.post(
"/v2/rerank",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
},
)
async def do_rerank_v2(request: RerankRequest, raw_request: Request):
return await do_rerank(request, raw_request)
@@ -1,978 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import Sequence
from typing import Any, TypeAlias, cast
import torch.nn.functional as F
from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
from vllm.exceptions import ChatTemplateResolutionError
from vllm.logger import init_logger
from vllm.renderers import TokenizeParams
from vllm.renderers.hf import safe_apply_chat_template
from vllm.renderers.inputs.preprocess import (
extract_target_prompt,
parse_model_prompt,
prompt_to_seq,
)
from vllm.tasks import PoolingTask
from vllm.utils.mistral import is_mistral_tokenizer
from ..base.io_processor import PoolingIOProcessor
from ..pooling.protocol import PoolingCompletionRequest
from ..typing import (
AnyOfflineInputsContext,
AnyPoolingRequest,
AnyRenderParam,
EncodeChatRenderParams,
EncodeCMPLRenderParams,
OfflineEncodeInputsContext,
OfflineOutputsContext,
OfflineScoringInputsContext,
PoolingEngineInput,
PoolingServeContext,
RequestFactory,
RequestGenerator,
ScoringRenderParams,
)
from .protocol import RerankRequest, ScoreRequest, ScoringRequest
from .typing import ScoreData, ScoreInput, ScoringData
from .utils import (
compress_token_type_ids,
compute_maxsim_score,
get_num_special_tokens_for_pair,
parse_score_data,
score_data_to_prompts,
truncate_text_to_tokens,
validate_score_input,
)
logger = init_logger(__name__)
ScoringServeContext: TypeAlias = PoolingServeContext[ScoringRequest]
def _apply_post_tokenization_to_token_type_ids(
tokenizer: Any,
tok_params: TokenizeParams,
token_type_ids: list[int],
) -> list[int]:
# Must stay in the same order as `TokenizeParams._validate_tokens`, which
# truncates before padding. These ids are parallel to the prompt tokens
# and locate the query/document boundary for the cross-encoder; applying
# the two steps in a different order silently misplaces that boundary.
max_length = tok_params.truncate_prompt_tokens
if max_length is not None and max_length < 0:
max_length = tok_params.max_input_tokens
if max_length is not None and max_length < len(token_type_ids):
if max_length == 0:
token_type_ids = token_type_ids[:0]
else:
side = tok_params.truncation_side or (
tokenizer.truncation_side if tokenizer is not None else None
)
token_type_ids = (
token_type_ids[-max_length:]
if side == "left"
else token_type_ids[:max_length]
)
pad_length = tok_params.pad_prompt_tokens
if pad_length is not None and pad_length < 0:
pad_length = tok_params.max_input_tokens
if pad_length is not None and pad_length > len(token_type_ids):
pad_token_type_id = getattr(tokenizer, "pad_token_type_id", 0) or 0
token_type_ids = token_type_ids + [pad_token_type_id] * (
pad_length - len(token_type_ids)
)
return token_type_ids
class ScoringIOProcessor(PoolingIOProcessor):
name: str
pooling_task: PoolingTask
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.tokenizer = self.renderer.get_tokenizer()
self.architecture = self.model_config.architecture
self.is_multimodal_model = self.model_config.is_multimodal_model
self.pad_token_id = self.tokenizer.pad_token_id
def create_pooling_params(self, request):
return request.to_pooling_params(self.pooling_task)
def _validate_token_limit(self, value: int, name: str) -> None:
if value < 0:
raise ValueError(f"{name} must be a non-negative integer")
if value >= self.model_config.max_model_len:
raise ValueError(
f"{name} ({value}) must be less "
f"than max_model_len ({self.model_config.max_model_len})."
)
def _get_token_limits(
self,
request: ScoringRequest | None = None,
pooling_params: PoolingParams | None = None,
) -> tuple[int, int]:
"""Extract and validate token limits from request or pooling_params."""
if request is not None:
max_tokens_per_query = getattr(request, "max_tokens_per_query", 0)
max_tokens_per_doc = getattr(request, "max_tokens_per_doc", 0)
else:
extra = (
(pooling_params.extra_kwargs or {})
if pooling_params is not None
else {}
)
max_tokens_per_query = extra.get("max_tokens_per_query", 0)
max_tokens_per_doc = extra.get("max_tokens_per_doc", 0)
if max_tokens_per_query != 0:
self._validate_token_limit(max_tokens_per_query, "max_tokens_per_query")
if max_tokens_per_doc != 0:
self._validate_token_limit(max_tokens_per_doc, "max_tokens_per_doc")
return max_tokens_per_query, max_tokens_per_doc
def _truncate_scoring_data(
self,
scoring_data: ScoringData,
max_tokens_per_query: int = 0,
max_tokens_per_doc: int = 0,
) -> ScoringData:
"""Truncate query/document texts to token limits."""
data_1 = scoring_data.data_1
data_2 = scoring_data.data_2
if max_tokens_per_query > 0:
data_1 = [
truncate_text_to_tokens(d, self.tokenizer, max_tokens_per_query)
if isinstance(d, str)
else d
for d in data_1
]
if max_tokens_per_doc > 0:
data_2 = [
truncate_text_to_tokens(d, self.tokenizer, max_tokens_per_doc)
if isinstance(d, str)
else d
for d in data_2
]
return ScoringData(data_1=data_1, data_2=data_2)
def valid_inputs(
self,
data_1: ScoreInput | list[ScoreInput],
data_2: ScoreInput | list[ScoreInput],
) -> ScoringData:
scoring_data = validate_score_input(
data_1,
data_2,
is_multimodal_model=self.is_multimodal_model,
architecture=self.architecture,
)
return scoring_data
def valid_inputs_online(self, request: AnyPoolingRequest):
if isinstance(request, ScoreRequest):
data_1 = request.data_1
data_2 = request.data_2
elif isinstance(request, RerankRequest):
data_1 = request.query
data_2 = request.documents
else:
raise ValueError(f"Invalid {request.__class__.__name__} request type")
scoring_data = self.valid_inputs(data_1, data_2)
return scoring_data
def _pair_error_output(
self,
output_1: PoolingRequestOutput,
output_2: PoolingRequestOutput,
) -> PoolingRequestOutput | None:
failed = output_1 if output_1.error is not None else output_2
if failed.error is None:
return None
padding = [self.pad_token_id] if self.pad_token_id is not None else []
return PoolingRequestOutput(
request_id=f"{output_1.request_id}_{output_2.request_id}",
outputs=failed.outputs,
prompt_token_ids=(
output_1.prompt_token_ids + padding + output_2.prompt_token_ids
),
num_cached_tokens=(output_1.num_cached_tokens + output_2.num_cached_tokens),
finished=failed.finished,
error=failed.error,
)
class BiEncoderIOProcessor(ScoringIOProcessor):
name = "bi-encoder"
pooling_task: PoolingTask = "embed"
#######################################
# online APIs
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
scoring_data = self.valid_inputs_online(request)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
request=request
)
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
scoring_data = self._truncate_scoring_data(
scoring_data, max_tokens_per_query, max_tokens_per_doc
)
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
data_2 = score_data_to_prompts(
scoring_data.data_2, "document", self.model_config
)
prompts = data_1 + data_2
ctx.n_queries = len(data_1)
prompts_seq = prompt_to_seq(prompts)
parsed_prompts = [
parse_model_prompt(self.model_config, prompt) for prompt in prompts_seq
]
num_requests = len(parsed_prompts)
tok_params = request.build_tok_params(self.model_config)
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
EncodeCMPLRenderParams(
prompts=parsed_prompts[i],
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=params_seq[i],
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
def post_process_online(
self,
ctx: ScoringServeContext,
):
assert ctx.final_res_batch is not None
assert isinstance(ctx.n_queries, int)
ctx.final_res_batch = self._post_process(
outputs=ctx.final_res_batch, n_queries=ctx.n_queries
)
#######################################
# offline APIs
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=ctx.pooling_params
)
scoring_data = ctx.scoring_data
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
scoring_data = self._truncate_scoring_data(
scoring_data, max_tokens_per_query, max_tokens_per_doc
)
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
data_2 = score_data_to_prompts(
scoring_data.data_2, "document", self.model_config
)
prompts = data_1 + data_2
return super().get_request_factory_offline(
OfflineEncodeInputsContext(
pooling_task=self.pooling_task,
prompts=prompts,
tokenization_kwargs=ctx.tokenization_kwargs,
pooling_params=ctx.pooling_params,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
)
)
def post_process_offline(
self,
ctx: OfflineOutputsContext,
) -> list[PoolingRequestOutput]:
assert ctx.n_queries is not None
return self._post_process(outputs=ctx.outputs, n_queries=ctx.n_queries)
#######################################
# helpers
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
emb_data_1 = outputs[:n_queries]
emb_data_2 = outputs[n_queries:]
if len(emb_data_1) == 1:
emb_data_1 = emb_data_1 * len(emb_data_2)
final_res_batch: list[PoolingRequestOutput] = []
for emb_1, emb_2 in zip(emb_data_1, emb_data_2):
if error_output := self._pair_error_output(emb_1, emb_2):
final_res_batch.append(error_output)
continue
pair_score = F.cosine_similarity(
emb_1.outputs.data.float(), emb_2.outputs.data.float(), dim=0
)
padding: list[int] = []
if self.pad_token_id is not None:
padding = [self.pad_token_id]
tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids
final_res_batch.append(
PoolingRequestOutput(
request_id=f"{emb_1.request_id}_{emb_2.request_id}",
outputs=pair_score,
prompt_token_ids=tokens,
num_cached_tokens=emb_1.num_cached_tokens + emb_2.num_cached_tokens,
finished=True,
)
)
return final_res_batch
class LateInteractionIOProcessor(BiEncoderIOProcessor):
name = "late-interaction"
pooling_task: PoolingTask = "token_embed"
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
# Split into query and document embeddings
emb_data_1 = outputs[:n_queries]
emb_data_2 = outputs[n_queries:]
# Expand queries if 1:N scoring
if len(emb_data_1) == 1:
emb_data_1 = emb_data_1 * len(emb_data_2)
final_res_batch: list[PoolingRequestOutput] = []
padding: list[int] = []
if (pad_token_id := self.pad_token_id) is not None:
padding = [pad_token_id]
# Compute MaxSim scores
for emb_1, emb_2 in zip(emb_data_1, emb_data_2):
if error_output := self._pair_error_output(emb_1, emb_2):
final_res_batch.append(error_output)
continue
# emb_1.outputs.data: [query_len, dim]
# emb_2.outputs.data: [doc_len, dim]
q_emb = emb_1.outputs.data
d_emb = emb_2.outputs.data
maxsim_score = compute_maxsim_score(q_emb, d_emb)
tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids
final_res_batch.append(
PoolingRequestOutput(
request_id=f"{emb_1.request_id}_{emb_2.request_id}",
outputs=maxsim_score,
prompt_token_ids=tokens,
num_cached_tokens=emb_1.num_cached_tokens + emb_2.num_cached_tokens,
finished=True,
)
)
return final_res_batch
class FlashLateInteractionIOProcessor(LateInteractionIOProcessor):
name = "flash-late-interaction"
def post_process_online(
self,
ctx: ScoringServeContext,
):
assert ctx.query_final_res_batch is not None
assert ctx.final_res_batch is not None
assert isinstance(ctx.n_queries, int)
# Expand queries if 1:N scoring
if len(ctx.query_final_res_batch) == 1:
ctx.query_final_res_batch = ctx.query_final_res_batch * len(
ctx.final_res_batch
)
final_res_batch: list[PoolingRequestOutput] = []
for d1, d2 in zip(ctx.query_final_res_batch, ctx.final_res_batch):
padding: list[int] = []
if (pad_token_id := self.pad_token_id) is not None:
padding = [pad_token_id]
tokens = d1.prompt_token_ids + padding + d2.prompt_token_ids
final_res_batch.append(
PoolingRequestOutput(
request_id=f"{d1.request_id}_{d2.request_id}",
outputs=d2.outputs,
prompt_token_ids=tokens,
num_cached_tokens=d1.num_cached_tokens + d2.num_cached_tokens,
finished=True,
)
)
ctx.final_res_batch = final_res_batch
class CrossEncoderIOProcessor(ScoringIOProcessor):
name = "cross-encoder"
pooling_task: PoolingTask = "classify"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if is_mistral_tokenizer(self.tokenizer):
raise ValueError("MistralTokenizer not supported for cross-encoding")
from vllm.model_executor.model_loader import get_model_cls
from vllm.model_executor.models.interfaces import supports_score_template
model = get_model_cls(self.model_config)
self.supports_score_template = supports_score_template(model)
self.model = model if self.supports_score_template else None
self.use_sep_token = self.model_config.use_sep_token
if (
getattr(self.model_config.hf_config, "is_original_qwen3_reranker", False)
and self.chat_template is None
):
suggested_template = (
"examples/pooling/score/template/qwen3_vl_reranker.jinja"
if self.model_config.is_multimodal_model
else "examples/pooling/score/template/qwen3_reranker.jinja"
)
logger.warning(
"Serving an original Qwen3 reranker (%s) without a "
"--chat-template. The model was trained with a specific prompt "
"template; running without it may produce inaccurate relevance "
"scores. Consider specifying `--chat-template %s`.",
self.model_config.model,
suggested_template,
)
#######################################
# online APIs
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
scoring_data = self.valid_inputs_online(request)
data_1 = scoring_data.data_1
data_2 = scoring_data.data_2
num_requests = len(data_2)
if len(data_1) == 1:
data_1 = data_1 * num_requests
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
request=request
)
tok_params = request.build_tok_params(self.model_config)
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
requests = [
ScoringRenderParams(
data_1=data_1[i],
data_2=data_2[i],
chat_template=self.chat_template,
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
tok_params=tok_params,
prompt_extras=ctx.prompt_extras,
skip_mm_cache=False,
params=ctx.pooling_params,
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
for i in range(num_requests)
]
return requests
#######################################
# offline APIs
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
data_1 = ctx.scoring_data.data_1
data_2 = ctx.scoring_data.data_2
num_requests = len(data_2)
if len(data_1) == 1:
data_1 = data_1 * num_requests
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=ctx.pooling_params
)
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
**(ctx.tokenization_kwargs or {})
)
prompt_extras = ctx.pooling_params.extra_kwargs
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
def request_factory() -> RequestGenerator:
for i in range(num_requests):
yield ScoringRenderParams(
data_1=data_1[i],
data_2=data_2[i],
chat_template=(
self.chat_template
if ctx.chat_template is None
else ctx.chat_template
),
max_tokens_per_query=max_tokens_per_query,
max_tokens_per_doc=max_tokens_per_doc,
tok_params=tok_params,
prompt_extras=prompt_extras,
skip_mm_cache=False,
params=ctx.pooling_params,
lora_requests=seq_lora_requests[i],
priorities=seq_priority[i],
)
return request_factory, num_requests
#######################################
# helpers
def render(
self,
render_params: EncodeCMPLRenderParams
| EncodeChatRenderParams
| ScoringRenderParams,
) -> PoolingEngineInput:
if "data_1" not in render_params:
raise ValueError(
f"Unsupported render_params type {render_params.__class__.__name__}"
)
render_params = cast(ScoringRenderParams, render_params)
arrival_time = time.time()
tok_params = render_params["tok_params"]
params = render_params["params"]
prompt_extras = render_params["prompt_extras"]
_, engine_prompt = self.get_score_prompt(
data_1=render_params["data_1"],
data_2=render_params["data_2"],
encode_kwargs=tok_params.get_encode_kwargs(),
chat_template=render_params["chat_template"],
max_tokens_per_query=render_params["max_tokens_per_query"],
max_tokens_per_doc=render_params["max_tokens_per_doc"],
chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
if prompt_extras
else None,
)
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
if token_type_ids := engine_prompt.pop("token_type_ids", None):
params = params.clone()
compressed = compress_token_type_ids(
_apply_post_tokenization_to_token_type_ids(
self.tokenizer, tok_params, token_type_ids
)
)
params.extra_kwargs = {
**(params.extra_kwargs or {}),
"compressed_token_type_ids": compressed,
}
engine_prompt_extras = (
{
k: v
for k in ("mm_processor_kwargs", "cache_salt")
if (v := prompt_extras.get(k)) is not None
}
if prompt_extras
else None
)
if engine_prompt_extras:
target_prompt = extract_target_prompt(self.model_config, engine_prompt)
target_prompt.update(engine_prompt_extras)
engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time)
return PoolingEngineInput(
prompts=engine_input,
params=params,
lora_requests=render_params["lora_requests"],
priorities=render_params["priorities"],
)
def get_score_prompt(
self,
data_1: ScoreData,
data_2: ScoreData,
encode_kwargs: dict[str, Any],
chat_template: str | None = None,
max_tokens_per_query: int = 0,
max_tokens_per_doc: int = 0,
chat_template_kwargs: dict[str, Any] | None = None,
):
model_config = self.model_config
tokenizer = self.tokenizer
prompt_1, prompt_2, mm_data, mm_uuids = parse_score_data(
data_1,
data_2,
model_config,
)
# Limit the query and document separately before composing them.
if max_tokens_per_query > 0 and isinstance(prompt_1, str):
prompt_1 = truncate_text_to_tokens(
prompt_1, tokenizer, max_tokens_per_query
)
if max_tokens_per_doc > 0 and isinstance(prompt_2, str):
prompt_2 = truncate_text_to_tokens(prompt_2, tokenizer, max_tokens_per_doc)
def default_tokenizer_encode():
local_kwargs = encode_kwargs.copy()
if self.supports_score_template:
assert self.model is not None
full_prompt = self.model.get_score_template(prompt_1, prompt_2)
if full_prompt is None:
raise ValueError("Get empty score template from model")
prompt_inputs = tokenizer(full_prompt, **local_kwargs)
else:
if self.use_sep_token:
# cross_encoder models defaults to using separating token.
if max_tokens_per_doc > 0 and isinstance(prompt_2, str):
query_tokens = tokenizer.encode(
prompt_1, add_special_tokens=False
)
num_special = get_num_special_tokens_for_pair(tokenizer)
doc_limit_max_length = (
len(query_tokens) + max_tokens_per_doc + num_special
)
existing_max_length = local_kwargs.get("max_length")
if existing_max_length is not None:
effective_max_length = min(
doc_limit_max_length, existing_max_length
)
else:
effective_max_length = doc_limit_max_length
local_kwargs["truncation"] = "only_second"
local_kwargs["max_length"] = effective_max_length
prompt_inputs = tokenizer(
text=prompt_1, text_pair=prompt_2, **local_kwargs
)
full_prompt = tokenizer.decode(prompt_inputs["input_ids"])
else:
# `llm as reranker` defaults to not using separating token.
full_prompt = prompt_1 + prompt_2
prompt_inputs = tokenizer(text=full_prompt, **local_kwargs)
return full_prompt, prompt_inputs
# FIXME: For now, we only apply a template when one is explicitly provided.
# We cannot rely on the tokenizer's chat template because many models
# inherit junk templates from their base LLM, which breaks both the models
# and the tests that use them.
if chat_template is None:
full_prompt, prompt_inputs = default_tokenizer_encode()
else:
# FIXME:
# Try applying a score template from the CLI arg or tokenizer_config.json
# If that fails because there is no such template,
# fall back to the default implementation.
try:
_safe_kwargs = chat_template_kwargs or {}
_reserved = {"chat_template", "tools", "tokenize"}
_unexpected = _reserved & _safe_kwargs.keys()
if _unexpected:
raise ValueError(
"chat_template_kwargs contains reserved keys that "
f"conflict with fixed scorer arguments: {_unexpected}"
)
full_prompt = safe_apply_chat_template(
model_config,
tokenizer,
[
{"role": "query", "content": prompt_1},
{"role": "document", "content": prompt_2},
],
chat_template=chat_template,
tools=None,
tokenize=False,
**_safe_kwargs,
)
prompt_inputs = tokenizer(full_prompt, **encode_kwargs)
except ChatTemplateResolutionError:
full_prompt, prompt_inputs = default_tokenizer_encode()
engine_prompt = TokensPrompt(prompt_token_ids=prompt_inputs["input_ids"])
if (token_type_ids := prompt_inputs.get("token_type_ids")) is not None:
engine_prompt["token_type_ids"] = token_type_ids
if self.model is not None:
self.model.post_process_tokens(engine_prompt)
if mm_data is not None:
engine_prompt["multi_modal_data"] = mm_data
if mm_uuids is not None:
engine_prompt["multi_modal_uuids"] = mm_uuids
return full_prompt, engine_prompt
class JinaRankingIOProcessorMixin:
@staticmethod
def format_docs_prompts_func(
query: str,
docs: list[str],
special_tokens: dict[str, str] | None = None,
instruction: str | None = None,
no_thinking: bool = True,
) -> str:
# TODO: Try converting the code below into a chat template.
default_special_tokens = {
"query_embed_token": "<|rerank_token|>",
"doc_embed_token": "<|embed_token|>",
}
if special_tokens is None:
special_tokens = default_special_tokens
def sanitize_input(text: str) -> str:
for token in special_tokens.values():
text = text.replace(token, "")
return text
query = sanitize_input(query)
docs = [sanitize_input(doc) for doc in docs]
prefix = (
"<|im_start|>system\n"
"You are a search relevance expert who can determine a ranking of the passages based on how relevant they are to the query. " # noqa: E501
"If the query is a question, how relevant a passage is depends on how well it answers the question. " # noqa: E501
"If not, try to analyze the intent of the query and assess how well each passage satisfies the intent. " # noqa: E501
"If an instruction is provided, you should follow the instruction when determining the ranking." # noqa: E501
"<|im_end|>\n<|im_start|>user\n"
)
suffix = "<|im_end|>\n<|im_start|>assistant\n"
if no_thinking:
suffix += "<think>\n\n</think>\n\n"
doc_emb_token = special_tokens["doc_embed_token"]
query_emb_token = special_tokens["query_embed_token"]
prompt = (
f"I will provide you with {len(docs)} passages, each indicated by a numerical identifier. " # noqa: E501
f"Rank the passages based on their relevance to query: {query}\n"
)
if instruction:
instruction = sanitize_input(instruction)
prompt += f"<instruct>\n{instruction}\n</instruct>\n"
doc_prompts = [
f'<passage id="{i}">\n{doc}{doc_emb_token}\n</passage>'
for i, doc in enumerate(docs)
]
prompt += "\n".join(doc_prompts) + "\n"
prompt += f"<query>\n{query}{query_emb_token}\n</query>"
return prefix + prompt + suffix
@staticmethod
def ensure_str(data: Sequence[Any]) -> list[str]:
text: list[str] = []
for prompt in data:
if not isinstance(prompt, str):
raise ValueError(
"The JinaForRanking model only supports text as input."
)
text.append(prompt)
return text
class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorMixin):
name = "jina-reranking-scoring"
pooling_task: PoolingTask = "token_embed"
def get_request_factory_online(
self, ctx: PoolingServeContext
) -> Sequence[AnyRenderParam]:
request = ctx.request
ctx.n_queries = 1
prompt_extras = ctx.prompt_extras
scoring_data = self.valid_inputs_online(request)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
request=request
)
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
scoring_data = self._truncate_scoring_data(
scoring_data, max_tokens_per_query, max_tokens_per_doc
)
queries = self.ensure_str(scoring_data.data_1)
docs = self.ensure_str(scoring_data.data_2)
chat_template_kwargs = (
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
)
instruction = (
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
)
if len(queries) == 1:
prompts = [
self.format_docs_prompts_func(
query=queries[0], docs=docs, instruction=instruction
)
]
else:
prompts = [
self.format_docs_prompts_func(
query=q, docs=[d], instruction=instruction
)
for q, d in zip(queries, docs)
]
# Forward truncation from the real request: the base factory reads
# these off ctx.request, so omitting them here silently drops
# truncate_prompt_tokens for Jina rerank/score (unlike the embed and
# bi/cross-encoder paths, which read them from the real request).
ctx.request = PoolingCompletionRequest(
task="token_embed",
input=prompts,
truncate_prompt_tokens=request.truncate_prompt_tokens,
truncation_side=request.truncation_side,
)
requests = PoolingIOProcessor.get_request_factory_online(self, ctx)
ctx.request = request
return requests
def get_request_factory_offline(
self, ctx: AnyOfflineInputsContext
) -> tuple[RequestFactory, int]:
assert isinstance(ctx, OfflineScoringInputsContext)
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
pooling_params=ctx.pooling_params
)
scoring_data = ctx.scoring_data
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
scoring_data = self._truncate_scoring_data(
scoring_data, max_tokens_per_query, max_tokens_per_doc
)
prompt_extras = ctx.pooling_params.extra_kwargs
queries = self.ensure_str(scoring_data.data_1)
docs = self.ensure_str(scoring_data.data_2)
chat_template_kwargs = (
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
)
instruction = (
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
)
if len(queries) == 1:
prompts = [
self.format_docs_prompts_func(
query=queries[0], docs=docs, instruction=instruction
)
]
else:
prompts = [
self.format_docs_prompts_func(
query=q, docs=[d], instruction=instruction
)
for q, d in zip(queries, docs)
]
return PoolingIOProcessor.get_request_factory_offline(
self,
OfflineEncodeInputsContext(
pooling_task=self.pooling_task,
prompts=prompts,
tokenization_kwargs=ctx.tokenization_kwargs,
pooling_params=ctx.pooling_params,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
),
)
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
final_res_batch: list[PoolingRequestOutput] = []
for i in range(len(outputs)):
if outputs[i].error is not None:
final_res_batch.append(outputs[i])
continue
embeds = outputs[i].outputs.data.float()
# The JinaForRanking model concatenates docs first, then query.
# Let's stay consistent with this novel design.
query_embeds = embeds[-1]
doc_embeds = embeds[:-1]
scores = F.cosine_similarity(query_embeds, doc_embeds)
for score in scores:
final_res_batch.append(
PoolingRequestOutput(
request_id=outputs[i].request_id,
outputs=score,
prompt_token_ids=outputs[i].prompt_token_ids,
num_cached_tokens=outputs[i].num_cached_tokens,
finished=True,
)
)
return final_res_batch
ScoringIOProcessors: dict[str, type[ScoringIOProcessor]] = {
p.name: p
for p in [
BiEncoderIOProcessor,
LateInteractionIOProcessor,
JinaRankingIOProcessor,
FlashLateInteractionIOProcessor,
CrossEncoderIOProcessor,
]
}
@@ -1,187 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from typing import Any, TypeAlias
from pydantic import BaseModel, Field, model_validator
from vllm import PoolingParams
from vllm.config import ModelConfig
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
from vllm.renderers import TokenizeParams
from vllm.tasks import PoolingTask
from vllm.utils import random_uuid
from ..base.protocol import ClassifyRequestMixin, PoolingBasicRequestMixin
from .typing import ScoreContentPartParam, ScoreInput
class ScoringRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
# --8<-- [start:scoring-common-params]
max_tokens_per_query: int = Field(
default=0,
description=(
"Maximum number of tokens per query. Queries longer than "
"this will be truncated to this length. 0 means no "
"query-level truncation is applied."
),
)
max_tokens_per_doc: int = Field(
default=0,
description=(
"Maximum number of tokens per document. Documents longer than "
"this will be truncated to this length. 0 means no "
"document-level truncation is applied (only truncate_prompt_tokens "
"applies to the combined query+document)."
),
)
instruction: str | None = Field(
default=None,
description=(
"Task instruction prepended to each scored pair via the chat "
"template. Equivalent to passing "
"chat_template_kwargs={'instruction': ...}."
),
)
chat_template_kwargs: dict[str, Any] | None = Field(
default=None,
description=(
"Additional keyword args to pass to the chat template renderer. "
"Will be accessible by the score/rerank chat template."
),
)
# --8<-- [end:scoring-common-params]
@model_validator(mode="after")
def _merge_instruction_into_kwargs(self) -> "ScoringRequestMixin":
"""Fold the top-level `instruction` field into `chat_template_kwargs`.
This allows callers to use either the convenience field or the generic
dict. Explicit keys inside `chat_template_kwargs` take precedence over
the top-level `instruction` field.
"""
if self.instruction is not None:
merged = dict(self.chat_template_kwargs or {})
merged.setdefault("instruction", self.instruction)
self.chat_template_kwargs = merged
return self
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
return self._build_pooling_tok_params(
model_config,
add_special_tokens=True,
max_total_tokens=model_config.max_model_len,
max_output_tokens=0,
)
def to_pooling_params(self, task: PoolingTask = "classify"):
return PoolingParams(
task=task,
use_activation=self.use_activation,
)
class ScoreDataRequest(ScoringRequestMixin):
data_1: ScoreInput | list[ScoreInput]
data_2: ScoreInput | list[ScoreInput]
class ScoreQueriesDocumentsRequest(ScoringRequestMixin):
# --8<-- [start:score-request-params]
queries: ScoreInput | list[ScoreInput]
documents: ScoreInput | list[ScoreInput]
# --8<-- [end:score-request-params]
@property
def data_1(self):
return self.queries
@property
def data_2(self):
return self.documents
class ScoreQueriesItemsRequest(ScoringRequestMixin):
queries: ScoreInput | list[ScoreInput]
items: ScoreInput | list[ScoreInput]
@property
def data_1(self):
return self.queries
@property
def data_2(self):
return self.items
class ScoreTextRequest(ScoringRequestMixin):
text_1: ScoreInput | list[ScoreInput]
text_2: ScoreInput | list[ScoreInput]
@property
def data_1(self):
return self.text_1
@property
def data_2(self):
return self.text_2
ScoreRequest: TypeAlias = (
ScoreQueriesDocumentsRequest
| ScoreQueriesItemsRequest
| ScoreDataRequest
| ScoreTextRequest
)
class RerankRequest(ScoringRequestMixin):
# --8<-- [start:rerank-request-params]
query: ScoreInput
documents: ScoreInput | list[ScoreInput]
top_n: int = Field(default=0, ge=0)
# --8<-- [end:rerank-request-params]
ScoringRequest: TypeAlias = ScoreRequest | RerankRequest
class RerankDocument(BaseModel):
text: str | None = None
multi_modal: list[ScoreContentPartParam] | None = None
class RerankResult(BaseModel):
index: int
document: RerankDocument
relevance_score: float
class RerankUsage(BaseModel):
prompt_tokens: int
total_tokens: int
class RerankResponse(OpenAIBaseModel):
id: str
model: str
usage: RerankUsage
results: list[RerankResult]
class ScoreResponseData(OpenAIBaseModel):
index: int
object: str = "score"
score: float
class ScoreResponse(OpenAIBaseModel):
id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
object: str = "list"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
data: list[ScoreResponseData]
usage: UsageInfo
ScoringResponse: TypeAlias = RerankResponse | ScoreResponse
-315
View File
@@ -1,315 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from fastapi.responses import JSONResponse, Response
from vllm import PoolingParams
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.engine.protocol import UsageInfo
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
from vllm.tasks import SCORE_TYPE_MAP, SupportedTask
from vllm.utils import random_uuid
from vllm.v1.pool.late_interaction import (
build_late_interaction_doc_params,
build_late_interaction_query_params,
)
from ..base.io_processor import PoolingIOProcessor
from ..base.serving import PoolingServing
from .io_processor import ScoringIOProcessors, ScoringServeContext
from .protocol import (
RerankDocument,
RerankRequest,
RerankResponse,
RerankResult,
RerankUsage,
ScoreRequest,
ScoreResponse,
ScoreResponseData,
)
from .typing import ScoreInput
logger = init_logger(__name__)
class ServingScores(PoolingServing):
request_id_prefix = "score"
def __init__(
self,
engine_client: EngineClient,
*args,
supported_tasks: tuple[SupportedTask, ...],
enable_flash_late_interaction: bool = True,
**kwargs,
):
pooling_task = engine_client.model_config.get_pooling_task(supported_tasks)
score_type = SCORE_TYPE_MAP.get(pooling_task, None) # type: ignore[arg-type]
assert score_type is not None
self.io_processor_name: str = score_type
self.enable_flash_late_interaction = (
self.io_processor_name == "late-interaction"
and enable_flash_late_interaction
)
if self.enable_flash_late_interaction:
self.io_processor_name = "flash-late-interaction"
if engine_client.model_config.architecture == "JinaForRanking":
self.io_processor_name = "jina-reranking-scoring"
self.enable_flash_late_interaction = False
super().__init__(engine_client, *args, **kwargs)
def init_io_processor(self, *args, **kwargs) -> PoolingIOProcessor:
return ScoringIOProcessors[self.io_processor_name](*args, **kwargs)
async def __call__(self, *args, **kwargs) -> Response:
if not self.enable_flash_late_interaction:
return await super().__call__(*args, **kwargs)
return await self.flash_late_interaction(*args, **kwargs)
def _build_response(
self,
ctx: ScoringServeContext,
) -> JSONResponse:
final_res_batch = ctx.final_res_batch
request_id = ctx.request_id
created_time = ctx.created_time
model_name = ctx.model_name
assert ctx.engine_inputs is not None
num_prompt_tokens = sum(
self._extract_prompt_len(engine_input["prompts"])
for engine_input in ctx.engine_inputs
)
if isinstance(ctx.request, ScoreRequest):
return self._request_output_to_score_response(
final_res_batch,
request_id,
created_time,
model_name,
num_prompt_tokens,
)
elif isinstance(ctx.request, RerankRequest):
return self._request_output_to_rerank_response(
final_res_batch,
request_id,
model_name,
ctx.request.documents,
ctx.request.top_n if ctx.request.top_n > 0 else len(final_res_batch),
num_prompt_tokens,
)
else:
raise ValueError(f"Invalid {self.request_id_prefix} request type")
def _request_output_to_score_response(
self,
final_res_batch: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
num_prompt_tokens: int,
) -> JSONResponse:
items: list[ScoreResponseData] = []
for idx, final_res in enumerate(final_res_batch):
classify_res = ScoringRequestOutput.from_base(final_res)
item = ScoreResponseData(
index=idx,
score=classify_res.outputs.score,
)
items.append(item)
usage = UsageInfo(
prompt_tokens=num_prompt_tokens,
total_tokens=num_prompt_tokens,
)
response = ScoreResponse(
id=request_id,
created=created_time,
model=model_name,
data=items,
usage=usage,
)
return JSONResponse(content=response.model_dump())
def _request_output_to_rerank_response(
self,
final_res_batch: list[PoolingRequestOutput],
request_id: str,
model_name: str,
documents: ScoreInput | list[ScoreInput],
top_n: int,
num_prompt_tokens: int,
) -> JSONResponse:
if not isinstance(documents, list):
documents = [documents]
results: list[RerankResult] = []
for idx, final_res in enumerate(final_res_batch):
classify_res = ScoringRequestOutput.from_base(final_res)
document = documents[idx]
if isinstance(document, str):
rerank_document = RerankDocument(text=document)
else:
rerank_document = RerankDocument(
multi_modal=document.get("content", [])
)
result = RerankResult(
index=idx,
document=rerank_document,
relevance_score=classify_res.outputs.score,
)
results.append(result)
# sort by relevance, then return the top n if set
results.sort(key=lambda x: x.relevance_score, reverse=True)
if top_n < len(documents):
results = results[:top_n]
response = RerankResponse(
id=request_id,
model=model_name,
results=results,
usage=RerankUsage(
total_tokens=num_prompt_tokens, prompt_tokens=num_prompt_tokens
),
)
return JSONResponse(content=response.model_dump())
###################################################################################
### Run pooling score MaxSim on worker side (GPU) in the API server process
### Can significantly improve late-interaction scoring performance.
async def flash_late_interaction(self, *args, **kwargs) -> Response:
ctx = await self._init_ctx(self.io_processor, *args, **kwargs)
await self._preprocessing(self.io_processor, ctx)
try:
# stage 1: encode queries and cache token embeddings on workers.
await self._flash_late_interaction_encode_queries(ctx)
# stage 2: encode docs and return scalar scores from workers.
await self._flash_late_interaction_encode_docs(ctx)
except BaseException:
try:
await self._cleanup_flash_late_interaction(ctx)
except Exception:
logger.exception("Failed to clean up late-interaction query cache.")
raise
return await self._postprocessing_async(self.io_processor, ctx)
async def _cleanup_flash_late_interaction(self, ctx: ScoringServeContext) -> None:
query_keys = ctx.late_interaction_query_keys
if not query_keys:
return
doc_keys = ctx.late_interaction_doc_keys or []
try:
# Stop documents before removing the query tensors they reference.
await self.engine_client.abort([*query_keys, *doc_keys])
finally:
await self.engine_client.collective_rpc(
"release_late_interaction_query_cache",
args=(query_keys,),
)
async def _flash_late_interaction_encode_queries(self, ctx: ScoringServeContext):
assert ctx.n_queries is not None
assert ctx.engine_inputs is not None
assert isinstance(ctx.pooling_params, PoolingParams)
n_queries = ctx.n_queries
n_docs = len(ctx.engine_inputs) - n_queries
query_engine_inputs = ctx.engine_inputs[:n_queries]
query_namespace = random_uuid()
query_keys = [
f"late-interaction-{query_namespace}-query-{i}" for i in range(n_queries)
]
ctx.late_interaction_query_keys = query_keys
query_uses = [n_docs if n_queries == 1 else 1] * n_queries
for i in range(n_queries):
pooling_params = ctx.pooling_params.clone()
pooling_params.late_interaction_params = (
build_late_interaction_query_params(
query_key=query_keys[i],
query_uses=query_uses[i],
)
)
query_engine_inputs[i]["params"] = pooling_params
assert n_queries == len(query_engine_inputs) == len(query_keys)
query_ctx = ScoringServeContext(
request=ctx.request,
raw_request=ctx.raw_request,
model_name=ctx.model_name,
request_id=ctx.request_id,
pooling_params=ctx.pooling_params,
prompt_request_ids=query_keys,
engine_inputs=query_engine_inputs,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
prompt_extras=ctx.prompt_extras,
)
await self._prepare_generators(query_ctx)
await self._collect_batch(query_ctx)
ctx.query_final_res_batch = query_ctx.final_res_batch
async def _flash_late_interaction_encode_docs(self, ctx: ScoringServeContext):
assert ctx.n_queries is not None
assert ctx.engine_inputs is not None
assert isinstance(ctx.pooling_params, PoolingParams)
n_queries = ctx.n_queries
n_docs = len(ctx.engine_inputs) - n_queries
doc_engine_inputs = ctx.engine_inputs[n_queries:]
query_keys = ctx.late_interaction_query_keys
if query_keys is None:
raise RuntimeError("Late-interaction query keys were not initialized.")
doc_namespace = random_uuid()
doc_keys = [f"late-interaction-{doc_namespace}-doc-{i}" for i in range(n_docs)]
ctx.late_interaction_doc_keys = doc_keys
for i in range(n_docs):
query_idx = 0 if n_queries == 1 else i
pooling_params = ctx.pooling_params.clone()
pooling_params.late_interaction_params = build_late_interaction_doc_params(
query_key=query_keys[query_idx]
)
doc_engine_inputs[i]["params"] = pooling_params
assert n_docs == len(doc_engine_inputs) == len(doc_keys)
doc_ctx = ScoringServeContext(
request=ctx.request,
raw_request=ctx.raw_request,
model_name=ctx.model_name,
request_id=ctx.request_id,
pooling_params=ctx.pooling_params,
prompt_request_ids=doc_keys,
engine_inputs=doc_engine_inputs,
lora_request=ctx.lora_request,
priorities=ctx.priorities,
prompt_extras=ctx.prompt_extras,
)
await self._prepare_generators(doc_ctx)
await self._collect_batch(doc_ctx)
ctx.final_res_batch = doc_ctx.final_res_batch
@@ -1,45 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from dataclasses import dataclass
from typing import Required, TypeAlias
from typing_extensions import TypedDict
from vllm.renderers.chat_utils import (
ChatCompletionContentPartImageEmbedsParam,
ChatCompletionContentPartImageParam,
ChatCompletionContentPartTextParam,
ChatCompletionContentPartVideoParam,
)
ScoreContentPartParam: TypeAlias = (
ChatCompletionContentPartImageParam
| ChatCompletionContentPartImageEmbedsParam
| ChatCompletionContentPartTextParam
| ChatCompletionContentPartVideoParam
)
class ScoreMultiModalParam(TypedDict, total=False):
"""A specialized parameter type for scoring multimodal content.
The reasons why don't reuse `CustomChatCompletionMessageParam` directly:
1. Score tasks don't need the 'role' field (user/assistant/system) that's required in chat completions
2. Including chat-specific fields would confuse users about their purpose in scoring
3. This is a more focused interface that only exposes what's needed for scoring
""" # noqa: E501
content: Required[list[ScoreContentPartParam]]
"""The multimodal contents"""
# Raw input data with content key in ScoreMultiModalParam.
ScoreInput = str | ScoreMultiModalParam
# Score data without content key.
ScoreData = str | list[ScoreContentPartParam]
@dataclass
class ScoringData:
data_1: list[ScoreData]
data_2: list[ScoreData]
-291
View File
@@ -1,291 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import cast
import torch
from vllm import PromptType, TextPrompt
from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
from vllm.renderers.chat_utils import (
BaseMultiModalItemTracker,
ChatCompletionContentPartParam,
ChatCompletionContentPartTextParam,
ConversationMessage,
MultiModalItemTracker,
parse_chat_message_content_parts,
)
from .typing import (
ScoreContentPartParam,
ScoreData,
ScoreInput,
ScoringData,
)
def get_num_special_tokens_for_pair(tokenizer) -> int:
"""Get number of special tokens added for a text pair encoding."""
method = getattr(tokenizer, "num_special_tokens_to_add", None)
if method is not None:
try:
return method(pair=True)
except TypeError:
pass
# Fallback: compute by tokenizing empty strings
empty_encoding = tokenizer("", text_pair="", add_special_tokens=True)
return len(empty_encoding["input_ids"])
def truncate_text_to_tokens(
text: str,
tokenizer,
max_tokens: int,
) -> str:
"""Truncate text to a maximum number of content tokens.
Uses offset_mapping to slice the original text at the exact character
boundary, avoiding lossy encode→decode round-trips that can shift
the token count due to BPE merge boundary changes.
"""
if max_tokens <= 0:
return ""
while text:
encoding = tokenizer(
text,
add_special_tokens=False,
return_offsets_mapping=True,
)
offsets = encoding["offset_mapping"]
if len(encoding["input_ids"]) <= max_tokens:
break
char_end = min(offsets[max_tokens - 1][1], offsets[max_tokens][0])
if char_end >= len(text):
char_end = len(text) - 1
text = text[:char_end]
return text
def compute_maxsim_score(q_emb: torch.Tensor, d_emb: torch.Tensor) -> torch.Tensor:
"""Compute ColBERT MaxSim score.
Args:
q_emb: Query token embeddings [query_len, dim]
d_emb: Document token embeddings [doc_len, dim]
Returns:
MaxSim score (sum over query tokens of max similarity to any doc token)
"""
# compute in float32 for numerical stability
# [query_len, doc_len]
token_scores = torch.matmul(q_emb.float(), d_emb.float().T)
# Max over document tokens, sum over query tokens
return token_scores.amax(dim=-1).sum()
def _validate_mm_score_input(
data: list[ScoreInput],
is_multimodal_model: bool,
architecture: str,
) -> list[ScoreData]:
out: list[ScoreData] = []
for d in data:
if isinstance(d, str):
out.append(d)
else:
if not is_multimodal_model:
raise VLLMValidationError(
f"MultiModalParam is not supported for {architecture}"
)
content = cast(list[ScoreContentPartParam], d.get("content", []))
out.append(content)
return out
def _validate_score_input_lens(
data_1: list[ScoreData],
data_2: list[ScoreData],
):
len_1 = len(data_1)
len_2 = len(data_2)
if len_1 > 1 and len_1 != len_2:
raise VLLMValidationError("Input lengths must be either 1:1, 1:N or N:N")
if len_1 == 0:
raise VLLMValidationError("At least one text element must be given")
if len_2 == 0:
raise VLLMValidationError("At least one text_pair element must be given")
def validate_score_input(
data_1: ScoreInput | list[ScoreInput],
data_2: ScoreInput | list[ScoreInput],
is_multimodal_model: bool,
architecture: str,
) -> ScoringData:
if not isinstance(data_1, list):
data_1 = [data_1]
if not isinstance(data_2, list):
data_2 = [data_2]
score_input_1 = _validate_mm_score_input(data_1, is_multimodal_model, architecture)
score_input_2 = _validate_mm_score_input(data_2, is_multimodal_model, architecture)
_validate_score_input_lens(score_input_1, score_input_2)
return ScoringData(data_1=score_input_1, data_2=score_input_2)
def score_data_to_prompts(
data_list: list[ScoreData],
role: str,
model_config: ModelConfig,
) -> list[PromptType]:
"""Convert a list of ScoreData into PromptType objects.
For plain text inputs, returns the string directly.
For multimodal inputs (list of content parts), parses them into
a :class:`TextPrompt` with attached ``multi_modal_data`` /
``multi_modal_uuids``.
This is used by late-interaction scoring where each query/document
is encoded independently.
"""
prompts: list[PromptType] = []
for data in data_list:
if isinstance(data, str):
prompts.append(data)
else:
text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config)
prompt: TextPrompt = TextPrompt(prompt=text)
if mm_data is not None:
prompt["multi_modal_data"] = mm_data
if mm_uuids is not None:
prompt["multi_modal_uuids"] = mm_uuids
prompts.append(prompt)
return prompts
def _ensure_str(content: list[ConversationMessage]) -> str:
"""Extract a single string prompt from parsed conversation content."""
assert len(content) == 1
prompt = content[0]["content"]
if prompt is not None and isinstance(prompt, str):
return cast(str, prompt)
raise ValueError(f"Only string content is supported, but got {content}.")
def _parse_score_content(
role: str,
data: ScoreData,
mm_tracker: BaseMultiModalItemTracker,
) -> list[ConversationMessage]:
parts: Iterable[ChatCompletionContentPartParam]
if isinstance(data, str):
parts = [ChatCompletionContentPartTextParam(type="text", text=data)]
else:
parts = cast(Iterable[ChatCompletionContentPartParam], data)
mm_parser = mm_tracker.create_parser()
parse_res = parse_chat_message_content_parts(
role=role,
parts=parts,
mm_tracker=mm_tracker,
wrap_dicts=False,
interleave_strings=False,
multimodal_content_part_separator="",
)
if parse_res:
return parse_res
mm_placeholder_storage = mm_parser.mm_placeholder_storage()
if (
len(mm_placeholder_storage) != 1
or len(next(iter(mm_placeholder_storage.values()))) != 1
):
raise ValueError("Only one multi-modal item is supported")
return next(iter(mm_placeholder_storage.values()))[0]
def parse_score_data_single(
data: ScoreData,
role: str,
model_config: ModelConfig,
) -> tuple[str, MultiModalDataDict | None, MultiModalUUIDDict | None]:
"""Parse **one** ScoreData into a text prompt and its own multi-modal
data.
Unlike :func:`parse_score_data`, each call creates an **independent**
:class:`MultiModalItemTracker` so multi-modal items are kept separate.
This is the correct behaviour for late-interaction scoring, where
query and document are encoded independently.
"""
mm_tracker = MultiModalItemTracker(model_config)
content = _parse_score_content(role, data, mm_tracker)
prompt = _ensure_str(content)
mm_items, mm_uuids = mm_tracker.resolve_items()
return prompt, mm_items, mm_uuids
def parse_score_data(
data_1: ScoreData,
data_2: ScoreData,
model_config: ModelConfig,
) -> tuple[str, str, MultiModalDataDict | None, MultiModalUUIDDict | None]:
"""Parse a query-document pair into text prompts and shared multi-modal
data.
Uses a **single** :class:`MultiModalItemTracker` so that multi-modal
items from both inputs are merged into one ``mm_data`` dict. This is
the correct behaviour for cross-encoder scoring, where query and
document are concatenated into a single model prompt.
"""
mm_tracker = MultiModalItemTracker(model_config)
content_1 = _parse_score_content("query", data_1, mm_tracker)
content_2 = _parse_score_content("document", data_2, mm_tracker)
prompt_1 = _ensure_str(content_1)
prompt_2 = _ensure_str(content_2)
mm_items, mm_uuids = mm_tracker.resolve_items()
return prompt_1, prompt_2, mm_items, mm_uuids
def compress_token_type_ids(token_type_ids: list[int]) -> tuple[int, int]:
"""Return the half-open range containing token type 1.
Right padding uses the tokenizer's padding token type, which is commonly
zero. Therefore, a padded query-document pair can contain zeros followed
by ones followed by trailing zeros.
"""
first_one = len(token_type_ids)
first_zero_after_one = len(token_type_ids)
err_msg = (
"Token type ids are expected to be a sequence"
" of zeros followed by ones and optional trailing zeros"
)
for i, type_id in enumerate(token_type_ids):
if type_id not in (0, 1):
raise ValueError(err_msg)
if type_id == 1:
if first_zero_after_one < len(token_type_ids):
raise ValueError(err_msg)
if first_one == len(token_type_ids):
first_one = i
elif first_one < len(token_type_ids) and first_zero_after_one == len(
token_type_ids
):
first_zero_after_one = i
return first_one, first_zero_after_one
-214
View File
@@ -1,214 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
from dataclasses import dataclass, field
from typing import Any, Generic, NotRequired, TypeAlias, TypedDict, TypeVar
from fastapi import Request
from pydantic import ConfigDict
from vllm import PoolingParams, PoolingRequestOutput, PromptType
from vllm.inputs import DataPrompt, EngineInput
from vllm.lora.request import LoRARequest
from vllm.outputs import LateChunkingMetadata
from vllm.renderers import ChatParams, TokenizeParams
from vllm.renderers.chat_utils import ChatCompletionMessageParam
from vllm.renderers.inputs import DictPrompt
from ...tasks import PoolingTask
from .classify.protocol import (
ClassificationChatRequest,
ClassificationCompletionRequest,
ClassificationResponse,
)
from .embed.protocol import (
CohereEmbedRequest,
EmbeddingBytesResponse,
EmbeddingChatInputRequest,
EmbeddingChatRequest,
EmbeddingCompletionRequest,
EmbeddingRequest,
EmbeddingResponse,
)
from .pooling.protocol import (
IOProcessorRequest,
PoolingBytesResponse,
PoolingChatRequest,
PoolingCompletionRequest,
PoolingResponse,
)
from .scoring.protocol import ScoringRequest, ScoringResponse
from .scoring.typing import ScoreData, ScoringData
PoolingCompletionLikeRequest: TypeAlias = (
EmbeddingCompletionRequest
| ClassificationCompletionRequest
| PoolingCompletionRequest
)
PoolingChatLikeRequest: TypeAlias = (
EmbeddingChatRequest
| EmbeddingChatInputRequest
| ClassificationChatRequest
| PoolingChatRequest
)
AnyPoolingRequest: TypeAlias = (
EmbeddingRequest
| PoolingCompletionLikeRequest
| PoolingChatLikeRequest
| IOProcessorRequest
| ScoringRequest
| CohereEmbedRequest
)
AnyPoolingResponse: TypeAlias = (
ClassificationResponse
| EmbeddingResponse
| EmbeddingBytesResponse
| PoolingResponse
| PoolingBytesResponse
| ScoringResponse
)
PoolingRequestT = TypeVar("PoolingRequestT", bound=AnyPoolingRequest)
@dataclass(kw_only=True)
class ChunkedEmbeddingMetadata:
prompt_index: int
chunk_index: int
@dataclass
class LateChunkingSlot:
"""Per-document source ranges shared by frontend rendering and output contexts."""
metadata: LateChunkingMetadata | None = None
@dataclass(kw_only=True)
class PoolingServeContext(Generic[PoolingRequestT]):
model_config = ConfigDict(arbitrary_types_allowed=True)
request: PoolingRequestT
raw_request: Request | None = None
model_name: str
request_id: str
pooling_params: PoolingParams
lora_request: LoRARequest | None
priorities: int | Sequence[int] | None
prompt_extras: dict[str, Any] | None
created_time: int = field(default_factory=lambda: int(time.time()))
engine_inputs: Sequence["PoolingEngineInput"] | None = None
prompt_request_ids: list[str] | None = None
result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (
None
)
final_res_batch: list[PoolingRequestOutput] = field(default_factory=list)
## for Long Text Embedding with Chunked Processing
original_engine_inputs: Sequence["PoolingEngineInput"] | None = None
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] | None = None
## for bi-encoder & late-interaction
n_queries: int | None = None
## for IOProcessorResponse
response: Any | None = None
## for flash-late-interaction
query_final_res_batch: list[PoolingRequestOutput] | None = None
late_interaction_query_keys: list[str] | None = None
late_interaction_doc_keys: list[str] | None = None
@dataclass
class OfflineInputsContext:
pooling_task: PoolingTask
tokenization_kwargs: dict[str, Any] | None
lora_request: Sequence[LoRARequest | None] | None
priorities: int | Sequence[int] | None
late_chunking: list[LateChunkingSlot | None] = field(
default_factory=list, kw_only=True
)
@dataclass
class OfflineEncodeInputsContext(OfflineInputsContext):
prompts: PromptType | Sequence[PromptType]
pooling_params: PoolingParams | Sequence[PoolingParams] | None
@dataclass
class OfflineScoringInputsContext(OfflineInputsContext):
scoring_data: ScoringData
chat_template: str | None
pooling_params: PoolingParams
@dataclass
class OfflinePluginInputsContext(OfflineInputsContext):
prompts: DataPrompt
pooling_params: PoolingParams | Sequence[PoolingParams] | None
AnyOfflineInputsContext: TypeAlias = (
OfflineEncodeInputsContext
| OfflineScoringInputsContext
| OfflinePluginInputsContext
)
@dataclass
class OfflineOutputsContext:
outputs: list[PoolingRequestOutput]
## for bi-encoder & late-interaction
n_queries: int | None = None
late_chunking: list[LateChunkingSlot | None] = field(default_factory=list)
class RenderParams(TypedDict):
tok_params: TokenizeParams
prompt_extras: dict[str, Any] | None
skip_mm_cache: bool
params: PoolingParams
lora_requests: LoRARequest | None
priorities: int
late_chunking: NotRequired[LateChunkingSlot]
class EncodeCMPLRenderParams(RenderParams):
prompts: DictPrompt
class EncodeChatRenderParams(RenderParams):
conversations: list["ChatCompletionMessageParam"]
chat_params: ChatParams
class ScoringRenderParams(RenderParams):
data_1: ScoreData
data_2: ScoreData
chat_template: str | None
max_tokens_per_query: int
max_tokens_per_doc: int
AnyRenderParam: TypeAlias = (
EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams
)
RequestGenerator: TypeAlias = Generator[AnyRenderParam]
RequestFactory: TypeAlias = Callable[[], RequestGenerator]
class PoolingEngineInput(TypedDict):
prompts: EngineInput
params: PoolingParams
lora_requests: LoRARequest | None
priorities: int
-250
View File
@@ -1,250 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
import json
import math
from collections.abc import Callable, Sequence
from dataclasses import dataclass
from functools import lru_cache, partial
from typing import Any, Literal, cast
import pybase64
import torch
from fastapi.responses import JSONResponse, StreamingResponse
from vllm.config import ModelConfig
from vllm.entrypoints.serve.engine.protocol import UsageInfo
from vllm.logger import init_logger
from vllm.outputs import PoolingRequestOutput
from vllm.tasks import SupportedTask
from vllm.utils.serial_utils import (
EMBED_DTYPES,
EmbedDType,
Endianness,
binary2tensor,
tensor2binary,
)
logger = init_logger(__name__)
JsonEncodingFormat = Literal["float", "base64"]
BytesEncodingFormat = Literal["bytes", "bytes_only"]
FloatEncodedPoolingOutput = list[float] | list[list[float]]
JsonEncodedPoolingOutput = FloatEncodedPoolingOutput | str
@dataclass
class MetadataItem:
index: int
embed_dtype: EmbedDType
endianness: Endianness
start: int
end: int
shape: tuple[int, ...]
def build_metadata_items(
embed_dtype: EmbedDType,
endianness: Endianness,
shape: tuple[int, ...],
n_request: int,
) -> list[MetadataItem]:
n_bytes = EMBED_DTYPES[embed_dtype].nbytes
size = math.prod(shape)
return [
MetadataItem(
index=i,
embed_dtype=embed_dtype,
endianness=endianness,
start=i * size * n_bytes,
end=(i + 1) * size * n_bytes,
shape=shape,
)
for i in range(n_request)
]
def encode_pooling_output_float(
output: PoolingRequestOutput,
) -> FloatEncodedPoolingOutput:
return output.outputs.data.tolist()
def encode_pooling_output_float_or_ndarray(output: PoolingRequestOutput) -> Any:
"""Return an ndarray when the response renderer can serialize NumPy."""
try:
data = output.outputs.data
if not data.is_contiguous():
data = data.contiguous()
return data.numpy()
except (RuntimeError, TypeError):
return output.outputs.data.tolist()
def encode_pooling_output_base64(
output: PoolingRequestOutput,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> str:
embedding_bytes = tensor2binary(output.outputs.data, embed_dtype, endianness)
return pybase64.b64encode(embedding_bytes).decode("utf-8")
def encode_pooling_bytes(
pooling_outputs: list[PoolingRequestOutput],
embed_dtype: EmbedDType,
endianness: Endianness,
) -> tuple[list[bytes], list[dict[str, Any]]]:
items: list[dict[str, Any]] = []
body: list[bytes] = []
offset = 0
for idx, output in enumerate(pooling_outputs):
binary = tensor2binary(
tensor=output.outputs.data,
embed_dtype=embed_dtype,
endianness=endianness,
)
size = len(binary)
# Dictionary form of MetadataItem
item = dict(
index=idx,
embed_dtype=embed_dtype,
endianness=endianness,
start=offset,
end=offset + size,
shape=output.outputs.data.shape,
)
body.append(binary)
items.append(item)
offset += size
return body, items
def get_pooling_output_encoder(
encoding_format: JsonEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> Callable[[PoolingRequestOutput], JsonEncodedPoolingOutput]:
return cast(
Callable[[PoolingRequestOutput], JsonEncodedPoolingOutput],
(
encode_pooling_output_float
if encoding_format == "float"
else partial(
encode_pooling_output_base64,
embed_dtype=embed_dtype,
endianness=endianness,
)
),
)
def get_pooling_usage(
pooling_outputs: Sequence[PoolingRequestOutput],
) -> UsageInfo:
num_prompt_tokens = sum(
len(output.prompt_token_ids) if output.prompt_token_ids is not None else 0
for output in pooling_outputs
)
return UsageInfo(
prompt_tokens=num_prompt_tokens,
total_tokens=num_prompt_tokens,
)
def get_pooling_usage_payload(
pooling_outputs: Sequence[PoolingRequestOutput],
) -> dict[str, int]:
usage = get_pooling_usage(pooling_outputs)
return {
"prompt_tokens": usage.prompt_tokens,
"total_tokens": usage.total_tokens,
}
def build_pooling_bytes_streaming_response(
pooling_outputs: list[PoolingRequestOutput],
request_id: str,
created_time: int,
model_name: str,
encoding_format: BytesEncodingFormat,
embed_dtype: EmbedDType,
endianness: Endianness,
) -> StreamingResponse:
content, items = encode_pooling_bytes(
pooling_outputs=pooling_outputs,
embed_dtype=embed_dtype,
endianness=endianness,
)
headers = (
None
if encoding_format == "bytes_only"
else {
"metadata": json.dumps(
{
"id": request_id,
"created": created_time,
"model": model_name,
"data": items,
"usage": get_pooling_usage_payload(pooling_outputs),
}
)
}
)
return StreamingResponse(
content=content,
headers=headers,
media_type="application/octet-stream",
)
def decode_pooling_output(items: list[MetadataItem], body: bytes) -> list[torch.Tensor]:
return [
binary2tensor(
body[item.start : item.end],
item.shape,
item.embed_dtype,
item.endianness,
)
for item in sorted(items, key=lambda x: x.index)
]
@lru_cache(maxsize=1)
def get_json_response_cls() -> type[JSONResponse]:
if importlib.util.find_spec("orjson") is not None:
from fastapi.responses import ORJSONResponse
return ORJSONResponse
logger.warning_once(
"To make v1/embeddings API fast, please install orjson by `pip install orjson`"
)
return JSONResponse
def enable_scoring_api(
supported_tasks: tuple["SupportedTask", ...],
model_config: ModelConfig | None = None,
) -> bool:
if model_config is None:
return False
pooling_task = model_config.get_pooling_task(supported_tasks)
if pooling_task in ("embed", "token_embed"):
return True
if pooling_task == "classify":
num_labels = getattr(model_config.hf_config, "num_labels", 0)
if num_labels != 1:
logger.debug_once("Scoring API is only enabled for num_labels == 1.")
return False
return True
return False
+4 -7
View File
@@ -11,7 +11,6 @@ from vllm.entrypoints.openai.models.serving import (
OpenAIModelRegistry,
OpenAIServingModels,
)
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
from vllm.entrypoints.serve import create_error_response
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.typing import AnyRequest
@@ -48,7 +47,7 @@ class BaseServing:
async def _check_model(
self,
request: AnyRequest | AnyPoolingRequest,
request: AnyRequest,
) -> ErrorResponse | None:
if self.request_logger is not None:
self.request_logger.log_request_body(request)
@@ -141,7 +140,7 @@ class BaseServing:
return random_uuid() if default is None else default
def _get_message_types(self, request: AnyRequest | AnyPoolingRequest) -> set[str]:
def _get_message_types(self, request: AnyRequest) -> set[str]:
"""Retrieve the set of types from message content dicts up
until `_`; we use this to match potential multimodal data
with default per modality loras.
@@ -166,9 +165,7 @@ class BaseServing:
message_types.add(content_dict["type"].split("_")[0])
return message_types
def _get_active_default_mm_loras(
self, request: AnyRequest | AnyPoolingRequest
) -> LoRARequest | None:
def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
"""Determine if there are any active default multimodal loras."""
# TODO: Currently this is only enabled for chat completions
# to be better aligned with only being enabled for .generate
@@ -193,7 +190,7 @@ class BaseServing:
def _maybe_get_adapters(
self,
request: AnyRequest | AnyPoolingRequest,
request: AnyRequest,
supports_default_mm_loras: bool = False,
) -> LoRARequest | None:
if request.model in self.models.lora_requests:
@@ -12,7 +12,6 @@ from fastapi.responses import JSONResponse, Response
from vllm.config import ModelConfig
from vllm.entrypoints.generate.factories import get_generate_invocation_types
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.instrumentator.basic import base
@@ -79,9 +78,7 @@ def _attach_router(
router = APIRouter()
# NOTE: Construct the TypeAdapters only once
INVOCATION_TYPES = get_generate_invocation_types(
supported_tasks, model_config
) + get_pooling_invocation_types(supported_tasks, model_config)
INVOCATION_TYPES = get_generate_invocation_types(supported_tasks, model_config)
INVOCATION_VALIDATORS = [
(pydantic.TypeAdapter(request_type), (get_handler, endpoint))
@@ -7,7 +7,6 @@ from collections.abc import Sequence
import torch
from vllm import envs
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
from vllm.entrypoints.serve.engine.typing import AnyRequest
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
@@ -70,7 +69,7 @@ class RequestLogger:
lora_request,
)
def log_request_body(self, request: AnyRequest | AnyPoolingRequest) -> None:
def log_request_body(self, request: AnyRequest) -> None:
try:
level = logging.getLevelNamesMapping()[envs.VLLM_REQUEST_BODY_LOG_LEVEL]
except KeyError: