[Mini-vLLM] SM86-only cut: models, backends, Rust, tests
- Keep only Qwen3.5, Qwen3.5-MTP and Qwen4Exp model paths; trim the model registry to 41 entries - Delete idle attention backends (MLA, FlashInfer, ROCm-only, quant backends) and dead GDN CuTeDSL kernels - Remove inc/quark quant loaders; keep compressed-tensors and Marlin - Drop the Rust frontend, pooling entrypoints, and idle test suites - Rebuild kernels for sm_80;sm_86 and strip the extensions - Verified: full import sweep clean (optional deps only) and a dummy-load Qwen3.5 engine run on RTX 3090 via the Triton GDN path Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,27 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkSubcommandBase(CLISubcommand):
|
||||
"""The base class of subcommands for `vllm bench`."""
|
||||
|
||||
help: str
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
"""Add the CLI arguments to the parser."""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
"""Run the benchmark.
|
||||
|
||||
Args:
|
||||
args: The arguments to the command.
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.benchmarks.forward.main import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkForwardSubcommand(BenchmarkSubcommandBase):
|
||||
"""The `forward` subcommand for `vllm bench`."""
|
||||
|
||||
name = "forward"
|
||||
help = "Benchmark the model forward of exact batches as CUDA graphs."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.benchmarks.latency import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
|
||||
"""The `latency` subcommand for `vllm bench`."""
|
||||
|
||||
name = "latency"
|
||||
help = "Benchmark the latency of a single batch of requests."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -1,100 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import typing
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if typing.TYPE_CHECKING:
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
else:
|
||||
FlexibleArgumentParser = argparse.ArgumentParser
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def maybe_exec_rust_bench() -> None:
|
||||
if sys.argv[1:3] != ["bench", "serve"] or not envs.VLLM_USE_RUST_BENCH:
|
||||
return
|
||||
|
||||
rust_cli = envs.VLLM_RUST_FRONTEND_PATH
|
||||
if rust_cli is None:
|
||||
raise RuntimeError(
|
||||
"VLLM_USE_RUST_BENCH=1 requires VLLM_RUST_FRONTEND_PATH "
|
||||
"to resolve to the vllm-rs binary."
|
||||
)
|
||||
|
||||
logger.info("Delegating `vllm bench serve` to Rust binary at %s.", rust_cli)
|
||||
os.execv(rust_cli, [rust_cli, "bench", "serve", *sys.argv[3:]])
|
||||
|
||||
|
||||
def _import_bench_subcommand_modules() -> None:
|
||||
# Imported lazily so `BenchmarkSubcommandBase` subclasses register only
|
||||
# when `vllm bench` is actually invoked.
|
||||
import vllm.entrypoints.cli.benchmark.forward # noqa: F401
|
||||
import vllm.entrypoints.cli.benchmark.latency # noqa: F401
|
||||
import vllm.entrypoints.cli.benchmark.mm_processor # noqa: F401
|
||||
import vllm.entrypoints.cli.benchmark.serve # noqa: F401
|
||||
import vllm.entrypoints.cli.benchmark.startup # noqa: F401
|
||||
import vllm.entrypoints.cli.benchmark.sweep # noqa: F401
|
||||
import vllm.entrypoints.cli.benchmark.throughput # noqa: F401
|
||||
|
||||
|
||||
class BenchmarkSubcommand(CLISubcommand):
|
||||
"""The `bench` subcommand for the vLLM CLI."""
|
||||
|
||||
name = "bench"
|
||||
help = "vLLM bench subcommand."
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
args.dispatch_function(args)
|
||||
|
||||
def validate(self, args: argparse.Namespace) -> None:
|
||||
pass
|
||||
|
||||
def subparser_init(
|
||||
self, subparsers: argparse._SubParsersAction
|
||||
) -> FlexibleArgumentParser:
|
||||
bench_parser = subparsers.add_parser(
|
||||
self.name,
|
||||
help=self.help,
|
||||
description=self.help,
|
||||
usage=f"vllm {self.name} <bench_type> [options]",
|
||||
)
|
||||
bench_subparsers = bench_parser.add_subparsers(required=True, dest="bench_type")
|
||||
|
||||
# Only build the nested bench subparsers when the user is actually
|
||||
# invoking `bench`; otherwise we'd drag in imports
|
||||
# unnecessarily on every `vllm --help` and `vllm serve`.
|
||||
# Scan for the first positional arg so global flags (e.g. `-v`)
|
||||
# before the subcommand don't break detection.
|
||||
first_positional = next(
|
||||
(arg for arg in sys.argv[1:] if not arg.startswith("-")), None
|
||||
)
|
||||
if first_positional == self.name:
|
||||
_import_bench_subcommand_modules()
|
||||
for cmd_cls in BenchmarkSubcommandBase.__subclasses__():
|
||||
cmd_subparser = bench_subparsers.add_parser(
|
||||
cmd_cls.name,
|
||||
help=cmd_cls.help,
|
||||
description=cmd_cls.help,
|
||||
usage=f"vllm {self.name} {cmd_cls.name} [options]",
|
||||
)
|
||||
cmd_subparser.set_defaults(dispatch_function=cmd_cls.cmd)
|
||||
cmd_cls.add_cli_args(cmd_subparser)
|
||||
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
|
||||
subcmd=f"{self.name} {cmd_cls.name}"
|
||||
)
|
||||
return bench_parser
|
||||
|
||||
|
||||
def cmd_init() -> list[CLISubcommand]:
|
||||
return [BenchmarkSubcommand()]
|
||||
@@ -1,69 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import inspect
|
||||
|
||||
from vllm.benchmarks.mm_processor import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):
|
||||
r"""`vllm bench mm-processor` profiles the multimodal input processor pipeline of
|
||||
vision-language models. It measures per-stage latency from the HuggingFace
|
||||
processor through to the encoder forward pass, helping you identify
|
||||
preprocessing bottlenecks and understand how different image resolutions or
|
||||
item counts affect end-to-end request time.
|
||||
|
||||
The benchmark supports two data sources: synthetic random multimodal inputs
|
||||
(`random-mm`) and HuggingFace datasets (`hf`). Warmup requests are run before
|
||||
measurement to ensure stable results.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
vllm bench mm-processor \
|
||||
--model Qwen/Qwen2-VL-7B-Instruct \
|
||||
--dataset-name random-mm \
|
||||
--num-prompts 50 \
|
||||
--random-input-len 300 \
|
||||
--random-output-len 40 \
|
||||
--random-mm-base-items-per-request 2 \
|
||||
--random-mm-limit-mm-per-prompt '{"image": 3, "video": 0}' \
|
||||
--random-mm-bucket-config '{(256, 256, 1): 0.7, (720, 1280, 1): 0.3}'
|
||||
```
|
||||
|
||||
## Measured Stages
|
||||
|
||||
| Stage | Description |
|
||||
| ----- | ----------- |
|
||||
| `get_mm_hashes_secs` | Time spent hashing multimodal inputs |
|
||||
| `get_cache_missing_items_secs` | Time spent looking up the processor cache |
|
||||
| `apply_hf_processor_secs` | Time spent in the HuggingFace processor |
|
||||
| `merge_mm_kwargs_secs` | Time spent merging multimodal kwargs |
|
||||
| `apply_prompt_updates_secs` | Time spent updating prompt tokens |
|
||||
| `preprocessor_total_secs` | Total preprocessing time |
|
||||
| `encoder_forward_secs` | Time spent in the encoder model forward pass |
|
||||
| `num_encoder_calls` | Number of encoder invocations per request |
|
||||
|
||||
The benchmark also reports end-to-end latency (TTFT + decode time) per
|
||||
request. Use `--metric-percentiles` to select which percentiles to report
|
||||
(default: p99) and `--output-json` to save results.
|
||||
|
||||
For more examples (HF datasets, warmup, JSON output), see the
|
||||
[multimodal processor benchmark guide](https://docs.vllm.ai/en/latest/benchmarking/cli/#multimodal-processor-benchmark).
|
||||
"""
|
||||
|
||||
name = "mm-processor"
|
||||
help = "Benchmark multimodal processor latency across different configurations."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
# The class docstring is the page overview / `--help` description.
|
||||
if cls.__doc__:
|
||||
parser.description = inspect.cleandoc(cls.__doc__)
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.benchmarks.serve import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
|
||||
"""The `serve` subcommand for `vllm bench`."""
|
||||
|
||||
name = "serve"
|
||||
help = "Benchmark the online serving throughput."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.benchmarks.startup import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkStartupSubcommand(BenchmarkSubcommandBase):
|
||||
"""The `startup` subcommand for `vllm bench`."""
|
||||
|
||||
name = "startup"
|
||||
help = "Benchmark the startup time of vLLM models."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.benchmarks.sweep.cli import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
|
||||
"""The `sweep` subcommand for `vllm bench`."""
|
||||
|
||||
name = "sweep"
|
||||
help = "Benchmark for a parameter sweep."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -1,22 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
|
||||
from vllm.benchmarks.throughput import add_cli_args, main
|
||||
from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
|
||||
class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
|
||||
"""The `throughput` subcommand for `vllm bench`."""
|
||||
|
||||
name = "throughput"
|
||||
help = "Benchmark offline inference throughput."
|
||||
|
||||
@classmethod
|
||||
def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:
|
||||
add_cli_args(parser)
|
||||
|
||||
@staticmethod
|
||||
def cmd(args: argparse.Namespace) -> None:
|
||||
main(args)
|
||||
@@ -40,7 +40,6 @@ def main():
|
||||
if maybe_run_omni():
|
||||
return
|
||||
|
||||
import vllm.entrypoints.cli.benchmark.main
|
||||
import vllm.entrypoints.cli.collect_env
|
||||
import vllm.entrypoints.cli.download_kernels
|
||||
import vllm.entrypoints.cli.launch
|
||||
@@ -59,7 +58,6 @@ def main():
|
||||
vllm.entrypoints.cli.openai,
|
||||
vllm.entrypoints.cli.serve,
|
||||
vllm.entrypoints.cli.launch,
|
||||
vllm.entrypoints.cli.benchmark.main,
|
||||
vllm.entrypoints.cli.collect_env,
|
||||
vllm.entrypoints.cli.download_kernels,
|
||||
vllm.entrypoints.cli.preload,
|
||||
@@ -70,24 +68,6 @@ def main():
|
||||
if sys.argv[1:2] != ["snapshot"]:
|
||||
cli_env_setup()
|
||||
|
||||
vllm.entrypoints.cli.benchmark.main.maybe_exec_rust_bench()
|
||||
|
||||
# For 'vllm bench *': use CPU instead of UnspecifiedPlatform by default
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "bench":
|
||||
logger.debug(
|
||||
"Bench command detected, must ensure current platform is not "
|
||||
"UnspecifiedPlatform to avoid device type inference error"
|
||||
)
|
||||
from vllm import platforms
|
||||
|
||||
if platforms.current_platform.is_unspecified():
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
|
||||
platforms.current_platform = CpuPlatform()
|
||||
logger.info(
|
||||
"Unspecified platform detected, switching to CPU Platform instead."
|
||||
)
|
||||
|
||||
parser = FlexibleArgumentParser(
|
||||
description="vLLM CLI",
|
||||
epilog=VLLM_SUBCMD_PARSER_EPILOG.format(subcmd="[subcommand]"),
|
||||
|
||||
@@ -17,7 +17,6 @@ from vllm.entrypoints.launchers.cli_args import (
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor
|
||||
from vllm.entrypoints.launchers.launcher import create_server_socket
|
||||
from vllm.entrypoints.serve.utils.api_utils import VLLM_SUBCMD_PARSER_EPILOG
|
||||
from vllm.logger import init_logger
|
||||
from vllm.reasoning import ReasoningParserManager
|
||||
@@ -30,7 +29,6 @@ from vllm.v1.executor.multiproc_executor import MultiprocExecutor
|
||||
from vllm.v1.metrics.prometheus import setup_multiprocess_prometheus
|
||||
from vllm.v1.utils import (
|
||||
APIServerProcessManager,
|
||||
RustFrontendProcessManager,
|
||||
wait_for_completion_or_failure,
|
||||
)
|
||||
|
||||
@@ -62,10 +60,6 @@ class ServeSubcommand(CLISubcommand):
|
||||
uvloop.run(serve_grpc(args))
|
||||
return
|
||||
|
||||
rust_frontend_path = (
|
||||
envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None
|
||||
)
|
||||
|
||||
if args.headless:
|
||||
if args.api_server_count is not None and args.api_server_count > 0:
|
||||
raise ValueError(
|
||||
@@ -106,12 +100,11 @@ class ServeSubcommand(CLISubcommand):
|
||||
|
||||
# Default api_server_count if not explicitly set.
|
||||
# - Multi-port: 1 (supervisor spawns one server per local DP rank)
|
||||
# - Rust frontend: 1 (not applicable as it's multithreaded)
|
||||
# - External LB: 1 (external LB handles distribution)
|
||||
# - Hybrid LB: Use local DP size (internal LB for local ranks only)
|
||||
# - Internal LB: Use full DP size
|
||||
if args.api_server_count is None:
|
||||
if is_multi_port or is_external_lb or rust_frontend_path:
|
||||
if is_multi_port or is_external_lb:
|
||||
args.api_server_count = 1
|
||||
elif is_hybrid_lb:
|
||||
args.api_server_count = args.data_parallel_size_local or 1
|
||||
@@ -128,12 +121,6 @@ class ServeSubcommand(CLISubcommand):
|
||||
"Defaulting api_server_count to data_parallel_size (%d).",
|
||||
args.api_server_count,
|
||||
)
|
||||
elif rust_frontend_path and args.api_server_count > 1:
|
||||
logger.warning(
|
||||
"Ignoring --api-server-count=%d when using rust front-end process",
|
||||
args.api_server_count,
|
||||
)
|
||||
args.api_server_count = 1
|
||||
|
||||
# Elastic EP currently only supports running with at most one API server.
|
||||
if getattr(args, "enable_elastic_ep", False) and args.api_server_count > 1:
|
||||
@@ -148,7 +135,7 @@ class ServeSubcommand(CLISubcommand):
|
||||
run_dp_supervisor(args)
|
||||
elif args.api_server_count < 1:
|
||||
run_headless(args)
|
||||
elif args.api_server_count > 1 or rust_frontend_path:
|
||||
elif args.api_server_count > 1:
|
||||
run_multi_api_server(args)
|
||||
else:
|
||||
# Single API server (this process).
|
||||
@@ -268,17 +255,9 @@ def run_headless(args: argparse.Namespace):
|
||||
|
||||
def run_multi_api_server(args: argparse.Namespace):
|
||||
assert not args.headless
|
||||
rust_frontend_path = (
|
||||
envs.VLLM_RUST_FRONTEND_PATH if envs.VLLM_USE_RUST_FRONTEND else None
|
||||
)
|
||||
num_api_servers: int = args.api_server_count
|
||||
assert num_api_servers > 0
|
||||
|
||||
if rust_frontend_path and num_api_servers > 1:
|
||||
raise ValueError(
|
||||
"VLLM_RUST_FRONTEND_PATH does not support api_server_count > 1"
|
||||
)
|
||||
|
||||
if num_api_servers > 1:
|
||||
setup_multiprocess_prometheus()
|
||||
|
||||
@@ -296,13 +275,6 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1)
|
||||
# `--grpc-port` is only accepted for the Rust frontend, which inherits this
|
||||
# listener like the HTTP one. gRPC follows the HTTP TCP host, or IPv4
|
||||
# loopback when HTTP uses a Unix socket.
|
||||
grpc_sock = None
|
||||
if args.grpc_port is not None:
|
||||
grpc_host = "127.0.0.1" if args.uds else (args.host or "")
|
||||
grpc_sock = create_server_socket((grpc_host, args.grpc_port), reuse_port=False)
|
||||
|
||||
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
|
||||
propagate_flash_late_interaction(args, engine_args)
|
||||
@@ -324,9 +296,7 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
dp_rank = parallel_config.data_parallel_rank
|
||||
assert parallel_config.local_engines_only or dp_rank == 0
|
||||
|
||||
api_server_manager: APIServerProcessManager | RustFrontendProcessManager | None = (
|
||||
None
|
||||
)
|
||||
api_server_manager: APIServerProcessManager | None = None
|
||||
|
||||
from vllm.v1.engine.utils import bind_engine_zmq_listeners
|
||||
|
||||
@@ -343,38 +313,16 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
coordinator.get_stats_publish_address() if coordinator else None
|
||||
)
|
||||
|
||||
if rust_frontend_path:
|
||||
if parallel_config.local_engines_only:
|
||||
expected_engine_start_index = parallel_config.data_parallel_rank
|
||||
expected_engine_count = parallel_config.data_parallel_size_local
|
||||
else:
|
||||
expected_engine_start_index = 0
|
||||
expected_engine_count = parallel_config.data_parallel_size
|
||||
# Start rust front-end process.
|
||||
api_server_manager = RustFrontendProcessManager(
|
||||
binary_path=rust_frontend_path,
|
||||
sock=sock,
|
||||
grpc_sock=grpc_sock,
|
||||
args=args,
|
||||
input_listener=zmq_listeners.inputs[0],
|
||||
output_listener=zmq_listeners.outputs[0],
|
||||
engine_start_index=expected_engine_start_index,
|
||||
engine_count=expected_engine_count,
|
||||
data_parallel_size=parallel_config.data_parallel_size,
|
||||
stats_update_address=stats_update_address,
|
||||
)
|
||||
else:
|
||||
# Start API server(s).
|
||||
api_server_manager = APIServerProcessManager(
|
||||
listen_address=listen_address,
|
||||
sock=sock,
|
||||
args=args,
|
||||
num_servers=num_api_servers,
|
||||
input_listeners=zmq_listeners.inputs,
|
||||
output_listeners=zmq_listeners.outputs,
|
||||
stats_update_address=stats_update_address,
|
||||
tensor_queue=engine_launch.tensor_queue,
|
||||
)
|
||||
api_server_manager = APIServerProcessManager(
|
||||
listen_address=listen_address,
|
||||
sock=sock,
|
||||
args=args,
|
||||
num_servers=num_api_servers,
|
||||
input_listeners=zmq_listeners.inputs,
|
||||
output_listeners=zmq_listeners.outputs,
|
||||
stats_update_address=stats_update_address,
|
||||
tensor_queue=engine_launch.tensor_queue,
|
||||
)
|
||||
|
||||
# Set frontend processes to watch during engine startup.
|
||||
# If any of these processes exit before the engines are up, the engine startup
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.plugins.endpoint_plugins.interface import init_endpoint_plugins_state
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.renderers.online_derenderer import OnlineDerenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, POOLING_TASKS, SupportedTask
|
||||
from vllm.tasks import FALLBACK_SUPPORTED_TASKS, SupportedTask
|
||||
|
||||
|
||||
async def init_app_state(
|
||||
@@ -152,11 +152,6 @@ async def init_app_state(
|
||||
engine_client, state, args, request_logger, supported_tasks
|
||||
)
|
||||
|
||||
if any(task in POOLING_TASKS for task in supported_tasks):
|
||||
from vllm.entrypoints.pooling.factories import init_pooling_state
|
||||
|
||||
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.api_router import (
|
||||
init_structured_decisions_state,
|
||||
)
|
||||
|
||||
@@ -6,7 +6,7 @@ from fastapi import FastAPI
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.tasks import POOLING_TASKS, SupportedTask
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
|
||||
def register_api_routers(
|
||||
@@ -77,11 +77,6 @@ def register_api_routers(
|
||||
|
||||
register_speech_to_text_api_routers(app, supported_tasks)
|
||||
|
||||
if any(task in POOLING_TASKS for task in supported_tasks):
|
||||
from vllm.entrypoints.pooling.factories import register_pooling_api_routers
|
||||
|
||||
register_pooling_api_routers(app, supported_tasks, model_config)
|
||||
|
||||
if getattr(args, "enable_fault_tolerance", False):
|
||||
from vllm.entrypoints.serve.fault_tolerance.api_router import (
|
||||
register_fault_tolerance_api_router,
|
||||
|
||||
@@ -285,11 +285,6 @@ class FrontendArgs(BaseFrontendArgs):
|
||||
"""Host name."""
|
||||
port: int = 8000
|
||||
"""Port number."""
|
||||
grpc_port: int | None = None
|
||||
"""Enable the Rust frontend's additional gRPC Inference and Control services
|
||||
on this port. Requires `VLLM_USE_RUST_FRONTEND=1 vllm serve`; HTTP remains on
|
||||
`--port`. Binds on `--host`, or 127.0.0.1 with `--uds`. Cannot be combined
|
||||
with the Python gRPC server's `--grpc` flag."""
|
||||
data_parallel_supervisor_port: int = 9256
|
||||
"""HTTP port for aggregated health endpoints in multi-port external LB
|
||||
mode."""
|
||||
@@ -468,38 +463,6 @@ def make_arg_parser(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
|
||||
return parser
|
||||
|
||||
|
||||
def validate_grpc_port_arg(args: argparse.Namespace) -> None:
|
||||
"""Validate the Rust frontend's optional gRPC listener."""
|
||||
if getattr(args, "grpc_port", None) is None:
|
||||
return
|
||||
|
||||
if args.grpc:
|
||||
raise ValueError(
|
||||
"--grpc and --grpc-port are mutually exclusive. Use --grpc "
|
||||
"--port for the Python gRPC server, or VLLM_USE_RUST_FRONTEND=1 "
|
||||
"with --grpc-port for the Rust frontend."
|
||||
)
|
||||
if not envs.VLLM_USE_RUST_FRONTEND or getattr(args, "subparser", None) != "serve":
|
||||
raise ValueError("--grpc-port requires VLLM_USE_RUST_FRONTEND=1 vllm serve")
|
||||
if args.headless or (
|
||||
args.api_server_count is not None and args.api_server_count <= 0
|
||||
):
|
||||
raise ValueError(
|
||||
"--grpc-port requires a Rust frontend; remove --headless "
|
||||
"and non-positive --api-server-count"
|
||||
)
|
||||
if args.data_parallel_multi_port_external_lb:
|
||||
raise ValueError(
|
||||
"--grpc-port is incompatible with "
|
||||
"--data-parallel-multi-port-external-lb: "
|
||||
"its frontends would share the same gRPC port"
|
||||
)
|
||||
if not 0 <= args.grpc_port <= 65535:
|
||||
raise ValueError("--grpc-port must be between 0 and 65535")
|
||||
if args.grpc_port != 0 and args.grpc_port == args.port and not args.uds:
|
||||
raise ValueError("--grpc-port must differ from --port when HTTP uses TCP")
|
||||
|
||||
|
||||
def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
"""Quick checks for model serve args that raise prior to loading."""
|
||||
# `vllm launch <component>` builds its parser with make_arg_parser too (see
|
||||
@@ -518,8 +481,6 @@ def validate_parsed_serve_args(args: argparse.Namespace):
|
||||
"the checkpoint's output format."
|
||||
)
|
||||
|
||||
validate_grpc_port_arg(args)
|
||||
|
||||
# Enable auto tool needs a tool call parser to be valid
|
||||
if args.enable_auto_tool_choice and not args.tool_call_parser:
|
||||
raise TypeError("Error: --enable-auto-tool-choice requires --tool-call-parser")
|
||||
|
||||
@@ -22,7 +22,6 @@ import uvicorn
|
||||
import uvloop
|
||||
from fastapi import FastAPI, Response
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import configure_logging_from_args, init_logger
|
||||
from vllm.utils.system_utils import (
|
||||
decorate_logs,
|
||||
@@ -243,12 +242,6 @@ def _run_python_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
uvloop.run(run_server(child_args))
|
||||
|
||||
|
||||
def _run_rust_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
from vllm.entrypoints.cli.serve import run_multi_api_server
|
||||
|
||||
run_multi_api_server(child_args)
|
||||
|
||||
|
||||
def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
"""Entrypoint function for the vLLM DP Server."""
|
||||
configure_logging_from_args(child_args)
|
||||
@@ -260,10 +253,7 @@ def _run_vllm_dp_server(child_args: argparse.Namespace) -> None:
|
||||
name = f"APIServer_DP{child_args.data_parallel_rank}"
|
||||
set_process_title(name)
|
||||
decorate_logs(name)
|
||||
if envs.VLLM_USE_RUST_FRONTEND and envs.VLLM_RUST_FRONTEND_PATH:
|
||||
_run_rust_vllm_dp_server(child_args)
|
||||
else:
|
||||
_run_python_vllm_dp_server(child_args)
|
||||
_run_python_vllm_dp_server(child_args)
|
||||
|
||||
|
||||
class DPSupervisor:
|
||||
|
||||
@@ -35,16 +35,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
)
|
||||
from vllm.entrypoints.pooling.embed.protocol import (
|
||||
EmbeddingRequest,
|
||||
EmbeddingResponse,
|
||||
)
|
||||
from vllm.entrypoints.pooling.scoring.protocol import (
|
||||
RerankRequest,
|
||||
RerankResponse,
|
||||
ScoreRequest,
|
||||
ScoreResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
ErrorInfo,
|
||||
@@ -147,12 +137,7 @@ class BatchTranslationRequest(TranslationRequest):
|
||||
|
||||
|
||||
BatchRequestInputBody: TypeAlias = (
|
||||
ChatCompletionRequest
|
||||
| EmbeddingRequest
|
||||
| ScoreRequest
|
||||
| RerankRequest
|
||||
| BatchTranscriptionRequest
|
||||
| BatchTranslationRequest
|
||||
ChatCompletionRequest | BatchTranscriptionRequest | BatchTranslationRequest
|
||||
)
|
||||
|
||||
|
||||
@@ -184,12 +169,6 @@ class BatchRequestInput(OpenAIBaseModel):
|
||||
url: str = info.data["url"]
|
||||
if url == "/v1/chat/completions":
|
||||
return ChatCompletionRequest.model_validate(value)
|
||||
if url == "/v1/embeddings":
|
||||
return TypeAdapter(EmbeddingRequest).validate_python(value)
|
||||
if url.endswith("/score"):
|
||||
return TypeAdapter(ScoreRequest).validate_python(value)
|
||||
if url.endswith("/rerank"):
|
||||
return RerankRequest.model_validate(value)
|
||||
if url == "/v1/audio/transcriptions":
|
||||
return BatchTranscriptionRequest.model_validate(value)
|
||||
if url == "/v1/audio/translations":
|
||||
@@ -199,9 +178,6 @@ class BatchRequestInput(OpenAIBaseModel):
|
||||
|
||||
AllResponse: TypeAlias = (
|
||||
ChatCompletionResponse
|
||||
| EmbeddingResponse
|
||||
| ScoreResponse
|
||||
| RerankResponse
|
||||
| TranscriptionResponseVariant
|
||||
| TranslationResponse
|
||||
| TranslationResponseVerbose
|
||||
@@ -738,8 +714,6 @@ async def build_endpoint_registry(
|
||||
openai_serving_chat = getattr(state, "openai_serving_chat", None)
|
||||
openai_serving_transcription = getattr(state, "openai_serving_transcription", None)
|
||||
openai_serving_translation = getattr(state, "openai_serving_translation", None)
|
||||
serving_embedding = getattr(state, "serving_embedding", None)
|
||||
serving_scores = getattr(state, "serving_scores", None)
|
||||
|
||||
allowed_media_domains = getattr(args, "allowed_media_domains", None)
|
||||
|
||||
@@ -754,27 +728,6 @@ async def build_endpoint_registry(
|
||||
),
|
||||
"wrapper_fn": None,
|
||||
},
|
||||
"embeddings": {
|
||||
"url_matcher": lambda url: url == "/v1/embeddings",
|
||||
"handler_getter": lambda: (
|
||||
serving_embedding if serving_embedding is not None else None
|
||||
),
|
||||
"wrapper_fn": None,
|
||||
},
|
||||
"score": {
|
||||
"url_matcher": lambda url: url.endswith("/score"),
|
||||
"handler_getter": lambda: (
|
||||
serving_scores if serving_scores is not None else None
|
||||
),
|
||||
"wrapper_fn": None,
|
||||
},
|
||||
"rerank": {
|
||||
"url_matcher": lambda url: url.endswith("/rerank"),
|
||||
"handler_getter": lambda: (
|
||||
serving_scores if serving_scores is not None else None
|
||||
),
|
||||
"wrapper_fn": None,
|
||||
},
|
||||
"transcriptions": {
|
||||
"url_matcher": lambda url: url == "/v1/audio/transcriptions",
|
||||
"handler_getter": lambda: (
|
||||
@@ -859,9 +812,8 @@ async def run_batch(
|
||||
make_async_error_request_output(
|
||||
request,
|
||||
error_msg=f"URL {request.url} was used. "
|
||||
"Supported endpoints: /v1/chat/completions, /v1/embeddings,"
|
||||
" /v1/audio/transcriptions, /v1/audio/translations, /score, "
|
||||
" /rerank.",
|
||||
"Supported endpoints: /v1/chat/completions,"
|
||||
" /v1/audio/transcriptions, /v1/audio/translations.",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -30,7 +30,6 @@ from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.engine.arg_utils import EngineArgs
|
||||
from vllm.entrypoints.common.offline import _O, _R, OfflineInferenceMixin
|
||||
from vllm.entrypoints.generate.beam_search.offline import BeamSearchOfflineMixin
|
||||
from vllm.entrypoints.pooling.offline import PoolingOfflineMixin
|
||||
from vllm.entrypoints.rl.offline import RLOfflineMixin
|
||||
from vllm.entrypoints.serve.utils.api_utils import log_non_default_args
|
||||
from vllm.inputs import PromptType
|
||||
@@ -60,9 +59,7 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LLM(
|
||||
BeamSearchOfflineMixin, PoolingOfflineMixin, RLOfflineMixin, OfflineInferenceMixin
|
||||
):
|
||||
class LLM(BeamSearchOfflineMixin, RLOfflineMixin, OfflineInferenceMixin):
|
||||
"""An LLM for generating texts from given prompts and sampling parameters.
|
||||
|
||||
This class includes a tokenizer, a language model (possibly distributed
|
||||
@@ -373,8 +370,6 @@ class LLM(
|
||||
self.model_config.renderer_num_workers,
|
||||
)
|
||||
|
||||
PoolingOfflineMixin.__init__(self)
|
||||
|
||||
# Cache for __repr__ to avoid repeated collective_rpc calls
|
||||
self._cached_repr: str | None = None
|
||||
|
||||
|
||||
@@ -1,356 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from concurrent.futures import Executor
|
||||
from typing import Any, Final, cast
|
||||
|
||||
from vllm import (
|
||||
PoolingParams,
|
||||
PoolingRequestOutput,
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers import BaseRenderer, merge_kwargs
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatTemplateConfig,
|
||||
)
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||
from vllm.utils.async_utils import make_async
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ..typing import (
|
||||
AnyOfflineInputsContext,
|
||||
AnyRenderParam,
|
||||
EncodeChatRenderParams,
|
||||
EncodeCMPLRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
PoolingChatLikeRequest,
|
||||
PoolingCompletionLikeRequest,
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
RequestGenerator,
|
||||
ScoringRenderParams,
|
||||
)
|
||||
|
||||
|
||||
class PoolingIOProcessor:
|
||||
"""Processor for handling preprocessing & postprocessing ops for pooling requests.
|
||||
|
||||
This class manages both online (serving) and offline (batch) processing of pooling
|
||||
requests, handling chat and completion formats.
|
||||
"""
|
||||
|
||||
name: str
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
renderer: BaseRenderer,
|
||||
chat_template_config: ChatTemplateConfig,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.renderer = renderer
|
||||
|
||||
self.chat_template = chat_template_config.chat_template
|
||||
self.chat_template_content_format: Final = (
|
||||
chat_template_config.chat_template_content_format
|
||||
)
|
||||
self.trust_request_chat_template = (
|
||||
chat_template_config.trust_request_chat_template
|
||||
)
|
||||
|
||||
self.template_kwargs = None
|
||||
self.tool_dicts = None
|
||||
|
||||
# Shared thread pool executor for preprocessing
|
||||
self._executor: Executor = self.renderer._executor
|
||||
self.render_async = make_async(self.render, executor=self._executor)
|
||||
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def create_pooling_params(self, request):
|
||||
return request.to_pooling_params()
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
renderer = self.renderer
|
||||
requests: Sequence[AnyRenderParam]
|
||||
|
||||
if isinstance(request, PoolingChatLikeRequest):
|
||||
self._validate_chat_template(
|
||||
request_chat_template=request.chat_template,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
|
||||
num_requests = 1
|
||||
default_template_kwargs = merge_kwargs(
|
||||
self.template_kwargs,
|
||||
dict(
|
||||
tools=self.tool_dicts,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
)
|
||||
|
||||
mm_config = self.model_config.multimodal_config
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
self.chat_template, self.chat_template_content_format
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(
|
||||
mm_config.media_io_kwargs if mm_config else None
|
||||
),
|
||||
)
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(
|
||||
ctx.lora_request, num_requests
|
||||
)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeChatRenderParams(
|
||||
conversations=request.messages,
|
||||
chat_params=chat_params,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
return requests
|
||||
|
||||
elif isinstance(request, PoolingCompletionLikeRequest):
|
||||
model_config = self.model_config
|
||||
prompts_seq = prompt_to_seq(request.input)
|
||||
num_requests = len(prompts_seq)
|
||||
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(model_config, prompt)
|
||||
)
|
||||
for prompt in prompts_seq
|
||||
]
|
||||
tok_params = request.build_tok_params(model_config)
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(
|
||||
ctx.lora_request, num_requests
|
||||
)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeCMPLRenderParams(
|
||||
prompts=parsed_prompts[i],
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
return requests
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.name} request type")
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
pass
|
||||
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
|
||||
prompts_seq = prompt_to_seq(ctx.prompts)
|
||||
num_requests = len(prompts_seq)
|
||||
pooling_task = ctx.pooling_task
|
||||
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(self.model_config, prompt)
|
||||
)
|
||||
for prompt in prompts_seq
|
||||
]
|
||||
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(ctx.tokenization_kwargs or {})
|
||||
)
|
||||
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams]
|
||||
if ctx.pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
else:
|
||||
pooling_params = ctx.pooling_params
|
||||
|
||||
# Clone to avoid modifying the caller's pooling parameters.
|
||||
params_seq = [
|
||||
param.clone() for param in self._params_to_seq(pooling_params, num_requests)
|
||||
]
|
||||
|
||||
for param in params_seq:
|
||||
if param.task is None:
|
||||
param.task = pooling_task
|
||||
elif pooling_task == "plugin":
|
||||
# `plugin` task uses io_processor.parse_data to verify inputs.
|
||||
# We actually allow plugin to overwrite pooling_task.
|
||||
pass
|
||||
elif param.task != pooling_task:
|
||||
msg = f"You cannot overwrite {param.task=!r} with {pooling_task=!r}!"
|
||||
raise VLLMValidationError(msg)
|
||||
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
def request_factory() -> RequestGenerator:
|
||||
for i in range(num_requests):
|
||||
yield EncodeCMPLRenderParams(
|
||||
prompts=parsed_prompts[i],
|
||||
tok_params=tok_params,
|
||||
prompt_extras=None,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
|
||||
return request_factory, num_requests
|
||||
|
||||
def post_process_offline(
|
||||
self,
|
||||
ctx: OfflineOutputsContext,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
return ctx.outputs
|
||||
|
||||
#######################################
|
||||
# helpers
|
||||
|
||||
def render(
|
||||
self,
|
||||
render_params: EncodeCMPLRenderParams
|
||||
| EncodeChatRenderParams
|
||||
| ScoringRenderParams,
|
||||
) -> PoolingEngineInput:
|
||||
if "conversations" in render_params:
|
||||
render_params = cast(EncodeChatRenderParams, render_params)
|
||||
(_,), engine_input = self.renderer.render_chat(
|
||||
conversations=[render_params["conversations"]],
|
||||
chat_params=render_params["chat_params"],
|
||||
tok_params=render_params["tok_params"],
|
||||
prompt_extras=render_params["prompt_extras"],
|
||||
skip_mm_cache=render_params["skip_mm_cache"],
|
||||
)
|
||||
elif "prompts" in render_params:
|
||||
render_params = cast(EncodeCMPLRenderParams, render_params)
|
||||
engine_input = self.renderer.render_cmpl(
|
||||
prompts=[render_params["prompts"]],
|
||||
tok_params=render_params["tok_params"],
|
||||
prompt_extras=render_params["prompt_extras"],
|
||||
skip_mm_cache=render_params["skip_mm_cache"],
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unsupported render_params type {render_params.__class__.__name__}"
|
||||
)
|
||||
|
||||
return PoolingEngineInput(
|
||||
prompts=engine_input[0],
|
||||
params=render_params["params"],
|
||||
lora_requests=render_params["lora_requests"],
|
||||
priorities=render_params["priorities"],
|
||||
)
|
||||
|
||||
def _validate_chat_template(
|
||||
self,
|
||||
request_chat_template: str | None,
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
trust_request_chat_template: bool,
|
||||
):
|
||||
if not trust_request_chat_template and (
|
||||
request_chat_template is not None
|
||||
or (
|
||||
chat_template_kwargs
|
||||
and chat_template_kwargs.get("chat_template") is not None
|
||||
)
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
"Chat template is passed with request, but "
|
||||
"--trust-request-chat-template is not set. "
|
||||
"Refused request with untrusted chat template."
|
||||
)
|
||||
return None
|
||||
|
||||
def _params_to_seq(
|
||||
self,
|
||||
params: PoolingParams | Sequence[PoolingParams],
|
||||
num_requests: int,
|
||||
) -> Sequence[PoolingParams]:
|
||||
if isinstance(params, Sequence):
|
||||
if len(params) != num_requests:
|
||||
raise VLLMValidationError(
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
f"and params ({len(params)}) must be the same."
|
||||
)
|
||||
|
||||
return params
|
||||
|
||||
return [params] * num_requests
|
||||
|
||||
def _lora_request_to_seq(
|
||||
self,
|
||||
lora_request: LoRARequest | None | Sequence[LoRARequest | None],
|
||||
num_requests: int,
|
||||
) -> Sequence[LoRARequest | None]:
|
||||
if isinstance(lora_request, Sequence):
|
||||
if len(lora_request) != num_requests:
|
||||
raise VLLMValidationError(
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
f"and lora_request ({len(lora_request)}) must be the same."
|
||||
)
|
||||
|
||||
return lora_request
|
||||
|
||||
return [lora_request] * num_requests
|
||||
|
||||
def _priority_to_seq(
|
||||
self,
|
||||
priority: int | Sequence[int] | None,
|
||||
num_requests: int,
|
||||
) -> Sequence[int]:
|
||||
if priority is not None:
|
||||
if isinstance(priority, int):
|
||||
return [priority] * num_requests
|
||||
|
||||
if len(priority) != num_requests:
|
||||
raise ValueError(
|
||||
f"The lengths of prompts ({num_requests}) "
|
||||
f"and priority ({len(priority)}) must be the same."
|
||||
)
|
||||
|
||||
return priority
|
||||
|
||||
return [0] * num_requests
|
||||
@@ -1,366 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionMessageParam,
|
||||
ChatTemplateContentFormatOption,
|
||||
)
|
||||
from vllm.tasks import check_removed_pooling_task
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness
|
||||
|
||||
|
||||
def reject_removed_pooling_parameters(data):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
if "normalize" in data:
|
||||
raise VLLMValidationError(
|
||||
"Parameter `normalize` was removed; use `use_activation` instead.",
|
||||
parameter="normalize",
|
||||
)
|
||||
check_removed_pooling_task(data.get("task"))
|
||||
return data
|
||||
|
||||
|
||||
class PoolingBasicRequestMixin(OpenAIBaseModel):
|
||||
# --8<-- [start:pooling-common-params]
|
||||
model: str | None = None
|
||||
user: str | None = None
|
||||
# --8<-- [end:pooling-common-params]
|
||||
|
||||
# --8<-- [start:pooling-common-extra-params]
|
||||
truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
|
||||
padding: Literal["max_length", "do_not_pad"] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Whether to pad the prompt, using the same names as the "
|
||||
"Transformers tokenizer. 'max_length' pads to the maximum input "
|
||||
"length, 'do_not_pad' leaves the prompt as is. Models trained "
|
||||
"with a fixed sequence length and no attention mask, such as "
|
||||
"SigLIP, need 'max_length' to match training, otherwise their "
|
||||
"embeddings are not comparable."
|
||||
),
|
||||
)
|
||||
truncation_side: Literal["left", "right"] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Which side to truncate from when truncate_prompt_tokens is active. "
|
||||
"'right' keeps the first N tokens. "
|
||||
"'left' keeps the last N tokens."
|
||||
),
|
||||
)
|
||||
request_id: str = Field(
|
||||
default_factory=random_uuid,
|
||||
description=(
|
||||
"The request_id related to this request. If the caller does "
|
||||
"not set it, a random_uuid will be generated. This id is used "
|
||||
"through out the inference process and return in response."
|
||||
),
|
||||
)
|
||||
priority: int = Field(
|
||||
default=0,
|
||||
ge=-(2**63),
|
||||
le=2**63 - 1,
|
||||
description=(
|
||||
"The priority of the request (lower means earlier handling; "
|
||||
"default: 0). Any priority other than 0 will raise an error "
|
||||
"if the served model does not use priority scheduling."
|
||||
),
|
||||
)
|
||||
mm_processor_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="Additional kwargs to pass to the HF processor.",
|
||||
)
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=1024,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:pooling-common-extra-params]
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_cache_salt_support(cls, data: Any) -> Any:
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
def _build_pooling_tok_params(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
*,
|
||||
add_special_tokens: bool,
|
||||
max_total_tokens: int | None,
|
||||
max_output_tokens: int,
|
||||
max_total_tokens_param: str = "max_model_len",
|
||||
max_output_tokens_param: str | None = None,
|
||||
) -> TokenizeParams:
|
||||
encoder_config = model_config.encoder_config or {}
|
||||
if max_output_tokens_param is None:
|
||||
tok_params = TokenizeParams(
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=add_special_tokens,
|
||||
max_total_tokens_param=max_total_tokens_param,
|
||||
)
|
||||
else:
|
||||
tok_params = TokenizeParams(
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
truncate_prompt_tokens=self.truncate_prompt_tokens,
|
||||
truncation_side=self.truncation_side,
|
||||
do_lower_case=encoder_config.get("do_lower_case", False),
|
||||
add_special_tokens=add_special_tokens,
|
||||
max_total_tokens_param=max_total_tokens_param,
|
||||
max_output_tokens_param=max_output_tokens_param,
|
||||
)
|
||||
|
||||
if self.padding is None:
|
||||
return tok_params
|
||||
|
||||
return tok_params.with_kwargs(padding=self.padding)
|
||||
|
||||
|
||||
class PoolingTokenizeParamsMixin:
|
||||
add_special_tokens: bool
|
||||
|
||||
def _build_pooling_tok_params(
|
||||
self,
|
||||
model_config: ModelConfig,
|
||||
*,
|
||||
add_special_tokens: bool,
|
||||
max_total_tokens: int | None,
|
||||
max_output_tokens: int,
|
||||
max_total_tokens_param: str = "max_model_len",
|
||||
max_output_tokens_param: str | None = None,
|
||||
) -> TokenizeParams:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FixedMaxLenTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingTokenizeParamsMixin(PoolingTokenizeParamsMixin):
|
||||
padding: Literal["max_length", "do_not_pad"] | None
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
default_max_total_tokens = model_config.max_model_len
|
||||
max_total_tokens: int | None = default_max_total_tokens
|
||||
max_output_tokens = 0
|
||||
|
||||
pooler_config = model_config.pooler_config
|
||||
if pooler_config is not None:
|
||||
if pooler_config.enable_chunked_processing:
|
||||
max_total_tokens = pooler_config.max_embed_len
|
||||
else:
|
||||
max_total_tokens = (
|
||||
pooler_config.max_embed_len or default_max_total_tokens
|
||||
)
|
||||
|
||||
tok_params = self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=self.add_special_tokens,
|
||||
max_total_tokens=max_total_tokens,
|
||||
max_output_tokens=max_output_tokens,
|
||||
)
|
||||
|
||||
if (
|
||||
self.padding == "max_length"
|
||||
and pooler_config is not None
|
||||
and pooler_config.enable_chunked_processing
|
||||
):
|
||||
max_padding_len = pooler_config.max_embed_len
|
||||
if max_padding_len is None:
|
||||
max_padding_len = default_max_total_tokens
|
||||
return tok_params.with_kwargs(
|
||||
pad_prompt_tokens=min(default_max_total_tokens, max_padding_len)
|
||||
)
|
||||
|
||||
return tok_params
|
||||
|
||||
|
||||
class CompletionRequestMixin(OpenAIBaseModel):
|
||||
# --8<-- [start:completion-params]
|
||||
input: list[int] | list[list[int]] | str | list[str]
|
||||
# --8<-- [end:completion-params]
|
||||
|
||||
# --8<-- [start:completion-extra-params]
|
||||
add_special_tokens: bool = Field(
|
||||
default=True,
|
||||
description=(
|
||||
"If true (the default), special tokens (e.g. BOS) will be added to "
|
||||
"the prompt."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:completion-extra-params]
|
||||
|
||||
|
||||
class ChatRequestOptionsMixin(OpenAIBaseModel):
|
||||
# --8<-- [start:chat-extra-params]
|
||||
add_generation_prompt: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, the generation prompt will be added to the chat template. "
|
||||
"This is a parameter used by chat template in tokenizer config of the "
|
||||
"model."
|
||||
),
|
||||
)
|
||||
continue_final_message: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If this is set, the chat will be formatted so that the final "
|
||||
"message in the chat is open-ended, without any EOS tokens. The "
|
||||
"model will continue this message rather than starting a new one. "
|
||||
'This allows you to "prefill" part of the model\'s response for it. '
|
||||
"Cannot be used at the same time as `add_generation_prompt`."
|
||||
),
|
||||
)
|
||||
add_special_tokens: bool = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, special tokens (e.g. BOS) will be added to the prompt "
|
||||
"on top of what is added by the chat template. "
|
||||
"For most models, the chat template takes care of adding the "
|
||||
"special tokens so this should be set to false (as is the "
|
||||
"default)."
|
||||
),
|
||||
)
|
||||
chat_template: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"A Jinja template to use for this conversion. "
|
||||
"As of transformers v4.44, default chat template is no longer "
|
||||
"allowed, so you must provide a chat template if the tokenizer "
|
||||
"does not define one."
|
||||
),
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the template renderer. "
|
||||
"Will be accessible by the chat template."
|
||||
),
|
||||
)
|
||||
media_io_kwargs: dict[str, dict[str, Any]] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional kwargs to pass to the media IO connectors, "
|
||||
"keyed by modality. Merged with engine-level media_io_kwargs."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:chat-extra-params]
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_generation_prompt(cls, data):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
if data.get("continue_final_message") and data.get("add_generation_prompt"):
|
||||
raise VLLMValidationError(
|
||||
"Cannot set both `continue_final_message` and "
|
||||
"`add_generation_prompt` to True.",
|
||||
)
|
||||
return data
|
||||
|
||||
def build_chat_params(
|
||||
self,
|
||||
default_template: str | None,
|
||||
default_template_content_format: ChatTemplateContentFormatOption,
|
||||
) -> ChatParams:
|
||||
return ChatParams(
|
||||
chat_template=self.chat_template or default_template,
|
||||
chat_template_content_format=default_template_content_format,
|
||||
chat_template_kwargs=merge_kwargs(
|
||||
self.chat_template_kwargs,
|
||||
dict(
|
||||
add_generation_prompt=self.add_generation_prompt,
|
||||
continue_final_message=self.continue_final_message,
|
||||
),
|
||||
),
|
||||
media_io_kwargs=self.media_io_kwargs,
|
||||
)
|
||||
|
||||
|
||||
class ChatRequestMixin(ChatRequestOptionsMixin):
|
||||
# --8<-- [start:chat-params]
|
||||
messages: list[ChatCompletionMessageParam]
|
||||
# --8<-- [end:chat-params]
|
||||
|
||||
|
||||
class EncodingRequestMixin(OpenAIBaseModel):
|
||||
# --8<-- [start:encoding-params]
|
||||
encoding_format: EncodingFormat = "float"
|
||||
# --8<-- [end:encoding-params]
|
||||
|
||||
# --8<-- [start:encoding-extra-params]
|
||||
embed_dtype: EmbedDType = Field(
|
||||
default="float32",
|
||||
description=(
|
||||
"What dtype to use for encoding. Default to using float32 for base64 "
|
||||
"encoding to match the OpenAI python client behavior. "
|
||||
"This parameter will affect base64 and binary_response."
|
||||
),
|
||||
)
|
||||
endianness: Endianness = Field(
|
||||
default="native",
|
||||
description=(
|
||||
"What endianness to use for encoding. Default to using native for "
|
||||
"base64 encoding to match the OpenAI python client behavior."
|
||||
"This parameter will affect base64 and binary_response."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:encoding-extra-params]
|
||||
|
||||
|
||||
class EmbedRequestMixin(EncodingRequestMixin):
|
||||
# --8<-- [start:embed-params]
|
||||
dimensions: int | None = None
|
||||
# --8<-- [end:embed-params]
|
||||
|
||||
# --8<-- [start:embed-extra-params]
|
||||
use_activation: bool | None = Field(
|
||||
default=None,
|
||||
description="Whether to use activation for the pooler outputs. "
|
||||
"`None` uses the pooler's default, which is `True` in most cases.",
|
||||
)
|
||||
# --8<-- [end:embed-extra-params]
|
||||
|
||||
|
||||
class ClassifyRequestMixin(OpenAIBaseModel):
|
||||
# --8<-- [start:classify-extra-params]
|
||||
use_activation: bool | None = Field(
|
||||
default=None,
|
||||
description="Whether to use activation for the pooler outputs. "
|
||||
"`None` uses the pooler's default, which is `True` in most cases.",
|
||||
)
|
||||
# --8<-- [end:classify-extra-params]
|
||||
@@ -1,280 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncGenerator
|
||||
from concurrent.futures import Executor
|
||||
from http import HTTPStatus
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from fastapi import Request
|
||||
from fastapi.responses import Response
|
||||
|
||||
from vllm import PoolingRequestOutput, envs
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.base.protocol import validate_request_mm_kwargs
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import GenerationError, RetryableRequestError
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers.base import BaseRenderer
|
||||
from vllm.renderers.chat_utils import ChatTemplateConfig
|
||||
from vllm.utils.async_utils import make_async, merge_async_iterators
|
||||
|
||||
from ...serve.engine.protocol import ErrorResponse
|
||||
from ..typing import AnyPoolingRequest, PoolingServeContext
|
||||
from .io_processor import PoolingIOProcessor
|
||||
|
||||
|
||||
class PoolingBaseServing(ABC, BaseServing):
|
||||
request_id_prefix: ClassVar[str]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
models: OpenAIServingModels,
|
||||
*,
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template_config: ChatTemplateConfig,
|
||||
return_tokens_as_token_ids: bool = False,
|
||||
log_error_stack: bool = False,
|
||||
):
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=models.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
self.engine_client = engine_client
|
||||
self.renderer = engine_client.renderer
|
||||
self.vllm_config = engine_client.vllm_config
|
||||
self.max_model_len = self.model_config.max_model_len
|
||||
self.return_tokens_as_token_ids = return_tokens_as_token_ids
|
||||
self.log_error_stack = log_error_stack
|
||||
self.chat_template_config = chat_template_config
|
||||
self.trust_request_mm_kwargs = chat_template_config.trust_request_mm_kwargs
|
||||
|
||||
# Shared thread pool executor for preprocessing and postprocessing.
|
||||
self._executor: Executor = self.renderer._executor
|
||||
self._postprocessing_async = make_async(
|
||||
self._postprocessing, executor=self._executor
|
||||
)
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
request: AnyPoolingRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> Response:
|
||||
io_processor = self.get_io_processor(request)
|
||||
ctx = await self._init_ctx(io_processor, request, raw_request)
|
||||
await self._preprocessing(io_processor, ctx)
|
||||
await self._prepare_generators(ctx)
|
||||
await self._collect_batch(ctx)
|
||||
return await self._postprocessing_async(io_processor, ctx)
|
||||
|
||||
@abstractmethod
|
||||
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
|
||||
raise NotImplementedError
|
||||
|
||||
async def _preprocessing(
|
||||
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
|
||||
):
|
||||
requests = io_processor.get_request_factory_online(ctx)
|
||||
|
||||
if len(requests) == 0:
|
||||
raise ValueError("You must pass at least one prompt")
|
||||
|
||||
ctx.engine_inputs = await asyncio.gather(
|
||||
*[io_processor.render_async(request) for request in requests]
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
def _postprocessing(
|
||||
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
|
||||
):
|
||||
io_processor.post_process_online(ctx)
|
||||
return self._build_response(ctx)
|
||||
|
||||
async def _init_ctx(
|
||||
self,
|
||||
io_processor: PoolingIOProcessor,
|
||||
request: AnyPoolingRequest,
|
||||
raw_request: Request | None = None,
|
||||
):
|
||||
validate_request_mm_kwargs(
|
||||
mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
|
||||
media_io_kwargs=getattr(request, "media_io_kwargs", None),
|
||||
trust_request_mm_kwargs=self.trust_request_mm_kwargs,
|
||||
)
|
||||
base_request_id = self._base_request_id(
|
||||
raw_request, getattr(request, "request_id", None)
|
||||
)
|
||||
request_id = f"{self.request_id_prefix}-{base_request_id}"
|
||||
await self._check_model(request)
|
||||
|
||||
pooling_params = io_processor.create_pooling_params(request)
|
||||
lora_request = self._maybe_get_adapters(request)
|
||||
model_name = self.models.model_name(lora_request)
|
||||
priorities = getattr(request, "priority", 0)
|
||||
prompt_extras = {
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt", "chat_template_kwargs")
|
||||
if (v := getattr(request, k, None)) is not None
|
||||
}
|
||||
|
||||
ctx = PoolingServeContext(
|
||||
request=request,
|
||||
raw_request=raw_request,
|
||||
model_name=model_name,
|
||||
pooling_params=pooling_params,
|
||||
request_id=request_id,
|
||||
lora_request=lora_request,
|
||||
priorities=priorities,
|
||||
prompt_extras=prompt_extras,
|
||||
)
|
||||
|
||||
self._validate_request(ctx)
|
||||
return ctx
|
||||
|
||||
async def _prepare_generators(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
if ctx.engine_inputs is None:
|
||||
raise ValueError("Engine inputs not available")
|
||||
|
||||
generators: list[AsyncGenerator[PoolingRequestOutput, None]] = []
|
||||
|
||||
trace_headers = (
|
||||
None
|
||||
if ctx.raw_request is None
|
||||
else await self._get_trace_headers(ctx.raw_request.headers)
|
||||
)
|
||||
|
||||
assert ctx.pooling_params is not None
|
||||
pooling_params = ctx.pooling_params
|
||||
pooling_params.verify(self.model_config)
|
||||
|
||||
for i, engine_input in enumerate(ctx.engine_inputs):
|
||||
prompt_request_id = (
|
||||
f"{ctx.request_id}-{i}"
|
||||
if ctx.prompt_request_ids is None
|
||||
else ctx.prompt_request_ids[i]
|
||||
)
|
||||
|
||||
self._log_inputs(
|
||||
prompt_request_id,
|
||||
engine_input["prompts"],
|
||||
params=engine_input["params"],
|
||||
lora_request=ctx.lora_request,
|
||||
)
|
||||
|
||||
generator = self.engine_client.encode(
|
||||
request_id=prompt_request_id,
|
||||
prompt=engine_input["prompts"],
|
||||
pooling_params=engine_input["params"],
|
||||
lora_request=engine_input["lora_requests"],
|
||||
priority=engine_input["priorities"],
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
|
||||
generators.append(generator)
|
||||
|
||||
ctx.result_generator = merge_async_iterators(*generators)
|
||||
|
||||
async def _collect_batch(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
if ctx.engine_inputs is None:
|
||||
raise ValueError("Engine prompts not available")
|
||||
|
||||
if ctx.result_generator is None:
|
||||
raise ValueError("Result generator not available")
|
||||
|
||||
num_inputs = len(ctx.engine_inputs)
|
||||
final_res_batch: list[PoolingRequestOutput | None]
|
||||
final_res_batch = [None] * num_inputs
|
||||
|
||||
async for i, res in ctx.result_generator:
|
||||
if res.error is not None:
|
||||
if res.error.retryable:
|
||||
raise RetryableRequestError(res.error.message)
|
||||
raise GenerationError(res.error.message)
|
||||
final_res_batch[i] = res
|
||||
|
||||
if None in final_res_batch:
|
||||
raise ValueError("Failed to generate results for all prompts")
|
||||
|
||||
ctx.final_res_batch = [res for res in final_res_batch if res is not None]
|
||||
|
||||
@abstractmethod
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> Response:
|
||||
raise NotImplementedError
|
||||
|
||||
async def _check_model(
|
||||
self,
|
||||
request: AnyRequest | AnyPoolingRequest,
|
||||
) -> ErrorResponse | None:
|
||||
if self._is_model_supported(request.model):
|
||||
return None
|
||||
if request.model in self.models.lora_requests:
|
||||
return None
|
||||
if (
|
||||
envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING
|
||||
and request.model
|
||||
and (load_result := await self.models.resolve_lora(request.model))
|
||||
):
|
||||
if isinstance(load_result, LoRARequest):
|
||||
return None
|
||||
if (
|
||||
isinstance(load_result, ErrorResponse)
|
||||
and load_result.error.code == HTTPStatus.BAD_REQUEST.value
|
||||
):
|
||||
raise ValueError(load_result.error.message)
|
||||
return None
|
||||
|
||||
def _validate_request(self, ctx: PoolingServeContext) -> None:
|
||||
truncate_prompt_tokens = getattr(ctx.request, "truncate_prompt_tokens", None)
|
||||
|
||||
if (
|
||||
truncate_prompt_tokens is not None
|
||||
and truncate_prompt_tokens > self.max_model_len
|
||||
):
|
||||
raise ValueError(
|
||||
"truncate_prompt_tokens value is "
|
||||
"greater than max_model_len."
|
||||
" Please request a smaller truncation size."
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
class PoolingServing(PoolingBaseServing, ABC):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.io_processor = self.init_io_processor(
|
||||
vllm_config=self.vllm_config,
|
||||
renderer=self.renderer,
|
||||
chat_template_config=self.chat_template_config,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def init_io_processor(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
renderer: BaseRenderer,
|
||||
chat_template_config: ChatTemplateConfig,
|
||||
) -> PoolingIOProcessor:
|
||||
raise NotImplementedError
|
||||
|
||||
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
|
||||
return self.io_processor
|
||||
@@ -1,33 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import Response
|
||||
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
from .protocol import ClassificationRequest
|
||||
from .serving import ServingClassification
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def classify(request: Request) -> ServingClassification:
|
||||
handler = getattr(request.app.state, "serving_classification", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Classification API")
|
||||
return handler
|
||||
|
||||
|
||||
@router.post("/classify", dependencies=[Depends(validate_json_request)])
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_classify(
|
||||
request: ClassificationRequest, raw_request: Request
|
||||
) -> Response:
|
||||
handler = classify(raw_request)
|
||||
return await handler(request, raw_request)
|
||||
@@ -1,12 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
|
||||
|
||||
class ClassifyIOProcessor(PoolingIOProcessor):
|
||||
name = "classify"
|
||||
|
||||
|
||||
class TokenClassifyIOProcessor(PoolingIOProcessor):
|
||||
name = "token_classify"
|
||||
@@ -1,71 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import time
|
||||
from typing import Annotated, TypeAlias
|
||||
|
||||
from pydantic import BeforeValidator, Field
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import (
|
||||
ChatRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
PoolingBasicRequestMixin,
|
||||
reject_removed_pooling_parameters,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ClassificationCompletionRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
):
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="classify",
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
class ClassificationChatRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
ChatRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
):
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="classify",
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
ClassificationRequest: TypeAlias = Annotated[
|
||||
ClassificationCompletionRequest | ClassificationChatRequest,
|
||||
BeforeValidator(reject_removed_pooling_parameters),
|
||||
]
|
||||
|
||||
|
||||
class ClassificationData(OpenAIBaseModel):
|
||||
index: int
|
||||
label: str | None
|
||||
probs: list[float]
|
||||
num_classes: int
|
||||
|
||||
|
||||
class ClassificationResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"classify-{random_uuid()}")
|
||||
object: str = "list"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
data: list[ClassificationData]
|
||||
usage: UsageInfo
|
||||
@@ -1,72 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TypeAlias
|
||||
|
||||
import numpy as np
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import ClassificationOutput
|
||||
|
||||
from ..base.serving import PoolingServing
|
||||
from ..typing import PoolingServeContext
|
||||
from .io_processor import ClassifyIOProcessor
|
||||
from .protocol import (
|
||||
ClassificationData,
|
||||
ClassificationRequest,
|
||||
ClassificationResponse,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
ClassificationServeContext: TypeAlias = PoolingServeContext[ClassificationRequest]
|
||||
|
||||
|
||||
class ServingClassification(PoolingServing):
|
||||
request_id_prefix = "classify"
|
||||
|
||||
def init_io_processor(self, *args, **kwargs) -> ClassifyIOProcessor:
|
||||
return ClassifyIOProcessor(*args, **kwargs)
|
||||
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: ClassificationServeContext,
|
||||
) -> JSONResponse:
|
||||
id2label = getattr(self.model_config.hf_config, "id2label", {})
|
||||
num_prompt_tokens = 0
|
||||
items: list[ClassificationData] = []
|
||||
for idx, final_res in enumerate(ctx.final_res_batch):
|
||||
classify_res = ClassificationOutput.from_base(final_res.outputs)
|
||||
|
||||
probs = classify_res.probs
|
||||
predicted_index = int(np.argmax(probs))
|
||||
label = id2label.get(predicted_index)
|
||||
|
||||
item = ClassificationData(
|
||||
index=idx,
|
||||
label=label,
|
||||
probs=probs,
|
||||
num_classes=len(probs),
|
||||
)
|
||||
|
||||
items.append(item)
|
||||
prompt_token_ids = final_res.prompt_token_ids
|
||||
num_prompt_tokens += len(prompt_token_ids)
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
total_tokens=num_prompt_tokens,
|
||||
)
|
||||
|
||||
response = ClassificationResponse(
|
||||
id=ctx.request_id,
|
||||
created=ctx.created_time,
|
||||
model=ctx.model_name,
|
||||
data=items,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
return JSONResponse(content=response.model_dump())
|
||||
@@ -1,61 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
from .protocol import CohereEmbedRequest, EmbeddingRequest
|
||||
from .serving import ServingEmbedding
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def embedding(request: Request) -> ServingEmbedding:
|
||||
handler = getattr(request.app.state, "serving_embedding", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Embeddings API")
|
||||
return handler
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/embeddings",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_embedding(
|
||||
request: EmbeddingRequest,
|
||||
raw_request: Request,
|
||||
):
|
||||
handler = embedding(raw_request)
|
||||
return await handler(request, raw_request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v2/embed",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_cohere_embedding(
|
||||
request: CohereEmbedRequest,
|
||||
raw_request: Request,
|
||||
):
|
||||
handler = embedding(raw_request)
|
||||
return await handler(request, raw_request)
|
||||
@@ -1,777 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
import torch
|
||||
from openai.types.chat import (
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
)
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingOutput, PoolingRequestOutput
|
||||
from vllm.renderers import merge_kwargs
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartParam,
|
||||
ChatCompletionMessageParam,
|
||||
CustomChatCompletionMessageParam,
|
||||
ImageURL,
|
||||
)
|
||||
from vllm.renderers.hf import resolve_chat_template
|
||||
from vllm.utils.collection_utils import chunk_list
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..late_chunking import (
|
||||
attach_late_chunking_metadata,
|
||||
build_late_chunking_metadata,
|
||||
prepare_late_chunking_input,
|
||||
)
|
||||
from ..scoring.io_processor import JinaRankingIOProcessorMixin
|
||||
from ..typing import (
|
||||
AnyOfflineInputsContext,
|
||||
AnyRenderParam,
|
||||
ChunkedEmbeddingMetadata,
|
||||
EncodeChatRenderParams,
|
||||
LateChunkingSlot,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
PoolingChatLikeRequest,
|
||||
PoolingCompletionLikeRequest,
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
RequestGenerator,
|
||||
)
|
||||
from .protocol import (
|
||||
CohereEmbedContent,
|
||||
CohereEmbedInput,
|
||||
CohereEmbedRequest,
|
||||
EmbeddingBatchChatInputRequest,
|
||||
EmbeddingBatchChatRequest,
|
||||
EmbeddingChatInputRequest,
|
||||
EmbeddingChatRequest,
|
||||
EmbeddingCompletionRequest,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _ChunkedPromptAggregator:
|
||||
weighted_sum: torch.Tensor | None = None
|
||||
total_weight: int = 0
|
||||
num_chunks: int = 0
|
||||
all_chunks_l2_normalized: bool = True
|
||||
|
||||
|
||||
class EmbedIOProcessor(PoolingIOProcessor):
|
||||
name = "embed"
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
assert self.model_config.pooler_config is not None
|
||||
|
||||
self.pooler_config = self.model_config.pooler_config
|
||||
self.enable_chunked_processing = self.pooler_config.enable_chunked_processing
|
||||
|
||||
# Load task instructions from HF config or sentence-transformers config
|
||||
self.task_instructions: dict[str, str] | None = self._load_task_instructions(
|
||||
self.model_config.hf_config
|
||||
) or self._load_st_prompts(self.model_config.model, self.model_config.revision)
|
||||
if self.task_instructions:
|
||||
logger.info(
|
||||
"Loaded prompt prefixes for input_type: %s",
|
||||
list(self.task_instructions.keys()),
|
||||
)
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
if isinstance(ctx.request, CohereEmbedRequest):
|
||||
requests = self._get_request_factory_cohere_online(ctx)
|
||||
elif isinstance(
|
||||
ctx.request,
|
||||
(
|
||||
EmbeddingChatRequest,
|
||||
EmbeddingBatchChatRequest,
|
||||
EmbeddingChatInputRequest,
|
||||
EmbeddingBatchChatInputRequest,
|
||||
),
|
||||
):
|
||||
requests = self._get_request_factory_chat_input_online(ctx)
|
||||
else:
|
||||
requests = super().get_request_factory_online(ctx)
|
||||
|
||||
return requests
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
if ctx.final_res_batch is None:
|
||||
raise ValueError("Final response batch not available")
|
||||
|
||||
if not self.enable_chunked_processing:
|
||||
self._enforce_cohere_max_tokens(ctx)
|
||||
return super().post_process_online(ctx)
|
||||
|
||||
self._post_process_chunked(ctx)
|
||||
self._enforce_cohere_max_tokens(ctx)
|
||||
|
||||
#################################################################
|
||||
# Long Text Embedding with Chunked Processing
|
||||
# PTAL: examples/pooling/embed/openai_embedding_long_text
|
||||
#################################################################
|
||||
|
||||
def maybe_pre_process_chunked(self, ctx: PoolingServeContext) -> None:
|
||||
if not self.enable_chunked_processing:
|
||||
return None
|
||||
|
||||
if ctx.engine_inputs is None:
|
||||
raise ValueError("Engine prompts not available")
|
||||
|
||||
ctx.original_engine_inputs = ctx.engine_inputs
|
||||
request_id = ctx.request_id
|
||||
max_model_len = self.model_config.max_model_len
|
||||
chunked_engine_inputs: list[PoolingEngineInput] = []
|
||||
prompt_request_ids: list[str] = []
|
||||
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] = []
|
||||
for prompt_idx, engine_input in enumerate(ctx.engine_inputs):
|
||||
token_ids = engine_input["prompts"].get("prompt_token_ids", None)
|
||||
if token_ids is None:
|
||||
raise NotImplementedError(
|
||||
"Long Text Embedding with Chunked Processing does "
|
||||
"not support EmbedsPrompt and EncoderDecoderInput."
|
||||
)
|
||||
|
||||
prompt_token_ids = cast(list[int], token_ids)
|
||||
cache_salt = cast(str | None, engine_input["prompts"].get("cache_salt"))
|
||||
|
||||
for chunk_idx, chunk_tokens in enumerate(
|
||||
chunk_list(prompt_token_ids, max_model_len)
|
||||
):
|
||||
chunked_engine_inputs.append(
|
||||
PoolingEngineInput(
|
||||
prompts=tokens_input(
|
||||
prompt_token_ids=chunk_tokens, cache_salt=cache_salt
|
||||
),
|
||||
params=engine_input["params"],
|
||||
lora_requests=engine_input["lora_requests"],
|
||||
priorities=engine_input["priorities"],
|
||||
)
|
||||
)
|
||||
prompt_request_ids.append(
|
||||
f"{request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}"
|
||||
)
|
||||
chunked_embedding_metadata.append(
|
||||
ChunkedEmbeddingMetadata(
|
||||
prompt_index=prompt_idx,
|
||||
chunk_index=chunk_idx,
|
||||
)
|
||||
)
|
||||
|
||||
ctx.engine_inputs = chunked_engine_inputs
|
||||
ctx.prompt_request_ids = prompt_request_ids
|
||||
ctx.chunked_embedding_metadata = chunked_embedding_metadata
|
||||
|
||||
return None
|
||||
|
||||
def _post_process_chunked(self, ctx: PoolingServeContext) -> None:
|
||||
# Online aggregation for chunked requests to
|
||||
# minimize memory usage
|
||||
# Track aggregation state for each prompt
|
||||
if ctx.chunked_embedding_metadata is None:
|
||||
raise ValueError("Chunked embedding metadata not available")
|
||||
if len(ctx.chunked_embedding_metadata) != len(ctx.final_res_batch):
|
||||
raise ValueError(
|
||||
"Chunked embedding metadata count does not match result count"
|
||||
)
|
||||
|
||||
prompt_aggregators: dict[int, _ChunkedPromptAggregator] = {}
|
||||
for result, chunk_metadata in zip(
|
||||
ctx.final_res_batch, ctx.chunked_embedding_metadata
|
||||
):
|
||||
prompt_idx = chunk_metadata.prompt_index
|
||||
aggregator = prompt_aggregators.setdefault(
|
||||
prompt_idx, _ChunkedPromptAggregator()
|
||||
)
|
||||
|
||||
# MEAN pooling with online weighted averaging
|
||||
# Ensure result is PoolingRequestOutput
|
||||
# for embedding processing
|
||||
if not isinstance(result, PoolingRequestOutput):
|
||||
raise ValueError(
|
||||
f"Expected PoolingRequestOutput for "
|
||||
f"chunked embedding, got "
|
||||
f"{type(result).__name__}"
|
||||
)
|
||||
if result.prompt_token_ids is None:
|
||||
raise ValueError(
|
||||
"prompt_token_ids cannot be None for chunked processing"
|
||||
)
|
||||
|
||||
weight = len(result.prompt_token_ids)
|
||||
embedding_data = result.outputs.data
|
||||
float_embedding = embedding_data.to(dtype=torch.float32)
|
||||
if ctx.pooling_params.use_activation:
|
||||
# Custom activations need not L2-normalize; allow BF16 rounding.
|
||||
embedding_norm = torch.linalg.vector_norm(float_embedding)
|
||||
aggregator.all_chunks_l2_normalized &= torch.allclose(
|
||||
embedding_norm,
|
||||
torch.ones_like(embedding_norm),
|
||||
rtol=1e-2,
|
||||
atol=1e-5,
|
||||
)
|
||||
weighted_embedding = float_embedding * weight
|
||||
|
||||
if aggregator.weighted_sum is None:
|
||||
# First chunk
|
||||
aggregator.weighted_sum = weighted_embedding
|
||||
else:
|
||||
# Accumulate
|
||||
aggregator.weighted_sum += weighted_embedding
|
||||
|
||||
aggregator.total_weight += weight
|
||||
aggregator.num_chunks += 1
|
||||
|
||||
if ctx.original_engine_inputs is None:
|
||||
raise ValueError("Original engine inputs not available")
|
||||
|
||||
original_engine_inputs = ctx.original_engine_inputs
|
||||
num_prompts = len(original_engine_inputs)
|
||||
|
||||
# Finalize aggregated results
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
for prompt_idx in range(num_prompts):
|
||||
if prompt_idx in prompt_aggregators:
|
||||
# Finalize MEAN aggregation for this chunked prompt
|
||||
aggregator = prompt_aggregators[prompt_idx]
|
||||
|
||||
weighted_sum = aggregator.weighted_sum
|
||||
total_weight = aggregator.total_weight
|
||||
|
||||
if (
|
||||
weighted_sum is not None
|
||||
and isinstance(weighted_sum, torch.Tensor)
|
||||
and isinstance(total_weight, (int, float))
|
||||
and total_weight > 0
|
||||
):
|
||||
# Compute final mean embedding
|
||||
final_embedding = weighted_sum / total_weight
|
||||
if (
|
||||
ctx.pooling_params.use_activation
|
||||
and aggregator.num_chunks > 1
|
||||
and aggregator.all_chunks_l2_normalized
|
||||
):
|
||||
final_embedding = torch.nn.functional.normalize(
|
||||
final_embedding, dim=-1
|
||||
)
|
||||
|
||||
# Create a PoolingRequestOutput
|
||||
# for the aggregated result
|
||||
pooling_output_data = PoolingOutput(data=final_embedding)
|
||||
|
||||
# Get original prompt token IDs for this prompt
|
||||
original_prompt = original_engine_inputs[prompt_idx]
|
||||
token_ids = original_prompt["prompts"].get("prompt_token_ids", None)
|
||||
if token_ids is None:
|
||||
raise NotImplementedError(
|
||||
"Long Text Embedding with Chunked Processing does "
|
||||
"not support EmbedsPrompt and EncoderDecoderInput."
|
||||
)
|
||||
|
||||
original_token_ids = cast(list[int], token_ids)
|
||||
pooling_request_output = PoolingRequestOutput(
|
||||
request_id=f"{ctx.request_id}-prompt-{prompt_idx}",
|
||||
prompt_token_ids=original_token_ids,
|
||||
outputs=pooling_output_data,
|
||||
num_cached_tokens=0,
|
||||
finished=True,
|
||||
)
|
||||
|
||||
final_res_batch.append(pooling_request_output)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Failed to aggregate chunks for prompt {prompt_idx}"
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Result not found for prompt {prompt_idx}")
|
||||
|
||||
ctx.final_res_batch = final_res_batch
|
||||
|
||||
return None
|
||||
|
||||
#################################################################
|
||||
# Chat input Request Preprocessing & Postprocessing
|
||||
#################################################################
|
||||
|
||||
def _get_request_factory_chat_input_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
renderer = self.renderer
|
||||
|
||||
self._validate_chat_template(
|
||||
request_chat_template=request.chat_template,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=self.trust_request_chat_template,
|
||||
)
|
||||
|
||||
if isinstance(
|
||||
request, (EmbeddingBatchChatRequest, EmbeddingBatchChatInputRequest)
|
||||
):
|
||||
all_messages = request.messages
|
||||
else:
|
||||
all_messages = [request.messages]
|
||||
num_requests = len(all_messages)
|
||||
|
||||
default_template_kwargs = merge_kwargs(
|
||||
self.template_kwargs,
|
||||
dict(
|
||||
tools=self.tool_dicts,
|
||||
tokenize=is_mistral_tokenizer(renderer.tokenizer),
|
||||
),
|
||||
)
|
||||
|
||||
mm_config = self.model_config.multimodal_config
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
chat_params = request.build_chat_params(
|
||||
self.chat_template, self.chat_template_content_format
|
||||
).with_defaults(
|
||||
default_template_kwargs,
|
||||
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
|
||||
)
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeChatRenderParams(
|
||||
conversations=all_messages[i],
|
||||
chat_params=chat_params,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
return requests
|
||||
|
||||
#################################################################
|
||||
# Cohere Request Preprocessing & Postprocessing
|
||||
#################################################################
|
||||
|
||||
@staticmethod
|
||||
def _load_task_instructions(hf_config: Any) -> dict[str, str] | None:
|
||||
"""Extract ``task_instructions`` from the HF model config."""
|
||||
ti = getattr(hf_config, "task_instructions", None)
|
||||
if not isinstance(ti, dict) or not ti:
|
||||
return None
|
||||
return {k: v for k, v in ti.items() if isinstance(v, str)}
|
||||
|
||||
@staticmethod
|
||||
def _load_st_prompts(
|
||||
model: str | Any,
|
||||
revision: str | None,
|
||||
) -> dict[str, str] | None:
|
||||
"""Load ``task_instructions`` from ``config_sentence_transformers.json``."""
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
||||
|
||||
try:
|
||||
cfg = get_hf_file_to_dict(
|
||||
"config_sentence_transformers.json", str(model), revision
|
||||
)
|
||||
except (ValueError, OSError):
|
||||
return None
|
||||
|
||||
if cfg is None:
|
||||
return None
|
||||
prompts = cfg.get("prompts")
|
||||
if not isinstance(prompts, dict) or not prompts:
|
||||
return None
|
||||
return {k: v for k, v in prompts.items() if isinstance(v, str)}
|
||||
|
||||
@staticmethod
|
||||
def _mixed_input_to_messages(
|
||||
inp: CohereEmbedInput,
|
||||
*,
|
||||
task_prefix: str | None = None,
|
||||
) -> list[ChatCompletionMessageParam]:
|
||||
"""Build chat messages from a mixed text+image input.
|
||||
|
||||
When *task_prefix* is given, it is used as the system prompt.
|
||||
"""
|
||||
messages: list[ChatCompletionMessageParam] = []
|
||||
if task_prefix is not None:
|
||||
messages.append(
|
||||
CustomChatCompletionMessageParam(
|
||||
role="system",
|
||||
content=[
|
||||
ChatCompletionContentPartTextParam(
|
||||
type="text", text=task_prefix
|
||||
)
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
parts: list[ChatCompletionContentPartParam] = []
|
||||
for item in inp.content:
|
||||
if item.type == "text" and item.text is not None:
|
||||
parts.append(
|
||||
ChatCompletionContentPartTextParam(type="text", text=item.text)
|
||||
)
|
||||
elif item.type == "image_url" and item.image_url is not None:
|
||||
parts.append(
|
||||
ChatCompletionContentPartImageParam(
|
||||
type="image_url",
|
||||
image_url=ImageURL(url=item.image_url["url"]),
|
||||
)
|
||||
)
|
||||
messages.append(CustomChatCompletionMessageParam(role="user", content=parts))
|
||||
return messages
|
||||
|
||||
@staticmethod
|
||||
def _check_cohere_max_tokens(
|
||||
outputs: list[PoolingRequestOutput],
|
||||
max_tokens_check: int | None,
|
||||
) -> None:
|
||||
"""Raise if any output exceeds *max_tokens_check* tokens.
|
||||
|
||||
Used to enforce ``truncate=NONE`` with an explicit ``max_tokens``:
|
||||
the pipeline runs without truncation and we reject afterwards.
|
||||
"""
|
||||
if max_tokens_check is None:
|
||||
return
|
||||
for out in outputs:
|
||||
n = len(out.prompt_token_ids)
|
||||
if n > max_tokens_check:
|
||||
raise ValueError(
|
||||
f"Input of {n} tokens exceeds max_tokens={max_tokens_check} "
|
||||
"with truncate=NONE. Set truncate to END or START to "
|
||||
"allow truncation."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_cohere_truncation(
|
||||
request: CohereEmbedRequest,
|
||||
) -> tuple[int | None, Literal["left", "right"] | None]:
|
||||
"""Return ``(truncate_prompt_tokens, truncation_side)``."""
|
||||
if request.truncate == "NONE":
|
||||
return None, None
|
||||
if request.truncate == "START":
|
||||
tokens = request.max_tokens if request.max_tokens is not None else -1
|
||||
return tokens, "left"
|
||||
if request.max_tokens is not None:
|
||||
return request.max_tokens, None
|
||||
return -1, None
|
||||
|
||||
def create_pooling_params(self, request):
|
||||
if isinstance(request, CohereEmbedRequest):
|
||||
return PoolingParams(
|
||||
task="embed",
|
||||
dimensions=request.output_dimension,
|
||||
)
|
||||
return super().create_pooling_params(request)
|
||||
|
||||
def _get_request_factory_cohere_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
"""Convert a ``CohereEmbedRequest`` into engine prompts.
|
||||
|
||||
If a model has a chat template the task instruction are rendered
|
||||
as a system prompt. Otherwise they are just prepended to the input text.
|
||||
|
||||
Images and mixed inputs are always batch-rendered through the chat
|
||||
template in one ``render_chat`` call.
|
||||
"""
|
||||
request = ctx.request
|
||||
assert isinstance(request, CohereEmbedRequest)
|
||||
|
||||
if request.texts is None and request.images is None and request.inputs is None:
|
||||
raise ValueError("One of texts, images, or inputs must be provided")
|
||||
|
||||
truncate_prompt_tokens, truncation_side = self._resolve_cohere_truncation(
|
||||
request
|
||||
)
|
||||
input_type = request.input_type
|
||||
self._validate_input_type(input_type)
|
||||
|
||||
if request.images is not None:
|
||||
input: list[CohereEmbedInput] = [
|
||||
CohereEmbedInput(
|
||||
content=[
|
||||
CohereEmbedContent(type="image_url", image_url={"url": uri})
|
||||
]
|
||||
)
|
||||
for uri in request.images
|
||||
]
|
||||
elif request.inputs is not None:
|
||||
input = request.inputs
|
||||
else:
|
||||
texts = request.texts or []
|
||||
task_prefix = self._get_task_instruction_prefix(input_type)
|
||||
|
||||
if task_prefix is None:
|
||||
return self._get_request_factory_cohere_text_completion(
|
||||
ctx,
|
||||
texts,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
|
||||
all_messages = [
|
||||
self._mixed_input_to_messages(
|
||||
CohereEmbedInput(
|
||||
content=[CohereEmbedContent(type="text", text=text)]
|
||||
),
|
||||
task_prefix=task_prefix,
|
||||
)
|
||||
for text in texts
|
||||
]
|
||||
if self._has_chat_template():
|
||||
return self._batch_render_chat(
|
||||
ctx,
|
||||
all_messages,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
else:
|
||||
return self._get_request_factory_cohere_text_completion(
|
||||
ctx,
|
||||
self._apply_task_instruction(texts, input_type),
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
|
||||
task_prefix = self._get_task_instruction_prefix(input_type)
|
||||
all_messages = [
|
||||
self._mixed_input_to_messages(inp, task_prefix=task_prefix) for inp in input
|
||||
]
|
||||
|
||||
return self._batch_render_chat(
|
||||
ctx,
|
||||
all_messages,
|
||||
truncate_prompt_tokens,
|
||||
truncation_side,
|
||||
)
|
||||
|
||||
def _has_chat_template(self) -> bool:
|
||||
return (
|
||||
resolve_chat_template(
|
||||
self.renderer.tokenizer,
|
||||
chat_template=self.chat_template,
|
||||
tools=None,
|
||||
model_config=self.model_config,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
|
||||
def _get_request_factory_cohere_text_completion(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
texts: list[str],
|
||||
truncate_prompt_tokens: int | None,
|
||||
truncation_side: Literal["left", "right"] | None,
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
proxy = EmbeddingCompletionRequest(
|
||||
model=request.model,
|
||||
input=texts,
|
||||
dimensions=request.output_dimension,
|
||||
encoding_format="float",
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
truncation_side=truncation_side,
|
||||
)
|
||||
ctx.request = proxy
|
||||
requests = super().get_request_factory_online(ctx)
|
||||
ctx.request = request
|
||||
return requests
|
||||
|
||||
def _batch_render_chat(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
all_messages: Sequence[list[ChatCompletionMessageParam]],
|
||||
truncate_prompt_tokens: int | None,
|
||||
truncation_side: Literal["left", "right"] | None,
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
"""Batch-render multiple conversations through the chat template."""
|
||||
request = ctx.request
|
||||
proxy = EmbeddingBatchChatRequest(
|
||||
model=request.model,
|
||||
messages=all_messages,
|
||||
dimensions=request.output_dimension,
|
||||
encoding_format="float",
|
||||
truncate_prompt_tokens=truncate_prompt_tokens,
|
||||
truncation_side=truncation_side,
|
||||
)
|
||||
ctx.request = proxy
|
||||
requests = self._get_request_factory_chat_input_online(ctx)
|
||||
ctx.request = request
|
||||
return requests
|
||||
|
||||
def _validate_input_type(self, input_type: str | None) -> None:
|
||||
"""Raise if *input_type* is not supported by this model."""
|
||||
if input_type is None:
|
||||
return
|
||||
if self.task_instructions is None:
|
||||
raise ValueError(
|
||||
f"Unsupported input_type {input_type!r}. "
|
||||
"This model does not define any input_type task instructions."
|
||||
)
|
||||
if input_type not in self.task_instructions:
|
||||
supported = ", ".join(sorted(self.task_instructions))
|
||||
raise ValueError(
|
||||
f"Unsupported input_type {input_type!r}. Supported values: {supported}"
|
||||
)
|
||||
|
||||
def _apply_task_instruction(
|
||||
self,
|
||||
texts: list[str],
|
||||
input_type: str | None,
|
||||
) -> list[str]:
|
||||
"""Prepend the task-instruction prefix for *input_type*.
|
||||
|
||||
Returns *texts* unchanged when no matching prefix is configured.
|
||||
"""
|
||||
prefix = self._get_task_instruction_prefix(input_type)
|
||||
if not prefix:
|
||||
return texts
|
||||
return [prefix + t for t in texts]
|
||||
|
||||
def _get_task_instruction_prefix(self, input_type: str | None) -> str | None:
|
||||
"""Return the task-instruction prefix for *input_type*, or ``None``."""
|
||||
if not self.task_instructions or input_type is None:
|
||||
return None
|
||||
return self.task_instructions.get(input_type) or None
|
||||
|
||||
def _enforce_cohere_max_tokens(self, ctx: PoolingServeContext) -> None:
|
||||
if isinstance(ctx.request, CohereEmbedRequest):
|
||||
request = ctx.request
|
||||
if request.truncate == "NONE" and request.max_tokens is not None:
|
||||
self._check_cohere_max_tokens(ctx.final_res_batch, request.max_tokens)
|
||||
|
||||
|
||||
class TokenEmbedIOProcessor(PoolingIOProcessor):
|
||||
name = "token_embed"
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
factory, num_requests = super().get_request_factory_offline(ctx)
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
if ctx.pooling_params is None or not any(
|
||||
p.late_chunking_params is not None
|
||||
for p in self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
):
|
||||
return factory, num_requests
|
||||
|
||||
if (ctx.tokenization_kwargs or {}).get("padding") not in (
|
||||
None,
|
||||
False,
|
||||
"do_not_pad",
|
||||
):
|
||||
raise VLLMValidationError("Late chunking does not support input padding")
|
||||
|
||||
def request_factory() -> RequestGenerator:
|
||||
for request in factory():
|
||||
slot = None
|
||||
if request["params"].late_chunking_params is not None:
|
||||
slot = LateChunkingSlot()
|
||||
request["late_chunking"] = slot
|
||||
ctx.late_chunking.append(slot)
|
||||
yield request
|
||||
|
||||
return request_factory, num_requests
|
||||
|
||||
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
|
||||
late_chunking = render_params["params"].late_chunking_params
|
||||
if late_chunking is None:
|
||||
return super().render(render_params)
|
||||
|
||||
text, tok_params = prepare_late_chunking_input(self.vllm_config, render_params)
|
||||
render_params = render_params.copy()
|
||||
render_params["tok_params"] = tok_params
|
||||
result = super().render(render_params)
|
||||
prompt = result["prompts"]
|
||||
if prompt["type"] != "token":
|
||||
raise VLLMValidationError("Late chunking requires tokenized text")
|
||||
render_params["late_chunking"].metadata = build_late_chunking_metadata(
|
||||
text,
|
||||
len(prompt["prompt_token_ids"]),
|
||||
prompt.pop("prompt_token_offsets", None),
|
||||
late_chunking.chunk_size,
|
||||
)
|
||||
return result
|
||||
|
||||
def post_process_offline(self, ctx: OfflineOutputsContext):
|
||||
if ctx.late_chunking:
|
||||
for output, slot in zip(ctx.outputs, ctx.late_chunking, strict=True):
|
||||
if slot is not None:
|
||||
assert slot.metadata is not None
|
||||
attach_late_chunking_metadata(output, slot.metadata)
|
||||
return ctx.outputs
|
||||
|
||||
|
||||
class JinaRankingTokenEmbedIOProcessor(
|
||||
TokenEmbedIOProcessor, JinaRankingIOProcessorMixin
|
||||
):
|
||||
def render(self, render_params: AnyRenderParam) -> PoolingEngineInput:
|
||||
if render_params["params"].late_chunking_params is not None:
|
||||
raise VLLMValidationError(
|
||||
"JinaForRanking returns document/query vectors and does not support "
|
||||
"late chunking"
|
||||
)
|
||||
return super().render(render_params)
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
if isinstance(request, PoolingCompletionLikeRequest):
|
||||
prompts = request.input
|
||||
if not isinstance(prompts, Sequence) or len(prompts) < 2:
|
||||
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
|
||||
|
||||
text_prompts = self.ensure_str(prompts)
|
||||
|
||||
# The JinaForRanking model concatenates docs first, then query.
|
||||
# Let's stay consistent with this novel design.
|
||||
prompt_input = self.format_docs_prompts_func(
|
||||
query=text_prompts[-1], docs=text_prompts[:-1]
|
||||
)
|
||||
|
||||
request.input = prompt_input
|
||||
return super().get_request_factory_online(ctx)
|
||||
elif isinstance(request, PoolingChatLikeRequest):
|
||||
raise ValueError("The JinaForRanking does not support chat Request.")
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.name} request type")
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineEncodeInputsContext)
|
||||
if not isinstance(ctx.prompts, Sequence) or len(ctx.prompts) < 2:
|
||||
raise ValueError("The JinaForRanking model requires at least 2 inputs.")
|
||||
|
||||
text_prompts = self.ensure_str(ctx.prompts)
|
||||
|
||||
# The JinaForRanking model concatenates docs first, then query.
|
||||
# Let's stay consistent with this novel design.
|
||||
ctx.prompts = self.format_docs_prompts_func(
|
||||
query=text_prompts[-1], docs=text_prompts[:-1]
|
||||
)
|
||||
|
||||
return super().get_request_factory_offline(ctx)
|
||||
@@ -1,354 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Embedding API protocol models for OpenAI and Cohere formats.
|
||||
|
||||
OpenAI: https://platform.openai.com/docs/api-reference/embeddings
|
||||
Cohere: https://docs.cohere.com/reference/embed
|
||||
"""
|
||||
|
||||
import builtins
|
||||
import struct
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from typing import Annotated, Any, Literal, TypeAlias
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
from pydantic import BaseModel, BeforeValidator, Field, model_validator
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import (
|
||||
ChatRequestMixin,
|
||||
ChatRequestOptionsMixin,
|
||||
CompletionRequestMixin,
|
||||
EmbeddingTokenizeParamsMixin,
|
||||
EmbedRequestMixin,
|
||||
PoolingBasicRequestMixin,
|
||||
reject_removed_pooling_parameters,
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingCompletionRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
EmbeddingTokenizeParamsMixin,
|
||||
):
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="embed",
|
||||
dimensions=self.dimensions,
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
def _is_chat_message(value: Any) -> bool:
|
||||
return isinstance(value, dict) and isinstance(value.get("role"), str)
|
||||
|
||||
|
||||
def _is_chat_messages(value: Any) -> bool:
|
||||
return (
|
||||
isinstance(value, list)
|
||||
and bool(value)
|
||||
and all(_is_chat_message(item) for item in value)
|
||||
)
|
||||
|
||||
|
||||
def _is_batched_chat_messages(value: Any) -> bool:
|
||||
return (
|
||||
isinstance(value, list)
|
||||
and bool(value)
|
||||
and all(_is_chat_messages(item) for item in value)
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingChatRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
ChatRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
EmbeddingTokenizeParamsMixin,
|
||||
):
|
||||
"""OpenAI embeddings request with one top-level chat conversation."""
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="embed",
|
||||
dimensions=self.dimensions,
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingBatchChatRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
ChatRequestOptionsMixin,
|
||||
EmbedRequestMixin,
|
||||
EmbeddingTokenizeParamsMixin,
|
||||
):
|
||||
"""OpenAI embeddings request with batched top-level chat conversations.
|
||||
|
||||
Mirrors ``BatchChatCompletionRequest`` by keeping batched conversations in
|
||||
``messages`` instead of introducing a separate batch-specific field.
|
||||
"""
|
||||
|
||||
messages: Sequence[
|
||||
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
|
||||
] = Field(..., min_length=1)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task="embed",
|
||||
dimensions=self.dimensions,
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
class EmbeddingChatInputRequest(
|
||||
EmbeddingChatRequest,
|
||||
):
|
||||
"""OpenAI embeddings request with one chat conversation in ``input``."""
|
||||
|
||||
input: list[ChatCompletionMessageParam]
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def normalize_input_messages(cls, data):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
if "messages" in data or "input" not in data:
|
||||
return data
|
||||
|
||||
input_data = data["input"]
|
||||
if not _is_chat_messages(input_data):
|
||||
return data
|
||||
|
||||
normalized = dict(data)
|
||||
normalized["messages"] = input_data
|
||||
return normalized
|
||||
|
||||
|
||||
class EmbeddingBatchChatInputRequest(EmbeddingBatchChatRequest):
|
||||
"""OpenAI embeddings request with batched chat conversations in ``input``."""
|
||||
|
||||
input: Sequence[
|
||||
Annotated[list[ChatCompletionMessageParam], Field(min_length=1)]
|
||||
] = Field(..., min_length=1)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def normalize_input_messages(cls, data):
|
||||
if not isinstance(data, dict):
|
||||
return data
|
||||
|
||||
if "messages" in data or "input" not in data:
|
||||
return data
|
||||
|
||||
input_data = data["input"]
|
||||
if not _is_batched_chat_messages(input_data):
|
||||
return data
|
||||
|
||||
normalized = dict(data)
|
||||
normalized["messages"] = input_data
|
||||
return normalized
|
||||
|
||||
|
||||
EmbeddingRequest: TypeAlias = Annotated[
|
||||
EmbeddingCompletionRequest
|
||||
| EmbeddingChatRequest
|
||||
| EmbeddingBatchChatRequest
|
||||
| EmbeddingChatInputRequest
|
||||
| EmbeddingBatchChatInputRequest,
|
||||
BeforeValidator(reject_removed_pooling_parameters),
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# OpenAI /v1/embeddings — response models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class EmbeddingResponseData(OpenAIBaseModel):
|
||||
index: int
|
||||
object: str = "embedding"
|
||||
embedding: list[float] | str
|
||||
|
||||
|
||||
class EmbeddingResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
|
||||
object: str = "list"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str | None = None
|
||||
data: list[EmbeddingResponseData]
|
||||
usage: UsageInfo
|
||||
|
||||
|
||||
class EmbeddingBytesResponse(OpenAIBaseModel):
|
||||
content: list[bytes]
|
||||
headers: dict[str, str] | None = None
|
||||
media_type: str = "application/octet-stream"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cohere /v2/embed — request models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CohereEmbeddingType = Literal[
|
||||
"float",
|
||||
"binary",
|
||||
"ubinary",
|
||||
"base64",
|
||||
]
|
||||
CohereTruncate = Literal["NONE", "START", "END"]
|
||||
|
||||
|
||||
class CohereEmbedContent(BaseModel):
|
||||
type: Literal["text", "image_url"]
|
||||
text: str | None = None
|
||||
image_url: dict[str, str] | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_content_payload(self):
|
||||
if self.type == "text":
|
||||
if self.text is None:
|
||||
raise ValueError("CohereEmbedContent with type='text' requires text")
|
||||
elif not self.image_url or not self.image_url.get("url"):
|
||||
raise ValueError(
|
||||
"CohereEmbedContent with type='image_url' requires image_url.url"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class CohereEmbedInput(BaseModel):
|
||||
content: list[CohereEmbedContent]
|
||||
|
||||
|
||||
class CohereEmbedRequest(BaseModel):
|
||||
model: str | None = None
|
||||
input_type: str | None = None
|
||||
texts: list[str] | None = None
|
||||
images: list[str] | None = None
|
||||
inputs: list[CohereEmbedInput] | None = None
|
||||
output_dimension: int | None = None
|
||||
embedding_types: list[CohereEmbeddingType] | None = None
|
||||
truncate: CohereTruncate = "END"
|
||||
max_tokens: int | None = None
|
||||
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_input_fields(self):
|
||||
input_fields = (self.texts, self.images, self.inputs)
|
||||
provided_fields = [field for field in input_fields if field is not None]
|
||||
if len(provided_fields) != 1 or not provided_fields[0]:
|
||||
raise ValueError(
|
||||
"Exactly one of texts, images, or inputs must be provided, "
|
||||
"and it must be non-empty"
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cohere /v2/embed — response models
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class CohereApiVersion(BaseModel):
|
||||
version: str = "2"
|
||||
|
||||
|
||||
class CohereBilledUnits(BaseModel):
|
||||
input_tokens: int | None = None
|
||||
image_tokens: int | None = None
|
||||
|
||||
|
||||
class CohereMeta(BaseModel):
|
||||
api_version: CohereApiVersion = Field(default_factory=CohereApiVersion)
|
||||
billed_units: CohereBilledUnits | None = None
|
||||
|
||||
|
||||
class CohereEmbedByTypeEmbeddings(BaseModel):
|
||||
# The field name ``float`` shadows the builtin type, so the annotation
|
||||
# must use ``builtins.float`` to avoid a self-referential type error.
|
||||
float: list[list[builtins.float]] | None = None
|
||||
binary: list[list[int]] | None = None
|
||||
ubinary: list[list[int]] | None = None
|
||||
base64: list[str] | None = None
|
||||
|
||||
|
||||
class CohereEmbedResponse(BaseModel):
|
||||
id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
|
||||
embeddings: CohereEmbedByTypeEmbeddings
|
||||
texts: list[str] | None = None
|
||||
meta: CohereMeta | None = None
|
||||
response_type: Literal["embeddings_by_type"] = "embeddings_by_type"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cohere embedding type conversion helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_UNSIGNED_TO_SIGNED_DIFF = 1 << 7 # 128
|
||||
|
||||
|
||||
def _pack_binary_embeddings(
|
||||
float_embeddings: list[list[float]],
|
||||
signed: bool,
|
||||
) -> list[list[int]]:
|
||||
"""Bit-pack float embeddings: positive -> 1, negative -> 0.
|
||||
|
||||
Bits are packed MSB-first, eight per byte.
|
||||
"""
|
||||
if not float_embeddings:
|
||||
return []
|
||||
|
||||
array = np.asarray(float_embeddings, dtype=np.float64)
|
||||
if array.ndim != 2:
|
||||
raise ValueError(
|
||||
f"Expected a 2D batch of embeddings, but got {array.ndim}D input."
|
||||
)
|
||||
|
||||
dim = array.shape[1]
|
||||
if dim % 8 != 0:
|
||||
raise ValueError(
|
||||
"Embedding dimension must be a multiple of 8 for binary "
|
||||
f"embedding types, but got {dim}."
|
||||
)
|
||||
|
||||
packed = np.packbits(array >= 0, axis=-1)
|
||||
if signed:
|
||||
packed = packed.astype(np.int16) - _UNSIGNED_TO_SIGNED_DIFF
|
||||
return packed.tolist()
|
||||
|
||||
|
||||
def _encode_base64_embeddings(
|
||||
float_embeddings: list[list[float]],
|
||||
) -> list[str]:
|
||||
"""Encode float embeddings as base64 (little-endian float32)."""
|
||||
result: list[str] = []
|
||||
for embedding in float_embeddings:
|
||||
buf = struct.pack(f"<{len(embedding)}f", *embedding)
|
||||
result.append(base64.b64encode(buf).decode("utf-8"))
|
||||
return result
|
||||
|
||||
|
||||
def build_typed_embeddings(
|
||||
float_embeddings: list[list[float]],
|
||||
embedding_types: Sequence[str],
|
||||
) -> CohereEmbedByTypeEmbeddings:
|
||||
"""Convert float embeddings to all requested Cohere embedding types."""
|
||||
result = CohereEmbedByTypeEmbeddings()
|
||||
|
||||
for emb_type in embedding_types:
|
||||
if emb_type == "float":
|
||||
result.float = float_embeddings
|
||||
elif emb_type == "binary":
|
||||
result.binary = _pack_binary_embeddings(float_embeddings, signed=True)
|
||||
elif emb_type == "ubinary":
|
||||
result.ubinary = _pack_binary_embeddings(float_embeddings, signed=False)
|
||||
elif emb_type == "base64":
|
||||
result.base64 = _encode_base64_embeddings(float_embeddings)
|
||||
|
||||
return result
|
||||
@@ -1,220 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TypeAlias, assert_never, cast
|
||||
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.utils.serial_utils import EmbedDType, Endianness
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..base.serving import PoolingServing
|
||||
from ..typing import PoolingServeContext
|
||||
from ..utils import (
|
||||
BytesEncodingFormat,
|
||||
JsonEncodingFormat,
|
||||
build_pooling_bytes_streaming_response,
|
||||
encode_pooling_output_float,
|
||||
encode_pooling_output_float_or_ndarray,
|
||||
get_json_response_cls,
|
||||
get_pooling_output_encoder,
|
||||
get_pooling_usage,
|
||||
)
|
||||
from .io_processor import EmbedIOProcessor
|
||||
from .protocol import (
|
||||
CohereBilledUnits,
|
||||
CohereEmbedRequest,
|
||||
CohereEmbedResponse,
|
||||
CohereMeta,
|
||||
EmbeddingRequest,
|
||||
EmbeddingResponse,
|
||||
EmbeddingResponseData,
|
||||
build_typed_embeddings,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
EmbeddingServeContext: TypeAlias = PoolingServeContext[EmbeddingRequest]
|
||||
|
||||
|
||||
class ServingEmbedding(PoolingServing):
|
||||
"""Embedding API supporting both OpenAI and Cohere formats."""
|
||||
|
||||
request_id_prefix = "embd"
|
||||
io_processor: EmbedIOProcessor
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.json_response_cls = get_json_response_cls()
|
||||
|
||||
def init_io_processor(self, *args, **kwargs) -> EmbedIOProcessor:
|
||||
return EmbedIOProcessor(*args, **kwargs)
|
||||
|
||||
async def _preprocessing(
|
||||
self, io_processor: PoolingIOProcessor, ctx: PoolingServeContext
|
||||
):
|
||||
await super()._preprocessing(io_processor, ctx)
|
||||
self.io_processor.maybe_pre_process_chunked(ctx)
|
||||
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> Response:
|
||||
if isinstance(ctx.request, CohereEmbedRequest):
|
||||
return self._build_cohere_response_from_ctx(ctx)
|
||||
return self._build_openai_response(ctx)
|
||||
|
||||
def _build_openai_response(
|
||||
self,
|
||||
ctx: EmbeddingServeContext,
|
||||
) -> JSONResponse | StreamingResponse:
|
||||
encoding_format = ctx.request.encoding_format
|
||||
embed_dtype = ctx.request.embed_dtype
|
||||
endianness = ctx.request.endianness
|
||||
|
||||
if encoding_format == "float" or encoding_format == "base64":
|
||||
return self._openai_json_response(
|
||||
ctx.final_res_batch,
|
||||
ctx.request_id,
|
||||
ctx.created_time,
|
||||
ctx.model_name,
|
||||
encoding_format,
|
||||
embed_dtype,
|
||||
endianness,
|
||||
)
|
||||
|
||||
if encoding_format == "bytes" or encoding_format == "bytes_only":
|
||||
return self._openai_bytes_response(
|
||||
ctx.final_res_batch,
|
||||
ctx.request_id,
|
||||
ctx.created_time,
|
||||
ctx.model_name,
|
||||
encoding_format,
|
||||
embed_dtype,
|
||||
endianness,
|
||||
)
|
||||
|
||||
assert_never(encoding_format)
|
||||
|
||||
def _openai_json_response(
|
||||
self,
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: JsonEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> JSONResponse:
|
||||
use_ndarray_response = (
|
||||
encoding_format == "float"
|
||||
and self.json_response_cls.__name__ == "ORJSONResponse"
|
||||
)
|
||||
if use_ndarray_response:
|
||||
ndarray_items: list[dict[str, object]] = []
|
||||
|
||||
for idx, final_res in enumerate(final_res_batch):
|
||||
item_dict = EmbeddingResponseData(
|
||||
index=idx,
|
||||
embedding=[],
|
||||
).model_dump()
|
||||
item_dict["embedding"] = encode_pooling_output_float_or_ndarray(
|
||||
final_res
|
||||
)
|
||||
ndarray_items.append(item_dict)
|
||||
ndarray_response = EmbeddingResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
data=[], # type: ignore[arg-type]
|
||||
usage=get_pooling_usage(final_res_batch),
|
||||
).model_dump()
|
||||
ndarray_response["data"] = ndarray_items
|
||||
|
||||
return self.json_response_cls(content=ndarray_response)
|
||||
|
||||
encode_fn = get_pooling_output_encoder(
|
||||
encoding_format=encoding_format,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
|
||||
items: list[EmbeddingResponseData] = []
|
||||
|
||||
for idx, final_res in enumerate(final_res_batch):
|
||||
item = EmbeddingResponseData(
|
||||
index=idx,
|
||||
embedding=cast(list[float] | str, encode_fn(final_res)),
|
||||
)
|
||||
|
||||
items.append(item)
|
||||
|
||||
response = EmbeddingResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
data=items,
|
||||
usage=get_pooling_usage(final_res_batch),
|
||||
)
|
||||
return self.json_response_cls(content=response.model_dump())
|
||||
|
||||
def _openai_bytes_response(
|
||||
self,
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: BytesEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> StreamingResponse:
|
||||
return build_pooling_bytes_streaming_response(
|
||||
pooling_outputs=final_res_batch,
|
||||
request_id=request_id,
|
||||
created_time=created_time,
|
||||
model_name=model_name,
|
||||
encoding_format=encoding_format,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
|
||||
def _build_cohere_response_from_ctx(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> JSONResponse:
|
||||
request = ctx.request
|
||||
assert isinstance(request, CohereEmbedRequest)
|
||||
|
||||
all_floats = [
|
||||
cast(list[float], encode_pooling_output_float(out))
|
||||
for out in ctx.final_res_batch
|
||||
]
|
||||
total_tokens = get_pooling_usage(ctx.final_res_batch).prompt_tokens
|
||||
|
||||
has_image_input = request.images is not None or any(
|
||||
content.type == "image_url"
|
||||
for input_item in request.inputs or []
|
||||
for content in input_item.content
|
||||
)
|
||||
image_tokens = total_tokens if has_image_input else 0
|
||||
texts_echo = request.texts
|
||||
|
||||
embedding_types = request.embedding_types or ["float"]
|
||||
embeddings_obj = build_typed_embeddings(all_floats, embedding_types)
|
||||
|
||||
input_tokens = total_tokens - image_tokens
|
||||
response = CohereEmbedResponse(
|
||||
id=ctx.request_id,
|
||||
embeddings=embeddings_obj,
|
||||
texts=texts_echo,
|
||||
meta=CohereMeta(
|
||||
billed_units=CohereBilledUnits(
|
||||
input_tokens=input_tokens,
|
||||
image_tokens=image_tokens,
|
||||
),
|
||||
),
|
||||
)
|
||||
return self.json_response_cls(content=response.model_dump(exclude_none=True))
|
||||
@@ -1,272 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.plugins.io_processors import has_io_processor
|
||||
from vllm.renderers import BaseRenderer
|
||||
from vllm.renderers.chat_utils import ChatTemplateConfig
|
||||
from vllm.tasks import POOLING_TASKS, SCORE_TYPE_MAP, SupportedTask
|
||||
|
||||
from .base.io_processor import PoolingIOProcessor
|
||||
from .utils import enable_scoring_api
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import Namespace
|
||||
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import (
|
||||
EndpointFn,
|
||||
GetHandlerFn,
|
||||
RequestType,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
|
||||
else:
|
||||
RequestLogger = object
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def init_pooling_io_processors(
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
vllm_config: VllmConfig,
|
||||
renderer: BaseRenderer,
|
||||
chat_template_config: ChatTemplateConfig,
|
||||
) -> dict[str, PoolingIOProcessor]:
|
||||
model_config = vllm_config.model_config
|
||||
processors: dict[str, type[PoolingIOProcessor]] = {}
|
||||
pooling_task = model_config.get_pooling_task(supported_tasks)
|
||||
|
||||
if pooling_task == "classify":
|
||||
from .classify.io_processor import ClassifyIOProcessor
|
||||
|
||||
processors["classify"] = ClassifyIOProcessor
|
||||
|
||||
if pooling_task == "token_classify":
|
||||
from .classify.io_processor import TokenClassifyIOProcessor
|
||||
|
||||
processors["token_classify"] = TokenClassifyIOProcessor
|
||||
|
||||
if pooling_task == "embed":
|
||||
from .embed.io_processor import EmbedIOProcessor
|
||||
|
||||
processors["embed"] = EmbedIOProcessor
|
||||
|
||||
if pooling_task == "token_embed":
|
||||
from .embed.io_processor import TokenEmbedIOProcessor
|
||||
|
||||
processors["token_embed"] = TokenEmbedIOProcessor
|
||||
|
||||
if pooling_task == "embed&token_classify":
|
||||
from .pooling.io_processor import UnsupportedCombinedTaskIOProcessor
|
||||
|
||||
processors[pooling_task] = UnsupportedCombinedTaskIOProcessor
|
||||
|
||||
has_plugin = has_io_processor(
|
||||
vllm_config,
|
||||
model_config.io_processor_plugin,
|
||||
)
|
||||
if has_plugin:
|
||||
from .pooling.io_processor import PluginWithIOProcessorPlugins
|
||||
|
||||
processors["plugin"] = PluginWithIOProcessorPlugins
|
||||
elif pooling_task == "plugin":
|
||||
from .pooling.io_processor import PluginWithoutIOProcessorPlugins
|
||||
|
||||
processors["plugin"] = PluginWithoutIOProcessorPlugins
|
||||
|
||||
if enable_scoring_api(supported_tasks, model_config):
|
||||
from .scoring.io_processor import ScoringIOProcessors
|
||||
|
||||
score_type: str | None = SCORE_TYPE_MAP.get(pooling_task, None) # type: ignore[arg-type]
|
||||
if score_type is not None and score_type in ScoringIOProcessors:
|
||||
processors[score_type] = ScoringIOProcessors[score_type]
|
||||
|
||||
if model_config.architecture == "JinaForRanking":
|
||||
from .embed.io_processor import JinaRankingTokenEmbedIOProcessor
|
||||
from .scoring.io_processor import ScoringIOProcessors
|
||||
|
||||
processors["token_embed"] = JinaRankingTokenEmbedIOProcessor
|
||||
processors["late-interaction"] = ScoringIOProcessors["jina-reranking-scoring"]
|
||||
|
||||
return {
|
||||
task: processor_cls(
|
||||
vllm_config=vllm_config,
|
||||
renderer=renderer,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
for task, processor_cls in processors.items()
|
||||
}
|
||||
|
||||
|
||||
def register_pooling_api_routers(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
if model_config is None:
|
||||
return
|
||||
|
||||
pooling_task = model_config.get_pooling_task(supported_tasks)
|
||||
|
||||
if pooling_task is not None:
|
||||
from .pooling.api_router import router as pooling_router
|
||||
|
||||
app.include_router(pooling_router)
|
||||
|
||||
if "classify" in supported_tasks:
|
||||
from .classify.api_router import (
|
||||
router as classify_router,
|
||||
)
|
||||
|
||||
app.include_router(classify_router)
|
||||
|
||||
if "embed" in supported_tasks:
|
||||
from .embed.api_router import router as embed_router
|
||||
|
||||
app.include_router(embed_router)
|
||||
|
||||
if enable_scoring_api(supported_tasks, model_config):
|
||||
from .scoring.api_router import router as score_router
|
||||
|
||||
app.include_router(score_router)
|
||||
|
||||
|
||||
def init_pooling_state(
|
||||
engine_client: "EngineClient",
|
||||
state: "State",
|
||||
args: "Namespace",
|
||||
request_logger: RequestLogger | None,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
):
|
||||
model_config = engine_client.model_config
|
||||
if model_config is None:
|
||||
return
|
||||
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.tasks import POOLING_TASKS
|
||||
|
||||
from .classify.serving import ServingClassification
|
||||
from .embed.serving import ServingEmbedding
|
||||
from .pooling.serving import ServingPooling
|
||||
from .scoring.serving import ServingScores
|
||||
|
||||
resolved_chat_template = load_chat_template(args.chat_template)
|
||||
pooling_task = model_config.get_pooling_task(supported_tasks)
|
||||
|
||||
chat_template_config = ChatTemplateConfig(
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
trust_request_chat_template=args.trust_request_chat_template,
|
||||
trust_request_mm_kwargs=args.trust_request_mm_kwargs,
|
||||
)
|
||||
|
||||
state.serving_pooling = (
|
||||
(
|
||||
ServingPooling(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
supported_tasks=supported_tasks,
|
||||
request_logger=request_logger,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
)
|
||||
if any(t in supported_tasks for t in POOLING_TASKS)
|
||||
else None
|
||||
)
|
||||
state.serving_embedding = (
|
||||
ServingEmbedding(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
if pooling_task == "embed"
|
||||
else None
|
||||
)
|
||||
state.serving_classification = (
|
||||
ServingClassification(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
chat_template_config=chat_template_config,
|
||||
)
|
||||
if pooling_task == "classify"
|
||||
else None
|
||||
)
|
||||
state.serving_scores = (
|
||||
ServingScores(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
supported_tasks=supported_tasks,
|
||||
request_logger=request_logger,
|
||||
chat_template_config=chat_template_config,
|
||||
enable_flash_late_interaction=getattr(
|
||||
args, "enable_flash_late_interaction", True
|
||||
),
|
||||
)
|
||||
if enable_scoring_api(supported_tasks, model_config)
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def get_pooling_invocation_types(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
# NOTE: Items defined earlier take higher priority
|
||||
invocation_types: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
|
||||
|
||||
if model_config is None:
|
||||
return invocation_types
|
||||
|
||||
pooling_task = model_config.get_pooling_task(supported_tasks)
|
||||
|
||||
if pooling_task == "embed":
|
||||
from .embed.api_router import create_embedding, embedding
|
||||
from .embed.protocol import EmbeddingRequest
|
||||
|
||||
invocation_types += [
|
||||
(EmbeddingRequest, (embedding, create_embedding)),
|
||||
]
|
||||
|
||||
if pooling_task == "classify":
|
||||
from .classify.api_router import classify, create_classify
|
||||
from .classify.protocol import ClassificationRequest
|
||||
|
||||
invocation_types += [
|
||||
(ClassificationRequest, (classify, create_classify)),
|
||||
]
|
||||
|
||||
if enable_scoring_api(supported_tasks, model_config):
|
||||
from .scoring.api_router import do_rerank, rerank
|
||||
from .scoring.protocol import RerankRequest
|
||||
|
||||
invocation_types += [
|
||||
(RerankRequest, (rerank, do_rerank)),
|
||||
]
|
||||
|
||||
from .scoring.api_router import create_score, score
|
||||
from .scoring.protocol import ScoreRequest
|
||||
|
||||
invocation_types += [
|
||||
(ScoreRequest, (score, create_score)),
|
||||
]
|
||||
|
||||
if any(task in POOLING_TASKS for task in supported_tasks):
|
||||
from .pooling.api_router import create_pooling, pooling
|
||||
from .pooling.protocol import PoolingRequest
|
||||
|
||||
invocation_types += [
|
||||
(PoolingRequest, (pooling, create_pooling)),
|
||||
]
|
||||
|
||||
return invocation_types
|
||||
@@ -1,92 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
from typing import cast
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.outputs import LateChunk, LateChunkingMetadata, PoolingRequestOutput
|
||||
from vllm.renderers import TokenizeParams
|
||||
|
||||
from .typing import AnyRenderParam, EncodeCMPLRenderParams
|
||||
|
||||
|
||||
def prepare_late_chunking_input(
|
||||
config: VllmConfig, render_params: AnyRenderParam
|
||||
) -> tuple[str, TokenizeParams]:
|
||||
"""Validate full-document text input before requesting tokenizer offsets."""
|
||||
params = render_params["params"]
|
||||
params.verify(config.model_config)
|
||||
if "prompts" not in render_params:
|
||||
raise VLLMValidationError("Late chunking requires plain-text input")
|
||||
prompt = cast(EncodeCMPLRenderParams, render_params)["prompts"]
|
||||
if (
|
||||
not isinstance(prompt, dict)
|
||||
or set(prompt) - {"prompt", "cache_salt"}
|
||||
or not isinstance(text := prompt.get("prompt"), str)
|
||||
or not text
|
||||
):
|
||||
raise VLLMValidationError("Late chunking requires nonempty plain-text input")
|
||||
tok_params = render_params["tok_params"]
|
||||
if (
|
||||
tok_params.pad_prompt_tokens is not None
|
||||
or tok_params.truncate_prompt_tokens is not None
|
||||
or tok_params.do_lower_case
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
"Late chunking does not support input padding, truncation or "
|
||||
"renderer text normalization"
|
||||
)
|
||||
return text, replace(tok_params, return_token_offsets=True)
|
||||
|
||||
|
||||
def build_late_chunking_metadata(
|
||||
text: str,
|
||||
num_tokens: int,
|
||||
offsets: Sequence[tuple[int, int]] | None,
|
||||
chunk_size: int,
|
||||
) -> LateChunkingMetadata:
|
||||
"""Map chunks to original text, keeping special-token-only chunks."""
|
||||
if num_tokens <= 0 or offsets is None or len(offsets) != num_tokens:
|
||||
raise VLLMValidationError(
|
||||
"Late chunking requires a fast tokenizer with offsets aligned to "
|
||||
"all input tokens"
|
||||
)
|
||||
previous_start = previous_end = 0
|
||||
for start, end in offsets:
|
||||
if not 0 <= start <= end <= len(text):
|
||||
raise VLLMValidationError("Token offsets are not aligned to the input text")
|
||||
if start < end:
|
||||
if start < previous_start or end < previous_end:
|
||||
raise VLLMValidationError("Token offsets are not in source-text order")
|
||||
previous_start, previous_end = start, end
|
||||
|
||||
chunks = []
|
||||
for start in range(0, num_tokens, chunk_size):
|
||||
end = min(start + chunk_size, num_tokens)
|
||||
source = [(a, b) for a, b in offsets[start:end] if a < b]
|
||||
char_range = (
|
||||
(min(a for a, _ in source), max(b for _, b in source)) if source else None
|
||||
)
|
||||
chunks.append(LateChunk(token_range=(start, end), char_range=char_range))
|
||||
return LateChunkingMetadata(
|
||||
chunk_size=chunk_size, input_tokens=num_tokens, chunks=chunks
|
||||
)
|
||||
|
||||
|
||||
def attach_late_chunking_metadata(
|
||||
output: PoolingRequestOutput, metadata: LateChunkingMetadata
|
||||
) -> None:
|
||||
"""Attach ranges only to successful, complete chunk-vector results."""
|
||||
if not output.finished or output.error is not None:
|
||||
return
|
||||
data = output.outputs.data
|
||||
if (
|
||||
data.ndim != 2
|
||||
or data.shape[0] != len(metadata.chunks)
|
||||
or len(output.prompt_token_ids) != metadata.input_tokens
|
||||
):
|
||||
raise ValueError("Late-chunking output does not match its token/source ranges")
|
||||
output.late_chunking = metadata
|
||||
@@ -1,479 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable, Sequence
|
||||
from typing import Any
|
||||
|
||||
from tqdm.auto import tqdm
|
||||
|
||||
from vllm.entrypoints.common.offline import OfflineInferenceMixin
|
||||
from vllm.inputs import DataPrompt, PromptType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import (
|
||||
ClassificationRequestOutput,
|
||||
EmbeddingRequestOutput,
|
||||
PoolingRequestOutput,
|
||||
ScoringRequestOutput,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers.chat_utils import ChatTemplateConfig
|
||||
from vllm.tasks import SCORE_TYPE_MAP, PoolingTask, SupportedTask
|
||||
|
||||
from .base.io_processor import PoolingIOProcessor
|
||||
from .factories import init_pooling_io_processors
|
||||
from .scoring.io_processor import ScoringIOProcessor
|
||||
from .scoring.typing import ScoreInput
|
||||
from .typing import (
|
||||
AnyOfflineInputsContext,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
OfflinePluginInputsContext,
|
||||
OfflineScoringInputsContext,
|
||||
RequestFactory,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class PoolingOfflineMixin(OfflineInferenceMixin):
|
||||
"""Offline inference for pooling models."""
|
||||
|
||||
runner_type: str
|
||||
chat_template: str | None
|
||||
supported_tasks: tuple[SupportedTask, ...]
|
||||
|
||||
def __init__(self):
|
||||
self.pooling_task = self.model_config.get_pooling_task(self.supported_tasks)
|
||||
if self.pooling_task is not None:
|
||||
logger.info("Supported pooling task: %s", self.pooling_task)
|
||||
|
||||
self.chat_template_config = ChatTemplateConfig(chat_template=self.chat_template)
|
||||
self.pooling_io_processors = init_pooling_io_processors(
|
||||
supported_tasks=self.supported_tasks,
|
||||
vllm_config=self.llm_engine.vllm_config,
|
||||
renderer=self.renderer,
|
||||
chat_template_config=self.chat_template_config,
|
||||
)
|
||||
|
||||
# Use thread pool executor to accelerate preprocessing.
|
||||
self._executor = self.renderer._executor
|
||||
|
||||
def encode(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType] | DataPrompt,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
pooling_task: PoolingTask | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
"""Apply pooling to the hidden states corresponding to the input
|
||||
prompts.
|
||||
|
||||
This class automatically batches the given prompts, considering
|
||||
the memory constraint. For the best performance, put all of your prompts
|
||||
into a single list and pass it to this method.
|
||||
|
||||
Args:
|
||||
prompts: The prompts to the LLM. You may pass a sequence of prompts
|
||||
for batch inference. See [PromptType][vllm.inputs.PromptType]
|
||||
for more details about the format of each prompt.
|
||||
pooling_params: The pooling parameters for pooling. If None, we
|
||||
use the default pooling parameters.
|
||||
use_tqdm: If `True`, shows a tqdm progress bar.
|
||||
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
pooling_task: Override the pooling task to use.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
|
||||
Returns:
|
||||
A list of `PoolingRequestOutput` objects containing the
|
||||
pooled hidden states in the same order as the input prompts.
|
||||
|
||||
"""
|
||||
if isinstance(prompts, dict) and "data" in prompts and pooling_task != "plugin":
|
||||
raise ValueError(
|
||||
"The 'data' field is only supported for the 'plugin' pooling task."
|
||||
)
|
||||
self._verify_pooling_task(pooling_task)
|
||||
assert pooling_task is not None and pooling_task in self.pooling_io_processors
|
||||
|
||||
io_processor = self.pooling_io_processors[pooling_task]
|
||||
|
||||
ctx: AnyOfflineInputsContext
|
||||
if isinstance(prompts, dict) and "data" in prompts:
|
||||
ctx = OfflinePluginInputsContext(
|
||||
pooling_task=pooling_task,
|
||||
prompts=prompts, # type: ignore[arg-type]
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
pooling_params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
priorities=None,
|
||||
)
|
||||
else:
|
||||
ctx = OfflineEncodeInputsContext(
|
||||
pooling_task=pooling_task,
|
||||
prompts=prompts,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
pooling_params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
priorities=None,
|
||||
)
|
||||
|
||||
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
|
||||
outputs = self._run_tiling_engine(
|
||||
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
|
||||
)
|
||||
outputs = io_processor.post_process_offline(
|
||||
ctx=OfflineOutputsContext(outputs=outputs, late_chunking=ctx.late_chunking)
|
||||
)
|
||||
return outputs
|
||||
|
||||
def _verify_pooling_task(self, pooling_task: PoolingTask | None):
|
||||
if self.runner_type != "pooling":
|
||||
raise ValueError(
|
||||
"LLM.encode() is only supported for pooling models. "
|
||||
"Try passing `--runner pooling` to use the model as a "
|
||||
"pooling model."
|
||||
)
|
||||
|
||||
if pooling_task is None:
|
||||
raise ValueError(
|
||||
"""
|
||||
pooling_task required for `LLM.encode`.
|
||||
Please use one of the more specific methods or set the pooling_task when using `LLM.encode`:
|
||||
- For embeddings, use `LLM.embed(...)` or `pooling_task="embed"`.
|
||||
- For classification logits, use `LLM.classify(...)` or `pooling_task="classify"`.
|
||||
- For similarity scores, use `LLM.score(...)`.
|
||||
- For rewards, `pooling_task="classify"` or `pooling_task="token_classify"`.
|
||||
- For token classification, use `pooling_task="token_classify"`.
|
||||
- For multi-vector retrieval, use `pooling_task="token_embed"`.
|
||||
""" # noqa: E501
|
||||
)
|
||||
|
||||
if (
|
||||
pooling_task in ("embed", "token_embed")
|
||||
and pooling_task not in self.supported_tasks
|
||||
):
|
||||
raise ValueError(
|
||||
"Embedding API is not supported by this model. "
|
||||
"Try converting the model using `--convert embed`."
|
||||
)
|
||||
|
||||
if (
|
||||
pooling_task in ("classify", "token_classify")
|
||||
and pooling_task not in self.supported_tasks
|
||||
):
|
||||
raise ValueError(
|
||||
"Classification API is not supported by this model. "
|
||||
"Try converting the model using `--convert classify`."
|
||||
)
|
||||
|
||||
# plugin task uses io_processor.parse_data to verify inputs
|
||||
if pooling_task != "plugin" and pooling_task != self.pooling_task:
|
||||
if pooling_task not in self.supported_tasks:
|
||||
raise ValueError(
|
||||
f"Unsupported task: {pooling_task!r} "
|
||||
f"Supported tasks: {self.supported_tasks}"
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Try switching the model's pooling_task "
|
||||
f'via `PoolerConfig(task="{pooling_task}")`'
|
||||
)
|
||||
|
||||
if pooling_task == "plugin" and "plugin" not in self.pooling_io_processors:
|
||||
raise ValueError(
|
||||
"No IOProcessor plugin installed. Please refer "
|
||||
"to the documentation and to the "
|
||||
"'prithvi_geospatial_mae_io_processor' "
|
||||
"offline inference example for more details."
|
||||
)
|
||||
|
||||
def embed(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[EmbeddingRequestOutput]:
|
||||
"""Generate an embedding vector for each prompt.
|
||||
|
||||
This class automatically batches the given prompts, considering
|
||||
the memory constraint. For the best performance, put all of your prompts
|
||||
into a single list and pass it to this method.
|
||||
|
||||
Args:
|
||||
prompts: The prompts to the LLM. You may pass a sequence of prompts
|
||||
for batch inference. See [PromptType][vllm.inputs.PromptType]
|
||||
for more details about the format of each prompt.
|
||||
pooling_params: The pooling parameters for pooling. If None, we
|
||||
use the default pooling parameters.
|
||||
use_tqdm: If `True`, shows a tqdm progress bar.
|
||||
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
|
||||
Returns:
|
||||
A list of `EmbeddingRequestOutput` objects containing the
|
||||
embedding vectors in the same order as the input prompts.
|
||||
|
||||
"""
|
||||
items = self.encode(
|
||||
prompts,
|
||||
use_tqdm=use_tqdm,
|
||||
pooling_params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
pooling_task="embed",
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return [EmbeddingRequestOutput.from_base(item) for item in items]
|
||||
|
||||
def classify(
|
||||
self,
|
||||
prompts: PromptType | Sequence[PromptType],
|
||||
*,
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[ClassificationRequestOutput]:
|
||||
"""Generate class logits for each prompt.
|
||||
|
||||
This class automatically batches the given prompts, considering
|
||||
the memory constraint. For the best performance, put all of your prompts
|
||||
into a single list and pass it to this method.
|
||||
|
||||
Args:
|
||||
prompts: The prompts to the LLM. You may pass a sequence of prompts
|
||||
for batch inference. See [PromptType][vllm.inputs.PromptType]
|
||||
for more details about the format of each prompt.
|
||||
pooling_params: The pooling parameters for pooling. If None, we
|
||||
use the default pooling parameters.
|
||||
use_tqdm: If `True`, shows a tqdm progress bar.
|
||||
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
|
||||
Returns:
|
||||
A list of `ClassificationRequestOutput` objects containing the
|
||||
embedding vectors in the same order as the input prompts.
|
||||
|
||||
"""
|
||||
items = self.encode(
|
||||
prompts,
|
||||
use_tqdm=use_tqdm,
|
||||
pooling_params=pooling_params,
|
||||
lora_request=lora_request,
|
||||
pooling_task="classify",
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
)
|
||||
|
||||
return [ClassificationRequestOutput.from_base(item) for item in items]
|
||||
|
||||
def score(
|
||||
self,
|
||||
data_1: ScoreInput | list[ScoreInput],
|
||||
data_2: ScoreInput | list[ScoreInput],
|
||||
/,
|
||||
*,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
pooling_params: PoolingParams | None = None,
|
||||
lora_request: list[LoRARequest] | LoRARequest | None = None,
|
||||
tokenization_kwargs: dict[str, Any] | None = None,
|
||||
chat_template: str | None = None,
|
||||
) -> list[ScoringRequestOutput]:
|
||||
"""Generate similarity scores for all pairs `<text,text_pair>` or
|
||||
`<multi-modal data, multi-modal data pair>`.
|
||||
|
||||
The inputs can be `1 -> 1`, `1 -> N` or `N -> N`.
|
||||
In the `1 - N` case the `data_1` input will be replicated `N`
|
||||
times to pair with the `data_2` inputs.
|
||||
The input pairs are used to build a list of prompts for the
|
||||
cross encoder model. This class automatically batches the prompts,
|
||||
considering the memory constraint. For the best performance, put all
|
||||
of your inputs into a single list and pass it to this method.
|
||||
|
||||
Supports both text and multi-modal data (images, etc.) when used with
|
||||
appropriate multi-modal models. For multi-modal inputs, ensure the
|
||||
prompt structure matches the model's expected input format.
|
||||
|
||||
Args:
|
||||
data_1: Can be a single prompt, a list of prompts or
|
||||
`ScoreMultiModalParam`, which can contain either text or
|
||||
multi-modal data. When a list, it must have the same length as
|
||||
the `data_2` list.
|
||||
data_2: The data to pair with the query to form the input to
|
||||
the LLM. Can be text or multi-modal data. See [PromptType]
|
||||
[vllm.inputs.PromptType] for more details about the format of
|
||||
each prompt.
|
||||
pooling_params: The pooling parameters for pooling. If None, we
|
||||
use the default pooling parameters.
|
||||
use_tqdm: If `True`, shows a tqdm progress bar.
|
||||
If a callable (e.g., `functools.partial(tqdm, leave=False)`),
|
||||
it is used to create the progress bar.
|
||||
If `False`, no progress bar is created.
|
||||
lora_request: LoRA request to use for generation, if any.
|
||||
chat_template: The chat template to use for scoring. If None, use
|
||||
the chat template passed to the LLM constructor.
|
||||
tokenization_kwargs: Overrides for `tokenizer.encode`.
|
||||
|
||||
Returns:
|
||||
A list of `ScoringRequestOutput` objects containing the
|
||||
generated scores in the same order as the input prompts.
|
||||
|
||||
"""
|
||||
if self.runner_type != "pooling":
|
||||
raise ValueError(
|
||||
"LLM.score() is only supported for pooling models. "
|
||||
"Try passing `--runner pooling` to use the model as a "
|
||||
"pooling model."
|
||||
)
|
||||
|
||||
score_type: str | None = SCORE_TYPE_MAP.get(self.pooling_task, None) # type: ignore[arg-type]
|
||||
if (
|
||||
score_type == "cross-encoder"
|
||||
and getattr(self.model_config.hf_config, "num_labels", 0) != 1
|
||||
):
|
||||
raise ValueError("Scoring API is only enabled for num_labels == 1.")
|
||||
|
||||
if score_type is None or score_type not in self.pooling_io_processors:
|
||||
raise ValueError("This model does not support the Scoring API.")
|
||||
|
||||
io_processor = self.pooling_io_processors[score_type]
|
||||
assert isinstance(io_processor, ScoringIOProcessor)
|
||||
|
||||
scoring_data = io_processor.valid_inputs(data_1, data_2)
|
||||
n_queries = len(scoring_data.data_1)
|
||||
|
||||
if pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
|
||||
assert isinstance(pooling_params, PoolingParams)
|
||||
pooling_task = io_processor.pooling_task
|
||||
# Clone to avoid modifying the caller's pooling parameters.
|
||||
pooling_params = pooling_params.clone()
|
||||
if pooling_params.task is None:
|
||||
pooling_params.task = pooling_task
|
||||
elif pooling_params.task != pooling_task:
|
||||
msg = (
|
||||
f"You cannot overwrite {pooling_params.task=!r} with {pooling_task=!r}!"
|
||||
)
|
||||
raise ValueError(msg)
|
||||
|
||||
ctx = OfflineScoringInputsContext(
|
||||
pooling_task=pooling_task,
|
||||
scoring_data=scoring_data,
|
||||
pooling_params=pooling_params,
|
||||
tokenization_kwargs=tokenization_kwargs,
|
||||
lora_request=lora_request,
|
||||
chat_template=chat_template,
|
||||
priorities=None,
|
||||
)
|
||||
|
||||
request_factory, num_requests = io_processor.get_request_factory_offline(ctx)
|
||||
outputs = self._run_tiling_engine(
|
||||
io_processor, request_factory, num_requests, use_tqdm=use_tqdm
|
||||
)
|
||||
|
||||
outputs = io_processor.post_process_offline(
|
||||
ctx=OfflineOutputsContext(outputs=outputs, n_queries=n_queries),
|
||||
)
|
||||
|
||||
return [ScoringRequestOutput.from_base(item) for item in outputs]
|
||||
|
||||
def _run_tiling_engine(
|
||||
self,
|
||||
io_processor: PoolingIOProcessor,
|
||||
request_factory: RequestFactory,
|
||||
num_requests: int,
|
||||
use_tqdm: bool | Callable[..., tqdm] = True,
|
||||
):
|
||||
if num_requests == 0:
|
||||
raise ValueError("You must pass at least one prompt")
|
||||
|
||||
# Keeping max_num_seqs * 2 requests in the core can already saturate the core.
|
||||
# Therefore, keep most requests waiting outside the core.
|
||||
max_requests_in_core = (
|
||||
self.llm_engine.vllm_config.scheduler_config.max_num_seqs * 2
|
||||
)
|
||||
num_requests_in_core = 0
|
||||
num_waited_requests = num_requests
|
||||
|
||||
if use_tqdm:
|
||||
tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
|
||||
pbar = tqdm_func(
|
||||
total=num_requests,
|
||||
desc="Processed prompts",
|
||||
dynamic_ncols=True,
|
||||
postfix=f"est. speed input: {0:.2f} toks/s, output: {0:.2f} toks/s",
|
||||
)
|
||||
|
||||
outputs: list[PoolingRequestOutput] = []
|
||||
added_request_ids: set[str] = set()
|
||||
|
||||
it = self._executor.map(io_processor.render, request_factory())
|
||||
|
||||
try:
|
||||
while num_waited_requests or self.llm_engine.has_unfinished_requests():
|
||||
requests = []
|
||||
for _ in range(max_requests_in_core - num_requests_in_core):
|
||||
if num_waited_requests == 0:
|
||||
break
|
||||
try:
|
||||
request = next(it)
|
||||
requests.append(request)
|
||||
except StopIteration:
|
||||
num_waited_requests = 0
|
||||
break
|
||||
|
||||
num_waited_requests -= 1
|
||||
num_requests_in_core += 1
|
||||
|
||||
if requests:
|
||||
request_ids = self._render_and_add_requests(
|
||||
prompts=[x["prompts"] for x in requests],
|
||||
params=[x["params"] for x in requests],
|
||||
lora_requests=[x["lora_requests"] for x in requests],
|
||||
priorities=[x["priorities"] for x in requests],
|
||||
)
|
||||
|
||||
for request_id in request_ids:
|
||||
# undo assign_request_id
|
||||
request_id = request_id.split("-", 1)[0]
|
||||
added_request_ids.add(request_id)
|
||||
|
||||
step_outputs = self.llm_engine.step()
|
||||
for output in step_outputs:
|
||||
assert isinstance(output, PoolingRequestOutput)
|
||||
assert output.finished
|
||||
outputs.append(output)
|
||||
added_request_ids.discard(output.request_id)
|
||||
num_requests_in_core -= 1
|
||||
|
||||
if use_tqdm:
|
||||
pbar.update(1)
|
||||
if pbar.n == num_requests:
|
||||
pbar.refresh()
|
||||
|
||||
except Exception:
|
||||
if added_request_ids:
|
||||
self.llm_engine.abort_request(list(added_request_ids))
|
||||
raise
|
||||
|
||||
finally:
|
||||
if use_tqdm:
|
||||
pbar.close()
|
||||
|
||||
return sorted(outputs, key=lambda x: int(x.request_id))
|
||||
@@ -1,39 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
from .protocol import PoolingRequest
|
||||
from .serving import ServingPooling
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def pooling(request: Request) -> ServingPooling:
|
||||
handler = getattr(request.app.state, "serving_pooling", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Pooling API")
|
||||
return handler
|
||||
|
||||
|
||||
@router.post(
|
||||
"/pooling",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_pooling(request: PoolingRequest, raw_request: Request):
|
||||
handler = pooling(raw_request)
|
||||
return await handler(request, raw_request)
|
||||
@@ -1,187 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput
|
||||
from vllm.plugins.io_processors import get_io_processor
|
||||
from vllm.renderers.inputs.preprocess import parse_model_prompt, prompt_to_seq
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..typing import (
|
||||
AnyOfflineInputsContext,
|
||||
AnyRenderParam,
|
||||
EncodeCMPLRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
OfflinePluginInputsContext,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
)
|
||||
from .protocol import IOProcessorRequest, IOProcessorResponse
|
||||
|
||||
|
||||
class UnsupportedCombinedTaskIOProcessor(PoolingIOProcessor):
|
||||
name = "embed&token_classify"
|
||||
|
||||
def create_pooling_params(self, request):
|
||||
raise ValueError(
|
||||
"The 'embed&token_classify' pooling task is only available "
|
||||
"through an IO processor plugin. Send a plugin request with "
|
||||
"a 'data' field, "
|
||||
"or select a concrete task with --pooler-config.task."
|
||||
)
|
||||
|
||||
|
||||
class PluginWithoutIOProcessorPlugins(PoolingIOProcessor):
|
||||
# Some models, such as Terratorch (tests/models/test_terratorch.py),
|
||||
# use plugin tasks in the pooler but do not use IO Processor plugins.
|
||||
name = "plugin"
|
||||
|
||||
|
||||
class PluginWithIOProcessorPlugins(PoolingIOProcessor):
|
||||
"""IO Processor plugins are a feature that allows pre- and post-processing
|
||||
of the model input and output for pooling models."""
|
||||
|
||||
name = "plugin"
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
io_processor = get_io_processor(
|
||||
self.vllm_config,
|
||||
self.renderer,
|
||||
self.model_config.io_processor_plugin,
|
||||
)
|
||||
|
||||
assert io_processor is not None
|
||||
self.io_processor = io_processor
|
||||
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
assert isinstance(ctx.request, IOProcessorRequest)
|
||||
|
||||
validated_prompt = self.io_processor.parse_data(ctx.request.data)
|
||||
|
||||
raw_prompts = self.io_processor.pre_process(
|
||||
prompt=validated_prompt, request_id=ctx.request_id
|
||||
)
|
||||
|
||||
parsed_prompts = [
|
||||
(
|
||||
prompt
|
||||
if isinstance(prompt, bytes)
|
||||
else parse_model_prompt(self.model_config, prompt)
|
||||
)
|
||||
for prompt in prompt_to_seq(raw_prompts)
|
||||
]
|
||||
num_requests = len(parsed_prompts)
|
||||
|
||||
tok_params = ctx.request.build_tok_params(self.model_config)
|
||||
|
||||
pooling_params = self.io_processor.merge_pooling_params()
|
||||
if pooling_params.task is None:
|
||||
pooling_params.task = "plugin"
|
||||
ctx.pooling_params = pooling_params
|
||||
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeCMPLRenderParams(
|
||||
prompts=parsed_prompts[i],
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
return requests
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
):
|
||||
output = self.io_processor.post_process(
|
||||
ctx.final_res_batch,
|
||||
request_id=ctx.request_id,
|
||||
)
|
||||
ctx.response = IOProcessorResponse(request_id=ctx.request_id, data=output)
|
||||
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflinePluginInputsContext)
|
||||
assert isinstance(ctx.prompts, dict) and "data" in ctx.prompts
|
||||
|
||||
# Validate the request data is valid for the loaded plugin
|
||||
prompt_data = ctx.prompts.get("data")
|
||||
if prompt_data is None:
|
||||
raise ValueError(
|
||||
"The 'data' field of the prompt is expected to contain "
|
||||
"the prompt data and it cannot be None. "
|
||||
"Refer to the documentation of the IOProcessor "
|
||||
"in use for more details."
|
||||
)
|
||||
validated_prompt = self.io_processor.parse_data(prompt_data)
|
||||
|
||||
# obtain the actual model prompts from the pre-processor
|
||||
prompts = self.io_processor.pre_process(prompt=validated_prompt)
|
||||
prompts_seq = prompt_to_seq(prompts)
|
||||
|
||||
num_requests = len(prompts_seq)
|
||||
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams]
|
||||
if ctx.pooling_params is None:
|
||||
pooling_params = PoolingParams()
|
||||
else:
|
||||
pooling_params = ctx.pooling_params
|
||||
|
||||
params_seq: list[PoolingParams] = [
|
||||
self.io_processor.merge_pooling_params(param.clone())
|
||||
for param in self._params_to_seq(
|
||||
pooling_params,
|
||||
num_requests,
|
||||
)
|
||||
]
|
||||
for p in params_seq:
|
||||
if p.task is None:
|
||||
p.task = "plugin"
|
||||
|
||||
return super().get_request_factory_offline(
|
||||
OfflineEncodeInputsContext(
|
||||
prompts=prompts_seq,
|
||||
pooling_params=params_seq,
|
||||
pooling_task="plugin",
|
||||
tokenization_kwargs=ctx.tokenization_kwargs,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
)
|
||||
)
|
||||
|
||||
def post_process_offline(
|
||||
self,
|
||||
ctx: OfflineOutputsContext,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
processed_outputs = self.io_processor.post_process(ctx.outputs)
|
||||
|
||||
return [
|
||||
PoolingRequestOutput[Any](
|
||||
request_id="",
|
||||
outputs=processed_outputs,
|
||||
num_cached_tokens=getattr(processed_outputs, "num_cached_tokens", 0),
|
||||
prompt_token_ids=[],
|
||||
finished=True,
|
||||
)
|
||||
]
|
||||
@@ -1,120 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from typing import Annotated, Generic, TypeAlias, TypeVar
|
||||
|
||||
from pydantic import BeforeValidator, Field
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import (
|
||||
ChatRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
EncodingRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
PoolingBasicRequestMixin,
|
||||
reject_removed_pooling_parameters,
|
||||
)
|
||||
|
||||
|
||||
class PoolingCompletionRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
CompletionRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
):
|
||||
task: PoolingTask | None = None
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task=self.task,
|
||||
use_activation=self.use_activation,
|
||||
dimensions=self.dimensions,
|
||||
)
|
||||
|
||||
|
||||
class PoolingChatRequest(
|
||||
PoolingBasicRequestMixin,
|
||||
ChatRequestMixin,
|
||||
EmbedRequestMixin,
|
||||
ClassifyRequestMixin,
|
||||
FixedMaxLenTokenizeParamsMixin,
|
||||
):
|
||||
task: PoolingTask | None = None
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task=self.task,
|
||||
use_activation=self.use_activation,
|
||||
dimensions=self.dimensions,
|
||||
)
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class IOProcessorRequest(PoolingBasicRequestMixin, EncodingRequestMixin, Generic[T]):
|
||||
data: T
|
||||
task: PoolingTask = "plugin"
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=not model_config.is_encoder_decoder,
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
)
|
||||
|
||||
def to_pooling_params(self):
|
||||
return PoolingParams(
|
||||
task=self.task,
|
||||
)
|
||||
|
||||
|
||||
class IOProcessorResponse(OpenAIBaseModel, Generic[T]):
|
||||
request_id: str | None = None
|
||||
"""
|
||||
The request_id associated with this response
|
||||
"""
|
||||
created_at: int = Field(default_factory=lambda: int(time.time()))
|
||||
|
||||
data: T
|
||||
"""
|
||||
When using plugins IOProcessor plugins, the actual output is generated
|
||||
by the plugin itself. Hence, we use a generic type for the response data
|
||||
"""
|
||||
|
||||
|
||||
PoolingRequest: TypeAlias = Annotated[
|
||||
PoolingCompletionRequest | PoolingChatRequest | IOProcessorRequest,
|
||||
BeforeValidator(reject_removed_pooling_parameters),
|
||||
]
|
||||
|
||||
|
||||
class PoolingResponseData(OpenAIBaseModel):
|
||||
index: int
|
||||
object: str = "pooling"
|
||||
data: list[list[float]] | list[float] | str
|
||||
|
||||
|
||||
class PoolingResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"pool-{random_uuid()}")
|
||||
object: str = "list"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
data: list[PoolingResponseData]
|
||||
usage: UsageInfo
|
||||
|
||||
|
||||
class PoolingBytesResponse(OpenAIBaseModel):
|
||||
content: list[bytes]
|
||||
headers: dict[str, str] | None = None
|
||||
media_type: str = "application/octet-stream"
|
||||
@@ -1,191 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import assert_never, cast
|
||||
|
||||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||||
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils.serial_utils import EmbedDType, Endianness
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..base.serving import PoolingBaseServing
|
||||
from ..factories import init_pooling_io_processors
|
||||
from ..typing import AnyPoolingRequest, PoolingServeContext
|
||||
from ..utils import (
|
||||
BytesEncodingFormat,
|
||||
JsonEncodingFormat,
|
||||
build_pooling_bytes_streaming_response,
|
||||
get_json_response_cls,
|
||||
get_pooling_output_encoder,
|
||||
get_pooling_usage,
|
||||
)
|
||||
from .protocol import (
|
||||
IOProcessorRequest,
|
||||
PoolingRequest,
|
||||
PoolingResponse,
|
||||
PoolingResponseData,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ServingPooling(PoolingBaseServing):
|
||||
request_id_prefix = "pooling"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*args,
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.supported_tasks = supported_tasks
|
||||
self.pooling_task = self.model_config.get_pooling_task(supported_tasks)
|
||||
self.io_processors = init_pooling_io_processors(
|
||||
supported_tasks=supported_tasks,
|
||||
vllm_config=self.vllm_config,
|
||||
renderer=self.renderer,
|
||||
chat_template_config=self.chat_template_config,
|
||||
)
|
||||
self.json_response_cls = get_json_response_cls()
|
||||
|
||||
def get_io_processor(self, request: AnyPoolingRequest) -> PoolingIOProcessor:
|
||||
request = cast(PoolingRequest, request)
|
||||
pooling_task = self._verify_pooling_task(request)
|
||||
return self.io_processors[pooling_task]
|
||||
|
||||
def _verify_pooling_task(self, request: PoolingRequest) -> str:
|
||||
if getattr(request, "dimensions", None) is not None:
|
||||
raise VLLMValidationError(
|
||||
"dimensions is currently not supported", parameter="dimensions"
|
||||
)
|
||||
|
||||
if request.task is None:
|
||||
request.task = self.pooling_task
|
||||
|
||||
if isinstance(request, IOProcessorRequest):
|
||||
request.task = "plugin"
|
||||
|
||||
assert request.task is not None
|
||||
pooling_task = request.task
|
||||
|
||||
# plugin task uses io_processor.parse_data to verify inputs
|
||||
if pooling_task != "plugin" and pooling_task != self.pooling_task:
|
||||
if pooling_task not in self.supported_tasks:
|
||||
raise VLLMValidationError(
|
||||
f"Unsupported task: {pooling_task!r} "
|
||||
f"Supported tasks: {self.supported_tasks}",
|
||||
parameter="task",
|
||||
)
|
||||
else:
|
||||
raise VLLMValidationError(
|
||||
"Try switching the model's pooling_task "
|
||||
f"via --pooler-config.task {request.task}.",
|
||||
parameter="task",
|
||||
)
|
||||
|
||||
if pooling_task == "plugin" and "plugin" not in self.io_processors:
|
||||
raise VLLMValidationError(
|
||||
"No IOProcessor plugin installed. Please refer "
|
||||
"to the documentation and to the "
|
||||
"'prithvi_geospatial_mae_io_processor' "
|
||||
"offline inference example for more details.",
|
||||
parameter="task",
|
||||
)
|
||||
|
||||
return pooling_task
|
||||
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: PoolingServeContext,
|
||||
) -> Response:
|
||||
if ctx.response is not None:
|
||||
# for IOProcessorResponse
|
||||
return self.json_response_cls(content=ctx.response.model_dump())
|
||||
|
||||
encoding_format = ctx.request.encoding_format
|
||||
embed_dtype = ctx.request.embed_dtype
|
||||
endianness = ctx.request.endianness
|
||||
|
||||
if encoding_format == "float" or encoding_format == "base64":
|
||||
return self.request_output_to_pooling_json_response(
|
||||
ctx.final_res_batch,
|
||||
ctx.request_id,
|
||||
ctx.created_time,
|
||||
ctx.model_name,
|
||||
encoding_format,
|
||||
embed_dtype,
|
||||
endianness,
|
||||
)
|
||||
|
||||
if encoding_format == "bytes" or encoding_format == "bytes_only":
|
||||
return self.request_output_to_pooling_bytes_response(
|
||||
ctx.final_res_batch,
|
||||
ctx.request_id,
|
||||
ctx.created_time,
|
||||
ctx.model_name,
|
||||
encoding_format,
|
||||
embed_dtype,
|
||||
endianness,
|
||||
)
|
||||
|
||||
assert_never(encoding_format)
|
||||
|
||||
def request_output_to_pooling_json_response(
|
||||
self,
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: JsonEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> JSONResponse:
|
||||
encode_fn = get_pooling_output_encoder(
|
||||
encoding_format=encoding_format,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
|
||||
items: list[PoolingResponseData] = []
|
||||
|
||||
for idx, final_res in enumerate(final_res_batch):
|
||||
item = PoolingResponseData(
|
||||
index=idx,
|
||||
data=encode_fn(final_res),
|
||||
)
|
||||
|
||||
items.append(item)
|
||||
|
||||
response = PoolingResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
data=items,
|
||||
usage=get_pooling_usage(final_res_batch),
|
||||
)
|
||||
return self.json_response_cls(content=response.model_dump())
|
||||
|
||||
def request_output_to_pooling_bytes_response(
|
||||
self,
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: BytesEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> StreamingResponse:
|
||||
return build_pooling_bytes_streaming_response(
|
||||
pooling_outputs=final_res_batch,
|
||||
request_id=request_id,
|
||||
created_time=created_time,
|
||||
model_name=model_name,
|
||||
encoding_format=encoding_format,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
@@ -1,111 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .protocol import RerankRequest, ScoreRequest
|
||||
from .serving import ServingScores
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def score(request: Request) -> ServingScores:
|
||||
handler = getattr(request.app.state, "serving_scores", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Score API")
|
||||
return handler
|
||||
|
||||
|
||||
def rerank(request: Request) -> ServingScores:
|
||||
handler = getattr(request.app.state, "serving_scores", None)
|
||||
if handler is None:
|
||||
raise NotImplementedError("The model does not support Rerank (Score) API")
|
||||
return handler
|
||||
|
||||
|
||||
@router.post(
|
||||
"/score",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_score(request: ScoreRequest, raw_request: Request):
|
||||
handler = score(raw_request)
|
||||
return await handler(request, raw_request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/score",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def create_score_v1(request: ScoreRequest, raw_request: Request):
|
||||
logger.warning(
|
||||
"To indicate that Score API is not part of standard OpenAI API, we "
|
||||
"have moved it to `/score`. Please update your client accordingly."
|
||||
)
|
||||
|
||||
return await create_score(request, raw_request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/rerank",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def do_rerank(request: RerankRequest, raw_request: Request):
|
||||
handler = rerank(raw_request)
|
||||
return await handler(request, raw_request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/rerank",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def do_rerank_v1(request: RerankRequest, raw_request: Request):
|
||||
logger.warning_once(
|
||||
"To indicate that the rerank API is not part of the standard OpenAI"
|
||||
" API, we have located it at `/rerank`. Please update your client "
|
||||
"accordingly. (Note: Conforms to JinaAI rerank API)"
|
||||
)
|
||||
|
||||
return await do_rerank(request, raw_request)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v2/rerank",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def do_rerank_v2(request: RerankRequest, raw_request: Request):
|
||||
return await do_rerank(request, raw_request)
|
||||
@@ -1,978 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, TypeAlias, cast
|
||||
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput, TokensPrompt
|
||||
from vllm.exceptions import ChatTemplateResolutionError
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.renderers.hf import safe_apply_chat_template
|
||||
from vllm.renderers.inputs.preprocess import (
|
||||
extract_target_prompt,
|
||||
parse_model_prompt,
|
||||
prompt_to_seq,
|
||||
)
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.mistral import is_mistral_tokenizer
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..pooling.protocol import PoolingCompletionRequest
|
||||
from ..typing import (
|
||||
AnyOfflineInputsContext,
|
||||
AnyPoolingRequest,
|
||||
AnyRenderParam,
|
||||
EncodeChatRenderParams,
|
||||
EncodeCMPLRenderParams,
|
||||
OfflineEncodeInputsContext,
|
||||
OfflineOutputsContext,
|
||||
OfflineScoringInputsContext,
|
||||
PoolingEngineInput,
|
||||
PoolingServeContext,
|
||||
RequestFactory,
|
||||
RequestGenerator,
|
||||
ScoringRenderParams,
|
||||
)
|
||||
from .protocol import RerankRequest, ScoreRequest, ScoringRequest
|
||||
from .typing import ScoreData, ScoreInput, ScoringData
|
||||
from .utils import (
|
||||
compress_token_type_ids,
|
||||
compute_maxsim_score,
|
||||
get_num_special_tokens_for_pair,
|
||||
parse_score_data,
|
||||
score_data_to_prompts,
|
||||
truncate_text_to_tokens,
|
||||
validate_score_input,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
ScoringServeContext: TypeAlias = PoolingServeContext[ScoringRequest]
|
||||
|
||||
|
||||
def _apply_post_tokenization_to_token_type_ids(
|
||||
tokenizer: Any,
|
||||
tok_params: TokenizeParams,
|
||||
token_type_ids: list[int],
|
||||
) -> list[int]:
|
||||
# Must stay in the same order as `TokenizeParams._validate_tokens`, which
|
||||
# truncates before padding. These ids are parallel to the prompt tokens
|
||||
# and locate the query/document boundary for the cross-encoder; applying
|
||||
# the two steps in a different order silently misplaces that boundary.
|
||||
max_length = tok_params.truncate_prompt_tokens
|
||||
if max_length is not None and max_length < 0:
|
||||
max_length = tok_params.max_input_tokens
|
||||
|
||||
if max_length is not None and max_length < len(token_type_ids):
|
||||
if max_length == 0:
|
||||
token_type_ids = token_type_ids[:0]
|
||||
else:
|
||||
side = tok_params.truncation_side or (
|
||||
tokenizer.truncation_side if tokenizer is not None else None
|
||||
)
|
||||
token_type_ids = (
|
||||
token_type_ids[-max_length:]
|
||||
if side == "left"
|
||||
else token_type_ids[:max_length]
|
||||
)
|
||||
|
||||
pad_length = tok_params.pad_prompt_tokens
|
||||
if pad_length is not None and pad_length < 0:
|
||||
pad_length = tok_params.max_input_tokens
|
||||
|
||||
if pad_length is not None and pad_length > len(token_type_ids):
|
||||
pad_token_type_id = getattr(tokenizer, "pad_token_type_id", 0) or 0
|
||||
token_type_ids = token_type_ids + [pad_token_type_id] * (
|
||||
pad_length - len(token_type_ids)
|
||||
)
|
||||
|
||||
return token_type_ids
|
||||
|
||||
|
||||
class ScoringIOProcessor(PoolingIOProcessor):
|
||||
name: str
|
||||
pooling_task: PoolingTask
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.tokenizer = self.renderer.get_tokenizer()
|
||||
self.architecture = self.model_config.architecture
|
||||
self.is_multimodal_model = self.model_config.is_multimodal_model
|
||||
self.pad_token_id = self.tokenizer.pad_token_id
|
||||
|
||||
def create_pooling_params(self, request):
|
||||
return request.to_pooling_params(self.pooling_task)
|
||||
|
||||
def _validate_token_limit(self, value: int, name: str) -> None:
|
||||
if value < 0:
|
||||
raise ValueError(f"{name} must be a non-negative integer")
|
||||
if value >= self.model_config.max_model_len:
|
||||
raise ValueError(
|
||||
f"{name} ({value}) must be less "
|
||||
f"than max_model_len ({self.model_config.max_model_len})."
|
||||
)
|
||||
|
||||
def _get_token_limits(
|
||||
self,
|
||||
request: ScoringRequest | None = None,
|
||||
pooling_params: PoolingParams | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""Extract and validate token limits from request or pooling_params."""
|
||||
if request is not None:
|
||||
max_tokens_per_query = getattr(request, "max_tokens_per_query", 0)
|
||||
max_tokens_per_doc = getattr(request, "max_tokens_per_doc", 0)
|
||||
else:
|
||||
extra = (
|
||||
(pooling_params.extra_kwargs or {})
|
||||
if pooling_params is not None
|
||||
else {}
|
||||
)
|
||||
max_tokens_per_query = extra.get("max_tokens_per_query", 0)
|
||||
max_tokens_per_doc = extra.get("max_tokens_per_doc", 0)
|
||||
|
||||
if max_tokens_per_query != 0:
|
||||
self._validate_token_limit(max_tokens_per_query, "max_tokens_per_query")
|
||||
if max_tokens_per_doc != 0:
|
||||
self._validate_token_limit(max_tokens_per_doc, "max_tokens_per_doc")
|
||||
return max_tokens_per_query, max_tokens_per_doc
|
||||
|
||||
def _truncate_scoring_data(
|
||||
self,
|
||||
scoring_data: ScoringData,
|
||||
max_tokens_per_query: int = 0,
|
||||
max_tokens_per_doc: int = 0,
|
||||
) -> ScoringData:
|
||||
"""Truncate query/document texts to token limits."""
|
||||
data_1 = scoring_data.data_1
|
||||
data_2 = scoring_data.data_2
|
||||
if max_tokens_per_query > 0:
|
||||
data_1 = [
|
||||
truncate_text_to_tokens(d, self.tokenizer, max_tokens_per_query)
|
||||
if isinstance(d, str)
|
||||
else d
|
||||
for d in data_1
|
||||
]
|
||||
if max_tokens_per_doc > 0:
|
||||
data_2 = [
|
||||
truncate_text_to_tokens(d, self.tokenizer, max_tokens_per_doc)
|
||||
if isinstance(d, str)
|
||||
else d
|
||||
for d in data_2
|
||||
]
|
||||
return ScoringData(data_1=data_1, data_2=data_2)
|
||||
|
||||
def valid_inputs(
|
||||
self,
|
||||
data_1: ScoreInput | list[ScoreInput],
|
||||
data_2: ScoreInput | list[ScoreInput],
|
||||
) -> ScoringData:
|
||||
scoring_data = validate_score_input(
|
||||
data_1,
|
||||
data_2,
|
||||
is_multimodal_model=self.is_multimodal_model,
|
||||
architecture=self.architecture,
|
||||
)
|
||||
return scoring_data
|
||||
|
||||
def valid_inputs_online(self, request: AnyPoolingRequest):
|
||||
if isinstance(request, ScoreRequest):
|
||||
data_1 = request.data_1
|
||||
data_2 = request.data_2
|
||||
elif isinstance(request, RerankRequest):
|
||||
data_1 = request.query
|
||||
data_2 = request.documents
|
||||
else:
|
||||
raise ValueError(f"Invalid {request.__class__.__name__} request type")
|
||||
|
||||
scoring_data = self.valid_inputs(data_1, data_2)
|
||||
return scoring_data
|
||||
|
||||
def _pair_error_output(
|
||||
self,
|
||||
output_1: PoolingRequestOutput,
|
||||
output_2: PoolingRequestOutput,
|
||||
) -> PoolingRequestOutput | None:
|
||||
failed = output_1 if output_1.error is not None else output_2
|
||||
if failed.error is None:
|
||||
return None
|
||||
|
||||
padding = [self.pad_token_id] if self.pad_token_id is not None else []
|
||||
return PoolingRequestOutput(
|
||||
request_id=f"{output_1.request_id}_{output_2.request_id}",
|
||||
outputs=failed.outputs,
|
||||
prompt_token_ids=(
|
||||
output_1.prompt_token_ids + padding + output_2.prompt_token_ids
|
||||
),
|
||||
num_cached_tokens=(output_1.num_cached_tokens + output_2.num_cached_tokens),
|
||||
finished=failed.finished,
|
||||
error=failed.error,
|
||||
)
|
||||
|
||||
|
||||
class BiEncoderIOProcessor(ScoringIOProcessor):
|
||||
name = "bi-encoder"
|
||||
pooling_task: PoolingTask = "embed"
|
||||
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
scoring_data = self.valid_inputs_online(request)
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
request=request
|
||||
)
|
||||
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
|
||||
scoring_data = self._truncate_scoring_data(
|
||||
scoring_data, max_tokens_per_query, max_tokens_per_doc
|
||||
)
|
||||
|
||||
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
|
||||
data_2 = score_data_to_prompts(
|
||||
scoring_data.data_2, "document", self.model_config
|
||||
)
|
||||
prompts = data_1 + data_2
|
||||
ctx.n_queries = len(data_1)
|
||||
|
||||
prompts_seq = prompt_to_seq(prompts)
|
||||
parsed_prompts = [
|
||||
parse_model_prompt(self.model_config, prompt) for prompt in prompts_seq
|
||||
]
|
||||
num_requests = len(parsed_prompts)
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
params_seq = self._params_to_seq(ctx.pooling_params, num_requests)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
EncodeCMPLRenderParams(
|
||||
prompts=parsed_prompts[i],
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=params_seq[i],
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
return requests
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: ScoringServeContext,
|
||||
):
|
||||
assert ctx.final_res_batch is not None
|
||||
assert isinstance(ctx.n_queries, int)
|
||||
|
||||
ctx.final_res_batch = self._post_process(
|
||||
outputs=ctx.final_res_batch, n_queries=ctx.n_queries
|
||||
)
|
||||
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineScoringInputsContext)
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
pooling_params=ctx.pooling_params
|
||||
)
|
||||
|
||||
scoring_data = ctx.scoring_data
|
||||
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
|
||||
scoring_data = self._truncate_scoring_data(
|
||||
scoring_data, max_tokens_per_query, max_tokens_per_doc
|
||||
)
|
||||
|
||||
data_1 = score_data_to_prompts(scoring_data.data_1, "query", self.model_config)
|
||||
data_2 = score_data_to_prompts(
|
||||
scoring_data.data_2, "document", self.model_config
|
||||
)
|
||||
prompts = data_1 + data_2
|
||||
|
||||
return super().get_request_factory_offline(
|
||||
OfflineEncodeInputsContext(
|
||||
pooling_task=self.pooling_task,
|
||||
prompts=prompts,
|
||||
tokenization_kwargs=ctx.tokenization_kwargs,
|
||||
pooling_params=ctx.pooling_params,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
)
|
||||
)
|
||||
|
||||
def post_process_offline(
|
||||
self,
|
||||
ctx: OfflineOutputsContext,
|
||||
) -> list[PoolingRequestOutput]:
|
||||
assert ctx.n_queries is not None
|
||||
return self._post_process(outputs=ctx.outputs, n_queries=ctx.n_queries)
|
||||
|
||||
#######################################
|
||||
# helpers
|
||||
|
||||
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
|
||||
emb_data_1 = outputs[:n_queries]
|
||||
emb_data_2 = outputs[n_queries:]
|
||||
|
||||
if len(emb_data_1) == 1:
|
||||
emb_data_1 = emb_data_1 * len(emb_data_2)
|
||||
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
for emb_1, emb_2 in zip(emb_data_1, emb_data_2):
|
||||
if error_output := self._pair_error_output(emb_1, emb_2):
|
||||
final_res_batch.append(error_output)
|
||||
continue
|
||||
|
||||
pair_score = F.cosine_similarity(
|
||||
emb_1.outputs.data.float(), emb_2.outputs.data.float(), dim=0
|
||||
)
|
||||
|
||||
padding: list[int] = []
|
||||
if self.pad_token_id is not None:
|
||||
padding = [self.pad_token_id]
|
||||
|
||||
tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids
|
||||
|
||||
final_res_batch.append(
|
||||
PoolingRequestOutput(
|
||||
request_id=f"{emb_1.request_id}_{emb_2.request_id}",
|
||||
outputs=pair_score,
|
||||
prompt_token_ids=tokens,
|
||||
num_cached_tokens=emb_1.num_cached_tokens + emb_2.num_cached_tokens,
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
return final_res_batch
|
||||
|
||||
|
||||
class LateInteractionIOProcessor(BiEncoderIOProcessor):
|
||||
name = "late-interaction"
|
||||
pooling_task: PoolingTask = "token_embed"
|
||||
|
||||
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
|
||||
# Split into query and document embeddings
|
||||
emb_data_1 = outputs[:n_queries]
|
||||
emb_data_2 = outputs[n_queries:]
|
||||
|
||||
# Expand queries if 1:N scoring
|
||||
if len(emb_data_1) == 1:
|
||||
emb_data_1 = emb_data_1 * len(emb_data_2)
|
||||
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
padding: list[int] = []
|
||||
if (pad_token_id := self.pad_token_id) is not None:
|
||||
padding = [pad_token_id]
|
||||
|
||||
# Compute MaxSim scores
|
||||
for emb_1, emb_2 in zip(emb_data_1, emb_data_2):
|
||||
if error_output := self._pair_error_output(emb_1, emb_2):
|
||||
final_res_batch.append(error_output)
|
||||
continue
|
||||
|
||||
# emb_1.outputs.data: [query_len, dim]
|
||||
# emb_2.outputs.data: [doc_len, dim]
|
||||
q_emb = emb_1.outputs.data
|
||||
d_emb = emb_2.outputs.data
|
||||
|
||||
maxsim_score = compute_maxsim_score(q_emb, d_emb)
|
||||
|
||||
tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids
|
||||
|
||||
final_res_batch.append(
|
||||
PoolingRequestOutput(
|
||||
request_id=f"{emb_1.request_id}_{emb_2.request_id}",
|
||||
outputs=maxsim_score,
|
||||
prompt_token_ids=tokens,
|
||||
num_cached_tokens=emb_1.num_cached_tokens + emb_2.num_cached_tokens,
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
return final_res_batch
|
||||
|
||||
|
||||
class FlashLateInteractionIOProcessor(LateInteractionIOProcessor):
|
||||
name = "flash-late-interaction"
|
||||
|
||||
def post_process_online(
|
||||
self,
|
||||
ctx: ScoringServeContext,
|
||||
):
|
||||
assert ctx.query_final_res_batch is not None
|
||||
assert ctx.final_res_batch is not None
|
||||
assert isinstance(ctx.n_queries, int)
|
||||
|
||||
# Expand queries if 1:N scoring
|
||||
if len(ctx.query_final_res_batch) == 1:
|
||||
ctx.query_final_res_batch = ctx.query_final_res_batch * len(
|
||||
ctx.final_res_batch
|
||||
)
|
||||
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
for d1, d2 in zip(ctx.query_final_res_batch, ctx.final_res_batch):
|
||||
padding: list[int] = []
|
||||
if (pad_token_id := self.pad_token_id) is not None:
|
||||
padding = [pad_token_id]
|
||||
|
||||
tokens = d1.prompt_token_ids + padding + d2.prompt_token_ids
|
||||
|
||||
final_res_batch.append(
|
||||
PoolingRequestOutput(
|
||||
request_id=f"{d1.request_id}_{d2.request_id}",
|
||||
outputs=d2.outputs,
|
||||
prompt_token_ids=tokens,
|
||||
num_cached_tokens=d1.num_cached_tokens + d2.num_cached_tokens,
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
ctx.final_res_batch = final_res_batch
|
||||
|
||||
|
||||
class CrossEncoderIOProcessor(ScoringIOProcessor):
|
||||
name = "cross-encoder"
|
||||
pooling_task: PoolingTask = "classify"
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
if is_mistral_tokenizer(self.tokenizer):
|
||||
raise ValueError("MistralTokenizer not supported for cross-encoding")
|
||||
|
||||
from vllm.model_executor.model_loader import get_model_cls
|
||||
from vllm.model_executor.models.interfaces import supports_score_template
|
||||
|
||||
model = get_model_cls(self.model_config)
|
||||
self.supports_score_template = supports_score_template(model)
|
||||
self.model = model if self.supports_score_template else None
|
||||
self.use_sep_token = self.model_config.use_sep_token
|
||||
|
||||
if (
|
||||
getattr(self.model_config.hf_config, "is_original_qwen3_reranker", False)
|
||||
and self.chat_template is None
|
||||
):
|
||||
suggested_template = (
|
||||
"examples/pooling/score/template/qwen3_vl_reranker.jinja"
|
||||
if self.model_config.is_multimodal_model
|
||||
else "examples/pooling/score/template/qwen3_reranker.jinja"
|
||||
)
|
||||
logger.warning(
|
||||
"Serving an original Qwen3 reranker (%s) without a "
|
||||
"--chat-template. The model was trained with a specific prompt "
|
||||
"template; running without it may produce inaccurate relevance "
|
||||
"scores. Consider specifying `--chat-template %s`.",
|
||||
self.model_config.model,
|
||||
suggested_template,
|
||||
)
|
||||
|
||||
#######################################
|
||||
# online APIs
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
scoring_data = self.valid_inputs_online(request)
|
||||
data_1 = scoring_data.data_1
|
||||
data_2 = scoring_data.data_2
|
||||
num_requests = len(data_2)
|
||||
|
||||
if len(data_1) == 1:
|
||||
data_1 = data_1 * num_requests
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
request=request
|
||||
)
|
||||
|
||||
tok_params = request.build_tok_params(self.model_config)
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
requests = [
|
||||
ScoringRenderParams(
|
||||
data_1=data_1[i],
|
||||
data_2=data_2[i],
|
||||
chat_template=self.chat_template,
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=ctx.pooling_params,
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
for i in range(num_requests)
|
||||
]
|
||||
|
||||
return requests
|
||||
|
||||
#######################################
|
||||
# offline APIs
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineScoringInputsContext)
|
||||
|
||||
data_1 = ctx.scoring_data.data_1
|
||||
data_2 = ctx.scoring_data.data_2
|
||||
num_requests = len(data_2)
|
||||
|
||||
if len(data_1) == 1:
|
||||
data_1 = data_1 * num_requests
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
pooling_params=ctx.pooling_params
|
||||
)
|
||||
tok_params = self.renderer.default_cmpl_tok_params.with_kwargs(
|
||||
**(ctx.tokenization_kwargs or {})
|
||||
)
|
||||
prompt_extras = ctx.pooling_params.extra_kwargs
|
||||
|
||||
seq_lora_requests = self._lora_request_to_seq(ctx.lora_request, num_requests)
|
||||
seq_priority = self._priority_to_seq(ctx.priorities, num_requests)
|
||||
|
||||
def request_factory() -> RequestGenerator:
|
||||
for i in range(num_requests):
|
||||
yield ScoringRenderParams(
|
||||
data_1=data_1[i],
|
||||
data_2=data_2[i],
|
||||
chat_template=(
|
||||
self.chat_template
|
||||
if ctx.chat_template is None
|
||||
else ctx.chat_template
|
||||
),
|
||||
max_tokens_per_query=max_tokens_per_query,
|
||||
max_tokens_per_doc=max_tokens_per_doc,
|
||||
tok_params=tok_params,
|
||||
prompt_extras=prompt_extras,
|
||||
skip_mm_cache=False,
|
||||
params=ctx.pooling_params,
|
||||
lora_requests=seq_lora_requests[i],
|
||||
priorities=seq_priority[i],
|
||||
)
|
||||
|
||||
return request_factory, num_requests
|
||||
|
||||
#######################################
|
||||
# helpers
|
||||
|
||||
def render(
|
||||
self,
|
||||
render_params: EncodeCMPLRenderParams
|
||||
| EncodeChatRenderParams
|
||||
| ScoringRenderParams,
|
||||
) -> PoolingEngineInput:
|
||||
if "data_1" not in render_params:
|
||||
raise ValueError(
|
||||
f"Unsupported render_params type {render_params.__class__.__name__}"
|
||||
)
|
||||
render_params = cast(ScoringRenderParams, render_params)
|
||||
|
||||
arrival_time = time.time()
|
||||
|
||||
tok_params = render_params["tok_params"]
|
||||
params = render_params["params"]
|
||||
prompt_extras = render_params["prompt_extras"]
|
||||
|
||||
_, engine_prompt = self.get_score_prompt(
|
||||
data_1=render_params["data_1"],
|
||||
data_2=render_params["data_2"],
|
||||
encode_kwargs=tok_params.get_encode_kwargs(),
|
||||
chat_template=render_params["chat_template"],
|
||||
max_tokens_per_query=render_params["max_tokens_per_query"],
|
||||
max_tokens_per_doc=render_params["max_tokens_per_doc"],
|
||||
chat_template_kwargs=prompt_extras.get("chat_template_kwargs")
|
||||
if prompt_extras
|
||||
else None,
|
||||
)
|
||||
|
||||
tok_params.apply_post_tokenization(self.tokenizer, engine_prompt)
|
||||
|
||||
if token_type_ids := engine_prompt.pop("token_type_ids", None):
|
||||
params = params.clone()
|
||||
compressed = compress_token_type_ids(
|
||||
_apply_post_tokenization_to_token_type_ids(
|
||||
self.tokenizer, tok_params, token_type_ids
|
||||
)
|
||||
)
|
||||
params.extra_kwargs = {
|
||||
**(params.extra_kwargs or {}),
|
||||
"compressed_token_type_ids": compressed,
|
||||
}
|
||||
|
||||
engine_prompt_extras = (
|
||||
{
|
||||
k: v
|
||||
for k in ("mm_processor_kwargs", "cache_salt")
|
||||
if (v := prompt_extras.get(k)) is not None
|
||||
}
|
||||
if prompt_extras
|
||||
else None
|
||||
)
|
||||
|
||||
if engine_prompt_extras:
|
||||
target_prompt = extract_target_prompt(self.model_config, engine_prompt)
|
||||
target_prompt.update(engine_prompt_extras)
|
||||
|
||||
engine_input = self.renderer.process_for_engine(engine_prompt, arrival_time)
|
||||
|
||||
return PoolingEngineInput(
|
||||
prompts=engine_input,
|
||||
params=params,
|
||||
lora_requests=render_params["lora_requests"],
|
||||
priorities=render_params["priorities"],
|
||||
)
|
||||
|
||||
def get_score_prompt(
|
||||
self,
|
||||
data_1: ScoreData,
|
||||
data_2: ScoreData,
|
||||
encode_kwargs: dict[str, Any],
|
||||
chat_template: str | None = None,
|
||||
max_tokens_per_query: int = 0,
|
||||
max_tokens_per_doc: int = 0,
|
||||
chat_template_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
model_config = self.model_config
|
||||
tokenizer = self.tokenizer
|
||||
|
||||
prompt_1, prompt_2, mm_data, mm_uuids = parse_score_data(
|
||||
data_1,
|
||||
data_2,
|
||||
model_config,
|
||||
)
|
||||
|
||||
# Limit the query and document separately before composing them.
|
||||
if max_tokens_per_query > 0 and isinstance(prompt_1, str):
|
||||
prompt_1 = truncate_text_to_tokens(
|
||||
prompt_1, tokenizer, max_tokens_per_query
|
||||
)
|
||||
if max_tokens_per_doc > 0 and isinstance(prompt_2, str):
|
||||
prompt_2 = truncate_text_to_tokens(prompt_2, tokenizer, max_tokens_per_doc)
|
||||
|
||||
def default_tokenizer_encode():
|
||||
local_kwargs = encode_kwargs.copy()
|
||||
|
||||
if self.supports_score_template:
|
||||
assert self.model is not None
|
||||
full_prompt = self.model.get_score_template(prompt_1, prompt_2)
|
||||
if full_prompt is None:
|
||||
raise ValueError("Get empty score template from model")
|
||||
|
||||
prompt_inputs = tokenizer(full_prompt, **local_kwargs)
|
||||
else:
|
||||
if self.use_sep_token:
|
||||
# cross_encoder models defaults to using separating token.
|
||||
if max_tokens_per_doc > 0 and isinstance(prompt_2, str):
|
||||
query_tokens = tokenizer.encode(
|
||||
prompt_1, add_special_tokens=False
|
||||
)
|
||||
num_special = get_num_special_tokens_for_pair(tokenizer)
|
||||
doc_limit_max_length = (
|
||||
len(query_tokens) + max_tokens_per_doc + num_special
|
||||
)
|
||||
existing_max_length = local_kwargs.get("max_length")
|
||||
if existing_max_length is not None:
|
||||
effective_max_length = min(
|
||||
doc_limit_max_length, existing_max_length
|
||||
)
|
||||
else:
|
||||
effective_max_length = doc_limit_max_length
|
||||
local_kwargs["truncation"] = "only_second"
|
||||
local_kwargs["max_length"] = effective_max_length
|
||||
|
||||
prompt_inputs = tokenizer(
|
||||
text=prompt_1, text_pair=prompt_2, **local_kwargs
|
||||
)
|
||||
full_prompt = tokenizer.decode(prompt_inputs["input_ids"])
|
||||
else:
|
||||
# `llm as reranker` defaults to not using separating token.
|
||||
full_prompt = prompt_1 + prompt_2
|
||||
prompt_inputs = tokenizer(text=full_prompt, **local_kwargs)
|
||||
return full_prompt, prompt_inputs
|
||||
|
||||
# FIXME: For now, we only apply a template when one is explicitly provided.
|
||||
# We cannot rely on the tokenizer's chat template because many models
|
||||
# inherit junk templates from their base LLM, which breaks both the models
|
||||
# and the tests that use them.
|
||||
if chat_template is None:
|
||||
full_prompt, prompt_inputs = default_tokenizer_encode()
|
||||
else:
|
||||
# FIXME:
|
||||
# Try applying a score template from the CLI arg or tokenizer_config.json
|
||||
# If that fails because there is no such template,
|
||||
# fall back to the default implementation.
|
||||
try:
|
||||
_safe_kwargs = chat_template_kwargs or {}
|
||||
_reserved = {"chat_template", "tools", "tokenize"}
|
||||
_unexpected = _reserved & _safe_kwargs.keys()
|
||||
if _unexpected:
|
||||
raise ValueError(
|
||||
"chat_template_kwargs contains reserved keys that "
|
||||
f"conflict with fixed scorer arguments: {_unexpected}"
|
||||
)
|
||||
full_prompt = safe_apply_chat_template(
|
||||
model_config,
|
||||
tokenizer,
|
||||
[
|
||||
{"role": "query", "content": prompt_1},
|
||||
{"role": "document", "content": prompt_2},
|
||||
],
|
||||
chat_template=chat_template,
|
||||
tools=None,
|
||||
tokenize=False,
|
||||
**_safe_kwargs,
|
||||
)
|
||||
prompt_inputs = tokenizer(full_prompt, **encode_kwargs)
|
||||
except ChatTemplateResolutionError:
|
||||
full_prompt, prompt_inputs = default_tokenizer_encode()
|
||||
|
||||
engine_prompt = TokensPrompt(prompt_token_ids=prompt_inputs["input_ids"])
|
||||
|
||||
if (token_type_ids := prompt_inputs.get("token_type_ids")) is not None:
|
||||
engine_prompt["token_type_ids"] = token_type_ids
|
||||
|
||||
if self.model is not None:
|
||||
self.model.post_process_tokens(engine_prompt)
|
||||
|
||||
if mm_data is not None:
|
||||
engine_prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
engine_prompt["multi_modal_uuids"] = mm_uuids
|
||||
|
||||
return full_prompt, engine_prompt
|
||||
|
||||
|
||||
class JinaRankingIOProcessorMixin:
|
||||
@staticmethod
|
||||
def format_docs_prompts_func(
|
||||
query: str,
|
||||
docs: list[str],
|
||||
special_tokens: dict[str, str] | None = None,
|
||||
instruction: str | None = None,
|
||||
no_thinking: bool = True,
|
||||
) -> str:
|
||||
# TODO: Try converting the code below into a chat template.
|
||||
|
||||
default_special_tokens = {
|
||||
"query_embed_token": "<|rerank_token|>",
|
||||
"doc_embed_token": "<|embed_token|>",
|
||||
}
|
||||
if special_tokens is None:
|
||||
special_tokens = default_special_tokens
|
||||
|
||||
def sanitize_input(text: str) -> str:
|
||||
for token in special_tokens.values():
|
||||
text = text.replace(token, "")
|
||||
return text
|
||||
|
||||
query = sanitize_input(query)
|
||||
docs = [sanitize_input(doc) for doc in docs]
|
||||
|
||||
prefix = (
|
||||
"<|im_start|>system\n"
|
||||
"You are a search relevance expert who can determine a ranking of the passages based on how relevant they are to the query. " # noqa: E501
|
||||
"If the query is a question, how relevant a passage is depends on how well it answers the question. " # noqa: E501
|
||||
"If not, try to analyze the intent of the query and assess how well each passage satisfies the intent. " # noqa: E501
|
||||
"If an instruction is provided, you should follow the instruction when determining the ranking." # noqa: E501
|
||||
"<|im_end|>\n<|im_start|>user\n"
|
||||
)
|
||||
suffix = "<|im_end|>\n<|im_start|>assistant\n"
|
||||
if no_thinking:
|
||||
suffix += "<think>\n\n</think>\n\n"
|
||||
|
||||
doc_emb_token = special_tokens["doc_embed_token"]
|
||||
query_emb_token = special_tokens["query_embed_token"]
|
||||
|
||||
prompt = (
|
||||
f"I will provide you with {len(docs)} passages, each indicated by a numerical identifier. " # noqa: E501
|
||||
f"Rank the passages based on their relevance to query: {query}\n"
|
||||
)
|
||||
|
||||
if instruction:
|
||||
instruction = sanitize_input(instruction)
|
||||
prompt += f"<instruct>\n{instruction}\n</instruct>\n"
|
||||
|
||||
doc_prompts = [
|
||||
f'<passage id="{i}">\n{doc}{doc_emb_token}\n</passage>'
|
||||
for i, doc in enumerate(docs)
|
||||
]
|
||||
prompt += "\n".join(doc_prompts) + "\n"
|
||||
prompt += f"<query>\n{query}{query_emb_token}\n</query>"
|
||||
|
||||
return prefix + prompt + suffix
|
||||
|
||||
@staticmethod
|
||||
def ensure_str(data: Sequence[Any]) -> list[str]:
|
||||
text: list[str] = []
|
||||
for prompt in data:
|
||||
if not isinstance(prompt, str):
|
||||
raise ValueError(
|
||||
"The JinaForRanking model only supports text as input."
|
||||
)
|
||||
text.append(prompt)
|
||||
return text
|
||||
|
||||
|
||||
class JinaRankingIOProcessor(LateInteractionIOProcessor, JinaRankingIOProcessorMixin):
|
||||
name = "jina-reranking-scoring"
|
||||
pooling_task: PoolingTask = "token_embed"
|
||||
|
||||
def get_request_factory_online(
|
||||
self, ctx: PoolingServeContext
|
||||
) -> Sequence[AnyRenderParam]:
|
||||
request = ctx.request
|
||||
ctx.n_queries = 1
|
||||
|
||||
prompt_extras = ctx.prompt_extras
|
||||
scoring_data = self.valid_inputs_online(request)
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
request=request
|
||||
)
|
||||
|
||||
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
|
||||
scoring_data = self._truncate_scoring_data(
|
||||
scoring_data, max_tokens_per_query, max_tokens_per_doc
|
||||
)
|
||||
|
||||
queries = self.ensure_str(scoring_data.data_1)
|
||||
docs = self.ensure_str(scoring_data.data_2)
|
||||
|
||||
chat_template_kwargs = (
|
||||
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
|
||||
)
|
||||
instruction = (
|
||||
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
|
||||
)
|
||||
|
||||
if len(queries) == 1:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=queries[0], docs=docs, instruction=instruction
|
||||
)
|
||||
]
|
||||
else:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=q, docs=[d], instruction=instruction
|
||||
)
|
||||
for q, d in zip(queries, docs)
|
||||
]
|
||||
|
||||
# Forward truncation from the real request: the base factory reads
|
||||
# these off ctx.request, so omitting them here silently drops
|
||||
# truncate_prompt_tokens for Jina rerank/score (unlike the embed and
|
||||
# bi/cross-encoder paths, which read them from the real request).
|
||||
ctx.request = PoolingCompletionRequest(
|
||||
task="token_embed",
|
||||
input=prompts,
|
||||
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
||||
truncation_side=request.truncation_side,
|
||||
)
|
||||
requests = PoolingIOProcessor.get_request_factory_online(self, ctx)
|
||||
ctx.request = request
|
||||
return requests
|
||||
|
||||
def get_request_factory_offline(
|
||||
self, ctx: AnyOfflineInputsContext
|
||||
) -> tuple[RequestFactory, int]:
|
||||
assert isinstance(ctx, OfflineScoringInputsContext)
|
||||
|
||||
max_tokens_per_query, max_tokens_per_doc = self._get_token_limits(
|
||||
pooling_params=ctx.pooling_params
|
||||
)
|
||||
scoring_data = ctx.scoring_data
|
||||
if max_tokens_per_query > 0 or max_tokens_per_doc > 0:
|
||||
scoring_data = self._truncate_scoring_data(
|
||||
scoring_data, max_tokens_per_query, max_tokens_per_doc
|
||||
)
|
||||
prompt_extras = ctx.pooling_params.extra_kwargs
|
||||
|
||||
queries = self.ensure_str(scoring_data.data_1)
|
||||
docs = self.ensure_str(scoring_data.data_2)
|
||||
chat_template_kwargs = (
|
||||
prompt_extras.get("chat_template_kwargs") if prompt_extras else None
|
||||
)
|
||||
instruction = (
|
||||
chat_template_kwargs.get("instruction") if chat_template_kwargs else None
|
||||
)
|
||||
|
||||
if len(queries) == 1:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=queries[0], docs=docs, instruction=instruction
|
||||
)
|
||||
]
|
||||
else:
|
||||
prompts = [
|
||||
self.format_docs_prompts_func(
|
||||
query=q, docs=[d], instruction=instruction
|
||||
)
|
||||
for q, d in zip(queries, docs)
|
||||
]
|
||||
|
||||
return PoolingIOProcessor.get_request_factory_offline(
|
||||
self,
|
||||
OfflineEncodeInputsContext(
|
||||
pooling_task=self.pooling_task,
|
||||
prompts=prompts,
|
||||
tokenization_kwargs=ctx.tokenization_kwargs,
|
||||
pooling_params=ctx.pooling_params,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
),
|
||||
)
|
||||
|
||||
def _post_process(self, outputs: list[PoolingRequestOutput], n_queries: int):
|
||||
final_res_batch: list[PoolingRequestOutput] = []
|
||||
|
||||
for i in range(len(outputs)):
|
||||
if outputs[i].error is not None:
|
||||
final_res_batch.append(outputs[i])
|
||||
continue
|
||||
|
||||
embeds = outputs[i].outputs.data.float()
|
||||
|
||||
# The JinaForRanking model concatenates docs first, then query.
|
||||
# Let's stay consistent with this novel design.
|
||||
query_embeds = embeds[-1]
|
||||
doc_embeds = embeds[:-1]
|
||||
|
||||
scores = F.cosine_similarity(query_embeds, doc_embeds)
|
||||
|
||||
for score in scores:
|
||||
final_res_batch.append(
|
||||
PoolingRequestOutput(
|
||||
request_id=outputs[i].request_id,
|
||||
outputs=score,
|
||||
prompt_token_ids=outputs[i].prompt_token_ids,
|
||||
num_cached_tokens=outputs[i].num_cached_tokens,
|
||||
finished=True,
|
||||
)
|
||||
)
|
||||
return final_res_batch
|
||||
|
||||
|
||||
ScoringIOProcessors: dict[str, type[ScoringIOProcessor]] = {
|
||||
p.name: p
|
||||
for p in [
|
||||
BiEncoderIOProcessor,
|
||||
LateInteractionIOProcessor,
|
||||
JinaRankingIOProcessor,
|
||||
FlashLateInteractionIOProcessor,
|
||||
CrossEncoderIOProcessor,
|
||||
]
|
||||
}
|
||||
@@ -1,187 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from typing import Any, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel, UsageInfo
|
||||
from vllm.renderers import TokenizeParams
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
from ..base.protocol import ClassifyRequestMixin, PoolingBasicRequestMixin
|
||||
from .typing import ScoreContentPartParam, ScoreInput
|
||||
|
||||
|
||||
class ScoringRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
|
||||
# --8<-- [start:scoring-common-params]
|
||||
max_tokens_per_query: int = Field(
|
||||
default=0,
|
||||
description=(
|
||||
"Maximum number of tokens per query. Queries longer than "
|
||||
"this will be truncated to this length. 0 means no "
|
||||
"query-level truncation is applied."
|
||||
),
|
||||
)
|
||||
max_tokens_per_doc: int = Field(
|
||||
default=0,
|
||||
description=(
|
||||
"Maximum number of tokens per document. Documents longer than "
|
||||
"this will be truncated to this length. 0 means no "
|
||||
"document-level truncation is applied (only truncate_prompt_tokens "
|
||||
"applies to the combined query+document)."
|
||||
),
|
||||
)
|
||||
instruction: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Task instruction prepended to each scored pair via the chat "
|
||||
"template. Equivalent to passing "
|
||||
"chat_template_kwargs={'instruction': ...}."
|
||||
),
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Additional keyword args to pass to the chat template renderer. "
|
||||
"Will be accessible by the score/rerank chat template."
|
||||
),
|
||||
)
|
||||
# --8<-- [end:scoring-common-params]
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _merge_instruction_into_kwargs(self) -> "ScoringRequestMixin":
|
||||
"""Fold the top-level `instruction` field into `chat_template_kwargs`.
|
||||
|
||||
This allows callers to use either the convenience field or the generic
|
||||
dict. Explicit keys inside `chat_template_kwargs` take precedence over
|
||||
the top-level `instruction` field.
|
||||
"""
|
||||
if self.instruction is not None:
|
||||
merged = dict(self.chat_template_kwargs or {})
|
||||
merged.setdefault("instruction", self.instruction)
|
||||
self.chat_template_kwargs = merged
|
||||
return self
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return self._build_pooling_tok_params(
|
||||
model_config,
|
||||
add_special_tokens=True,
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=0,
|
||||
)
|
||||
|
||||
def to_pooling_params(self, task: PoolingTask = "classify"):
|
||||
return PoolingParams(
|
||||
task=task,
|
||||
use_activation=self.use_activation,
|
||||
)
|
||||
|
||||
|
||||
class ScoreDataRequest(ScoringRequestMixin):
|
||||
data_1: ScoreInput | list[ScoreInput]
|
||||
data_2: ScoreInput | list[ScoreInput]
|
||||
|
||||
|
||||
class ScoreQueriesDocumentsRequest(ScoringRequestMixin):
|
||||
# --8<-- [start:score-request-params]
|
||||
queries: ScoreInput | list[ScoreInput]
|
||||
documents: ScoreInput | list[ScoreInput]
|
||||
# --8<-- [end:score-request-params]
|
||||
|
||||
@property
|
||||
def data_1(self):
|
||||
return self.queries
|
||||
|
||||
@property
|
||||
def data_2(self):
|
||||
return self.documents
|
||||
|
||||
|
||||
class ScoreQueriesItemsRequest(ScoringRequestMixin):
|
||||
queries: ScoreInput | list[ScoreInput]
|
||||
items: ScoreInput | list[ScoreInput]
|
||||
|
||||
@property
|
||||
def data_1(self):
|
||||
return self.queries
|
||||
|
||||
@property
|
||||
def data_2(self):
|
||||
return self.items
|
||||
|
||||
|
||||
class ScoreTextRequest(ScoringRequestMixin):
|
||||
text_1: ScoreInput | list[ScoreInput]
|
||||
text_2: ScoreInput | list[ScoreInput]
|
||||
|
||||
@property
|
||||
def data_1(self):
|
||||
return self.text_1
|
||||
|
||||
@property
|
||||
def data_2(self):
|
||||
return self.text_2
|
||||
|
||||
|
||||
ScoreRequest: TypeAlias = (
|
||||
ScoreQueriesDocumentsRequest
|
||||
| ScoreQueriesItemsRequest
|
||||
| ScoreDataRequest
|
||||
| ScoreTextRequest
|
||||
)
|
||||
|
||||
|
||||
class RerankRequest(ScoringRequestMixin):
|
||||
# --8<-- [start:rerank-request-params]
|
||||
query: ScoreInput
|
||||
documents: ScoreInput | list[ScoreInput]
|
||||
top_n: int = Field(default=0, ge=0)
|
||||
# --8<-- [end:rerank-request-params]
|
||||
|
||||
|
||||
ScoringRequest: TypeAlias = ScoreRequest | RerankRequest
|
||||
|
||||
|
||||
class RerankDocument(BaseModel):
|
||||
text: str | None = None
|
||||
multi_modal: list[ScoreContentPartParam] | None = None
|
||||
|
||||
|
||||
class RerankResult(BaseModel):
|
||||
index: int
|
||||
document: RerankDocument
|
||||
relevance_score: float
|
||||
|
||||
|
||||
class RerankUsage(BaseModel):
|
||||
prompt_tokens: int
|
||||
total_tokens: int
|
||||
|
||||
|
||||
class RerankResponse(OpenAIBaseModel):
|
||||
id: str
|
||||
model: str
|
||||
usage: RerankUsage
|
||||
results: list[RerankResult]
|
||||
|
||||
|
||||
class ScoreResponseData(OpenAIBaseModel):
|
||||
index: int
|
||||
object: str = "score"
|
||||
score: float
|
||||
|
||||
|
||||
class ScoreResponse(OpenAIBaseModel):
|
||||
id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
|
||||
object: str = "list"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
data: list[ScoreResponseData]
|
||||
usage: UsageInfo
|
||||
|
||||
|
||||
ScoringResponse: TypeAlias = RerankResponse | ScoreResponse
|
||||
@@ -1,315 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm import PoolingParams
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput
|
||||
from vllm.tasks import SCORE_TYPE_MAP, SupportedTask
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.v1.pool.late_interaction import (
|
||||
build_late_interaction_doc_params,
|
||||
build_late_interaction_query_params,
|
||||
)
|
||||
|
||||
from ..base.io_processor import PoolingIOProcessor
|
||||
from ..base.serving import PoolingServing
|
||||
from .io_processor import ScoringIOProcessors, ScoringServeContext
|
||||
from .protocol import (
|
||||
RerankDocument,
|
||||
RerankRequest,
|
||||
RerankResponse,
|
||||
RerankResult,
|
||||
RerankUsage,
|
||||
ScoreRequest,
|
||||
ScoreResponse,
|
||||
ScoreResponseData,
|
||||
)
|
||||
from .typing import ScoreInput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class ServingScores(PoolingServing):
|
||||
request_id_prefix = "score"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine_client: EngineClient,
|
||||
*args,
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
enable_flash_late_interaction: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
pooling_task = engine_client.model_config.get_pooling_task(supported_tasks)
|
||||
score_type = SCORE_TYPE_MAP.get(pooling_task, None) # type: ignore[arg-type]
|
||||
assert score_type is not None
|
||||
|
||||
self.io_processor_name: str = score_type
|
||||
self.enable_flash_late_interaction = (
|
||||
self.io_processor_name == "late-interaction"
|
||||
and enable_flash_late_interaction
|
||||
)
|
||||
|
||||
if self.enable_flash_late_interaction:
|
||||
self.io_processor_name = "flash-late-interaction"
|
||||
|
||||
if engine_client.model_config.architecture == "JinaForRanking":
|
||||
self.io_processor_name = "jina-reranking-scoring"
|
||||
self.enable_flash_late_interaction = False
|
||||
|
||||
super().__init__(engine_client, *args, **kwargs)
|
||||
|
||||
def init_io_processor(self, *args, **kwargs) -> PoolingIOProcessor:
|
||||
return ScoringIOProcessors[self.io_processor_name](*args, **kwargs)
|
||||
|
||||
async def __call__(self, *args, **kwargs) -> Response:
|
||||
if not self.enable_flash_late_interaction:
|
||||
return await super().__call__(*args, **kwargs)
|
||||
|
||||
return await self.flash_late_interaction(*args, **kwargs)
|
||||
|
||||
def _build_response(
|
||||
self,
|
||||
ctx: ScoringServeContext,
|
||||
) -> JSONResponse:
|
||||
final_res_batch = ctx.final_res_batch
|
||||
request_id = ctx.request_id
|
||||
created_time = ctx.created_time
|
||||
model_name = ctx.model_name
|
||||
assert ctx.engine_inputs is not None
|
||||
num_prompt_tokens = sum(
|
||||
self._extract_prompt_len(engine_input["prompts"])
|
||||
for engine_input in ctx.engine_inputs
|
||||
)
|
||||
|
||||
if isinstance(ctx.request, ScoreRequest):
|
||||
return self._request_output_to_score_response(
|
||||
final_res_batch,
|
||||
request_id,
|
||||
created_time,
|
||||
model_name,
|
||||
num_prompt_tokens,
|
||||
)
|
||||
elif isinstance(ctx.request, RerankRequest):
|
||||
return self._request_output_to_rerank_response(
|
||||
final_res_batch,
|
||||
request_id,
|
||||
model_name,
|
||||
ctx.request.documents,
|
||||
ctx.request.top_n if ctx.request.top_n > 0 else len(final_res_batch),
|
||||
num_prompt_tokens,
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Invalid {self.request_id_prefix} request type")
|
||||
|
||||
def _request_output_to_score_response(
|
||||
self,
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
num_prompt_tokens: int,
|
||||
) -> JSONResponse:
|
||||
items: list[ScoreResponseData] = []
|
||||
|
||||
for idx, final_res in enumerate(final_res_batch):
|
||||
classify_res = ScoringRequestOutput.from_base(final_res)
|
||||
|
||||
item = ScoreResponseData(
|
||||
index=idx,
|
||||
score=classify_res.outputs.score,
|
||||
)
|
||||
items.append(item)
|
||||
|
||||
usage = UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
total_tokens=num_prompt_tokens,
|
||||
)
|
||||
|
||||
response = ScoreResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
model=model_name,
|
||||
data=items,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
def _request_output_to_rerank_response(
|
||||
self,
|
||||
final_res_batch: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
model_name: str,
|
||||
documents: ScoreInput | list[ScoreInput],
|
||||
top_n: int,
|
||||
num_prompt_tokens: int,
|
||||
) -> JSONResponse:
|
||||
if not isinstance(documents, list):
|
||||
documents = [documents]
|
||||
|
||||
results: list[RerankResult] = []
|
||||
for idx, final_res in enumerate(final_res_batch):
|
||||
classify_res = ScoringRequestOutput.from_base(final_res)
|
||||
|
||||
document = documents[idx]
|
||||
if isinstance(document, str):
|
||||
rerank_document = RerankDocument(text=document)
|
||||
else:
|
||||
rerank_document = RerankDocument(
|
||||
multi_modal=document.get("content", [])
|
||||
)
|
||||
|
||||
result = RerankResult(
|
||||
index=idx,
|
||||
document=rerank_document,
|
||||
relevance_score=classify_res.outputs.score,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
# sort by relevance, then return the top n if set
|
||||
results.sort(key=lambda x: x.relevance_score, reverse=True)
|
||||
if top_n < len(documents):
|
||||
results = results[:top_n]
|
||||
|
||||
response = RerankResponse(
|
||||
id=request_id,
|
||||
model=model_name,
|
||||
results=results,
|
||||
usage=RerankUsage(
|
||||
total_tokens=num_prompt_tokens, prompt_tokens=num_prompt_tokens
|
||||
),
|
||||
)
|
||||
|
||||
return JSONResponse(content=response.model_dump())
|
||||
|
||||
###################################################################################
|
||||
### Run pooling score MaxSim on worker side (GPU) in the API server process
|
||||
### Can significantly improve late-interaction scoring performance.
|
||||
|
||||
async def flash_late_interaction(self, *args, **kwargs) -> Response:
|
||||
ctx = await self._init_ctx(self.io_processor, *args, **kwargs)
|
||||
await self._preprocessing(self.io_processor, ctx)
|
||||
|
||||
try:
|
||||
# stage 1: encode queries and cache token embeddings on workers.
|
||||
await self._flash_late_interaction_encode_queries(ctx)
|
||||
# stage 2: encode docs and return scalar scores from workers.
|
||||
await self._flash_late_interaction_encode_docs(ctx)
|
||||
except BaseException:
|
||||
try:
|
||||
await self._cleanup_flash_late_interaction(ctx)
|
||||
except Exception:
|
||||
logger.exception("Failed to clean up late-interaction query cache.")
|
||||
raise
|
||||
|
||||
return await self._postprocessing_async(self.io_processor, ctx)
|
||||
|
||||
async def _cleanup_flash_late_interaction(self, ctx: ScoringServeContext) -> None:
|
||||
query_keys = ctx.late_interaction_query_keys
|
||||
if not query_keys:
|
||||
return
|
||||
|
||||
doc_keys = ctx.late_interaction_doc_keys or []
|
||||
|
||||
try:
|
||||
# Stop documents before removing the query tensors they reference.
|
||||
await self.engine_client.abort([*query_keys, *doc_keys])
|
||||
finally:
|
||||
await self.engine_client.collective_rpc(
|
||||
"release_late_interaction_query_cache",
|
||||
args=(query_keys,),
|
||||
)
|
||||
|
||||
async def _flash_late_interaction_encode_queries(self, ctx: ScoringServeContext):
|
||||
assert ctx.n_queries is not None
|
||||
assert ctx.engine_inputs is not None
|
||||
assert isinstance(ctx.pooling_params, PoolingParams)
|
||||
|
||||
n_queries = ctx.n_queries
|
||||
n_docs = len(ctx.engine_inputs) - n_queries
|
||||
query_engine_inputs = ctx.engine_inputs[:n_queries]
|
||||
|
||||
query_namespace = random_uuid()
|
||||
query_keys = [
|
||||
f"late-interaction-{query_namespace}-query-{i}" for i in range(n_queries)
|
||||
]
|
||||
ctx.late_interaction_query_keys = query_keys
|
||||
query_uses = [n_docs if n_queries == 1 else 1] * n_queries
|
||||
|
||||
for i in range(n_queries):
|
||||
pooling_params = ctx.pooling_params.clone()
|
||||
pooling_params.late_interaction_params = (
|
||||
build_late_interaction_query_params(
|
||||
query_key=query_keys[i],
|
||||
query_uses=query_uses[i],
|
||||
)
|
||||
)
|
||||
query_engine_inputs[i]["params"] = pooling_params
|
||||
|
||||
assert n_queries == len(query_engine_inputs) == len(query_keys)
|
||||
|
||||
query_ctx = ScoringServeContext(
|
||||
request=ctx.request,
|
||||
raw_request=ctx.raw_request,
|
||||
model_name=ctx.model_name,
|
||||
request_id=ctx.request_id,
|
||||
pooling_params=ctx.pooling_params,
|
||||
prompt_request_ids=query_keys,
|
||||
engine_inputs=query_engine_inputs,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
)
|
||||
|
||||
await self._prepare_generators(query_ctx)
|
||||
await self._collect_batch(query_ctx)
|
||||
ctx.query_final_res_batch = query_ctx.final_res_batch
|
||||
|
||||
async def _flash_late_interaction_encode_docs(self, ctx: ScoringServeContext):
|
||||
assert ctx.n_queries is not None
|
||||
assert ctx.engine_inputs is not None
|
||||
assert isinstance(ctx.pooling_params, PoolingParams)
|
||||
|
||||
n_queries = ctx.n_queries
|
||||
n_docs = len(ctx.engine_inputs) - n_queries
|
||||
doc_engine_inputs = ctx.engine_inputs[n_queries:]
|
||||
|
||||
query_keys = ctx.late_interaction_query_keys
|
||||
if query_keys is None:
|
||||
raise RuntimeError("Late-interaction query keys were not initialized.")
|
||||
doc_namespace = random_uuid()
|
||||
doc_keys = [f"late-interaction-{doc_namespace}-doc-{i}" for i in range(n_docs)]
|
||||
ctx.late_interaction_doc_keys = doc_keys
|
||||
|
||||
for i in range(n_docs):
|
||||
query_idx = 0 if n_queries == 1 else i
|
||||
pooling_params = ctx.pooling_params.clone()
|
||||
pooling_params.late_interaction_params = build_late_interaction_doc_params(
|
||||
query_key=query_keys[query_idx]
|
||||
)
|
||||
doc_engine_inputs[i]["params"] = pooling_params
|
||||
|
||||
assert n_docs == len(doc_engine_inputs) == len(doc_keys)
|
||||
|
||||
doc_ctx = ScoringServeContext(
|
||||
request=ctx.request,
|
||||
raw_request=ctx.raw_request,
|
||||
model_name=ctx.model_name,
|
||||
request_id=ctx.request_id,
|
||||
pooling_params=ctx.pooling_params,
|
||||
prompt_request_ids=doc_keys,
|
||||
engine_inputs=doc_engine_inputs,
|
||||
lora_request=ctx.lora_request,
|
||||
priorities=ctx.priorities,
|
||||
prompt_extras=ctx.prompt_extras,
|
||||
)
|
||||
|
||||
await self._prepare_generators(doc_ctx)
|
||||
await self._collect_batch(doc_ctx)
|
||||
|
||||
ctx.final_res_batch = doc_ctx.final_res_batch
|
||||
@@ -1,45 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from typing import Required, TypeAlias
|
||||
|
||||
from typing_extensions import TypedDict
|
||||
|
||||
from vllm.renderers.chat_utils import (
|
||||
ChatCompletionContentPartImageEmbedsParam,
|
||||
ChatCompletionContentPartImageParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
ChatCompletionContentPartVideoParam,
|
||||
)
|
||||
|
||||
ScoreContentPartParam: TypeAlias = (
|
||||
ChatCompletionContentPartImageParam
|
||||
| ChatCompletionContentPartImageEmbedsParam
|
||||
| ChatCompletionContentPartTextParam
|
||||
| ChatCompletionContentPartVideoParam
|
||||
)
|
||||
|
||||
|
||||
class ScoreMultiModalParam(TypedDict, total=False):
|
||||
"""A specialized parameter type for scoring multimodal content.
|
||||
|
||||
The reasons why don't reuse `CustomChatCompletionMessageParam` directly:
|
||||
1. Score tasks don't need the 'role' field (user/assistant/system) that's required in chat completions
|
||||
2. Including chat-specific fields would confuse users about their purpose in scoring
|
||||
3. This is a more focused interface that only exposes what's needed for scoring
|
||||
""" # noqa: E501
|
||||
|
||||
content: Required[list[ScoreContentPartParam]]
|
||||
"""The multimodal contents"""
|
||||
|
||||
|
||||
# Raw input data with content key in ScoreMultiModalParam.
|
||||
ScoreInput = str | ScoreMultiModalParam
|
||||
# Score data without content key.
|
||||
ScoreData = str | list[ScoreContentPartParam]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScoringData:
|
||||
data_1: list[ScoreData]
|
||||
data_2: list[ScoreData]
|
||||
@@ -1,291 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm import PromptType, TextPrompt
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.exceptions import VLLMValidationError
|
||||
from vllm.inputs import MultiModalDataDict, MultiModalUUIDDict
|
||||
from vllm.renderers.chat_utils import (
|
||||
BaseMultiModalItemTracker,
|
||||
ChatCompletionContentPartParam,
|
||||
ChatCompletionContentPartTextParam,
|
||||
ConversationMessage,
|
||||
MultiModalItemTracker,
|
||||
parse_chat_message_content_parts,
|
||||
)
|
||||
|
||||
from .typing import (
|
||||
ScoreContentPartParam,
|
||||
ScoreData,
|
||||
ScoreInput,
|
||||
ScoringData,
|
||||
)
|
||||
|
||||
|
||||
def get_num_special_tokens_for_pair(tokenizer) -> int:
|
||||
"""Get number of special tokens added for a text pair encoding."""
|
||||
method = getattr(tokenizer, "num_special_tokens_to_add", None)
|
||||
if method is not None:
|
||||
try:
|
||||
return method(pair=True)
|
||||
except TypeError:
|
||||
pass
|
||||
# Fallback: compute by tokenizing empty strings
|
||||
empty_encoding = tokenizer("", text_pair="", add_special_tokens=True)
|
||||
return len(empty_encoding["input_ids"])
|
||||
|
||||
|
||||
def truncate_text_to_tokens(
|
||||
text: str,
|
||||
tokenizer,
|
||||
max_tokens: int,
|
||||
) -> str:
|
||||
"""Truncate text to a maximum number of content tokens.
|
||||
|
||||
Uses offset_mapping to slice the original text at the exact character
|
||||
boundary, avoiding lossy encode→decode round-trips that can shift
|
||||
the token count due to BPE merge boundary changes.
|
||||
"""
|
||||
if max_tokens <= 0:
|
||||
return ""
|
||||
|
||||
while text:
|
||||
encoding = tokenizer(
|
||||
text,
|
||||
add_special_tokens=False,
|
||||
return_offsets_mapping=True,
|
||||
)
|
||||
offsets = encoding["offset_mapping"]
|
||||
if len(encoding["input_ids"]) <= max_tokens:
|
||||
break
|
||||
|
||||
char_end = min(offsets[max_tokens - 1][1], offsets[max_tokens][0])
|
||||
if char_end >= len(text):
|
||||
char_end = len(text) - 1
|
||||
text = text[:char_end]
|
||||
|
||||
return text
|
||||
|
||||
|
||||
def compute_maxsim_score(q_emb: torch.Tensor, d_emb: torch.Tensor) -> torch.Tensor:
|
||||
"""Compute ColBERT MaxSim score.
|
||||
|
||||
Args:
|
||||
q_emb: Query token embeddings [query_len, dim]
|
||||
d_emb: Document token embeddings [doc_len, dim]
|
||||
|
||||
Returns:
|
||||
MaxSim score (sum over query tokens of max similarity to any doc token)
|
||||
|
||||
"""
|
||||
# compute in float32 for numerical stability
|
||||
# [query_len, doc_len]
|
||||
token_scores = torch.matmul(q_emb.float(), d_emb.float().T)
|
||||
# Max over document tokens, sum over query tokens
|
||||
return token_scores.amax(dim=-1).sum()
|
||||
|
||||
|
||||
def _validate_mm_score_input(
|
||||
data: list[ScoreInput],
|
||||
is_multimodal_model: bool,
|
||||
architecture: str,
|
||||
) -> list[ScoreData]:
|
||||
out: list[ScoreData] = []
|
||||
for d in data:
|
||||
if isinstance(d, str):
|
||||
out.append(d)
|
||||
else:
|
||||
if not is_multimodal_model:
|
||||
raise VLLMValidationError(
|
||||
f"MultiModalParam is not supported for {architecture}"
|
||||
)
|
||||
content = cast(list[ScoreContentPartParam], d.get("content", []))
|
||||
out.append(content)
|
||||
return out
|
||||
|
||||
|
||||
def _validate_score_input_lens(
|
||||
data_1: list[ScoreData],
|
||||
data_2: list[ScoreData],
|
||||
):
|
||||
len_1 = len(data_1)
|
||||
len_2 = len(data_2)
|
||||
|
||||
if len_1 > 1 and len_1 != len_2:
|
||||
raise VLLMValidationError("Input lengths must be either 1:1, 1:N or N:N")
|
||||
if len_1 == 0:
|
||||
raise VLLMValidationError("At least one text element must be given")
|
||||
if len_2 == 0:
|
||||
raise VLLMValidationError("At least one text_pair element must be given")
|
||||
|
||||
|
||||
def validate_score_input(
|
||||
data_1: ScoreInput | list[ScoreInput],
|
||||
data_2: ScoreInput | list[ScoreInput],
|
||||
is_multimodal_model: bool,
|
||||
architecture: str,
|
||||
) -> ScoringData:
|
||||
if not isinstance(data_1, list):
|
||||
data_1 = [data_1]
|
||||
|
||||
if not isinstance(data_2, list):
|
||||
data_2 = [data_2]
|
||||
|
||||
score_input_1 = _validate_mm_score_input(data_1, is_multimodal_model, architecture)
|
||||
score_input_2 = _validate_mm_score_input(data_2, is_multimodal_model, architecture)
|
||||
_validate_score_input_lens(score_input_1, score_input_2)
|
||||
return ScoringData(data_1=score_input_1, data_2=score_input_2)
|
||||
|
||||
|
||||
def score_data_to_prompts(
|
||||
data_list: list[ScoreData],
|
||||
role: str,
|
||||
model_config: ModelConfig,
|
||||
) -> list[PromptType]:
|
||||
"""Convert a list of ScoreData into PromptType objects.
|
||||
|
||||
For plain text inputs, returns the string directly.
|
||||
For multimodal inputs (list of content parts), parses them into
|
||||
a :class:`TextPrompt` with attached ``multi_modal_data`` /
|
||||
``multi_modal_uuids``.
|
||||
|
||||
This is used by late-interaction scoring where each query/document
|
||||
is encoded independently.
|
||||
"""
|
||||
prompts: list[PromptType] = []
|
||||
for data in data_list:
|
||||
if isinstance(data, str):
|
||||
prompts.append(data)
|
||||
else:
|
||||
text, mm_data, mm_uuids = parse_score_data_single(data, role, model_config)
|
||||
prompt: TextPrompt = TextPrompt(prompt=text)
|
||||
if mm_data is not None:
|
||||
prompt["multi_modal_data"] = mm_data
|
||||
if mm_uuids is not None:
|
||||
prompt["multi_modal_uuids"] = mm_uuids
|
||||
prompts.append(prompt)
|
||||
return prompts
|
||||
|
||||
|
||||
def _ensure_str(content: list[ConversationMessage]) -> str:
|
||||
"""Extract a single string prompt from parsed conversation content."""
|
||||
assert len(content) == 1
|
||||
prompt = content[0]["content"]
|
||||
if prompt is not None and isinstance(prompt, str):
|
||||
return cast(str, prompt)
|
||||
raise ValueError(f"Only string content is supported, but got {content}.")
|
||||
|
||||
|
||||
def _parse_score_content(
|
||||
role: str,
|
||||
data: ScoreData,
|
||||
mm_tracker: BaseMultiModalItemTracker,
|
||||
) -> list[ConversationMessage]:
|
||||
parts: Iterable[ChatCompletionContentPartParam]
|
||||
if isinstance(data, str):
|
||||
parts = [ChatCompletionContentPartTextParam(type="text", text=data)]
|
||||
else:
|
||||
parts = cast(Iterable[ChatCompletionContentPartParam], data)
|
||||
|
||||
mm_parser = mm_tracker.create_parser()
|
||||
|
||||
parse_res = parse_chat_message_content_parts(
|
||||
role=role,
|
||||
parts=parts,
|
||||
mm_tracker=mm_tracker,
|
||||
wrap_dicts=False,
|
||||
interleave_strings=False,
|
||||
multimodal_content_part_separator="",
|
||||
)
|
||||
|
||||
if parse_res:
|
||||
return parse_res
|
||||
|
||||
mm_placeholder_storage = mm_parser.mm_placeholder_storage()
|
||||
|
||||
if (
|
||||
len(mm_placeholder_storage) != 1
|
||||
or len(next(iter(mm_placeholder_storage.values()))) != 1
|
||||
):
|
||||
raise ValueError("Only one multi-modal item is supported")
|
||||
|
||||
return next(iter(mm_placeholder_storage.values()))[0]
|
||||
|
||||
|
||||
def parse_score_data_single(
|
||||
data: ScoreData,
|
||||
role: str,
|
||||
model_config: ModelConfig,
|
||||
) -> tuple[str, MultiModalDataDict | None, MultiModalUUIDDict | None]:
|
||||
"""Parse **one** ScoreData into a text prompt and its own multi-modal
|
||||
data.
|
||||
|
||||
Unlike :func:`parse_score_data`, each call creates an **independent**
|
||||
:class:`MultiModalItemTracker` so multi-modal items are kept separate.
|
||||
This is the correct behaviour for late-interaction scoring, where
|
||||
query and document are encoded independently.
|
||||
"""
|
||||
mm_tracker = MultiModalItemTracker(model_config)
|
||||
content = _parse_score_content(role, data, mm_tracker)
|
||||
|
||||
prompt = _ensure_str(content)
|
||||
mm_items, mm_uuids = mm_tracker.resolve_items()
|
||||
return prompt, mm_items, mm_uuids
|
||||
|
||||
|
||||
def parse_score_data(
|
||||
data_1: ScoreData,
|
||||
data_2: ScoreData,
|
||||
model_config: ModelConfig,
|
||||
) -> tuple[str, str, MultiModalDataDict | None, MultiModalUUIDDict | None]:
|
||||
"""Parse a query-document pair into text prompts and shared multi-modal
|
||||
data.
|
||||
|
||||
Uses a **single** :class:`MultiModalItemTracker` so that multi-modal
|
||||
items from both inputs are merged into one ``mm_data`` dict. This is
|
||||
the correct behaviour for cross-encoder scoring, where query and
|
||||
document are concatenated into a single model prompt.
|
||||
"""
|
||||
mm_tracker = MultiModalItemTracker(model_config)
|
||||
|
||||
content_1 = _parse_score_content("query", data_1, mm_tracker)
|
||||
content_2 = _parse_score_content("document", data_2, mm_tracker)
|
||||
|
||||
prompt_1 = _ensure_str(content_1)
|
||||
prompt_2 = _ensure_str(content_2)
|
||||
mm_items, mm_uuids = mm_tracker.resolve_items()
|
||||
|
||||
return prompt_1, prompt_2, mm_items, mm_uuids
|
||||
|
||||
|
||||
def compress_token_type_ids(token_type_ids: list[int]) -> tuple[int, int]:
|
||||
"""Return the half-open range containing token type 1.
|
||||
|
||||
Right padding uses the tokenizer's padding token type, which is commonly
|
||||
zero. Therefore, a padded query-document pair can contain zeros followed
|
||||
by ones followed by trailing zeros.
|
||||
"""
|
||||
first_one = len(token_type_ids)
|
||||
first_zero_after_one = len(token_type_ids)
|
||||
err_msg = (
|
||||
"Token type ids are expected to be a sequence"
|
||||
" of zeros followed by ones and optional trailing zeros"
|
||||
)
|
||||
for i, type_id in enumerate(token_type_ids):
|
||||
if type_id not in (0, 1):
|
||||
raise ValueError(err_msg)
|
||||
if type_id == 1:
|
||||
if first_zero_after_one < len(token_type_ids):
|
||||
raise ValueError(err_msg)
|
||||
if first_one == len(token_type_ids):
|
||||
first_one = i
|
||||
elif first_one < len(token_type_ids) and first_zero_after_one == len(
|
||||
token_type_ids
|
||||
):
|
||||
first_zero_after_one = i
|
||||
|
||||
return first_one, first_zero_after_one
|
||||
@@ -1,214 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Callable, Generator, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Generic, NotRequired, TypeAlias, TypedDict, TypeVar
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import ConfigDict
|
||||
|
||||
from vllm import PoolingParams, PoolingRequestOutput, PromptType
|
||||
from vllm.inputs import DataPrompt, EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import LateChunkingMetadata
|
||||
from vllm.renderers import ChatParams, TokenizeParams
|
||||
from vllm.renderers.chat_utils import ChatCompletionMessageParam
|
||||
from vllm.renderers.inputs import DictPrompt
|
||||
|
||||
from ...tasks import PoolingTask
|
||||
from .classify.protocol import (
|
||||
ClassificationChatRequest,
|
||||
ClassificationCompletionRequest,
|
||||
ClassificationResponse,
|
||||
)
|
||||
from .embed.protocol import (
|
||||
CohereEmbedRequest,
|
||||
EmbeddingBytesResponse,
|
||||
EmbeddingChatInputRequest,
|
||||
EmbeddingChatRequest,
|
||||
EmbeddingCompletionRequest,
|
||||
EmbeddingRequest,
|
||||
EmbeddingResponse,
|
||||
)
|
||||
from .pooling.protocol import (
|
||||
IOProcessorRequest,
|
||||
PoolingBytesResponse,
|
||||
PoolingChatRequest,
|
||||
PoolingCompletionRequest,
|
||||
PoolingResponse,
|
||||
)
|
||||
from .scoring.protocol import ScoringRequest, ScoringResponse
|
||||
from .scoring.typing import ScoreData, ScoringData
|
||||
|
||||
PoolingCompletionLikeRequest: TypeAlias = (
|
||||
EmbeddingCompletionRequest
|
||||
| ClassificationCompletionRequest
|
||||
| PoolingCompletionRequest
|
||||
)
|
||||
|
||||
PoolingChatLikeRequest: TypeAlias = (
|
||||
EmbeddingChatRequest
|
||||
| EmbeddingChatInputRequest
|
||||
| ClassificationChatRequest
|
||||
| PoolingChatRequest
|
||||
)
|
||||
|
||||
AnyPoolingRequest: TypeAlias = (
|
||||
EmbeddingRequest
|
||||
| PoolingCompletionLikeRequest
|
||||
| PoolingChatLikeRequest
|
||||
| IOProcessorRequest
|
||||
| ScoringRequest
|
||||
| CohereEmbedRequest
|
||||
)
|
||||
|
||||
AnyPoolingResponse: TypeAlias = (
|
||||
ClassificationResponse
|
||||
| EmbeddingResponse
|
||||
| EmbeddingBytesResponse
|
||||
| PoolingResponse
|
||||
| PoolingBytesResponse
|
||||
| ScoringResponse
|
||||
)
|
||||
|
||||
PoolingRequestT = TypeVar("PoolingRequestT", bound=AnyPoolingRequest)
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class ChunkedEmbeddingMetadata:
|
||||
prompt_index: int
|
||||
chunk_index: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class LateChunkingSlot:
|
||||
"""Per-document source ranges shared by frontend rendering and output contexts."""
|
||||
|
||||
metadata: LateChunkingMetadata | None = None
|
||||
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class PoolingServeContext(Generic[PoolingRequestT]):
|
||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||
|
||||
request: PoolingRequestT
|
||||
raw_request: Request | None = None
|
||||
model_name: str
|
||||
request_id: str
|
||||
pooling_params: PoolingParams
|
||||
lora_request: LoRARequest | None
|
||||
priorities: int | Sequence[int] | None
|
||||
prompt_extras: dict[str, Any] | None
|
||||
|
||||
created_time: int = field(default_factory=lambda: int(time.time()))
|
||||
engine_inputs: Sequence["PoolingEngineInput"] | None = None
|
||||
prompt_request_ids: list[str] | None = None
|
||||
|
||||
result_generator: AsyncGenerator[tuple[int, PoolingRequestOutput], None] | None = (
|
||||
None
|
||||
)
|
||||
final_res_batch: list[PoolingRequestOutput] = field(default_factory=list)
|
||||
|
||||
## for Long Text Embedding with Chunked Processing
|
||||
original_engine_inputs: Sequence["PoolingEngineInput"] | None = None
|
||||
chunked_embedding_metadata: list[ChunkedEmbeddingMetadata] | None = None
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
n_queries: int | None = None
|
||||
|
||||
## for IOProcessorResponse
|
||||
response: Any | None = None
|
||||
|
||||
## for flash-late-interaction
|
||||
query_final_res_batch: list[PoolingRequestOutput] | None = None
|
||||
late_interaction_query_keys: list[str] | None = None
|
||||
late_interaction_doc_keys: list[str] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflineInputsContext:
|
||||
pooling_task: PoolingTask
|
||||
tokenization_kwargs: dict[str, Any] | None
|
||||
lora_request: Sequence[LoRARequest | None] | None
|
||||
priorities: int | Sequence[int] | None
|
||||
late_chunking: list[LateChunkingSlot | None] = field(
|
||||
default_factory=list, kw_only=True
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflineEncodeInputsContext(OfflineInputsContext):
|
||||
prompts: PromptType | Sequence[PromptType]
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflineScoringInputsContext(OfflineInputsContext):
|
||||
scoring_data: ScoringData
|
||||
chat_template: str | None
|
||||
pooling_params: PoolingParams
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflinePluginInputsContext(OfflineInputsContext):
|
||||
prompts: DataPrompt
|
||||
pooling_params: PoolingParams | Sequence[PoolingParams] | None
|
||||
|
||||
|
||||
AnyOfflineInputsContext: TypeAlias = (
|
||||
OfflineEncodeInputsContext
|
||||
| OfflineScoringInputsContext
|
||||
| OfflinePluginInputsContext
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OfflineOutputsContext:
|
||||
outputs: list[PoolingRequestOutput]
|
||||
|
||||
## for bi-encoder & late-interaction
|
||||
n_queries: int | None = None
|
||||
late_chunking: list[LateChunkingSlot | None] = field(default_factory=list)
|
||||
|
||||
|
||||
class RenderParams(TypedDict):
|
||||
tok_params: TokenizeParams
|
||||
prompt_extras: dict[str, Any] | None
|
||||
skip_mm_cache: bool
|
||||
|
||||
params: PoolingParams
|
||||
lora_requests: LoRARequest | None
|
||||
priorities: int
|
||||
late_chunking: NotRequired[LateChunkingSlot]
|
||||
|
||||
|
||||
class EncodeCMPLRenderParams(RenderParams):
|
||||
prompts: DictPrompt
|
||||
|
||||
|
||||
class EncodeChatRenderParams(RenderParams):
|
||||
conversations: list["ChatCompletionMessageParam"]
|
||||
chat_params: ChatParams
|
||||
|
||||
|
||||
class ScoringRenderParams(RenderParams):
|
||||
data_1: ScoreData
|
||||
data_2: ScoreData
|
||||
chat_template: str | None
|
||||
max_tokens_per_query: int
|
||||
max_tokens_per_doc: int
|
||||
|
||||
|
||||
AnyRenderParam: TypeAlias = (
|
||||
EncodeCMPLRenderParams | EncodeChatRenderParams | ScoringRenderParams
|
||||
)
|
||||
RequestGenerator: TypeAlias = Generator[AnyRenderParam]
|
||||
RequestFactory: TypeAlias = Callable[[], RequestGenerator]
|
||||
|
||||
|
||||
class PoolingEngineInput(TypedDict):
|
||||
prompts: EngineInput
|
||||
params: PoolingParams
|
||||
lora_requests: LoRARequest | None
|
||||
priorities: int
|
||||
@@ -1,250 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache, partial
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
import pybase64
|
||||
import torch
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.serve.engine.protocol import UsageInfo
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import PoolingRequestOutput
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.utils.serial_utils import (
|
||||
EMBED_DTYPES,
|
||||
EmbedDType,
|
||||
Endianness,
|
||||
binary2tensor,
|
||||
tensor2binary,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
JsonEncodingFormat = Literal["float", "base64"]
|
||||
BytesEncodingFormat = Literal["bytes", "bytes_only"]
|
||||
FloatEncodedPoolingOutput = list[float] | list[list[float]]
|
||||
JsonEncodedPoolingOutput = FloatEncodedPoolingOutput | str
|
||||
|
||||
|
||||
@dataclass
|
||||
class MetadataItem:
|
||||
index: int
|
||||
embed_dtype: EmbedDType
|
||||
endianness: Endianness
|
||||
start: int
|
||||
end: int
|
||||
shape: tuple[int, ...]
|
||||
|
||||
|
||||
def build_metadata_items(
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
shape: tuple[int, ...],
|
||||
n_request: int,
|
||||
) -> list[MetadataItem]:
|
||||
n_bytes = EMBED_DTYPES[embed_dtype].nbytes
|
||||
size = math.prod(shape)
|
||||
|
||||
return [
|
||||
MetadataItem(
|
||||
index=i,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
start=i * size * n_bytes,
|
||||
end=(i + 1) * size * n_bytes,
|
||||
shape=shape,
|
||||
)
|
||||
for i in range(n_request)
|
||||
]
|
||||
|
||||
|
||||
def encode_pooling_output_float(
|
||||
output: PoolingRequestOutput,
|
||||
) -> FloatEncodedPoolingOutput:
|
||||
return output.outputs.data.tolist()
|
||||
|
||||
|
||||
def encode_pooling_output_float_or_ndarray(output: PoolingRequestOutput) -> Any:
|
||||
"""Return an ndarray when the response renderer can serialize NumPy."""
|
||||
try:
|
||||
data = output.outputs.data
|
||||
if not data.is_contiguous():
|
||||
data = data.contiguous()
|
||||
return data.numpy()
|
||||
except (RuntimeError, TypeError):
|
||||
return output.outputs.data.tolist()
|
||||
|
||||
|
||||
def encode_pooling_output_base64(
|
||||
output: PoolingRequestOutput,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> str:
|
||||
embedding_bytes = tensor2binary(output.outputs.data, embed_dtype, endianness)
|
||||
return pybase64.b64encode(embedding_bytes).decode("utf-8")
|
||||
|
||||
|
||||
def encode_pooling_bytes(
|
||||
pooling_outputs: list[PoolingRequestOutput],
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> tuple[list[bytes], list[dict[str, Any]]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
body: list[bytes] = []
|
||||
offset = 0
|
||||
for idx, output in enumerate(pooling_outputs):
|
||||
binary = tensor2binary(
|
||||
tensor=output.outputs.data,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
size = len(binary)
|
||||
|
||||
# Dictionary form of MetadataItem
|
||||
item = dict(
|
||||
index=idx,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
start=offset,
|
||||
end=offset + size,
|
||||
shape=output.outputs.data.shape,
|
||||
)
|
||||
|
||||
body.append(binary)
|
||||
items.append(item)
|
||||
offset += size
|
||||
|
||||
return body, items
|
||||
|
||||
|
||||
def get_pooling_output_encoder(
|
||||
encoding_format: JsonEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> Callable[[PoolingRequestOutput], JsonEncodedPoolingOutput]:
|
||||
return cast(
|
||||
Callable[[PoolingRequestOutput], JsonEncodedPoolingOutput],
|
||||
(
|
||||
encode_pooling_output_float
|
||||
if encoding_format == "float"
|
||||
else partial(
|
||||
encode_pooling_output_base64,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_pooling_usage(
|
||||
pooling_outputs: Sequence[PoolingRequestOutput],
|
||||
) -> UsageInfo:
|
||||
num_prompt_tokens = sum(
|
||||
len(output.prompt_token_ids) if output.prompt_token_ids is not None else 0
|
||||
for output in pooling_outputs
|
||||
)
|
||||
return UsageInfo(
|
||||
prompt_tokens=num_prompt_tokens,
|
||||
total_tokens=num_prompt_tokens,
|
||||
)
|
||||
|
||||
|
||||
def get_pooling_usage_payload(
|
||||
pooling_outputs: Sequence[PoolingRequestOutput],
|
||||
) -> dict[str, int]:
|
||||
usage = get_pooling_usage(pooling_outputs)
|
||||
return {
|
||||
"prompt_tokens": usage.prompt_tokens,
|
||||
"total_tokens": usage.total_tokens,
|
||||
}
|
||||
|
||||
|
||||
def build_pooling_bytes_streaming_response(
|
||||
pooling_outputs: list[PoolingRequestOutput],
|
||||
request_id: str,
|
||||
created_time: int,
|
||||
model_name: str,
|
||||
encoding_format: BytesEncodingFormat,
|
||||
embed_dtype: EmbedDType,
|
||||
endianness: Endianness,
|
||||
) -> StreamingResponse:
|
||||
content, items = encode_pooling_bytes(
|
||||
pooling_outputs=pooling_outputs,
|
||||
embed_dtype=embed_dtype,
|
||||
endianness=endianness,
|
||||
)
|
||||
|
||||
headers = (
|
||||
None
|
||||
if encoding_format == "bytes_only"
|
||||
else {
|
||||
"metadata": json.dumps(
|
||||
{
|
||||
"id": request_id,
|
||||
"created": created_time,
|
||||
"model": model_name,
|
||||
"data": items,
|
||||
"usage": get_pooling_usage_payload(pooling_outputs),
|
||||
}
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
return StreamingResponse(
|
||||
content=content,
|
||||
headers=headers,
|
||||
media_type="application/octet-stream",
|
||||
)
|
||||
|
||||
|
||||
def decode_pooling_output(items: list[MetadataItem], body: bytes) -> list[torch.Tensor]:
|
||||
return [
|
||||
binary2tensor(
|
||||
body[item.start : item.end],
|
||||
item.shape,
|
||||
item.embed_dtype,
|
||||
item.endianness,
|
||||
)
|
||||
for item in sorted(items, key=lambda x: x.index)
|
||||
]
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_json_response_cls() -> type[JSONResponse]:
|
||||
if importlib.util.find_spec("orjson") is not None:
|
||||
from fastapi.responses import ORJSONResponse
|
||||
|
||||
return ORJSONResponse
|
||||
logger.warning_once(
|
||||
"To make v1/embeddings API fast, please install orjson by `pip install orjson`"
|
||||
)
|
||||
return JSONResponse
|
||||
|
||||
|
||||
def enable_scoring_api(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
) -> bool:
|
||||
if model_config is None:
|
||||
return False
|
||||
|
||||
pooling_task = model_config.get_pooling_task(supported_tasks)
|
||||
if pooling_task in ("embed", "token_embed"):
|
||||
return True
|
||||
|
||||
if pooling_task == "classify":
|
||||
num_labels = getattr(model_config.hf_config, "num_labels", 0)
|
||||
if num_labels != 1:
|
||||
logger.debug_once("Scoring API is only enabled for num_labels == 1.")
|
||||
return False
|
||||
return True
|
||||
|
||||
return False
|
||||
@@ -11,7 +11,6 @@ from vllm.entrypoints.openai.models.serving import (
|
||||
OpenAIModelRegistry,
|
||||
OpenAIServingModels,
|
||||
)
|
||||
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
|
||||
from vllm.entrypoints.serve import create_error_response
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
@@ -48,7 +47,7 @@ class BaseServing:
|
||||
|
||||
async def _check_model(
|
||||
self,
|
||||
request: AnyRequest | AnyPoolingRequest,
|
||||
request: AnyRequest,
|
||||
) -> ErrorResponse | None:
|
||||
if self.request_logger is not None:
|
||||
self.request_logger.log_request_body(request)
|
||||
@@ -141,7 +140,7 @@ class BaseServing:
|
||||
|
||||
return random_uuid() if default is None else default
|
||||
|
||||
def _get_message_types(self, request: AnyRequest | AnyPoolingRequest) -> set[str]:
|
||||
def _get_message_types(self, request: AnyRequest) -> set[str]:
|
||||
"""Retrieve the set of types from message content dicts up
|
||||
until `_`; we use this to match potential multimodal data
|
||||
with default per modality loras.
|
||||
@@ -166,9 +165,7 @@ class BaseServing:
|
||||
message_types.add(content_dict["type"].split("_")[0])
|
||||
return message_types
|
||||
|
||||
def _get_active_default_mm_loras(
|
||||
self, request: AnyRequest | AnyPoolingRequest
|
||||
) -> LoRARequest | None:
|
||||
def _get_active_default_mm_loras(self, request: AnyRequest) -> LoRARequest | None:
|
||||
"""Determine if there are any active default multimodal loras."""
|
||||
# TODO: Currently this is only enabled for chat completions
|
||||
# to be better aligned with only being enabled for .generate
|
||||
@@ -193,7 +190,7 @@ class BaseServing:
|
||||
|
||||
def _maybe_get_adapters(
|
||||
self,
|
||||
request: AnyRequest | AnyPoolingRequest,
|
||||
request: AnyRequest,
|
||||
supports_default_mm_loras: bool = False,
|
||||
) -> LoRARequest | None:
|
||||
if request.model in self.models.lora_requests:
|
||||
|
||||
@@ -12,7 +12,6 @@ from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.factories import get_generate_invocation_types
|
||||
from vllm.entrypoints.pooling.factories import get_pooling_invocation_types
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.instrumentator.basic import base
|
||||
@@ -79,9 +78,7 @@ def _attach_router(
|
||||
router = APIRouter()
|
||||
|
||||
# NOTE: Construct the TypeAdapters only once
|
||||
INVOCATION_TYPES = get_generate_invocation_types(
|
||||
supported_tasks, model_config
|
||||
) + get_pooling_invocation_types(supported_tasks, model_config)
|
||||
INVOCATION_TYPES = get_generate_invocation_types(supported_tasks, model_config)
|
||||
|
||||
INVOCATION_VALIDATORS = [
|
||||
(pydantic.TypeAdapter(request_type), (get_handler, endpoint))
|
||||
|
||||
@@ -7,7 +7,6 @@ from collections.abc import Sequence
|
||||
import torch
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.pooling.typing import AnyPoolingRequest
|
||||
from vllm.entrypoints.serve.engine.typing import AnyRequest
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -70,7 +69,7 @@ class RequestLogger:
|
||||
lora_request,
|
||||
)
|
||||
|
||||
def log_request_body(self, request: AnyRequest | AnyPoolingRequest) -> None:
|
||||
def log_request_body(self, request: AnyRequest) -> None:
|
||||
try:
|
||||
level = logging.getLevelNamesMapping()[envs.VLLM_REQUEST_BODY_LOG_LEVEL]
|
||||
except KeyError:
|
||||
|
||||
Reference in New Issue
Block a user