[Perf] Integrate flash-maxsim Triton kernels for late-interaction scoring (#40337)
Signed-off-by: roi.pony <roi.pony@ibm.com> Co-authored-by: roi.pony <roi.pony@ibm.com> Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
co-authored by
roi.pony
Taneem Ibrahim
parent
083060d046
commit
e9be5323f0
@@ -13,6 +13,7 @@ from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.launchers.api_server.entry import run_server, setup_server
|
||||
from vllm.entrypoints.launchers.cli_args import (
|
||||
make_arg_parser,
|
||||
propagate_flash_late_interaction,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor
|
||||
@@ -185,6 +186,7 @@ def run_headless(args: argparse.Namespace):
|
||||
|
||||
# Create the EngineConfig.
|
||||
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
|
||||
propagate_flash_late_interaction(args, engine_args)
|
||||
usage_context = UsageContext.OPENAI_API_SERVER
|
||||
vllm_config = engine_args.create_engine_config(
|
||||
usage_context=usage_context, headless=True
|
||||
@@ -295,6 +297,7 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1)
|
||||
|
||||
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
|
||||
propagate_flash_late_interaction(args, engine_args)
|
||||
engine_args._api_process_count = num_api_servers
|
||||
engine_args._api_process_rank = -1
|
||||
|
||||
|
||||
@@ -51,6 +51,9 @@ async def build_async_engine_client(
|
||||
# Context manager to handle engine_client lifecycle
|
||||
# Ensures everything is shutdown and cleaned up on error/exit
|
||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||
from ..cli_args import propagate_flash_late_interaction
|
||||
|
||||
propagate_flash_late_interaction(args, engine_args)
|
||||
if client_config:
|
||||
engine_args._api_process_count = client_config.get("client_count", 1)
|
||||
engine_args._api_process_rank = client_config.get("client_index", 0)
|
||||
|
||||
@@ -363,7 +363,10 @@ class FrontendArgs(BaseFrontendArgs):
|
||||
"""
|
||||
enable_flash_late_interaction: bool = True
|
||||
"""If set, run pooling score MaxSim on GPU in the API server process.
|
||||
Can significantly improve late-interaction scoring performance."""
|
||||
Can significantly improve late-interaction scoring performance.
|
||||
When disabled, the setting is also propagated to the engine's
|
||||
`PoolerConfig`, so the engine-side scorer uses the reference MaxSim
|
||||
path instead of the fused Triton kernel."""
|
||||
|
||||
@classmethod
|
||||
def _customize_cli_kwargs(
|
||||
@@ -498,3 +501,20 @@ def create_parser_for_docs() -> FlexibleArgumentParser:
|
||||
prog="-m vllm.entrypoints.launchers.api_server.entry"
|
||||
)
|
||||
return make_arg_parser(parser_for_docs)
|
||||
|
||||
|
||||
def propagate_flash_late_interaction(args, engine_args) -> None:
|
||||
"""Propagate `--no-enable-flash-late-interaction` into the engine config.
|
||||
|
||||
The frontend flag alone only disables the API-server scoring path;
|
||||
mirroring it into `PoolerConfig.enable_flash_late_interaction` lets the
|
||||
engine-side scorer fall back to the reference MaxSim path too.
|
||||
"""
|
||||
if getattr(args, "enable_flash_late_interaction", True):
|
||||
return
|
||||
from vllm.config.pooler import PoolerConfig
|
||||
|
||||
if engine_args.pooler_config is None:
|
||||
engine_args.pooler_config = PoolerConfig(enable_flash_late_interaction=False)
|
||||
else:
|
||||
engine_args.pooler_config.enable_flash_late_interaction = False
|
||||
|
||||
Reference in New Issue
Block a user