[Perf] Integrate flash-maxsim Triton kernels for late-interaction scoring (#40337)
Signed-off-by: roi.pony <roi.pony@ibm.com> Co-authored-by: roi.pony <roi.pony@ibm.com> Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
co-authored by
roi.pony
Taneem Ibrahim
parent
083060d046
commit
e9be5323f0
@@ -51,6 +51,9 @@ async def build_async_engine_client(
|
||||
# Context manager to handle engine_client lifecycle
|
||||
# Ensures everything is shutdown and cleaned up on error/exit
|
||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||
from ..cli_args import propagate_flash_late_interaction
|
||||
|
||||
propagate_flash_late_interaction(args, engine_args)
|
||||
if client_config:
|
||||
engine_args._api_process_count = client_config.get("client_count", 1)
|
||||
engine_args._api_process_rank = client_config.get("client_index", 0)
|
||||
|
||||
Reference in New Issue
Block a user