[Perf] Integrate flash-maxsim Triton kernels for late-interaction scoring (#40337)
Signed-off-by: roi.pony <roi.pony@ibm.com> Co-authored-by: roi.pony <roi.pony@ibm.com> Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
co-authored by
roi.pony
Taneem Ibrahim
parent
083060d046
commit
e9be5323f0
@@ -10,6 +10,7 @@ from vllm.v1.pool.late_interaction import (
|
||||
LATE_INTERACTION_MODE_CACHE_QUERY,
|
||||
build_late_interaction_doc_params,
|
||||
build_late_interaction_query_params,
|
||||
compute_maxsim_score_batched,
|
||||
)
|
||||
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
|
||||
|
||||
@@ -152,3 +153,107 @@ def test_invalid_query_uses_raises():
|
||||
req_ids=["query-req"],
|
||||
finished_mask=[True],
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fused Triton scoring path (PR #40337).
|
||||
# ---------------------------------------------------------------------------
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
|
||||
def test_score_flash_matches_fp64_oracle():
|
||||
"""Shared-query scoring through the fused kernel must match a per-pair
|
||||
fp64 MaxSim oracle on ragged doc lengths, including length-1 docs."""
|
||||
torch.manual_seed(0)
|
||||
d = 128
|
||||
q = torch.randn(32, d, device="cuda", dtype=torch.float16)
|
||||
doc_lengths = [180, 37, 512, 1, 300, 64, 1030, 256]
|
||||
docs = [
|
||||
torch.randn(ld, d, device="cuda", dtype=torch.float16) for ld in doc_lengths
|
||||
]
|
||||
|
||||
runner = LateInteractionRunner()
|
||||
assert runner._flash_enabled
|
||||
scores = runner._score([q] * len(docs), docs)
|
||||
assert runner._flash_enabled, "kernel path must not have fallen back"
|
||||
|
||||
for i, doc in enumerate(docs):
|
||||
ref = (q.double() @ doc.double().T).max(dim=1).values.sum().float()
|
||||
torch.testing.assert_close(
|
||||
scores[i].to(torch.float32), ref, atol=5e-2, rtol=1e-3
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
|
||||
def test_score_multi_query_uses_reference_path():
|
||||
"""Distinct queries per pair route to the reference scorer and agree
|
||||
with it exactly (no kernel dispatch for the N:N pattern)."""
|
||||
torch.manual_seed(1)
|
||||
d = 128
|
||||
queries = [torch.randn(16, d, device="cuda") for _ in range(3)]
|
||||
docs = [torch.randn(ld, d, device="cuda") for ld in (50, 3, 77)]
|
||||
|
||||
runner = LateInteractionRunner()
|
||||
scores = runner._score(queries, docs)
|
||||
ref = compute_maxsim_score_batched(queries, docs)
|
||||
for s, r in zip(scores, ref):
|
||||
torch.testing.assert_close(s, r)
|
||||
|
||||
|
||||
def test_score_flash_failure_disables_and_falls_back(monkeypatch):
|
||||
"""A kernel failure must serve the batch via the reference scorer and
|
||||
disable the kernel path for the rest of the process."""
|
||||
runner = LateInteractionRunner()
|
||||
runner._flash_enabled = True
|
||||
|
||||
def _boom(query, docs):
|
||||
raise RuntimeError("synthetic kernel failure")
|
||||
|
||||
monkeypatch.setattr(runner, "_score_flash_shared_query", _boom)
|
||||
q = torch.randn(8, 32)
|
||||
docs = [torch.randn(5, 32), torch.randn(7, 32)]
|
||||
# CPU tensors already skip the kernel; force the dispatch condition.
|
||||
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
|
||||
scores = runner._score([q, q], docs)
|
||||
assert not runner._flash_enabled
|
||||
ref = compute_maxsim_score_batched([q, q], docs)
|
||||
for s, r in zip(scores, ref):
|
||||
torch.testing.assert_close(s, r)
|
||||
|
||||
|
||||
def test_disabled_flag_uses_reference_path():
|
||||
runner = LateInteractionRunner(enable_flash=False)
|
||||
assert not runner._flash_enabled
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Autotune config pruning.
|
||||
# ---------------------------------------------------------------------------
|
||||
def _prune(monkeypatch, budget: int, named_args: dict):
|
||||
from vllm.v1.pool.flash_maxsim import _common
|
||||
|
||||
monkeypatch.setattr(_common, "_smem_budget", lambda: budget)
|
||||
return _common._prune_configs(_common._get_configs(), named_args)
|
||||
|
||||
|
||||
def _est(cfg, d_pad: int) -> int:
|
||||
bq, bd = cfg.kwargs["BLOCK_Q"], cfg.kwargs["BLOCK_D"]
|
||||
return (bq * d_pad + bd * d_pad) * 2 + bq * bd * 4
|
||||
|
||||
|
||||
def test_prune_configs_uses_padded_dim(monkeypatch):
|
||||
"""d=513 pads to 1024: survivors must fit the budget at d_pad, not at
|
||||
the un-padded d (which would admit ~2x-oversized tiles)."""
|
||||
budget = 166_912 # A100 opt-in
|
||||
survivors = _prune(monkeypatch, budget, {"Lq": 1024, "d": 513, "d_pad": 1024})
|
||||
assert survivors
|
||||
assert all(_est(c, 1024) <= budget for c in survivors)
|
||||
|
||||
|
||||
def test_prune_configs_no_rejected_fallback(monkeypatch):
|
||||
"""When nothing fits, the fallback is the single smallest-footprint
|
||||
config — never a slice of rejected configs."""
|
||||
from vllm.v1.pool.flash_maxsim import _common
|
||||
|
||||
survivors = _prune(monkeypatch, 10_000, {"Lq": 1024, "d_pad": 1024})
|
||||
assert len(survivors) == 1
|
||||
smallest = min(_common._get_configs(), key=lambda c: _est(c, 1024))
|
||||
assert survivors[0].kwargs == smallest.kwargs
|
||||
|
||||
@@ -63,6 +63,14 @@ class PoolerConfig:
|
||||
`None` uses the pooler's default, which is `True` in most cases.
|
||||
"""
|
||||
|
||||
enable_flash_late_interaction: bool = True
|
||||
"""
|
||||
Whether the engine-side late-interaction scorer may use the fused
|
||||
flash-maxsim Triton kernel. Disabled automatically when the API server
|
||||
is started with `--no-enable-flash-late-interaction`; the reference
|
||||
scorer is used instead.
|
||||
"""
|
||||
|
||||
## for embedding models
|
||||
dimensions: int | None = None
|
||||
"""
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.entrypoints.cli.types import CLISubcommand
|
||||
from vllm.entrypoints.launchers.api_server.entry import run_server, setup_server
|
||||
from vllm.entrypoints.launchers.cli_args import (
|
||||
make_arg_parser,
|
||||
propagate_flash_late_interaction,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor
|
||||
@@ -185,6 +186,7 @@ def run_headless(args: argparse.Namespace):
|
||||
|
||||
# Create the EngineConfig.
|
||||
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
|
||||
propagate_flash_late_interaction(args, engine_args)
|
||||
usage_context = UsageContext.OPENAI_API_SERVER
|
||||
vllm_config = engine_args.create_engine_config(
|
||||
usage_context=usage_context, headless=True
|
||||
@@ -295,6 +297,7 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1)
|
||||
|
||||
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
|
||||
propagate_flash_late_interaction(args, engine_args)
|
||||
engine_args._api_process_count = num_api_servers
|
||||
engine_args._api_process_rank = -1
|
||||
|
||||
|
||||
@@ -51,6 +51,9 @@ async def build_async_engine_client(
|
||||
# Context manager to handle engine_client lifecycle
|
||||
# Ensures everything is shutdown and cleaned up on error/exit
|
||||
engine_args = AsyncEngineArgs.from_cli_args(args)
|
||||
from ..cli_args import propagate_flash_late_interaction
|
||||
|
||||
propagate_flash_late_interaction(args, engine_args)
|
||||
if client_config:
|
||||
engine_args._api_process_count = client_config.get("client_count", 1)
|
||||
engine_args._api_process_rank = client_config.get("client_index", 0)
|
||||
|
||||
@@ -363,7 +363,10 @@ class FrontendArgs(BaseFrontendArgs):
|
||||
"""
|
||||
enable_flash_late_interaction: bool = True
|
||||
"""If set, run pooling score MaxSim on GPU in the API server process.
|
||||
Can significantly improve late-interaction scoring performance."""
|
||||
Can significantly improve late-interaction scoring performance.
|
||||
When disabled, the setting is also propagated to the engine's
|
||||
`PoolerConfig`, so the engine-side scorer uses the reference MaxSim
|
||||
path instead of the fused Triton kernel."""
|
||||
|
||||
@classmethod
|
||||
def _customize_cli_kwargs(
|
||||
@@ -498,3 +501,20 @@ def create_parser_for_docs() -> FlexibleArgumentParser:
|
||||
prog="-m vllm.entrypoints.launchers.api_server.entry"
|
||||
)
|
||||
return make_arg_parser(parser_for_docs)
|
||||
|
||||
|
||||
def propagate_flash_late_interaction(args, engine_args) -> None:
|
||||
"""Propagate `--no-enable-flash-late-interaction` into the engine config.
|
||||
|
||||
The frontend flag alone only disables the API-server scoring path;
|
||||
mirroring it into `PoolerConfig.enable_flash_late_interaction` lets the
|
||||
engine-side scorer fall back to the reference MaxSim path too.
|
||||
"""
|
||||
if getattr(args, "enable_flash_late_interaction", True):
|
||||
return
|
||||
from vllm.config.pooler import PoolerConfig
|
||||
|
||||
if engine_args.pooler_config is None:
|
||||
engine_args.pooler_config = PoolerConfig(enable_flash_late_interaction=False)
|
||||
else:
|
||||
engine_args.pooler_config.enable_flash_late_interaction = False
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused Triton kernels for late-interaction (MaxSim) scoring."""
|
||||
|
||||
from .flash_maxsim_rerank import flash_maxsim_rerank_direct
|
||||
|
||||
__all__ = ["flash_maxsim_rerank_direct"]
|
||||
@@ -0,0 +1,85 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared utilities for flash-maxsim kernels: padding, autotune configs,
|
||||
and pruning."""
|
||||
|
||||
from vllm.triton_utils import triton
|
||||
|
||||
|
||||
def _next_pow2(x: int) -> int:
|
||||
return 1 << (x - 1).bit_length()
|
||||
|
||||
|
||||
def _get_configs() -> list:
|
||||
"""Autotune grid over (BLOCK_Q, BLOCK_D); `_prune_configs` filters by
|
||||
SMEM budget and Lq at runtime, so one list is safe across GPUs."""
|
||||
return [
|
||||
triton.Config({"BLOCK_Q": 16, "BLOCK_D": 16}, num_warps=2, num_stages=2),
|
||||
triton.Config({"BLOCK_Q": 16, "BLOCK_D": 32}, num_warps=2, num_stages=2),
|
||||
triton.Config({"BLOCK_Q": 32, "BLOCK_D": 16}, num_warps=2, num_stages=2),
|
||||
triton.Config({"BLOCK_Q": 32, "BLOCK_D": 32}, num_warps=4, num_stages=2),
|
||||
triton.Config({"BLOCK_Q": 32, "BLOCK_D": 64}, num_warps=4, num_stages=2),
|
||||
triton.Config({"BLOCK_Q": 32, "BLOCK_D": 128}, num_warps=8, num_stages=2),
|
||||
triton.Config({"BLOCK_Q": 64, "BLOCK_D": 64}, num_warps=4, num_stages=2),
|
||||
triton.Config({"BLOCK_Q": 64, "BLOCK_D": 128}, num_warps=8, num_stages=1),
|
||||
triton.Config({"BLOCK_Q": 128, "BLOCK_D": 64}, num_warps=8, num_stages=1),
|
||||
triton.Config({"BLOCK_Q": 128, "BLOCK_D": 128}, num_warps=8, num_stages=1),
|
||||
]
|
||||
|
||||
|
||||
def _smem_budget() -> int:
|
||||
"""Per-block opt-in shared-memory budget of the active device (bytes):
|
||||
Triton driver query, else a conservative capability table."""
|
||||
try:
|
||||
driver = triton.runtime.driver.active
|
||||
props = driver.utils.get_device_properties(driver.get_current_device())
|
||||
return int(props["max_shared_mem"])
|
||||
except Exception:
|
||||
try:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
cap = current_platform.get_device_capability()
|
||||
if cap is not None:
|
||||
if cap.major >= 9:
|
||||
return 232_448 # Hopper+: 227 KB opt-in
|
||||
if cap.major == 8:
|
||||
return 101_376 # worst sm_8x (86/89); A100 has more
|
||||
except Exception:
|
||||
pass
|
||||
return 98_304 # conservative pre-Ampere default
|
||||
|
||||
|
||||
def _prune_configs(configs, named_args, **kwargs):
|
||||
"""Reject configs that exceed the GPU's shared-memory budget.
|
||||
|
||||
BLOCK_Q and BLOCK_D load fp16 tiles of size [BLOCK_*, d_pad] — the
|
||||
kernels index with ``tl.arange(0, d_pad)``, so the estimate must use
|
||||
the padded dimension, not the real one (d=513 pads to 1024 and
|
||||
nearly doubles the tile) — and the [BLOCK_Q, BLOCK_D] score tile is
|
||||
fp32. Total:
|
||||
(BLOCK_Q*d_pad + BLOCK_D*d_pad)*2 + BLOCK_Q*BLOCK_D*4 bytes.
|
||||
"""
|
||||
Lq = named_args.get("Lq", 32)
|
||||
d_pad = named_args.get("d_pad")
|
||||
if d_pad is None:
|
||||
d_pad = _next_pow2(max(named_args.get("d", 128), 16))
|
||||
smem_limit = _smem_budget()
|
||||
|
||||
def _est(cfg):
|
||||
bq = cfg.kwargs["BLOCK_Q"]
|
||||
bd = cfg.kwargs["BLOCK_D"]
|
||||
return (bq * d_pad + bd * d_pad) * 2 + bq * bd * 4
|
||||
|
||||
pruned = []
|
||||
for cfg in configs:
|
||||
if cfg.kwargs["BLOCK_Q"] > Lq * 2:
|
||||
continue
|
||||
if _est(cfg) > smem_limit:
|
||||
continue
|
||||
pruned.append(cfg)
|
||||
if pruned:
|
||||
return pruned
|
||||
# Nothing fits the estimate (large d_pad on a small-SMEM device).
|
||||
# Return the single smallest-footprint config — a deterministic
|
||||
# least-bad launch attempt — never arbitrary rejected configs.
|
||||
return [min(configs, key=_est)]
|
||||
@@ -0,0 +1,204 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Flash-MaxSim rerank: one query vs variable-length docs in a packed tensor.
|
||||
|
||||
The kernel reads doc embeddings directly from a packed [total_tokens, d]
|
||||
tensor using per-doc (offset, length) pairs — no padding, no per-doc
|
||||
tensors, no materialised [B, Lq, Ld] similarity tensor.
|
||||
|
||||
Entry point:
|
||||
flash_maxsim_rerank_direct(Q, batch_tensor, doc_offsets, doc_lengths, max_ld)
|
||||
doc_offsets[i] = start token index of doc i in batch_tensor.
|
||||
doc_lengths[i] = number of tokens in doc i.
|
||||
Docs can be scattered (non-contiguous) in the batch tensor.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from ._common import _get_configs, _next_pow2, _prune_configs
|
||||
|
||||
|
||||
def _round_to_bucket(x):
|
||||
"""Round up to nearest bucket: 32, 64, 128, 256, 512, 1024, 2048, 4096."""
|
||||
for b in [32, 64, 128, 256, 512, 1024, 2048, 4096]:
|
||||
if x <= b:
|
||||
return b
|
||||
return _next_pow2(x)
|
||||
|
||||
|
||||
# Kernel: shared Q, scattered D via (offset, length) per doc
|
||||
|
||||
|
||||
@triton.autotune(
|
||||
configs=_get_configs(),
|
||||
key=["Lq_bucket", "max_Ld_bucket", "d_pad"],
|
||||
prune_configs_by={"early_config_prune": _prune_configs},
|
||||
)
|
||||
@triton.jit
|
||||
def _maxsim_rerank_kernel(
|
||||
Q_ptr,
|
||||
D_ptr,
|
||||
doc_offsets_ptr,
|
||||
doc_lengths_ptr,
|
||||
scores_ptr,
|
||||
B,
|
||||
Lq,
|
||||
Lq_bucket: tl.constexpr,
|
||||
max_Ld_bucket: tl.constexpr,
|
||||
d: tl.constexpr,
|
||||
d_pad: tl.constexpr,
|
||||
stride_q_t,
|
||||
stride_q_d,
|
||||
stride_d_t,
|
||||
stride_d_d,
|
||||
BLOCK_Q: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
):
|
||||
"""One program per doc. Q shared (read from same location). D scattered."""
|
||||
pid = tl.program_id(0)
|
||||
if pid >= B:
|
||||
return
|
||||
|
||||
# Each doc has its own offset and length in the batch tensor.
|
||||
# int64: (d_start + d_off) * stride overflows int32 past ~16.7M batch
|
||||
# tokens (same widen-on-load as the pairs kernel below; the host-side
|
||||
# offsets tensor stays int32, which indexes any realistic batch).
|
||||
d_start = tl.load(doc_offsets_ptr + pid).to(tl.int64)
|
||||
d_len = tl.load(doc_lengths_ptr + pid).to(tl.int32)
|
||||
|
||||
k_off = tl.arange(0, d_pad)
|
||||
k_mask = k_off < d
|
||||
score_acc = tl.zeros([], dtype=tl.float32)
|
||||
|
||||
# Q is shared — always read from start of Q_ptr
|
||||
for q_blk in tl.static_range(0, Lq_bucket, BLOCK_Q):
|
||||
q_off = q_blk + tl.arange(0, BLOCK_Q)
|
||||
q_valid = q_off < Lq
|
||||
|
||||
Q_block = tl.load(
|
||||
Q_ptr + q_off[:, None] * stride_q_t + k_off[None, :] * stride_q_d,
|
||||
mask=q_valid[:, None] & k_mask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float16)
|
||||
|
||||
m = tl.full([BLOCK_Q], float("-inf"), dtype=tl.float32)
|
||||
|
||||
for d_blk in range(0, max_Ld_bucket, BLOCK_D):
|
||||
d_off = d_blk + tl.arange(0, BLOCK_D)
|
||||
d_valid = d_off < d_len
|
||||
|
||||
D_block = tl.load(
|
||||
D_ptr
|
||||
+ (d_start + d_off[:, None]) * stride_d_t
|
||||
+ k_off[None, :] * stride_d_d,
|
||||
mask=d_valid[:, None] & k_mask[None, :],
|
||||
other=0.0,
|
||||
).to(tl.float16)
|
||||
|
||||
S = tl.dot(Q_block, tl.trans(D_block))
|
||||
S = tl.where(d_valid[None, :], S, float("-inf"))
|
||||
m = tl.maximum(m, tl.max(S, axis=1))
|
||||
|
||||
m = tl.where(q_valid, m, 0.0)
|
||||
score_acc += tl.sum(m)
|
||||
|
||||
tl.store(scores_ptr + pid, score_acc)
|
||||
|
||||
|
||||
# Internal helper
|
||||
|
||||
|
||||
def _run_rerank_kernel(Q, D_tensor, doc_offsets, doc_lengths, max_seqlen_d):
|
||||
"""Common kernel launch for both rerank and rerank_direct."""
|
||||
assert Q.is_cuda, f"Q must be on CUDA, got {Q.device}"
|
||||
assert D_tensor.is_cuda, f"D must be on CUDA, got {D_tensor.device}"
|
||||
Lq, d = Q.shape
|
||||
B = doc_offsets.shape[0]
|
||||
d_pad = _next_pow2(max(d, 16))
|
||||
|
||||
Q = Q.contiguous().half()
|
||||
# D_tensor is NOT copied — kernel reads from it directly
|
||||
if not D_tensor.is_contiguous():
|
||||
D_tensor = D_tensor.contiguous()
|
||||
if D_tensor.dtype != torch.float16:
|
||||
D_tensor = D_tensor.half()
|
||||
|
||||
doc_offsets = doc_offsets.to(torch.int32).contiguous()
|
||||
doc_lengths = doc_lengths.to(torch.int32).contiguous()
|
||||
scores = torch.empty(B, device=Q.device, dtype=torch.float32)
|
||||
|
||||
if B == 0:
|
||||
return scores
|
||||
|
||||
# Pad Q dimension if needed (Q is small, safe to copy).
|
||||
# D is NOT padded — the kernel uses k_mask to handle d < d_pad.
|
||||
# This preserves zero-copy: D_tensor is read in-place.
|
||||
if d < d_pad:
|
||||
Q_pad = torch.zeros(Lq, d_pad, device=Q.device, dtype=Q.dtype)
|
||||
Q_pad[:, :d] = Q
|
||||
else:
|
||||
Q_pad = Q
|
||||
D_pad = D_tensor
|
||||
|
||||
Lq_bucket = _round_to_bucket(Lq)
|
||||
max_ld_bucket = _round_to_bucket(max_seqlen_d)
|
||||
|
||||
_maxsim_rerank_kernel[(B,)](
|
||||
Q_pad,
|
||||
D_pad,
|
||||
doc_offsets,
|
||||
doc_lengths,
|
||||
scores,
|
||||
B,
|
||||
Lq,
|
||||
Lq_bucket,
|
||||
max_ld_bucket,
|
||||
d,
|
||||
d_pad,
|
||||
Q_pad.stride(0),
|
||||
Q_pad.stride(1),
|
||||
D_pad.stride(0),
|
||||
D_pad.stride(1),
|
||||
)
|
||||
return scores
|
||||
|
||||
|
||||
# Public API
|
||||
|
||||
|
||||
def flash_maxsim_rerank_direct(
|
||||
Q: torch.Tensor,
|
||||
batch_tensor: torch.Tensor,
|
||||
doc_offsets: torch.Tensor,
|
||||
doc_lengths: torch.Tensor,
|
||||
max_seqlen_d: int,
|
||||
) -> torch.Tensor:
|
||||
"""TRUE zero-copy: score query against docs scattered in a batch tensor.
|
||||
|
||||
The kernel reads doc embeddings directly from batch_tensor at the
|
||||
positions specified by doc_offsets. No torch.stack, no torch.cat,
|
||||
no copy of any kind. The batch tensor is the model's output.
|
||||
|
||||
Memory for doc scoring: 0 bytes additional.
|
||||
|
||||
Args:
|
||||
Q: [Lq, d] — single query embedding (from cache)
|
||||
batch_tensor: [total_tokens, d] — the model's projected output tensor.
|
||||
Contains ALL requests' tokens (queries + docs + others).
|
||||
doc_offsets: [B] int32 — start token index of each doc in batch_tensor
|
||||
doc_lengths: [B] int32 — number of tokens per doc
|
||||
max_seqlen_d: int — max(doc_lengths)
|
||||
|
||||
Returns:
|
||||
scores: [B] float32 — one MaxSim score per document
|
||||
|
||||
"""
|
||||
assert Q.dim() == 2, f"Q must be 2D [Lq, d], got {Q.dim()}D"
|
||||
assert batch_tensor.dim() == 2, (
|
||||
f"batch_tensor must be 2D, got {batch_tensor.dim()}D"
|
||||
)
|
||||
assert Q.shape[1] == batch_tensor.shape[1]
|
||||
|
||||
return _run_rerank_kernel(Q, batch_tensor, doc_offsets, doc_lengths, max_seqlen_d)
|
||||
@@ -4,6 +4,7 @@ from collections.abc import Iterable
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.v1.outputs import PoolerOutput
|
||||
from vllm.v1.pool.late_interaction import (
|
||||
@@ -12,17 +13,29 @@ from vllm.v1.pool.late_interaction import (
|
||||
compute_maxsim_score_batched,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class LateInteractionRunner:
|
||||
"""Worker-side state and postprocessing for late-interaction scoring."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, enable_flash: bool = True) -> None:
|
||||
# query_key -> token embeddings for late-interaction scoring.
|
||||
self._query_cache: dict[str, torch.Tensor] = {}
|
||||
# query_key -> remaining number of docs that should use this query.
|
||||
self._query_uses: dict[str, int] = {}
|
||||
# doc request id -> query key.
|
||||
self._doc_query_keys: dict[str, str] = {}
|
||||
# Fused Triton scoring (PoolerConfig.enable_flash_late_interaction
|
||||
# gates it; a runtime kernel failure disables it for the process).
|
||||
if enable_flash:
|
||||
try:
|
||||
from vllm.v1.pool.flash_maxsim import ( # noqa: F401
|
||||
flash_maxsim_rerank_direct,
|
||||
)
|
||||
except ImportError:
|
||||
enable_flash = False
|
||||
self._flash_enabled = enable_flash
|
||||
|
||||
def clear(self) -> None:
|
||||
self._query_cache.clear()
|
||||
@@ -116,7 +129,7 @@ class LateInteractionRunner:
|
||||
raise ValueError(f"Unsupported late-interaction mode: {mode!r}")
|
||||
|
||||
if score_indices:
|
||||
score_values = compute_maxsim_score_batched(score_queries, score_docs)
|
||||
score_values = self._score(score_queries, score_docs)
|
||||
for i, req_id, query_key, score in zip(
|
||||
score_indices, score_req_ids, score_query_keys, score_values
|
||||
):
|
||||
@@ -126,6 +139,58 @@ class LateInteractionRunner:
|
||||
|
||||
return outputs
|
||||
|
||||
def _score(
|
||||
self,
|
||||
queries: list[torch.Tensor],
|
||||
docs: list[torch.Tensor],
|
||||
) -> list[torch.Tensor]:
|
||||
"""Score (query_i, doc_i) pairs: the fused Triton kernel handles the
|
||||
common rerank pattern (one shared query, CUDA inputs); multiple
|
||||
distinct queries, CPU tensors, or a kernel failure fall back to the
|
||||
reference scorer."""
|
||||
if self._flash_enabled and docs and queries[0].is_cuda:
|
||||
first = queries[0]
|
||||
if all(q is first for q in queries):
|
||||
try:
|
||||
return self._score_flash_shared_query(first, docs)
|
||||
except Exception as exc:
|
||||
# A persistent compile/launch failure must not take down
|
||||
# requests: serve through the reference scorer and stop
|
||||
# trying the kernel for the rest of the process.
|
||||
self._flash_enabled = False
|
||||
logger.warning(
|
||||
"flash-maxsim scoring failed (%s); falling back to "
|
||||
"the reference MaxSim path for this process.",
|
||||
exc,
|
||||
)
|
||||
return compute_maxsim_score_batched(queries, docs)
|
||||
|
||||
@staticmethod
|
||||
def _score_flash_shared_query(
|
||||
query: torch.Tensor,
|
||||
docs: list[torch.Tensor],
|
||||
) -> list[torch.Tensor]:
|
||||
"""One kernel launch for all docs against a single shared query:
|
||||
docs pack into one [total_tokens, d] tensor (single cat, no padding)
|
||||
and the kernel reads each by (offset, length) — the [B, Lq, Ld]
|
||||
similarity tensor is never materialised."""
|
||||
from vllm.v1.pool.flash_maxsim import flash_maxsim_rerank_direct
|
||||
|
||||
device = docs[0].device
|
||||
lengths = [int(d.shape[0]) for d in docs]
|
||||
offsets = [0] * len(lengths)
|
||||
for i in range(1, len(lengths)):
|
||||
offsets[i] = offsets[i - 1] + lengths[i - 1]
|
||||
packed = docs[0] if len(docs) == 1 else torch.cat(docs, dim=0)
|
||||
scores = flash_maxsim_rerank_direct(
|
||||
query,
|
||||
packed,
|
||||
torch.tensor(offsets, device=device, dtype=torch.int32),
|
||||
torch.tensor(lengths, device=device, dtype=torch.int32),
|
||||
max(lengths),
|
||||
)
|
||||
return list(scores.unbind(0))
|
||||
|
||||
def _release_query_use(self, query_key: str) -> None:
|
||||
remaining = self._query_uses.get(query_key, 1) - 1
|
||||
if remaining <= 0:
|
||||
|
||||
@@ -51,7 +51,11 @@ class PoolingRunner:
|
||||
self.pooling_params: dict[int, PoolingParams] = {}
|
||||
self.pooling_states: dict[int, PoolingStates] = {}
|
||||
self.prompt_token_ids: dict[int, torch.Tensor] = {}
|
||||
self.late_interaction_runner = LateInteractionRunner()
|
||||
pooler_config = self.model_config.pooler_config
|
||||
self.late_interaction_runner = LateInteractionRunner(
|
||||
enable_flash=pooler_config is None
|
||||
or getattr(pooler_config, "enable_flash_late_interaction", True)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def get_supported_tasks(model: nn.Module) -> list[PoolingTask]:
|
||||
|
||||
Reference in New Issue
Block a user