diff --git a/tests/v1/worker/test_late_interaction_runner.py b/tests/v1/worker/test_late_interaction_runner.py index 054bfbdabd..4befbf72cc 100644 --- a/tests/v1/worker/test_late_interaction_runner.py +++ b/tests/v1/worker/test_late_interaction_runner.py @@ -10,6 +10,7 @@ from vllm.v1.pool.late_interaction import ( LATE_INTERACTION_MODE_CACHE_QUERY, build_late_interaction_doc_params, build_late_interaction_query_params, + compute_maxsim_score_batched, ) from vllm.v1.pool.late_interaction_runner import LateInteractionRunner @@ -152,3 +153,107 @@ def test_invalid_query_uses_raises(): req_ids=["query-req"], finished_mask=[True], ) + + +# --------------------------------------------------------------------------- +# Fused Triton scoring path (PR #40337). +# --------------------------------------------------------------------------- +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_score_flash_matches_fp64_oracle(): + """Shared-query scoring through the fused kernel must match a per-pair + fp64 MaxSim oracle on ragged doc lengths, including length-1 docs.""" + torch.manual_seed(0) + d = 128 + q = torch.randn(32, d, device="cuda", dtype=torch.float16) + doc_lengths = [180, 37, 512, 1, 300, 64, 1030, 256] + docs = [ + torch.randn(ld, d, device="cuda", dtype=torch.float16) for ld in doc_lengths + ] + + runner = LateInteractionRunner() + assert runner._flash_enabled + scores = runner._score([q] * len(docs), docs) + assert runner._flash_enabled, "kernel path must not have fallen back" + + for i, doc in enumerate(docs): + ref = (q.double() @ doc.double().T).max(dim=1).values.sum().float() + torch.testing.assert_close( + scores[i].to(torch.float32), ref, atol=5e-2, rtol=1e-3 + ) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA") +def test_score_multi_query_uses_reference_path(): + """Distinct queries per pair route to the reference scorer and agree + with it exactly (no kernel dispatch for the N:N pattern).""" + torch.manual_seed(1) + d = 128 + queries = [torch.randn(16, d, device="cuda") for _ in range(3)] + docs = [torch.randn(ld, d, device="cuda") for ld in (50, 3, 77)] + + runner = LateInteractionRunner() + scores = runner._score(queries, docs) + ref = compute_maxsim_score_batched(queries, docs) + for s, r in zip(scores, ref): + torch.testing.assert_close(s, r) + + +def test_score_flash_failure_disables_and_falls_back(monkeypatch): + """A kernel failure must serve the batch via the reference scorer and + disable the kernel path for the rest of the process.""" + runner = LateInteractionRunner() + runner._flash_enabled = True + + def _boom(query, docs): + raise RuntimeError("synthetic kernel failure") + + monkeypatch.setattr(runner, "_score_flash_shared_query", _boom) + q = torch.randn(8, 32) + docs = [torch.randn(5, 32), torch.randn(7, 32)] + # CPU tensors already skip the kernel; force the dispatch condition. + monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True)) + scores = runner._score([q, q], docs) + assert not runner._flash_enabled + ref = compute_maxsim_score_batched([q, q], docs) + for s, r in zip(scores, ref): + torch.testing.assert_close(s, r) + + +def test_disabled_flag_uses_reference_path(): + runner = LateInteractionRunner(enable_flash=False) + assert not runner._flash_enabled + + +# --------------------------------------------------------------------------- +# Autotune config pruning. +# --------------------------------------------------------------------------- +def _prune(monkeypatch, budget: int, named_args: dict): + from vllm.v1.pool.flash_maxsim import _common + + monkeypatch.setattr(_common, "_smem_budget", lambda: budget) + return _common._prune_configs(_common._get_configs(), named_args) + + +def _est(cfg, d_pad: int) -> int: + bq, bd = cfg.kwargs["BLOCK_Q"], cfg.kwargs["BLOCK_D"] + return (bq * d_pad + bd * d_pad) * 2 + bq * bd * 4 + + +def test_prune_configs_uses_padded_dim(monkeypatch): + """d=513 pads to 1024: survivors must fit the budget at d_pad, not at + the un-padded d (which would admit ~2x-oversized tiles).""" + budget = 166_912 # A100 opt-in + survivors = _prune(monkeypatch, budget, {"Lq": 1024, "d": 513, "d_pad": 1024}) + assert survivors + assert all(_est(c, 1024) <= budget for c in survivors) + + +def test_prune_configs_no_rejected_fallback(monkeypatch): + """When nothing fits, the fallback is the single smallest-footprint + config — never a slice of rejected configs.""" + from vllm.v1.pool.flash_maxsim import _common + + survivors = _prune(monkeypatch, 10_000, {"Lq": 1024, "d_pad": 1024}) + assert len(survivors) == 1 + smallest = min(_common._get_configs(), key=lambda c: _est(c, 1024)) + assert survivors[0].kwargs == smallest.kwargs diff --git a/vllm/config/pooler.py b/vllm/config/pooler.py index a5fbcbb622..04cf323015 100644 --- a/vllm/config/pooler.py +++ b/vllm/config/pooler.py @@ -63,6 +63,14 @@ class PoolerConfig: `None` uses the pooler's default, which is `True` in most cases. """ + enable_flash_late_interaction: bool = True + """ + Whether the engine-side late-interaction scorer may use the fused + flash-maxsim Triton kernel. Disabled automatically when the API server + is started with `--no-enable-flash-late-interaction`; the reference + scorer is used instead. + """ + ## for embedding models dimensions: int | None = None """ diff --git a/vllm/entrypoints/cli/serve.py b/vllm/entrypoints/cli/serve.py index 6a8492f4c4..1f6ca2b843 100644 --- a/vllm/entrypoints/cli/serve.py +++ b/vllm/entrypoints/cli/serve.py @@ -13,6 +13,7 @@ from vllm.entrypoints.cli.types import CLISubcommand from vllm.entrypoints.launchers.api_server.entry import run_server, setup_server from vllm.entrypoints.launchers.cli_args import ( make_arg_parser, + propagate_flash_late_interaction, validate_parsed_serve_args, ) from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor @@ -185,6 +186,7 @@ def run_headless(args: argparse.Namespace): # Create the EngineConfig. engine_args = vllm.AsyncEngineArgs.from_cli_args(args) + propagate_flash_late_interaction(args, engine_args) usage_context = UsageContext.OPENAI_API_SERVER vllm_config = engine_args.create_engine_config( usage_context=usage_context, headless=True @@ -295,6 +297,7 @@ def run_multi_api_server(args: argparse.Namespace): listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1) engine_args = vllm.AsyncEngineArgs.from_cli_args(args) + propagate_flash_late_interaction(args, engine_args) engine_args._api_process_count = num_api_servers engine_args._api_process_rank = -1 diff --git a/vllm/entrypoints/launchers/api_server/entry.py b/vllm/entrypoints/launchers/api_server/entry.py index 02837a451d..28bb696f2c 100644 --- a/vllm/entrypoints/launchers/api_server/entry.py +++ b/vllm/entrypoints/launchers/api_server/entry.py @@ -51,6 +51,9 @@ async def build_async_engine_client( # Context manager to handle engine_client lifecycle # Ensures everything is shutdown and cleaned up on error/exit engine_args = AsyncEngineArgs.from_cli_args(args) + from ..cli_args import propagate_flash_late_interaction + + propagate_flash_late_interaction(args, engine_args) if client_config: engine_args._api_process_count = client_config.get("client_count", 1) engine_args._api_process_rank = client_config.get("client_index", 0) diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index ce6692565c..3a52cf2e2b 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -363,7 +363,10 @@ class FrontendArgs(BaseFrontendArgs): """ enable_flash_late_interaction: bool = True """If set, run pooling score MaxSim on GPU in the API server process. - Can significantly improve late-interaction scoring performance.""" + Can significantly improve late-interaction scoring performance. + When disabled, the setting is also propagated to the engine's + `PoolerConfig`, so the engine-side scorer uses the reference MaxSim + path instead of the fused Triton kernel.""" @classmethod def _customize_cli_kwargs( @@ -498,3 +501,20 @@ def create_parser_for_docs() -> FlexibleArgumentParser: prog="-m vllm.entrypoints.launchers.api_server.entry" ) return make_arg_parser(parser_for_docs) + + +def propagate_flash_late_interaction(args, engine_args) -> None: + """Propagate `--no-enable-flash-late-interaction` into the engine config. + + The frontend flag alone only disables the API-server scoring path; + mirroring it into `PoolerConfig.enable_flash_late_interaction` lets the + engine-side scorer fall back to the reference MaxSim path too. + """ + if getattr(args, "enable_flash_late_interaction", True): + return + from vllm.config.pooler import PoolerConfig + + if engine_args.pooler_config is None: + engine_args.pooler_config = PoolerConfig(enable_flash_late_interaction=False) + else: + engine_args.pooler_config.enable_flash_late_interaction = False diff --git a/vllm/v1/pool/flash_maxsim/__init__.py b/vllm/v1/pool/flash_maxsim/__init__.py new file mode 100644 index 0000000000..dd5b20b102 --- /dev/null +++ b/vllm/v1/pool/flash_maxsim/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Fused Triton kernels for late-interaction (MaxSim) scoring.""" + +from .flash_maxsim_rerank import flash_maxsim_rerank_direct + +__all__ = ["flash_maxsim_rerank_direct"] diff --git a/vllm/v1/pool/flash_maxsim/_common.py b/vllm/v1/pool/flash_maxsim/_common.py new file mode 100644 index 0000000000..94633e743c --- /dev/null +++ b/vllm/v1/pool/flash_maxsim/_common.py @@ -0,0 +1,85 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared utilities for flash-maxsim kernels: padding, autotune configs, +and pruning.""" + +from vllm.triton_utils import triton + + +def _next_pow2(x: int) -> int: + return 1 << (x - 1).bit_length() + + +def _get_configs() -> list: + """Autotune grid over (BLOCK_Q, BLOCK_D); `_prune_configs` filters by + SMEM budget and Lq at runtime, so one list is safe across GPUs.""" + return [ + triton.Config({"BLOCK_Q": 16, "BLOCK_D": 16}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_Q": 16, "BLOCK_D": 32}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_Q": 32, "BLOCK_D": 16}, num_warps=2, num_stages=2), + triton.Config({"BLOCK_Q": 32, "BLOCK_D": 32}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_Q": 32, "BLOCK_D": 64}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_Q": 32, "BLOCK_D": 128}, num_warps=8, num_stages=2), + triton.Config({"BLOCK_Q": 64, "BLOCK_D": 64}, num_warps=4, num_stages=2), + triton.Config({"BLOCK_Q": 64, "BLOCK_D": 128}, num_warps=8, num_stages=1), + triton.Config({"BLOCK_Q": 128, "BLOCK_D": 64}, num_warps=8, num_stages=1), + triton.Config({"BLOCK_Q": 128, "BLOCK_D": 128}, num_warps=8, num_stages=1), + ] + + +def _smem_budget() -> int: + """Per-block opt-in shared-memory budget of the active device (bytes): + Triton driver query, else a conservative capability table.""" + try: + driver = triton.runtime.driver.active + props = driver.utils.get_device_properties(driver.get_current_device()) + return int(props["max_shared_mem"]) + except Exception: + try: + from vllm.platforms import current_platform + + cap = current_platform.get_device_capability() + if cap is not None: + if cap.major >= 9: + return 232_448 # Hopper+: 227 KB opt-in + if cap.major == 8: + return 101_376 # worst sm_8x (86/89); A100 has more + except Exception: + pass + return 98_304 # conservative pre-Ampere default + + +def _prune_configs(configs, named_args, **kwargs): + """Reject configs that exceed the GPU's shared-memory budget. + + BLOCK_Q and BLOCK_D load fp16 tiles of size [BLOCK_*, d_pad] — the + kernels index with ``tl.arange(0, d_pad)``, so the estimate must use + the padded dimension, not the real one (d=513 pads to 1024 and + nearly doubles the tile) — and the [BLOCK_Q, BLOCK_D] score tile is + fp32. Total: + (BLOCK_Q*d_pad + BLOCK_D*d_pad)*2 + BLOCK_Q*BLOCK_D*4 bytes. + """ + Lq = named_args.get("Lq", 32) + d_pad = named_args.get("d_pad") + if d_pad is None: + d_pad = _next_pow2(max(named_args.get("d", 128), 16)) + smem_limit = _smem_budget() + + def _est(cfg): + bq = cfg.kwargs["BLOCK_Q"] + bd = cfg.kwargs["BLOCK_D"] + return (bq * d_pad + bd * d_pad) * 2 + bq * bd * 4 + + pruned = [] + for cfg in configs: + if cfg.kwargs["BLOCK_Q"] > Lq * 2: + continue + if _est(cfg) > smem_limit: + continue + pruned.append(cfg) + if pruned: + return pruned + # Nothing fits the estimate (large d_pad on a small-SMEM device). + # Return the single smallest-footprint config — a deterministic + # least-bad launch attempt — never arbitrary rejected configs. + return [min(configs, key=_est)] diff --git a/vllm/v1/pool/flash_maxsim/flash_maxsim_rerank.py b/vllm/v1/pool/flash_maxsim/flash_maxsim_rerank.py new file mode 100644 index 0000000000..d25517691a --- /dev/null +++ b/vllm/v1/pool/flash_maxsim/flash_maxsim_rerank.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Flash-MaxSim rerank: one query vs variable-length docs in a packed tensor. + +The kernel reads doc embeddings directly from a packed [total_tokens, d] +tensor using per-doc (offset, length) pairs — no padding, no per-doc +tensors, no materialised [B, Lq, Ld] similarity tensor. + +Entry point: + flash_maxsim_rerank_direct(Q, batch_tensor, doc_offsets, doc_lengths, max_ld) + doc_offsets[i] = start token index of doc i in batch_tensor. + doc_lengths[i] = number of tokens in doc i. + Docs can be scattered (non-contiguous) in the batch tensor. +""" + +import torch + +from vllm.triton_utils import tl, triton + +from ._common import _get_configs, _next_pow2, _prune_configs + + +def _round_to_bucket(x): + """Round up to nearest bucket: 32, 64, 128, 256, 512, 1024, 2048, 4096.""" + for b in [32, 64, 128, 256, 512, 1024, 2048, 4096]: + if x <= b: + return b + return _next_pow2(x) + + +# Kernel: shared Q, scattered D via (offset, length) per doc + + +@triton.autotune( + configs=_get_configs(), + key=["Lq_bucket", "max_Ld_bucket", "d_pad"], + prune_configs_by={"early_config_prune": _prune_configs}, +) +@triton.jit +def _maxsim_rerank_kernel( + Q_ptr, + D_ptr, + doc_offsets_ptr, + doc_lengths_ptr, + scores_ptr, + B, + Lq, + Lq_bucket: tl.constexpr, + max_Ld_bucket: tl.constexpr, + d: tl.constexpr, + d_pad: tl.constexpr, + stride_q_t, + stride_q_d, + stride_d_t, + stride_d_d, + BLOCK_Q: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """One program per doc. Q shared (read from same location). D scattered.""" + pid = tl.program_id(0) + if pid >= B: + return + + # Each doc has its own offset and length in the batch tensor. + # int64: (d_start + d_off) * stride overflows int32 past ~16.7M batch + # tokens (same widen-on-load as the pairs kernel below; the host-side + # offsets tensor stays int32, which indexes any realistic batch). + d_start = tl.load(doc_offsets_ptr + pid).to(tl.int64) + d_len = tl.load(doc_lengths_ptr + pid).to(tl.int32) + + k_off = tl.arange(0, d_pad) + k_mask = k_off < d + score_acc = tl.zeros([], dtype=tl.float32) + + # Q is shared — always read from start of Q_ptr + for q_blk in tl.static_range(0, Lq_bucket, BLOCK_Q): + q_off = q_blk + tl.arange(0, BLOCK_Q) + q_valid = q_off < Lq + + Q_block = tl.load( + Q_ptr + q_off[:, None] * stride_q_t + k_off[None, :] * stride_q_d, + mask=q_valid[:, None] & k_mask[None, :], + other=0.0, + ).to(tl.float16) + + m = tl.full([BLOCK_Q], float("-inf"), dtype=tl.float32) + + for d_blk in range(0, max_Ld_bucket, BLOCK_D): + d_off = d_blk + tl.arange(0, BLOCK_D) + d_valid = d_off < d_len + + D_block = tl.load( + D_ptr + + (d_start + d_off[:, None]) * stride_d_t + + k_off[None, :] * stride_d_d, + mask=d_valid[:, None] & k_mask[None, :], + other=0.0, + ).to(tl.float16) + + S = tl.dot(Q_block, tl.trans(D_block)) + S = tl.where(d_valid[None, :], S, float("-inf")) + m = tl.maximum(m, tl.max(S, axis=1)) + + m = tl.where(q_valid, m, 0.0) + score_acc += tl.sum(m) + + tl.store(scores_ptr + pid, score_acc) + + +# Internal helper + + +def _run_rerank_kernel(Q, D_tensor, doc_offsets, doc_lengths, max_seqlen_d): + """Common kernel launch for both rerank and rerank_direct.""" + assert Q.is_cuda, f"Q must be on CUDA, got {Q.device}" + assert D_tensor.is_cuda, f"D must be on CUDA, got {D_tensor.device}" + Lq, d = Q.shape + B = doc_offsets.shape[0] + d_pad = _next_pow2(max(d, 16)) + + Q = Q.contiguous().half() + # D_tensor is NOT copied — kernel reads from it directly + if not D_tensor.is_contiguous(): + D_tensor = D_tensor.contiguous() + if D_tensor.dtype != torch.float16: + D_tensor = D_tensor.half() + + doc_offsets = doc_offsets.to(torch.int32).contiguous() + doc_lengths = doc_lengths.to(torch.int32).contiguous() + scores = torch.empty(B, device=Q.device, dtype=torch.float32) + + if B == 0: + return scores + + # Pad Q dimension if needed (Q is small, safe to copy). + # D is NOT padded — the kernel uses k_mask to handle d < d_pad. + # This preserves zero-copy: D_tensor is read in-place. + if d < d_pad: + Q_pad = torch.zeros(Lq, d_pad, device=Q.device, dtype=Q.dtype) + Q_pad[:, :d] = Q + else: + Q_pad = Q + D_pad = D_tensor + + Lq_bucket = _round_to_bucket(Lq) + max_ld_bucket = _round_to_bucket(max_seqlen_d) + + _maxsim_rerank_kernel[(B,)]( + Q_pad, + D_pad, + doc_offsets, + doc_lengths, + scores, + B, + Lq, + Lq_bucket, + max_ld_bucket, + d, + d_pad, + Q_pad.stride(0), + Q_pad.stride(1), + D_pad.stride(0), + D_pad.stride(1), + ) + return scores + + +# Public API + + +def flash_maxsim_rerank_direct( + Q: torch.Tensor, + batch_tensor: torch.Tensor, + doc_offsets: torch.Tensor, + doc_lengths: torch.Tensor, + max_seqlen_d: int, +) -> torch.Tensor: + """TRUE zero-copy: score query against docs scattered in a batch tensor. + + The kernel reads doc embeddings directly from batch_tensor at the + positions specified by doc_offsets. No torch.stack, no torch.cat, + no copy of any kind. The batch tensor is the model's output. + + Memory for doc scoring: 0 bytes additional. + + Args: + Q: [Lq, d] — single query embedding (from cache) + batch_tensor: [total_tokens, d] — the model's projected output tensor. + Contains ALL requests' tokens (queries + docs + others). + doc_offsets: [B] int32 — start token index of each doc in batch_tensor + doc_lengths: [B] int32 — number of tokens per doc + max_seqlen_d: int — max(doc_lengths) + + Returns: + scores: [B] float32 — one MaxSim score per document + + """ + assert Q.dim() == 2, f"Q must be 2D [Lq, d], got {Q.dim()}D" + assert batch_tensor.dim() == 2, ( + f"batch_tensor must be 2D, got {batch_tensor.dim()}D" + ) + assert Q.shape[1] == batch_tensor.shape[1] + + return _run_rerank_kernel(Q, batch_tensor, doc_offsets, doc_lengths, max_seqlen_d) diff --git a/vllm/v1/pool/late_interaction_runner.py b/vllm/v1/pool/late_interaction_runner.py index da87c8f05d..7ea39f5754 100644 --- a/vllm/v1/pool/late_interaction_runner.py +++ b/vllm/v1/pool/late_interaction_runner.py @@ -4,6 +4,7 @@ from collections.abc import Iterable import torch +from vllm.logger import init_logger from vllm.pooling_params import PoolingParams from vllm.v1.outputs import PoolerOutput from vllm.v1.pool.late_interaction import ( @@ -12,17 +13,29 @@ from vllm.v1.pool.late_interaction import ( compute_maxsim_score_batched, ) +logger = init_logger(__name__) + class LateInteractionRunner: """Worker-side state and postprocessing for late-interaction scoring.""" - def __init__(self) -> None: + def __init__(self, enable_flash: bool = True) -> None: # query_key -> token embeddings for late-interaction scoring. self._query_cache: dict[str, torch.Tensor] = {} # query_key -> remaining number of docs that should use this query. self._query_uses: dict[str, int] = {} # doc request id -> query key. self._doc_query_keys: dict[str, str] = {} + # Fused Triton scoring (PoolerConfig.enable_flash_late_interaction + # gates it; a runtime kernel failure disables it for the process). + if enable_flash: + try: + from vllm.v1.pool.flash_maxsim import ( # noqa: F401 + flash_maxsim_rerank_direct, + ) + except ImportError: + enable_flash = False + self._flash_enabled = enable_flash def clear(self) -> None: self._query_cache.clear() @@ -116,7 +129,7 @@ class LateInteractionRunner: raise ValueError(f"Unsupported late-interaction mode: {mode!r}") if score_indices: - score_values = compute_maxsim_score_batched(score_queries, score_docs) + score_values = self._score(score_queries, score_docs) for i, req_id, query_key, score in zip( score_indices, score_req_ids, score_query_keys, score_values ): @@ -126,6 +139,58 @@ class LateInteractionRunner: return outputs + def _score( + self, + queries: list[torch.Tensor], + docs: list[torch.Tensor], + ) -> list[torch.Tensor]: + """Score (query_i, doc_i) pairs: the fused Triton kernel handles the + common rerank pattern (one shared query, CUDA inputs); multiple + distinct queries, CPU tensors, or a kernel failure fall back to the + reference scorer.""" + if self._flash_enabled and docs and queries[0].is_cuda: + first = queries[0] + if all(q is first for q in queries): + try: + return self._score_flash_shared_query(first, docs) + except Exception as exc: + # A persistent compile/launch failure must not take down + # requests: serve through the reference scorer and stop + # trying the kernel for the rest of the process. + self._flash_enabled = False + logger.warning( + "flash-maxsim scoring failed (%s); falling back to " + "the reference MaxSim path for this process.", + exc, + ) + return compute_maxsim_score_batched(queries, docs) + + @staticmethod + def _score_flash_shared_query( + query: torch.Tensor, + docs: list[torch.Tensor], + ) -> list[torch.Tensor]: + """One kernel launch for all docs against a single shared query: + docs pack into one [total_tokens, d] tensor (single cat, no padding) + and the kernel reads each by (offset, length) — the [B, Lq, Ld] + similarity tensor is never materialised.""" + from vllm.v1.pool.flash_maxsim import flash_maxsim_rerank_direct + + device = docs[0].device + lengths = [int(d.shape[0]) for d in docs] + offsets = [0] * len(lengths) + for i in range(1, len(lengths)): + offsets[i] = offsets[i - 1] + lengths[i - 1] + packed = docs[0] if len(docs) == 1 else torch.cat(docs, dim=0) + scores = flash_maxsim_rerank_direct( + query, + packed, + torch.tensor(offsets, device=device, dtype=torch.int32), + torch.tensor(lengths, device=device, dtype=torch.int32), + max(lengths), + ) + return list(scores.unbind(0)) + def _release_query_use(self, query_key: str) -> None: remaining = self._query_uses.get(query_key, 1) - 1 if remaining <= 0: diff --git a/vllm/v1/worker/gpu/pool/pooling_runner.py b/vllm/v1/worker/gpu/pool/pooling_runner.py index ce94764851..1a81046b17 100644 --- a/vllm/v1/worker/gpu/pool/pooling_runner.py +++ b/vllm/v1/worker/gpu/pool/pooling_runner.py @@ -51,7 +51,11 @@ class PoolingRunner: self.pooling_params: dict[int, PoolingParams] = {} self.pooling_states: dict[int, PoolingStates] = {} self.prompt_token_ids: dict[int, torch.Tensor] = {} - self.late_interaction_runner = LateInteractionRunner() + pooler_config = self.model_config.pooler_config + self.late_interaction_runner = LateInteractionRunner( + enable_flash=pooler_config is None + or getattr(pooler_config, "enable_flash_late_interaction", True) + ) @staticmethod def get_supported_tasks(model: nn.Module) -> list[PoolingTask]: