[Perf] Integrate flash-maxsim Triton kernels for late-interaction scoring (#40337)

Signed-off-by: roi.pony <roi.pony@ibm.com>
Co-authored-by: roi.pony <roi.pony@ibm.com>
Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
roipony
2026-10-01 14:06:43 -04:00
committed by GitHub
co-authored by roi.pony Taneem Ibrahim
parent 083060d046
commit e9be5323f0
10 changed files with 508 additions and 4 deletions
@@ -10,6 +10,7 @@ from vllm.v1.pool.late_interaction import (
LATE_INTERACTION_MODE_CACHE_QUERY,
build_late_interaction_doc_params,
build_late_interaction_query_params,
compute_maxsim_score_batched,
)
from vllm.v1.pool.late_interaction_runner import LateInteractionRunner
@@ -152,3 +153,107 @@ def test_invalid_query_uses_raises():
req_ids=["query-req"],
finished_mask=[True],
)
# ---------------------------------------------------------------------------
# Fused Triton scoring path (PR #40337).
# ---------------------------------------------------------------------------
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
def test_score_flash_matches_fp64_oracle():
"""Shared-query scoring through the fused kernel must match a per-pair
fp64 MaxSim oracle on ragged doc lengths, including length-1 docs."""
torch.manual_seed(0)
d = 128
q = torch.randn(32, d, device="cuda", dtype=torch.float16)
doc_lengths = [180, 37, 512, 1, 300, 64, 1030, 256]
docs = [
torch.randn(ld, d, device="cuda", dtype=torch.float16) for ld in doc_lengths
]
runner = LateInteractionRunner()
assert runner._flash_enabled
scores = runner._score([q] * len(docs), docs)
assert runner._flash_enabled, "kernel path must not have fallen back"
for i, doc in enumerate(docs):
ref = (q.double() @ doc.double().T).max(dim=1).values.sum().float()
torch.testing.assert_close(
scores[i].to(torch.float32), ref, atol=5e-2, rtol=1e-3
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
def test_score_multi_query_uses_reference_path():
"""Distinct queries per pair route to the reference scorer and agree
with it exactly (no kernel dispatch for the N:N pattern)."""
torch.manual_seed(1)
d = 128
queries = [torch.randn(16, d, device="cuda") for _ in range(3)]
docs = [torch.randn(ld, d, device="cuda") for ld in (50, 3, 77)]
runner = LateInteractionRunner()
scores = runner._score(queries, docs)
ref = compute_maxsim_score_batched(queries, docs)
for s, r in zip(scores, ref):
torch.testing.assert_close(s, r)
def test_score_flash_failure_disables_and_falls_back(monkeypatch):
"""A kernel failure must serve the batch via the reference scorer and
disable the kernel path for the rest of the process."""
runner = LateInteractionRunner()
runner._flash_enabled = True
def _boom(query, docs):
raise RuntimeError("synthetic kernel failure")
monkeypatch.setattr(runner, "_score_flash_shared_query", _boom)
q = torch.randn(8, 32)
docs = [torch.randn(5, 32), torch.randn(7, 32)]
# CPU tensors already skip the kernel; force the dispatch condition.
monkeypatch.setattr(torch.Tensor, "is_cuda", property(lambda self: True))
scores = runner._score([q, q], docs)
assert not runner._flash_enabled
ref = compute_maxsim_score_batched([q, q], docs)
for s, r in zip(scores, ref):
torch.testing.assert_close(s, r)
def test_disabled_flag_uses_reference_path():
runner = LateInteractionRunner(enable_flash=False)
assert not runner._flash_enabled
# ---------------------------------------------------------------------------
# Autotune config pruning.
# ---------------------------------------------------------------------------
def _prune(monkeypatch, budget: int, named_args: dict):
from vllm.v1.pool.flash_maxsim import _common
monkeypatch.setattr(_common, "_smem_budget", lambda: budget)
return _common._prune_configs(_common._get_configs(), named_args)
def _est(cfg, d_pad: int) -> int:
bq, bd = cfg.kwargs["BLOCK_Q"], cfg.kwargs["BLOCK_D"]
return (bq * d_pad + bd * d_pad) * 2 + bq * bd * 4
def test_prune_configs_uses_padded_dim(monkeypatch):
"""d=513 pads to 1024: survivors must fit the budget at d_pad, not at
the un-padded d (which would admit ~2x-oversized tiles)."""
budget = 166_912 # A100 opt-in
survivors = _prune(monkeypatch, budget, {"Lq": 1024, "d": 513, "d_pad": 1024})
assert survivors
assert all(_est(c, 1024) <= budget for c in survivors)
def test_prune_configs_no_rejected_fallback(monkeypatch):
"""When nothing fits, the fallback is the single smallest-footprint
config — never a slice of rejected configs."""
from vllm.v1.pool.flash_maxsim import _common
survivors = _prune(monkeypatch, 10_000, {"Lq": 1024, "d_pad": 1024})
assert len(survivors) == 1
smallest = min(_common._get_configs(), key=lambda c: _est(c, 1024))
assert survivors[0].kwargs == smallest.kwargs
+8
View File
@@ -63,6 +63,14 @@ class PoolerConfig:
`None` uses the pooler's default, which is `True` in most cases.
"""
enable_flash_late_interaction: bool = True
"""
Whether the engine-side late-interaction scorer may use the fused
flash-maxsim Triton kernel. Disabled automatically when the API server
is started with `--no-enable-flash-late-interaction`; the reference
scorer is used instead.
"""
## for embedding models
dimensions: int | None = None
"""
+3
View File
@@ -13,6 +13,7 @@ from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.launchers.api_server.entry import run_server, setup_server
from vllm.entrypoints.launchers.cli_args import (
make_arg_parser,
propagate_flash_late_interaction,
validate_parsed_serve_args,
)
from vllm.entrypoints.launchers.dp_supervisor import run_dp_supervisor
@@ -185,6 +186,7 @@ def run_headless(args: argparse.Namespace):
# Create the EngineConfig.
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
propagate_flash_late_interaction(args, engine_args)
usage_context = UsageContext.OPENAI_API_SERVER
vllm_config = engine_args.create_engine_config(
usage_context=usage_context, headless=True
@@ -295,6 +297,7 @@ def run_multi_api_server(args: argparse.Namespace):
listen_address, sock = setup_server(args, reuse_port=num_api_servers > 1)
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
propagate_flash_late_interaction(args, engine_args)
engine_args._api_process_count = num_api_servers
engine_args._api_process_rank = -1
@@ -51,6 +51,9 @@ async def build_async_engine_client(
# Context manager to handle engine_client lifecycle
# Ensures everything is shutdown and cleaned up on error/exit
engine_args = AsyncEngineArgs.from_cli_args(args)
from ..cli_args import propagate_flash_late_interaction
propagate_flash_late_interaction(args, engine_args)
if client_config:
engine_args._api_process_count = client_config.get("client_count", 1)
engine_args._api_process_rank = client_config.get("client_index", 0)
+21 -1
View File
@@ -363,7 +363,10 @@ class FrontendArgs(BaseFrontendArgs):
"""
enable_flash_late_interaction: bool = True
"""If set, run pooling score MaxSim on GPU in the API server process.
Can significantly improve late-interaction scoring performance."""
Can significantly improve late-interaction scoring performance.
When disabled, the setting is also propagated to the engine's
`PoolerConfig`, so the engine-side scorer uses the reference MaxSim
path instead of the fused Triton kernel."""
@classmethod
def _customize_cli_kwargs(
@@ -498,3 +501,20 @@ def create_parser_for_docs() -> FlexibleArgumentParser:
prog="-m vllm.entrypoints.launchers.api_server.entry"
)
return make_arg_parser(parser_for_docs)
def propagate_flash_late_interaction(args, engine_args) -> None:
"""Propagate `--no-enable-flash-late-interaction` into the engine config.
The frontend flag alone only disables the API-server scoring path;
mirroring it into `PoolerConfig.enable_flash_late_interaction` lets the
engine-side scorer fall back to the reference MaxSim path too.
"""
if getattr(args, "enable_flash_late_interaction", True):
return
from vllm.config.pooler import PoolerConfig
if engine_args.pooler_config is None:
engine_args.pooler_config = PoolerConfig(enable_flash_late_interaction=False)
else:
engine_args.pooler_config.enable_flash_late_interaction = False
+7
View File
@@ -0,0 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused Triton kernels for late-interaction (MaxSim) scoring."""
from .flash_maxsim_rerank import flash_maxsim_rerank_direct
__all__ = ["flash_maxsim_rerank_direct"]
+85
View File
@@ -0,0 +1,85 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Shared utilities for flash-maxsim kernels: padding, autotune configs,
and pruning."""
from vllm.triton_utils import triton
def _next_pow2(x: int) -> int:
return 1 << (x - 1).bit_length()
def _get_configs() -> list:
"""Autotune grid over (BLOCK_Q, BLOCK_D); `_prune_configs` filters by
SMEM budget and Lq at runtime, so one list is safe across GPUs."""
return [
triton.Config({"BLOCK_Q": 16, "BLOCK_D": 16}, num_warps=2, num_stages=2),
triton.Config({"BLOCK_Q": 16, "BLOCK_D": 32}, num_warps=2, num_stages=2),
triton.Config({"BLOCK_Q": 32, "BLOCK_D": 16}, num_warps=2, num_stages=2),
triton.Config({"BLOCK_Q": 32, "BLOCK_D": 32}, num_warps=4, num_stages=2),
triton.Config({"BLOCK_Q": 32, "BLOCK_D": 64}, num_warps=4, num_stages=2),
triton.Config({"BLOCK_Q": 32, "BLOCK_D": 128}, num_warps=8, num_stages=2),
triton.Config({"BLOCK_Q": 64, "BLOCK_D": 64}, num_warps=4, num_stages=2),
triton.Config({"BLOCK_Q": 64, "BLOCK_D": 128}, num_warps=8, num_stages=1),
triton.Config({"BLOCK_Q": 128, "BLOCK_D": 64}, num_warps=8, num_stages=1),
triton.Config({"BLOCK_Q": 128, "BLOCK_D": 128}, num_warps=8, num_stages=1),
]
def _smem_budget() -> int:
"""Per-block opt-in shared-memory budget of the active device (bytes):
Triton driver query, else a conservative capability table."""
try:
driver = triton.runtime.driver.active
props = driver.utils.get_device_properties(driver.get_current_device())
return int(props["max_shared_mem"])
except Exception:
try:
from vllm.platforms import current_platform
cap = current_platform.get_device_capability()
if cap is not None:
if cap.major >= 9:
return 232_448 # Hopper+: 227 KB opt-in
if cap.major == 8:
return 101_376 # worst sm_8x (86/89); A100 has more
except Exception:
pass
return 98_304 # conservative pre-Ampere default
def _prune_configs(configs, named_args, **kwargs):
"""Reject configs that exceed the GPU's shared-memory budget.
BLOCK_Q and BLOCK_D load fp16 tiles of size [BLOCK_*, d_pad] — the
kernels index with ``tl.arange(0, d_pad)``, so the estimate must use
the padded dimension, not the real one (d=513 pads to 1024 and
nearly doubles the tile) — and the [BLOCK_Q, BLOCK_D] score tile is
fp32. Total:
(BLOCK_Q*d_pad + BLOCK_D*d_pad)*2 + BLOCK_Q*BLOCK_D*4 bytes.
"""
Lq = named_args.get("Lq", 32)
d_pad = named_args.get("d_pad")
if d_pad is None:
d_pad = _next_pow2(max(named_args.get("d", 128), 16))
smem_limit = _smem_budget()
def _est(cfg):
bq = cfg.kwargs["BLOCK_Q"]
bd = cfg.kwargs["BLOCK_D"]
return (bq * d_pad + bd * d_pad) * 2 + bq * bd * 4
pruned = []
for cfg in configs:
if cfg.kwargs["BLOCK_Q"] > Lq * 2:
continue
if _est(cfg) > smem_limit:
continue
pruned.append(cfg)
if pruned:
return pruned
# Nothing fits the estimate (large d_pad on a small-SMEM device).
# Return the single smallest-footprint config — a deterministic
# least-bad launch attempt — never arbitrary rejected configs.
return [min(configs, key=_est)]
@@ -0,0 +1,204 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Flash-MaxSim rerank: one query vs variable-length docs in a packed tensor.
The kernel reads doc embeddings directly from a packed [total_tokens, d]
tensor using per-doc (offset, length) pairs — no padding, no per-doc
tensors, no materialised [B, Lq, Ld] similarity tensor.
Entry point:
flash_maxsim_rerank_direct(Q, batch_tensor, doc_offsets, doc_lengths, max_ld)
doc_offsets[i] = start token index of doc i in batch_tensor.
doc_lengths[i] = number of tokens in doc i.
Docs can be scattered (non-contiguous) in the batch tensor.
"""
import torch
from vllm.triton_utils import tl, triton
from ._common import _get_configs, _next_pow2, _prune_configs
def _round_to_bucket(x):
"""Round up to nearest bucket: 32, 64, 128, 256, 512, 1024, 2048, 4096."""
for b in [32, 64, 128, 256, 512, 1024, 2048, 4096]:
if x <= b:
return b
return _next_pow2(x)
# Kernel: shared Q, scattered D via (offset, length) per doc
@triton.autotune(
configs=_get_configs(),
key=["Lq_bucket", "max_Ld_bucket", "d_pad"],
prune_configs_by={"early_config_prune": _prune_configs},
)
@triton.jit
def _maxsim_rerank_kernel(
Q_ptr,
D_ptr,
doc_offsets_ptr,
doc_lengths_ptr,
scores_ptr,
B,
Lq,
Lq_bucket: tl.constexpr,
max_Ld_bucket: tl.constexpr,
d: tl.constexpr,
d_pad: tl.constexpr,
stride_q_t,
stride_q_d,
stride_d_t,
stride_d_d,
BLOCK_Q: tl.constexpr,
BLOCK_D: tl.constexpr,
):
"""One program per doc. Q shared (read from same location). D scattered."""
pid = tl.program_id(0)
if pid >= B:
return
# Each doc has its own offset and length in the batch tensor.
# int64: (d_start + d_off) * stride overflows int32 past ~16.7M batch
# tokens (same widen-on-load as the pairs kernel below; the host-side
# offsets tensor stays int32, which indexes any realistic batch).
d_start = tl.load(doc_offsets_ptr + pid).to(tl.int64)
d_len = tl.load(doc_lengths_ptr + pid).to(tl.int32)
k_off = tl.arange(0, d_pad)
k_mask = k_off < d
score_acc = tl.zeros([], dtype=tl.float32)
# Q is shared — always read from start of Q_ptr
for q_blk in tl.static_range(0, Lq_bucket, BLOCK_Q):
q_off = q_blk + tl.arange(0, BLOCK_Q)
q_valid = q_off < Lq
Q_block = tl.load(
Q_ptr + q_off[:, None] * stride_q_t + k_off[None, :] * stride_q_d,
mask=q_valid[:, None] & k_mask[None, :],
other=0.0,
).to(tl.float16)
m = tl.full([BLOCK_Q], float("-inf"), dtype=tl.float32)
for d_blk in range(0, max_Ld_bucket, BLOCK_D):
d_off = d_blk + tl.arange(0, BLOCK_D)
d_valid = d_off < d_len
D_block = tl.load(
D_ptr
+ (d_start + d_off[:, None]) * stride_d_t
+ k_off[None, :] * stride_d_d,
mask=d_valid[:, None] & k_mask[None, :],
other=0.0,
).to(tl.float16)
S = tl.dot(Q_block, tl.trans(D_block))
S = tl.where(d_valid[None, :], S, float("-inf"))
m = tl.maximum(m, tl.max(S, axis=1))
m = tl.where(q_valid, m, 0.0)
score_acc += tl.sum(m)
tl.store(scores_ptr + pid, score_acc)
# Internal helper
def _run_rerank_kernel(Q, D_tensor, doc_offsets, doc_lengths, max_seqlen_d):
"""Common kernel launch for both rerank and rerank_direct."""
assert Q.is_cuda, f"Q must be on CUDA, got {Q.device}"
assert D_tensor.is_cuda, f"D must be on CUDA, got {D_tensor.device}"
Lq, d = Q.shape
B = doc_offsets.shape[0]
d_pad = _next_pow2(max(d, 16))
Q = Q.contiguous().half()
# D_tensor is NOT copied — kernel reads from it directly
if not D_tensor.is_contiguous():
D_tensor = D_tensor.contiguous()
if D_tensor.dtype != torch.float16:
D_tensor = D_tensor.half()
doc_offsets = doc_offsets.to(torch.int32).contiguous()
doc_lengths = doc_lengths.to(torch.int32).contiguous()
scores = torch.empty(B, device=Q.device, dtype=torch.float32)
if B == 0:
return scores
# Pad Q dimension if needed (Q is small, safe to copy).
# D is NOT padded — the kernel uses k_mask to handle d < d_pad.
# This preserves zero-copy: D_tensor is read in-place.
if d < d_pad:
Q_pad = torch.zeros(Lq, d_pad, device=Q.device, dtype=Q.dtype)
Q_pad[:, :d] = Q
else:
Q_pad = Q
D_pad = D_tensor
Lq_bucket = _round_to_bucket(Lq)
max_ld_bucket = _round_to_bucket(max_seqlen_d)
_maxsim_rerank_kernel[(B,)](
Q_pad,
D_pad,
doc_offsets,
doc_lengths,
scores,
B,
Lq,
Lq_bucket,
max_ld_bucket,
d,
d_pad,
Q_pad.stride(0),
Q_pad.stride(1),
D_pad.stride(0),
D_pad.stride(1),
)
return scores
# Public API
def flash_maxsim_rerank_direct(
Q: torch.Tensor,
batch_tensor: torch.Tensor,
doc_offsets: torch.Tensor,
doc_lengths: torch.Tensor,
max_seqlen_d: int,
) -> torch.Tensor:
"""TRUE zero-copy: score query against docs scattered in a batch tensor.
The kernel reads doc embeddings directly from batch_tensor at the
positions specified by doc_offsets. No torch.stack, no torch.cat,
no copy of any kind. The batch tensor is the model's output.
Memory for doc scoring: 0 bytes additional.
Args:
Q: [Lq, d] — single query embedding (from cache)
batch_tensor: [total_tokens, d] — the model's projected output tensor.
Contains ALL requests' tokens (queries + docs + others).
doc_offsets: [B] int32 — start token index of each doc in batch_tensor
doc_lengths: [B] int32 — number of tokens per doc
max_seqlen_d: int — max(doc_lengths)
Returns:
scores: [B] float32 — one MaxSim score per document
"""
assert Q.dim() == 2, f"Q must be 2D [Lq, d], got {Q.dim()}D"
assert batch_tensor.dim() == 2, (
f"batch_tensor must be 2D, got {batch_tensor.dim()}D"
)
assert Q.shape[1] == batch_tensor.shape[1]
return _run_rerank_kernel(Q, batch_tensor, doc_offsets, doc_lengths, max_seqlen_d)
+67 -2
View File
@@ -4,6 +4,7 @@ from collections.abc import Iterable
import torch
from vllm.logger import init_logger
from vllm.pooling_params import PoolingParams
from vllm.v1.outputs import PoolerOutput
from vllm.v1.pool.late_interaction import (
@@ -12,17 +13,29 @@ from vllm.v1.pool.late_interaction import (
compute_maxsim_score_batched,
)
logger = init_logger(__name__)
class LateInteractionRunner:
"""Worker-side state and postprocessing for late-interaction scoring."""
def __init__(self) -> None:
def __init__(self, enable_flash: bool = True) -> None:
# query_key -> token embeddings for late-interaction scoring.
self._query_cache: dict[str, torch.Tensor] = {}
# query_key -> remaining number of docs that should use this query.
self._query_uses: dict[str, int] = {}
# doc request id -> query key.
self._doc_query_keys: dict[str, str] = {}
# Fused Triton scoring (PoolerConfig.enable_flash_late_interaction
# gates it; a runtime kernel failure disables it for the process).
if enable_flash:
try:
from vllm.v1.pool.flash_maxsim import ( # noqa: F401
flash_maxsim_rerank_direct,
)
except ImportError:
enable_flash = False
self._flash_enabled = enable_flash
def clear(self) -> None:
self._query_cache.clear()
@@ -116,7 +129,7 @@ class LateInteractionRunner:
raise ValueError(f"Unsupported late-interaction mode: {mode!r}")
if score_indices:
score_values = compute_maxsim_score_batched(score_queries, score_docs)
score_values = self._score(score_queries, score_docs)
for i, req_id, query_key, score in zip(
score_indices, score_req_ids, score_query_keys, score_values
):
@@ -126,6 +139,58 @@ class LateInteractionRunner:
return outputs
def _score(
self,
queries: list[torch.Tensor],
docs: list[torch.Tensor],
) -> list[torch.Tensor]:
"""Score (query_i, doc_i) pairs: the fused Triton kernel handles the
common rerank pattern (one shared query, CUDA inputs); multiple
distinct queries, CPU tensors, or a kernel failure fall back to the
reference scorer."""
if self._flash_enabled and docs and queries[0].is_cuda:
first = queries[0]
if all(q is first for q in queries):
try:
return self._score_flash_shared_query(first, docs)
except Exception as exc:
# A persistent compile/launch failure must not take down
# requests: serve through the reference scorer and stop
# trying the kernel for the rest of the process.
self._flash_enabled = False
logger.warning(
"flash-maxsim scoring failed (%s); falling back to "
"the reference MaxSim path for this process.",
exc,
)
return compute_maxsim_score_batched(queries, docs)
@staticmethod
def _score_flash_shared_query(
query: torch.Tensor,
docs: list[torch.Tensor],
) -> list[torch.Tensor]:
"""One kernel launch for all docs against a single shared query:
docs pack into one [total_tokens, d] tensor (single cat, no padding)
and the kernel reads each by (offset, length) — the [B, Lq, Ld]
similarity tensor is never materialised."""
from vllm.v1.pool.flash_maxsim import flash_maxsim_rerank_direct
device = docs[0].device
lengths = [int(d.shape[0]) for d in docs]
offsets = [0] * len(lengths)
for i in range(1, len(lengths)):
offsets[i] = offsets[i - 1] + lengths[i - 1]
packed = docs[0] if len(docs) == 1 else torch.cat(docs, dim=0)
scores = flash_maxsim_rerank_direct(
query,
packed,
torch.tensor(offsets, device=device, dtype=torch.int32),
torch.tensor(lengths, device=device, dtype=torch.int32),
max(lengths),
)
return list(scores.unbind(0))
def _release_query_use(self, query_key: str) -> None:
remaining = self._query_uses.get(query_key, 1) - 1
if remaining <= 0:
+5 -1
View File
@@ -51,7 +51,11 @@ class PoolingRunner:
self.pooling_params: dict[int, PoolingParams] = {}
self.pooling_states: dict[int, PoolingStates] = {}
self.prompt_token_ids: dict[int, torch.Tensor] = {}
self.late_interaction_runner = LateInteractionRunner()
pooler_config = self.model_config.pooler_config
self.late_interaction_runner = LateInteractionRunner(
enable_flash=pooler_config is None
or getattr(pooler_config, "enable_flash_late_interaction", True)
)
@staticmethod
def get_supported_tasks(model: nn.Module) -> list[PoolingTask]: