[Feature] Add fixed-token prefill scoring (#54335)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-09-28 07:50:09 -07:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5
parent a7ff44355c
commit b721a4c709
36 changed files with 768 additions and 8 deletions
+3 -1
View File
@@ -19,6 +19,7 @@ def _topk_log_softmax_kernel(
logits_ptr,
logits_stride,
topk_ids_ptr,
topk_ids_stride,
topk,
vocab_size,
BLOCK_SIZE: tl.constexpr,
@@ -49,7 +50,7 @@ def _topk_log_softmax_kernel(
k_offset = j + tl.arange(0, TOPK_BLOCK_SIZE)
k_mask = k_offset < topk
topk_ids = tl.load(
topk_ids_ptr + req_idx * topk + k_offset, mask=k_mask, other=0
topk_ids_ptr + req_idx * topk_ids_stride + k_offset, mask=k_mask, other=0
)
logits = tl.load(row_ptr + topk_ids, mask=k_mask)
logits = logits.to(tl.float32)
@@ -98,6 +99,7 @@ def compute_token_logprobs(
logits,
logits.stride(0),
token_ids,
token_ids.stride(0),
num_logprobs,
vocab_size,
BLOCK_SIZE=1024, # type: ignore
+84 -3
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from dataclasses import dataclass
import numpy as np
import torch
@@ -8,20 +9,41 @@ import torch
from vllm.config.model import LogprobsMode
from vllm.sampling_params import SamplingParams
from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.outputs import LogprobsTensors
from vllm.v1.worker.gpu.input_batch import InputBatch
from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores
from vllm.v1.worker.gpu.sample.logprob import (
compute_token_logprobs,
compute_topk_scores,
)
CHUNK_SIZE = 1024
@dataclass
class _TokenIdScores:
token_ids: torch.Tensor # [num_ids], on device
start: int # first prompt row to score
scores: torch.Tensor | None = None # [num_rows, num_ids], filled per chunk
class PromptLogprobsWorker:
def __init__(self, max_num_reqs: int, logprobs_mode: LogprobsMode = "raw_logprobs"):
def __init__(
self,
max_num_reqs: int,
device: torch.device,
logprobs_mode: LogprobsMode = "raw_logprobs",
):
self.max_num_reqs = max_num_reqs
self.device = device
self.logprobs_mode = logprobs_mode
self.uses_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=bool)
self.num_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=np.int32)
# req_idx -> list of in-progress LogprobsTensors
self.in_progress_prompt_logprobs: dict[str, list[LogprobsTensors]] = {}
# req_id -> fixed-ID scoring state
self.token_id_scores: dict[str, _TokenIdScores] = {}
def add_request(self, req_id: str, req_idx: int, sampling_params: SamplingParams):
uses_prompt_logprobs = sampling_params.prompt_logprobs is not None
@@ -29,9 +51,69 @@ class PromptLogprobsWorker:
self.num_prompt_logprobs[req_idx] = sampling_params.prompt_logprobs or 0
if uses_prompt_logprobs:
self.in_progress_prompt_logprobs[req_id] = []
if sampling_params.prompt_logprob_token_ids is not None:
self.token_id_scores[req_id] = _TokenIdScores(
async_tensor_h2d(
sampling_params.prompt_logprob_token_ids, self.device, torch.int64
),
sampling_params.prompt_logprob_start or 0,
)
def remove_request(self, req_id: str) -> None:
self.in_progress_prompt_logprobs.pop(req_id, None)
self.token_id_scores.pop(req_id, None)
def compute_prompt_token_id_logprobs(
self,
logits_fn: Callable[[torch.Tensor], torch.Tensor],
hidden_states: torch.Tensor,
input_batch: InputBatch,
prompt_lens: np.ndarray,
) -> dict[str, torch.Tensor]:
"""Fill scores row (prompt row - req.start) per chunk; emit on the last."""
if not self.token_id_scores:
return {}
logits_mode = self.logprobs_mode in ("raw_logits", "processed_logits")
out: dict[str, torch.Tensor] = {}
for i, req_id in enumerate(input_batch.req_ids):
req = self.token_id_scores.get(req_id)
if req is None:
continue
prompt_len = int(prompt_lens[input_batch.idx_mapping_np[i]])
chunk_start = int(input_batch.num_computed_prefill_tokens_np[i])
chunk_end = chunk_start + int(input_batch.num_scheduled_tokens[i])
# Skip decode steps and, as compute_prompt_logprobs does, requests
# resumed after preemption: their scores were already emitted.
if chunk_start >= prompt_len or prompt_len < input_batch.prefill_len_np[i]:
continue
if req.scores is None:
if chunk_start > req.start:
# This prefill starts past the first scored row, so those
# rows will never be written; drop the request instead of
# emitting a buffer with unwritten rows.
self.token_id_scores.pop(req_id, None)
continue
req.scores = hidden_states.new_empty(
(max(prompt_len - 1 - req.start, 0), len(req.token_ids)),
dtype=torch.float32,
)
# The last prompt row predicts the first decode token; skip it.
lo = max(req.start, chunk_start)
hi = min(chunk_end, prompt_len - 1)
base = int(input_batch.query_start_loc_np[i]) - chunk_start
for a in range(lo, hi, CHUNK_SIZE):
b = min(a + CHUNK_SIZE, hi)
logits = logits_fn(hidden_states[base + a : base + b])
ids = req.token_ids.expand(b - a, -1)
req.scores[a - req.start : b - req.start] = (
logits.gather(-1, ids)
if logits_mode
else compute_token_logprobs(logits, ids)
)
if chunk_end >= prompt_len:
out[req_id] = req.scores
req.scores = None
return out
def compute_prompt_logprobs(
self,
@@ -205,7 +287,6 @@ def compute_prompt_logprobs_with_chunking(
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
# Since materializing the full prompt logits can take too much memory,
# we compute it in chunks.
CHUNK_SIZE = 1024
token_ids = []
scores = []
ranks = []