[Feature] Add fixed-token prefill scoring (#54335)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5
parent
a7ff44355c
commit
b721a4c709
@@ -19,6 +19,7 @@ def _topk_log_softmax_kernel(
|
||||
logits_ptr,
|
||||
logits_stride,
|
||||
topk_ids_ptr,
|
||||
topk_ids_stride,
|
||||
topk,
|
||||
vocab_size,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
@@ -49,7 +50,7 @@ def _topk_log_softmax_kernel(
|
||||
k_offset = j + tl.arange(0, TOPK_BLOCK_SIZE)
|
||||
k_mask = k_offset < topk
|
||||
topk_ids = tl.load(
|
||||
topk_ids_ptr + req_idx * topk + k_offset, mask=k_mask, other=0
|
||||
topk_ids_ptr + req_idx * topk_ids_stride + k_offset, mask=k_mask, other=0
|
||||
)
|
||||
logits = tl.load(row_ptr + topk_ids, mask=k_mask)
|
||||
logits = logits.to(tl.float32)
|
||||
@@ -98,6 +99,7 @@ def compute_token_logprobs(
|
||||
logits,
|
||||
logits.stride(0),
|
||||
token_ids,
|
||||
token_ids.stride(0),
|
||||
num_logprobs,
|
||||
vocab_size,
|
||||
BLOCK_SIZE=1024, # type: ignore
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -8,20 +9,41 @@ import torch
|
||||
from vllm.config.model import LogprobsMode
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.outputs import LogprobsTensors
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
from vllm.v1.worker.gpu.sample.logprob import compute_topk_scores
|
||||
from vllm.v1.worker.gpu.sample.logprob import (
|
||||
compute_token_logprobs,
|
||||
compute_topk_scores,
|
||||
)
|
||||
|
||||
CHUNK_SIZE = 1024
|
||||
|
||||
|
||||
@dataclass
|
||||
class _TokenIdScores:
|
||||
token_ids: torch.Tensor # [num_ids], on device
|
||||
start: int # first prompt row to score
|
||||
scores: torch.Tensor | None = None # [num_rows, num_ids], filled per chunk
|
||||
|
||||
|
||||
class PromptLogprobsWorker:
|
||||
def __init__(self, max_num_reqs: int, logprobs_mode: LogprobsMode = "raw_logprobs"):
|
||||
def __init__(
|
||||
self,
|
||||
max_num_reqs: int,
|
||||
device: torch.device,
|
||||
logprobs_mode: LogprobsMode = "raw_logprobs",
|
||||
):
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.device = device
|
||||
self.logprobs_mode = logprobs_mode
|
||||
|
||||
self.uses_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=bool)
|
||||
self.num_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=np.int32)
|
||||
# req_idx -> list of in-progress LogprobsTensors
|
||||
self.in_progress_prompt_logprobs: dict[str, list[LogprobsTensors]] = {}
|
||||
# req_id -> fixed-ID scoring state
|
||||
self.token_id_scores: dict[str, _TokenIdScores] = {}
|
||||
|
||||
def add_request(self, req_id: str, req_idx: int, sampling_params: SamplingParams):
|
||||
uses_prompt_logprobs = sampling_params.prompt_logprobs is not None
|
||||
@@ -29,9 +51,69 @@ class PromptLogprobsWorker:
|
||||
self.num_prompt_logprobs[req_idx] = sampling_params.prompt_logprobs or 0
|
||||
if uses_prompt_logprobs:
|
||||
self.in_progress_prompt_logprobs[req_id] = []
|
||||
if sampling_params.prompt_logprob_token_ids is not None:
|
||||
self.token_id_scores[req_id] = _TokenIdScores(
|
||||
async_tensor_h2d(
|
||||
sampling_params.prompt_logprob_token_ids, self.device, torch.int64
|
||||
),
|
||||
sampling_params.prompt_logprob_start or 0,
|
||||
)
|
||||
|
||||
def remove_request(self, req_id: str) -> None:
|
||||
self.in_progress_prompt_logprobs.pop(req_id, None)
|
||||
self.token_id_scores.pop(req_id, None)
|
||||
|
||||
def compute_prompt_token_id_logprobs(
|
||||
self,
|
||||
logits_fn: Callable[[torch.Tensor], torch.Tensor],
|
||||
hidden_states: torch.Tensor,
|
||||
input_batch: InputBatch,
|
||||
prompt_lens: np.ndarray,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Fill scores row (prompt row - req.start) per chunk; emit on the last."""
|
||||
if not self.token_id_scores:
|
||||
return {}
|
||||
logits_mode = self.logprobs_mode in ("raw_logits", "processed_logits")
|
||||
out: dict[str, torch.Tensor] = {}
|
||||
for i, req_id in enumerate(input_batch.req_ids):
|
||||
req = self.token_id_scores.get(req_id)
|
||||
if req is None:
|
||||
continue
|
||||
prompt_len = int(prompt_lens[input_batch.idx_mapping_np[i]])
|
||||
chunk_start = int(input_batch.num_computed_prefill_tokens_np[i])
|
||||
chunk_end = chunk_start + int(input_batch.num_scheduled_tokens[i])
|
||||
# Skip decode steps and, as compute_prompt_logprobs does, requests
|
||||
# resumed after preemption: their scores were already emitted.
|
||||
if chunk_start >= prompt_len or prompt_len < input_batch.prefill_len_np[i]:
|
||||
continue
|
||||
if req.scores is None:
|
||||
if chunk_start > req.start:
|
||||
# This prefill starts past the first scored row, so those
|
||||
# rows will never be written; drop the request instead of
|
||||
# emitting a buffer with unwritten rows.
|
||||
self.token_id_scores.pop(req_id, None)
|
||||
continue
|
||||
req.scores = hidden_states.new_empty(
|
||||
(max(prompt_len - 1 - req.start, 0), len(req.token_ids)),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
# The last prompt row predicts the first decode token; skip it.
|
||||
lo = max(req.start, chunk_start)
|
||||
hi = min(chunk_end, prompt_len - 1)
|
||||
base = int(input_batch.query_start_loc_np[i]) - chunk_start
|
||||
for a in range(lo, hi, CHUNK_SIZE):
|
||||
b = min(a + CHUNK_SIZE, hi)
|
||||
logits = logits_fn(hidden_states[base + a : base + b])
|
||||
ids = req.token_ids.expand(b - a, -1)
|
||||
req.scores[a - req.start : b - req.start] = (
|
||||
logits.gather(-1, ids)
|
||||
if logits_mode
|
||||
else compute_token_logprobs(logits, ids)
|
||||
)
|
||||
if chunk_end >= prompt_len:
|
||||
out[req_id] = req.scores
|
||||
req.scores = None
|
||||
return out
|
||||
|
||||
def compute_prompt_logprobs(
|
||||
self,
|
||||
@@ -205,7 +287,6 @@ def compute_prompt_logprobs_with_chunking(
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Since materializing the full prompt logits can take too much memory,
|
||||
# we compute it in chunks.
|
||||
CHUNK_SIZE = 1024
|
||||
token_ids = []
|
||||
scores = []
|
||||
ranks = []
|
||||
|
||||
Reference in New Issue
Block a user