Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
320 lines
13 KiB
Python
320 lines
13 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass
|
|
|
|
import numpy as np
|
|
import torch
|
|
|
|
from vllm.config.model import LogprobsMode
|
|
from vllm.sampling_params import SamplingParams
|
|
from vllm.triton_utils import tl, triton
|
|
from vllm.utils.torch_utils import async_tensor_h2d
|
|
from vllm.v1.outputs import LogprobsTensors
|
|
from vllm.v1.worker.gpu.input_batch import InputBatch
|
|
from vllm.v1.worker.gpu.sample.logprob import (
|
|
compute_token_logprobs,
|
|
compute_topk_scores,
|
|
)
|
|
|
|
CHUNK_SIZE = 1024
|
|
|
|
|
|
@dataclass
|
|
class _TokenIdScores:
|
|
token_ids: torch.Tensor # [num_rows, num_ids], on device
|
|
pad: torch.Tensor # [num_rows, num_ids] bool, True at padded columns
|
|
start: int # first prompt row to score
|
|
scores: torch.Tensor | None = None # [num_rows, num_ids], filled per chunk
|
|
|
|
|
|
class PromptLogprobsWorker:
|
|
def __init__(
|
|
self,
|
|
max_num_reqs: int,
|
|
device: torch.device,
|
|
logprobs_mode: LogprobsMode = "raw_logprobs",
|
|
):
|
|
self.max_num_reqs = max_num_reqs
|
|
self.device = device
|
|
self.logprobs_mode = logprobs_mode
|
|
|
|
self.uses_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=bool)
|
|
self.num_prompt_logprobs = np.zeros(self.max_num_reqs, dtype=np.int32)
|
|
# req_idx -> list of in-progress LogprobsTensors
|
|
self.in_progress_prompt_logprobs: dict[str, list[LogprobsTensors]] = {}
|
|
# req_id -> fixed-ID scoring state
|
|
self.token_id_scores: dict[str, _TokenIdScores] = {}
|
|
|
|
def add_request(self, req_id: str, req_idx: int, sampling_params: SamplingParams):
|
|
uses_prompt_logprobs = sampling_params.prompt_logprobs is not None
|
|
self.uses_prompt_logprobs[req_idx] = uses_prompt_logprobs
|
|
self.num_prompt_logprobs[req_idx] = sampling_params.prompt_logprobs or 0
|
|
if uses_prompt_logprobs:
|
|
self.in_progress_prompt_logprobs[req_id] = []
|
|
if sampling_params.prompt_logprob_token_ids is not None:
|
|
ids = sampling_params.prompt_logprob_token_ids
|
|
# Decoded arrays may be read-only; torch.from_numpy warns on those.
|
|
if not ids.flags.writeable:
|
|
ids = ids.copy()
|
|
ids = async_tensor_h2d(ids, self.device, torch.int64)
|
|
self.token_id_scores[req_id] = _TokenIdScores(
|
|
ids.clamp_min(0), ids < 0, sampling_params.prompt_logprob_start or 0
|
|
)
|
|
|
|
def remove_request(self, req_id: str) -> None:
|
|
self.in_progress_prompt_logprobs.pop(req_id, None)
|
|
self.token_id_scores.pop(req_id, None)
|
|
|
|
def compute_prompt_token_id_logprobs(
|
|
self,
|
|
logits_fn: Callable[[torch.Tensor], torch.Tensor],
|
|
hidden_states: torch.Tensor,
|
|
input_batch: InputBatch,
|
|
prompt_lens: np.ndarray,
|
|
) -> dict[str, torch.Tensor]:
|
|
"""Fill scores row (prompt row - req.start) per chunk; emit on the last."""
|
|
if not self.token_id_scores:
|
|
return {}
|
|
logits_mode = self.logprobs_mode in ("raw_logits", "processed_logits")
|
|
out: dict[str, torch.Tensor] = {}
|
|
for i, req_id in enumerate(input_batch.req_ids):
|
|
req = self.token_id_scores.get(req_id)
|
|
if req is None:
|
|
continue
|
|
prompt_len = int(prompt_lens[input_batch.idx_mapping_np[i]])
|
|
chunk_start = int(input_batch.num_computed_prefill_tokens_np[i])
|
|
chunk_end = chunk_start + int(input_batch.num_scheduled_tokens[i])
|
|
# Skip decode steps and, as compute_prompt_logprobs does, requests
|
|
# resumed after preemption: their scores were already emitted.
|
|
if chunk_start >= prompt_len or prompt_len < input_batch.prefill_len_np[i]:
|
|
self.token_id_scores.pop(req_id, None)
|
|
continue
|
|
if req.scores is None:
|
|
if chunk_start > req.start:
|
|
# This prefill starts past the first scored row, so those
|
|
# rows will never be written; drop the request instead of
|
|
# emitting a buffer with unwritten rows.
|
|
self.token_id_scores.pop(req_id, None)
|
|
continue
|
|
req.scores = torch.empty_like(req.token_ids, dtype=torch.float32)
|
|
# The last prompt row predicts the first decode token; skip it.
|
|
lo = max(req.start, chunk_start)
|
|
hi = min(chunk_end, prompt_len - 1)
|
|
base = int(input_batch.query_start_loc_np[i]) - chunk_start
|
|
for a in range(lo, hi, CHUNK_SIZE):
|
|
b = min(a + CHUNK_SIZE, hi)
|
|
logits = logits_fn(hidden_states[base + a : base + b])
|
|
ids = req.token_ids[a - req.start : b - req.start]
|
|
req.scores[a - req.start : b - req.start] = (
|
|
logits.gather(-1, ids)
|
|
if logits_mode
|
|
else compute_token_logprobs(logits, ids)
|
|
)
|
|
if chunk_end >= prompt_len:
|
|
req.scores.masked_fill_(req.pad, float("-inf"))
|
|
out[req_id] = req.scores
|
|
self.token_id_scores.pop(req_id, None)
|
|
return out
|
|
|
|
def compute_prompt_logprobs(
|
|
self,
|
|
logits_fn: Callable[[torch.Tensor], torch.Tensor],
|
|
hidden_states: torch.Tensor,
|
|
input_batch: InputBatch,
|
|
# [max_num_reqs, max_model_len]
|
|
all_token_ids: torch.Tensor,
|
|
# [max_num_reqs]
|
|
num_computed_tokens: torch.Tensor,
|
|
# [max_num_reqs]
|
|
prompt_lens: np.ndarray,
|
|
) -> dict[str, LogprobsTensors]:
|
|
idx_mapping_np = input_batch.idx_mapping_np
|
|
needs_prompt_logprobs = self.uses_prompt_logprobs[idx_mapping_np]
|
|
if not np.any(needs_prompt_logprobs):
|
|
# Common case: No request asks for prompt logprobs.
|
|
return {}
|
|
|
|
num_prompt_logprobs = self.num_prompt_logprobs[idx_mapping_np]
|
|
prompt_lens = prompt_lens[idx_mapping_np]
|
|
computed_prefill = input_batch.num_computed_prefill_tokens_np
|
|
includes_prompt = computed_prefill < prompt_lens
|
|
# NOTE(woosuk): If the request was resumed after preemption, its prompt
|
|
# logprobs must have been computed before preemption. Skip.
|
|
resumed_after_prompt = prompt_lens < input_batch.prefill_len_np
|
|
needs_prompt_logprobs &= includes_prompt & ~resumed_after_prompt
|
|
if not np.any(needs_prompt_logprobs):
|
|
return {}
|
|
|
|
# get the maximum number in this batch
|
|
requested_num_prompt_logprobs = num_prompt_logprobs[needs_prompt_logprobs]
|
|
max_num_prompt_logprobs = (
|
|
-1
|
|
if np.any(requested_num_prompt_logprobs == -1)
|
|
else int(requested_num_prompt_logprobs.max())
|
|
)
|
|
|
|
# Get the prompt logprobs token_ids.
|
|
prompt_logprobs_token_ids = get_prompt_logprobs_token_ids(
|
|
input_batch.num_tokens,
|
|
input_batch.query_start_loc,
|
|
input_batch.idx_mapping,
|
|
num_computed_tokens,
|
|
all_token_ids,
|
|
)
|
|
prompt_token_ids, prompt_logprobs, prompt_ranks = (
|
|
compute_prompt_logprobs_with_chunking(
|
|
prompt_logprobs_token_ids,
|
|
hidden_states[: input_batch.num_tokens],
|
|
logits_fn,
|
|
max_num_prompt_logprobs,
|
|
self.logprobs_mode,
|
|
)
|
|
)
|
|
|
|
pos_after_step = computed_prefill + input_batch.num_scheduled_tokens
|
|
is_prompt_chunked = pos_after_step < prompt_lens
|
|
|
|
query_start_loc_np = input_batch.query_start_loc_np
|
|
prompt_logprobs_dict: dict[str, LogprobsTensors] = {}
|
|
for i, req_id in enumerate(input_batch.req_ids):
|
|
if not needs_prompt_logprobs[i]:
|
|
continue
|
|
|
|
req_is_prompt_chunked = is_prompt_chunked[i]
|
|
req_num_prompt_logprobs = int(num_prompt_logprobs[i])
|
|
start_idx = query_start_loc_np[i]
|
|
end_idx = query_start_loc_np[i + 1]
|
|
assert start_idx < end_idx, (
|
|
f"start_idx ({start_idx}) >= end_idx ({end_idx})"
|
|
)
|
|
if not req_is_prompt_chunked:
|
|
end_idx -= 1
|
|
|
|
width = (
|
|
prompt_logprobs.shape[1]
|
|
if req_num_prompt_logprobs == -1
|
|
else req_num_prompt_logprobs + 1
|
|
)
|
|
# no logprobs if start_idx >= end_idx
|
|
logprobs = (
|
|
None
|
|
if start_idx >= end_idx
|
|
else LogprobsTensors(
|
|
logprob_token_ids=prompt_token_ids[start_idx:end_idx, :width],
|
|
logprobs=prompt_logprobs[start_idx:end_idx, :width],
|
|
selected_token_ranks=prompt_ranks[start_idx:end_idx],
|
|
)
|
|
)
|
|
|
|
prompt_logprobs_list = self.in_progress_prompt_logprobs[req_id]
|
|
if logprobs is not None and (req_is_prompt_chunked or prompt_logprobs_list):
|
|
prompt_logprobs_list.append(logprobs)
|
|
if req_is_prompt_chunked:
|
|
# Prompt is chunked. Do not return the logprobs yet.
|
|
continue
|
|
|
|
if prompt_logprobs_list:
|
|
# Merge the in-progress logprobs.
|
|
logprobs = LogprobsTensors.cat(prompt_logprobs_list)
|
|
prompt_logprobs_list.clear()
|
|
|
|
if logprobs is None:
|
|
continue
|
|
|
|
prompt_logprobs_dict[req_id] = logprobs
|
|
return prompt_logprobs_dict
|
|
|
|
|
|
@triton.jit
|
|
def _prompt_logprobs_token_ids_kernel(
|
|
prompt_logprobs_token_ids_ptr,
|
|
query_start_loc_ptr,
|
|
idx_mapping_ptr,
|
|
num_computed_tokens_ptr,
|
|
all_token_ids_ptr,
|
|
all_token_ids_stride,
|
|
BLOCK_SIZE: tl.constexpr,
|
|
):
|
|
batch_idx = tl.program_id(0)
|
|
req_state_idx = tl.load(idx_mapping_ptr + batch_idx)
|
|
|
|
query_start = tl.load(query_start_loc_ptr + batch_idx)
|
|
query_end = tl.load(query_start_loc_ptr + batch_idx + 1)
|
|
query_len = query_end - query_start
|
|
|
|
num_computed_tokens = tl.load(num_computed_tokens_ptr + req_state_idx)
|
|
for i in range(0, query_len, BLOCK_SIZE):
|
|
block = i + tl.arange(0, BLOCK_SIZE)
|
|
mask = block < query_len
|
|
# NOTE(woosuk): We should shift the pos by one
|
|
# because the logprob is computed for the next token.
|
|
target_pos = num_computed_tokens + 1 + block
|
|
token_ids = tl.load(
|
|
all_token_ids_ptr + req_state_idx * all_token_ids_stride + target_pos,
|
|
mask=mask,
|
|
)
|
|
tl.store(
|
|
prompt_logprobs_token_ids_ptr + query_start + block, token_ids, mask=mask
|
|
)
|
|
|
|
|
|
def get_prompt_logprobs_token_ids(
|
|
num_tokens: int,
|
|
query_start_loc: torch.Tensor,
|
|
idx_mapping: torch.Tensor,
|
|
num_computed_tokens: torch.Tensor,
|
|
all_token_ids: torch.Tensor,
|
|
) -> torch.Tensor:
|
|
token_ids = torch.empty(num_tokens, dtype=torch.int64, device=idx_mapping.device)
|
|
num_reqs = idx_mapping.shape[0]
|
|
_prompt_logprobs_token_ids_kernel[(num_reqs,)](
|
|
token_ids,
|
|
query_start_loc,
|
|
idx_mapping,
|
|
num_computed_tokens,
|
|
all_token_ids,
|
|
all_token_ids.stride(0),
|
|
BLOCK_SIZE=1024,
|
|
)
|
|
return token_ids
|
|
|
|
|
|
def compute_prompt_logprobs_with_chunking(
|
|
prompt_token_ids: torch.Tensor,
|
|
prompt_hidden_states: torch.Tensor,
|
|
logits_fn: Callable[[torch.Tensor], torch.Tensor],
|
|
num_prompt_logprobs: int,
|
|
logprobs_mode: LogprobsMode = "raw_logprobs",
|
|
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
# Since materializing the full prompt logits can take too much memory,
|
|
# we compute it in chunks.
|
|
token_ids = []
|
|
scores = []
|
|
ranks = []
|
|
logits_mode = logprobs_mode in ("raw_logits", "processed_logits")
|
|
prompt_token_ids = prompt_token_ids.to(torch.int64)
|
|
for start_idx in range(0, prompt_token_ids.shape[0], CHUNK_SIZE):
|
|
end_idx = start_idx + CHUNK_SIZE
|
|
# NOTE(woosuk): logits_fn can be slow because it involves all-gather.
|
|
prompt_logits = logits_fn(prompt_hidden_states[start_idx:end_idx])
|
|
requested_num = (
|
|
prompt_logits.shape[-1]
|
|
if num_prompt_logprobs == -1
|
|
else num_prompt_logprobs
|
|
)
|
|
result = compute_topk_scores(
|
|
prompt_logits,
|
|
requested_num,
|
|
prompt_token_ids[start_idx:end_idx],
|
|
logits_mode=logits_mode,
|
|
)
|
|
token_ids.append(result.logprob_token_ids)
|
|
scores.append(result.logprobs)
|
|
ranks.append(result.selected_token_ranks)
|
|
|
|
token_ids = torch.cat(token_ids, dim=0) if len(token_ids) > 1 else token_ids[0]
|
|
scores = torch.cat(scores, dim=0) if len(scores) > 1 else scores[0]
|
|
ranks = torch.cat(ranks, dim=0) if len(ranks) > 1 else ranks[0]
|
|
return token_ids, scores, ranks
|