[Feature] Per-row candidate IDs for prefill token scoring (M2 of #56860) (#56984)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-03 14:17:09 +00:00
committed by GitHub
co-authored by aoshen02 Claude Opus 5.5
parent bc21cba967
commit e319f86f15
17 changed files with 366 additions and 162 deletions
+13 -11
View File
@@ -22,7 +22,8 @@ CHUNK_SIZE = 1024
@dataclass
class _TokenIdScores:
token_ids: torch.Tensor # [num_ids], on device
token_ids: torch.Tensor # [num_rows, num_ids], on device
pad: torch.Tensor # [num_rows, num_ids] bool, True at padded columns
start: int # first prompt row to score
scores: torch.Tensor | None = None # [num_rows, num_ids], filled per chunk
@@ -52,11 +53,13 @@ class PromptLogprobsWorker:
if uses_prompt_logprobs:
self.in_progress_prompt_logprobs[req_id] = []
if sampling_params.prompt_logprob_token_ids is not None:
ids = sampling_params.prompt_logprob_token_ids
# Decoded arrays may be read-only; torch.from_numpy warns on those.
if not ids.flags.writeable:
ids = ids.copy()
ids = async_tensor_h2d(ids, self.device, torch.int64)
self.token_id_scores[req_id] = _TokenIdScores(
async_tensor_h2d(
sampling_params.prompt_logprob_token_ids, self.device, torch.int64
),
sampling_params.prompt_logprob_start or 0,
ids.clamp_min(0), ids < 0, sampling_params.prompt_logprob_start or 0
)
def remove_request(self, req_id: str) -> None:
@@ -85,6 +88,7 @@ class PromptLogprobsWorker:
# Skip decode steps and, as compute_prompt_logprobs does, requests
# resumed after preemption: their scores were already emitted.
if chunk_start >= prompt_len or prompt_len < input_batch.prefill_len_np[i]:
self.token_id_scores.pop(req_id, None)
continue
if req.scores is None:
if chunk_start > req.start:
@@ -93,10 +97,7 @@ class PromptLogprobsWorker:
# emitting a buffer with unwritten rows.
self.token_id_scores.pop(req_id, None)
continue
req.scores = hidden_states.new_empty(
(max(prompt_len - 1 - req.start, 0), len(req.token_ids)),
dtype=torch.float32,
)
req.scores = torch.empty_like(req.token_ids, dtype=torch.float32)
# The last prompt row predicts the first decode token; skip it.
lo = max(req.start, chunk_start)
hi = min(chunk_end, prompt_len - 1)
@@ -104,15 +105,16 @@ class PromptLogprobsWorker:
for a in range(lo, hi, CHUNK_SIZE):
b = min(a + CHUNK_SIZE, hi)
logits = logits_fn(hidden_states[base + a : base + b])
ids = req.token_ids.expand(b - a, -1)
ids = req.token_ids[a - req.start : b - req.start]
req.scores[a - req.start : b - req.start] = (
logits.gather(-1, ids)
if logits_mode
else compute_token_logprobs(logits, ids)
)
if chunk_end >= prompt_len:
req.scores.masked_fill_(req.pad, float("-inf"))
out[req_id] = req.scores
req.scores = None
self.token_id_scores.pop(req_id, None)
return out
def compute_prompt_logprobs(