Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
aoshen02
Claude Opus 5.5
parent
bc21cba967
commit
e319f86f15
@@ -22,7 +22,8 @@ CHUNK_SIZE = 1024
|
||||
|
||||
@dataclass
|
||||
class _TokenIdScores:
|
||||
token_ids: torch.Tensor # [num_ids], on device
|
||||
token_ids: torch.Tensor # [num_rows, num_ids], on device
|
||||
pad: torch.Tensor # [num_rows, num_ids] bool, True at padded columns
|
||||
start: int # first prompt row to score
|
||||
scores: torch.Tensor | None = None # [num_rows, num_ids], filled per chunk
|
||||
|
||||
@@ -52,11 +53,13 @@ class PromptLogprobsWorker:
|
||||
if uses_prompt_logprobs:
|
||||
self.in_progress_prompt_logprobs[req_id] = []
|
||||
if sampling_params.prompt_logprob_token_ids is not None:
|
||||
ids = sampling_params.prompt_logprob_token_ids
|
||||
# Decoded arrays may be read-only; torch.from_numpy warns on those.
|
||||
if not ids.flags.writeable:
|
||||
ids = ids.copy()
|
||||
ids = async_tensor_h2d(ids, self.device, torch.int64)
|
||||
self.token_id_scores[req_id] = _TokenIdScores(
|
||||
async_tensor_h2d(
|
||||
sampling_params.prompt_logprob_token_ids, self.device, torch.int64
|
||||
),
|
||||
sampling_params.prompt_logprob_start or 0,
|
||||
ids.clamp_min(0), ids < 0, sampling_params.prompt_logprob_start or 0
|
||||
)
|
||||
|
||||
def remove_request(self, req_id: str) -> None:
|
||||
@@ -85,6 +88,7 @@ class PromptLogprobsWorker:
|
||||
# Skip decode steps and, as compute_prompt_logprobs does, requests
|
||||
# resumed after preemption: their scores were already emitted.
|
||||
if chunk_start >= prompt_len or prompt_len < input_batch.prefill_len_np[i]:
|
||||
self.token_id_scores.pop(req_id, None)
|
||||
continue
|
||||
if req.scores is None:
|
||||
if chunk_start > req.start:
|
||||
@@ -93,10 +97,7 @@ class PromptLogprobsWorker:
|
||||
# emitting a buffer with unwritten rows.
|
||||
self.token_id_scores.pop(req_id, None)
|
||||
continue
|
||||
req.scores = hidden_states.new_empty(
|
||||
(max(prompt_len - 1 - req.start, 0), len(req.token_ids)),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
req.scores = torch.empty_like(req.token_ids, dtype=torch.float32)
|
||||
# The last prompt row predicts the first decode token; skip it.
|
||||
lo = max(req.start, chunk_start)
|
||||
hi = min(chunk_end, prompt_len - 1)
|
||||
@@ -104,15 +105,16 @@ class PromptLogprobsWorker:
|
||||
for a in range(lo, hi, CHUNK_SIZE):
|
||||
b = min(a + CHUNK_SIZE, hi)
|
||||
logits = logits_fn(hidden_states[base + a : base + b])
|
||||
ids = req.token_ids.expand(b - a, -1)
|
||||
ids = req.token_ids[a - req.start : b - req.start]
|
||||
req.scores[a - req.start : b - req.start] = (
|
||||
logits.gather(-1, ids)
|
||||
if logits_mode
|
||||
else compute_token_logprobs(logits, ids)
|
||||
)
|
||||
if chunk_end >= prompt_len:
|
||||
req.scores.masked_fill_(req.pad, float("-inf"))
|
||||
out[req_id] = req.scores
|
||||
req.scores = None
|
||||
self.token_id_scores.pop(req_id, None)
|
||||
return out
|
||||
|
||||
def compute_prompt_logprobs(
|
||||
|
||||
Reference in New Issue
Block a user