[V1] Logprobs and prompt logprobs support (#9880)
This PR is adding support for sample logprobs & prompt logprobs to vLLM v1. New behavior: - During model execution, model runner computes sample logprobs (if user-provided logprobs setting is not None) and prompt logprobs (if user-provided prompt_logprobs setting is not None). For both sample and prompt logprobs, the engine core returns 3 vectors: token ids, token logprob values, token ranks. Ranks reflect tokens' 1-indexed positions in the vocabulary vector after sorting the vocabulary by log probability in descending order. - In scheduler.update_from_output(), sample and prompt logprobs are incorporated into the EngineCoreOutput data structure which is transferred to the engine client. If multiprocessing is enabled, then sample and prompt logprobs will be (de)serialized when the EngineCoreOutput data structure is (de)serialized. - During output processing, the LogprobsProcessor transforms the triplet of token ids, token logprobs values, and token ranks into the OpenAI-compatible List[Dict[token id,Logprob]] format (for sample and prompt logprobs respectively.) - Each Logprob instance (whether sample- or prompt-) consists of a token's log-probability, rank, and detokenized string representation. Note that logprob detokenization is handled by the LogprobsProcessor not the detokenizer. Signed-off-by: Andrew Feldman <afeldman@neuralmagic.com> Signed-off-by: Nick Hill <nhill@redhat.com> Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com> Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
co-authored by
rshaw@neuralmagic.com <rshaw@neuralmagic.com>
Nick Hill
parent
538fab93cd
commit
0630d4537a
@@ -5,11 +5,12 @@ from dataclasses import dataclass
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.transformers_utils.detokenizer_utils import AnyTokenizer
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
|
||||
from vllm.v1.engine.detokenizer import (DetokenizerOutput,
|
||||
IncrementalDetokenizer)
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
|
||||
from vllm.v1.engine.detokenizer import IncrementalDetokenizer
|
||||
from vllm.v1.engine.logprobs import LogprobsProcessor
|
||||
from vllm.v1.metrics.stats import IterationStats, RequestStateStats
|
||||
|
||||
|
||||
@@ -26,16 +27,20 @@ class RequestState:
|
||||
def __init__(
|
||||
self,
|
||||
request_id: str,
|
||||
output_kind: RequestOutputKind,
|
||||
prompt: Optional[str],
|
||||
prompt_token_ids: List[int],
|
||||
logprobs_processor: LogprobsProcessor,
|
||||
detokenizer: IncrementalDetokenizer,
|
||||
arrival_time: float,
|
||||
queue: Optional[asyncio.Queue[RequestOutput]],
|
||||
):
|
||||
self.request_id = request_id
|
||||
self.output_kind = output_kind
|
||||
self.prompt = prompt
|
||||
self.prompt_token_ids = prompt_token_ids
|
||||
self.prompt_len = len(prompt_token_ids)
|
||||
self.logprobs_processor = logprobs_processor
|
||||
self.detokenizer = detokenizer
|
||||
self.is_prefilling = True
|
||||
self.queue = queue
|
||||
@@ -51,8 +56,13 @@ class RequestState:
|
||||
) -> "RequestState":
|
||||
return cls(
|
||||
request_id=request.request_id,
|
||||
output_kind=request.sampling_params.output_kind,
|
||||
prompt=request.prompt,
|
||||
prompt_token_ids=request.prompt_token_ids,
|
||||
logprobs_processor=LogprobsProcessor.from_new_request(
|
||||
tokenizer=tokenizer,
|
||||
request=request,
|
||||
),
|
||||
detokenizer=IncrementalDetokenizer.from_new_request(
|
||||
tokenizer=tokenizer,
|
||||
request=request,
|
||||
@@ -127,13 +137,8 @@ class OutputProcessor:
|
||||
batch to ensure system overheads are minimized. This is the
|
||||
only function that should loop over EngineCoreOutputs.
|
||||
|
||||
If you need to touch every element of the batch, implement a
|
||||
method called XXXClass.update_from_output() to be called
|
||||
within the loop below. For examples, see:
|
||||
* IterationStats.update_from_output()
|
||||
* Detokenizer.update_from_output()
|
||||
|
||||
TODO(rob): add Protocol makes update_from_output explicit.
|
||||
If you need to touch every element of the batch, do it from
|
||||
within the loop below.
|
||||
|
||||
**********************************************************
|
||||
"""
|
||||
@@ -154,17 +159,37 @@ class OutputProcessor:
|
||||
req_state.is_prefilling,
|
||||
req_state.prompt_len,
|
||||
req_state.stats)
|
||||
req_state.is_prefilling = False
|
||||
|
||||
# 2) Detokenize the token ids into text.
|
||||
detokenizer_output = req_state.detokenizer.update_from_output(
|
||||
engine_core_output)
|
||||
new_token_ids = engine_core_output.new_token_ids
|
||||
finish_reason = engine_core_output.finish_reason
|
||||
|
||||
# 3) Create and handle RequestOutput objects.
|
||||
if detokenizer_output is not None:
|
||||
request_output = self._make_request_output(
|
||||
req_state, detokenizer_output)
|
||||
# TODO(andy): prompt logprobs + chunked prefill can
|
||||
# result in engine core returning an output for a
|
||||
# partial prefill (in order to send back partial
|
||||
# prompt logprobs.) This breaks the invariant that
|
||||
# process_outputs is only operating on engine core
|
||||
# outputs associated with non-partial completions.
|
||||
# Currently this is handled by having `is_prefilling`
|
||||
# check for new decoded tokens, indicating that
|
||||
# the completion is not partial.
|
||||
#
|
||||
# Follow up will aggregate partial prompt logprobs
|
||||
# in the EngineCore.
|
||||
req_state.is_prefilling = not new_token_ids
|
||||
|
||||
# 2) Detokenize the token ids into text and check for stop
|
||||
# strings.
|
||||
stop_reason = req_state.detokenizer.update(new_token_ids)
|
||||
if stop_reason:
|
||||
finish_reason = FinishReason.STOP
|
||||
|
||||
# 3) Compute sample and prompt logprobs for request,
|
||||
# if required.
|
||||
req_state.logprobs_processor.update_from_output(engine_core_output)
|
||||
|
||||
# 4) Create and handle RequestOutput objects.
|
||||
if request_output := self._make_request_output(
|
||||
req_state, new_token_ids, finish_reason, stop_reason):
|
||||
if req_state.queue is not None:
|
||||
# AsyncLLM: put into queue for handling by generate().
|
||||
req_state.queue.put_nowait(request_output)
|
||||
@@ -174,18 +199,16 @@ class OutputProcessor:
|
||||
|
||||
# Free completed requests.
|
||||
if request_output.finished:
|
||||
assert detokenizer_output.finish_reason is not None
|
||||
|
||||
self.request_states.pop(req_id)
|
||||
if not engine_core_output.finished:
|
||||
# If req not finished in EngineCore, but Detokenizer
|
||||
# detected stop string, abort needed in EngineCore.
|
||||
reqs_to_abort.append(req_id)
|
||||
|
||||
# Track per-request stats
|
||||
# Track per-request stats.
|
||||
assert finish_reason is not None
|
||||
iteration_stats.update_from_finished_request(
|
||||
detokenizer_output.finish_reason, request_output,
|
||||
req_state.stats)
|
||||
finish_reason, request_output, req_state.stats)
|
||||
|
||||
return OutputProcessorOutput(
|
||||
request_outputs=request_outputs,
|
||||
@@ -196,20 +219,47 @@ class OutputProcessor:
|
||||
@staticmethod
|
||||
def _make_request_output(
|
||||
request_state: RequestState,
|
||||
detokenizer_output: DetokenizerOutput,
|
||||
) -> RequestOutput:
|
||||
new_token_ids: List[int],
|
||||
finish_reason: Optional[FinishReason],
|
||||
stop_reason: Optional[str],
|
||||
) -> Optional[RequestOutput]:
|
||||
|
||||
finished = finish_reason is not None
|
||||
output_kind = request_state.output_kind
|
||||
# In follow up, we will switch to invariant where EngineCore
|
||||
# does not stream partial prefills.
|
||||
if not finished and (request_state.is_prefilling
|
||||
or output_kind == RequestOutputKind.FINAL_ONLY):
|
||||
# Only the final output is required in FINAL_ONLY mode.
|
||||
return None
|
||||
|
||||
detokenizer = request_state.detokenizer
|
||||
logprobs_processor = request_state.logprobs_processor
|
||||
|
||||
delta = output_kind == RequestOutputKind.DELTA
|
||||
logprobs = logprobs_processor.logprobs
|
||||
if delta:
|
||||
if logprobs:
|
||||
logprobs = logprobs[-len(new_token_ids):]
|
||||
# Side effect: logprobs processor forgets prompt logprobs
|
||||
prompt_logprobs = logprobs_processor.pop_prompt_logprobs()
|
||||
else:
|
||||
prompt_logprobs = logprobs_processor.prompt_logprobs
|
||||
|
||||
request_output = RequestOutput.new(
|
||||
request_state.request_id,
|
||||
request_state.prompt,
|
||||
request_state.prompt_token_ids,
|
||||
detokenizer_output.output_text,
|
||||
detokenizer_output.token_ids,
|
||||
detokenizer_output.finished,
|
||||
request_id=request_state.request_id,
|
||||
prompt=request_state.prompt,
|
||||
prompt_token_ids=request_state.prompt_token_ids,
|
||||
text=detokenizer.get_next_output_text(finished, delta),
|
||||
token_ids=new_token_ids if delta else detokenizer.output_token_ids,
|
||||
logprobs=logprobs,
|
||||
prompt_logprobs=prompt_logprobs,
|
||||
cumulative_logprob=logprobs_processor.cumulative_logprob,
|
||||
finished=finished,
|
||||
)
|
||||
if detokenizer_output.finished:
|
||||
if finished:
|
||||
completion_output = request_output.outputs[0]
|
||||
completion_output.finish_reason = str(
|
||||
detokenizer_output.finish_reason)
|
||||
completion_output.stop_reason = detokenizer_output.stop_reason
|
||||
completion_output.finish_reason = str(finish_reason)
|
||||
completion_output.stop_reason = stop_reason
|
||||
|
||||
return request_output
|
||||
|
||||
Reference in New Issue
Block a user