Push logprob generation to LLMEngine (#3065)
Co-authored-by: Avnish Narayan <avnish@anyscale.com>
This commit is contained in:
co-authored by
Avnish Narayan
parent
76e8a70476
commit
22de45235c
@@ -18,7 +18,7 @@ from vllm.engine.ray_utils import RayWorkerVllm, initialize_cluster, ray
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.sequence import (SamplerOutput, Sequence, SequenceGroup,
|
||||
from vllm.sequence import (Logprob, SamplerOutput, Sequence, SequenceGroup,
|
||||
SequenceGroupOutput, SequenceOutput, SequenceStatus)
|
||||
from vllm.transformers_utils.tokenizer import (detokenize_incrementally,
|
||||
TokenizerGroup)
|
||||
@@ -473,6 +473,13 @@ class LLMEngine:
|
||||
if lora_request is not None and not self.lora_config:
|
||||
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
|
||||
"not enabled!")
|
||||
max_logprobs = self.get_model_config().max_logprobs
|
||||
if (sampling_params.logprobs
|
||||
and sampling_params.logprobs > max_logprobs) or (
|
||||
sampling_params.prompt_logprobs
|
||||
and sampling_params.prompt_logprobs > max_logprobs):
|
||||
raise ValueError(f"Cannot request more than "
|
||||
f"{max_logprobs} logprobs.")
|
||||
if arrival_time is None:
|
||||
arrival_time = time.monotonic()
|
||||
prompt_token_ids = self.encode_request(
|
||||
@@ -583,6 +590,13 @@ class LLMEngine:
|
||||
# Process prompt logprobs
|
||||
prompt_logprobs = outputs.prompt_logprobs
|
||||
if prompt_logprobs is not None:
|
||||
# We can pick any sequence for the prompt.
|
||||
seq = next(iter(seq_group.seqs_dict.values()))
|
||||
all_token_ids = seq.get_token_ids()
|
||||
for i, prompt_logprobs_for_token in enumerate(prompt_logprobs):
|
||||
self._decode_logprobs(seq, seq_group.sampling_params,
|
||||
prompt_logprobs_for_token,
|
||||
all_token_ids[:i])
|
||||
seq_group.prompt_logprobs = prompt_logprobs
|
||||
|
||||
# Process samples
|
||||
@@ -930,12 +944,36 @@ class LLMEngine:
|
||||
time_e2e_requests=time_e2e_requests,
|
||||
)
|
||||
|
||||
def _decode_logprobs(self, seq: Sequence, prms: SamplingParams,
|
||||
logprobs: Dict[int, Logprob],
|
||||
all_input_ids: List[int]) -> None:
|
||||
if not logprobs:
|
||||
return
|
||||
for token_id, sample_logprob in logprobs.items():
|
||||
if (sample_logprob.decoded_token is None and token_id != -1):
|
||||
all_input_ids_with_logprob = all_input_ids[:-1] + [token_id]
|
||||
_, new_text, prefix_offset, read_offset = detokenize_incrementally(
|
||||
self.get_tokenizer_for_seq(seq),
|
||||
all_input_ids=all_input_ids_with_logprob,
|
||||
prev_tokens=seq.tokens,
|
||||
prefix_offset=seq.prefix_offset,
|
||||
read_offset=seq.read_offset,
|
||||
skip_special_tokens=prms.skip_special_tokens,
|
||||
spaces_between_special_tokens=prms.
|
||||
spaces_between_special_tokens,
|
||||
)
|
||||
sample_logprob.decoded_token = new_text
|
||||
|
||||
def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
|
||||
"""Decodes the new token for a sequence."""
|
||||
all_input_ids = seq.get_token_ids()
|
||||
self._decode_logprobs(seq, prms, seq.output_logprobs[-1],
|
||||
all_input_ids)
|
||||
|
||||
(new_tokens, new_output_text, prefix_offset,
|
||||
read_offset) = detokenize_incrementally(
|
||||
self.get_tokenizer_for_seq(seq),
|
||||
all_input_ids=seq.get_token_ids(),
|
||||
all_input_ids=all_input_ids,
|
||||
prev_tokens=seq.tokens,
|
||||
prefix_offset=seq.prefix_offset,
|
||||
read_offset=seq.read_offset,
|
||||
|
||||
Reference in New Issue
Block a user