Push logprob generation to LLMEngine (#3065)

Co-authored-by: Avnish Narayan <avnish@anyscale.com>
This commit is contained in:
Antoni Baum
2024-03-04 19:54:06 +00:00
committed by GitHub
co-authored by Avnish Narayan
parent 76e8a70476
commit 22de45235c
13 changed files with 551 additions and 331 deletions
+17 -8
View File
@@ -8,8 +8,16 @@ from vllm.block import LogicalTokenBlock
from vllm.sampling_params import SamplingParams
from vllm.lora.request import LoRARequest
PromptLogprobs = List[Optional[Dict[int, float]]]
SampleLogprobs = List[Dict[int, float]]
@dataclass
class Logprob:
"""Infos for supporting OpenAI compatible logprobs."""
logprob: float
decoded_token: Optional[str] = None
PromptLogprobs = List[Optional[Dict[int, Logprob]]]
SampleLogprobs = List[Dict[int, Logprob]]
class SequenceStatus(enum.Enum):
@@ -196,12 +204,12 @@ class Sequence:
def append_token_id(
self,
token_id: int,
logprobs: Dict[int, float],
logprobs: Dict[int, Logprob],
) -> None:
assert token_id in logprobs
self._append_tokens_to_blocks([token_id])
self.output_logprobs.append(logprobs)
self.data.append_token_id(token_id, logprobs[token_id])
self.data.append_token_id(token_id, logprobs[token_id].logprob)
def get_len(self) -> int:
return self.data.get_len()
@@ -456,7 +464,7 @@ class SequenceOutput:
self,
parent_seq_id: int,
output_token: int,
logprobs: Dict[int, float],
logprobs: Dict[int, Logprob],
) -> None:
self.parent_seq_id = parent_seq_id
self.output_token = output_token
@@ -470,9 +478,10 @@ class SequenceOutput:
def __eq__(self, other: object) -> bool:
if not isinstance(other, SequenceOutput):
raise NotImplementedError()
return (self.parent_seq_id == other.parent_seq_id
and self.output_token == other.output_token
and self.logprobs == other.logprobs)
equal = (self.parent_seq_id == other.parent_seq_id
and self.output_token == other.output_token)
log_probs_equal = other.logprobs == self.logprobs
return equal and log_probs_equal
class SequenceGroupOutput: