Push logprob generation to LLMEngine (#3065)
Co-authored-by: Avnish Narayan <avnish@anyscale.com>
This commit is contained in:
co-authored by
Avnish Narayan
parent
76e8a70476
commit
22de45235c
+17
-8
@@ -8,8 +8,16 @@ from vllm.block import LogicalTokenBlock
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.lora.request import LoRARequest
|
||||
|
||||
PromptLogprobs = List[Optional[Dict[int, float]]]
|
||||
SampleLogprobs = List[Dict[int, float]]
|
||||
|
||||
@dataclass
|
||||
class Logprob:
|
||||
"""Infos for supporting OpenAI compatible logprobs."""
|
||||
logprob: float
|
||||
decoded_token: Optional[str] = None
|
||||
|
||||
|
||||
PromptLogprobs = List[Optional[Dict[int, Logprob]]]
|
||||
SampleLogprobs = List[Dict[int, Logprob]]
|
||||
|
||||
|
||||
class SequenceStatus(enum.Enum):
|
||||
@@ -196,12 +204,12 @@ class Sequence:
|
||||
def append_token_id(
|
||||
self,
|
||||
token_id: int,
|
||||
logprobs: Dict[int, float],
|
||||
logprobs: Dict[int, Logprob],
|
||||
) -> None:
|
||||
assert token_id in logprobs
|
||||
self._append_tokens_to_blocks([token_id])
|
||||
self.output_logprobs.append(logprobs)
|
||||
self.data.append_token_id(token_id, logprobs[token_id])
|
||||
self.data.append_token_id(token_id, logprobs[token_id].logprob)
|
||||
|
||||
def get_len(self) -> int:
|
||||
return self.data.get_len()
|
||||
@@ -456,7 +464,7 @@ class SequenceOutput:
|
||||
self,
|
||||
parent_seq_id: int,
|
||||
output_token: int,
|
||||
logprobs: Dict[int, float],
|
||||
logprobs: Dict[int, Logprob],
|
||||
) -> None:
|
||||
self.parent_seq_id = parent_seq_id
|
||||
self.output_token = output_token
|
||||
@@ -470,9 +478,10 @@ class SequenceOutput:
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if not isinstance(other, SequenceOutput):
|
||||
raise NotImplementedError()
|
||||
return (self.parent_seq_id == other.parent_seq_id
|
||||
and self.output_token == other.output_token
|
||||
and self.logprobs == other.logprobs)
|
||||
equal = (self.parent_seq_id == other.parent_seq_id
|
||||
and self.output_token == other.output_token)
|
||||
log_probs_equal = other.logprobs == self.logprobs
|
||||
return equal and log_probs_equal
|
||||
|
||||
|
||||
class SequenceGroupOutput:
|
||||
|
||||
Reference in New Issue
Block a user