[Core] Refactoring sampler and support prompt logprob for chunked prefill (#4309)
This commit is contained in:
+10
-1
@@ -28,7 +28,10 @@ class Logprob:
|
||||
decoded_token: Optional[str] = None
|
||||
|
||||
|
||||
# {token_id -> logprob} per each sequence group. None if the corresponding
|
||||
# sequence group doesn't require prompt logprob.
|
||||
PromptLogprobs = List[Optional[Dict[int, Logprob]]]
|
||||
# {token_id -> logprob} for each sequence group.
|
||||
SampleLogprobs = List[Dict[int, Logprob]]
|
||||
|
||||
|
||||
@@ -215,7 +218,7 @@ class Sequence:
|
||||
self.eos_token_id = eos_token_id
|
||||
self.lora_request = lora_request
|
||||
|
||||
self.data = SequenceData(prompt_token_ids)
|
||||
self.data: SequenceData = SequenceData(prompt_token_ids)
|
||||
self.output_logprobs: SampleLogprobs = []
|
||||
self.output_text = ""
|
||||
|
||||
@@ -559,6 +562,9 @@ class SequenceGroupMetadata:
|
||||
sampling_params: The sampling parameters used to generate the outputs.
|
||||
block_tables: The block tables. (Seq id -> list of physical block
|
||||
numbers)
|
||||
do_sample: True if sampling is required. Sampling is not required when
|
||||
e.g., prefill is chunked, and the current iteration only computes
|
||||
query tokens for prefill, we don't need sampling.
|
||||
token_chunk_size: The number of tokens to be processed (per sequence).
|
||||
None if chunking is not required.
|
||||
state: Internal state tied to this sequence group.
|
||||
@@ -573,6 +579,7 @@ class SequenceGroupMetadata:
|
||||
seq_data: Dict[int, SequenceData],
|
||||
sampling_params: SamplingParams,
|
||||
block_tables: Dict[int, List[int]],
|
||||
do_sample: bool = True,
|
||||
token_chunk_size: Optional[int] = None,
|
||||
lora_request: Optional[LoRARequest] = None,
|
||||
computed_block_nums: Optional[List[int]] = None,
|
||||
@@ -589,6 +596,7 @@ class SequenceGroupMetadata:
|
||||
self.multi_modal_data = multi_modal_data
|
||||
self.state = SequenceGroupState() if state is None else state
|
||||
self._token_chunk_size = token_chunk_size
|
||||
self.do_sample = do_sample
|
||||
|
||||
if self._token_chunk_size is None:
|
||||
if is_prompt:
|
||||
@@ -650,6 +658,7 @@ class SequenceGroupOutput:
|
||||
prompt_logprobs: Optional[PromptLogprobs],
|
||||
) -> None:
|
||||
self.samples = samples
|
||||
# Prompt logprob for each prompt query token.
|
||||
self.prompt_logprobs = prompt_logprobs
|
||||
|
||||
def __repr__(self) -> str:
|
||||
|
||||
Reference in New Issue
Block a user