[Core] Reduce TTFT with concurrent partial prefills (#10235)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Signed-off-by: Prashant Gupta <prashantgupta@us.ibm.com>
Co-authored-by: Prashant Gupta <prashantgupta@us.ibm.com>
Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
This commit is contained in:
Joe Runde
2025-02-14 15:36:07 -08:00
committed by GitHub
co-authored by Prashant Gupta Cody Yu
parent 5e5c8e091e
commit 3bcb8c75da
6 changed files with 699 additions and 106 deletions
+7 -1
View File
@@ -958,7 +958,9 @@ def get_logprobs(
if len(query_indices) == 0:
empty_sampled_logprob: SampleLogprobs = []
empty_prompt_logprob: Optional[PromptLogprobs] = None
return [empty_prompt_logprob], [empty_sampled_logprob]
num_seq_groups = len(sampling_metadata.seq_groups)
return [empty_prompt_logprob
] * num_seq_groups, [empty_sampled_logprob] * num_seq_groups
selected_logprobs, ranks = None, None
top_logprobs, top_token_ids = None, None
@@ -1225,6 +1227,10 @@ def _build_sampler_output(
assert sample_logprobs is not None
assert not isinstance(maybe_deferred_sample_results,
SampleResultArgsType)
assert len(sampling_metadata.seq_groups) \
== len(maybe_deferred_sample_results) \
== len(prompt_logprobs) \
== len(sample_logprobs)
deferred_sample_results_args = None
for (seq_group, sample_result, group_prompt_logprobs,