[Bugfix][Frontend] Keep streamed completion text_offset in step with tokens (#60668)

Signed-off-by: Vineeth Sai <vineethsai4444@gmail.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Vineeth Sai Varikuntla
2026-10-08 21:20:43 -07:00
committed by GitHub
co-authored by Flora Feng
parent 51ef239768
commit 112a282915
2 changed files with 105 additions and 3 deletions
@@ -289,7 +289,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
request_metadata: RequestResponseMetadata,
) -> AsyncGenerator[str, None]:
num_choices = 1 if request.n is None else request.n
previous_text_lens = [0] * num_choices * num_prompts
next_text_offsets = [0] * num_choices * num_prompts
previous_num_tokens = [0] * num_choices * num_prompts
has_echoed = [False] * num_choices * num_prompts
num_prompt_tokens = [0] * num_prompts
@@ -386,13 +386,18 @@ class OpenAIServingCompletion(GenerateBaseServing):
num_output_top_logprobs=request.logprobs,
tokenizer=tokenizer,
logprob_token_ids=request.logprob_token_ids,
initial_text_offset=previous_text_lens[i],
initial_text_offset=next_text_offsets[i],
return_as_token_id=request.return_tokens_as_token_ids,
)
# Advance by the tokens just reported, not by delta_text:
# text held back for stop strings lags the tokens.
if logprobs.tokens:
next_text_offsets[i] = logprobs.text_offset[-1] + len(
logprobs.tokens[-1]
)
else:
logprobs = None
previous_text_lens[i] += len(delta_text)
previous_num_tokens[i] += len(output.token_ids)
stop_reason = output.stop_reason