[Rust Frontend] Feed parser tests and benchmarks attributed input (#59048)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Bugen Zhao
2026-09-29 06:23:54 +00:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 35d6fb3187
commit e05095c9e1
7 changed files with 334 additions and 142 deletions
+27 -7
View File
@@ -780,10 +780,29 @@ fn decoded_completion_stream(
assistant_stop_suffix, text
)
})?;
// The engine returns token ids, so attribute the rendered text
// through the production decoder, then split it into small char
// chunks to exercise marker splits.
let token_ids =
tokenizer.encode(body, false).context("failed to encode text completion")?;
let mut rest = vllm_text::DecodedText::default();
for chunk in incremental_decode_chunks(
tokenizer,
&prompt_token_ids,
&token_ids,
TOKEN_COMPLETION_CHUNK_TOKENS,
)? {
rest.append(chunk.delta);
}
ensure!(
rest.text == body,
"text completion must roundtrip through the tokenizer\nbody: {body:?}\ndecoded: {:?}",
rest.text
);
split_by_chars(body, TEXT_COMPLETION_CHUNK_CHARS)
.into_iter()
.map(|delta| DecodedCompletionChunk {
delta,
delta: rest.drain_prefix(delta.len()),
token_ids: Vec::new(), // unused for text-level roundtrip cases
})
.collect()
@@ -836,7 +855,7 @@ fn decoded_completion_stream(
sampling_mask: None,
});
events.push(DecodedTextEvent::TextDelta {
decoded: vllm_text::DecodedText::unattributed(chunk.delta),
decoded: chunk.delta,
sampled: vllm_text::SampledDelta {
token_ids: chunk.token_ids,
logprobs: None,
@@ -851,7 +870,8 @@ fn decoded_completion_stream(
/// One decoded completion chunk fed into the output processor.
struct DecodedCompletionChunk {
delta: String,
/// Attributed for token-id completions, as the production decoder emits it.
delta: vllm_text::DecodedText,
token_ids: Vec<u32>,
}
@@ -893,11 +913,11 @@ fn incremental_decode_chunks(
let mut decoder = tokenizer.create_decode_stream(prompt_token_ids, false, 0);
let mut chunks = Vec::new();
for chunk_token_ids in split_by_count(token_ids, chunk_size) {
let mut delta = String::new();
let mut delta = vllm_text::DecodedText::default();
for token_id in chunk_token_ids.iter().copied() {
decoder.push_token(token_id)?;
while let Some(chunk) = decoder.next_chunk() {
delta.push_str(&chunk.text);
delta.append(chunk);
}
}
chunks.push(DecodedCompletionChunk {
@@ -909,10 +929,10 @@ fn incremental_decode_chunks(
let (last_chunk, _) = decoder.flush(None)?;
if let Some(last_chunk) = last_chunk {
if let Some(delta) = chunks.last_mut() {
delta.delta.push_str(&last_chunk.text);
delta.delta.append(last_chunk);
} else {
chunks.push(DecodedCompletionChunk {
delta: last_chunk.text,
delta: last_chunk,
token_ids: Vec::new(),
});
}