[Bugfix] Preserve sampling masks in DELTA and TITO streaming outputs (#55935)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: fh3 <a@b>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: fh3 <a@b>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-01 14:43:38 +08:00
committed by GitHub
co-authored by Codex fh3 Claude Fable 5.1
parent c9578f10ab
commit 4a9d6f9572
8 changed files with 135 additions and 6 deletions
@@ -34,7 +34,7 @@ from vllm.multimodal.inputs import (
MultiModalKwargsItem,
PlaceholderRange,
)
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask
from vllm.renderers import renderer_from_config
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
@@ -1154,3 +1154,44 @@ async def test_stream_prompt_tokens_details_zero_cached():
# Zero cached tokens must be present, not omitted
assert usage_chunk["usage"]["prompt_tokens_details"] is not None
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("with_mask", [False, True])
async def test_stream_sampling_mask_matches_each_token_chunk(with_mask):
engine = _mock_engine()
async def generate(*args, **kwargs):
for position, tokens in enumerate(([10], [20, 30])):
result = _make_request_output(
"req-mask",
token_ids=list(tokens),
finish_reason="length" if position else None,
finished=bool(position),
)
if with_mask:
result.outputs[0].sampling_mask = SamplingMask(
[[token, token + 1] for token in tokens]
)
yield result
engine.generate = MagicMock(side_effect=generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(
GenerateRequest(
model=MODEL_NAME, token_ids=[1, 2, 3], sampling_params={}, stream=True
)
)
chunks = _parse_sse_chunks([chunk async for chunk in response])
choices = [
choice
for chunk in chunks
if isinstance(chunk, dict)
for choice in chunk.get("choices", [])
]
assert [choice["token_ids"] for choice in choices] == [[10], [20, 30]]
for choice in choices:
expected = (
[[token, token + 1] for token in choice["token_ids"]] if with_mask else None
)
assert choice["sampling_mask"] == expected