[Bugfix] Preserve sampling masks in DELTA and TITO streaming outputs (#55935)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: fh3 <a@b> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: fh3 <a@b> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
fh3
Claude Fable 5.1
parent
c9578f10ab
commit
4a9d6f9572
@@ -34,7 +34,7 @@ from vllm.multimodal.inputs import (
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
@@ -1154,3 +1154,44 @@ async def test_stream_prompt_tokens_details_zero_cached():
|
||||
# Zero cached tokens must be present, not omitted
|
||||
assert usage_chunk["usage"]["prompt_tokens_details"] is not None
|
||||
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("with_mask", [False, True])
|
||||
async def test_stream_sampling_mask_matches_each_token_chunk(with_mask):
|
||||
engine = _mock_engine()
|
||||
|
||||
async def generate(*args, **kwargs):
|
||||
for position, tokens in enumerate(([10], [20, 30])):
|
||||
result = _make_request_output(
|
||||
"req-mask",
|
||||
token_ids=list(tokens),
|
||||
finish_reason="length" if position else None,
|
||||
finished=bool(position),
|
||||
)
|
||||
if with_mask:
|
||||
result.outputs[0].sampling_mask = SamplingMask(
|
||||
[[token, token + 1] for token in tokens]
|
||||
)
|
||||
yield result
|
||||
|
||||
engine.generate = MagicMock(side_effect=generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
response = await serving.serve_tokens(
|
||||
GenerateRequest(
|
||||
model=MODEL_NAME, token_ids=[1, 2, 3], sampling_params={}, stream=True
|
||||
)
|
||||
)
|
||||
chunks = _parse_sse_chunks([chunk async for chunk in response])
|
||||
choices = [
|
||||
choice
|
||||
for chunk in chunks
|
||||
if isinstance(chunk, dict)
|
||||
for choice in chunk.get("choices", [])
|
||||
]
|
||||
assert [choice["token_ids"] for choice in choices] == [[10], [20, 30]]
|
||||
for choice in choices:
|
||||
expected = (
|
||||
[[token, token + 1] for token in choice["token_ids"]] if with_mask else None
|
||||
)
|
||||
assert choice["sampling_mask"] == expected
|
||||
|
||||
Reference in New Issue
Block a user