[Bugfix] Preserve sampling masks in DELTA and TITO streaming outputs (#55935)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: fh3 <a@b>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: fh3 <a@b>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
aoshen02
2026-10-01 14:43:38 +08:00
committed by GitHub
co-authored by Codex fh3 Claude Fable 5.1
parent c9578f10ab
commit 4a9d6f9572
8 changed files with 135 additions and 6 deletions
+2 -2
View File
@@ -53,6 +53,8 @@ The mask is also available via the `/inference/v1/generate` HTTP endpoint:
}
```
With `stream=true`, each chunk carries the masks of the tokens it emits.
## Requirements
| Requirement | Reason |
@@ -145,5 +147,3 @@ Each position then carries the sampled token's logprob and the top-128
- **Engine-level flag:** `--return-sampling-mask` globally disables the
FlashInfer fused sampler. All requests pay the cost of the PyTorch sampling
path, even if they don't need the mask.
- **No streaming support:** The mask is returned only in the final response,
not in intermediate streaming chunks.
@@ -231,6 +231,7 @@ async fn generate_chunk_stream(
logprobs,
finish_reason: finish_reason.map(|reason| reason.as_str().to_string()),
token_ids,
sampling_mask: output.sampling_mask.map(|mask| mask.rows),
}],
usage: include_continuous_usage
.then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)),
@@ -331,6 +332,7 @@ fn collect_generate(
logprobs,
finish_reason: Some(finish_reason),
token_ids: collected.token_ids,
sampling_mask: collected.sampling_mask.map(|mask| mask.rows),
}],
prompt_logprobs,
prompt_token_id_logprobs: collected
@@ -516,6 +518,7 @@ mod tests {
use futures::{TryStreamExt as _, stream};
use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange};
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
use vllm_llm::GeneratePromptInfo;
use super::*;
@@ -657,6 +660,18 @@ mod tests {
.expect("collect chunks")
}
#[tokio::test]
async fn generate_chunk_stream_returns_sampling_mask() {
let mut output = stream_output(Some(&[11]), vec![33], Some(FinishReason::Length));
output.sampling_mask = Some(SamplingMask {
rows: vec![vec![33, 44]],
});
let chunks = collect_chunks(vec![output], false, None).await;
assert_eq!(chunks[0].choices[0].sampling_mask, Some(vec![vec![33, 44]]));
}
#[tokio::test]
async fn generate_chunk_stream_omits_prompt_metadata_by_default() {
let chunks = collect_chunks(
@@ -792,7 +807,9 @@ mod tests {
kv_transfer_params: None,
ec_transfer_params: None,
prompt_token_ids: vec![10, 20],
sampling_mask: None,
sampling_mask: Some(SamplingMask {
rows: vec![vec![30, 40]],
}),
spec_decode_metrics: None,
};
@@ -813,6 +830,7 @@ mod tests {
assert!(response.prompt_token_ids.is_none());
assert!(response.mm_placeholders.is_none());
assert_eq!(response.choices[0].sampling_mask, Some(vec![vec![30, 40]]));
}
#[test]
@@ -64,6 +64,7 @@ pub(super) struct GenerateResponseChoice {
pub logprobs: Option<ChatLogProbs>,
pub finish_reason: Option<String>,
pub token_ids: Vec<u32>,
pub sampling_mask: Option<Vec<Vec<u32>>>,
}
/// Mirrors the Python vLLM `GenerateResponseStreamChoice` class.
@@ -74,6 +75,7 @@ pub(super) struct GenerateResponseStreamChoice {
pub logprobs: Option<ChatLogProbs>,
pub finish_reason: Option<String>,
pub token_ids: Vec<u32>,
pub sampling_mask: Option<Vec<Vec<u32>>>,
}
/// Mirrors the Python vLLM `GenerateStreamResponse` class.
@@ -34,7 +34,7 @@ from vllm.multimodal.inputs import (
MultiModalKwargsItem,
PlaceholderRange,
)
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask
from vllm.renderers import renderer_from_config
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
@@ -1154,3 +1154,44 @@ async def test_stream_prompt_tokens_details_zero_cached():
# Zero cached tokens must be present, not omitted
assert usage_chunk["usage"]["prompt_tokens_details"] is not None
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 0
@pytest.mark.asyncio
@pytest.mark.parametrize("with_mask", [False, True])
async def test_stream_sampling_mask_matches_each_token_chunk(with_mask):
engine = _mock_engine()
async def generate(*args, **kwargs):
for position, tokens in enumerate(([10], [20, 30])):
result = _make_request_output(
"req-mask",
token_ids=list(tokens),
finish_reason="length" if position else None,
finished=bool(position),
)
if with_mask:
result.outputs[0].sampling_mask = SamplingMask(
[[token, token + 1] for token in tokens]
)
yield result
engine.generate = MagicMock(side_effect=generate)
serving = _build_serving_tokens(engine)
response = await serving.serve_tokens(
GenerateRequest(
model=MODEL_NAME, token_ids=[1, 2, 3], sampling_params={}, stream=True
)
)
chunks = _parse_sse_chunks([chunk async for chunk in response])
choices = [
choice
for chunk in chunks
if isinstance(chunk, dict)
for choice in chunk.get("choices", [])
]
assert [choice["token_ids"] for choice in choices] == [[10], [20, 30]]
for choice in choices:
expected = (
[[token, token + 1] for token in choice["token_ids"]] if with_mask else None
)
assert choice["sampling_mask"] == expected
+56 -1
View File
@@ -18,7 +18,7 @@ from tests.v1.engine.utils import (
from vllm import PoolingParams
from vllm.logprobs import FlatLogprobs, Logprob, PromptLogprobs, SampleLogprobs
from vllm.lora.request import LoRARequest
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.v1.engine import (
@@ -1575,3 +1575,58 @@ def test_abort_requests(runner: str, abort_by: str, dummy_test_vectors):
output_processor.abort_requests([request.request_id], internal=True)
else:
output_processor.abort_requests([request.external_req_id], internal=False)
@pytest.mark.parametrize("output_kind", list(RequestOutputKind))
def test_sampling_masks_follow_output_kind(output_kind):
state = RequestState.__new__(RequestState)
state.detokenizer = MagicMock()
state.detokenizer.get_next_output_text.return_value = ""
state.logprobs_processor = MagicMock()
state.logprobs_processor.logprobs = None
state.logprobs_processor.cumulative_logprob = None
state.output_kind = output_kind
state.request_index = 0
state.sampling_mask_chunks = []
state.routed_experts_chunks = []
state.spec_decode_metrics = None
supports = [[10, 11], [20], [30, 31]]
masks = []
for position, support in enumerate(supports):
state.sampling_mask_chunks.append(SamplingMaskLists(np.asarray(support)))
finish = FinishReason.LENGTH if position == len(supports) - 1 else None
mask = state._new_completion_output([support[0]], finish, None).sampling_mask
masks.append(mask.token_ids if mask is not None else None)
if output_kind == RequestOutputKind.DELTA:
assert masks == [[support] for support in supports]
else:
assert masks == [None, None, supports]
def test_request_output_add_merges_delta_sampling_masks():
def delta(token_ids, sampling_mask):
completion = CompletionOutput(
index=0,
text="",
token_ids=token_ids,
cumulative_logprob=None,
logprobs=None,
sampling_mask=sampling_mask,
)
return RequestOutput(
request_id="request",
prompt=None,
prompt_token_ids=[1],
prompt_logprobs=None,
outputs=[completion],
finished=False,
)
merged = delta([], None)
merged.add(delta([10], SamplingMask([[10, 11]])), aggregate=True)
merged.add(delta([20], SamplingMask([[20]])), aggregate=True)
assert merged.outputs[0].token_ids == [10, 20]
assert merged.outputs[0].sampling_mask.token_ids == [[10, 11], [20]]
@@ -515,6 +515,10 @@ class ServingTokens(GenerateBaseServing):
else None
)
sampling_mask = None
if output.sampling_mask is not None:
sampling_mask = output.sampling_mask.token_ids
chunk = GenerateStreamResponse(
request_id=request_id,
choices=[
@@ -524,6 +528,7 @@ class ServingTokens(GenerateBaseServing):
finish_reason=finish_reason,
token_ids=as_list(delta_token_ids),
routed_experts=routed_experts_b64,
sampling_mask=sampling_mask,
)
],
)
+6
View File
@@ -197,6 +197,12 @@ class RequestOutput:
if next_completion.logprobs:
assert completion.logprobs is not None
completion.logprobs.extend(next_completion.logprobs) # type: ignore[arg-type]
if next_completion.sampling_mask is not None:
if completion.sampling_mask is None:
completion.sampling_mask = SamplingMask([])
completion.sampling_mask.token_ids.extend(
next_completion.sampling_mask.token_ids
)
completion.cumulative_logprob = (
next_completion.cumulative_logprob
)
+3 -1
View File
@@ -439,7 +439,7 @@ class RequestState:
logprobs = logprobs[-num_new_tokens:] if num_new_tokens else logprobs[:0]
sampling_mask = None
if finished and self.sampling_mask_chunks:
if (delta or finished) and self.sampling_mask_chunks:
sampling_mask = SamplingMask(
[
position
@@ -447,6 +447,8 @@ class RequestState:
for position in chunk.to_nested_list()
]
)
if delta:
self.sampling_mask_chunks.clear()
# Concatenate routed experts on finish
routed_experts = None