[Bugfix] Preserve sampling masks in DELTA and TITO streaming outputs (#55935)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: fh3 <a@b> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: fh3 <a@b> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Codex
fh3
Claude Fable 5.1
parent
c9578f10ab
commit
4a9d6f9572
@@ -53,6 +53,8 @@ The mask is also available via the `/inference/v1/generate` HTTP endpoint:
|
||||
}
|
||||
```
|
||||
|
||||
With `stream=true`, each chunk carries the masks of the tokens it emits.
|
||||
|
||||
## Requirements
|
||||
|
||||
| Requirement | Reason |
|
||||
@@ -145,5 +147,3 @@ Each position then carries the sampled token's logprob and the top-128
|
||||
- **Engine-level flag:** `--return-sampling-mask` globally disables the
|
||||
FlashInfer fused sampler. All requests pay the cost of the PyTorch sampling
|
||||
path, even if they don't need the mask.
|
||||
- **No streaming support:** The mask is returned only in the final response,
|
||||
not in intermediate streaming chunks.
|
||||
|
||||
@@ -231,6 +231,7 @@ async fn generate_chunk_stream(
|
||||
logprobs,
|
||||
finish_reason: finish_reason.map(|reason| reason.as_str().to_string()),
|
||||
token_ids,
|
||||
sampling_mask: output.sampling_mask.map(|mask| mask.rows),
|
||||
}],
|
||||
usage: include_continuous_usage
|
||||
.then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)),
|
||||
@@ -331,6 +332,7 @@ fn collect_generate(
|
||||
logprobs,
|
||||
finish_reason: Some(finish_reason),
|
||||
token_ids: collected.token_ids,
|
||||
sampling_mask: collected.sampling_mask.map(|mask| mask.rows),
|
||||
}],
|
||||
prompt_logprobs,
|
||||
prompt_token_id_logprobs: collected
|
||||
@@ -516,6 +518,7 @@ mod tests {
|
||||
use futures::{TryStreamExt as _, stream};
|
||||
use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange};
|
||||
use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics;
|
||||
use vllm_engine_core_client::protocol::sampling_mask::SamplingMask;
|
||||
use vllm_llm::GeneratePromptInfo;
|
||||
|
||||
use super::*;
|
||||
@@ -657,6 +660,18 @@ mod tests {
|
||||
.expect("collect chunks")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_chunk_stream_returns_sampling_mask() {
|
||||
let mut output = stream_output(Some(&[11]), vec![33], Some(FinishReason::Length));
|
||||
output.sampling_mask = Some(SamplingMask {
|
||||
rows: vec![vec![33, 44]],
|
||||
});
|
||||
|
||||
let chunks = collect_chunks(vec![output], false, None).await;
|
||||
|
||||
assert_eq!(chunks[0].choices[0].sampling_mask, Some(vec![vec![33, 44]]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn generate_chunk_stream_omits_prompt_metadata_by_default() {
|
||||
let chunks = collect_chunks(
|
||||
@@ -792,7 +807,9 @@ mod tests {
|
||||
kv_transfer_params: None,
|
||||
ec_transfer_params: None,
|
||||
prompt_token_ids: vec![10, 20],
|
||||
sampling_mask: None,
|
||||
sampling_mask: Some(SamplingMask {
|
||||
rows: vec![vec![30, 40]],
|
||||
}),
|
||||
spec_decode_metrics: None,
|
||||
};
|
||||
|
||||
@@ -813,6 +830,7 @@ mod tests {
|
||||
|
||||
assert!(response.prompt_token_ids.is_none());
|
||||
assert!(response.mm_placeholders.is_none());
|
||||
assert_eq!(response.choices[0].sampling_mask, Some(vec![vec![30, 40]]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -64,6 +64,7 @@ pub(super) struct GenerateResponseChoice {
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_mask: Option<Vec<Vec<u32>>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateResponseStreamChoice` class.
|
||||
@@ -74,6 +75,7 @@ pub(super) struct GenerateResponseStreamChoice {
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
pub token_ids: Vec<u32>,
|
||||
pub sampling_mask: Option<Vec<Vec<u32>>>,
|
||||
}
|
||||
|
||||
/// Mirrors the Python vLLM `GenerateStreamResponse` class.
|
||||
|
||||
@@ -34,7 +34,7 @@ from vllm.multimodal.inputs import (
|
||||
MultiModalKwargsItem,
|
||||
PlaceholderRange,
|
||||
)
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask
|
||||
from vllm.renderers import renderer_from_config
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
@@ -1154,3 +1154,44 @@ async def test_stream_prompt_tokens_details_zero_cached():
|
||||
# Zero cached tokens must be present, not omitted
|
||||
assert usage_chunk["usage"]["prompt_tokens_details"] is not None
|
||||
assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("with_mask", [False, True])
|
||||
async def test_stream_sampling_mask_matches_each_token_chunk(with_mask):
|
||||
engine = _mock_engine()
|
||||
|
||||
async def generate(*args, **kwargs):
|
||||
for position, tokens in enumerate(([10], [20, 30])):
|
||||
result = _make_request_output(
|
||||
"req-mask",
|
||||
token_ids=list(tokens),
|
||||
finish_reason="length" if position else None,
|
||||
finished=bool(position),
|
||||
)
|
||||
if with_mask:
|
||||
result.outputs[0].sampling_mask = SamplingMask(
|
||||
[[token, token + 1] for token in tokens]
|
||||
)
|
||||
yield result
|
||||
|
||||
engine.generate = MagicMock(side_effect=generate)
|
||||
serving = _build_serving_tokens(engine)
|
||||
response = await serving.serve_tokens(
|
||||
GenerateRequest(
|
||||
model=MODEL_NAME, token_ids=[1, 2, 3], sampling_params={}, stream=True
|
||||
)
|
||||
)
|
||||
chunks = _parse_sse_chunks([chunk async for chunk in response])
|
||||
choices = [
|
||||
choice
|
||||
for chunk in chunks
|
||||
if isinstance(chunk, dict)
|
||||
for choice in chunk.get("choices", [])
|
||||
]
|
||||
assert [choice["token_ids"] for choice in choices] == [[10], [20, 30]]
|
||||
for choice in choices:
|
||||
expected = (
|
||||
[[token, token + 1] for token in choice["token_ids"]] if with_mask else None
|
||||
)
|
||||
assert choice["sampling_mask"] == expected
|
||||
|
||||
@@ -18,7 +18,7 @@ from tests.v1.engine.utils import (
|
||||
from vllm import PoolingParams
|
||||
from vllm.logprobs import FlatLogprobs, Logprob, PromptLogprobs, SampleLogprobs
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask
|
||||
from vllm.sampling_params import RequestOutputKind, SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.v1.engine import (
|
||||
@@ -1575,3 +1575,58 @@ def test_abort_requests(runner: str, abort_by: str, dummy_test_vectors):
|
||||
output_processor.abort_requests([request.request_id], internal=True)
|
||||
else:
|
||||
output_processor.abort_requests([request.external_req_id], internal=False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_kind", list(RequestOutputKind))
|
||||
def test_sampling_masks_follow_output_kind(output_kind):
|
||||
state = RequestState.__new__(RequestState)
|
||||
state.detokenizer = MagicMock()
|
||||
state.detokenizer.get_next_output_text.return_value = ""
|
||||
state.logprobs_processor = MagicMock()
|
||||
state.logprobs_processor.logprobs = None
|
||||
state.logprobs_processor.cumulative_logprob = None
|
||||
state.output_kind = output_kind
|
||||
state.request_index = 0
|
||||
state.sampling_mask_chunks = []
|
||||
state.routed_experts_chunks = []
|
||||
state.spec_decode_metrics = None
|
||||
|
||||
supports = [[10, 11], [20], [30, 31]]
|
||||
masks = []
|
||||
for position, support in enumerate(supports):
|
||||
state.sampling_mask_chunks.append(SamplingMaskLists(np.asarray(support)))
|
||||
finish = FinishReason.LENGTH if position == len(supports) - 1 else None
|
||||
mask = state._new_completion_output([support[0]], finish, None).sampling_mask
|
||||
masks.append(mask.token_ids if mask is not None else None)
|
||||
|
||||
if output_kind == RequestOutputKind.DELTA:
|
||||
assert masks == [[support] for support in supports]
|
||||
else:
|
||||
assert masks == [None, None, supports]
|
||||
|
||||
|
||||
def test_request_output_add_merges_delta_sampling_masks():
|
||||
def delta(token_ids, sampling_mask):
|
||||
completion = CompletionOutput(
|
||||
index=0,
|
||||
text="",
|
||||
token_ids=token_ids,
|
||||
cumulative_logprob=None,
|
||||
logprobs=None,
|
||||
sampling_mask=sampling_mask,
|
||||
)
|
||||
return RequestOutput(
|
||||
request_id="request",
|
||||
prompt=None,
|
||||
prompt_token_ids=[1],
|
||||
prompt_logprobs=None,
|
||||
outputs=[completion],
|
||||
finished=False,
|
||||
)
|
||||
|
||||
merged = delta([], None)
|
||||
merged.add(delta([10], SamplingMask([[10, 11]])), aggregate=True)
|
||||
merged.add(delta([20], SamplingMask([[20]])), aggregate=True)
|
||||
|
||||
assert merged.outputs[0].token_ids == [10, 20]
|
||||
assert merged.outputs[0].sampling_mask.token_ids == [[10, 11], [20]]
|
||||
|
||||
@@ -515,6 +515,10 @@ class ServingTokens(GenerateBaseServing):
|
||||
else None
|
||||
)
|
||||
|
||||
sampling_mask = None
|
||||
if output.sampling_mask is not None:
|
||||
sampling_mask = output.sampling_mask.token_ids
|
||||
|
||||
chunk = GenerateStreamResponse(
|
||||
request_id=request_id,
|
||||
choices=[
|
||||
@@ -524,6 +528,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
finish_reason=finish_reason,
|
||||
token_ids=as_list(delta_token_ids),
|
||||
routed_experts=routed_experts_b64,
|
||||
sampling_mask=sampling_mask,
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
@@ -197,6 +197,12 @@ class RequestOutput:
|
||||
if next_completion.logprobs:
|
||||
assert completion.logprobs is not None
|
||||
completion.logprobs.extend(next_completion.logprobs) # type: ignore[arg-type]
|
||||
if next_completion.sampling_mask is not None:
|
||||
if completion.sampling_mask is None:
|
||||
completion.sampling_mask = SamplingMask([])
|
||||
completion.sampling_mask.token_ids.extend(
|
||||
next_completion.sampling_mask.token_ids
|
||||
)
|
||||
completion.cumulative_logprob = (
|
||||
next_completion.cumulative_logprob
|
||||
)
|
||||
|
||||
@@ -439,7 +439,7 @@ class RequestState:
|
||||
logprobs = logprobs[-num_new_tokens:] if num_new_tokens else logprobs[:0]
|
||||
|
||||
sampling_mask = None
|
||||
if finished and self.sampling_mask_chunks:
|
||||
if (delta or finished) and self.sampling_mask_chunks:
|
||||
sampling_mask = SamplingMask(
|
||||
[
|
||||
position
|
||||
@@ -447,6 +447,8 @@ class RequestState:
|
||||
for position in chunk.to_nested_list()
|
||||
]
|
||||
)
|
||||
if delta:
|
||||
self.sampling_mask_chunks.clear()
|
||||
|
||||
# Concatenate routed experts on finish
|
||||
routed_experts = None
|
||||
|
||||
Reference in New Issue
Block a user