diff --git a/docs/training/sampling_mask.md b/docs/training/sampling_mask.md index c8425e0245..7de2366800 100644 --- a/docs/training/sampling_mask.md +++ b/docs/training/sampling_mask.md @@ -53,6 +53,8 @@ The mask is also available via the `/inference/v1/generate` HTTP endpoint: } ``` +With `stream=true`, each chunk carries the masks of the tokens it emits. + ## Requirements | Requirement | Reason | @@ -145,5 +147,3 @@ Each position then carries the sampled token's logprob and the top-128 - **Engine-level flag:** `--return-sampling-mask` globally disables the FlashInfer fused sampler. All requests pay the cost of the PyTorch sampling path, even if they don't need the mask. -- **No streaming support:** The mask is returned only in the final response, - not in intermediate streaming chunks. diff --git a/rust/src/server/src/routes/inference/generate.rs b/rust/src/server/src/routes/inference/generate.rs index 15caedaf17..0e682f7c45 100644 --- a/rust/src/server/src/routes/inference/generate.rs +++ b/rust/src/server/src/routes/inference/generate.rs @@ -231,6 +231,7 @@ async fn generate_chunk_stream( logprobs, finish_reason: finish_reason.map(|reason| reason.as_str().to_string()), token_ids, + sampling_mask: output.sampling_mask.map(|mask| mask.rows), }], usage: include_continuous_usage .then(|| Usage::from_token_usage(usage, enable_prompt_tokens_details)), @@ -331,6 +332,7 @@ fn collect_generate( logprobs, finish_reason: Some(finish_reason), token_ids: collected.token_ids, + sampling_mask: collected.sampling_mask.map(|mask| mask.rows), }], prompt_logprobs, prompt_token_id_logprobs: collected @@ -516,6 +518,7 @@ mod tests { use futures::{TryStreamExt as _, stream}; use vllm_engine_core_client::protocol::multimodal::{MmModality, PlaceholderRange}; use vllm_engine_core_client::protocol::output::RequestSpecDecodeMetrics; + use vllm_engine_core_client::protocol::sampling_mask::SamplingMask; use vllm_llm::GeneratePromptInfo; use super::*; @@ -657,6 +660,18 @@ mod tests { .expect("collect chunks") } + #[tokio::test] + async fn generate_chunk_stream_returns_sampling_mask() { + let mut output = stream_output(Some(&[11]), vec![33], Some(FinishReason::Length)); + output.sampling_mask = Some(SamplingMask { + rows: vec![vec![33, 44]], + }); + + let chunks = collect_chunks(vec![output], false, None).await; + + assert_eq!(chunks[0].choices[0].sampling_mask, Some(vec![vec![33, 44]])); + } + #[tokio::test] async fn generate_chunk_stream_omits_prompt_metadata_by_default() { let chunks = collect_chunks( @@ -792,7 +807,9 @@ mod tests { kv_transfer_params: None, ec_transfer_params: None, prompt_token_ids: vec![10, 20], - sampling_mask: None, + sampling_mask: Some(SamplingMask { + rows: vec![vec![30, 40]], + }), spec_decode_metrics: None, }; @@ -813,6 +830,7 @@ mod tests { assert!(response.prompt_token_ids.is_none()); assert!(response.mm_placeholders.is_none()); + assert_eq!(response.choices[0].sampling_mask, Some(vec![vec![30, 40]])); } #[test] diff --git a/rust/src/server/src/routes/inference/generate/types.rs b/rust/src/server/src/routes/inference/generate/types.rs index 78fe085b83..b0b93d7080 100644 --- a/rust/src/server/src/routes/inference/generate/types.rs +++ b/rust/src/server/src/routes/inference/generate/types.rs @@ -64,6 +64,7 @@ pub(super) struct GenerateResponseChoice { pub logprobs: Option, pub finish_reason: Option, pub token_ids: Vec, + pub sampling_mask: Option>>, } /// Mirrors the Python vLLM `GenerateResponseStreamChoice` class. @@ -74,6 +75,7 @@ pub(super) struct GenerateResponseStreamChoice { pub logprobs: Option, pub finish_reason: Option, pub token_ids: Vec, + pub sampling_mask: Option>>, } /// Mirrors the Python vLLM `GenerateStreamResponse` class. diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py index 28f8a505f8..53ff1dc598 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_generate_stream.py @@ -34,7 +34,7 @@ from vllm.multimodal.inputs import ( MultiModalKwargsItem, PlaceholderRange, ) -from vllm.outputs import CompletionOutput, RequestOutput +from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask from vllm.renderers import renderer_from_config from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import SamplingParams @@ -1154,3 +1154,44 @@ async def test_stream_prompt_tokens_details_zero_cached(): # Zero cached tokens must be present, not omitted assert usage_chunk["usage"]["prompt_tokens_details"] is not None assert usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"] == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("with_mask", [False, True]) +async def test_stream_sampling_mask_matches_each_token_chunk(with_mask): + engine = _mock_engine() + + async def generate(*args, **kwargs): + for position, tokens in enumerate(([10], [20, 30])): + result = _make_request_output( + "req-mask", + token_ids=list(tokens), + finish_reason="length" if position else None, + finished=bool(position), + ) + if with_mask: + result.outputs[0].sampling_mask = SamplingMask( + [[token, token + 1] for token in tokens] + ) + yield result + + engine.generate = MagicMock(side_effect=generate) + serving = _build_serving_tokens(engine) + response = await serving.serve_tokens( + GenerateRequest( + model=MODEL_NAME, token_ids=[1, 2, 3], sampling_params={}, stream=True + ) + ) + chunks = _parse_sse_chunks([chunk async for chunk in response]) + choices = [ + choice + for chunk in chunks + if isinstance(chunk, dict) + for choice in chunk.get("choices", []) + ] + assert [choice["token_ids"] for choice in choices] == [[10], [20, 30]] + for choice in choices: + expected = ( + [[token, token + 1] for token in choice["token_ids"]] if with_mask else None + ) + assert choice["sampling_mask"] == expected diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index 53b8e0f552..40a6883aa9 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -18,7 +18,7 @@ from tests.v1.engine.utils import ( from vllm import PoolingParams from vllm.logprobs import FlatLogprobs, Logprob, PromptLogprobs, SampleLogprobs from vllm.lora.request import LoRARequest -from vllm.outputs import CompletionOutput, RequestOutput +from vllm.outputs import CompletionOutput, RequestOutput, SamplingMask from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.v1.engine import ( @@ -1575,3 +1575,58 @@ def test_abort_requests(runner: str, abort_by: str, dummy_test_vectors): output_processor.abort_requests([request.request_id], internal=True) else: output_processor.abort_requests([request.external_req_id], internal=False) + + +@pytest.mark.parametrize("output_kind", list(RequestOutputKind)) +def test_sampling_masks_follow_output_kind(output_kind): + state = RequestState.__new__(RequestState) + state.detokenizer = MagicMock() + state.detokenizer.get_next_output_text.return_value = "" + state.logprobs_processor = MagicMock() + state.logprobs_processor.logprobs = None + state.logprobs_processor.cumulative_logprob = None + state.output_kind = output_kind + state.request_index = 0 + state.sampling_mask_chunks = [] + state.routed_experts_chunks = [] + state.spec_decode_metrics = None + + supports = [[10, 11], [20], [30, 31]] + masks = [] + for position, support in enumerate(supports): + state.sampling_mask_chunks.append(SamplingMaskLists(np.asarray(support))) + finish = FinishReason.LENGTH if position == len(supports) - 1 else None + mask = state._new_completion_output([support[0]], finish, None).sampling_mask + masks.append(mask.token_ids if mask is not None else None) + + if output_kind == RequestOutputKind.DELTA: + assert masks == [[support] for support in supports] + else: + assert masks == [None, None, supports] + + +def test_request_output_add_merges_delta_sampling_masks(): + def delta(token_ids, sampling_mask): + completion = CompletionOutput( + index=0, + text="", + token_ids=token_ids, + cumulative_logprob=None, + logprobs=None, + sampling_mask=sampling_mask, + ) + return RequestOutput( + request_id="request", + prompt=None, + prompt_token_ids=[1], + prompt_logprobs=None, + outputs=[completion], + finished=False, + ) + + merged = delta([], None) + merged.add(delta([10], SamplingMask([[10, 11]])), aggregate=True) + merged.add(delta([20], SamplingMask([[20]])), aggregate=True) + + assert merged.outputs[0].token_ids == [10, 20] + assert merged.outputs[0].sampling_mask.token_ids == [[10, 11], [20]] diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index ade103bbbc..249ed48093 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -515,6 +515,10 @@ class ServingTokens(GenerateBaseServing): else None ) + sampling_mask = None + if output.sampling_mask is not None: + sampling_mask = output.sampling_mask.token_ids + chunk = GenerateStreamResponse( request_id=request_id, choices=[ @@ -524,6 +528,7 @@ class ServingTokens(GenerateBaseServing): finish_reason=finish_reason, token_ids=as_list(delta_token_ids), routed_experts=routed_experts_b64, + sampling_mask=sampling_mask, ) ], ) diff --git a/vllm/outputs.py b/vllm/outputs.py index 049d795f78..8bbbbdf40c 100644 --- a/vllm/outputs.py +++ b/vllm/outputs.py @@ -197,6 +197,12 @@ class RequestOutput: if next_completion.logprobs: assert completion.logprobs is not None completion.logprobs.extend(next_completion.logprobs) # type: ignore[arg-type] + if next_completion.sampling_mask is not None: + if completion.sampling_mask is None: + completion.sampling_mask = SamplingMask([]) + completion.sampling_mask.token_ids.extend( + next_completion.sampling_mask.token_ids + ) completion.cumulative_logprob = ( next_completion.cumulative_logprob ) diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index e291b18806..89b3a9cf09 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -439,7 +439,7 @@ class RequestState: logprobs = logprobs[-num_new_tokens:] if num_new_tokens else logprobs[:0] sampling_mask = None - if finished and self.sampling_mask_chunks: + if (delta or finished) and self.sampling_mask_chunks: sampling_mask = SamplingMask( [ position @@ -447,6 +447,8 @@ class RequestState: for position in chunk.to_nested_list() ] ) + if delta: + self.sampling_mask_chunks.clear() # Concatenate routed experts on finish routed_experts = None