From 50ba4bc6b2cacd70c4711a1904dd7ad9740a578b Mon Sep 17 00:00:00 2001 From: vx120 <57470515+vx120@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:44:31 +0800 Subject: [PATCH] [Feature] Mask Replay (#49577) Signed-off-by: vx120 <893600387@qq.com> Signed-off-by: vx120 <57470515+vx120@users.noreply.github.com> Signed-off-by: aoshen02 Co-authored-by: aoshen02 Co-authored-by: Codex Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: OpenAI Codex --- docs/training/sampling_mask.md | 113 +++++++++++++++++ .../engine-core-client/src/protocol/output.rs | 3 + .../engine-core-client/src/tests/client.rs | 14 +++ .../src/tests/python_compat.py | 25 ++++ .../token_in_token_out/test_serving_tokens.py | 44 +++++++ tests/v1/core/test_async_scheduler.py | 1 + tests/v1/core/test_scheduler.py | 1 + tests/v1/test_outputs.py | 73 +++++++++++ vllm/config/model.py | 3 + vllm/config/vllm.py | 27 ++++ vllm/engine/arg_utils.py | 6 + vllm/entrypoints/llm.py | 4 + .../scale_out/token_in_token_out/protocol.py | 2 + .../scale_out/token_in_token_out/serving.py | 5 + vllm/outputs.py | 15 +++ vllm/v1/core/sched/scheduler.py | 10 ++ vllm/v1/engine/__init__.py | 4 +- vllm/v1/engine/input_processor.py | 11 ++ vllm/v1/engine/output_processor.py | 13 ++ vllm/v1/outputs.py | 42 +++++++ vllm/v1/worker/gpu/async_utils.py | 12 +- vllm/v1/worker/gpu/model_runner.py | 1 + vllm/v1/worker/gpu/sample/output.py | 119 +++++++++++++++++- vllm/v1/worker/gpu/sample/sampler.py | 15 ++- 24 files changed, 557 insertions(+), 6 deletions(-) create mode 100644 docs/training/sampling_mask.md diff --git a/docs/training/sampling_mask.md b/docs/training/sampling_mask.md new file mode 100644 index 0000000000..442906e488 --- /dev/null +++ b/docs/training/sampling_mask.md @@ -0,0 +1,113 @@ +# Sampling Mask (Distribution Replay) + +When using top-k/top-p sampling for RL rollouts (e.g. GRPO), there is a +systematic mismatch between the truncated distribution the sampler actually +drew from and the full-vocabulary softmax used to compute log-probabilities +during training. The **sampling mask** feature closes this gap by returning +the exact set of token IDs that survived top-k/top-p/min-p filtering at each +generation step, so the training side can normalize over the same support. + +## Background + +This feature implements the **Keep Sampling Mask** strategy described in the +[DeepSeek-V3.2 technical report](https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/assets/paper.pdf) +(Section 3.3). The key insight: top-k/top-p truncation during rollout sampling +introduces a mismatch between the action spaces of `π_old` and `π_θ`, which +violates the principles of importance sampling and destabilizes training. By +preserving the truncation masks from `π_old` and applying them to `π_θ` during +training, both policies share identical action subspaces. DeepSeek reports that +combining top-p sampling with the Keep Sampling Mask strategy effectively +preserves language consistency during RL training. + +## Quick start + +```bash +vllm serve \ + --return-sampling-mask \ + --logprobs-mode processed_logprobs +``` + +```python +from vllm import LLM, SamplingParams + +llm = LLM(model, return_sampling_mask=True, + logprobs_mode="processed_logprobs") +output = llm.generate( + "The capital of France is", + SamplingParams(temperature=1.0, top_k=50, top_p=0.95, logprobs=1), +) +mask = output[0].outputs[0].sampling_mask +# mask.token_ids: [[187, 326, 512], [42, 88], ...] +# mask.token_ids[i] = token IDs in the sampling support for generated token i +``` + +The mask is also available via the `/inference/v1/generate` HTTP endpoint: + +```json +{ + "choices": [{ + "token_ids": [187, 42, 303], + "sampling_mask": [[187, 326, 512], [42, 88], [303, 11, 22]], + "finish_reason": "stop" + }] +} +``` + +## Requirements + +| Requirement | Reason | +| --- | --- | +| `--return-sampling-mask` | Engine-level opt-in (disables FlashInfer sampler) | +| `--logprobs-mode processed_logprobs` | Returned logprobs are normalized over the nucleus, not full vocab | +| `temperature > 0` | Greedy has no truncated distribution | +| `top_k > 0` | Bounds mask size; pure top-p can produce vocab-sized masks | +| Model Runner V2 | Required by the async D2H copy pipeline | + +The engine rejects unsupported combinations at startup or request time: + +- Speculative decoding +- Diffusion models +- Custom logits processors (engine-level `--logits-processors`) + +## How it works + +1. The sampler applies all logit processors (penalties, logit bias, bad words, + temperature, min-p) and then top-k/top-p filtering, which sets excluded + logits to `-inf`. +2. After sampling, `torch.isfinite(processed_logits)` identifies the surviving + token IDs — this is the sampling mask. +3. The mask is transferred GPU → CPU asynchronously alongside sampled tokens. +4. On request completion, per-step masks are merged and converted to + `list[list[int]]` for the response. + +## RL training usage + +The training side needs two things for the importance ratio `π_θ/π_old`: + +**`π_old(a|s)` — old policy's nucleus-normalized logprob:** +Already returned by vLLM when `--logprobs-mode processed_logprobs` is set. +The `log_softmax` is computed over processed logits (where filtered tokens +are `-inf`), so the denominator only includes the nucleus. + +**`π_θ(a|s)` — current policy's nucleus-normalized logprob:** +Computed by the training framework using the mask: + +```python +# mask_ids: list[int], the sampling support for this token +# logits: the training model's raw logits for this position +keep = torch.zeros(vocab_size, dtype=torch.bool) +keep[mask_ids] = True +masked_logits = logits.masked_fill(~keep, float("-inf")) +log_prob = log_softmax(masked_logits)[sampled_token_id] +``` + +Both sides normalize over the same token set, so the importance ratio is +consistent. + +## Limitations + +- **Engine-level flag:** `--return-sampling-mask` globally disables the + FlashInfer fused sampler. All requests pay the cost of the PyTorch sampling + path, even if they don't need the mask. +- **No streaming support:** The mask is returned only in the final response, + not in intermediate streaming chunks. diff --git a/rust/src/engine-core-client/src/protocol/output.rs b/rust/src/engine-core-client/src/protocol/output.rs index 8919e5fa69..a9695bd1cf 100644 --- a/rust/src/engine-core-client/src/protocol/output.rs +++ b/rust/src/engine-core-client/src/protocol/output.rs @@ -123,6 +123,8 @@ pub struct EngineCoreOutput { /// frontend grows one. #[serde(default)] pub mm_cache_miss_hashes: Option>, + #[serde(default)] + pub new_sampling_mask: Option, } impl EngineCoreOutput { @@ -440,6 +442,7 @@ mod tests { routed_experts: None, num_nans_in_logits: 0, mm_cache_miss_hashes: None, + new_sampling_mask: None, }, ], scheduler_stats: None, diff --git a/rust/src/engine-core-client/src/tests/client.rs b/rust/src/engine-core-client/src/tests/client.rs index ca29ba7c9b..1d8d6e37cc 100644 --- a/rust/src/engine-core-client/src/tests/client.rs +++ b/rust/src/engine-core-client/src/tests/client.rs @@ -2498,6 +2498,8 @@ fn python_msgpack_fixtures_match_rust_encoding() { let defaults_request_hex = lines.next().expect("missing defaults request fixture line"); let multimodal_request_hex = lines.next().expect("missing multimodal request fixture line"); let outputs_hex = lines.next().expect("missing outputs fixture line"); + let sampling_mask_outputs_hex = + lines.next().expect("missing sampling mask outputs fixture line"); let inline_logprobs_frames = lines.next().expect("missing inline logprobs fixture line"); let multipart_logprobs_frames = lines.next().expect("missing multipart logprobs fixture line"); let inline_prompt_frames = lines.next().expect("missing inline prompt logprobs fixture line"); @@ -2508,6 +2510,7 @@ fn python_msgpack_fixtures_match_rust_encoding() { let request_bytes = hex::decode(request_hex).unwrap(); let multimodal_request_bytes = hex::decode(multimodal_request_hex).unwrap(); let outputs_bytes = hex::decode(outputs_hex).unwrap(); + let sampling_mask_outputs_bytes = hex::decode(sampling_mask_outputs_hex).unwrap(); let decoded_request: EngineCoreRequest = rmp_serde::from_slice(&request_bytes).unwrap(); let expected_request = sample_request(); @@ -2571,6 +2574,16 @@ fn python_msgpack_fixtures_match_rust_encoding() { decode_value(&rmp_serde::to_vec_named(&expected_multimodal_request.mm_features).unwrap()); assert_eq!(python_mm_features, rust_mm_features); + let decoded_sampling_mask_outputs: EngineCoreOutputs = + rmp_serde::from_slice(&sampling_mask_outputs_bytes).unwrap(); + let sampling_mask_output = + &decoded_sampling_mask_outputs.as_request_batch().unwrap().outputs[0]; + assert!(sampling_mask_output.mm_cache_miss_hashes.is_none()); + assert!(matches!( + sampling_mask_output.new_sampling_mask.as_ref(), + Some(Value::Array(fields)) if fields.len() == 3 + )); + let decoded_outputs: EngineCoreOutputs = rmp_serde::from_slice(&outputs_bytes).unwrap(); expect_test::expect![[r#" RequestBatch( @@ -2598,6 +2611,7 @@ fn python_msgpack_fixtures_match_rust_encoding() { routed_experts: None, num_nans_in_logits: 0, mm_cache_miss_hashes: None, + new_sampling_mask: None, }, ], scheduler_stats: None, diff --git a/rust/src/engine-core-client/src/tests/python_compat.py b/rust/src/engine-core-client/src/tests/python_compat.py index e306c35625..7883c7150c 100755 --- a/rust/src/engine-core-client/src/tests/python_compat.py +++ b/rust/src/engine-core-client/src/tests/python_compat.py @@ -98,6 +98,7 @@ class EngineCoreOutput( routed_experts: object | None = None num_nans_in_logits: int = 0 mm_cache_miss_hashes: list[str] | None = None + new_sampling_mask: object | None = None class EngineCoreOutputs( @@ -202,6 +203,29 @@ outputs = EngineCoreOutputs( finished_requests={"req-1"}, ) +sampling_mask_wire = [ + [ + " None: + model_config = self.model_config + if model_config is None or not model_config.return_sampling_mask: + return + if not self.use_v2_model_runner: + raise ValueError("sampling distribution replay requires Model Runner V2") + if self.speculative_config is not None: + raise ValueError( + "sampling distribution replay does not support speculative decoding" + ) + if model_config.is_diffusion: + raise ValueError( + "sampling distribution replay does not support diffusion models" + ) + if model_config.logits_processors: + raise ValueError( + "sampling distribution replay does not support custom logits processors" + ) + if model_config.logprobs_mode != "processed_logprobs": + raise ValueError( + "sampling distribution replay requires " + "logprobs_mode='processed_logprobs' so that returned logprobs " + "are normalized over the same nucleus as the sampling mask" + ) + def __post_init__(self): """Verify configs are valid & consistent with each other.""" @@ -1080,6 +1105,8 @@ class VllmConfig: "connectors (PD disaggregation, KV cache offload)." ) + self._verify_sampling_replay_config() + if self.lora_config is not None: self.lora_config.verify_with_model_config(self.model_config) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index bd8b42dc27..4e4448a09c 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -426,6 +426,7 @@ class EngineArgs: model: str = ModelConfig.model enable_return_routed_experts: bool = ModelConfig.enable_return_routed_experts + return_sampling_mask: bool = ModelConfig.return_sampling_mask model_weights: str = ModelConfig.model_weights served_model_name: str | list[str] | None = ModelConfig.served_model_name tokenizer: str | None = ModelConfig.tokenizer @@ -874,6 +875,10 @@ class EngineArgs: "--enable-return-routed-experts", **model_kwargs["enable_return_routed_experts"], ) + model_group.add_argument( + "--return-sampling-mask", + **model_kwargs["return_sampling_mask"], + ) model_group.add_argument("--max-logprobs", **model_kwargs["max_logprobs"]) model_group.add_argument("--logprobs-mode", **model_kwargs["logprobs_mode"]) model_group.add_argument("--use-fp64-gumbel", **model_kwargs["use_fp64_gumbel"]) @@ -1748,6 +1753,7 @@ class EngineArgs: allow_deprecated_quantization=self.allow_deprecated_quantization, enforce_eager=self.enforce_eager, enable_return_routed_experts=self.enable_return_routed_experts, + return_sampling_mask=self.return_sampling_mask, max_logprobs=self.max_logprobs, logprobs_mode=self.logprobs_mode, use_fp64_gumbel=self.use_fp64_gumbel, diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 478aa80357..e3d0b2f1e5 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -163,6 +163,8 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): dictionary or an AttentionConfig instance. If a dictionary, it will be converted to an AttentionConfig. Allows specifying the attention backend and other attention-related settings. + return_sampling_mask: Return each sampled token's post-processing + support set. Requires Model Runner V2 and processed log probabilities. spec_method: Top-level alias for `speculative_config["method"]`. spec_model: Top-level alias for `speculative_config["model"]`. spec_tokens: Top-level alias for @@ -201,6 +203,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): offload_params: set[str] | None = None, enforce_eager: bool = False, enable_return_routed_experts: bool = False, + return_sampling_mask: bool = False, disable_custom_all_reduce: bool = False, hf_token: bool | str | None = None, hf_overrides: HfOverrides | None = None, @@ -317,6 +320,7 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): offload_params=offload_params or set(), enforce_eager=enforce_eager, enable_return_routed_experts=enable_return_routed_experts, + return_sampling_mask=return_sampling_mask, disable_custom_all_reduce=disable_custom_all_reduce, hf_token=hf_token, hf_overrides=hf_overrides, diff --git a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py index 1db05282ba..966a878528 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/protocol.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/protocol.py @@ -210,6 +210,7 @@ class GenerateResponseChoice(BaseModel): # ``None`` if (a) the request was aborted before any forward pass, # or (b) ``enable_return_routed_experts`` is off server-side. routed_experts: str | None = None + sampling_mask: list[list[int]] | None = None @field_validator("token_ids") @classmethod @@ -225,6 +226,7 @@ class GenerateResponseStreamChoice(BaseModel): finish_reason: str | None = None token_ids: list[int] | None = None routed_experts: str | None = None + sampling_mask: list[list[int]] | None = None class GenerateStreamResponse(BaseModel): diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index c5b0ffea69..7751873025 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -315,12 +315,17 @@ class ServingTokens(GenerateBaseServing): np.save(buf, output.routed_experts) routed_experts_b64 = base64.b64encode(buf.getvalue()).decode("ascii") + sampling_mask = None + if output.sampling_mask is not None: + sampling_mask = output.sampling_mask.token_ids + choice_data = GenerateResponseChoice( index=output.index, logprobs=logprobs, finish_reason=output.finish_reason if output.finish_reason else "stop", token_ids=as_list(output.token_ids), routed_experts=routed_experts_b64, + sampling_mask=sampling_mask, ) choices.append(choice_data) diff --git a/vllm/outputs.py b/vllm/outputs.py index feee2a9527..986a315da5 100644 --- a/vllm/outputs.py +++ b/vllm/outputs.py @@ -18,6 +18,17 @@ from vllm.v1.metrics.stats import RequestStateStats logger = init_logger(__name__) +@dataclass +class SamplingMask: + """Per-token sampling support sets aligned with completion token IDs. + + Each inner list contains the vocabulary token IDs that survived + top-k / top-p / min-p filtering for the corresponding generated token. + """ + + token_ids: list[list[int]] + + @dataclass class CompletionOutput: """The output data of one completion output of a request. @@ -30,6 +41,8 @@ class CompletionOutput: output text. logprobs: The log probabilities of the top probability words at each position if the logprobs are requested. + sampling_mask: The post-processing token support set for each generated + token, if requested. finish_reason: The reason why the sequence is finished. stop_reason: The stop string or token id that caused the completion to stop, None if the completion finished for some other reason @@ -46,6 +59,7 @@ class CompletionOutput: finish_reason: str | None = None stop_reason: int | str | None = None lora_request: LoRARequest | None = None + sampling_mask: SamplingMask | None = None def finished(self) -> bool: return self.finish_reason is not None @@ -56,6 +70,7 @@ class CompletionOutput: f"text={self.text!r}, " f"token_ids={self.token_ids}, " f"routed_experts={self.routed_experts}, " + f"sampling_mask={self.sampling_mask}, " f"cumulative_logprob={self.cumulative_logprob}, " f"logprobs={self.logprobs}, " f"finish_reason={self.finish_reason}, " diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 455289a479..9d14d00edc 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -326,6 +326,7 @@ class Scheduler(SchedulerInterface): self.enable_return_routed_experts = ( vllm_config.model_config.enable_return_routed_experts ) + self.return_sampling_mask = vllm_config.model_config.return_sampling_mask if self.enable_return_routed_experts: assert self.dcp_world_size == 1 and self.pcp_world_size == 1, ( @@ -1809,6 +1810,7 @@ class Scheduler(SchedulerInterface): stopped = False new_logprobs = None + new_sampling_mask = None new_token_ids = generated_token_ids pooler_output = pooler_outputs[req_index] if pooler_outputs else None kv_transfer_params = None @@ -1938,6 +1940,13 @@ class Scheduler(SchedulerInterface): ): new_logprobs = logprobs.slice_request(req_index, len(new_token_ids)) + if self.return_sampling_mask: + sampling_masks = model_runner_output.sampling_masks + if new_token_ids and sampling_masks is not None: + new_sampling_mask = sampling_masks.slice_request( + req_index, len(new_token_ids) + ) + if num_nans_in_logits is not None and req_id in num_nans_in_logits: request.num_nans_in_logits = num_nans_in_logits[req_id] @@ -1951,6 +1960,7 @@ class Scheduler(SchedulerInterface): new_token_ids=new_token_ids, finish_reason=finish_reason, new_logprobs=new_logprobs, + new_sampling_mask=new_sampling_mask, new_prompt_logprobs_tensors=prompt_logprobs_tensors, pooling_output=pooler_output, stop_reason=request.stop_reason, diff --git a/vllm/v1/engine/__init__.py b/vllm/v1/engine/__init__.py index cfe3ba8c28..d89c9a3f5d 100644 --- a/vllm/v1/engine/__init__.py +++ b/vllm/v1/engine/__init__.py @@ -17,7 +17,7 @@ from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams from vllm.v1.metrics.stats import PrefillStats, SchedulerStats -from vllm.v1.outputs import LogprobsLists, LogprobsTensors +from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplingMaskLists from vllm.v1.serial_utils import UtilityResult # Type for pause_generation mode parameter. @@ -217,6 +217,8 @@ class EngineCoreOutput( # so `array_like` positional serialization stays backward compatible. mm_cache_miss_hashes: list[str] | None = None + new_sampling_mask: SamplingMaskLists | None = None + @property def finished(self) -> bool: return self.finish_reason is not None diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 0e5995ee6f..686044a403 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -108,6 +108,17 @@ class InputProcessor: self.tokenizer, ) + if self.model_config.return_sampling_mask: + if params.temperature <= 0: + raise ValueError( + "sampling distribution replay requires temperature > 0" + ) + if params.top_k <= 0: + raise ValueError( + "sampling distribution replay requires top_k > 0 to " + "bound sampling mask size, reduce transfer overhead, " + "and avoid potential OOMs" + ) if params.thinking_token_budget is not None and ( self.vllm_config.reasoning_config is None or not self.vllm_config.reasoning_config.enabled diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index c467703759..99f60d5d5d 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -17,6 +17,7 @@ from vllm.outputs import ( PoolingOutput, PoolingRequestOutput, RequestOutput, + SamplingMask, ) from vllm.sampling_params import RequestOutputKind from vllm.tokenizers import TokenizerLike @@ -37,6 +38,7 @@ from vllm.v1.metrics.stats import ( RequestStateStats, SchedulerStats, ) +from vllm.v1.outputs import SamplingMaskLists # shared empty CPU tensor used as a placeholder pooling output EMPTY_CPU_TENSOR = torch.empty(0, device="cpu") @@ -178,6 +180,7 @@ class RequestState: # Routed experts accumulation (prompt + sample chunks) self.routed_experts_chunks: list[np.ndarray] = [] + self.sampling_mask_chunks: list[SamplingMaskLists] = [] # Stream Interval self.stream_interval = stream_interval @@ -406,6 +409,11 @@ class RequestState: if delta and logprobs: logprobs = logprobs[-len(token_ids) :] + sampling_mask = None + if finished and self.sampling_mask_chunks: + merged = SamplingMaskLists.merge(self.sampling_mask_chunks) + sampling_mask = SamplingMask(merged.to_nested_list()) + # Concatenate routed experts on finish routed_experts = None if finished and self.routed_experts_chunks: @@ -416,6 +424,7 @@ class RequestState: text=text, token_ids=token_ids, routed_experts=routed_experts, + sampling_mask=sampling_mask, logprobs=logprobs, cumulative_logprob=self.logprobs_processor.cumulative_logprob, finish_reason=str(finish_reason) if finished else None, @@ -652,6 +661,10 @@ class OutputProcessor: if pooling_output is None: assert req_state.detokenizer is not None assert req_state.logprobs_processor is not None + if engine_core_output.new_sampling_mask is not None: + req_state.sampling_mask_chunks.append( + engine_core_output.new_sampling_mask + ) # 2) Detokenize the token ids into text and perform stop checks. stop_string = req_state.detokenizer.update( new_token_ids, finish_reason == FinishReason.STOP diff --git a/vllm/v1/outputs.py b/vllm/v1/outputs.py index c8876e20b2..0bbee76675 100644 --- a/vllm/v1/outputs.py +++ b/vllm/v1/outputs.py @@ -50,6 +50,45 @@ class LogprobsLists(NamedTuple): ) +class SamplingMaskLists(NamedTuple): + # [num_kept_tokens] + token_ids: np.ndarray + # [num_generated_tokens + 1] + offsets: np.ndarray + # [num_reqs + 1] + cu_num_generated_tokens: list[int] | None = None + + def slice_request(self, req_idx: int, num_positions: int) -> "SamplingMaskLists": + if self.cu_num_generated_tokens is None: + start_idx = req_idx + else: + start_idx = self.cu_num_generated_tokens[req_idx] + end_idx = start_idx + num_positions + flat_start = self.offsets[start_idx] + flat_end = self.offsets[end_idx] + return SamplingMaskLists( + self.token_ids[flat_start:flat_end], + self.offsets[start_idx : end_idx + 1] - flat_start, + None, + ) + + def to_nested_list(self) -> list[list[int]]: + """Convert CSR representation to ``list[list[int]]``.""" + return [ + self.token_ids[int(self.offsets[i]) : int(self.offsets[i + 1])].tolist() + for i in range(len(self.offsets) - 1) + ] + + @staticmethod + def merge(chunks: Sequence["SamplingMaskLists"]) -> "SamplingMaskLists": + token_ids = np.concatenate([chunk.token_ids for chunk in chunks]) + counts = np.concatenate([np.diff(chunk.offsets) for chunk in chunks]) + offsets = np.empty(len(counts) + 1, dtype=np.int64) + offsets[0] = 0 + np.cumsum(counts, dtype=np.int64, out=offsets[1:]) + return SamplingMaskLists(token_ids, offsets) + + class LogprobsTensors(NamedTuple): # [num_reqs x num_generated_tokens, max_num_logprobs + 1] logprob_token_ids: torch.Tensor @@ -307,6 +346,9 @@ class ModelRunnerOutput: # ``None`` when ``enable_return_routed_experts`` is off. routed_experts: RoutedExpertsLists | None = None + # ``None`` when ``return_sampling_mask`` is off. + sampling_masks: SamplingMaskLists | None = None + @staticmethod def with_kv_conn_output_only( kv_connector_output: KVConnectorOutput | None, diff --git a/vllm/v1/worker/gpu/async_utils.py b/vllm/v1/worker/gpu/async_utils.py index 1a8f8f7a2d..ba0a66d2bc 100644 --- a/vllm/v1/worker/gpu/async_utils.py +++ b/vllm/v1/worker/gpu/async_utils.py @@ -17,7 +17,7 @@ from vllm.v1.outputs import ( PoolerOutput, RoutedExpertsTensors, ) -from vllm.v1.worker.gpu.sample.output import SamplerOutput +from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors from vllm.v1.worker.utils import raise_if_nan_logits if TYPE_CHECKING: @@ -147,6 +147,11 @@ class AsyncOutput(AsyncModelRunnerOutput): if sampler_output.num_nans is not None: self.num_nans = async_copy_to_np(sampler_output.num_nans) self.num_sampled_tokens_np = async_copy_to_np(num_sampled_tokens) + self.sampling_mask_tensors: SamplingMaskTensors | None = None + if sampler_output.sampling_mask_tensors is not None: + self.sampling_mask_tensors = ( + sampler_output.sampling_mask_tensors.to_cpu_nonblocking() + ) self.routed_experts_cpu: RoutedExpertsTensors | None = None if routed_experts is not None: self.routed_experts_cpu = routed_experts.to_cpu_nonblocking() @@ -172,6 +177,11 @@ class AsyncOutput(AsyncModelRunnerOutput): del token_ids[num_tokens:] self.model_runner_output.sampled_token_ids = sampled_token_ids + if self.sampling_mask_tensors is not None: + self.model_runner_output.sampling_masks = ( + self.sampling_mask_tensors.tolists(self.num_sampled_tokens_np) + ) + if self.num_nans is not None: self.model_runner_output.num_nans_in_logits = dict( zip(self.model_runner_output.req_ids, self.num_nans.tolist()) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 23a037bef9..4b53defa47 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -398,6 +398,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): num_speculative_tokens=self.decode_query_len, use_fp64_gumbel=self.model_config.use_fp64_gumbel, reasoning_config=self.vllm_config.reasoning_config, + return_sampling_mask=self.model_config.return_sampling_mask, ) custom = self.model_state.custom_sampler(self.sampler) diff --git a/vllm/v1/worker/gpu/sample/output.py b/vllm/v1/worker/gpu/sample/output.py index 130f4ddbf8..02e86cc657 100644 --- a/vllm/v1/worker/gpu/sample/output.py +++ b/vllm/v1/worker/gpu/sample/output.py @@ -1,10 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from dataclasses import dataclass +from __future__ import annotations +from dataclasses import dataclass +from typing import NamedTuple + +import numpy as np import torch -from vllm.v1.outputs import LogprobsTensors +from vllm.triton_utils import tl, triton +from vllm.v1.outputs import LogprobsTensors, SamplingMaskLists @dataclass @@ -14,3 +19,113 @@ class SamplerOutput: num_nans: torch.Tensor | None num_sampled: torch.Tensor | None num_rejected: torch.Tensor | None = None + sampling_mask_tensors: SamplingMaskTensors | None = None + + +@triton.jit +def _pack_sampling_mask_kernel( + logits_ptr, + logits_row_stride, + logits_col_stride, + num_sampled_tokens_ptr, + packed_mask_ptr, + packed_mask_row_stride, + counts_ptr, + vocab_size, + BLOCK_SIZE: tl.constexpr, +): + req_idx = tl.program_id(0) + is_active = tl.load(num_sampled_tokens_ptr + req_idx) > 0 + count = tl.zeros((), dtype=tl.int32) + + for start_idx in range(0, vocab_size, BLOCK_SIZE): + offsets = start_idx + tl.arange(0, BLOCK_SIZE) + valid = offsets < vocab_size + logits = tl.load( + logits_ptr + req_idx * logits_row_stride + offsets * logits_col_stride, + mask=valid, + other=-float("inf"), + ) + keep = (logits > -float("inf")) & (logits < float("inf")) & is_active + count += tl.sum(keep).to(tl.int32) + + keep = tl.reshape(keep.to(tl.int32), (BLOCK_SIZE // 8, 8)) + bit_shifts = tl.arange(0, 8)[None, :] + packed = tl.sum(keep << bit_shifts, axis=1).to(tl.uint8) + byte_offsets = start_idx // 8 + tl.arange(0, BLOCK_SIZE // 8) + tl.store( + packed_mask_ptr + req_idx * packed_mask_row_stride + byte_offsets, + packed, + mask=byte_offsets < tl.cdiv(vocab_size, 8), + ) + + tl.store(counts_ptr + req_idx, count) + + +class SamplingMaskTensors(NamedTuple): + """Bit-packed device-side sampling mask data pending async D2H.""" + + # [num_requests, ceil(vocab_size / 8)] + packed_mask: torch.Tensor + # [num_requests] + counts: torch.Tensor + vocab_size: int + + @classmethod + def from_logits( + cls, + logits: torch.Tensor, + num_sampled_tokens: torch.Tensor, + ) -> SamplingMaskTensors: + """Pack the finite-logit support for requests that sampled tokens.""" + num_reqs, vocab_size = logits.shape + packed_width = (vocab_size + 7) // 8 + + packed_mask = torch.empty( + (num_reqs, packed_width), dtype=torch.uint8, device=logits.device + ) + counts = torch.empty(num_reqs, dtype=torch.int32, device=logits.device) + _pack_sampling_mask_kernel[(num_reqs,)]( + logits, + logits.stride(0), + logits.stride(1), + num_sampled_tokens, + packed_mask, + packed_mask.stride(0), + counts, + vocab_size, + BLOCK_SIZE=8192, + ) + + return cls(packed_mask, counts, vocab_size) + + def to_cpu_nonblocking(self) -> SamplingMaskTensors: + if self.packed_mask.device.type == "cpu": + return self + return SamplingMaskTensors( + self.packed_mask.to("cpu", non_blocking=True), + self.counts.to("cpu", non_blocking=True), + self.vocab_size, + ) + + def tolists(self, num_sampled_tokens: np.ndarray) -> SamplingMaskLists: + """Convert the packed masks to the scheduler's CSR representation.""" + sampled_rows = np.flatnonzero(num_sampled_tokens) + counts = self.counts.cpu().numpy()[sampled_rows] + offsets = np.empty(len(counts) + 1, dtype=np.int64) + offsets[0] = 0 + np.cumsum(counts, dtype=np.int64, out=offsets[1:]) + unpacked = np.unpackbits( + self.packed_mask.cpu().numpy()[sampled_rows], + axis=1, + count=self.vocab_size, + bitorder="little", + ) + token_ids = np.nonzero(unpacked)[1].astype(np.int32, copy=False) + return SamplingMaskLists( + token_ids=token_ids, + offsets=offsets, + cu_num_generated_tokens=np.cumsum( + np.concatenate(([0], num_sampled_tokens)) + ).tolist(), + ) diff --git a/vllm/v1/worker/gpu/sample/sampler.py b/vllm/v1/worker/gpu/sample/sampler.py index 2f03355e5d..bb7dc40b02 100644 --- a/vllm/v1/worker/gpu/sample/sampler.py +++ b/vllm/v1/worker/gpu/sample/sampler.py @@ -22,7 +22,7 @@ from vllm.v1.worker.gpu.sample.logprob import ( LogprobTokenIdsState, compute_topk_scores, ) -from vllm.v1.worker.gpu.sample.output import SamplerOutput +from vllm.v1.worker.gpu.sample.output import SamplerOutput, SamplingMaskTensors from vllm.v1.worker.gpu.sample.penalties import PenaltiesState from vllm.v1.worker.gpu.sample.states import NO_LOGPROBS, SamplingStates from vllm.v1.worker.gpu.sample.thinking_budget import ThinkingBudgetState @@ -40,6 +40,7 @@ class Sampler: num_speculative_tokens: int = 1, use_fp64_gumbel: bool = False, reasoning_config: ReasoningConfig | None = None, + return_sampling_mask: bool = False, ): self.logprobs_mode = logprobs_mode self.compute_nans = envs.VLLM_COMPUTE_NANS_IN_LOGITS # False by default. @@ -53,7 +54,10 @@ class Sampler: self.logprob_token_ids_state = LogprobTokenIdsState(max_num_reqs, device) self.thinking_budget_state = ThinkingBudgetState(req_states, reasoning_config) self.num_speculative_tokens = num_speculative_tokens - self.use_flashinfer = flashinfer_sampler_supported() + self.return_sampling_mask = return_sampling_mask + self.use_flashinfer = ( + not return_sampling_mask and flashinfer_sampler_supported() + ) def add_request( self, req_idx: int, prompt_len: int, sampling_params: SamplingParams @@ -137,6 +141,12 @@ class Sampler: self.req_states.prefill_len.gpu, ) + sampling_mask_tensors = None + if self.return_sampling_mask: + sampling_mask_tensors = SamplingMaskTensors.from_logits( + processed_logits, num_sampled + ) + # These are GPU tensors. sampler_output = SamplerOutput( # The sampled tokens are expanded to 2D tensor with shape @@ -147,6 +157,7 @@ class Sampler: num_nans=num_nans, num_sampled=num_sampled, num_rejected=num_rejected, + sampling_mask_tensors=sampling_mask_tensors, ) return sampler_output