From 81bc196913653c9e06b957e39a606002a09db171 Mon Sep 17 00:00:00 2001 From: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:12:46 -0700 Subject: [PATCH] [Model Runner V2] Cache draft logits in model's LM head dtype (#50910) Signed-off-by: Giancarlo Delfin --- .../test_rejection_sampler_utils.py | 28 ++++++-- tests/v1/worker/test_gpu_gumbel_sample.py | 66 +++++++++++++++++ vllm/v1/worker/gpu/sample/gumbel.py | 69 +++++++++--------- .../gpu/spec_decode/dspark/speculator.py | 4 +- .../spec_decode/rejection_sampler_utils.py | 72 +++++++++++-------- vllm/v1/worker/gpu/spec_decode/speculator.py | 7 +- 6 files changed, 170 insertions(+), 76 deletions(-) diff --git a/tests/v1/spec_decode/test_rejection_sampler_utils.py b/tests/v1/spec_decode/test_rejection_sampler_utils.py index 2663ab1496..0768380582 100644 --- a/tests/v1/spec_decode/test_rejection_sampler_utils.py +++ b/tests/v1/spec_decode/test_rejection_sampler_utils.py @@ -25,6 +25,12 @@ def _build_rejection_sample_inputs( temperature: float, num_trials: int, ) -> dict: + """Build rejection_sample kwargs from a fixed target and draft distribution. + + target_logits_1d must already have temperature applied (the sampler applies + sampling params before verification), whereas draft_logits_1d must not: + rejection_sample divides the draft logits by the temperature on load. + """ device = target_logits_1d.device vocab_size = target_logits_1d.shape[0] K = num_speculative_steps @@ -35,7 +41,10 @@ def _build_rejection_sample_inputs( draft_logits_1d.view(1, 1, vocab_size).expand(num_trials, K, -1).contiguous() ) - draft_probs = torch.softmax(draft_logits_1d, dim=0) + scaled_draft_logits_1d = draft_logits_1d.float() + if temperature > 0: + scaled_draft_logits_1d = scaled_draft_logits_1d / temperature + draft_probs = torch.softmax(scaled_draft_logits_1d, dim=0) draft_tokens = torch.multinomial( draft_probs.expand(num_trials, -1), K, replacement=True ) @@ -138,7 +147,10 @@ def _assert_distribution_match( (3, 1.0), ], ) -def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: float): +@pytest.mark.parametrize("draft_logits_dtype", [torch.float32, torch.bfloat16]) +def test_stochastic_rejection_sample( + num_speculative_steps: int, temperature: float, draft_logits_dtype: torch.dtype +): """ Verify that rejection sampling produces the target distribution. This is done by simulating many independent trials of speculative @@ -146,6 +158,9 @@ def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: fl run rejection sample on all of the trials (requests), and verify that the sampled tokens at every position follow the target distribution p(x). + + Parametrized over the draft-logits dtype: storing them in the draft head's + dtype must not bias the output distribution. """ torch.manual_seed(42) @@ -153,11 +168,12 @@ def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: fl num_trials = 10 * VOCAB_SIZE target_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32) - draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32) + draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32).to( + draft_logits_dtype + ) if temperature > 0: target_logits_1d /= temperature - draft_logits_1d /= temperature inputs = _build_rejection_sample_inputs( target_logits_1d, @@ -251,7 +267,6 @@ def test_synthetic_rejection_sample( if temperature > 0: target_logits_1d /= temperature - draft_logits_1d /= temperature inputs = _build_rejection_sample_inputs( target_logits_1d, @@ -334,7 +349,7 @@ def test_placeholder_draft_token_rejected(): temperature = 0.6 target_logits_1d = torch.randn(VOCAB_SIZE, device=device) / temperature - draft_logits_1d = torch.randn(VOCAB_SIZE, device=device) / temperature + draft_logits_1d = torch.randn(VOCAB_SIZE, device=device) inputs = _build_rejection_sample_inputs( target_logits_1d, @@ -383,7 +398,6 @@ def test_block_verification_rejection_sample( if temperature > 0: target_logits_1d /= temperature - draft_logits_1d /= temperature inputs = _build_rejection_sample_inputs( target_logits_1d, diff --git a/tests/v1/worker/test_gpu_gumbel_sample.py b/tests/v1/worker/test_gpu_gumbel_sample.py index 9db175113c..f64a3f1f24 100644 --- a/tests/v1/worker/test_gpu_gumbel_sample.py +++ b/tests/v1/worker/test_gpu_gumbel_sample.py @@ -225,3 +225,69 @@ def test_vocab_size_not_multiple_of_block(vocab_size: int): df = vocab_size - 1 if df >= 1: assert chi2 < df + 10 * math.sqrt(2 * df), f"chi2={chi2:.1f}, df={df}" + + +# ----------------------------- Logits cache -------------------------------- + + +def _float_bits(t: torch.Tensor) -> torch.Tensor: + """Reinterpret floats as same-width ints, so equality is truly bitwise.""" + int_dtype = torch.int32 if t.dtype == torch.float32 else torch.int16 + return t.view(int_dtype) + + +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32]) +@pytest.mark.parametrize("per_token_col", [False, True]) +def test_logits_cache_stores_input_logits_bitwise( + dtype: torch.dtype, per_token_col: bool +): + """`logits_cache` must receive the input logits, pre-temperature and bit-exact. + + The cache is kept in the head's dtype, which is only lossless because the + stored value is the input logit itself. Storing `logit / temp` instead would + generally not be representable there, and the rejection sampler -- which + divides by the same temperature on load -- would then verify against a `q` + the draft never sampled from. Temperatures 0.0 and 1.0 are included because + both make the divide a no-op and would mask such a bug. + + Both column-addressing modes are covered: a 0-d column (one draft step per + call, as MTP/EAGLE do) and a [num_tokens] column (as DFlash does, sampling + every step in one call). + """ + torch.manual_seed(0) + num_reqs, vocab_size, num_steps = 8, 4099, 3 + logits = torch.randn(num_reqs, vocab_size, device=DEVICE, dtype=dtype) + idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE) + temp = torch.tensor( + [0.0, 0.1, 0.5, 1.0, 1.5, 2.0, 0.7, 1.0], dtype=torch.float32, device=DEVICE + ) + seed = torch.arange(num_reqs, dtype=torch.int64, device=DEVICE) + pos = torch.arange(num_reqs, dtype=torch.int64, device=DEVICE) + + cache = torch.zeros(num_reqs, num_steps, vocab_size, device=DEVICE, dtype=dtype) + if per_token_col: + # Each token lands in its own column, cycling through the steps. + cols = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE) % num_steps + else: + cols = torch.tensor(1, dtype=torch.int32, device=DEVICE) + + gumbel_sample( + logits, + idx_mapping, + temp, + seed, + pos, + apply_temperature=True, + logits_cache=cache, + logits_cache_col=cols, + ) + + if per_token_col: + stored = cache[torch.arange(num_reqs, device=DEVICE), cols.long()] + else: + stored = cache[:, 1] + # Untouched columns must stay untouched. + assert not cache[:, 0].any() and not cache[:, 2].any() + assert torch.equal(_float_bits(stored), _float_bits(logits)), ( + "cached logits differ from the input logits" + ) diff --git a/vllm/v1/worker/gpu/sample/gumbel.py b/vllm/v1/worker/gpu/sample/gumbel.py index 190307d5e7..0a930b8183 100644 --- a/vllm/v1/worker/gpu/sample/gumbel.py +++ b/vllm/v1/worker/gpu/sample/gumbel.py @@ -91,9 +91,9 @@ def gumbel_block_argmax( temp_ptr, seeds_ptr, pos_ptr, - processed_logits_ptr, - processed_logits_stride, - processed_logits_col_ptr, + logits_cache_ptr, + logits_cache_stride, + logits_cache_col_ptr, vocab_size, APPLY_TEMPERATURE: tl.constexpr, USE_FP64: tl.constexpr, @@ -104,30 +104,30 @@ def gumbel_block_argmax( temp = tl.load(temp_ptr + req_state_idx, mask=is_valid_req, other=0.0).to( tl.float32 ) - if temp != 0.0 and APPLY_TEMPERATURE: - # Apply temperature. - # NOTE(woosuk): Match the behavior of _temperature_kernel. - # E.g., if the kernel uses tl.div_rn, we should use tl.div_rn here too. - logits = logits / temp - - if processed_logits_ptr is not None: - # Store the temperature-applied logits. - if processed_logits_col_ptr is not None: - if PER_TOKEN_COL: - col = tl.load(processed_logits_col_ptr + token_idx) - else: - col = tl.load(processed_logits_col_ptr) + if logits_cache_ptr is not None: + # Store the logits *before* temperature. Dividing first would produce a + # value that is generally not representable in the cache's dtype, forcing + # it to be fp32. Consumers (the rejection sampler) divide by the same + # temperature on load, which reproduces the value used below bitwise. + if PER_TOKEN_COL: + col = tl.load(logits_cache_col_ptr + token_idx) else: - col = 0 + col = tl.load(logits_cache_col_ptr) tl.store( - processed_logits_ptr - + req_state_idx * processed_logits_stride + logits_cache_ptr + + req_state_idx * logits_cache_stride + col * vocab_size + block, logits, mask=mask & is_valid_req, ) + if temp != 0.0 and APPLY_TEMPERATURE: + # Apply temperature. + # NOTE(woosuk): Match the behavior of _temperature_kernel. + # E.g., if the kernel uses tl.div_rn, we should use tl.div_rn here too. + logits = logits / temp + # fp32 is the default reduction dtype; fp64 is ~1/32–1/64x the throughput # on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max. if USE_FP64: @@ -164,9 +164,9 @@ def _gumbel_sample_kernel( local_argmax_stride, local_max_ptr, local_max_stride, - processed_logits_ptr, - processed_logits_stride, - processed_logits_col_ptr, + logits_cache_ptr, + logits_cache_stride, + logits_cache_col_ptr, logits_ptr, logits_stride, expanded_idx_mapping_ptr, @@ -199,9 +199,9 @@ def _gumbel_sample_kernel( temp_ptr, seeds_ptr, pos_ptr, - processed_logits_ptr, - processed_logits_stride, - processed_logits_col_ptr, + logits_cache_ptr, + logits_cache_stride, + logits_cache_col_ptr, vocab_size, APPLY_TEMPERATURE=APPLY_TEMPERATURE, USE_FP64=USE_FP64, @@ -219,33 +219,30 @@ def gumbel_sample( seed: torch.Tensor, # [max_num_reqs] pos: torch.Tensor, # [num_tokens] apply_temperature: bool, - output_processed_logits: torch.Tensor | None = None, - output_processed_logits_col: torch.Tensor | None = None, + logits_cache: torch.Tensor | None = None, # [max_num_reqs, num_cols, vocab_size] + logits_cache_col: torch.Tensor | None = None, # scalar or [num_tokens] use_fp64: bool = False, ) -> torch.Tensor: # Enforce contiguity on non-strided input tensors expanded_idx_mapping = expanded_idx_mapping.contiguous() pos = pos.contiguous() - if output_processed_logits_col is not None: - output_processed_logits_col = output_processed_logits_col.contiguous() + if logits_cache_col is not None: + logits_cache_col = logits_cache_col.contiguous() num_tokens, vocab_size = logits.shape BLOCK_SIZE = 1024 num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE) local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64) local_max_dtype = torch.float64 if use_fp64 else torch.float32 local_max = logits.new_empty(num_tokens, num_blocks, dtype=local_max_dtype) - per_token_col = ( - output_processed_logits_col is not None - and output_processed_logits_col.dim() > 0 - ) + per_token_col = logits_cache_col is not None and logits_cache_col.dim() > 0 _gumbel_sample_kernel[(num_tokens, num_blocks)]( local_argmax, local_argmax.stride(0), local_max, local_max.stride(0), - output_processed_logits, - output_processed_logits.stride(0) if output_processed_logits is not None else 0, - output_processed_logits_col, + logits_cache, + logits_cache.stride(0) if logits_cache is not None else 0, + logits_cache_col, logits, logits.stride(0), expanded_idx_mapping, diff --git a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py index a3207cd3da..7f48a3689f 100644 --- a/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/dspark/speculator.py @@ -127,8 +127,8 @@ class DSparkSpeculator(DFlashSpeculator): self.seeds, sample_pos - 1, apply_temperature=True, - output_processed_logits=self.draft_logits, - output_processed_logits_col=self._step_cols[step], + logits_cache=self.draft_logits, + logits_cache_col=self._step_cols[step], use_fp64=self.use_fp64_gumbel, ) diff --git a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py index 0ca3932a61..caab4c69a7 100644 --- a/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py +++ b/vllm/v1/worker/gpu/spec_decode/rejection_sampler_utils.py @@ -123,6 +123,7 @@ def _compute_global_logprobs_and_logsumexp( logit_idx, req_state_idx, draft_step, + temp, # [num_logits, V] target_logits_ptr, target_logits_stride, @@ -160,14 +161,18 @@ def _compute_global_logprobs_and_logsumexp( ) target_log_prob = target_logit - target_lse if HAS_DRAFT_LOGITS: - draft_logit = tl.load( - draft_logits_ptr - + req_state_idx * draft_logits_stride_0 - + draft_step * draft_logits_stride_1 - + token, - mask=mask, - other=float("-inf"), - ).to(tl.float32) + # draft_logits is stored pre-temperature, so apply scale first. + draft_logit = ( + tl.load( + draft_logits_ptr + + req_state_idx * draft_logits_stride_0 + + draft_step * draft_logits_stride_1 + + token, + mask=mask, + other=float("-inf"), + ).to(tl.float32) + / temp + ) draft_lse = _compute_global_logsumexp( draft_local_max_ptr, draft_local_max_stride, @@ -272,15 +277,19 @@ def _compute_local_logits_stats_kernel( target_sumexp, ) if HAS_DRAFT_LOGITS: - # Get local draft max and summed exponentials. - draft_logits = tl.load( - draft_logits_ptr - + req_state_idx * draft_logits_stride_0 - + draft_step_idx * draft_logits_stride_1 - + block_offsets, - mask=mask, - other=float("-inf"), - ).to(tl.float32) + # Get local draft max and summed exponentials. draft_logits is + # stored pre-temperature, so apply scale first. + draft_logits = ( + tl.load( + draft_logits_ptr + + req_state_idx * draft_logits_stride_0 + + draft_step_idx * draft_logits_stride_1 + + block_offsets, + mask=mask, + other=float("-inf"), + ).to(tl.float32) + / temp + ) draft_max, draft_sumexp = _compute_max_and_sumexp(draft_logits) tl.store( draft_local_max_ptr + logit_idx * draft_local_max_stride + block_idx, @@ -348,6 +357,7 @@ def _compute_cumulative_log_p_kernel( logit_idx, req_state_idx, step, + temp, target_logits_ptr, target_logits_stride, target_local_max_ptr, @@ -429,6 +439,7 @@ def _compute_local_residual_mass_kernel( logit_idx, req_state_idx, draft_step_idx, + temp, target_logits_ptr, target_logits_stride, target_local_max_ptr, @@ -599,6 +610,7 @@ def _rejection_kernel( logit_idx, req_state_idx, i, + temp, target_logits_ptr, target_logits_stride, target_local_max_ptr, @@ -724,14 +736,18 @@ def _resample_kernel( # Bonus token (no rejections). Directly use the target logits. residual_logits = target_logits elif HAS_DRAFT_LOGITS: - draft_logits = tl.load( - draft_logits_ptr - + req_state_idx * draft_logits_stride_0 - + resample_idx * draft_logits_stride_1 - + block, - mask=mask, - other=float("-inf"), - ).to(tl.float32) + # draft_logits is stored pre-temperature, so apply scale first. + draft_logits = ( + tl.load( + draft_logits_ptr + + req_state_idx * draft_logits_stride_0 + + resample_idx * draft_logits_stride_1 + + block, + mask=mask, + other=float("-inf"), + ).to(tl.float32) + / temp + ) target_lse = tl.load(target_rejected_logsumexp_ptr + req_idx) draft_lse = tl.load(draft_rejected_logsumexp_ptr + req_idx) target_log_probs = target_logits - target_lse @@ -784,9 +800,9 @@ def _resample_kernel( temp_ptr, seed_ptr, pos_ptr, - None, # processed_logits_ptr - 0, # processed_logits_stride - None, # processed_logits_col_ptr + None, # logits_cache_ptr + 0, # logits_cache_stride + None, # logits_cache_col_ptr vocab_size, APPLY_TEMPERATURE=False, USE_FP64=USE_FP64, diff --git a/vllm/v1/worker/gpu/spec_decode/speculator.py b/vllm/v1/worker/gpu/spec_decode/speculator.py index 71eb412ba2..69e0cc1091 100644 --- a/vllm/v1/worker/gpu/spec_decode/speculator.py +++ b/vllm/v1/worker/gpu/spec_decode/speculator.py @@ -131,11 +131,12 @@ class DraftModelSpeculator(BaseSpeculator): self.draft_logits: torch.Tensor | None = None if self.speculative_config.draft_sample_method == "probabilistic": + # Pre-temperature logits, cached from the previous decode step. self.draft_logits = torch.zeros( self.max_num_reqs, self.num_speculative_steps, self.vocab_size, - dtype=torch.float32, + dtype=vllm_config.model_config.head_dtype, device=device, ) @@ -332,8 +333,8 @@ class DraftModelSpeculator(BaseSpeculator): seeds, positions + 1, apply_temperature=True, - output_processed_logits=draft_logits, - output_processed_logits_col=draft_step, + logits_cache=draft_logits, + logits_cache_col=draft_step, use_fp64=self.use_fp64_gumbel, ) return self._greedy_sample_draft(hidden_states)