[Model Runner V2] Cache draft logits in model's LM head dtype (#50910)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-08-05 19:12:46 -07:00
committed by GitHub
parent f85c1d2f84
commit 81bc196913
6 changed files with 170 additions and 76 deletions
@@ -25,6 +25,12 @@ def _build_rejection_sample_inputs(
temperature: float,
num_trials: int,
) -> dict:
"""Build rejection_sample kwargs from a fixed target and draft distribution.
target_logits_1d must already have temperature applied (the sampler applies
sampling params before verification), whereas draft_logits_1d must not:
rejection_sample divides the draft logits by the temperature on load.
"""
device = target_logits_1d.device
vocab_size = target_logits_1d.shape[0]
K = num_speculative_steps
@@ -35,7 +41,10 @@ def _build_rejection_sample_inputs(
draft_logits_1d.view(1, 1, vocab_size).expand(num_trials, K, -1).contiguous()
)
draft_probs = torch.softmax(draft_logits_1d, dim=0)
scaled_draft_logits_1d = draft_logits_1d.float()
if temperature > 0:
scaled_draft_logits_1d = scaled_draft_logits_1d / temperature
draft_probs = torch.softmax(scaled_draft_logits_1d, dim=0)
draft_tokens = torch.multinomial(
draft_probs.expand(num_trials, -1), K, replacement=True
)
@@ -138,7 +147,10 @@ def _assert_distribution_match(
(3, 1.0),
],
)
def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: float):
@pytest.mark.parametrize("draft_logits_dtype", [torch.float32, torch.bfloat16])
def test_stochastic_rejection_sample(
num_speculative_steps: int, temperature: float, draft_logits_dtype: torch.dtype
):
"""
Verify that rejection sampling produces the target distribution.
This is done by simulating many independent trials of speculative
@@ -146,6 +158,9 @@ def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: fl
run rejection sample on all of the trials (requests), and verify
that the sampled tokens at every position follow the target
distribution p(x).
Parametrized over the draft-logits dtype: storing them in the draft head's
dtype must not bias the output distribution.
"""
torch.manual_seed(42)
@@ -153,11 +168,12 @@ def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: fl
num_trials = 10 * VOCAB_SIZE
target_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32).to(
draft_logits_dtype
)
if temperature > 0:
target_logits_1d /= temperature
draft_logits_1d /= temperature
inputs = _build_rejection_sample_inputs(
target_logits_1d,
@@ -251,7 +267,6 @@ def test_synthetic_rejection_sample(
if temperature > 0:
target_logits_1d /= temperature
draft_logits_1d /= temperature
inputs = _build_rejection_sample_inputs(
target_logits_1d,
@@ -334,7 +349,7 @@ def test_placeholder_draft_token_rejected():
temperature = 0.6
target_logits_1d = torch.randn(VOCAB_SIZE, device=device) / temperature
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device) / temperature
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device)
inputs = _build_rejection_sample_inputs(
target_logits_1d,
@@ -383,7 +398,6 @@ def test_block_verification_rejection_sample(
if temperature > 0:
target_logits_1d /= temperature
draft_logits_1d /= temperature
inputs = _build_rejection_sample_inputs(
target_logits_1d,
+66
View File
@@ -225,3 +225,69 @@ def test_vocab_size_not_multiple_of_block(vocab_size: int):
df = vocab_size - 1
if df >= 1:
assert chi2 < df + 10 * math.sqrt(2 * df), f"chi2={chi2:.1f}, df={df}"
# ----------------------------- Logits cache --------------------------------
def _float_bits(t: torch.Tensor) -> torch.Tensor:
"""Reinterpret floats as same-width ints, so equality is truly bitwise."""
int_dtype = torch.int32 if t.dtype == torch.float32 else torch.int16
return t.view(int_dtype)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32])
@pytest.mark.parametrize("per_token_col", [False, True])
def test_logits_cache_stores_input_logits_bitwise(
dtype: torch.dtype, per_token_col: bool
):
"""`logits_cache` must receive the input logits, pre-temperature and bit-exact.
The cache is kept in the head's dtype, which is only lossless because the
stored value is the input logit itself. Storing `logit / temp` instead would
generally not be representable there, and the rejection sampler -- which
divides by the same temperature on load -- would then verify against a `q`
the draft never sampled from. Temperatures 0.0 and 1.0 are included because
both make the divide a no-op and would mask such a bug.
Both column-addressing modes are covered: a 0-d column (one draft step per
call, as MTP/EAGLE do) and a [num_tokens] column (as DFlash does, sampling
every step in one call).
"""
torch.manual_seed(0)
num_reqs, vocab_size, num_steps = 8, 4099, 3
logits = torch.randn(num_reqs, vocab_size, device=DEVICE, dtype=dtype)
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE)
temp = torch.tensor(
[0.0, 0.1, 0.5, 1.0, 1.5, 2.0, 0.7, 1.0], dtype=torch.float32, device=DEVICE
)
seed = torch.arange(num_reqs, dtype=torch.int64, device=DEVICE)
pos = torch.arange(num_reqs, dtype=torch.int64, device=DEVICE)
cache = torch.zeros(num_reqs, num_steps, vocab_size, device=DEVICE, dtype=dtype)
if per_token_col:
# Each token lands in its own column, cycling through the steps.
cols = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE) % num_steps
else:
cols = torch.tensor(1, dtype=torch.int32, device=DEVICE)
gumbel_sample(
logits,
idx_mapping,
temp,
seed,
pos,
apply_temperature=True,
logits_cache=cache,
logits_cache_col=cols,
)
if per_token_col:
stored = cache[torch.arange(num_reqs, device=DEVICE), cols.long()]
else:
stored = cache[:, 1]
# Untouched columns must stay untouched.
assert not cache[:, 0].any() and not cache[:, 2].any()
assert torch.equal(_float_bits(stored), _float_bits(logits)), (
"cached logits differ from the input logits"
)
+33 -36
View File
@@ -91,9 +91,9 @@ def gumbel_block_argmax(
temp_ptr,
seeds_ptr,
pos_ptr,
processed_logits_ptr,
processed_logits_stride,
processed_logits_col_ptr,
logits_cache_ptr,
logits_cache_stride,
logits_cache_col_ptr,
vocab_size,
APPLY_TEMPERATURE: tl.constexpr,
USE_FP64: tl.constexpr,
@@ -104,30 +104,30 @@ def gumbel_block_argmax(
temp = tl.load(temp_ptr + req_state_idx, mask=is_valid_req, other=0.0).to(
tl.float32
)
if temp != 0.0 and APPLY_TEMPERATURE:
# Apply temperature.
# NOTE(woosuk): Match the behavior of _temperature_kernel.
# E.g., if the kernel uses tl.div_rn, we should use tl.div_rn here too.
logits = logits / temp
if processed_logits_ptr is not None:
# Store the temperature-applied logits.
if processed_logits_col_ptr is not None:
if PER_TOKEN_COL:
col = tl.load(processed_logits_col_ptr + token_idx)
else:
col = tl.load(processed_logits_col_ptr)
if logits_cache_ptr is not None:
# Store the logits *before* temperature. Dividing first would produce a
# value that is generally not representable in the cache's dtype, forcing
# it to be fp32. Consumers (the rejection sampler) divide by the same
# temperature on load, which reproduces the value used below bitwise.
if PER_TOKEN_COL:
col = tl.load(logits_cache_col_ptr + token_idx)
else:
col = 0
col = tl.load(logits_cache_col_ptr)
tl.store(
processed_logits_ptr
+ req_state_idx * processed_logits_stride
logits_cache_ptr
+ req_state_idx * logits_cache_stride
+ col * vocab_size
+ block,
logits,
mask=mask & is_valid_req,
)
if temp != 0.0 and APPLY_TEMPERATURE:
# Apply temperature.
# NOTE(woosuk): Match the behavior of _temperature_kernel.
# E.g., if the kernel uses tl.div_rn, we should use tl.div_rn here too.
logits = logits / temp
# fp32 is the default reduction dtype; fp64 is ~1/32–1/64x the throughput
# on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max.
if USE_FP64:
@@ -164,9 +164,9 @@ def _gumbel_sample_kernel(
local_argmax_stride,
local_max_ptr,
local_max_stride,
processed_logits_ptr,
processed_logits_stride,
processed_logits_col_ptr,
logits_cache_ptr,
logits_cache_stride,
logits_cache_col_ptr,
logits_ptr,
logits_stride,
expanded_idx_mapping_ptr,
@@ -199,9 +199,9 @@ def _gumbel_sample_kernel(
temp_ptr,
seeds_ptr,
pos_ptr,
processed_logits_ptr,
processed_logits_stride,
processed_logits_col_ptr,
logits_cache_ptr,
logits_cache_stride,
logits_cache_col_ptr,
vocab_size,
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
USE_FP64=USE_FP64,
@@ -219,33 +219,30 @@ def gumbel_sample(
seed: torch.Tensor, # [max_num_reqs]
pos: torch.Tensor, # [num_tokens]
apply_temperature: bool,
output_processed_logits: torch.Tensor | None = None,
output_processed_logits_col: torch.Tensor | None = None,
logits_cache: torch.Tensor | None = None, # [max_num_reqs, num_cols, vocab_size]
logits_cache_col: torch.Tensor | None = None, # scalar or [num_tokens]
use_fp64: bool = False,
) -> torch.Tensor:
# Enforce contiguity on non-strided input tensors
expanded_idx_mapping = expanded_idx_mapping.contiguous()
pos = pos.contiguous()
if output_processed_logits_col is not None:
output_processed_logits_col = output_processed_logits_col.contiguous()
if logits_cache_col is not None:
logits_cache_col = logits_cache_col.contiguous()
num_tokens, vocab_size = logits.shape
BLOCK_SIZE = 1024
num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE)
local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64)
local_max_dtype = torch.float64 if use_fp64 else torch.float32
local_max = logits.new_empty(num_tokens, num_blocks, dtype=local_max_dtype)
per_token_col = (
output_processed_logits_col is not None
and output_processed_logits_col.dim() > 0
)
per_token_col = logits_cache_col is not None and logits_cache_col.dim() > 0
_gumbel_sample_kernel[(num_tokens, num_blocks)](
local_argmax,
local_argmax.stride(0),
local_max,
local_max.stride(0),
output_processed_logits,
output_processed_logits.stride(0) if output_processed_logits is not None else 0,
output_processed_logits_col,
logits_cache,
logits_cache.stride(0) if logits_cache is not None else 0,
logits_cache_col,
logits,
logits.stride(0),
expanded_idx_mapping,
@@ -127,8 +127,8 @@ class DSparkSpeculator(DFlashSpeculator):
self.seeds,
sample_pos - 1,
apply_temperature=True,
output_processed_logits=self.draft_logits,
output_processed_logits_col=self._step_cols[step],
logits_cache=self.draft_logits,
logits_cache_col=self._step_cols[step],
use_fp64=self.use_fp64_gumbel,
)
@@ -123,6 +123,7 @@ def _compute_global_logprobs_and_logsumexp(
logit_idx,
req_state_idx,
draft_step,
temp,
# [num_logits, V]
target_logits_ptr,
target_logits_stride,
@@ -160,14 +161,18 @@ def _compute_global_logprobs_and_logsumexp(
)
target_log_prob = target_logit - target_lse
if HAS_DRAFT_LOGITS:
draft_logit = tl.load(
draft_logits_ptr
+ req_state_idx * draft_logits_stride_0
+ draft_step * draft_logits_stride_1
+ token,
mask=mask,
other=float("-inf"),
).to(tl.float32)
# draft_logits is stored pre-temperature, so apply scale first.
draft_logit = (
tl.load(
draft_logits_ptr
+ req_state_idx * draft_logits_stride_0
+ draft_step * draft_logits_stride_1
+ token,
mask=mask,
other=float("-inf"),
).to(tl.float32)
/ temp
)
draft_lse = _compute_global_logsumexp(
draft_local_max_ptr,
draft_local_max_stride,
@@ -272,15 +277,19 @@ def _compute_local_logits_stats_kernel(
target_sumexp,
)
if HAS_DRAFT_LOGITS:
# Get local draft max and summed exponentials.
draft_logits = tl.load(
draft_logits_ptr
+ req_state_idx * draft_logits_stride_0
+ draft_step_idx * draft_logits_stride_1
+ block_offsets,
mask=mask,
other=float("-inf"),
).to(tl.float32)
# Get local draft max and summed exponentials. draft_logits is
# stored pre-temperature, so apply scale first.
draft_logits = (
tl.load(
draft_logits_ptr
+ req_state_idx * draft_logits_stride_0
+ draft_step_idx * draft_logits_stride_1
+ block_offsets,
mask=mask,
other=float("-inf"),
).to(tl.float32)
/ temp
)
draft_max, draft_sumexp = _compute_max_and_sumexp(draft_logits)
tl.store(
draft_local_max_ptr + logit_idx * draft_local_max_stride + block_idx,
@@ -348,6 +357,7 @@ def _compute_cumulative_log_p_kernel(
logit_idx,
req_state_idx,
step,
temp,
target_logits_ptr,
target_logits_stride,
target_local_max_ptr,
@@ -429,6 +439,7 @@ def _compute_local_residual_mass_kernel(
logit_idx,
req_state_idx,
draft_step_idx,
temp,
target_logits_ptr,
target_logits_stride,
target_local_max_ptr,
@@ -599,6 +610,7 @@ def _rejection_kernel(
logit_idx,
req_state_idx,
i,
temp,
target_logits_ptr,
target_logits_stride,
target_local_max_ptr,
@@ -724,14 +736,18 @@ def _resample_kernel(
# Bonus token (no rejections). Directly use the target logits.
residual_logits = target_logits
elif HAS_DRAFT_LOGITS:
draft_logits = tl.load(
draft_logits_ptr
+ req_state_idx * draft_logits_stride_0
+ resample_idx * draft_logits_stride_1
+ block,
mask=mask,
other=float("-inf"),
).to(tl.float32)
# draft_logits is stored pre-temperature, so apply scale first.
draft_logits = (
tl.load(
draft_logits_ptr
+ req_state_idx * draft_logits_stride_0
+ resample_idx * draft_logits_stride_1
+ block,
mask=mask,
other=float("-inf"),
).to(tl.float32)
/ temp
)
target_lse = tl.load(target_rejected_logsumexp_ptr + req_idx)
draft_lse = tl.load(draft_rejected_logsumexp_ptr + req_idx)
target_log_probs = target_logits - target_lse
@@ -784,9 +800,9 @@ def _resample_kernel(
temp_ptr,
seed_ptr,
pos_ptr,
None, # processed_logits_ptr
0, # processed_logits_stride
None, # processed_logits_col_ptr
None, # logits_cache_ptr
0, # logits_cache_stride
None, # logits_cache_col_ptr
vocab_size,
APPLY_TEMPERATURE=False,
USE_FP64=USE_FP64,
+4 -3
View File
@@ -131,11 +131,12 @@ class DraftModelSpeculator(BaseSpeculator):
self.draft_logits: torch.Tensor | None = None
if self.speculative_config.draft_sample_method == "probabilistic":
# Pre-temperature logits, cached from the previous decode step.
self.draft_logits = torch.zeros(
self.max_num_reqs,
self.num_speculative_steps,
self.vocab_size,
dtype=torch.float32,
dtype=vllm_config.model_config.head_dtype,
device=device,
)
@@ -332,8 +333,8 @@ class DraftModelSpeculator(BaseSpeculator):
seeds,
positions + 1,
apply_temperature=True,
output_processed_logits=draft_logits,
output_processed_logits_col=draft_step,
logits_cache=draft_logits,
logits_cache_col=draft_step,
use_fp64=self.use_fp64_gumbel,
)
return self._greedy_sample_draft(hidden_states)