[Model Runner V2] Cache draft logits in model's LM head dtype (#50910)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -25,6 +25,12 @@ def _build_rejection_sample_inputs(
|
||||
temperature: float,
|
||||
num_trials: int,
|
||||
) -> dict:
|
||||
"""Build rejection_sample kwargs from a fixed target and draft distribution.
|
||||
|
||||
target_logits_1d must already have temperature applied (the sampler applies
|
||||
sampling params before verification), whereas draft_logits_1d must not:
|
||||
rejection_sample divides the draft logits by the temperature on load.
|
||||
"""
|
||||
device = target_logits_1d.device
|
||||
vocab_size = target_logits_1d.shape[0]
|
||||
K = num_speculative_steps
|
||||
@@ -35,7 +41,10 @@ def _build_rejection_sample_inputs(
|
||||
draft_logits_1d.view(1, 1, vocab_size).expand(num_trials, K, -1).contiguous()
|
||||
)
|
||||
|
||||
draft_probs = torch.softmax(draft_logits_1d, dim=0)
|
||||
scaled_draft_logits_1d = draft_logits_1d.float()
|
||||
if temperature > 0:
|
||||
scaled_draft_logits_1d = scaled_draft_logits_1d / temperature
|
||||
draft_probs = torch.softmax(scaled_draft_logits_1d, dim=0)
|
||||
draft_tokens = torch.multinomial(
|
||||
draft_probs.expand(num_trials, -1), K, replacement=True
|
||||
)
|
||||
@@ -138,7 +147,10 @@ def _assert_distribution_match(
|
||||
(3, 1.0),
|
||||
],
|
||||
)
|
||||
def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: float):
|
||||
@pytest.mark.parametrize("draft_logits_dtype", [torch.float32, torch.bfloat16])
|
||||
def test_stochastic_rejection_sample(
|
||||
num_speculative_steps: int, temperature: float, draft_logits_dtype: torch.dtype
|
||||
):
|
||||
"""
|
||||
Verify that rejection sampling produces the target distribution.
|
||||
This is done by simulating many independent trials of speculative
|
||||
@@ -146,6 +158,9 @@ def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: fl
|
||||
run rejection sample on all of the trials (requests), and verify
|
||||
that the sampled tokens at every position follow the target
|
||||
distribution p(x).
|
||||
|
||||
Parametrized over the draft-logits dtype: storing them in the draft head's
|
||||
dtype must not bias the output distribution.
|
||||
"""
|
||||
|
||||
torch.manual_seed(42)
|
||||
@@ -153,11 +168,12 @@ def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: fl
|
||||
num_trials = 10 * VOCAB_SIZE
|
||||
|
||||
target_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
|
||||
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
|
||||
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32).to(
|
||||
draft_logits_dtype
|
||||
)
|
||||
|
||||
if temperature > 0:
|
||||
target_logits_1d /= temperature
|
||||
draft_logits_1d /= temperature
|
||||
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
@@ -251,7 +267,6 @@ def test_synthetic_rejection_sample(
|
||||
|
||||
if temperature > 0:
|
||||
target_logits_1d /= temperature
|
||||
draft_logits_1d /= temperature
|
||||
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
@@ -334,7 +349,7 @@ def test_placeholder_draft_token_rejected():
|
||||
temperature = 0.6
|
||||
|
||||
target_logits_1d = torch.randn(VOCAB_SIZE, device=device) / temperature
|
||||
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device) / temperature
|
||||
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device)
|
||||
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
@@ -383,7 +398,6 @@ def test_block_verification_rejection_sample(
|
||||
|
||||
if temperature > 0:
|
||||
target_logits_1d /= temperature
|
||||
draft_logits_1d /= temperature
|
||||
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
|
||||
@@ -225,3 +225,69 @@ def test_vocab_size_not_multiple_of_block(vocab_size: int):
|
||||
df = vocab_size - 1
|
||||
if df >= 1:
|
||||
assert chi2 < df + 10 * math.sqrt(2 * df), f"chi2={chi2:.1f}, df={df}"
|
||||
|
||||
|
||||
# ----------------------------- Logits cache --------------------------------
|
||||
|
||||
|
||||
def _float_bits(t: torch.Tensor) -> torch.Tensor:
|
||||
"""Reinterpret floats as same-width ints, so equality is truly bitwise."""
|
||||
int_dtype = torch.int32 if t.dtype == torch.float32 else torch.int16
|
||||
return t.view(int_dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32])
|
||||
@pytest.mark.parametrize("per_token_col", [False, True])
|
||||
def test_logits_cache_stores_input_logits_bitwise(
|
||||
dtype: torch.dtype, per_token_col: bool
|
||||
):
|
||||
"""`logits_cache` must receive the input logits, pre-temperature and bit-exact.
|
||||
|
||||
The cache is kept in the head's dtype, which is only lossless because the
|
||||
stored value is the input logit itself. Storing `logit / temp` instead would
|
||||
generally not be representable there, and the rejection sampler -- which
|
||||
divides by the same temperature on load -- would then verify against a `q`
|
||||
the draft never sampled from. Temperatures 0.0 and 1.0 are included because
|
||||
both make the divide a no-op and would mask such a bug.
|
||||
|
||||
Both column-addressing modes are covered: a 0-d column (one draft step per
|
||||
call, as MTP/EAGLE do) and a [num_tokens] column (as DFlash does, sampling
|
||||
every step in one call).
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_reqs, vocab_size, num_steps = 8, 4099, 3
|
||||
logits = torch.randn(num_reqs, vocab_size, device=DEVICE, dtype=dtype)
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE)
|
||||
temp = torch.tensor(
|
||||
[0.0, 0.1, 0.5, 1.0, 1.5, 2.0, 0.7, 1.0], dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
seed = torch.arange(num_reqs, dtype=torch.int64, device=DEVICE)
|
||||
pos = torch.arange(num_reqs, dtype=torch.int64, device=DEVICE)
|
||||
|
||||
cache = torch.zeros(num_reqs, num_steps, vocab_size, device=DEVICE, dtype=dtype)
|
||||
if per_token_col:
|
||||
# Each token lands in its own column, cycling through the steps.
|
||||
cols = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE) % num_steps
|
||||
else:
|
||||
cols = torch.tensor(1, dtype=torch.int32, device=DEVICE)
|
||||
|
||||
gumbel_sample(
|
||||
logits,
|
||||
idx_mapping,
|
||||
temp,
|
||||
seed,
|
||||
pos,
|
||||
apply_temperature=True,
|
||||
logits_cache=cache,
|
||||
logits_cache_col=cols,
|
||||
)
|
||||
|
||||
if per_token_col:
|
||||
stored = cache[torch.arange(num_reqs, device=DEVICE), cols.long()]
|
||||
else:
|
||||
stored = cache[:, 1]
|
||||
# Untouched columns must stay untouched.
|
||||
assert not cache[:, 0].any() and not cache[:, 2].any()
|
||||
assert torch.equal(_float_bits(stored), _float_bits(logits)), (
|
||||
"cached logits differ from the input logits"
|
||||
)
|
||||
|
||||
@@ -91,9 +91,9 @@ def gumbel_block_argmax(
|
||||
temp_ptr,
|
||||
seeds_ptr,
|
||||
pos_ptr,
|
||||
processed_logits_ptr,
|
||||
processed_logits_stride,
|
||||
processed_logits_col_ptr,
|
||||
logits_cache_ptr,
|
||||
logits_cache_stride,
|
||||
logits_cache_col_ptr,
|
||||
vocab_size,
|
||||
APPLY_TEMPERATURE: tl.constexpr,
|
||||
USE_FP64: tl.constexpr,
|
||||
@@ -104,30 +104,30 @@ def gumbel_block_argmax(
|
||||
temp = tl.load(temp_ptr + req_state_idx, mask=is_valid_req, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
if temp != 0.0 and APPLY_TEMPERATURE:
|
||||
# Apply temperature.
|
||||
# NOTE(woosuk): Match the behavior of _temperature_kernel.
|
||||
# E.g., if the kernel uses tl.div_rn, we should use tl.div_rn here too.
|
||||
logits = logits / temp
|
||||
|
||||
if processed_logits_ptr is not None:
|
||||
# Store the temperature-applied logits.
|
||||
if processed_logits_col_ptr is not None:
|
||||
if PER_TOKEN_COL:
|
||||
col = tl.load(processed_logits_col_ptr + token_idx)
|
||||
else:
|
||||
col = tl.load(processed_logits_col_ptr)
|
||||
if logits_cache_ptr is not None:
|
||||
# Store the logits *before* temperature. Dividing first would produce a
|
||||
# value that is generally not representable in the cache's dtype, forcing
|
||||
# it to be fp32. Consumers (the rejection sampler) divide by the same
|
||||
# temperature on load, which reproduces the value used below bitwise.
|
||||
if PER_TOKEN_COL:
|
||||
col = tl.load(logits_cache_col_ptr + token_idx)
|
||||
else:
|
||||
col = 0
|
||||
col = tl.load(logits_cache_col_ptr)
|
||||
tl.store(
|
||||
processed_logits_ptr
|
||||
+ req_state_idx * processed_logits_stride
|
||||
logits_cache_ptr
|
||||
+ req_state_idx * logits_cache_stride
|
||||
+ col * vocab_size
|
||||
+ block,
|
||||
logits,
|
||||
mask=mask & is_valid_req,
|
||||
)
|
||||
|
||||
if temp != 0.0 and APPLY_TEMPERATURE:
|
||||
# Apply temperature.
|
||||
# NOTE(woosuk): Match the behavior of _temperature_kernel.
|
||||
# E.g., if the kernel uses tl.div_rn, we should use tl.div_rn here too.
|
||||
logits = logits / temp
|
||||
|
||||
# fp32 is the default reduction dtype; fp64 is ~1/32–1/64x the throughput
|
||||
# on H100/Ada/Blackwell and empirically indistinguishable for Gumbel-max.
|
||||
if USE_FP64:
|
||||
@@ -164,9 +164,9 @@ def _gumbel_sample_kernel(
|
||||
local_argmax_stride,
|
||||
local_max_ptr,
|
||||
local_max_stride,
|
||||
processed_logits_ptr,
|
||||
processed_logits_stride,
|
||||
processed_logits_col_ptr,
|
||||
logits_cache_ptr,
|
||||
logits_cache_stride,
|
||||
logits_cache_col_ptr,
|
||||
logits_ptr,
|
||||
logits_stride,
|
||||
expanded_idx_mapping_ptr,
|
||||
@@ -199,9 +199,9 @@ def _gumbel_sample_kernel(
|
||||
temp_ptr,
|
||||
seeds_ptr,
|
||||
pos_ptr,
|
||||
processed_logits_ptr,
|
||||
processed_logits_stride,
|
||||
processed_logits_col_ptr,
|
||||
logits_cache_ptr,
|
||||
logits_cache_stride,
|
||||
logits_cache_col_ptr,
|
||||
vocab_size,
|
||||
APPLY_TEMPERATURE=APPLY_TEMPERATURE,
|
||||
USE_FP64=USE_FP64,
|
||||
@@ -219,33 +219,30 @@ def gumbel_sample(
|
||||
seed: torch.Tensor, # [max_num_reqs]
|
||||
pos: torch.Tensor, # [num_tokens]
|
||||
apply_temperature: bool,
|
||||
output_processed_logits: torch.Tensor | None = None,
|
||||
output_processed_logits_col: torch.Tensor | None = None,
|
||||
logits_cache: torch.Tensor | None = None, # [max_num_reqs, num_cols, vocab_size]
|
||||
logits_cache_col: torch.Tensor | None = None, # scalar or [num_tokens]
|
||||
use_fp64: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# Enforce contiguity on non-strided input tensors
|
||||
expanded_idx_mapping = expanded_idx_mapping.contiguous()
|
||||
pos = pos.contiguous()
|
||||
if output_processed_logits_col is not None:
|
||||
output_processed_logits_col = output_processed_logits_col.contiguous()
|
||||
if logits_cache_col is not None:
|
||||
logits_cache_col = logits_cache_col.contiguous()
|
||||
num_tokens, vocab_size = logits.shape
|
||||
BLOCK_SIZE = 1024
|
||||
num_blocks = triton.cdiv(vocab_size, BLOCK_SIZE)
|
||||
local_argmax = logits.new_empty(num_tokens, num_blocks, dtype=torch.int64)
|
||||
local_max_dtype = torch.float64 if use_fp64 else torch.float32
|
||||
local_max = logits.new_empty(num_tokens, num_blocks, dtype=local_max_dtype)
|
||||
per_token_col = (
|
||||
output_processed_logits_col is not None
|
||||
and output_processed_logits_col.dim() > 0
|
||||
)
|
||||
per_token_col = logits_cache_col is not None and logits_cache_col.dim() > 0
|
||||
_gumbel_sample_kernel[(num_tokens, num_blocks)](
|
||||
local_argmax,
|
||||
local_argmax.stride(0),
|
||||
local_max,
|
||||
local_max.stride(0),
|
||||
output_processed_logits,
|
||||
output_processed_logits.stride(0) if output_processed_logits is not None else 0,
|
||||
output_processed_logits_col,
|
||||
logits_cache,
|
||||
logits_cache.stride(0) if logits_cache is not None else 0,
|
||||
logits_cache_col,
|
||||
logits,
|
||||
logits.stride(0),
|
||||
expanded_idx_mapping,
|
||||
|
||||
@@ -127,8 +127,8 @@ class DSparkSpeculator(DFlashSpeculator):
|
||||
self.seeds,
|
||||
sample_pos - 1,
|
||||
apply_temperature=True,
|
||||
output_processed_logits=self.draft_logits,
|
||||
output_processed_logits_col=self._step_cols[step],
|
||||
logits_cache=self.draft_logits,
|
||||
logits_cache_col=self._step_cols[step],
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
|
||||
|
||||
@@ -123,6 +123,7 @@ def _compute_global_logprobs_and_logsumexp(
|
||||
logit_idx,
|
||||
req_state_idx,
|
||||
draft_step,
|
||||
temp,
|
||||
# [num_logits, V]
|
||||
target_logits_ptr,
|
||||
target_logits_stride,
|
||||
@@ -160,14 +161,18 @@ def _compute_global_logprobs_and_logsumexp(
|
||||
)
|
||||
target_log_prob = target_logit - target_lse
|
||||
if HAS_DRAFT_LOGITS:
|
||||
draft_logit = tl.load(
|
||||
draft_logits_ptr
|
||||
+ req_state_idx * draft_logits_stride_0
|
||||
+ draft_step * draft_logits_stride_1
|
||||
+ token,
|
||||
mask=mask,
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
# draft_logits is stored pre-temperature, so apply scale first.
|
||||
draft_logit = (
|
||||
tl.load(
|
||||
draft_logits_ptr
|
||||
+ req_state_idx * draft_logits_stride_0
|
||||
+ draft_step * draft_logits_stride_1
|
||||
+ token,
|
||||
mask=mask,
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
/ temp
|
||||
)
|
||||
draft_lse = _compute_global_logsumexp(
|
||||
draft_local_max_ptr,
|
||||
draft_local_max_stride,
|
||||
@@ -272,15 +277,19 @@ def _compute_local_logits_stats_kernel(
|
||||
target_sumexp,
|
||||
)
|
||||
if HAS_DRAFT_LOGITS:
|
||||
# Get local draft max and summed exponentials.
|
||||
draft_logits = tl.load(
|
||||
draft_logits_ptr
|
||||
+ req_state_idx * draft_logits_stride_0
|
||||
+ draft_step_idx * draft_logits_stride_1
|
||||
+ block_offsets,
|
||||
mask=mask,
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
# Get local draft max and summed exponentials. draft_logits is
|
||||
# stored pre-temperature, so apply scale first.
|
||||
draft_logits = (
|
||||
tl.load(
|
||||
draft_logits_ptr
|
||||
+ req_state_idx * draft_logits_stride_0
|
||||
+ draft_step_idx * draft_logits_stride_1
|
||||
+ block_offsets,
|
||||
mask=mask,
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
/ temp
|
||||
)
|
||||
draft_max, draft_sumexp = _compute_max_and_sumexp(draft_logits)
|
||||
tl.store(
|
||||
draft_local_max_ptr + logit_idx * draft_local_max_stride + block_idx,
|
||||
@@ -348,6 +357,7 @@ def _compute_cumulative_log_p_kernel(
|
||||
logit_idx,
|
||||
req_state_idx,
|
||||
step,
|
||||
temp,
|
||||
target_logits_ptr,
|
||||
target_logits_stride,
|
||||
target_local_max_ptr,
|
||||
@@ -429,6 +439,7 @@ def _compute_local_residual_mass_kernel(
|
||||
logit_idx,
|
||||
req_state_idx,
|
||||
draft_step_idx,
|
||||
temp,
|
||||
target_logits_ptr,
|
||||
target_logits_stride,
|
||||
target_local_max_ptr,
|
||||
@@ -599,6 +610,7 @@ def _rejection_kernel(
|
||||
logit_idx,
|
||||
req_state_idx,
|
||||
i,
|
||||
temp,
|
||||
target_logits_ptr,
|
||||
target_logits_stride,
|
||||
target_local_max_ptr,
|
||||
@@ -724,14 +736,18 @@ def _resample_kernel(
|
||||
# Bonus token (no rejections). Directly use the target logits.
|
||||
residual_logits = target_logits
|
||||
elif HAS_DRAFT_LOGITS:
|
||||
draft_logits = tl.load(
|
||||
draft_logits_ptr
|
||||
+ req_state_idx * draft_logits_stride_0
|
||||
+ resample_idx * draft_logits_stride_1
|
||||
+ block,
|
||||
mask=mask,
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
# draft_logits is stored pre-temperature, so apply scale first.
|
||||
draft_logits = (
|
||||
tl.load(
|
||||
draft_logits_ptr
|
||||
+ req_state_idx * draft_logits_stride_0
|
||||
+ resample_idx * draft_logits_stride_1
|
||||
+ block,
|
||||
mask=mask,
|
||||
other=float("-inf"),
|
||||
).to(tl.float32)
|
||||
/ temp
|
||||
)
|
||||
target_lse = tl.load(target_rejected_logsumexp_ptr + req_idx)
|
||||
draft_lse = tl.load(draft_rejected_logsumexp_ptr + req_idx)
|
||||
target_log_probs = target_logits - target_lse
|
||||
@@ -784,9 +800,9 @@ def _resample_kernel(
|
||||
temp_ptr,
|
||||
seed_ptr,
|
||||
pos_ptr,
|
||||
None, # processed_logits_ptr
|
||||
0, # processed_logits_stride
|
||||
None, # processed_logits_col_ptr
|
||||
None, # logits_cache_ptr
|
||||
0, # logits_cache_stride
|
||||
None, # logits_cache_col_ptr
|
||||
vocab_size,
|
||||
APPLY_TEMPERATURE=False,
|
||||
USE_FP64=USE_FP64,
|
||||
|
||||
@@ -131,11 +131,12 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
|
||||
self.draft_logits: torch.Tensor | None = None
|
||||
if self.speculative_config.draft_sample_method == "probabilistic":
|
||||
# Pre-temperature logits, cached from the previous decode step.
|
||||
self.draft_logits = torch.zeros(
|
||||
self.max_num_reqs,
|
||||
self.num_speculative_steps,
|
||||
self.vocab_size,
|
||||
dtype=torch.float32,
|
||||
dtype=vllm_config.model_config.head_dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
@@ -332,8 +333,8 @@ class DraftModelSpeculator(BaseSpeculator):
|
||||
seeds,
|
||||
positions + 1,
|
||||
apply_temperature=True,
|
||||
output_processed_logits=draft_logits,
|
||||
output_processed_logits_col=draft_step,
|
||||
logits_cache=draft_logits,
|
||||
logits_cache_col=draft_step,
|
||||
use_fp64=self.use_fp64_gumbel,
|
||||
)
|
||||
return self._greedy_sample_draft(hidden_states)
|
||||
|
||||
Reference in New Issue
Block a user