[Model Runner V2] Cache draft logits in model's LM head dtype (#50910)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-08-05 19:12:46 -07:00
committed by GitHub
parent f85c1d2f84
commit 81bc196913
6 changed files with 170 additions and 76 deletions
@@ -25,6 +25,12 @@ def _build_rejection_sample_inputs(
temperature: float,
num_trials: int,
) -> dict:
"""Build rejection_sample kwargs from a fixed target and draft distribution.
target_logits_1d must already have temperature applied (the sampler applies
sampling params before verification), whereas draft_logits_1d must not:
rejection_sample divides the draft logits by the temperature on load.
"""
device = target_logits_1d.device
vocab_size = target_logits_1d.shape[0]
K = num_speculative_steps
@@ -35,7 +41,10 @@ def _build_rejection_sample_inputs(
draft_logits_1d.view(1, 1, vocab_size).expand(num_trials, K, -1).contiguous()
)
draft_probs = torch.softmax(draft_logits_1d, dim=0)
scaled_draft_logits_1d = draft_logits_1d.float()
if temperature > 0:
scaled_draft_logits_1d = scaled_draft_logits_1d / temperature
draft_probs = torch.softmax(scaled_draft_logits_1d, dim=0)
draft_tokens = torch.multinomial(
draft_probs.expand(num_trials, -1), K, replacement=True
)
@@ -138,7 +147,10 @@ def _assert_distribution_match(
(3, 1.0),
],
)
def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: float):
@pytest.mark.parametrize("draft_logits_dtype", [torch.float32, torch.bfloat16])
def test_stochastic_rejection_sample(
num_speculative_steps: int, temperature: float, draft_logits_dtype: torch.dtype
):
"""
Verify that rejection sampling produces the target distribution.
This is done by simulating many independent trials of speculative
@@ -146,6 +158,9 @@ def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: fl
run rejection sample on all of the trials (requests), and verify
that the sampled tokens at every position follow the target
distribution p(x).
Parametrized over the draft-logits dtype: storing them in the draft head's
dtype must not bias the output distribution.
"""
torch.manual_seed(42)
@@ -153,11 +168,12 @@ def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: fl
num_trials = 10 * VOCAB_SIZE
target_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32).to(
draft_logits_dtype
)
if temperature > 0:
target_logits_1d /= temperature
draft_logits_1d /= temperature
inputs = _build_rejection_sample_inputs(
target_logits_1d,
@@ -251,7 +267,6 @@ def test_synthetic_rejection_sample(
if temperature > 0:
target_logits_1d /= temperature
draft_logits_1d /= temperature
inputs = _build_rejection_sample_inputs(
target_logits_1d,
@@ -334,7 +349,7 @@ def test_placeholder_draft_token_rejected():
temperature = 0.6
target_logits_1d = torch.randn(VOCAB_SIZE, device=device) / temperature
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device) / temperature
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device)
inputs = _build_rejection_sample_inputs(
target_logits_1d,
@@ -383,7 +398,6 @@ def test_block_verification_rejection_sample(
if temperature > 0:
target_logits_1d /= temperature
draft_logits_1d /= temperature
inputs = _build_rejection_sample_inputs(
target_logits_1d,
+66
View File
@@ -225,3 +225,69 @@ def test_vocab_size_not_multiple_of_block(vocab_size: int):
df = vocab_size - 1
if df >= 1:
assert chi2 < df + 10 * math.sqrt(2 * df), f"chi2={chi2:.1f}, df={df}"
# ----------------------------- Logits cache --------------------------------
def _float_bits(t: torch.Tensor) -> torch.Tensor:
"""Reinterpret floats as same-width ints, so equality is truly bitwise."""
int_dtype = torch.int32 if t.dtype == torch.float32 else torch.int16
return t.view(int_dtype)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32])
@pytest.mark.parametrize("per_token_col", [False, True])
def test_logits_cache_stores_input_logits_bitwise(
dtype: torch.dtype, per_token_col: bool
):
"""`logits_cache` must receive the input logits, pre-temperature and bit-exact.
The cache is kept in the head's dtype, which is only lossless because the
stored value is the input logit itself. Storing `logit / temp` instead would
generally not be representable there, and the rejection sampler -- which
divides by the same temperature on load -- would then verify against a `q`
the draft never sampled from. Temperatures 0.0 and 1.0 are included because
both make the divide a no-op and would mask such a bug.
Both column-addressing modes are covered: a 0-d column (one draft step per
call, as MTP/EAGLE do) and a [num_tokens] column (as DFlash does, sampling
every step in one call).
"""
torch.manual_seed(0)
num_reqs, vocab_size, num_steps = 8, 4099, 3
logits = torch.randn(num_reqs, vocab_size, device=DEVICE, dtype=dtype)
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE)
temp = torch.tensor(
[0.0, 0.1, 0.5, 1.0, 1.5, 2.0, 0.7, 1.0], dtype=torch.float32, device=DEVICE
)
seed = torch.arange(num_reqs, dtype=torch.int64, device=DEVICE)
pos = torch.arange(num_reqs, dtype=torch.int64, device=DEVICE)
cache = torch.zeros(num_reqs, num_steps, vocab_size, device=DEVICE, dtype=dtype)
if per_token_col:
# Each token lands in its own column, cycling through the steps.
cols = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE) % num_steps
else:
cols = torch.tensor(1, dtype=torch.int32, device=DEVICE)
gumbel_sample(
logits,
idx_mapping,
temp,
seed,
pos,
apply_temperature=True,
logits_cache=cache,
logits_cache_col=cols,
)
if per_token_col:
stored = cache[torch.arange(num_reqs, device=DEVICE), cols.long()]
else:
stored = cache[:, 1]
# Untouched columns must stay untouched.
assert not cache[:, 0].any() and not cache[:, 2].any()
assert torch.equal(_float_bits(stored), _float_bits(logits)), (
"cached logits differ from the input logits"
)