[Model Runner V2] Cache draft logits in model's LM head dtype (#50910)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -25,6 +25,12 @@ def _build_rejection_sample_inputs(
|
||||
temperature: float,
|
||||
num_trials: int,
|
||||
) -> dict:
|
||||
"""Build rejection_sample kwargs from a fixed target and draft distribution.
|
||||
|
||||
target_logits_1d must already have temperature applied (the sampler applies
|
||||
sampling params before verification), whereas draft_logits_1d must not:
|
||||
rejection_sample divides the draft logits by the temperature on load.
|
||||
"""
|
||||
device = target_logits_1d.device
|
||||
vocab_size = target_logits_1d.shape[0]
|
||||
K = num_speculative_steps
|
||||
@@ -35,7 +41,10 @@ def _build_rejection_sample_inputs(
|
||||
draft_logits_1d.view(1, 1, vocab_size).expand(num_trials, K, -1).contiguous()
|
||||
)
|
||||
|
||||
draft_probs = torch.softmax(draft_logits_1d, dim=0)
|
||||
scaled_draft_logits_1d = draft_logits_1d.float()
|
||||
if temperature > 0:
|
||||
scaled_draft_logits_1d = scaled_draft_logits_1d / temperature
|
||||
draft_probs = torch.softmax(scaled_draft_logits_1d, dim=0)
|
||||
draft_tokens = torch.multinomial(
|
||||
draft_probs.expand(num_trials, -1), K, replacement=True
|
||||
)
|
||||
@@ -138,7 +147,10 @@ def _assert_distribution_match(
|
||||
(3, 1.0),
|
||||
],
|
||||
)
|
||||
def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: float):
|
||||
@pytest.mark.parametrize("draft_logits_dtype", [torch.float32, torch.bfloat16])
|
||||
def test_stochastic_rejection_sample(
|
||||
num_speculative_steps: int, temperature: float, draft_logits_dtype: torch.dtype
|
||||
):
|
||||
"""
|
||||
Verify that rejection sampling produces the target distribution.
|
||||
This is done by simulating many independent trials of speculative
|
||||
@@ -146,6 +158,9 @@ def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: fl
|
||||
run rejection sample on all of the trials (requests), and verify
|
||||
that the sampled tokens at every position follow the target
|
||||
distribution p(x).
|
||||
|
||||
Parametrized over the draft-logits dtype: storing them in the draft head's
|
||||
dtype must not bias the output distribution.
|
||||
"""
|
||||
|
||||
torch.manual_seed(42)
|
||||
@@ -153,11 +168,12 @@ def test_stochastic_rejection_sample(num_speculative_steps: int, temperature: fl
|
||||
num_trials = 10 * VOCAB_SIZE
|
||||
|
||||
target_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
|
||||
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32)
|
||||
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device, dtype=torch.float32).to(
|
||||
draft_logits_dtype
|
||||
)
|
||||
|
||||
if temperature > 0:
|
||||
target_logits_1d /= temperature
|
||||
draft_logits_1d /= temperature
|
||||
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
@@ -251,7 +267,6 @@ def test_synthetic_rejection_sample(
|
||||
|
||||
if temperature > 0:
|
||||
target_logits_1d /= temperature
|
||||
draft_logits_1d /= temperature
|
||||
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
@@ -334,7 +349,7 @@ def test_placeholder_draft_token_rejected():
|
||||
temperature = 0.6
|
||||
|
||||
target_logits_1d = torch.randn(VOCAB_SIZE, device=device) / temperature
|
||||
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device) / temperature
|
||||
draft_logits_1d = torch.randn(VOCAB_SIZE, device=device)
|
||||
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
@@ -383,7 +398,6 @@ def test_block_verification_rejection_sample(
|
||||
|
||||
if temperature > 0:
|
||||
target_logits_1d /= temperature
|
||||
draft_logits_1d /= temperature
|
||||
|
||||
inputs = _build_rejection_sample_inputs(
|
||||
target_logits_1d,
|
||||
|
||||
@@ -225,3 +225,69 @@ def test_vocab_size_not_multiple_of_block(vocab_size: int):
|
||||
df = vocab_size - 1
|
||||
if df >= 1:
|
||||
assert chi2 < df + 10 * math.sqrt(2 * df), f"chi2={chi2:.1f}, df={df}"
|
||||
|
||||
|
||||
# ----------------------------- Logits cache --------------------------------
|
||||
|
||||
|
||||
def _float_bits(t: torch.Tensor) -> torch.Tensor:
|
||||
"""Reinterpret floats as same-width ints, so equality is truly bitwise."""
|
||||
int_dtype = torch.int32 if t.dtype == torch.float32 else torch.int16
|
||||
return t.view(int_dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16, torch.float32])
|
||||
@pytest.mark.parametrize("per_token_col", [False, True])
|
||||
def test_logits_cache_stores_input_logits_bitwise(
|
||||
dtype: torch.dtype, per_token_col: bool
|
||||
):
|
||||
"""`logits_cache` must receive the input logits, pre-temperature and bit-exact.
|
||||
|
||||
The cache is kept in the head's dtype, which is only lossless because the
|
||||
stored value is the input logit itself. Storing `logit / temp` instead would
|
||||
generally not be representable there, and the rejection sampler -- which
|
||||
divides by the same temperature on load -- would then verify against a `q`
|
||||
the draft never sampled from. Temperatures 0.0 and 1.0 are included because
|
||||
both make the divide a no-op and would mask such a bug.
|
||||
|
||||
Both column-addressing modes are covered: a 0-d column (one draft step per
|
||||
call, as MTP/EAGLE do) and a [num_tokens] column (as DFlash does, sampling
|
||||
every step in one call).
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_reqs, vocab_size, num_steps = 8, 4099, 3
|
||||
logits = torch.randn(num_reqs, vocab_size, device=DEVICE, dtype=dtype)
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE)
|
||||
temp = torch.tensor(
|
||||
[0.0, 0.1, 0.5, 1.0, 1.5, 2.0, 0.7, 1.0], dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
seed = torch.arange(num_reqs, dtype=torch.int64, device=DEVICE)
|
||||
pos = torch.arange(num_reqs, dtype=torch.int64, device=DEVICE)
|
||||
|
||||
cache = torch.zeros(num_reqs, num_steps, vocab_size, device=DEVICE, dtype=dtype)
|
||||
if per_token_col:
|
||||
# Each token lands in its own column, cycling through the steps.
|
||||
cols = torch.arange(num_reqs, dtype=torch.int32, device=DEVICE) % num_steps
|
||||
else:
|
||||
cols = torch.tensor(1, dtype=torch.int32, device=DEVICE)
|
||||
|
||||
gumbel_sample(
|
||||
logits,
|
||||
idx_mapping,
|
||||
temp,
|
||||
seed,
|
||||
pos,
|
||||
apply_temperature=True,
|
||||
logits_cache=cache,
|
||||
logits_cache_col=cols,
|
||||
)
|
||||
|
||||
if per_token_col:
|
||||
stored = cache[torch.arange(num_reqs, device=DEVICE), cols.long()]
|
||||
else:
|
||||
stored = cache[:, 1]
|
||||
# Untouched columns must stay untouched.
|
||||
assert not cache[:, 0].any() and not cache[:, 2].any()
|
||||
assert torch.equal(_float_bits(stored), _float_bits(logits)), (
|
||||
"cached logits differ from the input logits"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user