[Bugfix][CPU] Fix fused Gumbel-max wrapping-window bias (#60221)
Signed-off-by: Rehan Khan <Rehan.Khan7@ibm.com> Co-authored-by: Bob <bob@ibm.com>
This commit is contained in:
@@ -43,20 +43,20 @@ class TestGreedyArgmax:
|
||||
|
||||
|
||||
class TestFusedGumbelArgmax:
|
||||
@pytest.mark.parametrize("vocab_size", VOCAB_SIZES)
|
||||
def test_distribution_chi_squared(self, vocab_size: int):
|
||||
def test_distribution_chi_squared(self):
|
||||
"""Verify sampling distribution via chi-squared goodness of fit."""
|
||||
small_vocab = min(vocab_size, 100)
|
||||
# Small support is enough for chi-squared power; large-vocab bias
|
||||
# is covered by test_noise_not_sliding_window (#59786).
|
||||
small_vocab = 100
|
||||
logits = torch.randn(1, small_vocab, dtype=torch.float32)
|
||||
probs = logits.softmax(dim=-1).squeeze(0)
|
||||
|
||||
n_samples = 100_000
|
||||
counts = torch.zeros(small_vocab)
|
||||
for trial in range(n_samples):
|
||||
seed = torch.tensor([trial * 7 + 13], dtype=torch.long)
|
||||
tile = logits.expand(1, -1).contiguous()
|
||||
idx = torch.ops._C.fused_gumbel_argmax(tile, seed)
|
||||
counts[idx.item()] += 1
|
||||
seeds = torch.arange(n_samples, dtype=torch.long) * 7 + 13
|
||||
idx = torch.ops._C.fused_gumbel_argmax(
|
||||
logits.expand(n_samples, -1).contiguous(), seeds
|
||||
)
|
||||
counts = torch.bincount(idx, minlength=small_vocab).float()
|
||||
|
||||
expected = probs * n_samples
|
||||
mask = expected > 5
|
||||
@@ -70,6 +70,39 @@ class TestFusedGumbelArgmax:
|
||||
f"dof={dof}, p={p_value:.6f}"
|
||||
)
|
||||
|
||||
def test_noise_not_sliding_window(self):
|
||||
"""Table[(seed+i)&MASK] made consecutive seeds a 1-token shift.
|
||||
|
||||
On uniform logits that yields P(w(s+1) == w(s)-1) ~ 1-2/V. Independent
|
||||
per-token noise yields ~1/V. See #59786.
|
||||
"""
|
||||
vocab, n = 128, 2048
|
||||
logits = torch.zeros(n, vocab, dtype=torch.float32)
|
||||
seeds = torch.arange(n, dtype=torch.long)
|
||||
winners = torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
shifted = (winners[:-1] - 1) % vocab
|
||||
frac = (winners[1:] == shifted).float().mean().item()
|
||||
assert frac < 0.05, (
|
||||
f"consecutive-seed winners follow a table shift (frac={frac:.3f})"
|
||||
)
|
||||
|
||||
def test_zipf_head_matches_softmax(self):
|
||||
"""Zipf head probability should match softmax (#59786)."""
|
||||
vocab, n_samples, batch = 8192, 16384, 256
|
||||
logits = -1.1 * torch.arange(1, vocab + 1, dtype=torch.float32).log()
|
||||
p0 = torch.softmax(logits.double(), dim=-1)[0].item()
|
||||
row = logits.view(1, -1).expand(batch, vocab).contiguous()
|
||||
n_top = 0
|
||||
for start in range(0, n_samples, batch):
|
||||
seeds = torch.arange(start, start + batch, dtype=torch.long)
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(row, seeds)
|
||||
n_top += int((tokens == 0).sum())
|
||||
q0 = n_top / n_samples
|
||||
se = (p0 * (1.0 - p0) / n_samples) ** 0.5
|
||||
assert abs(q0 - p0) < 8.0 * se, (
|
||||
f"P(top-1) kernel={q0:.4f} exact={p0:.4f} se={se:.4f}"
|
||||
)
|
||||
|
||||
def test_deterministic_same_seed(self):
|
||||
"""Same seed produces same result."""
|
||||
logits = torch.randn(4, 32000, dtype=torch.float32)
|
||||
@@ -95,6 +128,108 @@ class TestFusedGumbelArgmax:
|
||||
assert result.min() >= 0
|
||||
assert result.max() < vocab_size
|
||||
|
||||
def test_samples_only_from_finite_logits(self):
|
||||
"""top-k/top-p mask with -inf; those IDs must never win."""
|
||||
allowed = torch.tensor([3, 17, 99, 1024, 8191], dtype=torch.long)
|
||||
logits = torch.full((64, 32000), float("-inf"), dtype=torch.float32)
|
||||
logits[:, allowed] = torch.randn(64, allowed.numel())
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(
|
||||
logits, torch.arange(64, dtype=torch.long)
|
||||
)
|
||||
assert torch.isin(tokens, allowed).all()
|
||||
|
||||
def test_single_unmasked_token(self):
|
||||
logits = torch.full((8, 4096), float("-inf"), dtype=torch.float32)
|
||||
logits[:, 123] = 0.0
|
||||
seeds = torch.tensor([0, -1, -(2**63), 2**63 - 1, 42, 99, 1000, 7])
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
assert (tokens == 123).all()
|
||||
|
||||
def test_tokens_beyond_gumbel_max_cannot_win(self):
|
||||
"""A gap larger than GUMBEL_MAX is unreachable."""
|
||||
logits = torch.zeros(256, 4, dtype=torch.float32)
|
||||
logits[:, 1:] = -80.0
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(
|
||||
logits, torch.arange(256, dtype=torch.long)
|
||||
)
|
||||
assert (tokens == 0).all()
|
||||
|
||||
def test_tied_huge_logits_not_index_biased(self):
|
||||
"""Noise must still break ties when logits sit at float32 max."""
|
||||
logits = torch.full(
|
||||
(4096, 2), torch.finfo(torch.float32).max, dtype=torch.float32
|
||||
)
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(
|
||||
logits, torch.arange(4096, dtype=torch.long)
|
||||
)
|
||||
frac0 = (tokens == 0).float().mean().item()
|
||||
assert set(tokens.tolist()) == {0, 1}, tokens.unique().tolist()
|
||||
assert 0.35 < frac0 < 0.65, f"P(index 0)={frac0:.3f} expected ~0.5"
|
||||
|
||||
def test_serial_batch1_matches_batched(self):
|
||||
"""OpenMP rows must match the batch=1 serial path."""
|
||||
logits = torch.randn(8, 2048, dtype=torch.float32)
|
||||
seeds = torch.arange(8, dtype=torch.long)
|
||||
batched = torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
serial = torch.stack(
|
||||
[
|
||||
torch.ops._C.fused_gumbel_argmax(logits[i : i + 1], seeds[i : i + 1])[0]
|
||||
for i in range(8)
|
||||
]
|
||||
)
|
||||
torch.testing.assert_close(batched, serial)
|
||||
|
||||
def test_noncontiguous_matches_contiguous(self):
|
||||
logits = torch.randn(8, 1024, dtype=torch.float32)[:, ::2]
|
||||
seeds = torch.arange(16, dtype=torch.long)[::2]
|
||||
assert not logits.is_contiguous() and not seeds.is_contiguous()
|
||||
expected = torch.ops._C.fused_gumbel_argmax(
|
||||
logits.contiguous(), seeds.contiguous()
|
||||
)
|
||||
actual = torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_empty_batch(self):
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(
|
||||
torch.empty(0, 7), torch.empty(0, dtype=torch.long)
|
||||
)
|
||||
assert tokens.shape == (0,) and tokens.dtype == torch.int64
|
||||
|
||||
def test_masked_support_matches_softmax(self):
|
||||
"""Finite subset after -inf mask should match softmax."""
|
||||
from scipy.stats import chisquare
|
||||
|
||||
logits = torch.tensor(
|
||||
[0.0, -0.2, -0.7, -1.5, -3.0, float("-inf")], dtype=torch.float32
|
||||
)
|
||||
n_samples = 20_000
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(
|
||||
logits.expand(n_samples, -1).contiguous(),
|
||||
torch.arange(n_samples, dtype=torch.long),
|
||||
)
|
||||
counts = torch.bincount(tokens, minlength=logits.numel()).double()
|
||||
assert counts[-1] == 0
|
||||
probs = logits.double().softmax(-1)
|
||||
_, p_value = chisquare(counts[:-1].numpy(), (probs[:-1] * n_samples).numpy())
|
||||
assert p_value > 1e-7, f"masked-support p-value: {p_value}"
|
||||
|
||||
def test_zipf_tail_matches_softmax(self):
|
||||
"""Independent 53-bit Gumbel must not starve the Zipf tail (#59786)."""
|
||||
vocab, n_samples, batch = 8192, 16384, 256
|
||||
logits = -1.1 * torch.arange(1, vocab + 1, dtype=torch.float32).log()
|
||||
p_tail = torch.softmax(logits.double(), dim=-1)[vocab // 2 :].sum().item()
|
||||
row = logits.view(1, -1).expand(batch, vocab).contiguous()
|
||||
n_tail = 0
|
||||
for start in range(0, n_samples, batch):
|
||||
seeds = torch.arange(start, start + batch, dtype=torch.long)
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(row, seeds)
|
||||
n_tail += int((tokens >= vocab // 2).sum())
|
||||
q_tail = n_tail / n_samples
|
||||
se = (p_tail * (1.0 - p_tail) / n_samples) ** 0.5
|
||||
assert abs(q_tail - p_tail) < 8.0 * se, (
|
||||
f"P(tail) kernel={q_tail:.4f} exact={p_tail:.4f} se={se:.4f}"
|
||||
)
|
||||
|
||||
|
||||
class TestMemory:
|
||||
def test_fused_no_intermediate_allocs(self):
|
||||
|
||||
Reference in New Issue
Block a user