[Feature] Support batch invariance on ROCm (#52231)

Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Matt
2026-10-12 10:53:26 -05:00
committed by GitHub
co-authored by Claude Opus 5.5 Andreas Karatzas
parent 50b21a2667
commit e0317b28ce
31 changed files with 1106 additions and 122 deletions
@@ -32,11 +32,9 @@ def torch_scaled_mm(
out = torch.mm(a.to(torch.float32), b.to(torch.float32))
out = scale_a * out
out = scale_b.T * out
out = out.to(out_dtype)
if bias is not None:
out = out + bias
return out
out = out + bias.float()
return out.to(out_dtype)
def get_8bit_types():
@@ -125,6 +123,25 @@ def test_scaled_mm(
torch.testing.assert_close(c_check, c_actual, rtol=1e-1, atol=1e-1)
@pytest.mark.parametrize("in_dtype", get_8bit_types())
@pytest.mark.parametrize("use_heuristic", [True, False])
def test_scaled_mm_adds_bias_before_rounding(in_dtype, use_heuristic):
# The accumulator is 1 + 3 * 2**-10 and the bias 2**-8. Rounding to bf16
# first gives 1.0 (1 + 2**-8 is a tie, to even); adding first, 1 + 2**-7.
a = torch.ones((1, 32), device=device).to(in_dtype)
col = torch.tensor([64.0] * 16 + [2.0, 1.0] + [0.0] * 14, device=device)
b = col[:, None].expand(32, 16).to(in_dtype).contiguous()
scale_a = torch.ones((1, 1), device=device)
scale_b = torch.full((1, 1), 2**-10, device=device)
bias = torch.full((16,), 2**-8, device=device, dtype=torch.bfloat16)
args = (a, b, scale_a, scale_b, torch.bfloat16, bias)
out = triton_scaled_mm(*args, use_heuristic=use_heuristic)
expected = torch_scaled_mm(*args)
assert (expected == 1 + 2**-7).all()
torch.testing.assert_close(out, expected, rtol=0, atol=0)
# TD operand loads must be bit-exact vs the plain masked-load path.
@pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),