[Perf] Autotune batch invariance triton kernel in blackwell, 33.6% E2E latency reduction (#53649)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-08-25 15:06:01 -04:00
committed by GitHub
parent e6cc0899b6
commit 384d079697
2 changed files with 85 additions and 2 deletions
@@ -111,7 +111,8 @@ def test_matmul_batch_invariance(dtype):
@skip_unsupported
@pytest.mark.parametrize("m", [8, 32, 256, 2048])
def test_matmul_batch_invariance_across_tuned_m_buckets(m):
@pytest.mark.parametrize("transpose_b", [False, True], ids=["contiguous", "transposed"])
def test_matmul_batch_invariance_across_tuned_m_buckets(m, transpose_b):
# Tuned M buckets must preserve each row's K-reduction order.
capability = (
current_platform.get_device_capability() if current_platform.is_cuda() else None
@@ -124,7 +125,10 @@ def test_matmul_batch_invariance_across_tuned_m_buckets(m):
n = k = 2048
torch.manual_seed(42)
a = torch.rand((m, k), dtype=torch.bfloat16, device=device)
b = torch.rand((k, n), dtype=torch.bfloat16, device=device)
if transpose_b:
b = torch.rand((n, k), dtype=torch.bfloat16, device=device).t()
else:
b = torch.rand((k, n), dtype=torch.bfloat16, device=device)
single_output = matmul_batch_invariant(a[:1], b)
batch_output = matmul_batch_invariant(a, b)
@@ -177,6 +177,83 @@ _BATCH_INVARIANT_MATMUL_TUNED_CONFIGS: dict[
),
),
},
"blackwell": {
(12288, 2048): _MatmulShapeConfig(
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 128, 8, 5)),
(4, _MatmulMConfig(16, 128, 4, 5)),
(8, _MatmulMConfig(16, 128, 4, 5)),
(16, _MatmulMConfig(16, 128, 4, 5)),
(32, _MatmulMConfig(32, 128, 4, 5)),
(64, _MatmulMConfig(64, 128, 8, 5)),
(256, _MatmulMConfig(128, 256, 4, 4)),
(512, _MatmulMConfig(128, 256, 4, 4)),
(1024, _MatmulMConfig(128, 256, 4, 4)),
(2048, _MatmulMConfig(128, 256, 4, 4)),
),
),
(2048, 6144): _MatmulShapeConfig(
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 32, 4, 5)),
(4, _MatmulMConfig(16, 32, 4, 5)),
(8, _MatmulMConfig(16, 32, 4, 5)),
(16, _MatmulMConfig(16, 32, 4, 5)),
(32, _MatmulMConfig(16, 32, 4, 5)),
(64, _MatmulMConfig(32, 32, 4, 5)),
(256, _MatmulMConfig(64, 64, 4, 5)),
(512, _MatmulMConfig(128, 64, 4, 4)),
(1024, _MatmulMConfig(128, 128, 8, 3)),
(2048, _MatmulMConfig(128, 128, 8, 3)),
),
),
(4096, 2048): _MatmulShapeConfig(
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 64, 4, 5)),
(4, _MatmulMConfig(16, 64, 4, 5)),
(8, _MatmulMConfig(16, 32, 4, 5)),
(16, _MatmulMConfig(16, 32, 4, 5)),
(32, _MatmulMConfig(32, 32, 4, 5)),
(64, _MatmulMConfig(32, 64, 4, 4)),
(256, _MatmulMConfig(128, 64, 4, 4)),
(512, _MatmulMConfig(128, 128, 8, 3)),
(1024, _MatmulMConfig(128, 128, 8, 3)),
(2048, _MatmulMConfig(128, 128, 8, 3)),
),
),
(151936, 2048): _MatmulShapeConfig(
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 256, 4, 5)),
(4, _MatmulMConfig(16, 256, 4, 5)),
(8, _MatmulMConfig(16, 256, 4, 5)),
(16, _MatmulMConfig(16, 256, 4, 5)),
(32, _MatmulMConfig(64, 256, 4, 5)),
(64, _MatmulMConfig(64, 256, 4, 5)),
(256, _MatmulMConfig(128, 256, 4, 4)),
(512, _MatmulMConfig(128, 256, 4, 4)),
(1024, _MatmulMConfig(128, 256, 4, 4)),
(2048, _MatmulMConfig(128, 256, 4, 4)),
),
),
(2048, 2048): _MatmulShapeConfig(
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 32, 4, 4)),
(4, _MatmulMConfig(16, 32, 4, 4)),
(8, _MatmulMConfig(16, 32, 4, 5)),
(16, _MatmulMConfig(16, 32, 8, 5)),
(32, _MatmulMConfig(16, 32, 4, 4)),
(64, _MatmulMConfig(32, 32, 4, 5)),
(256, _MatmulMConfig(64, 64, 4, 4)),
(512, _MatmulMConfig(128, 64, 4, 4)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 8, 3)),
),
),
},
}
_TUNED_MATMUL_CONFIGS_FOR_DEVICE: dict[tuple[int, int], _MatmulShapeConfig] | None = (
@@ -188,6 +265,8 @@ _TUNED_MATMUL_CONFIGS_RESOLVED = False
def _get_tuned_matmul_arch_family(capability: DeviceCapability | None) -> str | None:
if capability is None:
return None
if capability.major == 10:
return "blackwell"
if capability.major == 9:
return "hopper"
if capability.major == 8 and capability.minor == 9: