[Perf] Autotune batch invariance triton kernel in blackwell, 33.6% E2E latency reduction (#53649)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -177,6 +177,83 @@ _BATCH_INVARIANT_MATMUL_TUNED_CONFIGS: dict[
|
||||
),
|
||||
),
|
||||
},
|
||||
"blackwell": {
|
||||
(12288, 2048): _MatmulShapeConfig(
|
||||
block_k=64,
|
||||
m_buckets=(
|
||||
(1, _MatmulMConfig(16, 128, 8, 5)),
|
||||
(4, _MatmulMConfig(16, 128, 4, 5)),
|
||||
(8, _MatmulMConfig(16, 128, 4, 5)),
|
||||
(16, _MatmulMConfig(16, 128, 4, 5)),
|
||||
(32, _MatmulMConfig(32, 128, 4, 5)),
|
||||
(64, _MatmulMConfig(64, 128, 8, 5)),
|
||||
(256, _MatmulMConfig(128, 256, 4, 4)),
|
||||
(512, _MatmulMConfig(128, 256, 4, 4)),
|
||||
(1024, _MatmulMConfig(128, 256, 4, 4)),
|
||||
(2048, _MatmulMConfig(128, 256, 4, 4)),
|
||||
),
|
||||
),
|
||||
(2048, 6144): _MatmulShapeConfig(
|
||||
block_k=128,
|
||||
m_buckets=(
|
||||
(1, _MatmulMConfig(16, 32, 4, 5)),
|
||||
(4, _MatmulMConfig(16, 32, 4, 5)),
|
||||
(8, _MatmulMConfig(16, 32, 4, 5)),
|
||||
(16, _MatmulMConfig(16, 32, 4, 5)),
|
||||
(32, _MatmulMConfig(16, 32, 4, 5)),
|
||||
(64, _MatmulMConfig(32, 32, 4, 5)),
|
||||
(256, _MatmulMConfig(64, 64, 4, 5)),
|
||||
(512, _MatmulMConfig(128, 64, 4, 4)),
|
||||
(1024, _MatmulMConfig(128, 128, 8, 3)),
|
||||
(2048, _MatmulMConfig(128, 128, 8, 3)),
|
||||
),
|
||||
),
|
||||
(4096, 2048): _MatmulShapeConfig(
|
||||
block_k=128,
|
||||
m_buckets=(
|
||||
(1, _MatmulMConfig(16, 64, 4, 5)),
|
||||
(4, _MatmulMConfig(16, 64, 4, 5)),
|
||||
(8, _MatmulMConfig(16, 32, 4, 5)),
|
||||
(16, _MatmulMConfig(16, 32, 4, 5)),
|
||||
(32, _MatmulMConfig(32, 32, 4, 5)),
|
||||
(64, _MatmulMConfig(32, 64, 4, 4)),
|
||||
(256, _MatmulMConfig(128, 64, 4, 4)),
|
||||
(512, _MatmulMConfig(128, 128, 8, 3)),
|
||||
(1024, _MatmulMConfig(128, 128, 8, 3)),
|
||||
(2048, _MatmulMConfig(128, 128, 8, 3)),
|
||||
),
|
||||
),
|
||||
(151936, 2048): _MatmulShapeConfig(
|
||||
block_k=64,
|
||||
m_buckets=(
|
||||
(1, _MatmulMConfig(16, 256, 4, 5)),
|
||||
(4, _MatmulMConfig(16, 256, 4, 5)),
|
||||
(8, _MatmulMConfig(16, 256, 4, 5)),
|
||||
(16, _MatmulMConfig(16, 256, 4, 5)),
|
||||
(32, _MatmulMConfig(64, 256, 4, 5)),
|
||||
(64, _MatmulMConfig(64, 256, 4, 5)),
|
||||
(256, _MatmulMConfig(128, 256, 4, 4)),
|
||||
(512, _MatmulMConfig(128, 256, 4, 4)),
|
||||
(1024, _MatmulMConfig(128, 256, 4, 4)),
|
||||
(2048, _MatmulMConfig(128, 256, 4, 4)),
|
||||
),
|
||||
),
|
||||
(2048, 2048): _MatmulShapeConfig(
|
||||
block_k=128,
|
||||
m_buckets=(
|
||||
(1, _MatmulMConfig(16, 32, 4, 4)),
|
||||
(4, _MatmulMConfig(16, 32, 4, 4)),
|
||||
(8, _MatmulMConfig(16, 32, 4, 5)),
|
||||
(16, _MatmulMConfig(16, 32, 8, 5)),
|
||||
(32, _MatmulMConfig(16, 32, 4, 4)),
|
||||
(64, _MatmulMConfig(32, 32, 4, 5)),
|
||||
(256, _MatmulMConfig(64, 64, 4, 4)),
|
||||
(512, _MatmulMConfig(128, 64, 4, 4)),
|
||||
(1024, _MatmulMConfig(128, 128, 4, 3)),
|
||||
(2048, _MatmulMConfig(128, 128, 8, 3)),
|
||||
),
|
||||
),
|
||||
},
|
||||
}
|
||||
|
||||
_TUNED_MATMUL_CONFIGS_FOR_DEVICE: dict[tuple[int, int], _MatmulShapeConfig] | None = (
|
||||
@@ -188,6 +265,8 @@ _TUNED_MATMUL_CONFIGS_RESOLVED = False
|
||||
def _get_tuned_matmul_arch_family(capability: DeviceCapability | None) -> str | None:
|
||||
if capability is None:
|
||||
return None
|
||||
if capability.major == 10:
|
||||
return "blackwell"
|
||||
if capability.major == 9:
|
||||
return "hopper"
|
||||
if capability.major == 8 and capability.minor == 9:
|
||||
|
||||
Reference in New Issue
Block a user