[Kernel][Perf] Per-architecture tuned configs for batch-invariant persistent matmul (~3x decode kernels on RTX 4090D/H20) (#53247)
Signed-off-by: LioEinaudi <zhao3024667639@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
29c9af5211
commit
7797b6022c
@@ -12,6 +12,10 @@ import torch
|
||||
from utils import skip_unsupported
|
||||
|
||||
from vllm.model_executor.layers.batch_invariant import matmul_batch_invariant
|
||||
from vllm.model_executor.layers.batch_invariant_configs import (
|
||||
_BATCH_INVARIANT_MATMUL_TUNED_CONFIGS,
|
||||
_get_tuned_matmul_arch_family,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
@@ -103,3 +107,26 @@ def test_matmul_batch_invariance(dtype):
|
||||
batch_output_a = batch_output[3]
|
||||
|
||||
assert torch.equal(standard_output[0], batch_output_a)
|
||||
|
||||
|
||||
@skip_unsupported
|
||||
@pytest.mark.parametrize("m", [8, 32, 256, 2048])
|
||||
def test_matmul_batch_invariance_across_tuned_m_buckets(m):
|
||||
# Tuned M buckets must preserve each row's K-reduction order.
|
||||
capability = (
|
||||
current_platform.get_device_capability() if current_platform.is_cuda() else None
|
||||
)
|
||||
arch_family = _get_tuned_matmul_arch_family(capability)
|
||||
if arch_family not in _BATCH_INVARIANT_MATMUL_TUNED_CONFIGS:
|
||||
pytest.skip("No tuned persistent matmul config for this architecture")
|
||||
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
n = k = 2048
|
||||
torch.manual_seed(42)
|
||||
a = torch.rand((m, k), dtype=torch.bfloat16, device=device)
|
||||
b = torch.rand((k, n), dtype=torch.bfloat16, device=device)
|
||||
|
||||
single_output = matmul_batch_invariant(a[:1], b)
|
||||
batch_output = matmul_batch_invariant(a, b)
|
||||
|
||||
assert torch.equal(single_output[0], batch_output[0])
|
||||
|
||||
Reference in New Issue
Block a user