diff --git a/tests/v1/determinism/test_matmul_batch_invariant.py b/tests/v1/determinism/test_matmul_batch_invariant.py index a162511e9e..086490526e 100644 --- a/tests/v1/determinism/test_matmul_batch_invariant.py +++ b/tests/v1/determinism/test_matmul_batch_invariant.py @@ -111,7 +111,8 @@ def test_matmul_batch_invariance(dtype): @skip_unsupported @pytest.mark.parametrize("m", [8, 32, 256, 2048]) -def test_matmul_batch_invariance_across_tuned_m_buckets(m): +@pytest.mark.parametrize("transpose_b", [False, True], ids=["contiguous", "transposed"]) +def test_matmul_batch_invariance_across_tuned_m_buckets(m, transpose_b): # Tuned M buckets must preserve each row's K-reduction order. capability = ( current_platform.get_device_capability() if current_platform.is_cuda() else None @@ -124,7 +125,10 @@ def test_matmul_batch_invariance_across_tuned_m_buckets(m): n = k = 2048 torch.manual_seed(42) a = torch.rand((m, k), dtype=torch.bfloat16, device=device) - b = torch.rand((k, n), dtype=torch.bfloat16, device=device) + if transpose_b: + b = torch.rand((n, k), dtype=torch.bfloat16, device=device).t() + else: + b = torch.rand((k, n), dtype=torch.bfloat16, device=device) single_output = matmul_batch_invariant(a[:1], b) batch_output = matmul_batch_invariant(a, b) diff --git a/vllm/model_executor/determinism/batch_invariant_configs.py b/vllm/model_executor/determinism/batch_invariant_configs.py index 4570a491ce..86dd51147f 100644 --- a/vllm/model_executor/determinism/batch_invariant_configs.py +++ b/vllm/model_executor/determinism/batch_invariant_configs.py @@ -177,6 +177,83 @@ _BATCH_INVARIANT_MATMUL_TUNED_CONFIGS: dict[ ), ), }, + "blackwell": { + (12288, 2048): _MatmulShapeConfig( + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 128, 8, 5)), + (4, _MatmulMConfig(16, 128, 4, 5)), + (8, _MatmulMConfig(16, 128, 4, 5)), + (16, _MatmulMConfig(16, 128, 4, 5)), + (32, _MatmulMConfig(32, 128, 4, 5)), + (64, _MatmulMConfig(64, 128, 8, 5)), + (256, _MatmulMConfig(128, 256, 4, 4)), + (512, _MatmulMConfig(128, 256, 4, 4)), + (1024, _MatmulMConfig(128, 256, 4, 4)), + (2048, _MatmulMConfig(128, 256, 4, 4)), + ), + ), + (2048, 6144): _MatmulShapeConfig( + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 32, 4, 5)), + (4, _MatmulMConfig(16, 32, 4, 5)), + (8, _MatmulMConfig(16, 32, 4, 5)), + (16, _MatmulMConfig(16, 32, 4, 5)), + (32, _MatmulMConfig(16, 32, 4, 5)), + (64, _MatmulMConfig(32, 32, 4, 5)), + (256, _MatmulMConfig(64, 64, 4, 5)), + (512, _MatmulMConfig(128, 64, 4, 4)), + (1024, _MatmulMConfig(128, 128, 8, 3)), + (2048, _MatmulMConfig(128, 128, 8, 3)), + ), + ), + (4096, 2048): _MatmulShapeConfig( + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 64, 4, 5)), + (4, _MatmulMConfig(16, 64, 4, 5)), + (8, _MatmulMConfig(16, 32, 4, 5)), + (16, _MatmulMConfig(16, 32, 4, 5)), + (32, _MatmulMConfig(32, 32, 4, 5)), + (64, _MatmulMConfig(32, 64, 4, 4)), + (256, _MatmulMConfig(128, 64, 4, 4)), + (512, _MatmulMConfig(128, 128, 8, 3)), + (1024, _MatmulMConfig(128, 128, 8, 3)), + (2048, _MatmulMConfig(128, 128, 8, 3)), + ), + ), + (151936, 2048): _MatmulShapeConfig( + block_k=64, + m_buckets=( + (1, _MatmulMConfig(16, 256, 4, 5)), + (4, _MatmulMConfig(16, 256, 4, 5)), + (8, _MatmulMConfig(16, 256, 4, 5)), + (16, _MatmulMConfig(16, 256, 4, 5)), + (32, _MatmulMConfig(64, 256, 4, 5)), + (64, _MatmulMConfig(64, 256, 4, 5)), + (256, _MatmulMConfig(128, 256, 4, 4)), + (512, _MatmulMConfig(128, 256, 4, 4)), + (1024, _MatmulMConfig(128, 256, 4, 4)), + (2048, _MatmulMConfig(128, 256, 4, 4)), + ), + ), + (2048, 2048): _MatmulShapeConfig( + block_k=128, + m_buckets=( + (1, _MatmulMConfig(16, 32, 4, 4)), + (4, _MatmulMConfig(16, 32, 4, 4)), + (8, _MatmulMConfig(16, 32, 4, 5)), + (16, _MatmulMConfig(16, 32, 8, 5)), + (32, _MatmulMConfig(16, 32, 4, 4)), + (64, _MatmulMConfig(32, 32, 4, 5)), + (256, _MatmulMConfig(64, 64, 4, 4)), + (512, _MatmulMConfig(128, 64, 4, 4)), + (1024, _MatmulMConfig(128, 128, 4, 3)), + (2048, _MatmulMConfig(128, 128, 8, 3)), + ), + ), + }, } _TUNED_MATMUL_CONFIGS_FOR_DEVICE: dict[tuple[int, int], _MatmulShapeConfig] | None = ( @@ -188,6 +265,8 @@ _TUNED_MATMUL_CONFIGS_RESOLVED = False def _get_tuned_matmul_arch_family(capability: DeviceCapability | None) -> str | None: if capability is None: return None + if capability.major == 10: + return "blackwell" if capability.major == 9: return "hopper" if capability.major == 8 and capability.minor == 9: