[Refactor] Refactor batch invariance folder (#53619)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-08-24 14:31:36 -07:00
committed by GitHub
parent 6a9c69fa85
commit f59bb0bd8c
11 changed files with 12 additions and 10 deletions
File diff suppressed because it is too large Load Diff
@@ -1,250 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import NamedTuple
import torch
from vllm.platforms import current_platform
from vllm.platforms.interface import DeviceCapability
class _MatmulMConfig(NamedTuple):
block_m: int
block_n: int
num_warps: int
num_stages: int
class _MatmulShapeConfig(NamedTuple):
block_k: int
m_buckets: tuple[tuple[int, _MatmulMConfig], ...]
_BATCH_INVARIANT_MATMUL_TUNED_CONFIGS: dict[
str, dict[tuple[int, int], _MatmulShapeConfig]
] = {
"ada": {
(12288, 2048): _MatmulShapeConfig(
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 128, 8, 2)),
(4, _MatmulMConfig(16, 64, 8, 5)),
(8, _MatmulMConfig(16, 128, 8, 4)),
(16, _MatmulMConfig(16, 128, 4, 5)),
(32, _MatmulMConfig(32, 128, 4, 5)),
(64, _MatmulMConfig(32, 256, 4, 3)),
(256, _MatmulMConfig(128, 128, 8, 3)),
(512, _MatmulMConfig(128, 64, 4, 4)),
(1024, _MatmulMConfig(128, 128, 8, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(2048, 6144): _MatmulShapeConfig(
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 64, 4, 4)),
(4, _MatmulMConfig(16, 64, 8, 4)),
(8, _MatmulMConfig(16, 32, 8, 4)),
(16, _MatmulMConfig(16, 32, 8, 4)),
(32, _MatmulMConfig(16, 64, 8, 3)),
(64, _MatmulMConfig(32, 64, 4, 4)),
(256, _MatmulMConfig(32, 64, 4, 4)),
(512, _MatmulMConfig(64, 64, 4, 3)),
(1024, _MatmulMConfig(64, 64, 4, 3)),
(2048, _MatmulMConfig(64, 64, 4, 3)),
),
),
(4096, 2048): _MatmulShapeConfig(
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 256, 8, 3)),
(4, _MatmulMConfig(16, 64, 4, 3)),
(8, _MatmulMConfig(16, 128, 8, 2)),
(16, _MatmulMConfig(16, 32, 4, 5)),
(32, _MatmulMConfig(16, 64, 4, 4)),
(64, _MatmulMConfig(64, 64, 8, 3)),
(256, _MatmulMConfig(64, 64, 4, 5)),
(512, _MatmulMConfig(64, 64, 4, 5)),
(1024, _MatmulMConfig(128, 64, 4, 4)),
(2048, _MatmulMConfig(128, 64, 8, 4)),
),
),
(151936, 2048): _MatmulShapeConfig(
block_k=64,
m_buckets=(
(1, _MatmulMConfig(16, 32, 8, 5)),
(4, _MatmulMConfig(16, 32, 8, 5)),
(8, _MatmulMConfig(16, 32, 8, 5)),
(16, _MatmulMConfig(16, 32, 4, 4)),
(32, _MatmulMConfig(16, 64, 4, 4)),
(64, _MatmulMConfig(32, 64, 4, 4)),
(256, _MatmulMConfig(128, 128, 4, 3)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(128, 128, 4, 3)),
(2048, _MatmulMConfig(128, 128, 4, 3)),
),
),
(2048, 2048): _MatmulShapeConfig(
block_k=64,
m_buckets=(
(1, _MatmulMConfig(32, 128, 8, 5)),
(4, _MatmulMConfig(16, 256, 8, 3)),
(8, _MatmulMConfig(32, 128, 8, 3)),
(16, _MatmulMConfig(32, 128, 8, 3)),
(32, _MatmulMConfig(16, 128, 8, 5)),
(64, _MatmulMConfig(16, 256, 4, 3)),
(256, _MatmulMConfig(64, 128, 8, 4)),
(512, _MatmulMConfig(64, 64, 4, 5)),
(1024, _MatmulMConfig(64, 64, 4, 5)),
(2048, _MatmulMConfig(64, 64, 4, 5)),
),
),
},
"hopper": {
(12288, 2048): _MatmulShapeConfig(
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 256, 4, 3)),
(4, _MatmulMConfig(16, 256, 4, 4)),
(8, _MatmulMConfig(16, 64, 4, 4)),
(16, _MatmulMConfig(16, 64, 4, 4)),
(32, _MatmulMConfig(32, 64, 8, 5)),
(64, _MatmulMConfig(64, 64, 4, 4)),
(256, _MatmulMConfig(64, 128, 4, 3)),
(512, _MatmulMConfig(128, 128, 8, 3)),
(1024, _MatmulMConfig(128, 256, 8, 2)),
(2048, _MatmulMConfig(128, 256, 8, 2)),
),
),
(2048, 6144): _MatmulShapeConfig(
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 32, 4, 4)),
(4, _MatmulMConfig(16, 64, 4, 4)),
(8, _MatmulMConfig(16, 64, 4, 4)),
(16, _MatmulMConfig(16, 32, 4, 3)),
(32, _MatmulMConfig(32, 32, 8, 3)),
(64, _MatmulMConfig(64, 32, 4, 5)),
(256, _MatmulMConfig(64, 128, 4, 3)),
(512, _MatmulMConfig(128, 128, 4, 3)),
(1024, _MatmulMConfig(64, 64, 4, 5)),
(2048, _MatmulMConfig(64, 128, 4, 4)),
),
),
(4096, 2048): _MatmulShapeConfig(
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 64, 8, 5)),
(4, _MatmulMConfig(16, 64, 4, 5)),
(8, _MatmulMConfig(16, 128, 4, 3)),
(16, _MatmulMConfig(16, 64, 4, 5)),
(32, _MatmulMConfig(16, 64, 4, 4)),
(64, _MatmulMConfig(64, 32, 4, 3)),
(256, _MatmulMConfig(64, 256, 4, 2)),
(512, _MatmulMConfig(64, 64, 4, 3)),
(1024, _MatmulMConfig(64, 128, 4, 3)),
(2048, _MatmulMConfig(64, 64, 4, 4)),
),
),
(151936, 2048): _MatmulShapeConfig(
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 256, 4, 4)),
(4, _MatmulMConfig(16, 256, 4, 4)),
(8, _MatmulMConfig(16, 256, 4, 3)),
(16, _MatmulMConfig(16, 256, 4, 3)),
(32, _MatmulMConfig(32, 256, 8, 3)),
(64, _MatmulMConfig(64, 128, 4, 3)),
(256, _MatmulMConfig(64, 128, 4, 3)),
(512, _MatmulMConfig(128, 128, 8, 3)),
(1024, _MatmulMConfig(128, 256, 8, 2)),
(2048, _MatmulMConfig(128, 256, 8, 2)),
),
),
(2048, 2048): _MatmulShapeConfig(
block_k=128,
m_buckets=(
(1, _MatmulMConfig(16, 64, 4, 5)),
(4, _MatmulMConfig(16, 128, 8, 3)),
(8, _MatmulMConfig(16, 32, 4, 3)),
(16, _MatmulMConfig(32, 64, 8, 5)),
(32, _MatmulMConfig(16, 128, 4, 5)),
(64, _MatmulMConfig(32, 64, 4, 2)),
(256, _MatmulMConfig(128, 64, 4, 4)),
(512, _MatmulMConfig(64, 256, 4, 2)),
(1024, _MatmulMConfig(64, 64, 4, 3)),
(2048, _MatmulMConfig(64, 128, 4, 3)),
),
),
},
}
_TUNED_MATMUL_CONFIGS_FOR_DEVICE: dict[tuple[int, int], _MatmulShapeConfig] | None = (
None
)
_TUNED_MATMUL_CONFIGS_RESOLVED = False
def _get_tuned_matmul_arch_family(capability: DeviceCapability | None) -> str | None:
if capability is None:
return None
if capability.major == 9:
return "hopper"
if capability.major == 8 and capability.minor == 9:
return "ada"
return None
def resolve_tuned_matmul_configs() -> None:
global _TUNED_MATMUL_CONFIGS_FOR_DEVICE
global _TUNED_MATMUL_CONFIGS_RESOLVED
if _TUNED_MATMUL_CONFIGS_RESOLVED:
return
# Normal vLLM initialization resolves this before torch.compile tracing.
capability = (
current_platform.get_device_capability() if current_platform.is_cuda() else None
)
arch_family = _get_tuned_matmul_arch_family(capability)
if arch_family is None:
_TUNED_MATMUL_CONFIGS_FOR_DEVICE = None
else:
_TUNED_MATMUL_CONFIGS_FOR_DEVICE = _BATCH_INVARIANT_MATMUL_TUNED_CONFIGS.get(
arch_family
)
_TUNED_MATMUL_CONFIGS_RESOLVED = True
def _get_matmul_config(
M: int,
N: int,
K: int,
dtype: torch.dtype,
default: dict[str, int],
) -> dict[str, int]:
if not _TUNED_MATMUL_CONFIGS_RESOLVED:
resolve_tuned_matmul_configs()
if dtype != torch.bfloat16:
return default
device_configs = _TUNED_MATMUL_CONFIGS_FOR_DEVICE
if device_configs is None:
return default
shape_config = device_configs.get((N, K))
if shape_config is None:
return default
# Values above the tuned range reuse the largest bucket;
# shape-wide BLOCK_K keeps this batch-invariant.
m_config = shape_config.m_buckets[-1][1]
for max_m, bucket_config in shape_config.m_buckets:
if max_m >= M:
m_config = bucket_config
break
return {
"BLOCK_SIZE_M": m_config.block_m,
"BLOCK_SIZE_N": m_config.block_n,
"BLOCK_SIZE_K": shape_config.block_k,
"GROUP_SIZE_M": 8,
"num_warps": m_config.num_warps,
"num_stages": m_config.num_stages,
}
+1 -1
View File
@@ -11,7 +11,7 @@ import vllm.kernels # noqa: F401
from vllm import envs, ir
from vllm.logger import init_logger
from vllm.model_executor.custom_op import CustomOp
from vllm.model_executor.layers.batch_invariant import rms_norm_batch_invariant
from vllm.model_executor.determinism.batch_invariant import rms_norm_batch_invariant
logger = init_logger(__name__)
+1 -1
View File
@@ -21,7 +21,7 @@ from vllm.distributed import (
)
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.batch_invariant import (
from vllm.model_executor.determinism.batch_invariant import (
linear_batch_invariant,
)
from vllm.model_executor.layers.quantization.base_config import (
@@ -16,7 +16,7 @@ from vllm.distributed import (
tensor_model_parallel_all_reduce,
)
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.batch_invariant import (
from vllm.model_executor.determinism.batch_invariant import (
linear_batch_invariant,
)
from vllm.model_executor.layers.quantization.base_config import (