diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 9bb018b3d3..6b63588150 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -175,18 +175,26 @@ def is_aiter_found_and_supported_on_rdna4() -> bool: @functools.cache def _load_gemm_tuned_configs( - q_dtype_w: torch.dtype, csv_path: str -) -> set[tuple[int, int, int]]: + csv_path: str, + filters: tuple[tuple[str, object], ...], + key_cols: tuple[str, ...] = ("N", "K", "M"), +) -> set[tuple[int, ...]]: try: df = pd.read_csv(csv_path).drop_duplicates() - df = df[df["q_dtype_w"] == str(q_dtype_w)] - return set(zip(df["N"].astype(int), df["K"].astype(int), df["M"].astype(int))) + for col, val in filters: + if col not in df.columns: + continue + if isinstance(val, int): + df = df[df[col].astype(int) == val] + else: + df = df[df[col].astype(str) == str(val)] + return set(zip(*(df[c].astype(int) for c in key_cols))) except Exception: return set() def _check_kernel_tuned(N: int, K: int, q_dtype_w: torch.dtype, csv_path: str) -> bool: - configs = _load_gemm_tuned_configs(q_dtype_w, csv_path) + configs = _load_gemm_tuned_configs(csv_path, (("q_dtype_w", q_dtype_w),)) l_m = ( [1, 2, 4] + list(range(8, 513, 8)) @@ -3200,6 +3208,20 @@ class rocm_aiter_ops: csv_path = aiter_gemm_a8w8_ops.AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_FILE return _check_kernel_tuned(N, K, q_dtype_w, csv_path) + @staticmethod + def is_blockscale_bpreshuffle_tuned(n: int, k: int) -> bool: + """Whether (N, K) has a tuned aiter blockscale bpreshuffle config.""" + if not current_platform.is_rocm(): + return False + import aiter.ops.gemm_op_a8w8 as aiter_gemm_a8w8_ops + + csv_path = aiter_gemm_a8w8_ops.AITER_CONFIGS.AITER_CONFIG_GEMM_A8W8_BLOCKSCALE_BPRESHUFFLE_FILE + gfx = aiter_gemm_a8w8_ops.get_gfx() + cu_num = aiter_gemm_a8w8_ops.get_cu_num() + return (n, k) in _load_gemm_tuned_configs( + csv_path, (("gfx", gfx), ("cu_num", cu_num)), key_cols=("N", "K") + ) + @staticmethod def shuffle_weight( tensor: torch.Tensor, layout: tuple[int, int] = (16, 16) diff --git a/vllm/model_executor/kernels/linear/__init__.py b/vllm/model_executor/kernels/linear/__init__.py index 9c52e7b413..1df73273ca 100644 --- a/vllm/model_executor/kernels/linear/__init__.py +++ b/vllm/model_executor/kernels/linear/__init__.py @@ -178,6 +178,7 @@ from vllm.model_executor.kernels.linear.scaled_mm.aiter import ( AiterHipbMMPerTokenFp8ScaledMMLinearKernel, AiterInt8ScaledMMLinearKernel, AiterPerTokenFp8ScaledMMLinearKernel, + AiterPreshuffledFp8BlockScaledMMKernel, AiterPreshuffledPerTokenFp8ScaledMMLinearKernel, ) from vllm.model_executor.kernels.linear.scaled_mm.b12x import ( @@ -316,6 +317,7 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = { "aiter": { AiterInt8ScaledMMLinearKernel, AiterFp8BlockScaledMMKernel, + AiterPreshuffledFp8BlockScaledMMKernel, AiterPerTokenFp8ScaledMMLinearKernel, AiterPreshuffledPerTokenFp8ScaledMMLinearKernel, AiterMxfp4LinearKernel, @@ -456,6 +458,7 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[ BlockWiseTorchFP8ScaledMMLinearKernel, ], PlatformEnum.ROCM: [ + AiterPreshuffledFp8BlockScaledMMKernel, AiterFp8BlockScaledMMKernel, TritonFp8BlockScaledMMKernel, ], @@ -1210,6 +1213,7 @@ __all__ = [ "ScaledMMLinearLayerConfig", "AiterHipbMMPerTokenFp8ScaledMMLinearKernel", "AiterPreshuffledPerTokenFp8ScaledMMLinearKernel", + "AiterPreshuffledFp8BlockScaledMMKernel", "AiterPerTokenFp8ScaledMMLinearKernel", "NvFp4LinearKernel", "NvFp4LinearLayerConfig", diff --git a/vllm/model_executor/kernels/linear/scaled_mm/aiter.py b/vllm/model_executor/kernels/linear/scaled_mm/aiter.py index b1518f21cc..45f6d418b9 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/aiter.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/aiter.py @@ -457,3 +457,131 @@ class AiterFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): return gemm_a8w8_blockscale_op( A, B, As, Bs, list(self.weight_group_shape), output_dtype=out_dtype ) + + +class AiterPreshuffledFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): + """Aiter FP8 block-scaled GEMM using a pre-shuffled (bpreshuffle) weight.""" + + @classmethod + def is_supported( + cls, compute_capability: int | None = None + ) -> tuple[bool, str | None]: + return AiterPreshuffledPerTokenFp8ScaledMMLinearKernel.is_supported( + compute_capability + ) + + @classmethod + def can_implement(cls, config: FP8ScaledMMLinearLayerConfig): + can_implement_base, reason = super().can_implement(config) + if not can_implement_base: + return can_implement_base, reason + + act_quant_desc = config.activation_quant_key.scale + if act_quant_desc.group_shape != GroupShape(1, 128): + return ( + False, + ( + "Supports only dynamic per token group activation " + "quantization with group_shape=(1,128)." + ), + ) + + # bpreshuffle GEMM requires aiter fp8 linear to be enabled and an fp8 + # 2D weight with N and K divisible by 128. Require a tuned config as the + # fallback can trigger faults or numerical errors. + if not rocm_aiter_ops.is_linear_fp8_enabled(): + return ( + False, + ( + "requires setting `VLLM_ROCM_USE_AITER=1` " + "and `VLLM_ROCM_USE_AITER_LINEAR=1`." + ), + ) + + n, k = config.weight_shape + if not (n % 128 == 0 and k % 128 == 0): + return ( + False, + ( + f"requires N and K dimensions divisible by 128, received " + f"N={n} and K={k}." + ), + ) + + if not rocm_aiter_ops.is_blockscale_bpreshuffle_tuned(n, k): + return ( + False, + ( + f"requires a tuned aiter blockscale bpreshuffle config for " + f"N={n} and K={k}." + ), + ) + + return True, None + + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: + super().process_weights_after_loading(layer) + + params = FP8BlockParams.from_layer(layer) + if params.weight_scale_inv is not None: + ws, attr = params.weight_scale_inv, params.WEIGHT_SCALE_INV + else: + ws, attr = params.weight_scale, params.WEIGHT_SCALE + if ws is not None and ws.dtype == torch.float8_e8m0fnu: + replace_parameter(layer, attr, _upcast_e8m0_to_fp32(ws).contiguous()) + + weight = params.weight + # runtime safety net + assert ( + weight.dim() == 2 + and weight.dtype == current_platform.fp8_dtype() + and weight.shape[0] % 128 == 0 + and weight.shape[1] % 128 == 0 + ), ( + "AiterPreshuffledFp8BlockScaledMMKernel requires a 2D fp8 weight " + "with N and K divisible by 128." + ) + + shuffled_weight = rocm_aiter_ops.shuffle_weight( + weight.contiguous(), layout=(16, 16) + ) + replace_parameter( + layer, + params.WEIGHT, + torch.nn.Parameter(shuffled_weight.data, requires_grad=False), + ) + + def apply_weights( + self, + layer: torch.nn.Module, + x: torch.Tensor, + bias: torch.Tensor | None = None, + **kwargs, + ) -> torch.Tensor: + params = self._get_layer_params(layer) + Bs = ( + params.weight_scale + if params.weight_scale_inv is None + else params.weight_scale_inv + ) + + x_2d = x.view(-1, x.shape[-1]) + A, As = rocm_aiter_ops.group_fp8_quant(x_2d, transpose_scale=True) + output = rocm_aiter_ops.gemm_a8w8_blockscale_bpreshuffle( + A, params.weight, As, Bs, output_dtype=self.config.out_dtype + ) + if bias is not None: + output = output + bias + return output.view(*x.shape[:-1], params.weight.shape[0]) + + def apply_block_scaled_mm( + self, + A: torch.Tensor, + B: torch.Tensor, + As: torch.Tensor, + Bs: torch.Tensor, + ) -> torch.Tensor: + raise NotImplementedError( + "AiterPreshuffledFp8BlockScaledMMKernel overrides apply_weights and " + "does not use apply_block_scaled_mm." + )