diff --git a/tests/model_executor/test_deep_gemm_warmup.py b/tests/model_executor/test_deep_gemm_warmup.py new file mode 100644 index 0000000000..6f2a31cf27 --- /dev/null +++ b/tests/model_executor/test_deep_gemm_warmup.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.model_executor.kernels.linear.scaled_mm.deep_gemm import ( + DeepGemmFp8BlockScaledMMKernel, +) +from vllm.model_executor.warmup import deep_gemm_warmup + + +def _block_fp8_layer(n: int, k: int, scale_name: str = "weight_scale"): + layer = SimpleNamespace( + weight=torch.empty((n, k), dtype=torch.float8_e4m3fn), + weight_block_size=[128, 128], + deep_gemm_warmup_provider=object.__new__(DeepGemmFp8BlockScaledMMKernel), + ) + setattr(layer, scale_name, torch.empty((n // 128, k // 128), dtype=torch.float32)) + return layer + + +def _run_warmup(monkeypatch, layers) -> list[dict]: + calls: list[dict] = [] + monkeypatch.setattr( + deep_gemm_warmup, + "get_mk_alignment_for_contiguous_layout", + lambda: [128, 128], + ) + monkeypatch.setattr( + deep_gemm_warmup, + "_deepgemm_fp8_gemm_nt_warmup", + lambda **kwargs: calls.append(kwargs), + ) + model = SimpleNamespace(modules=lambda: iter(layers)) + deep_gemm_warmup.deepgemm_fp8_gemm_nt_warmup(model, max_tokens=16) + return calls + + +@pytest.mark.parametrize("scale_name", ["weight_scale", "weight_scale_inv"]) +def test_registered_deep_gemm_layer_is_warmed_up(monkeypatch, scale_name) -> None: + """Any layer stamped by the DeepGEMM kernel is warmed, regardless of the + quantization method that owns it or the name of its scale parameter.""" + layer = _block_fp8_layer(256, 128, scale_name) + + calls = _run_warmup(monkeypatch, [layer]) + + assert len(calls) == 1 + assert calls[0]["w"] is layer.weight + assert calls[0]["ws"] is getattr(layer, scale_name) + + +def test_n_multiple_of_64_matches_kernel_selection(monkeypatch) -> None: + """DeepGEMM accepts N % 64 == 0 (e.g. DeepSeek kv_a_proj_with_mqa, N=576), + so warmup must not require N % 128 == 0.""" + assert len(_run_warmup(monkeypatch, [_block_fp8_layer(576, 256)])) == 1 + + +def test_unstamped_and_mismatched_block_layers_are_skipped(monkeypatch) -> None: + mismatched = _block_fp8_layer(256, 128) + mismatched.weight_block_size = [1, 32] + layers = [SimpleNamespace(), mismatched, _block_fp8_layer(256, 128)] + + calls = _run_warmup(monkeypatch, layers) + + assert len(calls) == 1 + assert calls[0]["w"] is layers[-1].weight + + +@pytest.mark.parametrize("is_bmm", [False, True]) +def test_kernel_registers_itself_as_warmup_provider(is_bmm) -> None: + """bmm layers bypass the kernel at runtime, so they must not be warmed.""" + kernel = object.__new__(DeepGemmFp8BlockScaledMMKernel) + kernel.is_deep_gemm_supported = False + layer = torch.nn.Module() + layer.weight = torch.nn.Parameter( + torch.empty((256, 128), dtype=torch.float8_e4m3fn), requires_grad=False + ) + layer.weight_scale = torch.nn.Parameter( + torch.empty((2, 1), dtype=torch.float32), requires_grad=False + ) + layer.weight_block_size = [128, 128] + layer.is_bmm = is_bmm + + kernel.process_weights_after_loading(layer) + + provider = getattr(layer, "deep_gemm_warmup_provider", None) + assert provider is (None if is_bmm else kernel) diff --git a/vllm/model_executor/kernels/linear/scaled_mm/BlockScaledMMLinearKernel.py b/vllm/model_executor/kernels/linear/scaled_mm/BlockScaledMMLinearKernel.py index c83885ca51..3a053412f3 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/BlockScaledMMLinearKernel.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/BlockScaledMMLinearKernel.py @@ -38,6 +38,22 @@ class FP8BlockParams(FP8Params): input_scale_ub=getattr(layer, cls.INPUT_SCALE_UB, None), ) + @property + def block_scale_attr(self) -> str: + """Fp8LinearMethod registers the block scale as ``weight_scale_inv``, + compressed-tensors as ``weight_scale``.""" + return ( + self.WEIGHT_SCALE + if self.weight_scale_inv is None + else self.WEIGHT_SCALE_INV + ) + + @property + def block_scale(self) -> torch.Tensor: + scale = getattr(self, self.block_scale_attr) + assert scale is not None + return scale + class Fp8BlockScaledMMLinearKernel( MMLinearKernel[FP8ScaledMMLinearLayerConfig, FP8BlockParams], ABC @@ -74,25 +90,13 @@ class Fp8BlockScaledMMLinearKernel( def process_weights_after_loading(self, layer: torch.nn.Module): params = self._get_layer_params(layer) - # Fp8LinearMethod registered weight scale - # buffer as weight_scale_inv unlike compressed tensors. - weight_scale = ( - params.weight_scale - if params.weight_scale_inv is None - else params.weight_scale_inv - ) - scale_attr_name = ( - params.WEIGHT_SCALE - if params.weight_scale_inv is None - else params.WEIGHT_SCALE_INV - ) new_weight, new_weight_scale = process_fp8_weight_block_strategy( params.weight, - weight_scale, + params.block_scale, ) replace_parameter(layer, params.WEIGHT, new_weight.data) - replace_parameter(layer, scale_attr_name, new_weight_scale.data) + replace_parameter(layer, params.block_scale_attr, new_weight_scale.data) def apply_weights( self, @@ -104,11 +108,7 @@ class Fp8BlockScaledMMLinearKernel( out_dtype = self.config.out_dtype params = self._get_layer_params(layer) weight = params.weight - weight_scale = ( - params.weight_scale - if params.weight_scale_inv is None - else params.weight_scale_inv - ) + weight_scale = params.block_scale input_scale = params.input_scale scale_up = params.input_scale_ub diff --git a/vllm/model_executor/kernels/linear/scaled_mm/deep_gemm.py b/vllm/model_executor/kernels/linear/scaled_mm/deep_gemm.py index 79458f5cda..19f9347e31 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/deep_gemm.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/deep_gemm.py @@ -86,25 +86,28 @@ class DeepGemmFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel): params = self._get_layer_params(layer) assert layer.weight_block_size is not None + is_bmm = getattr(layer, "is_bmm", False) + if self.is_deep_gemm_supported: - weight_scale_invs = params.weight_scale_inv - scale_attr = ( - params.WEIGHT_SCALE_INV - if weight_scale_invs is not None - else params.WEIGHT_SCALE - ) dg_weight, dg_weight_scale = deepgemm_post_process_fp8_weight_block( wq=params.weight, - ws=weight_scale_invs - if weight_scale_invs is not None - else params.weight_scale, + ws=params.block_scale, quant_block_shape=tuple(layer.weight_block_size), use_e8m0=self.use_deep_gemm_e8m0, - is_bmm=getattr(layer, "is_bmm", False), + is_bmm=is_bmm, bmm_batch_size=getattr(layer, "bmm_batch_size", 0), ) replace_parameter(layer, params.WEIGHT, dg_weight) - replace_parameter(layer, scale_attr, dg_weight_scale) + replace_parameter(layer, params.block_scale_attr, dg_weight_scale) + # bmm layers bypass this kernel and run through fp8_einsum. + if not is_bmm: + layer.deep_gemm_warmup_provider = self + + def get_deep_gemm_warmup_weights( + self, layer: torch.nn.Module + ) -> tuple[torch.Tensor, torch.Tensor]: + params = self._get_layer_params(layer) + return params.weight, params.block_scale def apply_block_scaled_mm( self, diff --git a/vllm/model_executor/warmup/deep_gemm_warmup.py b/vllm/model_executor/warmup/deep_gemm_warmup.py index 44f2121659..35fd6bb8f9 100644 --- a/vllm/model_executor/warmup/deep_gemm_warmup.py +++ b/vllm/model_executor/warmup/deep_gemm_warmup.py @@ -22,9 +22,6 @@ from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import DeepGemmE from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import ( TritonOrDeepGemmExperts, ) -from vllm.model_executor.layers.linear import LinearBase -from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod -from vllm.model_executor.layers.quantization.online.mxfp8 import Mxfp8OnlineLinearMethod from vllm.tracing import instrument from vllm.utils.deep_gemm import ( fp8_gemm_nt, @@ -82,28 +79,6 @@ def _generate_optimal_warmup_m_values( return sorted(m_values) -def _extract_data_from_linear_base_module( - m: torch.nn.Module, -) -> tuple[torch.Tensor, torch.Tensor, list[int]]: - """ - Extract weights, weight scales and quantization block sizes from the given - LinearBase module. - """ - assert isinstance(m, LinearBase) - assert isinstance(m.quant_method, Fp8LinearMethod) - assert m.quant_method.block_quant - assert m.quant_method.quant_config is not None - - w = m.weight - ws = m.weight_scale_inv if hasattr(m, "weight_scale_inv") else m.weight_scale - quant_block_size = m.quant_method.quant_config.weight_block_size - - assert isinstance(w, torch.Tensor) - assert isinstance(ws, torch.Tensor) - assert quant_block_size is not None - return (w, ws, quant_block_size) - - def _extract_data_from_fused_moe_module( m_: torch.nn.Module, ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]: @@ -133,45 +108,19 @@ def _extract_data_from_fused_moe_module( return w13, w13_s, w2, w2_s, num_topk -def _is_deep_gemm_backed_kernel(fp8_linear: object) -> bool: +def _deep_gemm_linear_data( + module: torch.nn.Module, +) -> tuple[torch.Tensor, torch.Tensor] | None: """ - Return True if the selected linear kernel dispatches to DeepGEMM, either - directly or as the fallback branch of a dynamic wrapper. + Return (weight, weight_scale) for a layer whose kernel stamped itself as + ``deep_gemm_warmup_provider`` in ``process_weights_after_loading``, else None. """ - if isinstance(fp8_linear, DeepGemmFp8BlockScaledMMKernel): - return True - return isinstance( - getattr(fp8_linear, "fallback", None), DeepGemmFp8BlockScaledMMKernel - ) - - -def _fp8_linear_may_use_deep_gemm(module: torch.nn.Module) -> bool: - """ - Return True if the input module/layer could be processed with DeepGEMM. - """ - - if not ( - isinstance(module, LinearBase) - and isinstance(module.quant_method, Fp8LinearMethod) - and not isinstance(module.quant_method, Mxfp8OnlineLinearMethod) - and getattr(module.quant_method, "block_quant", False) - and not getattr(module.quant_method, "use_marlin", True) - ): - return False - - fp8_linear = getattr(module.quant_method, "fp8_linear", None) - if not _is_deep_gemm_backed_kernel(fp8_linear): - return False - - block_size = get_mk_alignment_for_contiguous_layout()[0] - - w, _, block_sizes = _extract_data_from_linear_base_module(module) - return ( - block_sizes == get_mk_alignment_for_contiguous_layout() - and w.ndim == 2 - and w.shape[0] % block_size == 0 - and w.shape[1] % block_size == 0 - ) + provider = getattr(module, "deep_gemm_warmup_provider", None) + if not isinstance(provider, DeepGemmFp8BlockScaledMMKernel): + return None + if module.weight_block_size != get_mk_alignment_for_contiguous_layout(): + return None + return provider.get_deep_gemm_warmup_weights(module) def _fused_moe_grouped_gemm_may_use_deep_gemm(module: torch.nn.Module) -> bool: @@ -361,11 +310,10 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup( def deepgemm_fp8_gemm_nt_warmup( model: torch.nn.Module, max_tokens: int, pbar: tqdm | None = None ): - dg_modules = [m for m in model.modules() if _fp8_linear_may_use_deep_gemm(m)] - - for dgm in dg_modules: - w, ws, _ = _extract_data_from_linear_base_module(dgm) - _deepgemm_fp8_gemm_nt_warmup(w=w, ws=ws, max_tokens=max_tokens, pbar=pbar) + for m in model.modules(): + if (data := _deep_gemm_linear_data(m)) is not None: + w, ws = data + _deepgemm_fp8_gemm_nt_warmup(w=w, ws=ws, max_tokens=max_tokens, pbar=pbar) def deepgemm_grouped_fp8_gemm_nt_contiguous_warmup( @@ -392,8 +340,8 @@ def _count_warmup_iterations(model: torch.nn.Module, max_tokens: int) -> int: total = 0 for m in model.modules(): - if _fp8_linear_may_use_deep_gemm(m): - w, _, _ = _extract_data_from_linear_base_module(m) + if (data := _deep_gemm_linear_data(m)) is not None: + w, _ = data if w.size() not in seen_fp8_sizes: total += len(_get_fp8_gemm_nt_m_values(w, max_tokens)) seen_fp8_sizes.add(w.size())