[Bugfix] Fix DeepGEMM FP8 warmup coverage (#56452)
Signed-off-by: Ayushman Singh <40520701+ayush1399@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
29332cf936
commit
7bcca16729
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.deep_gemm import (
|
||||
DeepGemmFp8BlockScaledMMKernel,
|
||||
)
|
||||
from vllm.model_executor.warmup import deep_gemm_warmup
|
||||
|
||||
|
||||
def _block_fp8_layer(n: int, k: int, scale_name: str = "weight_scale"):
|
||||
layer = SimpleNamespace(
|
||||
weight=torch.empty((n, k), dtype=torch.float8_e4m3fn),
|
||||
weight_block_size=[128, 128],
|
||||
deep_gemm_warmup_provider=object.__new__(DeepGemmFp8BlockScaledMMKernel),
|
||||
)
|
||||
setattr(layer, scale_name, torch.empty((n // 128, k // 128), dtype=torch.float32))
|
||||
return layer
|
||||
|
||||
|
||||
def _run_warmup(monkeypatch, layers) -> list[dict]:
|
||||
calls: list[dict] = []
|
||||
monkeypatch.setattr(
|
||||
deep_gemm_warmup,
|
||||
"get_mk_alignment_for_contiguous_layout",
|
||||
lambda: [128, 128],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
deep_gemm_warmup,
|
||||
"_deepgemm_fp8_gemm_nt_warmup",
|
||||
lambda **kwargs: calls.append(kwargs),
|
||||
)
|
||||
model = SimpleNamespace(modules=lambda: iter(layers))
|
||||
deep_gemm_warmup.deepgemm_fp8_gemm_nt_warmup(model, max_tokens=16)
|
||||
return calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scale_name", ["weight_scale", "weight_scale_inv"])
|
||||
def test_registered_deep_gemm_layer_is_warmed_up(monkeypatch, scale_name) -> None:
|
||||
"""Any layer stamped by the DeepGEMM kernel is warmed, regardless of the
|
||||
quantization method that owns it or the name of its scale parameter."""
|
||||
layer = _block_fp8_layer(256, 128, scale_name)
|
||||
|
||||
calls = _run_warmup(monkeypatch, [layer])
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["w"] is layer.weight
|
||||
assert calls[0]["ws"] is getattr(layer, scale_name)
|
||||
|
||||
|
||||
def test_n_multiple_of_64_matches_kernel_selection(monkeypatch) -> None:
|
||||
"""DeepGEMM accepts N % 64 == 0 (e.g. DeepSeek kv_a_proj_with_mqa, N=576),
|
||||
so warmup must not require N % 128 == 0."""
|
||||
assert len(_run_warmup(monkeypatch, [_block_fp8_layer(576, 256)])) == 1
|
||||
|
||||
|
||||
def test_unstamped_and_mismatched_block_layers_are_skipped(monkeypatch) -> None:
|
||||
mismatched = _block_fp8_layer(256, 128)
|
||||
mismatched.weight_block_size = [1, 32]
|
||||
layers = [SimpleNamespace(), mismatched, _block_fp8_layer(256, 128)]
|
||||
|
||||
calls = _run_warmup(monkeypatch, layers)
|
||||
|
||||
assert len(calls) == 1
|
||||
assert calls[0]["w"] is layers[-1].weight
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_bmm", [False, True])
|
||||
def test_kernel_registers_itself_as_warmup_provider(is_bmm) -> None:
|
||||
"""bmm layers bypass the kernel at runtime, so they must not be warmed."""
|
||||
kernel = object.__new__(DeepGemmFp8BlockScaledMMKernel)
|
||||
kernel.is_deep_gemm_supported = False
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.empty((256, 128), dtype=torch.float8_e4m3fn), requires_grad=False
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
torch.empty((2, 1), dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
layer.weight_block_size = [128, 128]
|
||||
layer.is_bmm = is_bmm
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
provider = getattr(layer, "deep_gemm_warmup_provider", None)
|
||||
assert provider is (None if is_bmm else kernel)
|
||||
@@ -38,6 +38,22 @@ class FP8BlockParams(FP8Params):
|
||||
input_scale_ub=getattr(layer, cls.INPUT_SCALE_UB, None),
|
||||
)
|
||||
|
||||
@property
|
||||
def block_scale_attr(self) -> str:
|
||||
"""Fp8LinearMethod registers the block scale as ``weight_scale_inv``,
|
||||
compressed-tensors as ``weight_scale``."""
|
||||
return (
|
||||
self.WEIGHT_SCALE
|
||||
if self.weight_scale_inv is None
|
||||
else self.WEIGHT_SCALE_INV
|
||||
)
|
||||
|
||||
@property
|
||||
def block_scale(self) -> torch.Tensor:
|
||||
scale = getattr(self, self.block_scale_attr)
|
||||
assert scale is not None
|
||||
return scale
|
||||
|
||||
|
||||
class Fp8BlockScaledMMLinearKernel(
|
||||
MMLinearKernel[FP8ScaledMMLinearLayerConfig, FP8BlockParams], ABC
|
||||
@@ -74,25 +90,13 @@ class Fp8BlockScaledMMLinearKernel(
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module):
|
||||
params = self._get_layer_params(layer)
|
||||
# Fp8LinearMethod registered weight scale
|
||||
# buffer as weight_scale_inv unlike compressed tensors.
|
||||
weight_scale = (
|
||||
params.weight_scale
|
||||
if params.weight_scale_inv is None
|
||||
else params.weight_scale_inv
|
||||
)
|
||||
scale_attr_name = (
|
||||
params.WEIGHT_SCALE
|
||||
if params.weight_scale_inv is None
|
||||
else params.WEIGHT_SCALE_INV
|
||||
)
|
||||
new_weight, new_weight_scale = process_fp8_weight_block_strategy(
|
||||
params.weight,
|
||||
weight_scale,
|
||||
params.block_scale,
|
||||
)
|
||||
|
||||
replace_parameter(layer, params.WEIGHT, new_weight.data)
|
||||
replace_parameter(layer, scale_attr_name, new_weight_scale.data)
|
||||
replace_parameter(layer, params.block_scale_attr, new_weight_scale.data)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -104,11 +108,7 @@ class Fp8BlockScaledMMLinearKernel(
|
||||
out_dtype = self.config.out_dtype
|
||||
params = self._get_layer_params(layer)
|
||||
weight = params.weight
|
||||
weight_scale = (
|
||||
params.weight_scale
|
||||
if params.weight_scale_inv is None
|
||||
else params.weight_scale_inv
|
||||
)
|
||||
weight_scale = params.block_scale
|
||||
input_scale = params.input_scale
|
||||
scale_up = params.input_scale_ub
|
||||
|
||||
|
||||
@@ -86,25 +86,28 @@ class DeepGemmFp8BlockScaledMMKernel(Fp8BlockScaledMMLinearKernel):
|
||||
params = self._get_layer_params(layer)
|
||||
assert layer.weight_block_size is not None
|
||||
|
||||
is_bmm = getattr(layer, "is_bmm", False)
|
||||
|
||||
if self.is_deep_gemm_supported:
|
||||
weight_scale_invs = params.weight_scale_inv
|
||||
scale_attr = (
|
||||
params.WEIGHT_SCALE_INV
|
||||
if weight_scale_invs is not None
|
||||
else params.WEIGHT_SCALE
|
||||
)
|
||||
dg_weight, dg_weight_scale = deepgemm_post_process_fp8_weight_block(
|
||||
wq=params.weight,
|
||||
ws=weight_scale_invs
|
||||
if weight_scale_invs is not None
|
||||
else params.weight_scale,
|
||||
ws=params.block_scale,
|
||||
quant_block_shape=tuple(layer.weight_block_size),
|
||||
use_e8m0=self.use_deep_gemm_e8m0,
|
||||
is_bmm=getattr(layer, "is_bmm", False),
|
||||
is_bmm=is_bmm,
|
||||
bmm_batch_size=getattr(layer, "bmm_batch_size", 0),
|
||||
)
|
||||
replace_parameter(layer, params.WEIGHT, dg_weight)
|
||||
replace_parameter(layer, scale_attr, dg_weight_scale)
|
||||
replace_parameter(layer, params.block_scale_attr, dg_weight_scale)
|
||||
# bmm layers bypass this kernel and run through fp8_einsum.
|
||||
if not is_bmm:
|
||||
layer.deep_gemm_warmup_provider = self
|
||||
|
||||
def get_deep_gemm_warmup_weights(
|
||||
self, layer: torch.nn.Module
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
params = self._get_layer_params(layer)
|
||||
return params.weight, params.block_scale
|
||||
|
||||
def apply_block_scaled_mm(
|
||||
self,
|
||||
|
||||
@@ -22,9 +22,6 @@ from vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe import DeepGemmE
|
||||
from vllm.model_executor.layers.fused_moe.experts.triton_deep_gemm_moe import (
|
||||
TritonOrDeepGemmExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8LinearMethod
|
||||
from vllm.model_executor.layers.quantization.online.mxfp8 import Mxfp8OnlineLinearMethod
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.deep_gemm import (
|
||||
fp8_gemm_nt,
|
||||
@@ -82,28 +79,6 @@ def _generate_optimal_warmup_m_values(
|
||||
return sorted(m_values)
|
||||
|
||||
|
||||
def _extract_data_from_linear_base_module(
|
||||
m: torch.nn.Module,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, list[int]]:
|
||||
"""
|
||||
Extract weights, weight scales and quantization block sizes from the given
|
||||
LinearBase module.
|
||||
"""
|
||||
assert isinstance(m, LinearBase)
|
||||
assert isinstance(m.quant_method, Fp8LinearMethod)
|
||||
assert m.quant_method.block_quant
|
||||
assert m.quant_method.quant_config is not None
|
||||
|
||||
w = m.weight
|
||||
ws = m.weight_scale_inv if hasattr(m, "weight_scale_inv") else m.weight_scale
|
||||
quant_block_size = m.quant_method.quant_config.weight_block_size
|
||||
|
||||
assert isinstance(w, torch.Tensor)
|
||||
assert isinstance(ws, torch.Tensor)
|
||||
assert quant_block_size is not None
|
||||
return (w, ws, quant_block_size)
|
||||
|
||||
|
||||
def _extract_data_from_fused_moe_module(
|
||||
m_: torch.nn.Module,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]:
|
||||
@@ -133,45 +108,19 @@ def _extract_data_from_fused_moe_module(
|
||||
return w13, w13_s, w2, w2_s, num_topk
|
||||
|
||||
|
||||
def _is_deep_gemm_backed_kernel(fp8_linear: object) -> bool:
|
||||
def _deep_gemm_linear_data(
|
||||
module: torch.nn.Module,
|
||||
) -> tuple[torch.Tensor, torch.Tensor] | None:
|
||||
"""
|
||||
Return True if the selected linear kernel dispatches to DeepGEMM, either
|
||||
directly or as the fallback branch of a dynamic wrapper.
|
||||
Return (weight, weight_scale) for a layer whose kernel stamped itself as
|
||||
``deep_gemm_warmup_provider`` in ``process_weights_after_loading``, else None.
|
||||
"""
|
||||
if isinstance(fp8_linear, DeepGemmFp8BlockScaledMMKernel):
|
||||
return True
|
||||
return isinstance(
|
||||
getattr(fp8_linear, "fallback", None), DeepGemmFp8BlockScaledMMKernel
|
||||
)
|
||||
|
||||
|
||||
def _fp8_linear_may_use_deep_gemm(module: torch.nn.Module) -> bool:
|
||||
"""
|
||||
Return True if the input module/layer could be processed with DeepGEMM.
|
||||
"""
|
||||
|
||||
if not (
|
||||
isinstance(module, LinearBase)
|
||||
and isinstance(module.quant_method, Fp8LinearMethod)
|
||||
and not isinstance(module.quant_method, Mxfp8OnlineLinearMethod)
|
||||
and getattr(module.quant_method, "block_quant", False)
|
||||
and not getattr(module.quant_method, "use_marlin", True)
|
||||
):
|
||||
return False
|
||||
|
||||
fp8_linear = getattr(module.quant_method, "fp8_linear", None)
|
||||
if not _is_deep_gemm_backed_kernel(fp8_linear):
|
||||
return False
|
||||
|
||||
block_size = get_mk_alignment_for_contiguous_layout()[0]
|
||||
|
||||
w, _, block_sizes = _extract_data_from_linear_base_module(module)
|
||||
return (
|
||||
block_sizes == get_mk_alignment_for_contiguous_layout()
|
||||
and w.ndim == 2
|
||||
and w.shape[0] % block_size == 0
|
||||
and w.shape[1] % block_size == 0
|
||||
)
|
||||
provider = getattr(module, "deep_gemm_warmup_provider", None)
|
||||
if not isinstance(provider, DeepGemmFp8BlockScaledMMKernel):
|
||||
return None
|
||||
if module.weight_block_size != get_mk_alignment_for_contiguous_layout():
|
||||
return None
|
||||
return provider.get_deep_gemm_warmup_weights(module)
|
||||
|
||||
|
||||
def _fused_moe_grouped_gemm_may_use_deep_gemm(module: torch.nn.Module) -> bool:
|
||||
@@ -361,11 +310,10 @@ def _deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
|
||||
def deepgemm_fp8_gemm_nt_warmup(
|
||||
model: torch.nn.Module, max_tokens: int, pbar: tqdm | None = None
|
||||
):
|
||||
dg_modules = [m for m in model.modules() if _fp8_linear_may_use_deep_gemm(m)]
|
||||
|
||||
for dgm in dg_modules:
|
||||
w, ws, _ = _extract_data_from_linear_base_module(dgm)
|
||||
_deepgemm_fp8_gemm_nt_warmup(w=w, ws=ws, max_tokens=max_tokens, pbar=pbar)
|
||||
for m in model.modules():
|
||||
if (data := _deep_gemm_linear_data(m)) is not None:
|
||||
w, ws = data
|
||||
_deepgemm_fp8_gemm_nt_warmup(w=w, ws=ws, max_tokens=max_tokens, pbar=pbar)
|
||||
|
||||
|
||||
def deepgemm_grouped_fp8_gemm_nt_contiguous_warmup(
|
||||
@@ -392,8 +340,8 @@ def _count_warmup_iterations(model: torch.nn.Module, max_tokens: int) -> int:
|
||||
|
||||
total = 0
|
||||
for m in model.modules():
|
||||
if _fp8_linear_may_use_deep_gemm(m):
|
||||
w, _, _ = _extract_data_from_linear_base_module(m)
|
||||
if (data := _deep_gemm_linear_data(m)) is not None:
|
||||
w, _ = data
|
||||
if w.size() not in seen_fp8_sizes:
|
||||
total += len(_get_fp8_gemm_nt_m_values(w, max_tokens))
|
||||
seen_fp8_sizes.add(w.size())
|
||||
|
||||
Reference in New Issue
Block a user