[Bugfix][Quantization] Support CT block FP8 with Marlin (#52966)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
cb58bb9c1e
commit
541c6d64c1
@@ -21,6 +21,9 @@ from tests.models.utils import check_logprobs_close
|
|||||||
from vllm.model_executor.kernels.linear import (
|
from vllm.model_executor.kernels.linear import (
|
||||||
Fp8BlockScaledMMLinearKernel,
|
Fp8BlockScaledMMLinearKernel,
|
||||||
)
|
)
|
||||||
|
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||||
|
MarlinFP8ScaledMMLinearKernel,
|
||||||
|
)
|
||||||
from vllm.model_executor.layers.fused_moe import UnquantizedFusedMoEMethod
|
from vllm.model_executor.layers.fused_moe import UnquantizedFusedMoEMethod
|
||||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
|
from vllm.model_executor.layers.fused_moe.oracle.fp8 import Fp8MoeBackend
|
||||||
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tensors import ( # noqa: E501
|
||||||
@@ -527,9 +530,14 @@ def test_compressed_tensors_transforms_perplexity(
|
|||||||
assert perplexity <= exp_perplexity
|
assert perplexity <= exp_perplexity
|
||||||
|
|
||||||
|
|
||||||
def test_compressed_tensors_fp8_block_enabled(vllm_runner):
|
@pytest.mark.parametrize(
|
||||||
|
"linear_backend", ["auto", "marlin"] if current_platform.is_cuda() else ["auto"]
|
||||||
|
)
|
||||||
|
def test_compressed_tensors_fp8_block_enabled(vllm_runner, linear_backend):
|
||||||
model_path = "RedHatAI/Qwen3-0.6B-FP8-BLOCK"
|
model_path = "RedHatAI/Qwen3-0.6B-FP8-BLOCK"
|
||||||
with vllm_runner(model_path, enforce_eager=True) as llm:
|
with vllm_runner(
|
||||||
|
model_path, enforce_eager=True, linear_backend=linear_backend
|
||||||
|
) as llm:
|
||||||
fp8_dtype = current_platform.fp8_dtype()
|
fp8_dtype = current_platform.fp8_dtype()
|
||||||
|
|
||||||
def check_model(model):
|
def check_model(model):
|
||||||
@@ -538,20 +546,29 @@ def test_compressed_tensors_fp8_block_enabled(vllm_runner):
|
|||||||
qkv_proj = layer.self_attn.qkv_proj
|
qkv_proj = layer.self_attn.qkv_proj
|
||||||
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
|
assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
|
||||||
assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8)
|
assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8)
|
||||||
assert isinstance(qkv_proj.scheme.fp8_linear, Fp8BlockScaledMMLinearKernel)
|
if linear_backend == "marlin":
|
||||||
|
assert isinstance(
|
||||||
assert qkv_proj.weight.dtype is fp8_dtype
|
qkv_proj.scheme.fp8_linear, MarlinFP8ScaledMMLinearKernel
|
||||||
assert qkv_proj.weight_scale.dtype is torch.float32
|
)
|
||||||
|
assert qkv_proj.weight.dtype is torch.int32
|
||||||
|
assert qkv_proj.weight_scale.dtype is qkv_proj.orig_dtype
|
||||||
|
else:
|
||||||
|
assert isinstance(
|
||||||
|
qkv_proj.scheme.fp8_linear, Fp8BlockScaledMMLinearKernel
|
||||||
|
)
|
||||||
|
assert qkv_proj.weight.dtype is fp8_dtype
|
||||||
|
assert qkv_proj.weight_scale.dtype is torch.float32
|
||||||
assert len(qkv_proj.weight.shape) == 2
|
assert len(qkv_proj.weight.shape) == 2
|
||||||
assert len(qkv_proj.weight_scale.shape) == 2
|
assert len(qkv_proj.weight_scale.shape) == 2
|
||||||
|
|
||||||
input_quant_op = qkv_proj.scheme.fp8_linear.quant_fp8
|
if linear_backend == "auto":
|
||||||
assert isinstance(input_quant_op, QuantFP8)
|
input_quant_op = qkv_proj.scheme.fp8_linear.quant_fp8
|
||||||
assert input_quant_op._forward_method in (
|
assert isinstance(input_quant_op, QuantFP8)
|
||||||
input_quant_op.forward_cuda,
|
assert input_quant_op._forward_method in (
|
||||||
input_quant_op.forward_hip,
|
input_quant_op.forward_cuda,
|
||||||
input_quant_op.forward_xpu,
|
input_quant_op.forward_hip,
|
||||||
)
|
input_quant_op.forward_xpu,
|
||||||
|
)
|
||||||
|
|
||||||
llm.apply_model(check_model)
|
llm.apply_model(check_model)
|
||||||
|
|
||||||
|
|||||||
@@ -57,14 +57,21 @@ class MarlinFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
|||||||
self.block_quant = self.config.weight_quant_key in {kFp8Static128BlockSym}
|
self.block_quant = self.config.weight_quant_key in {kFp8Static128BlockSym}
|
||||||
self.size_k_first = not self.block_quant
|
self.size_k_first = not self.block_quant
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _block_scale_name(layer: torch.nn.Module) -> str:
|
||||||
|
if getattr(layer, "weight_scale_inv", None) is not None:
|
||||||
|
return "weight_scale_inv"
|
||||||
|
return "weight_scale"
|
||||||
|
|
||||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||||
if self.block_quant:
|
if self.block_quant:
|
||||||
weight, weight_scale_inv = process_fp8_weight_block_strategy(
|
scale_name = self._block_scale_name(layer)
|
||||||
layer.weight, layer.weight_scale_inv
|
weight, weight_scale = process_fp8_weight_block_strategy(
|
||||||
|
layer.weight, getattr(layer, scale_name)
|
||||||
)
|
)
|
||||||
# Update layer with new values
|
# Update layer with new values
|
||||||
replace_parameter(layer, "weight", weight.data)
|
replace_parameter(layer, "weight", weight.data)
|
||||||
replace_parameter(layer, "weight_scale_inv", weight_scale_inv.data)
|
replace_parameter(layer, scale_name, weight_scale.data)
|
||||||
# Non-block: callers must pass weight in (K, N) layout.
|
# Non-block: callers must pass weight in (K, N) layout.
|
||||||
|
|
||||||
layer.input_scale = None
|
layer.input_scale = None
|
||||||
@@ -80,7 +87,7 @@ class MarlinFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
|||||||
bias: torch.Tensor | None = None,
|
bias: torch.Tensor | None = None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
if self.block_quant:
|
if self.block_quant:
|
||||||
weight_scale = layer.weight_scale_inv
|
weight_scale = getattr(layer, self._block_scale_name(layer))
|
||||||
else:
|
else:
|
||||||
weight_scale = layer.weight_scale
|
weight_scale = layer.weight_scale
|
||||||
return apply_fp8_marlin_linear(
|
return apply_fp8_marlin_linear(
|
||||||
|
|||||||
Reference in New Issue
Block a user