Feature/silu block quant fusion v1 (#32996)
Signed-off-by: Monishver Chandrasekaran <monishverchandrasekaran@gmail.com>
This commit is contained in:
@@ -17,6 +17,8 @@ from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8Dynamic64Sym,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
@@ -43,6 +45,10 @@ silu_and_mul_nvfp4_quant_supported = current_platform.is_cuda() and hasattr(
|
||||
if silu_and_mul_nvfp4_quant_supported:
|
||||
FUSED_OPS[kNvfp4Dynamic] = torch.ops._C.silu_and_mul_nvfp4_quant.default # noqa: E501
|
||||
|
||||
if current_platform.is_cuda():
|
||||
FUSED_OPS[kFp8Dynamic128Sym] = torch.ops._C.silu_and_mul_per_block_quant.default
|
||||
FUSED_OPS[kFp8Dynamic64Sym] = torch.ops._C.silu_and_mul_per_block_quant.default
|
||||
|
||||
|
||||
class ActivationQuantPattern(ABC):
|
||||
"""
|
||||
@@ -174,6 +180,102 @@ class SiluMulNvfp4QuantPattern(ActivationQuantPattern):
|
||||
register_replacement(pattern, replacement, self.get_inputs(), fwd_only, pm_pass)
|
||||
|
||||
|
||||
class SiluMulBlockQuantPattern(ActivationQuantPattern):
|
||||
"""
|
||||
Fusion for SiluMul+BlockQuant (FP8 dynamic per-group) Pattern.
|
||||
Supports group_size 128 and 64 via QuantKey.
|
||||
Parameterized on is_scale_transposed for different scale layouts.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
quant_key: QuantKey,
|
||||
is_scale_transposed: bool = False,
|
||||
is_e8m0: bool = False,
|
||||
is_tma_aligned: bool = False,
|
||||
) -> None:
|
||||
super().__init__(quant_key)
|
||||
self.quant_matcher = MatcherQuantFP8(
|
||||
quant_key,
|
||||
has_col_major_scales=is_scale_transposed,
|
||||
is_e8m0=is_e8m0,
|
||||
is_tma_aligned=is_tma_aligned,
|
||||
)
|
||||
self.group_size = quant_key.scale.group_shape[1]
|
||||
self.is_scale_transposed = is_scale_transposed
|
||||
self.is_e8m0 = is_e8m0
|
||||
self.is_tma_aligned = is_tma_aligned
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
scale = self.quant_matcher.empty_f32(1, 1)
|
||||
return self.silu_and_mul_matcher.inputs() + [scale]
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
is_scale_transposed = self.is_scale_transposed
|
||||
|
||||
def pattern(
|
||||
input: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
silu_out = self.silu_and_mul_matcher(input)
|
||||
result = torch.empty(
|
||||
silu_out.shape,
|
||||
device=silu_out.device,
|
||||
dtype=self.quant_dtype,
|
||||
)
|
||||
assert scale is not None
|
||||
finfo = torch.finfo(self.quant_dtype)
|
||||
_, result, scale = auto_functionalized(
|
||||
self.quant_matcher.QUANT_OP,
|
||||
input=silu_out,
|
||||
output_q=result,
|
||||
output_s=scale,
|
||||
group_size=self.group_size,
|
||||
eps=1e-10,
|
||||
fp8_min=finfo.min,
|
||||
fp8_max=finfo.max,
|
||||
scale_ue8m0=self.is_e8m0,
|
||||
dummy_is_scale_transposed=is_scale_transposed,
|
||||
dummy_is_tma_aligned=self.is_tma_aligned,
|
||||
)
|
||||
return result, scale
|
||||
|
||||
def replacement(
|
||||
input: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
d = input.shape[-1] // 2
|
||||
output_shape = input.shape[:-1] + (d,)
|
||||
result = torch.empty(
|
||||
output_shape, device=input.device, dtype=self.quant_dtype
|
||||
)
|
||||
if is_scale_transposed:
|
||||
scale = torch.empty(
|
||||
(d // self.group_size, input.shape[0]),
|
||||
device=input.device,
|
||||
dtype=torch.float32,
|
||||
).permute(-1, -2)
|
||||
else:
|
||||
scale = torch.empty(
|
||||
(input.shape[0], d // self.group_size),
|
||||
device=input.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
at = auto_functionalized(
|
||||
self.FUSED_OP,
|
||||
out=result,
|
||||
input=input,
|
||||
scales=scale,
|
||||
group_size=self.group_size,
|
||||
scale_ub=None,
|
||||
is_scale_transposed=is_scale_transposed,
|
||||
)
|
||||
return at[1], at[2]
|
||||
|
||||
inps = self.get_inputs()
|
||||
register_replacement(pattern, replacement, inps, fwd_only, pm_pass)
|
||||
|
||||
|
||||
class ActivationQuantFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
This pass fuses a pre-defined set of custom ops into fused ops.
|
||||
@@ -199,6 +301,18 @@ class ActivationQuantFusionPass(VllmPatternMatcherPass):
|
||||
pattern_silu_mul_nvfp4 = SiluMulNvfp4QuantPattern()
|
||||
pattern_silu_mul_nvfp4.register(self.patterns)
|
||||
|
||||
if current_platform.is_cuda():
|
||||
for quant_key in [kFp8Dynamic128Sym, kFp8Dynamic64Sym]:
|
||||
for is_scale_transposed in [False, True]:
|
||||
for is_e8m0 in [True, False]:
|
||||
for is_tma_aligned in [False, True]:
|
||||
SiluMulBlockQuantPattern(
|
||||
quant_key,
|
||||
is_scale_transposed=is_scale_transposed,
|
||||
is_e8m0=is_e8m0,
|
||||
is_tma_aligned=is_tma_aligned,
|
||||
).register(self.patterns)
|
||||
|
||||
self.dump_patterns(config, self.patterns)
|
||||
|
||||
@VllmInductorPass.time_and_log
|
||||
@@ -212,4 +326,5 @@ class ActivationQuantFusionPass(VllmPatternMatcherPass):
|
||||
ActivationQuantPattern,
|
||||
SiluMulFp8StaticQuantPattern,
|
||||
SiluMulNvfp4QuantPattern,
|
||||
SiluMulBlockQuantPattern,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user