[Fusion] Manual ActivationQuantFusionPass initial application (#51415)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: ElizaWszola <ewszola@redhat.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: ElizaWszola <ewszola@redhat.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
ElizaWszola
parent
d4d703caf9
commit
b7624069ff
@@ -7,7 +7,11 @@ import torch
|
||||
|
||||
from vllm.config import CompilationMode, get_current_vllm_config
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .BlockScaledMMLinearKernel import Fp8BlockScaledMMLinearKernel
|
||||
@@ -88,6 +92,11 @@ class PerTensorTorchFP8ScaledMMLinearKernel(TorchFP8ScaledMMLinearKernel):
|
||||
return False, "requires per tensor activation and weight scales."
|
||||
return True, None
|
||||
|
||||
def input_quant_key(self) -> QuantKey | None:
|
||||
if self.config.activation_quant_key == kFp8StaticTensorSym:
|
||||
return kFp8StaticTensorSym
|
||||
return None
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
*,
|
||||
|
||||
Reference in New Issue
Block a user