[ROCm][FEAT] Integrate aiter gemm w8a8 ptpc (#33773)
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
This commit is contained in:
+4
-5
@@ -31,8 +31,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
create_fp8_quant_key,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8StaticChannelSym,
|
||||
kFp8StaticTensorSym,
|
||||
kFp8StaticTokenSym,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
cutlass_block_fp8_supported,
|
||||
@@ -47,7 +47,7 @@ activation_quant_key_mapping = {
|
||||
DYNAMIC_QUANT: kFp8DynamicTokenSym,
|
||||
}
|
||||
weight_quant_key_mapping = {
|
||||
QuantizationStrategy.CHANNEL: kFp8StaticTokenSym,
|
||||
QuantizationStrategy.CHANNEL: kFp8StaticChannelSym,
|
||||
QuantizationStrategy.TENSOR: kFp8StaticTensorSym,
|
||||
}
|
||||
logger = init_logger(__name__)
|
||||
@@ -67,7 +67,6 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
|
||||
self.use_aiter_and_is_supported = rocm_aiter_ops.is_linear_fp8_enabled()
|
||||
assert not self.is_static_input_scheme
|
||||
self.act_q_group_shape = GroupShape(1, self.weight_block_size[0])
|
||||
|
||||
self.weight_quant_key = create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(*self.weight_block_size)
|
||||
)
|
||||
@@ -76,7 +75,7 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
|
||||
)
|
||||
else:
|
||||
self.activation_quant_key = activation_quant_key_mapping[
|
||||
is_static_input_scheme
|
||||
self.is_static_input_scheme
|
||||
]
|
||||
self.weight_quant_key = weight_quant_key_mapping[self.strategy]
|
||||
|
||||
@@ -138,9 +137,9 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
|
||||
self.fp8_linear = init_fp8_linear_kernel(
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
weight_shape=layer.weight.shape,
|
||||
input_dtype=self.input_dtype,
|
||||
out_dtype=self.out_dtype,
|
||||
weight_shape=(output_size_per_partition, input_size_per_partition),
|
||||
module_name=self.__class__.__name__,
|
||||
)
|
||||
|
||||
|
||||
@@ -175,6 +175,8 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
|
||||
# Activations not quantized for marlin.
|
||||
del layer.input_scale_ub
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
|
||||
@@ -397,8 +397,6 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
if self.block_quant:
|
||||
assert not self.act_q_static
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
# If checkpoint not serialized fp8, quantize the weights.
|
||||
else:
|
||||
# If checkpoint is fp8 per-tensor, handle that there are N scales for N
|
||||
@@ -428,6 +426,8 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
else:
|
||||
layer.input_scale = None
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
|
||||
@@ -517,6 +517,7 @@ class ModelOptFp8LinearMethod(LinearMethodBase):
|
||||
layer.weight = Parameter(weight.t(), requires_grad=False)
|
||||
layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
|
||||
layer.input_scale = Parameter(layer.input_scale.max(), requires_grad=False)
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -597,6 +598,7 @@ class ModelOptFp8PcPtLinearMethod(LinearMethodBase):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
layer.weight = Parameter(layer.weight.t(), requires_grad=False)
|
||||
layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
|
||||
@@ -120,6 +120,8 @@ class QuarkW8A8Fp8(QuarkScheme):
|
||||
if self.is_static_input_scheme:
|
||||
layer.input_scale = Parameter(layer.input_scale.max(), requires_grad=False)
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
|
||||
Reference in New Issue
Block a user