[ROCm][FEAT] Integrate aiter gemm w8a8 ptpc (#33773)

Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
This commit is contained in:
vllmellm
2026-04-16 09:55:29 +08:00
committed by GitHub
parent 343f65234b
commit 5f7fab881a
9 changed files with 279 additions and 23 deletions
@@ -31,8 +31,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
create_fp8_quant_key,
kFp8DynamicTokenSym,
kFp8StaticChannelSym,
kFp8StaticTensorSym,
kFp8StaticTokenSym,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
cutlass_block_fp8_supported,
@@ -47,7 +47,7 @@ activation_quant_key_mapping = {
DYNAMIC_QUANT: kFp8DynamicTokenSym,
}
weight_quant_key_mapping = {
QuantizationStrategy.CHANNEL: kFp8StaticTokenSym,
QuantizationStrategy.CHANNEL: kFp8StaticChannelSym,
QuantizationStrategy.TENSOR: kFp8StaticTensorSym,
}
logger = init_logger(__name__)
@@ -67,7 +67,6 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
self.use_aiter_and_is_supported = rocm_aiter_ops.is_linear_fp8_enabled()
assert not self.is_static_input_scheme
self.act_q_group_shape = GroupShape(1, self.weight_block_size[0])
self.weight_quant_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(*self.weight_block_size)
)
@@ -76,7 +75,7 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
)
else:
self.activation_quant_key = activation_quant_key_mapping[
is_static_input_scheme
self.is_static_input_scheme
]
self.weight_quant_key = weight_quant_key_mapping[self.strategy]
@@ -138,9 +137,9 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
self.fp8_linear = init_fp8_linear_kernel(
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
weight_shape=layer.weight.shape,
input_dtype=self.input_dtype,
out_dtype=self.out_dtype,
weight_shape=(output_size_per_partition, input_size_per_partition),
module_name=self.__class__.__name__,
)
@@ -175,6 +175,8 @@ class FBGEMMFp8LinearMethod(LinearMethodBase):
# Activations not quantized for marlin.
del layer.input_scale_ub
self.fp8_linear.process_weights_after_loading(layer)
def apply(
self,
layer: torch.nn.Module,
@@ -397,8 +397,6 @@ class Fp8LinearMethod(LinearMethodBase):
if self.block_quant:
assert not self.act_q_static
self.fp8_linear.process_weights_after_loading(layer)
# If checkpoint not serialized fp8, quantize the weights.
else:
# If checkpoint is fp8 per-tensor, handle that there are N scales for N
@@ -428,6 +426,8 @@ class Fp8LinearMethod(LinearMethodBase):
else:
layer.input_scale = None
self.fp8_linear.process_weights_after_loading(layer)
def apply(
self,
layer: torch.nn.Module,
@@ -517,6 +517,7 @@ class ModelOptFp8LinearMethod(LinearMethodBase):
layer.weight = Parameter(weight.t(), requires_grad=False)
layer.weight_scale = Parameter(max_w_scale, requires_grad=False)
layer.input_scale = Parameter(layer.input_scale.max(), requires_grad=False)
self.fp8_linear.process_weights_after_loading(layer)
def apply(
self,
@@ -597,6 +598,7 @@ class ModelOptFp8PcPtLinearMethod(LinearMethodBase):
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.weight = Parameter(layer.weight.t(), requires_grad=False)
layer.weight_scale = Parameter(layer.weight_scale.data, requires_grad=False)
self.fp8_linear.process_weights_after_loading(layer)
def apply(
self,
@@ -120,6 +120,8 @@ class QuarkW8A8Fp8(QuarkScheme):
if self.is_static_input_scheme:
layer.input_scale = Parameter(layer.input_scale.max(), requires_grad=False)
self.fp8_linear.process_weights_after_loading(layer)
def create_weights(
self,
layer: torch.nn.Module,