[ROCm] Expose kFp8DynamicTokenSym on AITER PTPC linears (#54248)
Signed-off-by: Rebecca Lee <rebecca.lee@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -9,6 +9,10 @@ from torch.nn import Parameter
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
expose_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.quark.schemes import QuarkScheme
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
create_fp8_scale_parameter,
|
||||
@@ -195,10 +199,12 @@ class QuarkW8A8Fp8(QuarkScheme):
|
||||
module_name=self.__class__.__name__,
|
||||
)
|
||||
|
||||
expose_input_quant_key(layer, self.fp8_linear)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
x: torch.Tensor,
|
||||
x: torch.Tensor | QuantizedActivation,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
Reference in New Issue
Block a user