[ROCm] Expose kFp8DynamicTokenSym on AITER PTPC linears (#54248)
Signed-off-by: Rebecca Lee <rebecca.lee@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -27,6 +27,10 @@ from vllm.model_executor.layers.fused_moe.config import FusedMoEConfig
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
select_fp8_moe_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
expose_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearMethodBase,
|
||||
)
|
||||
@@ -405,6 +409,7 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
replace_parameter(layer, "weight_scale", weight_scale)
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
expose_input_quant_key(layer, self.fp8_linear)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
@@ -415,8 +420,10 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# if batch invariant mode is enabled dequant
|
||||
if envs.VLLM_BATCH_INVARIANT and not isinstance(
|
||||
self.fp8_linear, CutlassFP8ScaledMMLinearKernel
|
||||
if (
|
||||
envs.VLLM_BATCH_INVARIANT
|
||||
and not isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel)
|
||||
and not isinstance(x, QuantizedActivation)
|
||||
):
|
||||
weight_dequant = (
|
||||
layer.weight.to(x.dtype) * layer.weight_scale.to(x.dtype).t()
|
||||
|
||||
Reference in New Issue
Block a user