[Feature] Support batch invariance on ROCm (#52231)
Signed-off-by: Matthew Wong <Matthew.Wong2@amd.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Andreas Karatzas
parent
50b21a2667
commit
e0317b28ce
@@ -15,9 +15,11 @@ if TYPE_CHECKING:
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
fp8_scaled_mm_is_batch_invariant,
|
||||
init_fp8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
@@ -278,7 +280,7 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
) -> torch.Tensor:
|
||||
# if batch invariant mode is enabled, use BF16 dequant
|
||||
if envs.VLLM_BATCH_INVARIANT:
|
||||
if isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel):
|
||||
if fp8_scaled_mm_is_batch_invariant(self.fp8_linear):
|
||||
return self.fp8_linear.apply_weights(layer, x, bias)
|
||||
|
||||
weight_fp8 = layer.weight.to(torch.bfloat16)
|
||||
@@ -466,7 +468,7 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
# if batch invariant mode is enabled dequant
|
||||
if (
|
||||
envs.VLLM_BATCH_INVARIANT
|
||||
and not isinstance(self.fp8_linear, CutlassFP8ScaledMMLinearKernel)
|
||||
and not fp8_scaled_mm_is_batch_invariant(self.fp8_linear)
|
||||
and not isinstance(x, QuantizedActivation)
|
||||
):
|
||||
weight_dequant = (
|
||||
|
||||
Reference in New Issue
Block a user