[BugFix] Make FP8 Linear compatible with torch.compile (#13918)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
Woosuk Kwon
2025-02-26 13:48:55 -08:00
committed by GitHub
parent 4cb6fa0a9c
commit b382a7f28f
2 changed files with 21 additions and 4 deletions
@@ -369,12 +369,9 @@ class Fp8LinearMethod(LinearMethodBase):
size_k=layer.input_size_per_partition,
bias=bias)
# Note: lazy import to avoid triton import error.
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
apply_w8a8_block_fp8_linear)
if self.block_quant:
assert self.quant_config.weight_block_size is not None
return apply_w8a8_block_fp8_linear(
return torch.ops.vllm.apply_w8a8_block_fp8_linear(
input=x,
weight=layer.weight,
block_size=self.quant_config.weight_block_size,