[Refactor] Move NVFP4 GEMM management into NvFp4LinearKernel (#39129)
Signed-off-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -10,7 +10,10 @@ from torch.nn.parameter import Parameter
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
init_fp8_linear_kernel,
|
||||
init_nvfp4_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
@@ -70,12 +73,6 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
Mxfp8LinearOp,
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
|
||||
NvFp4LinearBackend,
|
||||
apply_nvfp4_linear,
|
||||
convert_to_nvfp4_linear_kernel_format,
|
||||
select_nvfp4_linear_backend,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
create_fp8_quant_key,
|
||||
@@ -1090,11 +1087,7 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
|
||||
def __init__(self, quant_config: ModelOptNvFp4Config) -> None:
|
||||
self.quant_config = quant_config
|
||||
self.marlin_input_dtype = None
|
||||
self.backend = select_nvfp4_linear_backend()
|
||||
|
||||
self.swizzle = None
|
||||
if self.backend == NvFp4LinearBackend.EMULATION:
|
||||
self.swizzle = False
|
||||
self.kernel = init_nvfp4_linear_kernel()
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -1201,7 +1194,7 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
|
||||
)
|
||||
|
||||
# Convert layer to NVFP4 linear kernel format
|
||||
convert_to_nvfp4_linear_kernel_format(self.backend, layer)
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
@@ -1209,13 +1202,7 @@ class ModelOptNvFp4LinearMethod(LinearMethodBase):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return apply_nvfp4_linear(
|
||||
backend=self.backend,
|
||||
layer=layer,
|
||||
x=x,
|
||||
bias=bias,
|
||||
swizzle=self.swizzle,
|
||||
)
|
||||
return self.kernel.apply_weights(layer=layer, x=x, bias=bias)
|
||||
|
||||
|
||||
class ModelOptNvFp4FusedMoE(FusedMoEMethodBase):
|
||||
|
||||
Reference in New Issue
Block a user