[Refactor] refactor humming linear and moe backends to use explicit layer configs (#49610)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
e229fdbc87
commit
fcde8e1460
@@ -28,7 +28,8 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_linear_layer_to_humming_standard,
|
||||
prepare_humming_layer,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
|
||||
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
|
||||
@@ -46,9 +47,11 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
input_quant_config = getattr(layer, "_humming_input_quant_config", None)
|
||||
prepare_humming_layer(
|
||||
self.layer_config = prepare_humming_linear_layer_config(
|
||||
layer, quant_config, input_quant_config=input_quant_config
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -56,12 +59,14 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
apply_humming_linear,
|
||||
)
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
@@ -5,11 +5,14 @@ import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
apply_humming_linear,
|
||||
convert_linear_layer_to_humming_standard,
|
||||
prepare_humming_layer,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
|
||||
@@ -26,6 +29,9 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
@@ -56,7 +62,9 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
}
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -64,12 +72,10 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
@@ -4,10 +4,13 @@
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
apply_humming_linear,
|
||||
convert_linear_layer_to_humming_standard,
|
||||
prepare_humming_layer,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
|
||||
@@ -22,6 +25,9 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
@@ -44,7 +50,9 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
}
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -52,12 +60,10 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
@@ -5,9 +5,12 @@ import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
prepare_humming_layer,
|
||||
apply_humming_linear,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
|
||||
|
||||
@@ -24,6 +27,9 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
@@ -54,7 +60,9 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
layer.weight_global_scale = torch.nn.Parameter(
|
||||
1.0 / layer.weight_global_scale, requires_grad=False
|
||||
)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -62,12 +70,10 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
@@ -5,10 +5,13 @@ import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
apply_humming_linear,
|
||||
convert_linear_layer_to_humming_standard,
|
||||
prepare_humming_layer,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .ScaledMMLinearKernel import (
|
||||
FP8ScaledMMLinearKernel,
|
||||
@@ -30,6 +33,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
@@ -58,9 +64,6 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
scale_group_shape = self.config.weight_quant_key.scale.group_shape
|
||||
if scale_group_shape.is_per_tensor():
|
||||
quant_config["weight_scale_type"] = "tensor"
|
||||
if not hasattr(layer, "global_scale") and hasattr(layer, "weight_scale"):
|
||||
del name_map["weight_scale"]
|
||||
name_map["global_scale"] = "weight_scale"
|
||||
elif scale_group_shape.is_per_channel():
|
||||
quant_config["weight_scale_type"] = "channel"
|
||||
elif scale_group_shape.is_per_group():
|
||||
@@ -76,7 +79,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
name_map["weight_scale"] = "weight_scale_inv"
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -84,15 +89,13 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
def apply_scaled_mm(
|
||||
self,
|
||||
@@ -105,7 +108,9 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
bias: torch.Tensor | None,
|
||||
output_shape: list,
|
||||
) -> torch.Tensor:
|
||||
pass
|
||||
raise NotImplementedError(
|
||||
"HummingFP8ScaledMMLinearKernel uses apply_weights directly"
|
||||
)
|
||||
|
||||
|
||||
class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
@@ -118,6 +123,9 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
if not current_platform.is_cuda():
|
||||
return False, "Humming only supported on CUDA"
|
||||
|
||||
if not has_humming():
|
||||
return False, "Humming is not installed"
|
||||
|
||||
if not current_platform.has_device_capability(75):
|
||||
return False, "Humming only supported on SM75+"
|
||||
|
||||
@@ -137,7 +145,9 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
weight.data = weight.data + 128
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
prepare_humming_layer(layer, quant_config)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -145,12 +155,10 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
x: torch.Tensor,
|
||||
bias: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
flatten_inputs = x.view(-1, x.size(-1))
|
||||
output = HummingMethod.forward_layer(
|
||||
layer=layer,
|
||||
inputs=flatten_inputs,
|
||||
compute_config=layer.compute_config,
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
)
|
||||
return output.view(*x.shape[:-1], output.size(-1))
|
||||
|
||||
Reference in New Issue
Block a user