[Feature][Humming] Humming feature integration (#56685)
Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
mgoin
parent
27757dde02
commit
7d99c2c4fd
@@ -3,21 +3,18 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
apply_humming_linear,
|
||||
convert_linear_layer_to_humming_standard,
|
||||
get_humming_linear_compute_config,
|
||||
prepare_humming_linear_layer_config,
|
||||
quant_key_to_input_schema,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
"""Humming GEMM Kernel for MXFP4."""
|
||||
@@ -39,14 +36,10 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, config: MxFp4LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
if config.activation_quant_key not in (None, kMxfp4Dynamic):
|
||||
return False, "only supports MXFP4 dynamic or unquantized activations"
|
||||
if config.activation_quant_key is not None:
|
||||
logger.warning_once(
|
||||
"HummingMxFp4LinearKernel is a weight-only (A16) kernel; "
|
||||
"the requested activation quantization (%s) is ignored.",
|
||||
config.activation_quant_key,
|
||||
)
|
||||
try:
|
||||
quant_key_to_input_schema(config.activation_quant_key)
|
||||
except ValueError as error:
|
||||
return False, str(error)
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
@@ -62,7 +55,10 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
}
|
||||
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
input_schema = quant_key_to_input_schema(self.config.activation_quant_key)
|
||||
self.layer_config = prepare_humming_linear_layer_config(
|
||||
layer, quant_config, input_schema=input_schema
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
|
||||
@@ -75,6 +71,7 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
return apply_humming_linear(
|
||||
layer,
|
||||
x,
|
||||
skip_bias_add=bias is None,
|
||||
layer_config=self.layer_config,
|
||||
compute_config=self.compute_config,
|
||||
locks=self.locks,
|
||||
|
||||
Reference in New Issue
Block a user