[Quantization] Prefer Humming before Marlin backends on SM90 (#56997)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jinzhen Lin
2026-10-06 12:31:35 -07:00
committed by GitHub
co-authored by Codex mgoin Claude Opus 5.5
parent bb87d227d4
commit d1f3d8b870
13 changed files with 159 additions and 15 deletions
@@ -4,11 +4,21 @@
import torch
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8DynamicTokenSym,
kInt8DynamicTokenSym,
)
from vllm.platforms import current_platform
from vllm.utils.import_utils import has_humming
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
# Per-token activation quantization requested through `act_type`.
_ACT_QUANT_KEYS = {
torch.int8: kInt8DynamicTokenSym,
torch.float8_e4m3fn: kFp8DynamicTokenSym,
}
class HummingLinearKernel(MPLinearKernel):
@classmethod
@@ -21,6 +31,8 @@ class HummingLinearKernel(MPLinearKernel):
return False, "Humming is only supported on CUDA"
if not has_humming():
return False, "Humming is not installed"
if c.act_type.itemsize == 1 and c.act_type not in _ACT_QUANT_KEYS:
return False, f"Activation type ({c.act_type}) not supported by Humming"
return True, None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
@@ -28,6 +40,7 @@ class HummingLinearKernel(MPLinearKernel):
convert_linear_layer_to_humming_standard,
get_humming_linear_compute_config,
prepare_humming_linear_layer_config,
quant_key_to_input_schema,
)
name_map = {"weight": self.w_q_name, "weight_scale": self.w_s_name}
@@ -48,8 +61,14 @@ class HummingLinearKernel(MPLinearKernel):
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
input_quant_config = getattr(layer, "_humming_input_quant_config", None)
input_schema = None
if (act_key := _ACT_QUANT_KEYS.get(self.config.act_type)) is not None:
input_schema = quant_key_to_input_schema(act_key)
self.layer_config = prepare_humming_linear_layer_config(
layer, quant_config, input_quant_config=input_quant_config
layer,
quant_config,
input_quant_config=input_quant_config,
input_schema=input_schema,
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)