[Dependency] Update Humming and remove NVFP4 scale workaround (#61556)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Jinzhen Lin
2026-10-14 13:32:36 -07:00
committed by GitHub
co-authored by Codex Misha Goin Kevin H. Luu
parent 2e8a5930f6
commit fe98d90a9d
13 changed files with 54 additions and 47 deletions
+1 -1
View File
@@ -28,4 +28,4 @@ quack-kernels==0.6.5 # Required by MSA
tokenspeed-mla==2.0.2; platform_system == "Linux"
# Humming kernels for quantization gemm
humming-kernels[cu13]==0.1.16
humming-kernels[cu13]==0.1.17
@@ -9,6 +9,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt8DynamicTokenSym,
)
from vllm.platforms import current_platform
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
@@ -71,7 +72,7 @@ class HummingLinearKernel(MPLinearKernel):
input_schema=input_schema,
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
quant_key_to_input_schema,
)
from vllm.platforms import current_platform
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
@@ -60,7 +61,7 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
layer, quant_config, input_schema=input_schema
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp6E2M3Static
from vllm.platforms import current_platform
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from .base import MxFp6LinearKernel, MxFp6LinearLayerConfig
@@ -66,7 +67,7 @@ class HummingMxFp6LinearKernel(MxFp6LinearKernel):
layer, quant_config, input_schema=input_schema
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
from vllm.platforms import current_platform
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
@@ -57,7 +58,7 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
layer, quant_config, input_schema=input_schema
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
)
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic
from vllm.platforms import current_platform
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
@@ -71,7 +72,7 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
layer, quant_config, input_schema=input_schema
)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
prepare_humming_linear_layer_config,
)
from vllm.platforms import current_platform
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from .ScaledMMLinearKernel import (
@@ -81,7 +82,7 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -148,7 +149,7 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = create_humming_locks(layer.weight.device)
def apply_weights(
self,
@@ -66,6 +66,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.platforms import current_platform
from vllm.scalar_type import ScalarType
from vllm.utils.humming import create_humming_locks
from vllm.utils.import_utils import has_humming
from vllm.utils.platform_utils import num_compute_units
from vllm.v1.worker.workspace import current_workspace_manager
@@ -140,7 +141,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
"w13": humming_quant_config.w1_humming_config,
"w2": humming_quant_config.w2_humming_config,
}
self.locks = torch.zeros(1024, dtype=torch.int32, device=moe_config.device)
self.locks = create_humming_locks(moe_config.device)
self.num_experts = moe_config.num_local_experts
self.global_num_experts = moe_config.num_experts
self.quant_config = humming_quant_config
@@ -584,7 +584,9 @@ class HummingLinearMethod(LinearMethodBase):
param_dtype=layer.param_dtype,
allow_fallback=self.quant_config.allow_input_schema_fallback,
)
self.layer_config = _hm.prepare_layer_config(
source_tensors = dict(layer.named_parameters())
self.layer_config, tensors = _hm.prepare_and_transform_humming_tensors(
tensors=source_tensors,
shape_n=layer.output_partition_sizes_sum,
shape_k=layer.input_size_per_partition,
weight_schema=self.weight_schema,
@@ -595,8 +597,6 @@ class HummingLinearMethod(LinearMethodBase):
torch_dtype=layer.param_dtype,
device=layer.weight.device,
)
source_tensors = dict(layer.named_parameters())
tensors = _hm.transform_humming_tensors(self.layer_config, source_tensors)
name = self.input_schema.static_tensor_scale_name
if name is not None:
tensors[name] = source_tensors[name]
@@ -610,7 +610,7 @@ class HummingLinearMethod(LinearMethodBase):
setattr(layer, name, param)
self.compute_config = get_humming_linear_compute_config()
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
self.locks = _hm.create_humming_locks(layer.weight.device)
def apply(
self,
@@ -64,8 +64,7 @@ def prepare_humming_linear_layer_config(
BaseInputSchema,
BaseWeightSchema,
HummingInputSchema,
prepare_layer_config,
transform_humming_tensors,
prepare_and_transform_humming_tensors,
)
weight_schema = BaseWeightSchema.from_config(quant_config)
@@ -127,7 +126,8 @@ def prepare_humming_linear_layer_config(
input_tensors.pop(name, None)
# Step 2: transform weight (humming standard format) for forwarding.
config = prepare_layer_config(
config, tensors = prepare_and_transform_humming_tensors(
tensors=tensors,
shape_n=sum(layer.output_partition_sizes),
shape_k=input_size_per_partition,
weight_schema=weight_schema,
@@ -138,7 +138,6 @@ def prepare_humming_linear_layer_config(
torch_dtype=layer.params_dtype,
device=tensors["weight"].device,
)
tensors = transform_humming_tensors(config, tensors)
tensors.update(input_tensors)
if "bias" in tensors:
zero_bias = torch.zeros_like(tensors["bias"])
@@ -397,11 +397,15 @@ def _prepare_and_transform_sublayer(
) -> "LayerConfig":
"""Prepare Humming configuration and transform one sublayer's tensors."""
from vllm.utils.humming import (
prepare_layer_config,
transform_humming_tensors,
prepare_and_transform_humming_tensors,
)
config = prepare_layer_config(
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
for name in ("input_scale", "input_scale_2"):
if name != input_schema.static_tensor_scale_name:
source_tensors.pop(name, None)
config, tensors = prepare_and_transform_humming_tensors(
tensors=source_tensors,
shape_n=shape_n,
shape_k=shape_k,
pad_n_to_multiple=256,
@@ -413,11 +417,6 @@ def _prepare_and_transform_sublayer(
torch_dtype=param_dtype,
device=getattr(layer, f"{sublayer_name}_weight").device,
)
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
for name in ("input_scale", "input_scale_2"):
if name != input_schema.static_tensor_scale_name:
source_tensors.pop(name, None)
tensors = transform_humming_tensors(config, source_tensors)
for name in ("input_scale", "input_scale_2"):
if name in source_tensors:
tensors[name] = source_tensors[name]
@@ -322,22 +322,6 @@ def check_and_fallback_input_schema(
input_group_size = input_schema.input_scale_group_size
input_scale_dtype = input_schema.input_scale_dtype
def is_compatible(candidate: "HummingInputSchema") -> bool:
check_weight_schema = weight_schema
if (
candidate.a_dtype in (None, humming_dtypes.float16, humming_dtypes.bfloat16)
and weight_schema.b_dtype == humming_dtypes.float4e2m1
and weight_group_size == 16
and weight_schema.bs_dtype
in (None, humming_dtypes.float16, humming_dtypes.bfloat16)
):
# Humming rejects folded NVFP4 scales despite A16 software dequant
# supporting them. Relax only this check, preserving kernel metadata.
check_weight_schema = dataclasses.replace(
weight_schema, bs_dtype=humming_dtypes.float8e4m3
)
return candidate.is_compatible_with(check_weight_schema, param_dtype)
def is_deprecated(dtype: "humming_dtypes.DataType | None") -> bool:
if dtype is None or dtype.num_bits == 16:
return False
@@ -349,7 +333,7 @@ def check_and_fallback_input_schema(
is_fp8_deprecated = dtype.num_bits < 16 and 100 <= sm_version < 120
return is_int4_deprecated or is_int8_deprecated or is_fp8_deprecated
if is_compatible(input_schema):
if input_schema.is_compatible_with(weight_schema, param_dtype):
if not allow_fallback:
if is_deprecated(a_dtype):
msg = f"Humming {a_dtype} activation is slow on SM{sm_version} now."
@@ -406,7 +390,7 @@ def check_and_fallback_input_schema(
input_scale_dtype=scale_dtype,
input_quant_mode=quant_mode,
)
if is_compatible(candidate):
if candidate.is_compatible_with(weight_schema, param_dtype):
return candidate
raise ValueError(
+22 -4
View File
@@ -11,6 +11,10 @@ symbols to both the ``TYPE_CHECKING`` imports and ``_EXPORTS``, using
import importlib
from typing import TYPE_CHECKING, Any
import torch
from vllm.utils.platform_utils import num_compute_units
if TYPE_CHECKING:
from humming import dtypes as dtypes
from humming.config import GemmType as GemmType
@@ -50,8 +54,9 @@ if TYPE_CHECKING:
from humming.schema.modelopt import (
ModeloptNvfp4WeightSchema as ModeloptNvfp4WeightSchema,
)
from humming.transform import prepare_layer_config as prepare_layer_config
from humming.transform import transform_humming_tensors as transform_humming_tensors
from humming.transform import (
prepare_and_transform_humming_tensors as prepare_and_transform_humming_tensors,
)
from humming.tune import get_heuristics_config as get_heuristics_config
from humming.utils.weight import quantize_weight as quantize_weight
@@ -68,8 +73,9 @@ _EXPORTS: dict[str, str] = {
"may_process_input": "humming.forward:may_process_input",
"process_input": "humming.ops:process_input",
"may_quant_input": "humming.forward:may_quant_input",
"prepare_layer_config": "humming.transform:prepare_layer_config",
"transform_humming_tensors": "humming.transform:transform_humming_tensors",
"prepare_and_transform_humming_tensors": (
"humming.transform:prepare_and_transform_humming_tensors"
),
"get_heuristics_config": "humming.tune:get_heuristics_config",
"BaseInputSchema": "humming.schema:BaseInputSchema",
"BaseWeightSchema": "humming.schema:BaseWeightSchema",
@@ -91,6 +97,18 @@ _EXPORTS: dict[str, str] = {
}
def create_humming_locks(device: torch.device | str) -> torch.Tensor:
device = torch.device(device)
device_index = device.index
if device_index is None:
device_index = torch.accelerator.current_device_index()
return torch.zeros(
num_compute_units(device_index) * 8,
dtype=torch.int32,
device=device,
)
def __getattr__(name: str) -> Any:
spec = _EXPORTS.get(name)
if spec is None: