[Dependency] Update Humming and remove NVFP4 scale workaround (#61556)
Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Codex
Misha Goin
Kevin H. Luu
parent
2e8a5930f6
commit
fe98d90a9d
@@ -28,4 +28,4 @@ quack-kernels==0.6.5 # Required by MSA
|
||||
tokenspeed-mla==2.0.2; platform_system == "Linux"
|
||||
|
||||
# Humming kernels for quantization gemm
|
||||
humming-kernels[cu13]==0.1.16
|
||||
humming-kernels[cu13]==0.1.17
|
||||
|
||||
@@ -9,6 +9,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt8DynamicTokenSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig
|
||||
@@ -71,7 +72,7 @@ class HummingLinearKernel(MPLinearKernel):
|
||||
input_schema=input_schema,
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
quant_key_to_input_schema,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig
|
||||
@@ -60,7 +61,7 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel):
|
||||
layer, quant_config, input_schema=input_schema
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp6E2M3Static
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .base import MxFp6LinearKernel, MxFp6LinearLayerConfig
|
||||
@@ -66,7 +67,7 @@ class HummingMxFp6LinearKernel(MxFp6LinearKernel):
|
||||
layer, quant_config, input_schema=input_schema
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
@@ -57,7 +58,7 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
layer, quant_config, input_schema=input_schema
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig
|
||||
@@ -71,7 +72,7 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel):
|
||||
layer, quant_config, input_schema=input_schema
|
||||
)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.humming import (
|
||||
prepare_humming_linear_layer_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
|
||||
from .ScaledMMLinearKernel import (
|
||||
@@ -81,7 +82,7 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
@@ -148,7 +149,7 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel):
|
||||
convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map)
|
||||
self.layer_config = prepare_humming_linear_layer_config(layer, quant_config)
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply_weights(
|
||||
self,
|
||||
|
||||
@@ -66,6 +66,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType
|
||||
from vllm.utils.humming import create_humming_locks
|
||||
from vllm.utils.import_utils import has_humming
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
@@ -140,7 +141,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
"w13": humming_quant_config.w1_humming_config,
|
||||
"w2": humming_quant_config.w2_humming_config,
|
||||
}
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=moe_config.device)
|
||||
self.locks = create_humming_locks(moe_config.device)
|
||||
self.num_experts = moe_config.num_local_experts
|
||||
self.global_num_experts = moe_config.num_experts
|
||||
self.quant_config = humming_quant_config
|
||||
|
||||
@@ -584,7 +584,9 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
param_dtype=layer.param_dtype,
|
||||
allow_fallback=self.quant_config.allow_input_schema_fallback,
|
||||
)
|
||||
self.layer_config = _hm.prepare_layer_config(
|
||||
source_tensors = dict(layer.named_parameters())
|
||||
self.layer_config, tensors = _hm.prepare_and_transform_humming_tensors(
|
||||
tensors=source_tensors,
|
||||
shape_n=layer.output_partition_sizes_sum,
|
||||
shape_k=layer.input_size_per_partition,
|
||||
weight_schema=self.weight_schema,
|
||||
@@ -595,8 +597,6 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
torch_dtype=layer.param_dtype,
|
||||
device=layer.weight.device,
|
||||
)
|
||||
source_tensors = dict(layer.named_parameters())
|
||||
tensors = _hm.transform_humming_tensors(self.layer_config, source_tensors)
|
||||
name = self.input_schema.static_tensor_scale_name
|
||||
if name is not None:
|
||||
tensors[name] = source_tensors[name]
|
||||
@@ -610,7 +610,7 @@ class HummingLinearMethod(LinearMethodBase):
|
||||
setattr(layer, name, param)
|
||||
|
||||
self.compute_config = get_humming_linear_compute_config()
|
||||
self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device)
|
||||
self.locks = _hm.create_humming_locks(layer.weight.device)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
|
||||
@@ -64,8 +64,7 @@ def prepare_humming_linear_layer_config(
|
||||
BaseInputSchema,
|
||||
BaseWeightSchema,
|
||||
HummingInputSchema,
|
||||
prepare_layer_config,
|
||||
transform_humming_tensors,
|
||||
prepare_and_transform_humming_tensors,
|
||||
)
|
||||
|
||||
weight_schema = BaseWeightSchema.from_config(quant_config)
|
||||
@@ -127,7 +126,8 @@ def prepare_humming_linear_layer_config(
|
||||
input_tensors.pop(name, None)
|
||||
|
||||
# Step 2: transform weight (humming standard format) for forwarding.
|
||||
config = prepare_layer_config(
|
||||
config, tensors = prepare_and_transform_humming_tensors(
|
||||
tensors=tensors,
|
||||
shape_n=sum(layer.output_partition_sizes),
|
||||
shape_k=input_size_per_partition,
|
||||
weight_schema=weight_schema,
|
||||
@@ -138,7 +138,6 @@ def prepare_humming_linear_layer_config(
|
||||
torch_dtype=layer.params_dtype,
|
||||
device=tensors["weight"].device,
|
||||
)
|
||||
tensors = transform_humming_tensors(config, tensors)
|
||||
tensors.update(input_tensors)
|
||||
if "bias" in tensors:
|
||||
zero_bias = torch.zeros_like(tensors["bias"])
|
||||
|
||||
@@ -397,11 +397,15 @@ def _prepare_and_transform_sublayer(
|
||||
) -> "LayerConfig":
|
||||
"""Prepare Humming configuration and transform one sublayer's tensors."""
|
||||
from vllm.utils.humming import (
|
||||
prepare_layer_config,
|
||||
transform_humming_tensors,
|
||||
prepare_and_transform_humming_tensors,
|
||||
)
|
||||
|
||||
config = prepare_layer_config(
|
||||
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
|
||||
for name in ("input_scale", "input_scale_2"):
|
||||
if name != input_schema.static_tensor_scale_name:
|
||||
source_tensors.pop(name, None)
|
||||
config, tensors = prepare_and_transform_humming_tensors(
|
||||
tensors=source_tensors,
|
||||
shape_n=shape_n,
|
||||
shape_k=shape_k,
|
||||
pad_n_to_multiple=256,
|
||||
@@ -413,11 +417,6 @@ def _prepare_and_transform_sublayer(
|
||||
torch_dtype=param_dtype,
|
||||
device=getattr(layer, f"{sublayer_name}_weight").device,
|
||||
)
|
||||
source_tensors = _extract_sublayer_tensors(layer, sublayer_name)
|
||||
for name in ("input_scale", "input_scale_2"):
|
||||
if name != input_schema.static_tensor_scale_name:
|
||||
source_tensors.pop(name, None)
|
||||
tensors = transform_humming_tensors(config, source_tensors)
|
||||
for name in ("input_scale", "input_scale_2"):
|
||||
if name in source_tensors:
|
||||
tensors[name] = source_tensors[name]
|
||||
|
||||
@@ -322,22 +322,6 @@ def check_and_fallback_input_schema(
|
||||
input_group_size = input_schema.input_scale_group_size
|
||||
input_scale_dtype = input_schema.input_scale_dtype
|
||||
|
||||
def is_compatible(candidate: "HummingInputSchema") -> bool:
|
||||
check_weight_schema = weight_schema
|
||||
if (
|
||||
candidate.a_dtype in (None, humming_dtypes.float16, humming_dtypes.bfloat16)
|
||||
and weight_schema.b_dtype == humming_dtypes.float4e2m1
|
||||
and weight_group_size == 16
|
||||
and weight_schema.bs_dtype
|
||||
in (None, humming_dtypes.float16, humming_dtypes.bfloat16)
|
||||
):
|
||||
# Humming rejects folded NVFP4 scales despite A16 software dequant
|
||||
# supporting them. Relax only this check, preserving kernel metadata.
|
||||
check_weight_schema = dataclasses.replace(
|
||||
weight_schema, bs_dtype=humming_dtypes.float8e4m3
|
||||
)
|
||||
return candidate.is_compatible_with(check_weight_schema, param_dtype)
|
||||
|
||||
def is_deprecated(dtype: "humming_dtypes.DataType | None") -> bool:
|
||||
if dtype is None or dtype.num_bits == 16:
|
||||
return False
|
||||
@@ -349,7 +333,7 @@ def check_and_fallback_input_schema(
|
||||
is_fp8_deprecated = dtype.num_bits < 16 and 100 <= sm_version < 120
|
||||
return is_int4_deprecated or is_int8_deprecated or is_fp8_deprecated
|
||||
|
||||
if is_compatible(input_schema):
|
||||
if input_schema.is_compatible_with(weight_schema, param_dtype):
|
||||
if not allow_fallback:
|
||||
if is_deprecated(a_dtype):
|
||||
msg = f"Humming {a_dtype} activation is slow on SM{sm_version} now."
|
||||
@@ -406,7 +390,7 @@ def check_and_fallback_input_schema(
|
||||
input_scale_dtype=scale_dtype,
|
||||
input_quant_mode=quant_mode,
|
||||
)
|
||||
if is_compatible(candidate):
|
||||
if candidate.is_compatible_with(weight_schema, param_dtype):
|
||||
return candidate
|
||||
|
||||
raise ValueError(
|
||||
|
||||
+22
-4
@@ -11,6 +11,10 @@ symbols to both the ``TYPE_CHECKING`` imports and ``_EXPORTS``, using
|
||||
import importlib
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from humming import dtypes as dtypes
|
||||
from humming.config import GemmType as GemmType
|
||||
@@ -50,8 +54,9 @@ if TYPE_CHECKING:
|
||||
from humming.schema.modelopt import (
|
||||
ModeloptNvfp4WeightSchema as ModeloptNvfp4WeightSchema,
|
||||
)
|
||||
from humming.transform import prepare_layer_config as prepare_layer_config
|
||||
from humming.transform import transform_humming_tensors as transform_humming_tensors
|
||||
from humming.transform import (
|
||||
prepare_and_transform_humming_tensors as prepare_and_transform_humming_tensors,
|
||||
)
|
||||
from humming.tune import get_heuristics_config as get_heuristics_config
|
||||
from humming.utils.weight import quantize_weight as quantize_weight
|
||||
|
||||
@@ -68,8 +73,9 @@ _EXPORTS: dict[str, str] = {
|
||||
"may_process_input": "humming.forward:may_process_input",
|
||||
"process_input": "humming.ops:process_input",
|
||||
"may_quant_input": "humming.forward:may_quant_input",
|
||||
"prepare_layer_config": "humming.transform:prepare_layer_config",
|
||||
"transform_humming_tensors": "humming.transform:transform_humming_tensors",
|
||||
"prepare_and_transform_humming_tensors": (
|
||||
"humming.transform:prepare_and_transform_humming_tensors"
|
||||
),
|
||||
"get_heuristics_config": "humming.tune:get_heuristics_config",
|
||||
"BaseInputSchema": "humming.schema:BaseInputSchema",
|
||||
"BaseWeightSchema": "humming.schema:BaseWeightSchema",
|
||||
@@ -91,6 +97,18 @@ _EXPORTS: dict[str, str] = {
|
||||
}
|
||||
|
||||
|
||||
def create_humming_locks(device: torch.device | str) -> torch.Tensor:
|
||||
device = torch.device(device)
|
||||
device_index = device.index
|
||||
if device_index is None:
|
||||
device_index = torch.accelerator.current_device_index()
|
||||
return torch.zeros(
|
||||
num_compute_units(device_index) * 8,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
spec = _EXPORTS.get(name)
|
||||
if spec is None:
|
||||
|
||||
Reference in New Issue
Block a user