diff --git a/requirements/cuda.txt b/requirements/cuda.txt index e1d0bc932b..ad2bc63df1 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -28,4 +28,4 @@ quack-kernels==0.6.5 # Required by MSA tokenspeed-mla==2.0.2; platform_system == "Linux" # Humming kernels for quantization gemm -humming-kernels[cu13]==0.1.16 +humming-kernels[cu13]==0.1.17 diff --git a/vllm/model_executor/kernels/linear/mixed_precision/humming.py b/vllm/model_executor/kernels/linear/mixed_precision/humming.py index a2d88705d2..6d6933776d 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/humming.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/humming.py @@ -9,6 +9,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( kInt8DynamicTokenSym, ) from vllm.platforms import current_platform +from vllm.utils.humming import create_humming_locks from vllm.utils.import_utils import has_humming from .MPLinearKernel import MPLinearKernel, MPLinearLayerConfig @@ -71,7 +72,7 @@ class HummingLinearKernel(MPLinearKernel): input_schema=input_schema, ) self.compute_config = get_humming_linear_compute_config() - self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) + self.locks = create_humming_locks(layer.weight.device) def apply_weights( self, diff --git a/vllm/model_executor/kernels/linear/mxfp4/humming.py b/vllm/model_executor/kernels/linear/mxfp4/humming.py index 0504fc5482..4325b917db 100644 --- a/vllm/model_executor/kernels/linear/mxfp4/humming.py +++ b/vllm/model_executor/kernels/linear/mxfp4/humming.py @@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.humming import ( quant_key_to_input_schema, ) from vllm.platforms import current_platform +from vllm.utils.humming import create_humming_locks from vllm.utils.import_utils import has_humming from .base import MxFp4LinearKernel, MxFp4LinearLayerConfig @@ -60,7 +61,7 @@ class HummingMxFp4LinearKernel(MxFp4LinearKernel): layer, quant_config, input_schema=input_schema ) self.compute_config = get_humming_linear_compute_config() - self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) + self.locks = create_humming_locks(layer.weight.device) def apply_weights( self, diff --git a/vllm/model_executor/kernels/linear/mxfp6/humming.py b/vllm/model_executor/kernels/linear/mxfp6/humming.py index fff7584d93..bdc36758cd 100644 --- a/vllm/model_executor/kernels/linear/mxfp6/humming.py +++ b/vllm/model_executor/kernels/linear/mxfp6/humming.py @@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import ( ) from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp6E2M3Static from vllm.platforms import current_platform +from vllm.utils.humming import create_humming_locks from vllm.utils.import_utils import has_humming from .base import MxFp6LinearKernel, MxFp6LinearLayerConfig @@ -66,7 +67,7 @@ class HummingMxFp6LinearKernel(MxFp6LinearKernel): layer, quant_config, input_schema=input_schema ) self.compute_config = get_humming_linear_compute_config() - self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) + self.locks = create_humming_locks(layer.weight.device) def apply_weights( self, diff --git a/vllm/model_executor/kernels/linear/mxfp8/humming.py b/vllm/model_executor/kernels/linear/mxfp8/humming.py index 63e9514f4e..bc5bc71175 100644 --- a/vllm/model_executor/kernels/linear/mxfp8/humming.py +++ b/vllm/model_executor/kernels/linear/mxfp8/humming.py @@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import ( ) from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic from vllm.platforms import current_platform +from vllm.utils.humming import create_humming_locks from vllm.utils.import_utils import has_humming from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig @@ -57,7 +58,7 @@ class HummingMxfp8LinearKernel(Mxfp8LinearKernel): layer, quant_config, input_schema=input_schema ) self.compute_config = get_humming_linear_compute_config() - self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) + self.locks = create_humming_locks(layer.weight.device) def apply_weights( self, diff --git a/vllm/model_executor/kernels/linear/nvfp4/humming.py b/vllm/model_executor/kernels/linear/nvfp4/humming.py index 3829c39b74..3e92e4ec7a 100644 --- a/vllm/model_executor/kernels/linear/nvfp4/humming.py +++ b/vllm/model_executor/kernels/linear/nvfp4/humming.py @@ -12,6 +12,7 @@ from vllm.model_executor.layers.quantization.utils.humming import ( ) from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic from vllm.platforms import current_platform +from vllm.utils.humming import create_humming_locks from vllm.utils.import_utils import has_humming from .base import NvFp4LinearKernel, NvFp4LinearLayerConfig @@ -71,7 +72,7 @@ class HummingNvFp4LinearKernel(NvFp4LinearKernel): layer, quant_config, input_schema=input_schema ) self.compute_config = get_humming_linear_compute_config() - self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) + self.locks = create_humming_locks(layer.weight.device) def apply_weights( self, diff --git a/vllm/model_executor/kernels/linear/scaled_mm/humming.py b/vllm/model_executor/kernels/linear/scaled_mm/humming.py index 3381a3381f..4a68c3dbe4 100644 --- a/vllm/model_executor/kernels/linear/scaled_mm/humming.py +++ b/vllm/model_executor/kernels/linear/scaled_mm/humming.py @@ -11,6 +11,7 @@ from vllm.model_executor.layers.quantization.utils.humming import ( prepare_humming_linear_layer_config, ) from vllm.platforms import current_platform +from vllm.utils.humming import create_humming_locks from vllm.utils.import_utils import has_humming from .ScaledMMLinearKernel import ( @@ -81,7 +82,7 @@ class HummingFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel): convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) self.layer_config = prepare_humming_linear_layer_config(layer, quant_config) self.compute_config = get_humming_linear_compute_config() - self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) + self.locks = create_humming_locks(layer.weight.device) def apply_weights( self, @@ -148,7 +149,7 @@ class HummingInt8ScaledMMLinearKernel(Int8ScaledMMLinearKernel): convert_linear_layer_to_humming_standard(layer=layer, name_map=name_map) self.layer_config = prepare_humming_linear_layer_config(layer, quant_config) self.compute_config = get_humming_linear_compute_config() - self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) + self.locks = create_humming_locks(layer.weight.device) def apply_weights( self, diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py index efc2b3f6c8..a7be558599 100644 --- a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py @@ -66,6 +66,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import ( ) from vllm.platforms import current_platform from vllm.scalar_type import ScalarType +from vllm.utils.humming import create_humming_locks from vllm.utils.import_utils import has_humming from vllm.utils.platform_utils import num_compute_units from vllm.v1.worker.workspace import current_workspace_manager @@ -140,7 +141,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): "w13": humming_quant_config.w1_humming_config, "w2": humming_quant_config.w2_humming_config, } - self.locks = torch.zeros(1024, dtype=torch.int32, device=moe_config.device) + self.locks = create_humming_locks(moe_config.device) self.num_experts = moe_config.num_local_experts self.global_num_experts = moe_config.num_experts self.quant_config = humming_quant_config diff --git a/vllm/model_executor/layers/quantization/humming.py b/vllm/model_executor/layers/quantization/humming.py index 6bb0faa7c8..d0c225cebc 100644 --- a/vllm/model_executor/layers/quantization/humming.py +++ b/vllm/model_executor/layers/quantization/humming.py @@ -584,7 +584,9 @@ class HummingLinearMethod(LinearMethodBase): param_dtype=layer.param_dtype, allow_fallback=self.quant_config.allow_input_schema_fallback, ) - self.layer_config = _hm.prepare_layer_config( + source_tensors = dict(layer.named_parameters()) + self.layer_config, tensors = _hm.prepare_and_transform_humming_tensors( + tensors=source_tensors, shape_n=layer.output_partition_sizes_sum, shape_k=layer.input_size_per_partition, weight_schema=self.weight_schema, @@ -595,8 +597,6 @@ class HummingLinearMethod(LinearMethodBase): torch_dtype=layer.param_dtype, device=layer.weight.device, ) - source_tensors = dict(layer.named_parameters()) - tensors = _hm.transform_humming_tensors(self.layer_config, source_tensors) name = self.input_schema.static_tensor_scale_name if name is not None: tensors[name] = source_tensors[name] @@ -610,7 +610,7 @@ class HummingLinearMethod(LinearMethodBase): setattr(layer, name, param) self.compute_config = get_humming_linear_compute_config() - self.locks = torch.zeros(1024, dtype=torch.int32, device=layer.weight.device) + self.locks = _hm.create_humming_locks(layer.weight.device) def apply( self, diff --git a/vllm/model_executor/layers/quantization/utils/humming/linear.py b/vllm/model_executor/layers/quantization/utils/humming/linear.py index fcce3dabfe..c82f4fd33f 100644 --- a/vllm/model_executor/layers/quantization/utils/humming/linear.py +++ b/vllm/model_executor/layers/quantization/utils/humming/linear.py @@ -64,8 +64,7 @@ def prepare_humming_linear_layer_config( BaseInputSchema, BaseWeightSchema, HummingInputSchema, - prepare_layer_config, - transform_humming_tensors, + prepare_and_transform_humming_tensors, ) weight_schema = BaseWeightSchema.from_config(quant_config) @@ -127,7 +126,8 @@ def prepare_humming_linear_layer_config( input_tensors.pop(name, None) # Step 2: transform weight (humming standard format) for forwarding. - config = prepare_layer_config( + config, tensors = prepare_and_transform_humming_tensors( + tensors=tensors, shape_n=sum(layer.output_partition_sizes), shape_k=input_size_per_partition, weight_schema=weight_schema, @@ -138,7 +138,6 @@ def prepare_humming_linear_layer_config( torch_dtype=layer.params_dtype, device=tensors["weight"].device, ) - tensors = transform_humming_tensors(config, tensors) tensors.update(input_tensors) if "bias" in tensors: zero_bias = torch.zeros_like(tensors["bias"]) diff --git a/vllm/model_executor/layers/quantization/utils/humming/moe.py b/vllm/model_executor/layers/quantization/utils/humming/moe.py index 37eb9c26f0..418e9acbe7 100644 --- a/vllm/model_executor/layers/quantization/utils/humming/moe.py +++ b/vllm/model_executor/layers/quantization/utils/humming/moe.py @@ -397,11 +397,15 @@ def _prepare_and_transform_sublayer( ) -> "LayerConfig": """Prepare Humming configuration and transform one sublayer's tensors.""" from vllm.utils.humming import ( - prepare_layer_config, - transform_humming_tensors, + prepare_and_transform_humming_tensors, ) - config = prepare_layer_config( + source_tensors = _extract_sublayer_tensors(layer, sublayer_name) + for name in ("input_scale", "input_scale_2"): + if name != input_schema.static_tensor_scale_name: + source_tensors.pop(name, None) + config, tensors = prepare_and_transform_humming_tensors( + tensors=source_tensors, shape_n=shape_n, shape_k=shape_k, pad_n_to_multiple=256, @@ -413,11 +417,6 @@ def _prepare_and_transform_sublayer( torch_dtype=param_dtype, device=getattr(layer, f"{sublayer_name}_weight").device, ) - source_tensors = _extract_sublayer_tensors(layer, sublayer_name) - for name in ("input_scale", "input_scale_2"): - if name != input_schema.static_tensor_scale_name: - source_tensors.pop(name, None) - tensors = transform_humming_tensors(config, source_tensors) for name in ("input_scale", "input_scale_2"): if name in source_tensors: tensors[name] = source_tensors[name] diff --git a/vllm/model_executor/layers/quantization/utils/humming/schema.py b/vllm/model_executor/layers/quantization/utils/humming/schema.py index aaf6d4a66c..c84e8a370d 100644 --- a/vllm/model_executor/layers/quantization/utils/humming/schema.py +++ b/vllm/model_executor/layers/quantization/utils/humming/schema.py @@ -322,22 +322,6 @@ def check_and_fallback_input_schema( input_group_size = input_schema.input_scale_group_size input_scale_dtype = input_schema.input_scale_dtype - def is_compatible(candidate: "HummingInputSchema") -> bool: - check_weight_schema = weight_schema - if ( - candidate.a_dtype in (None, humming_dtypes.float16, humming_dtypes.bfloat16) - and weight_schema.b_dtype == humming_dtypes.float4e2m1 - and weight_group_size == 16 - and weight_schema.bs_dtype - in (None, humming_dtypes.float16, humming_dtypes.bfloat16) - ): - # Humming rejects folded NVFP4 scales despite A16 software dequant - # supporting them. Relax only this check, preserving kernel metadata. - check_weight_schema = dataclasses.replace( - weight_schema, bs_dtype=humming_dtypes.float8e4m3 - ) - return candidate.is_compatible_with(check_weight_schema, param_dtype) - def is_deprecated(dtype: "humming_dtypes.DataType | None") -> bool: if dtype is None or dtype.num_bits == 16: return False @@ -349,7 +333,7 @@ def check_and_fallback_input_schema( is_fp8_deprecated = dtype.num_bits < 16 and 100 <= sm_version < 120 return is_int4_deprecated or is_int8_deprecated or is_fp8_deprecated - if is_compatible(input_schema): + if input_schema.is_compatible_with(weight_schema, param_dtype): if not allow_fallback: if is_deprecated(a_dtype): msg = f"Humming {a_dtype} activation is slow on SM{sm_version} now." @@ -406,7 +390,7 @@ def check_and_fallback_input_schema( input_scale_dtype=scale_dtype, input_quant_mode=quant_mode, ) - if is_compatible(candidate): + if candidate.is_compatible_with(weight_schema, param_dtype): return candidate raise ValueError( diff --git a/vllm/utils/humming.py b/vllm/utils/humming.py index 574a00f16b..8004812ea4 100644 --- a/vllm/utils/humming.py +++ b/vllm/utils/humming.py @@ -11,6 +11,10 @@ symbols to both the ``TYPE_CHECKING`` imports and ``_EXPORTS``, using import importlib from typing import TYPE_CHECKING, Any +import torch + +from vllm.utils.platform_utils import num_compute_units + if TYPE_CHECKING: from humming import dtypes as dtypes from humming.config import GemmType as GemmType @@ -50,8 +54,9 @@ if TYPE_CHECKING: from humming.schema.modelopt import ( ModeloptNvfp4WeightSchema as ModeloptNvfp4WeightSchema, ) - from humming.transform import prepare_layer_config as prepare_layer_config - from humming.transform import transform_humming_tensors as transform_humming_tensors + from humming.transform import ( + prepare_and_transform_humming_tensors as prepare_and_transform_humming_tensors, + ) from humming.tune import get_heuristics_config as get_heuristics_config from humming.utils.weight import quantize_weight as quantize_weight @@ -68,8 +73,9 @@ _EXPORTS: dict[str, str] = { "may_process_input": "humming.forward:may_process_input", "process_input": "humming.ops:process_input", "may_quant_input": "humming.forward:may_quant_input", - "prepare_layer_config": "humming.transform:prepare_layer_config", - "transform_humming_tensors": "humming.transform:transform_humming_tensors", + "prepare_and_transform_humming_tensors": ( + "humming.transform:prepare_and_transform_humming_tensors" + ), "get_heuristics_config": "humming.tune:get_heuristics_config", "BaseInputSchema": "humming.schema:BaseInputSchema", "BaseWeightSchema": "humming.schema:BaseWeightSchema", @@ -91,6 +97,18 @@ _EXPORTS: dict[str, str] = { } +def create_humming_locks(device: torch.device | str) -> torch.Tensor: + device = torch.device(device) + device_index = device.index + if device_index is None: + device_index = torch.accelerator.current_device_index() + return torch.zeros( + num_compute_units(device_index) * 8, + dtype=torch.int32, + device=device, + ) + + def __getattr__(name: str) -> Any: spec = _EXPORTS.get(name) if spec is None: