diff --git a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py index 5177fa0cde..047ae46c0d 100644 --- a/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py +++ b/vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py @@ -36,28 +36,42 @@ from vllm.model_executor.layers.fused_moe.utils import ( _resize_cache, swiglu_limit_func, ) -from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + QuantKey, + kFp8DynamicTokenSym, + kFp8Static128BlockSym, + kFp8StaticChannelSym, + kInt4Static, + kInt8Static, + kMxfp4Dynamic, + kMxfp4Static, + kMxfp8Dynamic, + kMxfp8Static, + kNvfp4Static, +) from vllm.platforms import current_platform -from vllm.utils.humming import GemmType as HummingGemmType -from vllm.utils.humming import HummingLayerMeta, HummingMethod, dtypes +from vllm.utils.import_utils import has_humming from vllm.v1.worker.workspace import current_workspace_manager if TYPE_CHECKING: from vllm.model_executor.layers.fused_moe import RoutedExperts + from vllm.utils.humming import GemmType as HummingGemmType logger = init_logger(__name__) -def get_humming_moe_gemm_type() -> str: - env_gemm_type: str = envs.VLLM_HUMMING_MOE_GEMM_TYPE or "" - env_gemm_type = env_gemm_type.lower() - if env_gemm_type == "indexed": - gemm_type = env_gemm_type - elif env_gemm_type in ["grouped_contiguous", "grouped"]: - gemm_type = "grouped_contiguous" - else: - gemm_type = "indexed" +def get_humming_moe_gemm_type() -> str | None: + env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE + gemm_type = None + if env_gemm_type is not None: + env_gemm_type = env_gemm_type.lower() + if env_gemm_type == "indexed": + gemm_type = env_gemm_type + elif env_gemm_type in ["grouped_contiguous", "grouped"]: + gemm_type = "grouped_contiguous" + else: + gemm_type = "indexed" logger.info_once(f"Using {gemm_type} gemm for humming moe") # noqa return gemm_type @@ -89,6 +103,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): self._permute_scratch: MoEPermuteScratch | None = None def init_humming_moe(self): + from vllm.utils.humming import HummingMethod + self.compute_config = { "use_batch_invariant": envs.VLLM_BATCH_INVARIANT, "use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM, @@ -141,7 +157,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): return math.ceil(global_valid_shape_m * num_experts / global_num_experts) @staticmethod - def humming_gemm_type() -> HummingGemmType: + def humming_gemm_type() -> "HummingGemmType": raise NotImplementedError @classmethod @@ -153,12 +169,51 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): weight_key: QuantKey | None, activation_key: QuantKey | None, ) -> bool: + SUPPORTED_W_A = [ + (kMxfp4Static, None), + (kMxfp4Static, kMxfp4Dynamic), + (kMxfp4Static, kMxfp8Dynamic), + (kMxfp4Static, kFp8DynamicTokenSym), + (kNvfp4Static, None), + (kNvfp4Static, kFp8DynamicTokenSym), + (kMxfp8Static, None), + (kMxfp8Static, kFp8DynamicTokenSym), + (kFp8StaticChannelSym, None), + (kFp8StaticChannelSym, kFp8DynamicTokenSym), + (kFp8Static128BlockSym, None), + (kFp8Static128BlockSym, kFp8DynamicTokenSym), + (kInt4Static, None), + (kInt4Static, kFp8DynamicTokenSym), + (kInt8Static, None), + (kInt8Static, kFp8DynamicTokenSym), + ] + return (weight_key, activation_key) in SUPPORTED_W_A + + @property + def expects_unquantized_inputs(self) -> bool: + """ + Humming kernels handle input quantization internally via + HummingMethod.may_quant_input() in the apply() method. + + This property tells the prepare/finalize step to skip input + quantization (by setting defer_input_quant=True) and pass + unquantized inputs to the experts. This prevents double + quantization: once in prepare and once in Humming's apply(). + + Returns: + True to indicate that this expert expects unquantized inputs + and will handle quantization internally. + """ return True @staticmethod def _supports_current_device() -> bool: platform = current_platform - return platform.is_cuda() and platform.has_device_capability((7, 5)) + return ( + has_humming() + and platform.is_cuda() + and platform.has_device_capability((7, 5)) + ) @staticmethod def _supports_no_act_and_mul() -> bool: @@ -182,10 +237,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): @staticmethod def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool: - return not ( - moe_parallel_config.use_fi_nvl_two_sided_kernels - or moe_parallel_config.use_fi_nvl_one_sided_kernels - ) + return True def moe_problem_size( self, @@ -194,6 +246,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): w2: torch.Tensor, topk_ids: torch.Tensor, ) -> tuple[int, int, int, int, int]: + from vllm.utils.humming import HummingLayerMeta + meta1: HummingLayerMeta = self.layer.humming_metas["w13"] meta2: HummingLayerMeta = self.layer.humming_metas["w2"] @@ -215,6 +269,9 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): return meta1.num_experts, num_tokens, meta1.shape_n // 2, meta1.shape_k, top_k def get_buffer_metas(self, M: int, topk: int, activation: MoEActivation): + from vllm.utils.humming import GemmType as HummingGemmType + from vllm.utils.humming import dtypes + num_experts = self.num_experts N = self.layer.intermediate_size_per_partition K = self.layer.hidden_size @@ -254,7 +311,9 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): torch_dtype_map = { dtypes.float16: torch.float16, dtypes.bfloat16: torch.bfloat16, + dtypes.float32: torch.float32, dtypes.float8e4m3: torch.float8_e4m3fn, + dtypes.float8e5m2: torch.float8_e5m2, dtypes.int8: torch.int8, dtypes.int4: torch.uint8, } @@ -289,7 +348,13 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): for key in buffer_metas: meta = buffer_metas[key] if "quanted" in key and a_dtype.num_bits == 4: - meta["shape"] = meta["shape"][:-1] + (meta["shape"][-1] // 2,) + last_dim = meta["shape"][-1] + if last_dim % 2 != 0: + raise ValueError( + f"Int4 packing requires last dimension to be even, " + f"got {last_dim} for buffer '{key}'" + ) + meta["shape"] = meta["shape"][:-1] + (last_dim // 2,) if num_bits == 16: required_buffers = ["gate_up_output", "activation_output", "down_output"] @@ -325,8 +390,13 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): output_key = "down_output" if self.is_batched() else "output" output_shape = buffer_metas[output_key]["shape"] + elem_size = self.layer.params_dtype.itemsize - return (workspace1_nbytes // 2,), (workspace2_nbytes // 2,), output_shape + return ( + (workspace1_nbytes // elem_size,), + (workspace2_nbytes // elem_size,), + output_shape, + ) def workspace_shapes( self, @@ -344,7 +414,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): def make_workspaces(self, M: int, topk: int, activation: MoEActivation): shapes = self._workspace_shapes(M, topk, activation) workspace1_shape, workspace2_shape, output_shape = shapes - torch_dtype = self.layer.param_dtype + torch_dtype = self.layer.params_dtype workspace1, workspace2 = current_workspace_manager().get_simultaneous( (workspace1_shape, torch_dtype), (workspace2_shape, torch_dtype), @@ -370,45 +440,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): return buffers - def apply( - self, - output: torch.Tensor, - hidden_states: torch.Tensor, - w1: torch.Tensor, - w2: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - activation: MoEActivation, - global_num_experts: int, - expert_map: torch.Tensor | None, - a1q_scale: torch.Tensor | None, - a2_scale: torch.Tensor | None, - workspace13: torch.Tensor, - workspace2: torch.Tensor, - expert_tokens_meta: mk.ExpertTokensMetadata | None, - apply_router_weight_on_input: bool, - ): - assert not apply_router_weight_on_input - - self.main_apply( - hidden_states=hidden_states, - topk_weights=topk_weights, - topk_ids=topk_ids, - workspace1=workspace13, - workspace2=workspace2, - expert_tokens_meta=expert_tokens_meta, - ) - - def main_apply( - self, - hidden_states: torch.Tensor, - topk_weights: torch.Tensor, - topk_ids: torch.Tensor, - workspace1: torch.Tensor, - workspace2: torch.Tensor, - expert_tokens_meta: mk.ExpertTokensMetadata | None, - ): - raise NotImplementedError + # Note: apply method is implemented by subclasses following the + # standard FusedMoEExpertsModular.apply signature @staticmethod def is_supported_config( @@ -418,24 +451,27 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular): activation_key: QuantKey | None, activation_format: mk.FusedMoEActivationFormat, ) -> tuple[bool, str | None]: - if activation_format == mk.FusedMoEActivationFormat.BatchedExperts: - supported = cls.activation_format() == activation_format - reason = "activation_format mismatched" - elif activation_format == mk.FusedMoEActivationFormat.Standard: - if cls.activation_format() != mk.FusedMoEActivationFormat.Standard: - supported = False - reason = "activation_format mismatched" - else: - assert hasattr(cls, "humming_gemm_type") - gemm_type = cls.humming_gemm_type().value.lower() - preferred_gemm_type = get_humming_moe_gemm_type().lower() - supported = preferred_gemm_type == gemm_type - reason = "preferred gemm type mismatched" - else: - supported = False - reason = "unsupported activation_format" + supported, reason = mk.FusedMoEExpertsModular.is_supported_config( + cls, + moe_config, + weight_key, + activation_key, + activation_format, + ) - return supported, None if supported else reason + if supported: + assert hasattr(cls, "humming_gemm_type") + gemm_type = cls.humming_gemm_type().value.lower() + preferred_gemm_type = get_humming_moe_gemm_type() + if preferred_gemm_type is not None: + supported = preferred_gemm_type.lower() == gemm_type + if not supported: + reason = ( + f"preferred gemm type {preferred_gemm_type} != " + f"supported gemm type {gemm_type}" + ) + + return supported, reason def apply_activation( self, @@ -459,7 +495,9 @@ class HummingIndexedExperts(HummingExpertsBase): return mk.FusedMoEActivationFormat.Standard @staticmethod - def humming_gemm_type() -> HummingGemmType: + def humming_gemm_type() -> "HummingGemmType": + from vllm.utils.humming import GemmType as HummingGemmType + return HummingGemmType.INDEXED def prepare_humming_moe_kwargs( @@ -470,12 +508,18 @@ class HummingIndexedExperts(HummingExpertsBase): ) -> tuple[dict[str, Any], dict[str, Any]]: valid_shape_m = self.estimate_local_valid_shape_m(topk_ids) + moe_block_size = None for min_shape_m, max_shape_m, config in self.w13_tuning_config: if valid_shape_m > min_shape_m and valid_shape_m <= max_shape_m: moe_block_size = config["block_shape"][0] break - else: - raise ValueError(f"cannot found moe_block_size for shape {valid_shape_m}") + + if moe_block_size is None: + logger.warning_once( + "No tuning config found for shape %s, using default block_size=64", + valid_shape_m, + ) + moe_block_size = 64 sorted_ids, expert_ids, num_tokens_padded = moe_align_block_size( topk_ids=topk_ids, @@ -501,27 +545,47 @@ class HummingIndexedExperts(HummingExpertsBase): return moe_kwargs1, moe_kwargs2 - def main_apply( + def apply( self, + output: torch.Tensor, hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, - workspace1: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, workspace2: torch.Tensor, expert_tokens_meta: mk.ExpertTokensMetadata | None, - ): + apply_router_weight_on_input: bool, + ) -> None: + """ + Standard apply implementation for Humming indexed experts. + + Note: Humming kernels handle weights and quantization internally through + the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used. + The output is written into workspace13 via the buffer management. + """ + from vllm.utils.humming import HummingMethod + + assert not apply_router_weight_on_input + hidden_states = hidden_states.view(-1, hidden_states.size(-1)) buffers = self.prepare_buffers( - workspace1, + workspace13, workspace2, topk_ids.size(0), topk_ids.size(1), - self.layer.activation, + activation, ) moe_kwargs1, moe_kwargs2 = self.prepare_humming_moe_kwargs( topk_ids=topk_ids, - expert_map=self.layer.expert_map, + expert_map=expert_map, expert_tokens_meta=expert_tokens_meta, ) @@ -542,7 +606,7 @@ class HummingIndexedExperts(HummingExpertsBase): ) self.apply_activation( - activation=self.layer.activation, + activation=activation, input=buffers["gate_up_output"], output=buffers["activation_output"], ) @@ -567,10 +631,13 @@ class HummingIndexedExperts(HummingExpertsBase): inputs=buffers["down_output"].view(*topk_ids.shape, -1), topk_weights=topk_weights, topk_ids=topk_ids, - expert_map=self.layer.expert_map, + expert_map=expert_map, outputs=buffers["output"], ) + # Note: output is already written to buffers["output"] + # which aliases workspace13/output + class HummingGroupedExperts(HummingExpertsBase): def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: @@ -581,35 +648,57 @@ class HummingGroupedExperts(HummingExpertsBase): return mk.FusedMoEActivationFormat.Standard @staticmethod - def humming_gemm_type() -> HummingGemmType: + def humming_gemm_type() -> "HummingGemmType": + from vllm.utils.humming import GemmType as HummingGemmType + return HummingGemmType.GROUPED_CONTIGUOUS - def main_apply( + def apply( self, + output: torch.Tensor, hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, - workspace1: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, workspace2: torch.Tensor, expert_tokens_meta: mk.ExpertTokensMetadata | None, - ): + apply_router_weight_on_input: bool, + ) -> None: + """ + Standard apply implementation for Humming grouped experts. + + Note: Humming kernels handle weights and quantization internally through + the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used. + The output is written into workspace13 via the buffer management. + """ + from vllm.utils.humming import HummingMethod + + assert not apply_router_weight_on_input + valid_shape_m = self.estimate_local_valid_shape_m(topk_ids) buffers = self.prepare_buffers( - workspace1, + workspace13, workspace2, topk_ids.size(0), topk_ids.size(1), - self.layer.activation, + activation, ) hidden_states, _, expert_first_token_offset, inv_perm, _ = moe_permute( hidden_states=hidden_states, a1q_scale=None, topk_ids=topk_ids, - n_expert=self.global_num_experts, + n_expert=global_num_experts, n_local_expert=self.num_experts, - expert_map=self.layer.expert_map, + expert_map=expert_map, scratch=self._get_permute_scratch(), ) @@ -633,7 +722,7 @@ class HummingGroupedExperts(HummingExpertsBase): ) self.apply_activation( - activation=self.layer.activation, + activation=activation, input=buffers["gate_up_output"], output=buffers["activation_output"], ) @@ -665,6 +754,9 @@ class HummingGroupedExperts(HummingExpertsBase): expert_first_token_offset=expert_first_token_offset, ) + # Note: output is already written to buffers["output"] + # which aliases workspace13/output + class BatchedHummingGroupedExperts(HummingExpertsBase): def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce: @@ -675,29 +767,51 @@ class BatchedHummingGroupedExperts(HummingExpertsBase): return mk.FusedMoEActivationFormat.BatchedExperts @staticmethod - def humming_gemm_type() -> HummingGemmType: + def humming_gemm_type() -> "HummingGemmType": + from vllm.utils.humming import GemmType as HummingGemmType + return HummingGemmType.GROUPED_MASKED - def main_apply( + def apply( self, + output: torch.Tensor, hidden_states: torch.Tensor, + w1: torch.Tensor, + w2: torch.Tensor, topk_weights: torch.Tensor, topk_ids: torch.Tensor, - workspace1: torch.Tensor, + activation: MoEActivation, + global_num_experts: int, + expert_map: torch.Tensor | None, + a1q_scale: torch.Tensor | None, + a2_scale: torch.Tensor | None, + workspace13: torch.Tensor, workspace2: torch.Tensor, expert_tokens_meta: mk.ExpertTokensMetadata | None, - ): + apply_router_weight_on_input: bool, + ) -> None: + """ + Standard apply implementation for Humming batched grouped experts. + + Note: Humming kernels handle weights and quantization internally through + the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used. + The output is written into workspace13 via the buffer management. + """ + from vllm.utils.humming import HummingMethod + + assert not apply_router_weight_on_input assert expert_tokens_meta is not None + hidden_states = hidden_states.view(-1, hidden_states.size(-1)) valid_shape_m = self.estimate_local_valid_shape_m(topk_ids) expert_num_tokens = expert_tokens_meta.expert_num_tokens buffers = self.prepare_buffers( - workspace1, + workspace13, workspace2, topk_ids.size(0), topk_ids.size(1), - self.layer.activation, + activation, ) inputs, input_scale = HummingMethod.may_quant_input( @@ -720,7 +834,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase): ) self.apply_activation( - activation=self.layer.activation, + activation=activation, input=buffers["gate_up_output"], output=buffers["activation_output"], ) @@ -743,3 +857,6 @@ class BatchedHummingGroupedExperts(HummingExpertsBase): tuning_config=self.w2_tuning_config_str, sublayer_name="w2", ) + + # Note: output is already written to buffers["down_output"] + # which aliases workspace13/output diff --git a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py index b1b41ded11..55a767f060 100644 --- a/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py +++ b/vllm/model_executor/layers/fused_moe/oracle/mxfp4.py @@ -711,10 +711,12 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format( if mxfp4_backend == Mxfp4MoeBackend.HUMMING: from vllm.model_executor.layers.quantization.utils.humming_utils import ( - prepare_humming_moe_layer, + convert_to_humming_moe_kernel_format, ) - prepare_humming_moe_layer(layer, {"quant_method": "gpt_oss_mxfp4"}) + convert_to_humming_moe_kernel_format( + layer, quant_config={"quant_method": "gpt_oss_mxfp4"} + ) return ( layer.w13_weight, layer.w2_weight, @@ -1277,10 +1279,12 @@ def convert_weight_to_mxfp4_moe_kernel_format( if mxfp4_backend == Mxfp4MoeBackend.HUMMING: from vllm.model_executor.layers.quantization.utils.humming_utils import ( - prepare_humming_moe_layer, + convert_to_humming_moe_kernel_format, ) - prepare_humming_moe_layer(layer, {"quant_method": "mxfp4"}) + convert_to_humming_moe_kernel_format( + layer, quant_config={"quant_method": "mxfp4"} + ) return ( layer.w13_weight, layer.w2_weight, @@ -1569,7 +1573,7 @@ def make_mxfp4_moe_quant_config( w2_bias: torch.Tensor | None = None, a1_scale: torch.Tensor | None = None, a2_scale: torch.Tensor | None = None, - layer: torch.nn.Module | None = None, + layer: "RoutedExperts | None" = None, ) -> FusedMoEQuantConfig | None: """Create a FusedMoEQuantConfig for the given MXFP4 backend.""" if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4: @@ -1662,7 +1666,7 @@ def make_mxfp4_moe_quant_config( get_humming_moe_quant_config, ) - assert isinstance(layer, RoutedExperts) + assert layer is not None return get_humming_moe_quant_config( layer, gemm1_alpha=gemm1_alpha, @@ -1703,6 +1707,7 @@ def make_mxfp4_moe_kernel( assert prepare_finalize is not None logger.info_once("Using %s", prepare_finalize.__class__.__name__) + logger.info_once("Using %s", experts_cls.__name__) extra_kwargs = {} if mxfp4_backend == Mxfp4MoeBackend.HUMMING: diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py index 906b0727b1..1a0cd8ed10 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_w4a4_mxfp4.py @@ -143,6 +143,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod): mxfp4_backend=self.mxfp4_backend, w1_scale=layer.w13_weight_scale, w2_scale=layer.w2_weight_scale, + layer=layer, ) def process_weights_after_loading(self, layer: RoutedExperts) -> None: diff --git a/vllm/model_executor/layers/quantization/fp8.py b/vllm/model_executor/layers/quantization/fp8.py index a71754769a..86818ed4b7 100644 --- a/vllm/model_executor/layers/quantization/fp8.py +++ b/vllm/model_executor/layers/quantization/fp8.py @@ -706,15 +706,15 @@ class Fp8MoEMethod(FusedMoEMethodBase): layer.w2_weight.is_shuffled = True self.moe_quant_config = self.get_fused_moe_quant_config(layer) - if self.moe_quant_config: - assert self.experts_cls is not None - self.moe_kernel = make_fp8_moe_kernel( - moe_quant_config=self.moe_quant_config, - moe_config=self.moe, - fp8_backend=self.fp8_backend, - experts_cls=self.experts_cls, - routing_tables=layer._expert_routing_tables(), - ) + assert self.moe_quant_config is not None + assert self.experts_cls is not None + self.moe_kernel = make_fp8_moe_kernel( + moe_quant_config=self.moe_quant_config, + moe_config=self.moe, + fp8_backend=self.fp8_backend, + experts_cls=self.experts_cls, + routing_tables=layer._expert_routing_tables(), + ) def process_weights_after_loading(self, layer: RoutedExperts) -> None: # Allow for accessing weights and scales in standard way. diff --git a/vllm/model_executor/layers/quantization/humming.py b/vllm/model_executor/layers/quantization/humming.py index eb598ff7f7..9bbbf41115 100644 --- a/vllm/model_executor/layers/quantization/humming.py +++ b/vllm/model_executor/layers/quantization/humming.py @@ -30,6 +30,14 @@ from vllm.model_executor.layers.quantization.base_config import ( QuantizationConfig, QuantizeMethodBase, ) +from vllm.model_executor.layers.quantization.utils.humming_utils import ( + convert_to_humming_moe_kernel_format, + get_humming_moe_quant_config, + input_schema_to_quant_key, + make_humming_moe_kernel, + select_humming_moe_experts, + weight_schema_to_quant_key, +) from vllm.model_executor.model_loader.weight_utils import default_weight_loader from vllm.model_executor.parameter import ( BasevLLMParameter, @@ -106,7 +114,7 @@ def prepare_param(tensor, name, extra_attrs): return param -def prepare_moe_param(tensor, name, extra_attrs): +def prepare_moe_param(tensor: torch.Tensor, name: str, extra_attrs: dict[str, Any]): param = torch.nn.Parameter(tensor, requires_grad=False) if "scale_type" in extra_attrs: extra_attrs["quant_method"] = extra_attrs["scale_type"] @@ -605,12 +613,27 @@ class HummingMoEMethod(FusedMoEMethodBase): ) -> None: super().__init__(moe) self.quant_config = quant_config - self.moe = moe self.weight_schema = quant_config.weight_schema self.input_schema = quant_config.input_schema self.force_weight_schema = quant_config.force_weight_schema self.force_input_schema = quant_config.force_input_schema + # Derive QuantKeys from humming schemas. + # Prefer force schemas (the final format after requant) over base. + weight_key = weight_schema_to_quant_key( + self.force_weight_schema or self.weight_schema + ) + activation_key = input_schema_to_quant_key( + self.force_input_schema or self.input_schema + ) + + # Select Humming MoE experts + self.experts_cls = select_humming_moe_experts( + config=self.moe, + weight_key=weight_key, + activation_key=activation_key, + ) + def prepare_weight_loader(self, layer, weight_loader): def new_weight_loader( param: torch.nn.Parameter, @@ -647,7 +670,7 @@ class HummingMoEMethod(FusedMoEMethodBase): sublayer_name = "w2" if shard_id == "w2" else "w13" param = getattr(layer, sublayer_name + "_" + key) - part_subccess = param.weight_loader( + part_success = param.weight_loader( param=param, loaded_weight=tensor.cpu(), weight_name=shard_id + "_" + key, @@ -655,7 +678,7 @@ class HummingMoEMethod(FusedMoEMethodBase): expert_id=expert_id, return_success=return_success, ) - success = success and part_subccess + success = success and part_success return success if return_success else None @@ -677,7 +700,7 @@ class HummingMoEMethod(FusedMoEMethodBase): def create_weights( self, - layer: torch.nn.Module, + layer: RoutedExperts, num_experts: int, hidden_size: int, intermediate_size_per_partition: int, @@ -734,160 +757,35 @@ class HummingMoEMethod(FusedMoEMethodBase): locks = torch.zeros(1024, dtype=torch.int32) layer.register_buffer("locks", locks) - def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig: - from vllm.model_executor.layers.quantization.utils.humming_utils import ( - get_humming_moe_quant_config, - ) - + def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig: return get_humming_moe_quant_config(layer) def process_weights_after_loading(self, layer: RoutedExperts) -> None: if getattr(self, "processed", False): return self.processed = True - layer.weight_schemas = {} - layer.input_schemas = {} - for sublayer_name, configs in layer.sublayer_configs.items(): - input_schema = self.input_schema - weight_schema = self.weight_schema - # convert from checkpoint format to humming format - if not isinstance(weight_schema, _hm.HummingWeightSchema): - tensors: dict[str, torch.Tensor] = dict( - (key.removeprefix(sublayer_name + "_"), value) - for key, value in layer.state_dict().items() - if key.startswith(sublayer_name + "_") - ) - shape_k_stacks = [configs["shape_k"]] - shape_n_stacks = [configs["shape_n"]] - if sublayer_name == "w13": - shape_n_stacks = [configs["shape_n"] // 2] * 2 - - weight_schema, tensors = weight_schema.convert_humming( - tensors=tensors, - shape_n_stacks=shape_n_stacks, - shape_k_stacks=shape_k_stacks, - param_dtype=layer.param_dtype, - num_experts=layer.num_experts, - ) - - input_schema, _ = input_schema.convert_humming( - tensors=tensors, - shape_n_stacks=shape_n_stacks, - shape_k_stacks=shape_k_stacks, - param_dtype=layer.param_dtype, - num_experts=layer.num_experts, - ) - - for name, _ in list(layer.named_parameters()): - if not name.startswith(sublayer_name + "_"): - continue - delattr(layer, name) - - for name, tensor in tensors.items(): - name = f"{sublayer_name}_{name}" - param = torch.nn.Parameter(tensor, requires_grad=False) - setattr(layer, name, param) - - layer.weight_schemas[sublayer_name] = weight_schema - layer.input_schemas[sublayer_name] = input_schema - - # force requant (origin quant setting -> fp16/bf16 -> new_quant setting) - assert isinstance(weight_schema, _hm.HummingWeightSchema) - force_requant = self.force_weight_schema is not None - if force_requant and weight_schema != self.force_weight_schema: - tensors = dict( - (key.removeprefix(sublayer_name + "_"), value) - for key, value in layer.state_dict().items() - if key.startswith(sublayer_name + "_") - ) - - tensors = weight_schema.requant_tensors( - tensors=tensors, - target_weight_schema=self.force_weight_schema, - param_dtype=layer.param_dtype, - ) - - weight_schema = self.force_weight_schema - - for name, _ in list(layer.named_parameters()): - if not name.startswith(sublayer_name + "_"): - continue - if name == sublayer_name + "_bias": - continue - delattr(layer, name) - - for name, tensor in tensors.items(): - name = f"{sublayer_name}_{name}" - param = torch.nn.Parameter(tensor, requires_grad=False) - setattr(layer, name, param) - - del tensors - - # prepare layer config from humming kernel - _hm.HummingMethod.prepare_layer_meta( - layer=layer, - shape_n=configs["shape_n"], - shape_k=configs["shape_k"], - pad_n_to_multiple=256, - pad_k_to_multiple=128, - input_schema=input_schema, - weight_schema=weight_schema, - has_bias=self.moe.has_bias, - num_experts=layer.num_experts, - torch_dtype=layer.param_dtype, - sublayer_name=sublayer_name, - ) - - # preprocess weight for inference - _hm.HummingMethod.transform_humming_layer( - layer, sublayer_name=sublayer_name - ) - - from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( - HummingGroupedExperts, - HummingIndexedExperts, - get_humming_moe_gemm_type, + # Convert weights to Humming kernel format + convert_to_humming_moe_kernel_format( + layer=layer, + sublayer_configs=layer.sublayer_configs, + weight_schema=self.weight_schema, + input_schema=self.input_schema, + force_weight_schema=self.force_weight_schema, ) - # use moe modular - experts: HummingIndexedExperts | HummingGroupedExperts - layer._ensure_moe_quant_config_init() + # Build the MoE kernel + self.moe_quant_config = self.get_fused_moe_quant_config(layer) assert self.moe_quant_config is not None - if get_humming_moe_gemm_type() == "indexed": - experts = HummingIndexedExperts(layer, self.moe, self.moe_quant_config) - else: - experts = HummingGroupedExperts(layer, self.moe, self.moe_quant_config) - self.experts = experts - - def select_gemm_impl( - self, - prepare_finalize, - layer: torch.nn.Module, - ): - from vllm.model_executor.layers.fused_moe import modular_kernel as mk - from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( - BatchedHummingGroupedExperts, - HummingGroupedExperts, - HummingIndexedExperts, - get_humming_moe_gemm_type, + assert self.experts_cls is not None + self.moe_kernel = make_humming_moe_kernel( + self.moe_quant_config, + self.moe, + self.experts_cls, + layer=layer, + routing_tables=layer._expert_routing_tables(), ) - activation_format = prepare_finalize.activation_format - assert self.moe_quant_config is not None - if activation_format == mk.FusedMoEActivationFormat.BatchedExperts: - return BatchedHummingGroupedExperts( - layer=layer, - moe_config=self.moe, - quant_config=self.moe_quant_config, - max_num_tokens=prepare_finalize.max_num_tokens_per_rank(), - num_dispatchers=prepare_finalize.num_dispatchers(), - ) - elif get_humming_moe_gemm_type() == "indexed": - return HummingIndexedExperts(layer, self.moe, self.moe_quant_config) - else: - return HummingGroupedExperts(layer, self.moe, self.moe_quant_config) - def apply( self, layer: RoutedExperts, @@ -896,22 +794,36 @@ class HummingMoEMethod(FusedMoEMethodBase): topk_ids: torch.Tensor, shared_experts: SharedExperts | None, shared_experts_input: torch.Tensor | None, - ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: - workspace1, workspace2, output = self.experts.make_workspaces( - M=topk_ids.size(0), - topk=topk_ids.size(1), - activation=layer.activation, - ) + ) -> torch.Tensor: + """ + Apply Humming-quantized MoE computation using the standard kernel flow. - assert workspace1.data_ptr() == output.data_ptr() + This method uses FusedMoEKernel.apply() which orchestrates: + 1. Preparation (quantization if needed - skipped for Humming via + expects_unquantized_inputs=True to prevent double quantization) + 2. Expert computation (via experts.apply()) + 3. Finalization (weight application & reduction - no-op for Humming + since it's already done internally) - self.experts.main_apply( + Humming handles all quantization, weight application, and reduction + internally in the experts.apply() method via HummingMethod calls. + + Note: Although w1/w2 weights are passed to the kernel for interface + consistency, Humming's experts.apply() reads weights directly from + the layer object via HummingMethod.forward_layer() and ignores the + w1/w2 parameters. + """ + assert self.moe_kernel is not None + return self.moe_kernel.apply( hidden_states=x, - topk_weights=topk_weights, + w1=layer.w13_weight, + w2=layer.w2_weight, topk_ids=topk_ids, - workspace1=workspace1, - workspace2=workspace2, - expert_tokens_meta=None, + topk_weights=topk_weights, + activation=layer.activation, + global_num_experts=layer.global_num_experts, + expert_map=layer.expert_map, + apply_router_weight_on_input=False, + shared_experts=shared_experts, + shared_experts_input=shared_experts_input, ) - - return output diff --git a/vllm/model_executor/layers/quantization/quark/quark_moe.py b/vllm/model_executor/layers/quantization/quark/quark_moe.py index 70b1e25959..ebe37d8dc5 100644 --- a/vllm/model_executor/layers/quantization/quark/quark_moe.py +++ b/vllm/model_executor/layers/quantization/quark/quark_moe.py @@ -1280,6 +1280,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod): gemm1_alpha=getattr(layer, "swiglu_alpha", None), gemm1_beta=getattr(layer, "swiglu_beta", None), swiglu_limit=getattr(layer, "swiglu_limit", None), + layer=layer, ) # Emulation and other schemes diff --git a/vllm/model_executor/layers/quantization/utils/humming_utils.py b/vllm/model_executor/layers/quantization/utils/humming_utils.py index 617158ae13..d84a2e12f5 100644 --- a/vllm/model_executor/layers/quantization/utils/humming_utils.py +++ b/vllm/model_executor/layers/quantization/utils/humming_utils.py @@ -1,20 +1,376 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json -from typing import Any +from typing import TYPE_CHECKING, Any import regex as re import torch +import vllm.model_executor.layers.fused_moe.modular_kernel as mk from vllm import envs -from vllm.model_executor.layers.fused_moe import RoutedExperts +from vllm.logger import init_logger +from vllm.model_executor.layers.fused_moe.all2all_utils import ( + maybe_make_prepare_finalize, +) from vllm.model_executor.layers.fused_moe.config import ( + FusedMoEConfig, FusedMoEQuantConfig, FusedMoEQuantDesc, ) +from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import ( + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, +) from vllm.model_executor.layers.linear import LinearBase -from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape -from vllm.utils.humming import BaseWeightSchema, HummingInputSchema, HummingMethod +from vllm.model_executor.layers.quantization.utils.quant_utils import ( + FP4_DTYPE, + FP8_DTYPE, + INT4_DTYPE, + INT8_DTYPE, + MXFP_SCALE_DTYPE, + GroupShape, + QuantKey, + ScaleDesc, +) +from vllm.utils.import_utils import has_humming + +if TYPE_CHECKING: + from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts + from vllm.utils.humming import ( + AWQWeightSchema, + BaseInputSchema, + BaseWeightSchema, + CompressedTensorsInputSchema, + CompressedTensorsWeightSchema, + Fp8WeightSchema, + GPTQWeightSchema, + HummingInputSchema, + HummingWeightSchema, + ) + from vllm.utils.humming import dtypes as humming_dtypes + +logger = init_logger(__name__) + +if has_humming(): + from vllm.utils.humming import dtypes as humming_dtypes + + _HUMMING_TO_QUANT_DTYPE: dict[humming_dtypes.DataType, Any] = { + humming_dtypes.float4e2m1: FP4_DTYPE, + humming_dtypes.float8e4m3: FP8_DTYPE, + humming_dtypes.float8e5m2: torch.float8_e5m2, + humming_dtypes.int8: torch.int8, + humming_dtypes.uint4: INT4_DTYPE, + humming_dtypes.uint8: INT8_DTYPE, + humming_dtypes.uint2: torch.uint8, + humming_dtypes.uint3: torch.uint8, + } + + _HUMMING_TO_SCALE_DTYPE: dict[humming_dtypes.DataType, torch.dtype] = { + humming_dtypes.float8e8m0: MXFP_SCALE_DTYPE, + humming_dtypes.float8e4m3: FP8_DTYPE, + humming_dtypes.float16: torch.float16, + humming_dtypes.bfloat16: torch.bfloat16, + humming_dtypes.float32: torch.float32, + } + + +def _group_shape(group_size: int, group_size_n: int = 0) -> GroupShape: + """ + Map humming group sizes to QuantKey GroupShape. + + group_size: elements per group along K (col); 0 means full dimension. + group_size_n: elements per group along N (row); 0 means 1 (per-row). + + GroupShape convention: row = N dim, col = K dim. + """ + if group_size == 0 and group_size_n == 0: + return GroupShape.PER_CHANNEL + + row = group_size_n if group_size_n > 0 else 1 + col = group_size if group_size > 0 else -1 + return GroupShape(row=row, col=col) + + +# ---- HummingWeightSchema (post-conversion) -------------------------------- + + +def _humming_weight_schema_to_quant_key( + schema: "HummingWeightSchema", +) -> QuantKey: + from vllm.utils.humming import WeightScaleType + + """Convert a HummingWeightSchema to a QuantKey.""" + dtype = _HUMMING_TO_QUANT_DTYPE[schema.b_dtype] + + if schema.bs_dtype is not None: + scale_dtype = _HUMMING_TO_SCALE_DTYPE[schema.bs_dtype] + else: + scale_dtype = torch.float32 + + group_shape = _group_shape( + schema.weight_scale_group_size, + schema.weight_scale_group_size_n, + ) + + scale = ScaleDesc(dtype=scale_dtype, static=True, group_shape=group_shape) + + scale2 = None + if schema.weight_scale_type == WeightScaleType.GROUP_TENSOR: + scale2 = ScaleDesc( + dtype=torch.float32, + static=True, + group_shape=GroupShape.PER_TENSOR, + ) + + return QuantKey( + dtype=dtype, + scale=scale, + scale2=scale2, + symmetric=not schema.has_zero_point, + ) + + +# ---- Checkpoint-format weight schemas (pre-conversion) -------------------- + + +def _fp8_weight_schema_to_quant_key(schema: "Fp8WeightSchema") -> QuantKey: + if schema.weight_block_size is not None: + gs_n, gs_k = schema.weight_block_size + group_shape = GroupShape(row=gs_n, col=gs_k) + else: + group_shape = GroupShape.PER_CHANNEL + + scale = ScaleDesc(dtype=torch.float32, static=True, group_shape=group_shape) + return QuantKey(dtype=FP8_DTYPE, scale=scale, symmetric=True) + + +def _awq_weight_schema_to_quant_key(schema: "AWQWeightSchema") -> QuantKey: + group_shape = _group_shape(schema.group_size) + scale = ScaleDesc( + dtype=torch.float16, + static=True, + group_shape=group_shape, + ) + return QuantKey( + dtype=INT4_DTYPE, + scale=scale, + symmetric=not schema.zero_point, + ) + + +def _gptq_weight_schema_to_quant_key(schema: "GPTQWeightSchema") -> QuantKey: + group_shape = _group_shape(schema.group_size) + scale = ScaleDesc( + dtype=torch.float16, + static=True, + group_shape=group_shape, + ) + return QuantKey(dtype=INT4_DTYPE, scale=scale, symmetric=schema.sym) + + +def _compressed_tensors_weight_schema_to_quant_key( + schema: "CompressedTensorsWeightSchema", +) -> QuantKey: + # Determine dtype from format/type/num_bits + fmt = schema.format + if fmt in ("int-quantized", "float-quantized", "naive-quantized"): + dtype = INT8_DTYPE if schema.type == "int" else FP8_DTYPE + elif "nvfp4" in fmt or "mxfp4" in fmt: + dtype = FP4_DTYPE + else: + dtype = _HUMMING_TO_QUANT_DTYPE[ + humming_dtypes.DataType.from_str(f"uint{schema.num_bits}") + ] + + # Determine group shape from strategy + if schema.strategy in ("group", "tensor_group"): + group_shape = _group_shape(schema.group_size or 0) + elif schema.strategy == "block" and schema.block_structure is not None: + group_shape = GroupShape( + row=schema.block_structure[0], + col=schema.block_structure[1], + ) + else: + group_shape = GroupShape.PER_CHANNEL + + # Determine scale dtype + if "mxfp" in fmt: + scale_dtype = MXFP_SCALE_DTYPE + elif "nvfp4" in fmt: + scale_dtype = FP8_DTYPE + else: + scale_dtype = torch.float32 + + scale = ScaleDesc(dtype=scale_dtype, static=True, group_shape=group_shape) + + scale2 = None + if "nvfp4" in fmt or schema.strategy == "tensor_group": + scale2 = ScaleDesc( + dtype=torch.float32, + static=True, + group_shape=GroupShape.PER_TENSOR, + ) + + return QuantKey( + dtype=dtype, + scale=scale, + scale2=scale2, + symmetric=schema.symmetric, + ) + + +# ---- Dispatch for any BaseWeightSchema ------------------------------------ + + +def weight_schema_to_quant_key( + schema: "BaseWeightSchema", +) -> QuantKey: + from vllm.utils.humming import ( + AWQWeightSchema, + BitnetWeightSchema, + CompressedTensorsWeightSchema, + Fp8WeightSchema, + GptOssMxfp4WeightSchema, + GPTQWeightSchema, + HummingWeightSchema, + ModeloptMxfp8WeightSchema, + ModeloptNvfp4WeightSchema, + Mxfp4WeightSchema, + ) + + """Convert any BaseWeightSchema to a QuantKey.""" + if isinstance(schema, HummingWeightSchema): + return _humming_weight_schema_to_quant_key(schema) + + # Schemas with fixed QuantKeys + if isinstance(schema, (Mxfp4WeightSchema, GptOssMxfp4WeightSchema)): + return QuantKey( + dtype=FP4_DTYPE, + scale=ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32)), + ) + if isinstance(schema, ModeloptMxfp8WeightSchema): + return QuantKey( + dtype=FP8_DTYPE, + scale=ScaleDesc(MXFP_SCALE_DTYPE, True, GroupShape(1, 32)), + ) + if isinstance(schema, ModeloptNvfp4WeightSchema): + return QuantKey( + dtype=FP4_DTYPE, + scale=ScaleDesc(FP8_DTYPE, True, GroupShape(1, 16)), + scale2=ScaleDesc(torch.float32, True, GroupShape.PER_TENSOR), + ) + if isinstance(schema, BitnetWeightSchema): + return QuantKey( + dtype=torch.uint8, + scale=ScaleDesc(torch.float32, True, GroupShape.PER_CHANNEL), + ) + + # Schemas requiring config inspection + if isinstance(schema, Fp8WeightSchema): + return _fp8_weight_schema_to_quant_key(schema) + if isinstance(schema, AWQWeightSchema): + return _awq_weight_schema_to_quant_key(schema) + if isinstance(schema, GPTQWeightSchema): + return _gptq_weight_schema_to_quant_key(schema) + if isinstance(schema, CompressedTensorsWeightSchema): + return _compressed_tensors_weight_schema_to_quant_key(schema) + + raise TypeError(f"Unsupported weight schema type: {type(schema)}") + + +# ---- HummingInputSchema (post-conversion) ---------------------------------- + + +def _humming_input_schema_to_quant_key( + schema: "HummingInputSchema", +) -> QuantKey | None: + """Convert a HummingInputSchema to a QuantKey. Returns None if + the schema represents unquantized (bf16/fp16) inputs.""" + if schema.a_dtype is None or schema.a_dtype.num_bits >= 16: + return None + + dtype = _HUMMING_TO_QUANT_DTYPE[schema.a_dtype] + + gs = schema.input_scale_group_size + group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN + + scale_dtype = MXFP_SCALE_DTYPE if gs > 0 else torch.float32 + + scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape) + + return QuantKey(dtype=dtype, scale=scale, symmetric=True) + + +# ---- Checkpoint-format input schemas (pre-conversion) ---------------------- + + +def _resolve_input_quant_key( + origin_a_dtype: "humming_dtypes.DataType", + group_size: int, +) -> QuantKey | None: + from vllm.utils.humming import HummingInputSchema + + """Resolve the actual activation QuantKey after platform fallback.""" + a_dtype = HummingInputSchema().get_fallback_input_dtype(origin_a_dtype) + if a_dtype is None or a_dtype.num_bits >= 16: + return None + + dtype = _HUMMING_TO_QUANT_DTYPE[a_dtype] + gs = group_size if a_dtype == humming_dtypes.float4e2m1 else 0 + group_shape = GroupShape(row=1, col=gs) if gs > 0 else GroupShape.PER_TOKEN + scale_dtype = MXFP_SCALE_DTYPE if gs > 0 else torch.float32 + + scale = ScaleDesc(dtype=scale_dtype, static=False, group_shape=group_shape) + return QuantKey(dtype=dtype, scale=scale, symmetric=True) + + +def _compressed_tensors_input_schema_to_quant_key( + schema: "CompressedTensorsInputSchema", +) -> QuantKey | None: + type_bits_to_dtype = { + ("float", 8): humming_dtypes.float8e4m3, + ("float", 4): humming_dtypes.float4e2m1, + ("int", 8): humming_dtypes.int8, + ("int", 4): humming_dtypes.int4, + } + origin = type_bits_to_dtype.get((schema.type, schema.num_bits)) + if origin is None: + return None + return _resolve_input_quant_key(origin, schema.group_size) + + +# ---- Dispatch for any BaseInputSchema ------------------------------------- + + +def input_schema_to_quant_key( + schema: "BaseInputSchema", +) -> QuantKey | None: + from vllm.utils.humming import ( + CompressedTensorsInputSchema, + Fp8InputSchema, + HummingInputSchema, + ModeloptNvfp4InputSchema, + ) + + """Convert any BaseInputSchema to a QuantKey. Returns None if + the schema represents unquantized (bf16/fp16) inputs.""" + if isinstance(schema, HummingInputSchema): + return _humming_input_schema_to_quant_key(schema) + + if isinstance(schema, Fp8InputSchema): + return _resolve_input_quant_key(humming_dtypes.float8e4m3, 0) + + if isinstance(schema, ModeloptNvfp4InputSchema): + return _resolve_input_quant_key( + humming_dtypes.float8e4m3, + schema.group_size, + ) + + if isinstance(schema, CompressedTensorsInputSchema): + return _compressed_tensors_input_schema_to_quant_key(schema) + + raise TypeError(f"Unsupported input schema type: {type(schema)}") def humming_is_layer_skipped(config: dict[str, Any], prefix: str): @@ -24,8 +380,9 @@ def humming_is_layer_skipped(config: dict[str, Any], prefix: str): keys = ["ignored_layers", "ignore", "modules_to_not_convert"] ignored_layers: list[str] = [] for key in keys: - ignored_layers = config.get(key, []) or [] - if not ignored_layers: + candidate = config.get(key, []) or [] + if candidate: + ignored_layers = candidate break if any(module_name in prefix for module_name in ignored_layers): @@ -79,6 +436,12 @@ def convert_linear_layer_to_humming_standard( def prepare_humming_layer(layer: LinearBase, quant_config: dict): + from vllm.utils.humming import ( + BaseWeightSchema, + HummingInputSchema, + HummingMethod, + ) + weight_schema = BaseWeightSchema.from_config(quant_config) input_schema = HummingInputSchema() @@ -140,90 +503,59 @@ def prepare_humming_layer(layer: LinearBase, quant_config: dict): layer.compute_config = json.dumps(compute_config) -def prepare_humming_moe_layer(layer: RoutedExperts, quant_config: dict): - weight_schema = BaseWeightSchema.from_config(quant_config) - input_quant_config = envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {} - if humming_is_layer_skipped(input_quant_config, layer.layer_name): - input_schema = HummingInputSchema() +def make_humming_moe_quant_config( + quant_dtype: torch.dtype | str | None, + weight_dtype: torch.dtype | str | None, + weight_group_shape: GroupShape | None = None, + w1_scale: torch.Tensor | None = None, + w2_scale: torch.Tensor | None = None, + w1_zp: torch.Tensor | None = None, + w2_zp: torch.Tensor | None = None, + w1_bias: torch.Tensor | None = None, + w2_bias: torch.Tensor | None = None, + w1_gscale: torch.Tensor | None = None, + w2_gscale: torch.Tensor | None = None, + gemm1_alpha: float | None = None, + gemm1_beta: float | None = None, + gemm1_clamp_limit: float | None = None, +) -> FusedMoEQuantConfig: + if quant_dtype is None: + a_quant_desc = FusedMoEQuantDesc(dtype=None) else: - # TODO: read input_quant_config from quant_config - input_schema = HummingInputSchema.from_config(input_quant_config) + shape = GroupShape(row=1, col=-1) + a_quant_desc = FusedMoEQuantDesc(dtype=quant_dtype, shape=shape) - is_gated = layer.activation.is_gated - shape_config = { - "w13": ( - layer.moe_config.intermediate_size_per_partition * 2, - layer.moe_config.hidden_dim, - ), - "w2": ( - layer.moe_config.hidden_dim, - layer.moe_config.intermediate_size_per_partition * (1 if is_gated else 2), - ), - } + w1_quant_desc = FusedMoEQuantDesc( + dtype=weight_dtype, + shape=weight_group_shape, + scale=w1_scale, + alpha_or_gscale=w1_gscale, + zp=w1_zp, + bias=w1_bias, + ) - layer.weight_schemas = {} - layer.input_schemas = {} + w2_quant_desc = FusedMoEQuantDesc( + dtype=weight_dtype, + shape=weight_group_shape, + scale=w2_scale, + alpha_or_gscale=w2_gscale, + zp=w2_zp, + bias=w2_bias, + ) - for sublayer_name in shape_config: - # Step 1: convert weight to humming standard format - tensors: dict[str, torch.Tensor] = dict( - (key.removeprefix(sublayer_name + "_"), value) - for key, value in layer.state_dict().items() - if key.startswith(sublayer_name + "_") - ) - - shape_n, shape_k = shape_config[sublayer_name] - shape_n_stacks = [shape_n] - shape_k_stacks = [shape_k] - if sublayer_name == "w13": - shape_n_stacks = [shape_n // 2] * 2 - - weight_schema_new, tensors = weight_schema.convert_humming( - tensors=tensors, - shape_n_stacks=shape_n_stacks, - shape_k_stacks=shape_k_stacks, - num_experts=layer.local_num_experts, - param_dtype=layer.params_dtype, - ) - - layer.weight_schemas[sublayer_name] = weight_schema_new - layer.input_schemas[sublayer_name] = input_schema - - for name, _ in list(layer.named_parameters()): - if not name.startswith(sublayer_name + "_"): - continue - delattr(layer, name) - - for name, tensor in tensors.items(): - name = f"{sublayer_name}_{name}" - param = torch.nn.Parameter(tensor, requires_grad=False) - setattr(layer, name, param) - - # Step 2: transform weight (humming standard format) for forwarding - HummingMethod.prepare_layer_meta( - layer=layer, - shape_n=shape_n, - shape_k=shape_k, - pad_n_to_multiple=256, - pad_k_to_multiple=128, - input_schema=input_schema, - weight_schema=weight_schema_new, - has_bias=layer.moe_config.has_bias, - num_experts=layer.num_experts, - torch_dtype=layer.params_dtype, - sublayer_name=sublayer_name, - ) - - HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name) - - if not hasattr(layer, "locks"): - device = layer.w13_weight.device - locks = torch.zeros(1024, dtype=torch.int32, device=device) - layer.register_buffer("locks", locks) + return FusedMoEQuantConfig( + _a1=a_quant_desc, + _a2=a_quant_desc, + _w1=w1_quant_desc, + _w2=w2_quant_desc, + gemm1_alpha=gemm1_alpha, + gemm1_beta=gemm1_beta, + gemm1_clamp_limit=gemm1_clamp_limit, + ) def get_humming_moe_quant_config( - layer: RoutedExperts, + layer: "RoutedExperts", gemm1_alpha: float | None = None, gemm1_beta: float | None = None, gemm1_clamp_limit: float | None = None, @@ -231,12 +563,10 @@ def get_humming_moe_quant_config( input_schema = layer.input_schemas["w13"] weight_schema = layer.weight_schemas["w13"] - a_dtype = input_schema.a_dtype - if a_dtype is None or a_dtype.num_bits == 16: - a_quant_desc = FusedMoEQuantDesc(dtype=None) + if input_schema.a_dtype is None or input_schema.a_dtype.num_bits == 16: + q_dtype = None else: - shape = GroupShape(row=1, col=-1) - a_quant_desc = FusedMoEQuantDesc(dtype=str(a_dtype), shape=shape) + q_dtype = str(input_schema.a_dtype) weight_scale_group_size = weight_schema.weight_scale_group_size weight_scale_group_size_n = weight_schema.weight_scale_group_size_n @@ -251,30 +581,437 @@ def get_humming_moe_quant_config( else: weight_group_shape = GroupShape(row=weight_scale_group_size, col=1) - w1_quant_desc = FusedMoEQuantDesc( - dtype=str(weight_schema.b_dtype), - shape=weight_group_shape, - scale=getattr(layer, "w13_weight_scale", None), - alpha_or_gscale=getattr(layer, "w13_global_scale", None), - zp=getattr(layer, "w13_zero_point", None), - bias=getattr(layer, "w13_bias", None), + return make_humming_moe_quant_config( + quant_dtype=q_dtype, + weight_dtype=str(weight_schema.b_dtype), + weight_group_shape=weight_group_shape, + w1_scale=getattr(layer, "w13_weight_scale", None), + w1_gscale=getattr(layer, "w13_global_scale", None), + w1_zp=getattr(layer, "w13_zero_point", None), + w1_bias=getattr(layer, "w13_bias", None), + w2_scale=getattr(layer, "w2_weight_scale", None), + w2_gscale=getattr(layer, "w2_global_scale", None), + w2_zp=getattr(layer, "w2_zero_point", None), + w2_bias=getattr(layer, "w2_bias", None), ) - w2_quant_desc = FusedMoEQuantDesc( - dtype=str(weight_schema.b_dtype), - shape=weight_group_shape, - scale=getattr(layer, "w2_weight_scale", None), - alpha_or_gscale=getattr(layer, "w2_global_scale", None), - zp=getattr(layer, "w2_zero_point", None), - bias=getattr(layer, "w2_bias", None), + +def select_humming_moe_experts( + config: FusedMoEConfig, + weight_key: QuantKey | None, + activation_key: QuantKey | None, +) -> type[mk.FusedMoEExperts] | None: + """ + Select the primary Humming MoE Experts class + Note: Shape-specific fallbacks may still occur at runtime. + """ + + if not has_humming(): + return None + + # NOTE: the kernels are selected in the following order. + AVAILABLE_EXPERTS: list[type[mk.FusedMoEExperts]] = [ + BatchedHummingGroupedExperts, + HummingGroupedExperts, + HummingIndexedExperts, + ] + + # NOTE(rob): We need to peak into the P/F selection to determine + # if we are using the batched or standard expert format, which + # if not ideal. Once we unify TP + DP/EP, we can select P/F first. + activation_format = ( + mk.FusedMoEActivationFormat.BatchedExperts + if config.moe_parallel_config.use_batched_activation_format + else mk.FusedMoEActivationFormat.Standard ) - return FusedMoEQuantConfig( - _a1=a_quant_desc, - _a2=a_quant_desc, - _w1=w1_quant_desc, - _w2=w2_quant_desc, - gemm1_alpha=gemm1_alpha, - gemm1_beta=gemm1_beta, - gemm1_clamp_limit=gemm1_clamp_limit, + def _make_log_backend(experts_cls: type[mk.FusedMoEExperts]): + return f"Using {experts_cls.__name__} Humming MoE backend." + + def _make_log_unsupported( + experts_cls: type[mk.FusedMoEExperts], reason: str | None + ) -> str: + if reason: + return ( + f"Humming MoE experts {experts_cls.__name__} does not support the " + f"deployment configuration since {reason}." + ) + else: + return ( + f"Humming MoE experts '{experts_cls.__name__}' does not support the " + "deployment configuration." + ) + + for k_cls in AVAILABLE_EXPERTS: + supported, reason = k_cls.is_supported_config( + k_cls, + config, + weight_key, + activation_key, + activation_format, + ) + if supported: + logger.info_once(_make_log_backend(k_cls)) + return k_cls + else: + logger.debug_once(_make_log_unsupported(k_cls, reason)) + + return None + + +def make_humming_moe_kernel( + moe_quant_config: FusedMoEQuantConfig, + moe_config: FusedMoEConfig, + experts_cls: type[mk.FusedMoEExperts], + layer: "RoutedExperts", + routing_tables: tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None = None, +) -> mk.FusedMoEKernel: + # Create Prepare/Finalize. + prepare_finalize = maybe_make_prepare_finalize( + moe=moe_config, + quant_config=moe_quant_config, + routing_tables=routing_tables, + allow_new_interface=True, + use_monolithic=issubclass(experts_cls, mk.FusedMoEExpertsMonolithic), ) + assert prepare_finalize is not None + + logger.info_once("Using %s", prepare_finalize.__class__.__name__) + + extra_args: dict[str, Any] = {"layer": layer} + + # Create Experts. + if prepare_finalize.activation_format == mk.FusedMoEActivationFormat.BatchedExperts: + max_num_tokens = prepare_finalize.max_num_tokens_per_rank() + assert max_num_tokens is not None + experts = experts_cls( + moe_config=moe_config, + quant_config=moe_quant_config, + max_num_tokens=max_num_tokens, + num_dispatchers=prepare_finalize.num_dispatchers(), + **extra_args, + ) + else: + experts = experts_cls( + moe_config=moe_config, + quant_config=moe_quant_config, + **extra_args, + ) + + kernel = mk.FusedMoEKernel( + prepare_finalize, + experts, + ) + + return kernel + + +def _extract_sublayer_tensors( + layer: "RoutedExperts", + sublayer_name: str, +) -> dict[str, torch.Tensor]: + """Extract tensors for a specific sublayer from the layer's state dict.""" + return dict( + (key.removeprefix(sublayer_name + "_"), value) + for key, value in layer.state_dict().items() + if key.startswith(sublayer_name + "_") + ) + + +def _replace_layer_parameters( + layer: "RoutedExperts", + sublayer_name: str, + tensors: dict[str, torch.Tensor], + preserve_bias: bool = False, +) -> None: + """ + Replace layer parameters for a sublayer with new tensors. + + Args: + layer: The RoutedExperts layer + sublayer_name: Name of the sublayer (e.g., "w13", "w2") + tensors: Dict of parameter name to tensor + preserve_bias: If True, don't delete bias parameters + """ + # Delete old parameters + for name, _ in list(layer.named_parameters()): + if not name.startswith(sublayer_name + "_"): + continue + if preserve_bias and name == sublayer_name + "_bias": + continue + delattr(layer, name) + + # Set new parameters + for name, tensor in tensors.items(): + param_name = f"{sublayer_name}_{name}" + param = torch.nn.Parameter(tensor, requires_grad=False) + setattr(layer, param_name, param) + + +def _convert_sublayer_to_humming( + layer: "RoutedExperts", + sublayer_name: str, + shape_n: int, + shape_k: int, + weight_schema: Any, + input_schema: Any, + num_experts: int, + param_dtype: torch.dtype, +) -> tuple[Any, Any]: + """ + Convert a sublayer's weights from checkpoint format to Humming format. + + Returns: + Tuple of (converted_weight_schema, converted_input_schema) + """ + from humming.schema import HummingWeightSchema + + if isinstance(weight_schema, HummingWeightSchema): + # Already in Humming format + return weight_schema, input_schema + + tensors = _extract_sublayer_tensors(layer, sublayer_name) + + shape_k_stacks = [shape_k] + shape_n_stacks = [shape_n] + if sublayer_name == "w13": + shape_n_stacks = [shape_n // 2] * 2 + + converted_weight_schema, converted_tensors = weight_schema.convert_humming( + tensors=tensors, + shape_n_stacks=shape_n_stacks, + shape_k_stacks=shape_k_stacks, + param_dtype=param_dtype, + num_experts=num_experts, + ) + + converted_input_schema, _ = input_schema.convert_humming( + tensors=converted_tensors, + shape_n_stacks=shape_n_stacks, + shape_k_stacks=shape_k_stacks, + param_dtype=param_dtype, + num_experts=num_experts, + ) + + _replace_layer_parameters(layer, sublayer_name, converted_tensors) + + return converted_weight_schema, converted_input_schema + + +def _prepare_and_transform_sublayer( + layer: "RoutedExperts", + sublayer_name: str, + shape_n: int, + shape_k: int, + weight_schema: Any, + input_schema: Any, + has_bias: bool, + num_experts: int, + param_dtype: torch.dtype, +) -> None: + """ + Prepare layer metadata and transform weights for a sublayer. + + This calls Humming's prepare_layer_meta and transform_humming_layer. + """ + from humming.layer import HummingMethod + + HummingMethod.prepare_layer_meta( + layer=layer, + shape_n=shape_n, + shape_k=shape_k, + pad_n_to_multiple=256, + pad_k_to_multiple=128, + input_schema=input_schema, + weight_schema=weight_schema, + has_bias=has_bias, + num_experts=num_experts, + torch_dtype=param_dtype, + sublayer_name=sublayer_name, + ) + + HummingMethod.transform_humming_layer(layer, sublayer_name=sublayer_name) + + +def _process_single_sublayer( + layer: "RoutedExperts", + sublayer_name: str, + shape_n: int, + shape_k: int, + weight_schema: Any, + input_schema: Any, + has_bias: bool, + num_experts: int, + param_dtype: torch.dtype, + force_weight_schema: Any | None = None, +) -> tuple[Any, Any]: + """ + Process a single sublayer: convert, optionally requant, prepare, and transform. + + This combines the common logic from convert_to_humming_moe_kernel_format + for processing a single sublayer. + + Args: + layer: The RoutedExperts layer + sublayer_name: Name of the sublayer (e.g., "w13", "w2") + shape_n: Output dimension size + shape_k: Input dimension size + weight_schema: Initial weight quantization schema + input_schema: Initial input quantization schema + has_bias: Whether the layer has bias terms + num_experts: Number of experts + param_dtype: Parameter data type + force_weight_schema: Optional schema to force requantization to + + Returns: + Tuple of (final_weight_schema, final_input_schema) + """ + from humming.schema import HummingWeightSchema + + # Step 1: Convert from checkpoint format to humming format if needed + current_weight_schema, current_input_schema = _convert_sublayer_to_humming( + layer=layer, + sublayer_name=sublayer_name, + shape_n=shape_n, + shape_k=shape_k, + weight_schema=weight_schema, + input_schema=input_schema, + num_experts=num_experts, + param_dtype=param_dtype, + ) + + # Step 2: Force requant if needed + assert isinstance(current_weight_schema, HummingWeightSchema) + if force_weight_schema is not None and current_weight_schema != force_weight_schema: + tensors = _extract_sublayer_tensors(layer, sublayer_name) + + tensors = current_weight_schema.requant_tensors( + tensors=tensors, + target_weight_schema=force_weight_schema, + param_dtype=param_dtype, + ) + + current_weight_schema = force_weight_schema + _replace_layer_parameters(layer, sublayer_name, tensors, preserve_bias=True) + del tensors + + # Step 3: Prepare layer metadata and transform weights + _prepare_and_transform_sublayer( + layer=layer, + sublayer_name=sublayer_name, + shape_n=shape_n, + shape_k=shape_k, + weight_schema=current_weight_schema, + input_schema=current_input_schema, + has_bias=has_bias, + num_experts=num_experts, + param_dtype=param_dtype, + ) + + return current_weight_schema, current_input_schema + + +def convert_to_humming_moe_kernel_format( + layer: "RoutedExperts", + quant_config: dict | None = None, + sublayer_configs: dict[str, Any] | None = None, + weight_schema: Any | None = None, + input_schema: Any | None = None, + force_weight_schema: Any | None = None, +) -> None: + """ + Convert MoE weights from checkpoint format to Humming kernel format. + + This function processes weights for each sublayer (w13, w2) by: + 1. Converting from checkpoint format to humming format if needed + 2. Force requanting if a different quantization schema is specified + 3. Preparing layer metadata for the Humming kernel + 4. Transforming weights for inference + + Args: + layer: The RoutedExperts layer containing weights to process + quant_config: Optional quantization config dict. Required if weight_schema + or input_schema are None. Used to build schemas via + BaseWeightSchema.from_config(). + sublayer_configs: Optional configuration dict for each sublayer (w13, w2). + Each config must have "shape_n" and "shape_k" keys. + If None, configs are built from layer.moe_config properties. + weight_schema: Optional initial weight quantization schema. + If None, built from quant_config. + input_schema: Optional initial input quantization schema. + If None, built from quant_config or env vars. + force_weight_schema: Optional schema to force requantization to + + Side effects: + - Modifies layer parameters in place + - Sets layer.weight_schemas and layer.input_schemas + """ + + # Build schemas from quant_config if not provided + has_bias = layer.moe_config.has_bias + num_experts = layer.moe_config.num_local_experts + param_dtype = layer.params_dtype + + if weight_schema is None or input_schema is None: + if quant_config is None: + raise ValueError( + "Must provide either weight_schema/input_schema or quant_config" + ) + + from humming.layer import HummingInputSchema + from humming.schema import BaseWeightSchema + + from vllm.model_executor.layers.quantization.utils.humming_utils import ( + humming_is_layer_skipped, + ) + + if weight_schema is None: + weight_schema = BaseWeightSchema.from_config(quant_config) + + if input_schema is None: + input_quant_config = envs.VLLM_HUMMING_INPUT_QUANT_CONFIG or {} + if humming_is_layer_skipped(input_quant_config, layer.layer_name): + input_schema = HummingInputSchema() + else: + # TODO: read input_quant_config from quant_config + input_schema = HummingInputSchema.from_config(input_quant_config) + + # Build sublayer configs from layer properties if not provided + if sublayer_configs is None: + is_gated = layer.moe_config.activation.is_gated + sublayer_configs = { + "w13": { + "shape_n": layer.moe_config.intermediate_size_per_partition * 2, + "shape_k": layer.moe_config.hidden_dim, + }, + "w2": { + "shape_n": layer.moe_config.hidden_dim, + "shape_k": layer.moe_config.intermediate_size_per_partition + * (1 if is_gated else 2), + }, + } + + layer.weight_schemas = {} + layer.input_schemas = {} + + for sublayer_name, configs in sublayer_configs.items(): + final_weight_schema, final_input_schema = _process_single_sublayer( + layer=layer, + sublayer_name=sublayer_name, + shape_n=configs["shape_n"], + shape_k=configs["shape_k"], + weight_schema=weight_schema, + input_schema=input_schema, + has_bias=has_bias, + num_experts=num_experts, + param_dtype=param_dtype, + force_weight_schema=force_weight_schema, + ) + + layer.weight_schemas[sublayer_name] = final_weight_schema + layer.input_schemas[sublayer_name] = final_input_schema + + if not hasattr(layer, "locks"): + device = layer.w13_weight.device + locks = torch.zeros(1024, dtype=torch.int32, device=device) + layer.register_buffer("locks", locks) diff --git a/vllm/utils/humming.py b/vllm/utils/humming.py index b8d9445c3f..bdd519bd8c 100644 --- a/vllm/utils/humming.py +++ b/vllm/utils/humming.py @@ -15,6 +15,7 @@ _EXPORTS: dict[str, str] = { "dtypes": "humming.dtypes", "DataType": "humming.dtypes:DataType", "GemmType": "humming.config:GemmType", + "WeightScaleType": "humming.config:WeightScaleType", "HummingMethod": "humming.layer:HummingMethod", "HummingLayerMeta": "humming.layer:HummingLayerMeta", "BaseInputSchema": "humming.schema:BaseInputSchema", @@ -22,6 +23,18 @@ _EXPORTS: dict[str, str] = { "HummingInputSchema": "humming.schema:HummingInputSchema", "HummingWeightSchema": "humming.schema:HummingWeightSchema", "quantize_weight": "humming.utils.weight:quantize_weight", + "AWQWeightSchema": "humming.schema:AWQWeightSchema", + "BitnetWeightSchema": "humming.schema:BitnetWeightSchema", + "ModeloptMxfp8WeightSchema": "humming.schema.modelopt:ModeloptMxfp8WeightSchema", + "ModeloptNvfp4InputSchema": "humming.schema.modelopt:ModeloptNvfp4InputSchema", + "ModeloptNvfp4WeightSchema": "humming.schema.modelopt:ModeloptNvfp4WeightSchema", + "CompressedTensorsInputSchema": "humming.schema:CompressedTensorsInputSchema", + "CompressedTensorsWeightSchema": "humming.schema:CompressedTensorsWeightSchema", + "Fp8InputSchema": "humming.schema:Fp8InputSchema", + "Fp8WeightSchema": "humming.schema.fp8:Fp8WeightSchema", + "Mxfp4WeightSchema": "humming.schema:Mxfp4WeightSchema", + "GptOssMxfp4WeightSchema": "humming.schema:GptOssMxfp4WeightSchema", + "GPTQWeightSchema": "humming.schema:GPTQWeightSchema", }