[MoE Refactor] Standardize Humming MoE experts + utilities (#43373)
Signed-off-by: Bill Nell <bnell@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -36,28 +36,42 @@ from vllm.model_executor.layers.fused_moe.utils import (
|
||||
_resize_cache,
|
||||
swiglu_limit_func,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8DynamicTokenSym,
|
||||
kFp8Static128BlockSym,
|
||||
kFp8StaticChannelSym,
|
||||
kInt4Static,
|
||||
kInt8Static,
|
||||
kMxfp4Dynamic,
|
||||
kMxfp4Static,
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
kNvfp4Static,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.humming import GemmType as HummingGemmType
|
||||
from vllm.utils.humming import HummingLayerMeta, HummingMethod, dtypes
|
||||
from vllm.utils.import_utils import has_humming
|
||||
from vllm.v1.worker.workspace import current_workspace_manager
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.layers.fused_moe import RoutedExperts
|
||||
from vllm.utils.humming import GemmType as HummingGemmType
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_humming_moe_gemm_type() -> str:
|
||||
env_gemm_type: str = envs.VLLM_HUMMING_MOE_GEMM_TYPE or ""
|
||||
env_gemm_type = env_gemm_type.lower()
|
||||
if env_gemm_type == "indexed":
|
||||
gemm_type = env_gemm_type
|
||||
elif env_gemm_type in ["grouped_contiguous", "grouped"]:
|
||||
gemm_type = "grouped_contiguous"
|
||||
else:
|
||||
gemm_type = "indexed"
|
||||
def get_humming_moe_gemm_type() -> str | None:
|
||||
env_gemm_type: str | None = envs.VLLM_HUMMING_MOE_GEMM_TYPE
|
||||
gemm_type = None
|
||||
if env_gemm_type is not None:
|
||||
env_gemm_type = env_gemm_type.lower()
|
||||
if env_gemm_type == "indexed":
|
||||
gemm_type = env_gemm_type
|
||||
elif env_gemm_type in ["grouped_contiguous", "grouped"]:
|
||||
gemm_type = "grouped_contiguous"
|
||||
else:
|
||||
gemm_type = "indexed"
|
||||
|
||||
logger.info_once(f"Using {gemm_type} gemm for humming moe") # noqa
|
||||
return gemm_type
|
||||
@@ -89,6 +103,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
self._permute_scratch: MoEPermuteScratch | None = None
|
||||
|
||||
def init_humming_moe(self):
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
self.compute_config = {
|
||||
"use_batch_invariant": envs.VLLM_BATCH_INVARIANT,
|
||||
"use_f16_accum": envs.VLLM_HUMMING_USE_F16_ACCUM,
|
||||
@@ -141,7 +157,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
return math.ceil(global_valid_shape_m * num_experts / global_num_experts)
|
||||
|
||||
@staticmethod
|
||||
def humming_gemm_type() -> HummingGemmType:
|
||||
def humming_gemm_type() -> "HummingGemmType":
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
@@ -153,12 +169,51 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
weight_key: QuantKey | None,
|
||||
activation_key: QuantKey | None,
|
||||
) -> bool:
|
||||
SUPPORTED_W_A = [
|
||||
(kMxfp4Static, None),
|
||||
(kMxfp4Static, kMxfp4Dynamic),
|
||||
(kMxfp4Static, kMxfp8Dynamic),
|
||||
(kMxfp4Static, kFp8DynamicTokenSym),
|
||||
(kNvfp4Static, None),
|
||||
(kNvfp4Static, kFp8DynamicTokenSym),
|
||||
(kMxfp8Static, None),
|
||||
(kMxfp8Static, kFp8DynamicTokenSym),
|
||||
(kFp8StaticChannelSym, None),
|
||||
(kFp8StaticChannelSym, kFp8DynamicTokenSym),
|
||||
(kFp8Static128BlockSym, None),
|
||||
(kFp8Static128BlockSym, kFp8DynamicTokenSym),
|
||||
(kInt4Static, None),
|
||||
(kInt4Static, kFp8DynamicTokenSym),
|
||||
(kInt8Static, None),
|
||||
(kInt8Static, kFp8DynamicTokenSym),
|
||||
]
|
||||
return (weight_key, activation_key) in SUPPORTED_W_A
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
"""
|
||||
Humming kernels handle input quantization internally via
|
||||
HummingMethod.may_quant_input() in the apply() method.
|
||||
|
||||
This property tells the prepare/finalize step to skip input
|
||||
quantization (by setting defer_input_quant=True) and pass
|
||||
unquantized inputs to the experts. This prevents double
|
||||
quantization: once in prepare and once in Humming's apply().
|
||||
|
||||
Returns:
|
||||
True to indicate that this expert expects unquantized inputs
|
||||
and will handle quantization internally.
|
||||
"""
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _supports_current_device() -> bool:
|
||||
platform = current_platform
|
||||
return platform.is_cuda() and platform.has_device_capability((7, 5))
|
||||
return (
|
||||
has_humming()
|
||||
and platform.is_cuda()
|
||||
and platform.has_device_capability((7, 5))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _supports_no_act_and_mul() -> bool:
|
||||
@@ -182,10 +237,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
|
||||
@staticmethod
|
||||
def _supports_parallel_config(moe_parallel_config: FusedMoEParallelConfig) -> bool:
|
||||
return not (
|
||||
moe_parallel_config.use_fi_nvl_two_sided_kernels
|
||||
or moe_parallel_config.use_fi_nvl_one_sided_kernels
|
||||
)
|
||||
return True
|
||||
|
||||
def moe_problem_size(
|
||||
self,
|
||||
@@ -194,6 +246,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
w2: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
) -> tuple[int, int, int, int, int]:
|
||||
from vllm.utils.humming import HummingLayerMeta
|
||||
|
||||
meta1: HummingLayerMeta = self.layer.humming_metas["w13"]
|
||||
meta2: HummingLayerMeta = self.layer.humming_metas["w2"]
|
||||
|
||||
@@ -215,6 +269,9 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
return meta1.num_experts, num_tokens, meta1.shape_n // 2, meta1.shape_k, top_k
|
||||
|
||||
def get_buffer_metas(self, M: int, topk: int, activation: MoEActivation):
|
||||
from vllm.utils.humming import GemmType as HummingGemmType
|
||||
from vllm.utils.humming import dtypes
|
||||
|
||||
num_experts = self.num_experts
|
||||
N = self.layer.intermediate_size_per_partition
|
||||
K = self.layer.hidden_size
|
||||
@@ -254,7 +311,9 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
torch_dtype_map = {
|
||||
dtypes.float16: torch.float16,
|
||||
dtypes.bfloat16: torch.bfloat16,
|
||||
dtypes.float32: torch.float32,
|
||||
dtypes.float8e4m3: torch.float8_e4m3fn,
|
||||
dtypes.float8e5m2: torch.float8_e5m2,
|
||||
dtypes.int8: torch.int8,
|
||||
dtypes.int4: torch.uint8,
|
||||
}
|
||||
@@ -289,7 +348,13 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
for key in buffer_metas:
|
||||
meta = buffer_metas[key]
|
||||
if "quanted" in key and a_dtype.num_bits == 4:
|
||||
meta["shape"] = meta["shape"][:-1] + (meta["shape"][-1] // 2,)
|
||||
last_dim = meta["shape"][-1]
|
||||
if last_dim % 2 != 0:
|
||||
raise ValueError(
|
||||
f"Int4 packing requires last dimension to be even, "
|
||||
f"got {last_dim} for buffer '{key}'"
|
||||
)
|
||||
meta["shape"] = meta["shape"][:-1] + (last_dim // 2,)
|
||||
|
||||
if num_bits == 16:
|
||||
required_buffers = ["gate_up_output", "activation_output", "down_output"]
|
||||
@@ -325,8 +390,13 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
|
||||
output_key = "down_output" if self.is_batched() else "output"
|
||||
output_shape = buffer_metas[output_key]["shape"]
|
||||
elem_size = self.layer.params_dtype.itemsize
|
||||
|
||||
return (workspace1_nbytes // 2,), (workspace2_nbytes // 2,), output_shape
|
||||
return (
|
||||
(workspace1_nbytes // elem_size,),
|
||||
(workspace2_nbytes // elem_size,),
|
||||
output_shape,
|
||||
)
|
||||
|
||||
def workspace_shapes(
|
||||
self,
|
||||
@@ -344,7 +414,7 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
def make_workspaces(self, M: int, topk: int, activation: MoEActivation):
|
||||
shapes = self._workspace_shapes(M, topk, activation)
|
||||
workspace1_shape, workspace2_shape, output_shape = shapes
|
||||
torch_dtype = self.layer.param_dtype
|
||||
torch_dtype = self.layer.params_dtype
|
||||
workspace1, workspace2 = current_workspace_manager().get_simultaneous(
|
||||
(workspace1_shape, torch_dtype),
|
||||
(workspace2_shape, torch_dtype),
|
||||
@@ -370,45 +440,8 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
|
||||
return buffers
|
||||
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
apply_router_weight_on_input: bool,
|
||||
):
|
||||
assert not apply_router_weight_on_input
|
||||
|
||||
self.main_apply(
|
||||
hidden_states=hidden_states,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
workspace1=workspace13,
|
||||
workspace2=workspace2,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
)
|
||||
|
||||
def main_apply(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
workspace1: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
):
|
||||
raise NotImplementedError
|
||||
# Note: apply method is implemented by subclasses following the
|
||||
# standard FusedMoEExpertsModular.apply signature
|
||||
|
||||
@staticmethod
|
||||
def is_supported_config(
|
||||
@@ -418,24 +451,27 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
|
||||
activation_key: QuantKey | None,
|
||||
activation_format: mk.FusedMoEActivationFormat,
|
||||
) -> tuple[bool, str | None]:
|
||||
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
supported = cls.activation_format() == activation_format
|
||||
reason = "activation_format mismatched"
|
||||
elif activation_format == mk.FusedMoEActivationFormat.Standard:
|
||||
if cls.activation_format() != mk.FusedMoEActivationFormat.Standard:
|
||||
supported = False
|
||||
reason = "activation_format mismatched"
|
||||
else:
|
||||
assert hasattr(cls, "humming_gemm_type")
|
||||
gemm_type = cls.humming_gemm_type().value.lower()
|
||||
preferred_gemm_type = get_humming_moe_gemm_type().lower()
|
||||
supported = preferred_gemm_type == gemm_type
|
||||
reason = "preferred gemm type mismatched"
|
||||
else:
|
||||
supported = False
|
||||
reason = "unsupported activation_format"
|
||||
supported, reason = mk.FusedMoEExpertsModular.is_supported_config(
|
||||
cls,
|
||||
moe_config,
|
||||
weight_key,
|
||||
activation_key,
|
||||
activation_format,
|
||||
)
|
||||
|
||||
return supported, None if supported else reason
|
||||
if supported:
|
||||
assert hasattr(cls, "humming_gemm_type")
|
||||
gemm_type = cls.humming_gemm_type().value.lower()
|
||||
preferred_gemm_type = get_humming_moe_gemm_type()
|
||||
if preferred_gemm_type is not None:
|
||||
supported = preferred_gemm_type.lower() == gemm_type
|
||||
if not supported:
|
||||
reason = (
|
||||
f"preferred gemm type {preferred_gemm_type} != "
|
||||
f"supported gemm type {gemm_type}"
|
||||
)
|
||||
|
||||
return supported, reason
|
||||
|
||||
def apply_activation(
|
||||
self,
|
||||
@@ -459,7 +495,9 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def humming_gemm_type() -> HummingGemmType:
|
||||
def humming_gemm_type() -> "HummingGemmType":
|
||||
from vllm.utils.humming import GemmType as HummingGemmType
|
||||
|
||||
return HummingGemmType.INDEXED
|
||||
|
||||
def prepare_humming_moe_kwargs(
|
||||
@@ -470,12 +508,18 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
valid_shape_m = self.estimate_local_valid_shape_m(topk_ids)
|
||||
|
||||
moe_block_size = None
|
||||
for min_shape_m, max_shape_m, config in self.w13_tuning_config:
|
||||
if valid_shape_m > min_shape_m and valid_shape_m <= max_shape_m:
|
||||
moe_block_size = config["block_shape"][0]
|
||||
break
|
||||
else:
|
||||
raise ValueError(f"cannot found moe_block_size for shape {valid_shape_m}")
|
||||
|
||||
if moe_block_size is None:
|
||||
logger.warning_once(
|
||||
"No tuning config found for shape %s, using default block_size=64",
|
||||
valid_shape_m,
|
||||
)
|
||||
moe_block_size = 64
|
||||
|
||||
sorted_ids, expert_ids, num_tokens_padded = moe_align_block_size(
|
||||
topk_ids=topk_ids,
|
||||
@@ -501,27 +545,47 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
|
||||
return moe_kwargs1, moe_kwargs2
|
||||
|
||||
def main_apply(
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
workspace1: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
):
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
"""
|
||||
Standard apply implementation for Humming indexed experts.
|
||||
|
||||
Note: Humming kernels handle weights and quantization internally through
|
||||
the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used.
|
||||
The output is written into workspace13 via the buffer management.
|
||||
"""
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
assert not apply_router_weight_on_input
|
||||
|
||||
hidden_states = hidden_states.view(-1, hidden_states.size(-1))
|
||||
buffers = self.prepare_buffers(
|
||||
workspace1,
|
||||
workspace13,
|
||||
workspace2,
|
||||
topk_ids.size(0),
|
||||
topk_ids.size(1),
|
||||
self.layer.activation,
|
||||
activation,
|
||||
)
|
||||
|
||||
moe_kwargs1, moe_kwargs2 = self.prepare_humming_moe_kwargs(
|
||||
topk_ids=topk_ids,
|
||||
expert_map=self.layer.expert_map,
|
||||
expert_map=expert_map,
|
||||
expert_tokens_meta=expert_tokens_meta,
|
||||
)
|
||||
|
||||
@@ -542,7 +606,7 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
)
|
||||
|
||||
self.apply_activation(
|
||||
activation=self.layer.activation,
|
||||
activation=activation,
|
||||
input=buffers["gate_up_output"],
|
||||
output=buffers["activation_output"],
|
||||
)
|
||||
@@ -567,10 +631,13 @@ class HummingIndexedExperts(HummingExpertsBase):
|
||||
inputs=buffers["down_output"].view(*topk_ids.shape, -1),
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
expert_map=self.layer.expert_map,
|
||||
expert_map=expert_map,
|
||||
outputs=buffers["output"],
|
||||
)
|
||||
|
||||
# Note: output is already written to buffers["output"]
|
||||
# which aliases workspace13/output
|
||||
|
||||
|
||||
class HummingGroupedExperts(HummingExpertsBase):
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
@@ -581,35 +648,57 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
return mk.FusedMoEActivationFormat.Standard
|
||||
|
||||
@staticmethod
|
||||
def humming_gemm_type() -> HummingGemmType:
|
||||
def humming_gemm_type() -> "HummingGemmType":
|
||||
from vllm.utils.humming import GemmType as HummingGemmType
|
||||
|
||||
return HummingGemmType.GROUPED_CONTIGUOUS
|
||||
|
||||
def main_apply(
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
workspace1: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
):
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
"""
|
||||
Standard apply implementation for Humming grouped experts.
|
||||
|
||||
Note: Humming kernels handle weights and quantization internally through
|
||||
the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used.
|
||||
The output is written into workspace13 via the buffer management.
|
||||
"""
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
assert not apply_router_weight_on_input
|
||||
|
||||
valid_shape_m = self.estimate_local_valid_shape_m(topk_ids)
|
||||
|
||||
buffers = self.prepare_buffers(
|
||||
workspace1,
|
||||
workspace13,
|
||||
workspace2,
|
||||
topk_ids.size(0),
|
||||
topk_ids.size(1),
|
||||
self.layer.activation,
|
||||
activation,
|
||||
)
|
||||
|
||||
hidden_states, _, expert_first_token_offset, inv_perm, _ = moe_permute(
|
||||
hidden_states=hidden_states,
|
||||
a1q_scale=None,
|
||||
topk_ids=topk_ids,
|
||||
n_expert=self.global_num_experts,
|
||||
n_expert=global_num_experts,
|
||||
n_local_expert=self.num_experts,
|
||||
expert_map=self.layer.expert_map,
|
||||
expert_map=expert_map,
|
||||
scratch=self._get_permute_scratch(),
|
||||
)
|
||||
|
||||
@@ -633,7 +722,7 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
)
|
||||
|
||||
self.apply_activation(
|
||||
activation=self.layer.activation,
|
||||
activation=activation,
|
||||
input=buffers["gate_up_output"],
|
||||
output=buffers["activation_output"],
|
||||
)
|
||||
@@ -665,6 +754,9 @@ class HummingGroupedExperts(HummingExpertsBase):
|
||||
expert_first_token_offset=expert_first_token_offset,
|
||||
)
|
||||
|
||||
# Note: output is already written to buffers["output"]
|
||||
# which aliases workspace13/output
|
||||
|
||||
|
||||
class BatchedHummingGroupedExperts(HummingExpertsBase):
|
||||
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
|
||||
@@ -675,29 +767,51 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
||||
return mk.FusedMoEActivationFormat.BatchedExperts
|
||||
|
||||
@staticmethod
|
||||
def humming_gemm_type() -> HummingGemmType:
|
||||
def humming_gemm_type() -> "HummingGemmType":
|
||||
from vllm.utils.humming import GemmType as HummingGemmType
|
||||
|
||||
return HummingGemmType.GROUPED_MASKED
|
||||
|
||||
def main_apply(
|
||||
def apply(
|
||||
self,
|
||||
output: torch.Tensor,
|
||||
hidden_states: torch.Tensor,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
workspace1: torch.Tensor,
|
||||
activation: MoEActivation,
|
||||
global_num_experts: int,
|
||||
expert_map: torch.Tensor | None,
|
||||
a1q_scale: torch.Tensor | None,
|
||||
a2_scale: torch.Tensor | None,
|
||||
workspace13: torch.Tensor,
|
||||
workspace2: torch.Tensor,
|
||||
expert_tokens_meta: mk.ExpertTokensMetadata | None,
|
||||
):
|
||||
apply_router_weight_on_input: bool,
|
||||
) -> None:
|
||||
"""
|
||||
Standard apply implementation for Humming batched grouped experts.
|
||||
|
||||
Note: Humming kernels handle weights and quantization internally through
|
||||
the layer object, so w1, w2, a1q_scale, a2_scale parameters are not used.
|
||||
The output is written into workspace13 via the buffer management.
|
||||
"""
|
||||
from vllm.utils.humming import HummingMethod
|
||||
|
||||
assert not apply_router_weight_on_input
|
||||
assert expert_tokens_meta is not None
|
||||
|
||||
hidden_states = hidden_states.view(-1, hidden_states.size(-1))
|
||||
valid_shape_m = self.estimate_local_valid_shape_m(topk_ids)
|
||||
expert_num_tokens = expert_tokens_meta.expert_num_tokens
|
||||
|
||||
buffers = self.prepare_buffers(
|
||||
workspace1,
|
||||
workspace13,
|
||||
workspace2,
|
||||
topk_ids.size(0),
|
||||
topk_ids.size(1),
|
||||
self.layer.activation,
|
||||
activation,
|
||||
)
|
||||
|
||||
inputs, input_scale = HummingMethod.may_quant_input(
|
||||
@@ -720,7 +834,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
||||
)
|
||||
|
||||
self.apply_activation(
|
||||
activation=self.layer.activation,
|
||||
activation=activation,
|
||||
input=buffers["gate_up_output"],
|
||||
output=buffers["activation_output"],
|
||||
)
|
||||
@@ -743,3 +857,6 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
|
||||
tuning_config=self.w2_tuning_config_str,
|
||||
sublayer_name="w2",
|
||||
)
|
||||
|
||||
# Note: output is already written to buffers["down_output"]
|
||||
# which aliases workspace13/output
|
||||
|
||||
@@ -711,10 +711,12 @@ def convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
prepare_humming_moe_layer,
|
||||
convert_to_humming_moe_kernel_format,
|
||||
)
|
||||
|
||||
prepare_humming_moe_layer(layer, {"quant_method": "gpt_oss_mxfp4"})
|
||||
convert_to_humming_moe_kernel_format(
|
||||
layer, quant_config={"quant_method": "gpt_oss_mxfp4"}
|
||||
)
|
||||
return (
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
@@ -1277,10 +1279,12 @@ def convert_weight_to_mxfp4_moe_kernel_format(
|
||||
|
||||
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
prepare_humming_moe_layer,
|
||||
convert_to_humming_moe_kernel_format,
|
||||
)
|
||||
|
||||
prepare_humming_moe_layer(layer, {"quant_method": "mxfp4"})
|
||||
convert_to_humming_moe_kernel_format(
|
||||
layer, quant_config={"quant_method": "mxfp4"}
|
||||
)
|
||||
return (
|
||||
layer.w13_weight,
|
||||
layer.w2_weight,
|
||||
@@ -1569,7 +1573,7 @@ def make_mxfp4_moe_quant_config(
|
||||
w2_bias: torch.Tensor | None = None,
|
||||
a1_scale: torch.Tensor | None = None,
|
||||
a2_scale: torch.Tensor | None = None,
|
||||
layer: torch.nn.Module | None = None,
|
||||
layer: "RoutedExperts | None" = None,
|
||||
) -> FusedMoEQuantConfig | None:
|
||||
"""Create a FusedMoEQuantConfig for the given MXFP4 backend."""
|
||||
if mxfp4_backend == Mxfp4MoeBackend.DEEPGEMM_MXFP4:
|
||||
@@ -1662,7 +1666,7 @@ def make_mxfp4_moe_quant_config(
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
assert isinstance(layer, RoutedExperts)
|
||||
assert layer is not None
|
||||
return get_humming_moe_quant_config(
|
||||
layer,
|
||||
gemm1_alpha=gemm1_alpha,
|
||||
@@ -1703,6 +1707,7 @@ def make_mxfp4_moe_kernel(
|
||||
assert prepare_finalize is not None
|
||||
|
||||
logger.info_once("Using %s", prepare_finalize.__class__.__name__)
|
||||
logger.info_once("Using %s", experts_cls.__name__)
|
||||
|
||||
extra_kwargs = {}
|
||||
if mxfp4_backend == Mxfp4MoeBackend.HUMMING:
|
||||
|
||||
+1
@@ -143,6 +143,7 @@ class CompressedTensorsW4A4Mxfp4MoEMethod(CompressedTensorsMoEMethod):
|
||||
mxfp4_backend=self.mxfp4_backend,
|
||||
w1_scale=layer.w13_weight_scale,
|
||||
w2_scale=layer.w2_weight_scale,
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
|
||||
@@ -706,15 +706,15 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
if self.moe_quant_config:
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_fp8_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
assert self.moe_quant_config is not None
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_fp8_moe_kernel(
|
||||
moe_quant_config=self.moe_quant_config,
|
||||
moe_config=self.moe,
|
||||
fp8_backend=self.fp8_backend,
|
||||
experts_cls=self.experts_cls,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
# Allow for accessing weights and scales in standard way.
|
||||
|
||||
@@ -30,6 +30,14 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
convert_to_humming_moe_kernel_format,
|
||||
get_humming_moe_quant_config,
|
||||
input_schema_to_quant_key,
|
||||
make_humming_moe_kernel,
|
||||
select_humming_moe_experts,
|
||||
weight_schema_to_quant_key,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.parameter import (
|
||||
BasevLLMParameter,
|
||||
@@ -106,7 +114,7 @@ def prepare_param(tensor, name, extra_attrs):
|
||||
return param
|
||||
|
||||
|
||||
def prepare_moe_param(tensor, name, extra_attrs):
|
||||
def prepare_moe_param(tensor: torch.Tensor, name: str, extra_attrs: dict[str, Any]):
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
if "scale_type" in extra_attrs:
|
||||
extra_attrs["quant_method"] = extra_attrs["scale_type"]
|
||||
@@ -605,12 +613,27 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
) -> None:
|
||||
super().__init__(moe)
|
||||
self.quant_config = quant_config
|
||||
self.moe = moe
|
||||
self.weight_schema = quant_config.weight_schema
|
||||
self.input_schema = quant_config.input_schema
|
||||
self.force_weight_schema = quant_config.force_weight_schema
|
||||
self.force_input_schema = quant_config.force_input_schema
|
||||
|
||||
# Derive QuantKeys from humming schemas.
|
||||
# Prefer force schemas (the final format after requant) over base.
|
||||
weight_key = weight_schema_to_quant_key(
|
||||
self.force_weight_schema or self.weight_schema
|
||||
)
|
||||
activation_key = input_schema_to_quant_key(
|
||||
self.force_input_schema or self.input_schema
|
||||
)
|
||||
|
||||
# Select Humming MoE experts
|
||||
self.experts_cls = select_humming_moe_experts(
|
||||
config=self.moe,
|
||||
weight_key=weight_key,
|
||||
activation_key=activation_key,
|
||||
)
|
||||
|
||||
def prepare_weight_loader(self, layer, weight_loader):
|
||||
def new_weight_loader(
|
||||
param: torch.nn.Parameter,
|
||||
@@ -647,7 +670,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
sublayer_name = "w2" if shard_id == "w2" else "w13"
|
||||
|
||||
param = getattr(layer, sublayer_name + "_" + key)
|
||||
part_subccess = param.weight_loader(
|
||||
part_success = param.weight_loader(
|
||||
param=param,
|
||||
loaded_weight=tensor.cpu(),
|
||||
weight_name=shard_id + "_" + key,
|
||||
@@ -655,7 +678,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
expert_id=expert_id,
|
||||
return_success=return_success,
|
||||
)
|
||||
success = success and part_subccess
|
||||
success = success and part_success
|
||||
|
||||
return success if return_success else None
|
||||
|
||||
@@ -677,7 +700,7 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
layer: RoutedExperts,
|
||||
num_experts: int,
|
||||
hidden_size: int,
|
||||
intermediate_size_per_partition: int,
|
||||
@@ -734,160 +757,35 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
locks = torch.zeros(1024, dtype=torch.int32)
|
||||
layer.register_buffer("locks", locks)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: torch.nn.Module) -> FusedMoEQuantConfig:
|
||||
from vllm.model_executor.layers.quantization.utils.humming_utils import (
|
||||
get_humming_moe_quant_config,
|
||||
)
|
||||
|
||||
def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
|
||||
return get_humming_moe_quant_config(layer)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
if getattr(self, "processed", False):
|
||||
return
|
||||
self.processed = True
|
||||
layer.weight_schemas = {}
|
||||
layer.input_schemas = {}
|
||||
for sublayer_name, configs in layer.sublayer_configs.items():
|
||||
input_schema = self.input_schema
|
||||
weight_schema = self.weight_schema
|
||||
# convert from checkpoint format to humming format
|
||||
if not isinstance(weight_schema, _hm.HummingWeightSchema):
|
||||
tensors: dict[str, torch.Tensor] = dict(
|
||||
(key.removeprefix(sublayer_name + "_"), value)
|
||||
for key, value in layer.state_dict().items()
|
||||
if key.startswith(sublayer_name + "_")
|
||||
)
|
||||
|
||||
shape_k_stacks = [configs["shape_k"]]
|
||||
shape_n_stacks = [configs["shape_n"]]
|
||||
if sublayer_name == "w13":
|
||||
shape_n_stacks = [configs["shape_n"] // 2] * 2
|
||||
|
||||
weight_schema, tensors = weight_schema.convert_humming(
|
||||
tensors=tensors,
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.param_dtype,
|
||||
num_experts=layer.num_experts,
|
||||
)
|
||||
|
||||
input_schema, _ = input_schema.convert_humming(
|
||||
tensors=tensors,
|
||||
shape_n_stacks=shape_n_stacks,
|
||||
shape_k_stacks=shape_k_stacks,
|
||||
param_dtype=layer.param_dtype,
|
||||
num_experts=layer.num_experts,
|
||||
)
|
||||
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
if not name.startswith(sublayer_name + "_"):
|
||||
continue
|
||||
delattr(layer, name)
|
||||
|
||||
for name, tensor in tensors.items():
|
||||
name = f"{sublayer_name}_{name}"
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
layer.weight_schemas[sublayer_name] = weight_schema
|
||||
layer.input_schemas[sublayer_name] = input_schema
|
||||
|
||||
# force requant (origin quant setting -> fp16/bf16 -> new_quant setting)
|
||||
assert isinstance(weight_schema, _hm.HummingWeightSchema)
|
||||
force_requant = self.force_weight_schema is not None
|
||||
if force_requant and weight_schema != self.force_weight_schema:
|
||||
tensors = dict(
|
||||
(key.removeprefix(sublayer_name + "_"), value)
|
||||
for key, value in layer.state_dict().items()
|
||||
if key.startswith(sublayer_name + "_")
|
||||
)
|
||||
|
||||
tensors = weight_schema.requant_tensors(
|
||||
tensors=tensors,
|
||||
target_weight_schema=self.force_weight_schema,
|
||||
param_dtype=layer.param_dtype,
|
||||
)
|
||||
|
||||
weight_schema = self.force_weight_schema
|
||||
|
||||
for name, _ in list(layer.named_parameters()):
|
||||
if not name.startswith(sublayer_name + "_"):
|
||||
continue
|
||||
if name == sublayer_name + "_bias":
|
||||
continue
|
||||
delattr(layer, name)
|
||||
|
||||
for name, tensor in tensors.items():
|
||||
name = f"{sublayer_name}_{name}"
|
||||
param = torch.nn.Parameter(tensor, requires_grad=False)
|
||||
setattr(layer, name, param)
|
||||
|
||||
del tensors
|
||||
|
||||
# prepare layer config from humming kernel
|
||||
_hm.HummingMethod.prepare_layer_meta(
|
||||
layer=layer,
|
||||
shape_n=configs["shape_n"],
|
||||
shape_k=configs["shape_k"],
|
||||
pad_n_to_multiple=256,
|
||||
pad_k_to_multiple=128,
|
||||
input_schema=input_schema,
|
||||
weight_schema=weight_schema,
|
||||
has_bias=self.moe.has_bias,
|
||||
num_experts=layer.num_experts,
|
||||
torch_dtype=layer.param_dtype,
|
||||
sublayer_name=sublayer_name,
|
||||
)
|
||||
|
||||
# preprocess weight for inference
|
||||
_hm.HummingMethod.transform_humming_layer(
|
||||
layer, sublayer_name=sublayer_name
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
get_humming_moe_gemm_type,
|
||||
# Convert weights to Humming kernel format
|
||||
convert_to_humming_moe_kernel_format(
|
||||
layer=layer,
|
||||
sublayer_configs=layer.sublayer_configs,
|
||||
weight_schema=self.weight_schema,
|
||||
input_schema=self.input_schema,
|
||||
force_weight_schema=self.force_weight_schema,
|
||||
)
|
||||
|
||||
# use moe modular
|
||||
experts: HummingIndexedExperts | HummingGroupedExperts
|
||||
layer._ensure_moe_quant_config_init()
|
||||
# Build the MoE kernel
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
if get_humming_moe_gemm_type() == "indexed":
|
||||
experts = HummingIndexedExperts(layer, self.moe, self.moe_quant_config)
|
||||
else:
|
||||
experts = HummingGroupedExperts(layer, self.moe, self.moe_quant_config)
|
||||
self.experts = experts
|
||||
|
||||
def select_gemm_impl(
|
||||
self,
|
||||
prepare_finalize,
|
||||
layer: torch.nn.Module,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe import modular_kernel as mk
|
||||
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
|
||||
BatchedHummingGroupedExperts,
|
||||
HummingGroupedExperts,
|
||||
HummingIndexedExperts,
|
||||
get_humming_moe_gemm_type,
|
||||
assert self.experts_cls is not None
|
||||
self.moe_kernel = make_humming_moe_kernel(
|
||||
self.moe_quant_config,
|
||||
self.moe,
|
||||
self.experts_cls,
|
||||
layer=layer,
|
||||
routing_tables=layer._expert_routing_tables(),
|
||||
)
|
||||
|
||||
activation_format = prepare_finalize.activation_format
|
||||
assert self.moe_quant_config is not None
|
||||
if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:
|
||||
return BatchedHummingGroupedExperts(
|
||||
layer=layer,
|
||||
moe_config=self.moe,
|
||||
quant_config=self.moe_quant_config,
|
||||
max_num_tokens=prepare_finalize.max_num_tokens_per_rank(),
|
||||
num_dispatchers=prepare_finalize.num_dispatchers(),
|
||||
)
|
||||
elif get_humming_moe_gemm_type() == "indexed":
|
||||
return HummingIndexedExperts(layer, self.moe, self.moe_quant_config)
|
||||
else:
|
||||
return HummingGroupedExperts(layer, self.moe, self.moe_quant_config)
|
||||
|
||||
def apply(
|
||||
self,
|
||||
layer: RoutedExperts,
|
||||
@@ -896,22 +794,36 @@ class HummingMoEMethod(FusedMoEMethodBase):
|
||||
topk_ids: torch.Tensor,
|
||||
shared_experts: SharedExperts | None,
|
||||
shared_experts_input: torch.Tensor | None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
workspace1, workspace2, output = self.experts.make_workspaces(
|
||||
M=topk_ids.size(0),
|
||||
topk=topk_ids.size(1),
|
||||
activation=layer.activation,
|
||||
)
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Apply Humming-quantized MoE computation using the standard kernel flow.
|
||||
|
||||
assert workspace1.data_ptr() == output.data_ptr()
|
||||
This method uses FusedMoEKernel.apply() which orchestrates:
|
||||
1. Preparation (quantization if needed - skipped for Humming via
|
||||
expects_unquantized_inputs=True to prevent double quantization)
|
||||
2. Expert computation (via experts.apply())
|
||||
3. Finalization (weight application & reduction - no-op for Humming
|
||||
since it's already done internally)
|
||||
|
||||
self.experts.main_apply(
|
||||
Humming handles all quantization, weight application, and reduction
|
||||
internally in the experts.apply() method via HummingMethod calls.
|
||||
|
||||
Note: Although w1/w2 weights are passed to the kernel for interface
|
||||
consistency, Humming's experts.apply() reads weights directly from
|
||||
the layer object via HummingMethod.forward_layer() and ignores the
|
||||
w1/w2 parameters.
|
||||
"""
|
||||
assert self.moe_kernel is not None
|
||||
return self.moe_kernel.apply(
|
||||
hidden_states=x,
|
||||
topk_weights=topk_weights,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
topk_ids=topk_ids,
|
||||
workspace1=workspace1,
|
||||
workspace2=workspace2,
|
||||
expert_tokens_meta=None,
|
||||
topk_weights=topk_weights,
|
||||
activation=layer.activation,
|
||||
global_num_experts=layer.global_num_experts,
|
||||
expert_map=layer.expert_map,
|
||||
apply_router_weight_on_input=False,
|
||||
shared_experts=shared_experts,
|
||||
shared_experts_input=shared_experts_input,
|
||||
)
|
||||
|
||||
return output
|
||||
|
||||
@@ -1280,6 +1280,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
|
||||
gemm1_alpha=getattr(layer, "swiglu_alpha", None),
|
||||
gemm1_beta=getattr(layer, "swiglu_beta", None),
|
||||
swiglu_limit=getattr(layer, "swiglu_limit", None),
|
||||
layer=layer,
|
||||
)
|
||||
|
||||
# Emulation and other schemes
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,6 +15,7 @@ _EXPORTS: dict[str, str] = {
|
||||
"dtypes": "humming.dtypes",
|
||||
"DataType": "humming.dtypes:DataType",
|
||||
"GemmType": "humming.config:GemmType",
|
||||
"WeightScaleType": "humming.config:WeightScaleType",
|
||||
"HummingMethod": "humming.layer:HummingMethod",
|
||||
"HummingLayerMeta": "humming.layer:HummingLayerMeta",
|
||||
"BaseInputSchema": "humming.schema:BaseInputSchema",
|
||||
@@ -22,6 +23,18 @@ _EXPORTS: dict[str, str] = {
|
||||
"HummingInputSchema": "humming.schema:HummingInputSchema",
|
||||
"HummingWeightSchema": "humming.schema:HummingWeightSchema",
|
||||
"quantize_weight": "humming.utils.weight:quantize_weight",
|
||||
"AWQWeightSchema": "humming.schema:AWQWeightSchema",
|
||||
"BitnetWeightSchema": "humming.schema:BitnetWeightSchema",
|
||||
"ModeloptMxfp8WeightSchema": "humming.schema.modelopt:ModeloptMxfp8WeightSchema",
|
||||
"ModeloptNvfp4InputSchema": "humming.schema.modelopt:ModeloptNvfp4InputSchema",
|
||||
"ModeloptNvfp4WeightSchema": "humming.schema.modelopt:ModeloptNvfp4WeightSchema",
|
||||
"CompressedTensorsInputSchema": "humming.schema:CompressedTensorsInputSchema",
|
||||
"CompressedTensorsWeightSchema": "humming.schema:CompressedTensorsWeightSchema",
|
||||
"Fp8InputSchema": "humming.schema:Fp8InputSchema",
|
||||
"Fp8WeightSchema": "humming.schema.fp8:Fp8WeightSchema",
|
||||
"Mxfp4WeightSchema": "humming.schema:Mxfp4WeightSchema",
|
||||
"GptOssMxfp4WeightSchema": "humming.schema:GptOssMxfp4WeightSchema",
|
||||
"GPTQWeightSchema": "humming.schema:GPTQWeightSchema",
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user