[Feature][Humming] Humming feature integration (#56685)

Signed-off-by: jinzhen.ljz <jinzhen.ljz@antgroup.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Jinzhen Lin
2026-09-20 08:11:01 -07:00
committed by GitHub
co-authored by Codex mgoin
parent 27757dde02
commit 7d99c2c4fd
62 changed files with 2040 additions and 1086 deletions
@@ -9,3 +9,5 @@ server_args: >-
--moe-backend humming
--linear-backend humming
--dtype bfloat16
env:
VLLM_HUMMING_MOE_GEMM_TYPE: "grouped_contiguous"
@@ -0,0 +1,14 @@
model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
--data-parallel-size 2
--enable-expert-parallel
--all2all-backend deepep_high_throughput
env:
VLLM_HUMMING_MOE_GEMM_TYPE: "grouped_contiguous"
@@ -0,0 +1,14 @@
model_name: "nm-testing/Qwen3-30B-A3B-FP8-block"
accuracy_threshold: 0.86
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--moe-backend humming
--linear-backend humming
--data-parallel-size 2
--enable-expert-parallel
--all2all-backend deepep_high_throughput
env:
VLLM_HUMMING_MOE_GEMM_TYPE: "indexed"
@@ -7,3 +7,5 @@ server_args: >-
--max-model-len 8192
--quantization humming
--linear-backend humming
env:
VLLM_HUMMING_MOE_GEMM_TYPE: "grouped_contiguous"
@@ -7,3 +7,5 @@ server_args: >-
--max-model-len 8192
--moe-backend humming
--linear-backend humming
env:
VLLM_HUMMING_MOE_GEMM_TYPE: "grouped_contiguous"
@@ -1,13 +0,0 @@
model_name: "Qwen/Qwen3-30B-A3B"
accuracy_threshold: 0.80
num_questions: 1319
num_fewshot: 5
server_args: >-
--enforce-eager
--max-model-len 8192
--tensor-parallel-size 1
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
env:
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"float8e4m3"}'
@@ -1,7 +1,8 @@
model_name: "Qwen/Qwen3-30B-A3B"
accuracy_threshold: 0.80
model_name: "Qwen/Qwen3.5-35B-A3B"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
@@ -9,5 +10,5 @@ server_args: >-
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
env:
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int6","hadamard_block_size":-1}'
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -1,7 +1,8 @@
model_name: "Qwen/Qwen3-30B-A3B"
accuracy_threshold: 0.80
model_name: "Qwen/Qwen3.5-35B-A3B"
accuracy_threshold: 0.90
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
@@ -9,4 +10,4 @@ server_args: >-
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
env:
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int5","hadamard_block_size":-1}'
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int6","hadamard_block_size":-1}'
@@ -0,0 +1,13 @@
model_name: "Qwen/Qwen3.5-9B"
accuracy_threshold: 0.88
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
env:
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int6","hadamard_block_size":-1}'
VLLM_HUMMING_INPUT_QUANT_CONFIG: '{"dtype":"int8"}'
@@ -0,0 +1,12 @@
model_name: "Qwen/Qwen3.5-9B"
accuracy_threshold: 0.88
num_questions: 1319
num_fewshot: 5
gen_prefix: " <think>\n\n</think>\n"
server_args: >-
--enforce-eager
--max-model-len 8192
--quantization humming
--kernel-config.enable_flashinfer_autotune=False
env:
VLLM_HUMMING_ONLINE_QUANT_CONFIG: '{"dtype":"int6","hadamard_block_size":-1}'
@@ -0,0 +1,3 @@
gpt-oss-20b-humming.yaml
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml
@@ -0,0 +1,4 @@
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
Qwen3-4B-mixed-quant-RTN-humming.yaml
@@ -1,4 +1,5 @@
Qwen3.5-35B-A3B-experts-int8-humming.yaml
Qwen3.5-35B-A3B-int6wc-hadamard-humming.yaml
gpt-oss-20b-humming.yaml
Qwen3-30B-A3B-GPTQ-Int4-humming.yaml
Qwen3-30B-A3B-FP8-block-humming.yaml
@@ -3,4 +3,5 @@ Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
Qwen3.5-35B-A3B-experts-int8-humming-act-int8.yaml
Qwen3.5-35B-A3B-int6wc-hadamard-humming-act-int8.yaml
Qwen3-4B-mixed-quant-RTN-humming.yaml
@@ -0,0 +1,2 @@
Qwen3-30B-A3B-FP8-block-humming-deepep-ht-grouped.yaml
Qwen3-30B-A3B-FP8-block-humming-deepep-ht-indexed.yaml
@@ -1,4 +1,5 @@
Qwen3.5-35B-A3B-experts-int8-humming.yaml
Qwen3.5-35B-A3B-int6wc-hadamard-humming.yaml
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming.yaml
Qwen3-30B-A3B-Fp8-v1-humming.yaml
Qwen3-30B-A3B-NVFP4-humming.yaml
@@ -1,3 +0,0 @@
Qwen3-30B-A3B-int5wc-hadamard-humming.yaml
Qwen3-30B-A3B-int5wc-hadamard-humming-act-fp8.yaml
Qwen3-30B-A3B-int5wc-hadamard-humming-act-int8.yaml
@@ -0,0 +1,9 @@
Qwen2-1.5B-Instruct-FP8W8-humming.yaml
Qwen2-1.5B-Instruct-FP8W8-humming-act-fp8.yaml
Qwen3-0.6B-MXFP8-humming.yaml
Qwen3-0.6B-MXFP8-humming-act-fp8.yaml
Qwen3-30B-A3B-AWQ-humming.yaml
Qwen3-30B-A3B-GPTQ-Int4-humming-act-fp8.yaml
gpt-oss-20b-humming.yaml
Qwen3.5-9B-int6wc-hadamard-humming.yaml
Qwen3.5-9B-int6wc-hadamard-humming-act-int8.yaml
+109 -42
View File
@@ -352,49 +352,46 @@ def test_masked_situ_and_mul(
)
@pytest.mark.parametrize(
("activation", "activation_config"),
[
(MoEActivation.SILU, ApplyMoEActivationConfig()),
(
MoEActivation.SILU,
ApplyMoEActivationConfig(clamp_limit=3.0),
MOE_ACTIVATION_CASES = [
pytest.param(MoEActivation.SILU, ApplyMoEActivationConfig(), id="silu"),
pytest.param(
MoEActivation.SILU, ApplyMoEActivationConfig(clamp_limit=3.0), id="silu_clamp"
),
pytest.param(MoEActivation.GELU, ApplyMoEActivationConfig(), id="gelu"),
pytest.param(MoEActivation.GELU_TANH, ApplyMoEActivationConfig(), id="gelu_tanh"),
pytest.param(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
(MoEActivation.GELU, ApplyMoEActivationConfig()),
(MoEActivation.GELU_TANH, ApplyMoEActivationConfig()),
(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=2.0,
),
),
(MoEActivation.SWIGLUOAI, ApplyMoEActivationConfig()),
(
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
),
(MoEActivation.SWIGLUSTEP, ApplyMoEActivationConfig()),
(MoEActivation.SILU_NO_MUL, ApplyMoEActivationConfig()),
(MoEActivation.GELU_NO_MUL, ApplyMoEActivationConfig()),
(MoEActivation.GELU_TANH_NO_MUL, ApplyMoEActivationConfig()),
(MoEActivation.RELU2_NO_MUL, ApplyMoEActivationConfig()),
],
ids=[
"silu",
"silu_clamp",
"gelu",
"gelu_tanh",
"situ",
"swigluoai",
"swigluoai_uninterleave",
"swiglustep",
"silu_no_mul",
"gelu_no_mul",
"gelu_tanh_no_mul",
"relu2_no_mul",
],
)
id="situ",
),
pytest.param(MoEActivation.SWIGLUOAI, ApplyMoEActivationConfig(), id="swigluoai"),
pytest.param(
MoEActivation.SWIGLUOAI_UNINTERLEAVE,
ApplyMoEActivationConfig(clamp_limit=3.0, alpha=1.3, beta=0.5),
id="swigluoai_uninterleave",
),
pytest.param(MoEActivation.SWIGLUSTEP, ApplyMoEActivationConfig(), id="swiglustep"),
pytest.param(
MoEActivation.SILU_NO_MUL, ApplyMoEActivationConfig(), id="silu_no_mul"
),
pytest.param(
MoEActivation.GELU_NO_MUL, ApplyMoEActivationConfig(), id="gelu_no_mul"
),
pytest.param(
MoEActivation.GELU_TANH_NO_MUL,
ApplyMoEActivationConfig(),
id="gelu_tanh_no_mul",
),
pytest.param(
MoEActivation.RELU2_NO_MUL, ApplyMoEActivationConfig(), id="relu2_no_mul"
),
]
@pytest.mark.parametrize(("activation", "activation_config"), MOE_ACTIVATION_CASES)
@torch.inference_mode()
def test_masked_moe_activation_dispatch(
default_vllm_config,
@@ -497,3 +494,73 @@ def test_activation(
out = torch.empty_like(x)
opcheck(fn, (out, x))
HUMMING_ACTIVATION_CASES = MOE_ACTIVATION_CASES + [
pytest.param(MoEActivation.RELU2, ApplyMoEActivationConfig(), id="relu2"),
]
HUMMING_ACTIVATION_CASES += [
pytest.param(
MoEActivation.SITU,
ApplyMoEActivationConfig(
activation_situ_beta=1.5,
activation_situ_linear_beta=linear_beta,
),
id=f"situ-linear-beta-{linear_beta}",
)
for linear_beta in (None, 0.0, -1.0)
]
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Humming requires CUDA")
@pytest.mark.parametrize(("activation", "activation_config"), HUMMING_ACTIVATION_CASES)
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize(("num_tokens", "d"), [(1, 512), (7, 768), (83, 512)])
@torch.inference_mode()
def test_humming_activation_matches_framework(
activation: MoEActivation,
activation_config: ApplyMoEActivationConfig,
dtype: torch.dtype,
num_tokens: int,
d: int,
) -> None:
"""Compare activation math/layouts without quantization or Hadamard error."""
pytest.importorskip("humming")
from humming.ops import process_input
from vllm.model_executor.layers.quantization.utils.humming.activation import (
get_humming_activation,
)
set_random_seed(0)
width = d * 2 if activation.is_gated else d
x = 4 * torch.randn(num_tokens, width, dtype=dtype, device=CUDA_DEVICES[0])
# Exercise zero, saturation, and both sides of the clamp limits (3 and 7).
edges = x.new_tensor([0, 0.001, 1, 2.99, 3, 3.01, 6.99, 7, 7.01, 8, 16])
edges = torch.cat((-edges[1:].flip(0), edges))
x[0] = edges[torch.arange(width, device=x.device) % edges.numel()]
actual, _, _ = process_input(
x,
quant_mode="none",
hadamard_block_size=0,
**get_humming_activation(activation, activation_config),
)
expected = torch.empty(num_tokens, d, dtype=dtype, device=x.device)
if activation == MoEActivation.RELU2:
# apply_moe_activation has only the non-gated ReLU2 variant.
gate, up = x.float().chunk(2, dim=-1)
activated_gate = torch.empty_like(gate)
apply_moe_activation(MoEActivation.RELU2_NO_MUL, activated_gate, gate.clone())
expected.copy_(activated_gate * up)
else:
# The framework's non-gated ReLU2 path modifies its input in place.
apply_moe_activation(
activation, expected, x.clone(), activation_config=activation_config
)
torch.testing.assert_close(
actual,
expected,
atol=get_default_atol(expected),
rtol=get_default_rtol(expected),
)
+75 -26
View File
@@ -1245,7 +1245,7 @@ def _make_humming_indexed_experts(activation: MoEActivation):
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingIndexedExperts,
)
from vllm.model_executor.layers.quantization.utils import humming_utils
from vllm.model_executor.layers.quantization.utils import humming as humming_utils
from vllm.utils import humming
top_k, num_experts = 6, 12
@@ -1306,6 +1306,56 @@ def _make_humming_indexed_experts(activation: MoEActivation):
return experts, layer
@pytest.mark.parametrize("valid_shape_m", [8192, 8193])
@pytest.mark.parametrize("w2_block_size", [64, 96, 128])
def test_humming_indexed_routing_matches_each_projection_tile(
valid_shape_m: int, w2_block_size: int
):
"""Independently tuned projections preserve routed rows and expert identity."""
from types import SimpleNamespace
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingIndexedExperts,
)
topk_ids = torch.tensor([[0, 1], [1, 2], [2, 0]], device="cuda")
expert_map = torch.tensor([1, -1, 0], dtype=torch.int32, device="cuda")
experts = SimpleNamespace(
global_num_experts=3,
estimate_local_valid_shape_m=lambda _: valid_shape_m,
w13_tuning_config=[
[0, 8192, {"block_shape": [64, 128, 64]}],
[8192, 16384, {"block_shape": [128, 128, 64]}],
],
w2_tuning_config=[
[0, 8192, {"block_shape": [64, 128, 64]}],
[8192, 16384, {"block_shape": [w2_block_size, 128, 64]}],
],
compute_config_str="compute",
w13_tuning_config_str="w13",
w2_tuning_config_str="w2",
)
first, second, _ = HummingIndexedExperts.prepare_humming_moe_kwargs(
experts, topk_ids, expert_map, None
)
block_sizes = (64, 64) if valid_shape_m == 8192 else (128, w2_block_size)
expected_rows = torch.where(expert_map[topk_ids.flatten()] >= 0)[0]
for kwargs, block_size in zip((first, second), block_sizes):
padded = kwargs["num_tokens_padded"].item()
assert padded % block_size == 0
rows = kwargs["sorted_ids"][:padded].long()
valid = rows < topk_ids.numel()
torch.testing.assert_close(rows[valid].sort().values, expected_rows)
routed_experts = kwargs["expert_ids"][: padded // block_size]
torch.testing.assert_close(
routed_experts.repeat_interleave(block_size)[valid],
expert_map[topk_ids.flatten()[rows[valid]]],
)
assert (first["top_k"], second["top_k"]) == (2, 1)
for key in ("sorted_ids", "expert_ids", "num_tokens_padded"):
assert (first[key] is second[key]) == (block_sizes[0] == block_sizes[1])
@pytest.mark.parametrize("activation", list(MoEActivation))
def test_humming_activation_metadata_tracks_shared_apply(activation: MoEActivation):
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
@@ -1464,54 +1514,46 @@ def test_humming_delegates_to_instance_activation():
def test_humming_grouped_apply_forwards_valid_prefix(
monkeypatch: pytest.MonkeyPatch,
):
from types import MethodType, SimpleNamespace
from types import SimpleNamespace
from unittest.mock import Mock
import vllm.model_executor.layers.fused_moe.experts.fused_humming_moe as humming
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingExpertsBase,
HummingGroupedExperts,
)
activation_func = Mock()
buffers = {
"gate_up_output": torch.empty(6, 4),
"activation_output": torch.empty(6, 2),
"quanted_down_input": torch.empty(6, 2),
"down_output": torch.empty(6, 4),
}
experts = SimpleNamespace(
activation=activation_func,
num_experts=2,
estimate_local_valid_shape_m=lambda _: 6,
prepare_buffers=lambda *_: buffers,
_get_permute_scratch=lambda _: None,
quantize_input=lambda _, *, inputs, input_scale=None, quanted_input: (
inputs,
input_scale,
_get_permute_scratch=lambda _, *, indices_only=False: object(),
process_input=Mock(
side_effect=lambda _, **kwargs: (
kwargs["inputs"],
kwargs.get("input_scale"),
None,
)
),
humming_forward=Mock(),
compute_config_str="",
w13_tuning_config_str="",
w2_tuning_config_str="",
)
experts.apply_activation = MethodType(HummingExpertsBase.apply_activation, experts)
hidden_states = torch.empty(3, 4)
output = torch.empty_like(hidden_states)
topk_ids = torch.zeros((3, 2), dtype=torch.int64)
topk_weights = torch.ones_like(topk_ids, dtype=torch.float32)
expected_counts = torch.tensor([4], dtype=torch.int32)
expert_offsets = torch.tensor([0, 2, 4], dtype=torch.int64)
monkeypatch.setattr(
humming,
"moe_permute",
lambda **_: (
torch.empty(6, 4),
None,
expert_offsets,
torch.arange(6),
None,
),
"moe_prepare_scatter",
lambda *_: (expert_offsets, torch.arange(6).view(3, 2)),
)
monkeypatch.setattr(humming, "moe_unpermute", lambda **_: None)
@@ -1534,12 +1576,11 @@ def test_humming_grouped_apply_forwards_valid_prefix(
apply_router_weight_on_input=False,
)
activation_func.assert_called_once()
call_kwargs = activation_func.call_args.kwargs
call_kwargs = experts.process_input.call_args.kwargs
assert call_kwargs["activation"] == MoEActivation.SITU
assert call_kwargs["input"].shape == (6, 4)
assert call_kwargs["output"].shape == (6, 2)
torch.testing.assert_close(call_kwargs["valid_token_counts"], expected_counts)
assert call_kwargs["inputs"].shape == (6, 4)
assert call_kwargs["quanted_input"].shape == (6, 2)
torch.testing.assert_close(call_kwargs["num_valid_tokens"], expert_offsets[-1:])
def test_batched_marlin_activation_uses_expert_token_counts(
@@ -1653,7 +1694,8 @@ def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):
activation=moe_config.activation,
)
assert buffer_metas["gate_up_output"]["shape"][-1] == gate_up_size
assert buffer_metas["activation_output"]["shape"][-1] == intermediate_size
assert "activation_output" not in buffer_metas
assert buffer_metas["quanted_down_input"]["shape"][-1] == intermediate_size
assert experts.moe_problem_size(
a1=torch.empty(1, hidden_size),
w1=torch.empty(num_experts, 1),
@@ -1661,6 +1703,13 @@ def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):
topk_ids=torch.empty(1, top_k, dtype=torch.long),
) == (num_experts, 1, intermediate_size, hidden_size, top_k)
# Quantized batched input storage must cover every dispatcher's token slots.
experts.is_batched = lambda: True
experts.max_num_tokens = 17
experts.num_dispatchers = 2
batched_metas, _ = experts.get_buffer_metas(17, top_k, activation)
assert batched_metas["quanted_gate_up_input"]["shape"][0] == num_experts * 17 * 2
def test_humming_indexed_writes_supplied_output_buffer():
from vllm.forward_context import set_forward_context
@@ -22,6 +22,46 @@ BLOCK_SIZES = [32, 128]
set_random_seed(0)
@pytest.mark.parametrize("num_experts,num_tokens", [(32, 8), (256, 8), (256, 1024)])
@pytest.mark.parametrize("use_expert_map", [False, True])
def test_moe_align_scatter_masks_invalid_routes(
num_experts, num_tokens, use_expert_map
):
topk_ids = torch.arange(num_tokens * 4, device="cuda").reshape(-1, 4)
topk_ids %= num_experts
topk_ids[0] = torch.tensor([-1, num_experts, 0, 1], device="cuda")
expert_map = None
valid = (topk_ids >= 0) & (topk_ids < num_experts)
if use_expert_map:
expert_map = torch.full((num_experts,), -1, dtype=torch.int32, device="cuda")
expert_map[::2] = torch.arange(
num_experts // 2, device="cuda", dtype=torch.int32
)
valid &= topk_ids % 2 == 0
sorted_ids, expert_ids, num_padded, scatter_idx = moe_align_block_size(
topk_ids,
32,
num_experts,
expert_map,
ignore_invalid_experts=True,
return_scatter_idx=True,
)
expected = torch.arange(topk_ids.numel(), device="cuda", dtype=torch.int32)
expected[~valid.flatten()] = -1
torch.testing.assert_close(scatter_idx, expected[:, None])
active_sorted = sorted_ids[: num_padded.item()]
active_sorted = active_sorted[active_sorted < topk_ids.numel()]
torch.testing.assert_close(active_sorted.sort().values, expected[expected >= 0])
baseline = moe_align_block_size(
topk_ids, 32, num_experts, expert_map, ignore_invalid_experts=True
)
assert len(baseline) == 3
torch.testing.assert_close(num_padded, baseline[2])
torch.testing.assert_close(expert_ids, baseline[1])
def _group_tokens_by_expert(
sorted_ids: torch.Tensor,
expert_ids: torch.Tensor,
@@ -18,6 +18,7 @@ from vllm.model_executor.layers.fused_moe.moe_permute_unpermute import (
get_moe_permute_scratch,
moe_permute,
moe_permute_unpermute_supported,
moe_prepare_scatter,
moe_unpermute,
)
from vllm.platforms import current_platform
@@ -458,7 +459,7 @@ def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
hidden_dtype=hidden_states.dtype,
)
permuted, _, expert_offsets, _, _ = moe_permute(
permuted, _, expert_offsets, inverse, _ = moe_permute(
hidden_states=hidden_states,
a1q_scale=None,
topk_ids=topk_ids,
@@ -473,3 +474,27 @@ def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
torch.tensor([0, 1, 2], dtype=torch.int64, device="cuda"),
)
torch.testing.assert_close(permuted[:2], hidden_states[[0, 2]])
assert torch.all(inverse[[1, 3, 4]] >= expert_offsets[-1])
expected = torch.zeros_like(hidden_states)
expected[[0, 2]] = hidden_states[[0, 2]]
output = torch.empty_like(hidden_states)
moe_unpermute(
output, permuted, torch.ones(5, 1, device="cuda"), inverse, expert_offsets
)
torch.testing.assert_close(output, expected)
expected_inverse = inverse.clone()
expected_offsets = expert_offsets.clone()
expert_offsets, indices = moe_prepare_scatter(topk_ids, expert_map, scratch)
torch.testing.assert_close(indices.flatten(), expected_inverse)
torch.testing.assert_close(expert_offsets, expected_offsets)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
expert_offsets, indices = moe_prepare_scatter(topk_ids, expert_map, scratch)
topk_ids.zero_()
graph.replay()
expected_inverse = torch.arange(5, dtype=torch.int32, device="cuda")
expected_offsets = torch.tensor([0, 5, 5], dtype=torch.int64, device="cuda")
torch.testing.assert_close(indices.flatten(), expected_inverse)
torch.testing.assert_close(expert_offsets, expected_offsets)
@@ -14,7 +14,6 @@ from vllm.model_executor.kernels.linear import (
AiterMxfp4LinearKernel,
EmulationMxfp4LinearKernel,
FlashInferMxFp4LinearKernel,
HummingMxFp4LinearKernel,
MarlinMxFp4LinearKernel,
MxFp4LinearKernel,
MxFp4LinearLayerConfig,
@@ -44,7 +43,7 @@ _TRUE_W4A4_KERNELS = [
# Weight-only (A16) kernels: they never quantize activations. They still accept
# MXFP4 activation keys as an intentional compatibility fallback.
_WEIGHT_ONLY_KERNELS = [MarlinMxFp4LinearKernel, HummingMxFp4LinearKernel]
_WEIGHT_ONLY_KERNELS = [MarlinMxFp4LinearKernel]
def test_can_implement_is_abstract():
@@ -15,6 +15,7 @@ it in vLLM comes down to two things, both exercised here:
"""
import pytest
import torch
from vllm.model_executor.layers.fused_moe.experts.fused_humming_moe import (
HummingExpertsBase,
@@ -56,8 +57,8 @@ def test_mxfp4_weight_with_per_token_fp8_activation_still_supported():
],
)
def test_humming_fp8_input_schema_to_quant_key(group_size, expected_key):
from vllm.model_executor.layers.quantization.utils.humming_utils import (
_humming_input_schema_to_quant_key,
from vllm.model_executor.layers.quantization.utils.humming import (
input_schema_to_quant_key,
)
from vllm.utils.humming import HummingInputSchema
from vllm.utils.humming import dtypes as humming_dtypes
@@ -66,18 +67,21 @@ def test_humming_fp8_input_schema_to_quant_key(group_size, expected_key):
a_dtype=humming_dtypes.float8e4m3,
input_scale_group_size=group_size,
)
assert _humming_input_schema_to_quant_key(schema) == expected_key
assert input_schema_to_quant_key(schema, param_dtype=torch.bfloat16) == expected_key
@pytest.mark.skipif(not has_humming(), reason="humming is not installed")
def test_humming_bf16_input_schema_is_unquantized():
from vllm.model_executor.layers.quantization.utils.humming_utils import (
_humming_input_schema_to_quant_key,
from vllm.model_executor.layers.quantization.utils.humming import (
input_schema_to_quant_key,
)
from vllm.utils.humming import HummingInputSchema
# No a_dtype -> unquantized (bf16/fp16) inputs -> None.
assert _humming_input_schema_to_quant_key(HummingInputSchema()) is None
assert (
input_schema_to_quant_key(HummingInputSchema(), param_dtype=torch.bfloat16)
is None
)
def test_block_fp8_activation_quant_config_is_block_quantized():
@@ -86,7 +90,7 @@ def test_block_fp8_activation_quant_config_is_block_quantized():
DeepEP prepare/finalize step quantize activations to block FP8 *before* the
all-to-all dispatch instead of deferring to Humming -- and what makes
HummingExpertsBase.expects_unquantized_inputs return False."""
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
make_humming_moe_quant_config,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
@@ -108,7 +112,7 @@ def test_default_activation_quant_config_defers_to_humming():
"""Without a block activation shape the config stays per-token (the deferred
path): Humming quantizes internally, preserving pre-existing behavior for
every non-block-FP8 scheme."""
from vllm.model_executor.layers.quantization.utils.humming_utils import (
from vllm.model_executor.layers.quantization.utils.humming import (
make_humming_moe_quant_config,
)
from vllm.platforms import current_platform
+1 -1
View File
@@ -229,7 +229,7 @@ def test_moe_wna16_humming_adapter_repacks_uint8_tensors():
def test_moe_wna16_uses_humming_quant_config(monkeypatch):
from vllm.model_executor.layers.quantization.utils import humming_utils
from vllm.model_executor.layers.quantization.utils import humming as humming_utils
method = object.__new__(MoeWNA16Method)
method.wna16_backend = WNA16MoEBackend.HUMMING