[Refactor] refactor humming linear and moe backends to use explicit layer configs (#49610)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
e229fdbc87
commit
fcde8e1460
@@ -0,0 +1,9 @@
|
||||
model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"
|
||||
accuracy_threshold: 0.29
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
--enforce-eager
|
||||
--max-model-len 8192
|
||||
--moe-backend humming
|
||||
--linear-backend humming
|
||||
@@ -1,5 +1,5 @@
|
||||
model_name: "nm-testing/Qwen3-4B-mixed-quant-RTN-wNaM"
|
||||
accuracy_threshold: 0.85
|
||||
accuracy_threshold: 0.78
|
||||
num_questions: 1319
|
||||
num_fewshot: 5
|
||||
server_args: >-
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
gpt-oss-20b-humming.yaml
|
||||
Qwen3-30B-A3B-Instruct-2507-quantized.w8a8-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-GPTQ-Int4-humming-act-int8.yaml
|
||||
Qwen3-30B-A3B-AWQ-humming.yaml
|
||||
|
||||
@@ -11,3 +11,4 @@ Qwen3-30B-A3B-NVFP4-humming.yaml
|
||||
Qwen3-30B-A3B-AWQ-humming.yaml
|
||||
Qwen3.5-35B-A3B-FP8-humming.yaml
|
||||
Qwen3.5-35B-A3B-experts-int8-humming.yaml
|
||||
NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4-humming.yaml
|
||||
|
||||
@@ -1265,11 +1265,10 @@ def _make_humming_indexed_experts(activation: MoEActivation):
|
||||
layer.intermediate_size_per_partition = intermediate_size
|
||||
quant_config = humming_utils.get_humming_moe_quant_config(layer)
|
||||
experts = HummingIndexedExperts(
|
||||
layer,
|
||||
moe_config,
|
||||
quant_config,
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
return experts
|
||||
return experts, layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("activation", list(MoEActivation))
|
||||
@@ -1328,8 +1327,7 @@ def test_humming_delegates_to_instance_activation():
|
||||
def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):
|
||||
from vllm.utils import humming
|
||||
|
||||
experts = _make_humming_indexed_experts(activation)
|
||||
layer = experts.layer
|
||||
experts, layer = _make_humming_indexed_experts(activation)
|
||||
moe_config = experts.moe_config
|
||||
top_k = moe_config.experts_per_token
|
||||
num_experts = moe_config.num_experts
|
||||
@@ -1337,7 +1335,7 @@ def test_humming_gated_non_gated_shape_contract(activation: MoEActivation):
|
||||
intermediate_size = moe_config.intermediate_size
|
||||
gate_up_size = intermediate_size * 2 if activation.is_gated else intermediate_size
|
||||
|
||||
w13_meta, w2_meta = (layer.humming_metas[name] for name in ("w13", "w2"))
|
||||
w13_meta, w2_meta = (experts.humming_configs[name] for name in ("w13", "w2"))
|
||||
for meta in (w13_meta, w2_meta):
|
||||
assert meta.a_dtype == humming.dtypes.bfloat16
|
||||
assert meta.b_dtype == humming.dtypes.float4e2m1
|
||||
@@ -1364,7 +1362,7 @@ def test_humming_indexed_writes_supplied_output_buffer():
|
||||
from vllm.forward_context import set_forward_context
|
||||
|
||||
activation = MoEActivation.SILU
|
||||
experts = _make_humming_indexed_experts(activation)
|
||||
experts, layer = _make_humming_indexed_experts(activation)
|
||||
moe_config = experts.moe_config
|
||||
num_tokens = 1
|
||||
top_k = moe_config.experts_per_token
|
||||
@@ -1382,7 +1380,7 @@ def test_humming_indexed_writes_supplied_output_buffer():
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
dtype = experts.layer.params_dtype
|
||||
dtype = layer.params_dtype
|
||||
workspace13 = torch.empty(workspace13_shape, dtype=dtype, device=device)
|
||||
workspace2 = torch.empty(workspace2_shape, dtype=dtype, device=device)
|
||||
hidden_states = torch.ones((num_tokens, hidden_size), dtype=dtype, device=device)
|
||||
@@ -1394,14 +1392,12 @@ def test_humming_indexed_writes_supplied_output_buffer():
|
||||
device=device,
|
||||
)
|
||||
topk_ids = torch.arange(top_k, dtype=torch.int32, device=device).unsqueeze(0)
|
||||
unused = torch.empty((num_experts, 0), device=device)
|
||||
|
||||
with set_forward_context(None, vllm_config, num_tokens=num_tokens):
|
||||
experts.apply(
|
||||
output=output,
|
||||
hidden_states=hidden_states,
|
||||
w1=unused,
|
||||
w2=unused,
|
||||
w1=layer.w13_weight,
|
||||
w2=layer.w2_weight,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
|
||||
@@ -1489,7 +1489,6 @@ def test_rocm_mxfp4_moe_oracle(
|
||||
mxfp4_backend=backend,
|
||||
experts_cls=experts_cls,
|
||||
routing_tables=None,
|
||||
layer=None,
|
||||
)
|
||||
|
||||
# Create inputs
|
||||
|
||||
@@ -161,7 +161,6 @@ def test_moe_wna16_setup_forwards_selected_backend(monkeypatch):
|
||||
|
||||
assert method.moe_kernel is kernel
|
||||
assert captured["backend"] == WNA16MoEBackend.HUMMING
|
||||
assert captured["layer"] is layer
|
||||
|
||||
|
||||
def test_moe_wna16_humming_adapter_repacks_uint8_tensors():
|
||||
@@ -198,7 +197,9 @@ def test_moe_wna16_uses_humming_quant_config(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
humming_utils,
|
||||
"get_humming_moe_quant_config",
|
||||
lambda actual_layer: quant_config if actual_layer is layer else None,
|
||||
lambda actual_layer, *args, **kwargs: (
|
||||
quant_config if actual_layer is layer else None
|
||||
),
|
||||
)
|
||||
|
||||
assert method.get_fused_moe_quant_config(layer) is quant_config
|
||||
|
||||
Reference in New Issue
Block a user