[MoE Refactor] Remove MoE DP chunking (#39107)

Signed-off-by: Bill Nell <bnell@redhat.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
bnellnm
2026-04-14 09:48:05 -04:00
committed by GitHub
co-authored by Robert Shaw
parent f7e62e3d66
commit e1e318af01
20 changed files with 76 additions and 389 deletions
@@ -46,6 +46,7 @@ from vllm.utils.import_utils import (
has_deep_gemm,
has_mori,
)
from vllm.utils.math_utils import next_power_of_2
from .mk_objects import (
TestMoEQuantConfig,
@@ -604,13 +605,6 @@ def make_modular_kernel(
vllm_config: VllmConfig,
quant_config: FusedMoEQuantConfig,
) -> mk.FusedMoEKernel:
def next_power_of_2(x):
import math
if x == 0:
return 1
return 2 ** math.ceil(math.log2(x))
# make moe config
moe_parallel_config: FusedMoEParallelConfig = FusedMoEParallelConfig.make(
tp_size_=get_tensor_model_parallel_world_size(),
@@ -126,7 +126,7 @@ def parallel_launch_with_config(
world_size: int,
worker: Callable[Concatenate[ProcessGroupInfo, VllmConfig, Any, P], None],
vllm_config: VllmConfig,
env_dict: dict[Any, Any],
env_dict: dict[Any, Any] | None,
*args: P.args,
**kwargs: P.kwargs,
) -> None:
@@ -29,6 +29,7 @@ from vllm.utils.deep_gemm import (
is_deep_gemm_supported,
)
from vllm.utils.import_utils import has_deep_ep, has_deep_gemm
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.worker.workspace import init_workspace_manager
@@ -84,14 +85,6 @@ def with_dp_metadata(M: int, world_size: int):
yield
def next_power_of_2(x):
import math
if x == 0:
return 1
return 2 ** math.ceil(math.log2(x))
def make_block_quant_fp8_weights(
e: int,
n: int,
+3
View File
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8
from vllm.model_executor.models.llama4 import Llama4MoE
from vllm.platforms import current_platform
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import set_random_seed
try:
@@ -174,6 +175,7 @@ class TestData:
routing_method=layer.routing_method_type,
activation=activation,
device=w13_quantized.device,
max_num_tokens=next_power_of_2(m),
)
return TestData(
@@ -348,6 +350,7 @@ def test_flashinfer_cutlass_moe_fp8_no_graph(
in_dtype=torch.bfloat16,
is_act_and_mul=activation.is_gated,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
kernel = mk.FusedMoEKernel(
+2
View File
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import set_random_seed
if not has_flashinfer_cutlass_fused_moe() or not current_platform.has_device_capability(
@@ -105,6 +106,7 @@ def test_flashinfer_fp4_moe_no_graph(
in_dtype=dtype,
is_act_and_mul=is_gated_act,
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
flashinfer_experts = FusedMoEKernel(
+2 -1
View File
@@ -59,6 +59,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import quantize_w
from vllm.model_executor.models.mixtral import MixtralMoE
from vllm.platforms import current_platform
from vllm.scalar_type import ScalarType, scalar_types
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.worker.workspace import init_workspace_manager
@@ -1676,7 +1677,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
in_dtype=dtype,
is_act_and_mul=True,
routing_method=RoutingMethodType.Renormalize,
max_num_tokens=m,
max_num_tokens=next_power_of_2(m),
)
with set_current_vllm_config(vllm_config):
+13 -10
View File
@@ -26,6 +26,7 @@ from tests.kernels.moe.utils import TestMLP, make_test_weights, moe_quantize_wei
from vllm.config import (
CompilationConfig,
ParallelConfig,
SchedulerConfig,
VllmConfig,
set_current_vllm_config,
)
@@ -53,7 +54,7 @@ from vllm.utils.flashinfer import (
has_flashinfer_nvlink_two_sided,
)
from vllm.utils.import_utils import has_deep_ep, has_mori, has_nixl_ep
from vllm.utils.math_utils import cdiv
from vllm.utils.math_utils import cdiv, next_power_of_2
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.worker.workspace import (
init_workspace_manager,
@@ -65,8 +66,9 @@ fp8_dtype = torch.float8_e4m3fn # current_platform.fp8_dtype
SHAPE_COMBOS = [
(1, 128, 256),
(32, 1024, 512),
(222, 2048, 2048), # should be big enough to exercise DP chunking
(222, 2048, 2048),
]
MAX_M = max([x[0] for x in SHAPE_COMBOS])
NUM_EXPERTS = [8, 64]
TOP_KS = [2, 6]
@@ -112,7 +114,7 @@ BACKEND_SUPPORTED_QUANTS: dict[str, set[str | None]] = {
"mori": {None, "fp8", "modelopt_fp8"},
"flashinfer_nvlink_two_sided": {None, "modelopt_fp8", "modelopt_fp4"},
"flashinfer_nvlink_one_sided": {None, "modelopt_fp8", "modelopt_fp4"},
"deepep_low_latency": {None, "fp8", "modelopt_fp8", "modelopt_fp4"},
"deepep_low_latency": {None, "modelopt_fp8", "modelopt_fp4"},
"deepep_high_throughput": {None, "fp8", "modelopt_fp8", "modelopt_fp4"},
"nixl_ep": {None, "fp8", "modelopt_fp8"},
}
@@ -363,9 +365,9 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]:
)
# routed_input_transform + quantization + high hidden dimensions
# TODO: Disable >= 2048 w/fp8 + deepep LL for now due to insane errors.
# TODO: Disable >= 2048 for now due to insane errors.
if (
(config.use_routed_input_transform or config.backend == "deepep_low_latency")
config.use_routed_input_transform
and config.quantization is not None
and config.k >= 2048
):
@@ -1663,9 +1665,6 @@ def test_moe_layer(
verbosity = pytestconfig.getoption("verbose")
test_env = dict()
test_env["VLLM_MOE_DP_CHUNK_SIZE"] = "128"
monkeypatch.setenv("VLLM_MOE_DP_CHUNK_SIZE", "128")
if os.environ.get("VLLM_LOGGING_LEVEL") is None:
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
@@ -1690,7 +1689,11 @@ def test_moe_layer(
compilation_config.pass_config.fuse_allreduce_rms = False # for now
vllm_config = VllmConfig(
parallel_config=parallel_config, compilation_config=compilation_config
parallel_config=parallel_config,
compilation_config=compilation_config,
scheduler_config=SchedulerConfig.default_factory(
max_num_batched_tokens=next_power_of_2(MAX_M)
),
)
test_configs = generate_valid_test_configs(
@@ -1718,7 +1721,7 @@ def test_moe_layer(
world_size,
_parallel_worker,
vllm_config,
test_env,
None,
test_configs,
verbosity,
)
+1
View File
@@ -69,6 +69,7 @@ def make_dummy_moe_config(
in_dtype=in_dtype,
device="cuda",
routing_method=RoutingMethodType.TopK,
max_num_tokens=512,
)