[MoE Refactor] Remove MoE DP chunking (#39107)
Signed-off-by: Bill Nell <bnell@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -46,6 +46,7 @@ from vllm.utils.import_utils import (
|
||||
has_deep_gemm,
|
||||
has_mori,
|
||||
)
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
|
||||
from .mk_objects import (
|
||||
TestMoEQuantConfig,
|
||||
@@ -604,13 +605,6 @@ def make_modular_kernel(
|
||||
vllm_config: VllmConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
) -> mk.FusedMoEKernel:
|
||||
def next_power_of_2(x):
|
||||
import math
|
||||
|
||||
if x == 0:
|
||||
return 1
|
||||
return 2 ** math.ceil(math.log2(x))
|
||||
|
||||
# make moe config
|
||||
moe_parallel_config: FusedMoEParallelConfig = FusedMoEParallelConfig.make(
|
||||
tp_size_=get_tensor_model_parallel_world_size(),
|
||||
|
||||
@@ -126,7 +126,7 @@ def parallel_launch_with_config(
|
||||
world_size: int,
|
||||
worker: Callable[Concatenate[ProcessGroupInfo, VllmConfig, Any, P], None],
|
||||
vllm_config: VllmConfig,
|
||||
env_dict: dict[Any, Any],
|
||||
env_dict: dict[Any, Any] | None,
|
||||
*args: P.args,
|
||||
**kwargs: P.kwargs,
|
||||
) -> None:
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.utils.deep_gemm import (
|
||||
is_deep_gemm_supported,
|
||||
)
|
||||
from vllm.utils.import_utils import has_deep_ep, has_deep_gemm
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
@@ -84,14 +85,6 @@ def with_dp_metadata(M: int, world_size: int):
|
||||
yield
|
||||
|
||||
|
||||
def next_power_of_2(x):
|
||||
import math
|
||||
|
||||
if x == 0:
|
||||
return 1
|
||||
return 2 ** math.ceil(math.log2(x))
|
||||
|
||||
|
||||
def make_block_quant_fp8_weights(
|
||||
e: int,
|
||||
n: int,
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8
|
||||
from vllm.model_executor.models.llama4 import Llama4MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
try:
|
||||
@@ -174,6 +175,7 @@ class TestData:
|
||||
routing_method=layer.routing_method_type,
|
||||
activation=activation,
|
||||
device=w13_quantized.device,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
|
||||
return TestData(
|
||||
@@ -348,6 +350,7 @@ def test_flashinfer_cutlass_moe_fp8_no_graph(
|
||||
in_dtype=torch.bfloat16,
|
||||
is_act_and_mul=activation.is_gated,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
|
||||
kernel = mk.FusedMoEKernel(
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.model_executor.layers.fused_moe.flashinfer_cutlass_moe import (
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import FusedMoEKernel
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
if not has_flashinfer_cutlass_fused_moe() or not current_platform.has_device_capability(
|
||||
@@ -105,6 +106,7 @@ def test_flashinfer_fp4_moe_no_graph(
|
||||
in_dtype=dtype,
|
||||
is_act_and_mul=is_gated_act,
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
|
||||
flashinfer_experts = FusedMoEKernel(
|
||||
|
||||
@@ -59,6 +59,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import quantize_w
|
||||
from vllm.model_executor.models.mixtral import MixtralMoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
@@ -1676,7 +1677,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
|
||||
in_dtype=dtype,
|
||||
is_act_and_mul=True,
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
max_num_tokens=m,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
|
||||
@@ -26,6 +26,7 @@ from tests.kernels.moe.utils import TestMLP, make_test_weights, moe_quantize_wei
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
ParallelConfig,
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
@@ -53,7 +54,7 @@ from vllm.utils.flashinfer import (
|
||||
has_flashinfer_nvlink_two_sided,
|
||||
)
|
||||
from vllm.utils.import_utils import has_deep_ep, has_mori, has_nixl_ep
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.worker.workspace import (
|
||||
init_workspace_manager,
|
||||
@@ -65,8 +66,9 @@ fp8_dtype = torch.float8_e4m3fn # current_platform.fp8_dtype
|
||||
SHAPE_COMBOS = [
|
||||
(1, 128, 256),
|
||||
(32, 1024, 512),
|
||||
(222, 2048, 2048), # should be big enough to exercise DP chunking
|
||||
(222, 2048, 2048),
|
||||
]
|
||||
MAX_M = max([x[0] for x in SHAPE_COMBOS])
|
||||
|
||||
NUM_EXPERTS = [8, 64]
|
||||
TOP_KS = [2, 6]
|
||||
@@ -112,7 +114,7 @@ BACKEND_SUPPORTED_QUANTS: dict[str, set[str | None]] = {
|
||||
"mori": {None, "fp8", "modelopt_fp8"},
|
||||
"flashinfer_nvlink_two_sided": {None, "modelopt_fp8", "modelopt_fp4"},
|
||||
"flashinfer_nvlink_one_sided": {None, "modelopt_fp8", "modelopt_fp4"},
|
||||
"deepep_low_latency": {None, "fp8", "modelopt_fp8", "modelopt_fp4"},
|
||||
"deepep_low_latency": {None, "modelopt_fp8", "modelopt_fp4"},
|
||||
"deepep_high_throughput": {None, "fp8", "modelopt_fp8", "modelopt_fp4"},
|
||||
"nixl_ep": {None, "fp8", "modelopt_fp8"},
|
||||
}
|
||||
@@ -363,9 +365,9 @@ def is_valid_config(config: MoETestConfig) -> tuple[bool, str | None]:
|
||||
)
|
||||
|
||||
# routed_input_transform + quantization + high hidden dimensions
|
||||
# TODO: Disable >= 2048 w/fp8 + deepep LL for now due to insane errors.
|
||||
# TODO: Disable >= 2048 for now due to insane errors.
|
||||
if (
|
||||
(config.use_routed_input_transform or config.backend == "deepep_low_latency")
|
||||
config.use_routed_input_transform
|
||||
and config.quantization is not None
|
||||
and config.k >= 2048
|
||||
):
|
||||
@@ -1663,9 +1665,6 @@ def test_moe_layer(
|
||||
|
||||
verbosity = pytestconfig.getoption("verbose")
|
||||
|
||||
test_env = dict()
|
||||
test_env["VLLM_MOE_DP_CHUNK_SIZE"] = "128"
|
||||
monkeypatch.setenv("VLLM_MOE_DP_CHUNK_SIZE", "128")
|
||||
if os.environ.get("VLLM_LOGGING_LEVEL") is None:
|
||||
monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
|
||||
|
||||
@@ -1690,7 +1689,11 @@ def test_moe_layer(
|
||||
compilation_config.pass_config.fuse_allreduce_rms = False # for now
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
parallel_config=parallel_config, compilation_config=compilation_config
|
||||
parallel_config=parallel_config,
|
||||
compilation_config=compilation_config,
|
||||
scheduler_config=SchedulerConfig.default_factory(
|
||||
max_num_batched_tokens=next_power_of_2(MAX_M)
|
||||
),
|
||||
)
|
||||
|
||||
test_configs = generate_valid_test_configs(
|
||||
@@ -1718,7 +1721,7 @@ def test_moe_layer(
|
||||
world_size,
|
||||
_parallel_worker,
|
||||
vllm_config,
|
||||
test_env,
|
||||
None,
|
||||
test_configs,
|
||||
verbosity,
|
||||
)
|
||||
|
||||
@@ -69,6 +69,7 @@ def make_dummy_moe_config(
|
||||
in_dtype=in_dtype,
|
||||
device="cuda",
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=512,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user