Signed-off-by: Felix Marty <Felix.Marty@amd.com>
This commit is contained in:
@@ -30,9 +30,32 @@ from vllm.model_executor.layers.fused_moe.prepare_finalize.flashinfer_nvlink_two
|
|||||||
FlashInferNVLinkTwoSidedPrepareAndFinalize,
|
FlashInferNVLinkTwoSidedPrepareAndFinalize,
|
||||||
)
|
)
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
|
from vllm.utils.import_utils import (
|
||||||
|
has_deep_ep,
|
||||||
|
has_deep_ep_v2,
|
||||||
|
has_mori,
|
||||||
|
has_nixl_ep,
|
||||||
|
)
|
||||||
|
|
||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
if current_platform.is_cuda_alike():
|
||||||
|
if has_deep_ep():
|
||||||
|
from .prepare_finalize.deepep_ht import DeepEPHTPrepareAndFinalize
|
||||||
|
from .prepare_finalize.deepep_ll import (
|
||||||
|
DEEPEP_QUANT_BLOCK_SHAPE,
|
||||||
|
DeepEPLLPrepareAndFinalize,
|
||||||
|
)
|
||||||
|
if has_deep_ep_v2():
|
||||||
|
from .prepare_finalize.deepep_v2 import DeepEPV2PrepareAndFinalize
|
||||||
|
if has_mori():
|
||||||
|
from .prepare_finalize.mori import MoriPrepareAndFinalize
|
||||||
|
if has_nixl_ep():
|
||||||
|
from .prepare_finalize.nixl_ep import (
|
||||||
|
NIXL_EP_QUANT_BLOCK_SHAPE,
|
||||||
|
NixlEPPrepareAndFinalize,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def get_ep_all2all_manager(eep_stage: bool = False) -> Any:
|
def get_ep_all2all_manager(eep_stage: bool = False) -> Any:
|
||||||
if eep_stage:
|
if eep_stage:
|
||||||
@@ -70,29 +93,21 @@ def maybe_roundup_layer_hidden_size(
|
|||||||
Original hidden size otherwise.
|
Original hidden size otherwise.
|
||||||
"""
|
"""
|
||||||
if moe_parallel_config.use_deepep_ht_kernels:
|
if moe_parallel_config.use_deepep_ht_kernels:
|
||||||
from .prepare_finalize.deepep_ht import DeepEPHTPrepareAndFinalize
|
|
||||||
|
|
||||||
hidden_size = DeepEPHTPrepareAndFinalize.maybe_roundup_layer_hidden_size(
|
hidden_size = DeepEPHTPrepareAndFinalize.maybe_roundup_layer_hidden_size(
|
||||||
hidden_size, act_dtype
|
hidden_size, act_dtype
|
||||||
)
|
)
|
||||||
|
|
||||||
if moe_parallel_config.use_deepep_ll_kernels:
|
if moe_parallel_config.use_deepep_ll_kernels:
|
||||||
from .prepare_finalize.deepep_ll import DeepEPLLPrepareAndFinalize
|
|
||||||
|
|
||||||
hidden_size = DeepEPLLPrepareAndFinalize.maybe_roundup_layer_hidden_size(
|
hidden_size = DeepEPLLPrepareAndFinalize.maybe_roundup_layer_hidden_size(
|
||||||
hidden_size
|
hidden_size
|
||||||
)
|
)
|
||||||
|
|
||||||
if moe_parallel_config.use_deepep_v2_kernels:
|
if moe_parallel_config.use_deepep_v2_kernels:
|
||||||
from .prepare_finalize.deepep_v2 import DeepEPV2PrepareAndFinalize
|
|
||||||
|
|
||||||
hidden_size = DeepEPV2PrepareAndFinalize.maybe_roundup_layer_hidden_size(
|
hidden_size = DeepEPV2PrepareAndFinalize.maybe_roundup_layer_hidden_size(
|
||||||
hidden_size, act_dtype
|
hidden_size, act_dtype
|
||||||
)
|
)
|
||||||
|
|
||||||
if moe_parallel_config.use_nixl_ep_kernels:
|
if moe_parallel_config.use_nixl_ep_kernels:
|
||||||
from .prepare_finalize.nixl_ep import NixlEPPrepareAndFinalize
|
|
||||||
|
|
||||||
hidden_size = NixlEPPrepareAndFinalize.maybe_roundup_layer_hidden_size(
|
hidden_size = NixlEPPrepareAndFinalize.maybe_roundup_layer_hidden_size(
|
||||||
hidden_size
|
hidden_size
|
||||||
)
|
)
|
||||||
@@ -156,8 +171,6 @@ def maybe_make_prepare_finalize(
|
|||||||
prepare_finalize: FusedMoEPrepareAndFinalize | None = None
|
prepare_finalize: FusedMoEPrepareAndFinalize | None = None
|
||||||
|
|
||||||
if moe.use_deepep_ht_kernels:
|
if moe.use_deepep_ht_kernels:
|
||||||
from .prepare_finalize.deepep_ht import DeepEPHTPrepareAndFinalize
|
|
||||||
|
|
||||||
assert moe.dp_size == all2all_manager.dp_world_size
|
assert moe.dp_size == all2all_manager.dp_world_size
|
||||||
|
|
||||||
all_to_all_args: dict[str, Any] = dict()
|
all_to_all_args: dict[str, Any] = dict()
|
||||||
@@ -170,11 +183,6 @@ def maybe_make_prepare_finalize(
|
|||||||
)
|
)
|
||||||
|
|
||||||
elif moe.use_deepep_ll_kernels:
|
elif moe.use_deepep_ll_kernels:
|
||||||
from .prepare_finalize.deepep_ll import (
|
|
||||||
DEEPEP_QUANT_BLOCK_SHAPE,
|
|
||||||
DeepEPLLPrepareAndFinalize,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert quant_config is not None
|
assert quant_config is not None
|
||||||
global_to_physical = physical_to_global = local_expert_global_ids = None
|
global_to_physical = physical_to_global = local_expert_global_ids = None
|
||||||
if routing_tables is not None:
|
if routing_tables is not None:
|
||||||
@@ -209,8 +217,6 @@ def maybe_make_prepare_finalize(
|
|||||||
local_expert_global_ids=local_expert_global_ids,
|
local_expert_global_ids=local_expert_global_ids,
|
||||||
)
|
)
|
||||||
elif moe.use_deepep_v2_kernels:
|
elif moe.use_deepep_v2_kernels:
|
||||||
from .prepare_finalize.deepep_v2 import DeepEPV2PrepareAndFinalize
|
|
||||||
|
|
||||||
assert moe.dp_size == all2all_manager.dp_world_size
|
assert moe.dp_size == all2all_manager.dp_world_size
|
||||||
|
|
||||||
use_fp8_dispatch = (
|
use_fp8_dispatch = (
|
||||||
@@ -241,8 +247,6 @@ def maybe_make_prepare_finalize(
|
|||||||
)
|
)
|
||||||
|
|
||||||
elif moe.use_mori_kernels:
|
elif moe.use_mori_kernels:
|
||||||
from .prepare_finalize.mori import MoriPrepareAndFinalize
|
|
||||||
|
|
||||||
assert quant_config is not None
|
assert quant_config is not None
|
||||||
|
|
||||||
# Note: We may want to use FP8 dispatch just to reduce
|
# Note: We may want to use FP8 dispatch just to reduce
|
||||||
@@ -330,11 +334,6 @@ def maybe_make_prepare_finalize(
|
|||||||
)
|
)
|
||||||
|
|
||||||
elif moe.use_nixl_ep_kernels:
|
elif moe.use_nixl_ep_kernels:
|
||||||
from .prepare_finalize.nixl_ep import (
|
|
||||||
NIXL_EP_QUANT_BLOCK_SHAPE,
|
|
||||||
NixlEPPrepareAndFinalize,
|
|
||||||
)
|
|
||||||
|
|
||||||
assert quant_config is not None
|
assert quant_config is not None
|
||||||
global_to_physical = physical_to_global = local_expert_global_ids = None
|
global_to_physical = physical_to_global = local_expert_global_ids = None
|
||||||
if routing_tables is not None:
|
if routing_tables is not None:
|
||||||
|
|||||||
Reference in New Issue
Block a user