[ROCm] mori: add InterNodeV1LL inter-node kernel selection via VLLM_MORI_INTERNODE_KERNEL (#41751)
Signed-off-by: jatseng-ai <jatseng@amd.com>
This commit is contained in:
@@ -249,7 +249,7 @@ class Config:
|
||||
|
||||
def needs_mori(self):
|
||||
info = prepare_finalize_info(self.prepare_finalize_type)
|
||||
return info.backend == "mori"
|
||||
return info.backend in ("mori_high_throughput", "mori_low_latency")
|
||||
|
||||
def all2all_backend(self):
|
||||
info = prepare_finalize_info(self.prepare_finalize_type)
|
||||
|
||||
@@ -232,7 +232,7 @@ if has_mori():
|
||||
standard_format,
|
||||
fp8_types,
|
||||
blocked_quantization_support=True,
|
||||
backend="mori",
|
||||
backend="mori_high_throughput",
|
||||
supports_apply_weight_on_input=False,
|
||||
)
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ PARALLEL_COMBOS = [
|
||||
BACKENDS = ["allgather_reducescatter"]
|
||||
|
||||
if has_mori():
|
||||
BACKENDS += ["mori"]
|
||||
BACKENDS += ["mori_high_throughput", "mori_low_latency"]
|
||||
|
||||
if has_flashinfer_nvlink_two_sided():
|
||||
BACKENDS += ["flashinfer_nvlink_two_sided"]
|
||||
@@ -118,7 +118,8 @@ QUANT_METHODS = [
|
||||
# fmt: off
|
||||
BACKEND_SUPPORTED_QUANTS: dict[str, set[str | None]] = {
|
||||
"allgather_reducescatter": {None, "fp8", "modelopt_fp8", "modelopt_fp4"}, # noqa: E501
|
||||
"mori": {None, "fp8", "modelopt_fp8"},
|
||||
"mori_high_throughput": {None, "fp8", "modelopt_fp8"},
|
||||
"mori_low_latency": {None, "fp8", "modelopt_fp8"},
|
||||
"flashinfer_nvlink_two_sided": {None, "fp8_blocked", "modelopt_fp4"}, # noqa: E501
|
||||
"flashinfer_nvlink_one_sided": {None, "modelopt_fp4"}, # noqa: E501
|
||||
"deepep_low_latency": {None, "fp8_blocked", "modelopt_fp4"}, # noqa: E501
|
||||
@@ -129,7 +130,8 @@ BACKEND_SUPPORTED_QUANTS: dict[str, set[str | None]] = {
|
||||
# Map from backend -> (DP/EP support, DP support, TP support, SP support)
|
||||
BACKEND_EP_DP_TP_SUPPORT: dict[str, tuple[bool, bool, bool, bool]] = {
|
||||
"allgather_reducescatter": (True, True, True, True),
|
||||
"mori": (True, False, False, True),
|
||||
"mori_high_throughput": (True, False, False, True),
|
||||
"mori_low_latency": (True, False, False, True),
|
||||
"flashinfer_nvlink_two_sided": (False, True, False, False),
|
||||
"flashinfer_nvlink_one_sided": (False, True, False, False),
|
||||
"deepep_low_latency": (True, False, False, True),
|
||||
|
||||
Reference in New Issue
Block a user