[ROCm][AITER] Directly Implement AITER Custom All-reduce in CudaCommunicator (#46065)

Signed-off-by: BadrBasowid <badr.basowid@gmail.com>
This commit is contained in:
BadrBasowid
2026-07-06 12:16:32 +00:00
committed by GitHub
parent 40cc2e8327
commit 740f379fae
12 changed files with 499 additions and 148 deletions
+34 -95
View File
@@ -2,12 +2,9 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
from collections.abc import Callable
from contextlib import contextmanager
from typing import Protocol
import torch
from torch._ops import OpOverload
from torch.distributed import ProcessGroup
import vllm.envs as envs
from vllm.platforms import current_platform
@@ -52,42 +49,6 @@ def is_aiter_found() -> bool:
IS_AITER_FOUND = is_aiter_found()
class AiterCustomAllreduceProto(Protocol):
max_size: int
world_size: int
fully_connected: bool
@contextmanager
def capture(self): ...
def close(self) -> None: ...
def fused_ar_rms(
self,
inp: torch.Tensor,
res_inp: torch.Tensor,
*,
w: torch.Tensor,
eps: float,
registered: bool = False,
use_1stage: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]: ...
def fused_ar_rms_per_group_quant(
self,
inp: torch.Tensor,
res_inp: torch.Tensor,
*,
w: torch.Tensor,
eps: float,
group_size: int = 128,
registered: bool = False,
use_1stage: bool = False,
emit_bf16: bool = False,
) -> (
tuple[torch.Tensor, torch.Tensor, torch.Tensor]
| tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]
): ...
def should_custom_ar(self, inp: torch.Tensor) -> bool: ...
def is_aiter_found_and_supported() -> bool:
"""Check if AITER library is available and platform supports it.
@@ -830,6 +791,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
) -> tuple[torch.Tensor, torch.Tensor]:
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
assert aiter_ar is not None, "aiter allreduce must be initialized"
ca = aiter_ar.aiter_ca
total_bytes = input_.numel() * input_.element_size()
hidden_dim = input_.shape[-1]
@@ -840,8 +802,8 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
else:
hidden_ok = False
token_ok = token_num <= 80
world_size = aiter_ar.world_size
full_nvlink = aiter_ar.fully_connected
world_size = ca.world_size
full_nvlink = ca.fully_connected
if world_size == 2:
size_ok = True
@@ -854,12 +816,11 @@ def _rocm_aiter_fused_allreduce_rmsnorm_impl(
use_1stage = hidden_ok and token_ok and size_ok
result = aiter_ar.fused_ar_rms(
result = ca.custom_fused_ar_rms(
input_,
residual,
w=weight,
eps=epsilon,
registered=torch.cuda.is_current_stream_capturing(),
weight,
epsilon,
use_1stage=use_1stage,
)
assert result is not None
@@ -890,6 +851,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
"""
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
assert aiter_ar is not None, "aiter allreduce must be initialized"
ca = aiter_ar.aiter_ca
total_bytes = input_.numel() * input_.element_size()
hidden_dim = input_.shape[-1]
@@ -900,8 +862,8 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
else:
hidden_ok = False
token_ok = token_num <= 80
world_size = aiter_ar.world_size
full_nvlink = aiter_ar.fully_connected
world_size = ca.world_size
full_nvlink = ca.fully_connected
if world_size == 2:
size_ok = True
@@ -914,7 +876,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_impl(
use_1stage = hidden_ok and token_ok and size_ok
result = aiter_ar.fused_ar_rms_per_group_quant(
result = ca.fused_ar_rms_per_group_quant(
input_,
residual,
w=weight,
@@ -962,6 +924,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl(
"""
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
assert aiter_ar is not None, "aiter allreduce must be initialized"
ca = aiter_ar.aiter_ca
total_bytes = input_.numel() * input_.element_size()
hidden_dim = input_.shape[-1]
@@ -972,8 +935,8 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl(
else:
hidden_ok = False
token_ok = token_num <= 80
world_size = aiter_ar.world_size
full_nvlink = aiter_ar.fully_connected
world_size = ca.world_size
full_nvlink = ca.fully_connected
if world_size == 2:
size_ok = True
@@ -986,7 +949,7 @@ def _rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_impl(
use_1stage = hidden_ok and token_ok and size_ok
result = aiter_ar.fused_ar_rms_per_group_quant(
result = ca.fused_ar_rms_per_group_quant(
input_,
residual,
w=weight,
@@ -1577,6 +1540,7 @@ class rocm_aiter_ops:
# Check if the env variable is set
_AITER_ENABLED = envs.VLLM_ROCM_USE_AITER
_CUSTOM_ALL_REDUCE_ENABLED = envs.VLLM_ROCM_USE_AITER_CUSTOM_AR
_LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR
_FMOE_ENABLED = envs.VLLM_ROCM_USE_AITER_MOE
_MLA_ENABLED = envs.VLLM_ROCM_USE_AITER_MLA
@@ -1598,9 +1562,6 @@ class rocm_aiter_ops:
# num_shared_experts / shared_expert_scoring_func args (7-arg form).
_TOPK_SOFTMAX_FUSED_SIGMOID: bool | None = None
_ALL_REDUCE_MAX_SIZE: int = 8192 * 1024 * 8 * 2
_CUSTOM_ALL_REDUCE: AiterCustomAllreduceProto | None = None
@classmethod
def refresh_env_variables(cls):
"""
@@ -1611,6 +1572,7 @@ class rocm_aiter_ops:
you can call this function to reload the env variables.
"""
cls._AITER_ENABLED = envs.VLLM_ROCM_USE_AITER
cls._CUSTOM_ALL_REDUCE_ENABLED = envs.VLLM_ROCM_USE_AITER_CUSTOM_AR
cls._LINEAR_ENABLED = envs.VLLM_ROCM_USE_AITER_LINEAR
cls._FMOE_ENABLED = envs.VLLM_ROCM_USE_AITER_MOE
cls._MLA_ENABLED = envs.VLLM_ROCM_USE_AITER_MLA
@@ -1770,6 +1732,11 @@ class rocm_aiter_ops:
def is_mha_enabled(cls) -> bool:
return cls._AITER_ENABLED and cls._MHA_ENABLED
@classmethod
@if_aiter_supported
def is_custom_all_reduce_enabled(cls) -> bool:
return cls._AITER_ENABLED and cls._CUSTOM_ALL_REDUCE_ENABLED
@classmethod
@if_aiter_supported
def is_shuffle_kv_cache_enabled(cls) -> bool:
@@ -1824,33 +1791,20 @@ class rocm_aiter_ops:
return cls.is_linear_enabled() and on_gfx950()
@classmethod
def initialize_aiter_allreduce(
cls, group: ProcessGroup, device: torch.device
) -> None:
try:
from aiter.dist.device_communicators.custom_all_reduce import (
CustomAllreduce as AiterCustomAllreduce,
)
def get_aiter_allreduce(cls):
"""Return the TP device communicator's AITER custom-allreduce if it has
one, return None otherwise
"""
from vllm.distributed.device_communicators.aiter_custom_all_reduce import (
AiterCustomAllreduce,
)
from vllm.distributed.parallel_state import get_tp_group
cls._CUSTOM_ALL_REDUCE = AiterCustomAllreduce(group, device)
except Exception:
cls._CUSTOM_ALL_REDUCE = None
@classmethod
def get_aiter_allreduce(cls) -> AiterCustomAllreduceProto | None:
return cls._CUSTOM_ALL_REDUCE
@classmethod
def destroy_aiter_allreduce(cls) -> None:
if cls._CUSTOM_ALL_REDUCE is not None:
cls._CUSTOM_ALL_REDUCE.close()
cls._CUSTOM_ALL_REDUCE = None
@classmethod
def get_aiter_allreduce_max_size(cls) -> int | None:
# effective max input size (based on upstream aiter version: v0.1.10.post3)
# https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/aiter/dist/device_communicators/custom_all_reduce.py#L272-L273
return int(cls._ALL_REDUCE_MAX_SIZE / 2)
device_comm = get_tp_group().device_communicator
aiter_ar_comm = getattr(device_comm, "aiter_ar_comm", None)
return (
aiter_ar_comm if isinstance(aiter_ar_comm, AiterCustomAllreduce) else None
)
@classmethod
@if_aiter_supported
@@ -2165,21 +2119,6 @@ class rocm_aiter_ops:
def get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op() -> OpOverload: # noqa: E501
return torch.ops.vllm.rocm_aiter_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm.default # noqa: E501
# TODO(frida-andersson): drop once vLLM pins AITER >= 0.1.14 (ROCm/aiter#2823).
@classmethod
def has_fused_allreduce_rmsnorm_quant_per_group(cls) -> bool:
"""True if the running AITER build exposes the per-group AR+RMS+quant
kernel (added in ROCm/aiter PR #2823).
The pattern registration in ``RocmAiterAllReduceFusionPass`` keys off
this so vLLM degrades to the AR+RMS-only fusion when run against an
older aiter that lacks the per-group launcher.
"""
aiter_ar = cls.get_aiter_allreduce()
return aiter_ar is not None and hasattr(
aiter_ar, "fused_ar_rms_per_group_quant"
)
@staticmethod
def get_fused_mla_dual_rms_norm_op() -> OpOverload:
return torch.ops.vllm.fused_mla_dual_rms_norm.default
@@ -19,7 +19,6 @@ from vllm.compilation.passes.fusion.rms_quant_fusion import (
from vllm.config import VllmConfig
from vllm.config.utils import Range
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
from vllm.distributed.device_communicators.custom_all_reduce import CustomAllreduce
from vllm.distributed.parallel_state import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
@@ -1473,39 +1472,23 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
)
return
device_comm = get_tp_group().device_communicator
if device_comm is None:
logger.warning_once("Device communicator is required.")
return
ca_comm = getattr(device_comm, "ca_comm", None)
ca_comm = rocm_aiter_ops.get_aiter_allreduce()
if ca_comm is None:
logger.warning_once("Custom Allreduce is required.")
logger.warning_once(
"AITER allreduce fusions are disabled "
"because AITER Custom All Reduce is not enabled. "
"Set VLLM_ROCM_USE_AITER_CUSTOM_AR=1 "
"to enable it."
)
return
self.ca_comm = ca_comm
assert isinstance(ca_comm, CustomAllreduce)
group = get_tp_group().cpu_group
rocm_aiter_ops.initialize_aiter_allreduce(group, self.device)
hidden_dim = config.model_config.get_hidden_size()
element_size = torch.tensor([], dtype=self.model_dtype).element_size()
max_size = rocm_aiter_ops.get_aiter_allreduce_max_size()
if max_size is None:
logger.warning("AITER allreduce fusion must be initialized")
return
# Aiter's fused_allreduce_rmsnorm kernel dispatches on hidden_dim.
# Before aiter v0.1.12 the launcher was template-specialized on HIDDEN_DIM
# and silently no-op'd for sizes outside {512, 1024, 2048, 4096}. From v0.1.12
# hidden_dim is a runtime argument. Detect the older API via the missing
# `_pool` attribute and skip fusion for unsupported sizes.
# Ref (old kernel): https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/csrc/include/custom_all_reduce.cuh#L2590
aiter_ar = rocm_aiter_ops.get_aiter_allreduce()
max_size = ca_comm.effective_max_size()
_AITER_OLD_FUSED_AR_RMS_HIDDEN = (512, 1024, 2048, 4096)
if (
aiter_ar is not None
and not hasattr(aiter_ar, "_pool")
not ca_comm.supports_dynamic_hidden_dim
and hidden_dim not in _AITER_OLD_FUSED_AR_RMS_HIDDEN
):
logger.warning_once(
@@ -1515,10 +1498,6 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
_AITER_OLD_FUSED_AR_RMS_HIDDEN,
hidden_dim,
)
# Tear down aiter's custom-allreduce so its IPC handles don't
# race with vllm's ca_comm on the unfused fallback path.
with contextlib.suppress(Exception):
rocm_aiter_ops.destroy_aiter_allreduce()
return
max_token_num = max_size // (hidden_dim * element_size)
@@ -1532,9 +1511,7 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
# fall back to the AR+RMS-only fusion paired with PR #41825's
# standalone RMS+quant fusion -- still correct, just leaves the
# post-AR quant as a standalone kernel.
supports_per_group_quant = (
rocm_aiter_ops.has_fused_allreduce_rmsnorm_quant_per_group()
)
supports_per_group_quant = ca_comm.supports_per_group_quant
if not supports_per_group_quant:
logger.warning_once(
"AITER AR+RMS+per-group-FP8-quant fusion disabled: aiter "
@@ -1609,9 +1586,3 @@ class RocmAiterAllReduceFusionPass(VllmFusionPatternMatcherPass):
logger.debug(
"%s Replaced %s patterns", self.__class__.__name__, self.matched_count
)
def __del__(self) -> None:
if getattr(self, "disabled", True):
return
with contextlib.suppress(Exception):
rocm_aiter_ops.destroy_aiter_allreduce()
+7 -2
View File
@@ -1850,8 +1850,13 @@ class VllmConfig:
tp_size = self.parallel_config.tensor_parallel_size
from vllm._aiter_ops import rocm_aiter_ops
if rocm_aiter_ops.is_enabled():
max_size = rocm_aiter_ops.get_aiter_allreduce_max_size()
max_size: int | None = None
if rocm_aiter_ops.is_custom_all_reduce_enabled():
from vllm.distributed.device_communicators.aiter_custom_all_reduce import ( # noqa: E501
AiterCustomAllreduce,
)
max_size = AiterCustomAllreduce.effective_max_size()
else:
max_size = compilation_config.pass_config.flashinfer_max_size(tp_size)
if max_size is not None and self.model_config is not None:
@@ -0,0 +1,95 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""vLLM-owned wrapper over AITER's ``CustomAllreduce``.
vLLM's ``CudaCommunicator`` stores one of these as ``aiter_ar_comm`` (when
``VLLM_ROCM_USE_AITER_CUSTOM_AR`` is set) so the plain allreduce and
the fused allreduce+RMSNorm path share a single AITER instance with its IPC buffers.
"""
import torch
from torch.distributed import ProcessGroup
from vllm.logger import init_logger
logger = init_logger(__name__)
class AiterCustomAllreduce:
# Default IPC buffer size for AITER's CustomAllreduce.
MAX_SIZE: int = 8192 * 1024 * 8 * 2
@classmethod
def effective_max_size(cls) -> int:
"""
Max input byte size eligible for AITER custom allreduce.
"""
return cls.MAX_SIZE // 2
def __init__(
self,
group: ProcessGroup,
device: int | str | torch.device,
max_size: int | None = None,
):
from aiter.dist.device_communicators.custom_all_reduce import (
CustomAllreduce as _AiterCustomAllreduce,
)
if max_size is None:
max_size = self.MAX_SIZE
self._impl = _AiterCustomAllreduce(group, device, max_size=max_size)
@property
def aiter_ca(self):
return self._impl
@property
def disabled(self) -> bool:
return self._impl.disabled
def should_custom_ar(self, inp: torch.Tensor) -> bool:
return self._impl.should_custom_ar(inp)
def custom_all_reduce(self, inp: torch.Tensor) -> torch.Tensor | None:
return self._impl.custom_all_reduce(inp)
def capture(self):
return self._impl.capture()
def close(self) -> None:
self._impl.close()
@property
def supports_dynamic_hidden_dim(self) -> bool:
"""Aiter's fused_allreduce_rmsnorm kernel dispatches on hidden_dim.
Before aiter v0.1.12 the launcher was template-specialized on HIDDEN_DIM
and silently no-op'd for sizes outside {512, 1024, 2048, 4096}. From v0.1.12
hidden_dim is a runtime argument. Older builds are detected via
AiterCustomAllreduce.supports_dynamic_hidden_dim; This function is used to
skip fusion for unsupported sizes on them.
Ref (old kernel): https://github.com/ROCm/aiter/blob/6a0e7b26ccf33164785531212cc2ec2cde0b9243/csrc/include/custom_all_reduce.cuh#L2590
"""
return hasattr(self._impl, "_pool")
@staticmethod
def build_supports_per_group_quant() -> bool:
"""True if the running AITER build exposes the per-group AR+RMS+quant
kernel (added in ROCm/aiter PR #2823).
The pattern registration in ``RocmAiterAllReduceFusionPass`` keys off
this so vLLM degrades to the AR+RMS-only fusion when run against an
older aiter that lacks the per-group launcher.
"""
from aiter.dist.device_communicators.custom_all_reduce import (
CustomAllreduce as _AiterCustomAllreduce,
)
return hasattr(_AiterCustomAllreduce, "fused_ar_rms_per_group_quant")
# TODO(frida-andersson): drop once vLLM pins AITER >= 0.1.14 (ROCm/aiter#2823).
@property
def supports_per_group_quant(self) -> bool:
return self.build_supports_per_group_quant()
@@ -6,6 +6,7 @@ import torch
from torch.distributed import ProcessGroup
import vllm.envs as envs
from vllm._aiter_ops import rocm_aiter_ops
from vllm.distributed.device_communicators.all_reduce_utils import (
NCCL_SYMM_MEM_ALL_REDUCE_CONFIG,
should_nccl_symm_mem_ag_rs,
@@ -19,6 +20,7 @@ from vllm.logger import init_logger
from vllm.platforms import current_platform
from ..utils import StatelessProcessGroup
from .aiter_custom_all_reduce import AiterCustomAllreduce
from .base_device_communicator import DeviceCommunicatorBase
logger = init_logger(__name__)
@@ -48,16 +50,21 @@ class CudaCommunicator(DeviceCommunicatorBase):
use_custom_allreduce = False
use_torch_symm_mem = False
use_flashinfer_allreduce = False
use_aiter_allreduce = False
else:
from vllm.distributed.parallel_state import _ENABLE_CUSTOM_ALL_REDUCE
use_custom_allreduce = _ENABLE_CUSTOM_ALL_REDUCE
use_torch_symm_mem = envs.VLLM_ALLREDUCE_USE_SYMM_MEM
use_flashinfer_allreduce = envs.VLLM_ALLREDUCE_USE_FLASHINFER
use_aiter_allreduce = use_custom_allreduce and bool(
rocm_aiter_ops.is_custom_all_reduce_enabled()
)
self.use_custom_allreduce = use_custom_allreduce
self.use_torch_symm_mem = use_torch_symm_mem
self.use_flashinfer_allreduce = use_flashinfer_allreduce
self.use_aiter_allreduce = use_aiter_allreduce
# lazy import to avoid documentation build error
from vllm.distributed.device_communicators.custom_all_reduce import (
@@ -85,6 +92,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.qr_comm: QuickAllReduce | None = None
self.symm_mem_comm: SymmMemCommunicator | None = None
self.fi_ar_comm: FlashInferAllReduce | None = None
self.aiter_ar_comm: AiterCustomAllreduce | None = None
if use_torch_symm_mem and current_platform.is_cuda():
self.symm_mem_comm = SymmMemCommunicator(
@@ -98,7 +106,13 @@ class CudaCommunicator(DeviceCommunicatorBase):
device=self.device,
)
if use_custom_allreduce and self.world_size > 1:
if self.use_aiter_allreduce and self.world_size > 1:
self.aiter_ar_comm = AiterCustomAllreduce(
group=self.cpu_group,
device=self.device,
)
if use_custom_allreduce and self.aiter_ar_comm is None and self.world_size > 1:
# Initialize a custom fast all-reduce implementation.
self.ca_comm = CustomAllreduce(
group=self.cpu_group,
@@ -108,13 +122,14 @@ class CudaCommunicator(DeviceCommunicatorBase):
),
)
if current_platform.is_rocm():
# Initialize a custom quick all-reduce implementation for AMD.
# Quick reduce is designed as a complement to custom allreduce.
# Based on quickreduce (https://github.com/mk1-project/quickreduce).
# If it's a rocm, 'use_custom_allreduce==True' means it must
# currently be an MI300 series.
self.qr_comm = QuickAllReduce(group=self.cpu_group, device=self.device)
if use_custom_allreduce and self.world_size > 1 and current_platform.is_rocm():
# Initialize a custom quick all-reduce implementation for AMD.
# Quick reduce is designed as a complement to custom allreduce
# (vLLM's or AITER's), so it is initialized for either backend.
# Based on quickreduce (https://github.com/mk1-project/quickreduce).
# On ROCm, 'use_custom_allreduce==True' means it must currently be
# an MI300 series.
self.qr_comm = QuickAllReduce(group=self.cpu_group, device=self.device)
if self.world_size > 1:
self._log_all_reduce_backend_selection()
@@ -203,6 +218,7 @@ class CudaCommunicator(DeviceCommunicatorBase):
"NCCL_SYMM_MEM",
"QUICK_REDUCE",
"FLASHINFER",
"AITER_CUSTOM",
"CUSTOM",
"SYMM_MEM",
"PYNCCL",
@@ -236,6 +252,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
enabled_ar_backends.append("QUICK_REDUCE")
if self.fi_ar_comm is not None and not self.fi_ar_comm.disabled:
enabled_ar_backends.append("FLASHINFER")
if self.aiter_ar_comm is not None and not self.aiter_ar_comm.disabled:
enabled_ar_backends.append("AITER_CUSTOM")
if self.ca_comm is not None and not self.ca_comm.disabled:
enabled_ar_backends.append("CUSTOM")
if self.symm_mem_comm is not None and not self.symm_mem_comm.disabled:
@@ -261,8 +279,8 @@ class CudaCommunicator(DeviceCommunicatorBase):
out = torch.ops.vllm.all_reduce_symmetric_with_copy(input_)
if out is not None:
return out
# always try quick reduce first, then flashinfer, then custom allreduce,
# and then pynccl. (quick reduce just for ROCM MI3*)
# always try quick reduce first, then flashinfer, then the AITER or vLLM
# custom allreduce, and then pynccl. (quick reduce just for ROCM MI3*)
qr_comm = self.qr_comm
if (
qr_comm is not None
@@ -281,6 +299,15 @@ class CudaCommunicator(DeviceCommunicatorBase):
out = fi_ar_comm.all_reduce(input_)
assert out is not None
return out
aiter_ar_comm = self.aiter_ar_comm
if (
aiter_ar_comm is not None
and not aiter_ar_comm.disabled
and aiter_ar_comm.should_custom_ar(input_)
):
out = aiter_ar_comm.custom_all_reduce(input_)
assert out is not None
return out
ca_comm = self.ca_comm
if (
ca_comm is not None
@@ -509,6 +536,9 @@ class CudaCommunicator(DeviceCommunicatorBase):
self.pynccl_comm = None
if self.ca_comm is not None:
self.ca_comm = None
if self.aiter_ar_comm is not None:
self.aiter_ar_comm.close()
self.aiter_ar_comm = None
if self.fi_ar_comm is not None:
self.fi_ar_comm.destroy()
self.fi_ar_comm = None
+6
View File
@@ -119,6 +119,7 @@ if TYPE_CHECKING:
VLLM_USE_OINK_OPS: bool = False
VLLM_MXFP8_EMULATION_DEQUANT_AT_LOAD: bool = True
VLLM_ROCM_USE_AITER: bool = False
VLLM_ROCM_USE_AITER_CUSTOM_AR: bool = True
VLLM_ROCM_USE_AITER_PAGED_ATTN: bool = False
VLLM_ROCM_USE_AITER_LINEAR: bool = True
VLLM_ROCM_USE_AITER_LINEAR_HIPBMM: bool = False
@@ -1146,6 +1147,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ROCM_USE_AITER": lambda: (
os.getenv("VLLM_ROCM_USE_AITER", "False").lower() in ("true", "1")
),
# Use AITER's CustomAllreduce as the custom-allreduce backend inside vLLM's
# CudaCommunicator on ROCm.
"VLLM_ROCM_USE_AITER_CUSTOM_AR": lambda: (
os.getenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "True").lower() in ("true", "1")
),
# Whether to use aiter paged attention.
# By default is disabled.
"VLLM_ROCM_USE_AITER_PAGED_ATTN": lambda: (