[Core] Disable fuse_allreduce_rms under VLLM_BATCH_INVARIANT (non-deterministic under TP) (#51292)
Signed-off-by: Don Tolley <tolleybot@gmail.com> Signed-off-by: tolleybot <tolleybot@gmail.com> Co-authored-by: yewentao256 <yewentao256@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
yewentao256
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
0e30bd62fe
commit
80771bbbdd
@@ -183,6 +183,10 @@ def enable_allreduce_rms_fusion(cfg: "VllmConfig") -> bool:
|
|||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
from vllm.utils.flashinfer import has_flashinfer
|
from vllm.utils.flashinfer import has_flashinfer
|
||||||
|
|
||||||
|
# The fused all-reduce + RMSNorm path is not batch-invariant
|
||||||
|
if envs.VLLM_BATCH_INVARIANT:
|
||||||
|
return False
|
||||||
|
|
||||||
if current_platform.is_rocm():
|
if current_platform.is_rocm():
|
||||||
from vllm._aiter_ops import rocm_aiter_ops
|
from vllm._aiter_ops import rocm_aiter_ops
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user