[Distributed] Enable custom all-reduce under VLLM_BATCH_INVARIANT (#58623)
Signed-off-by: Flora Feng <4florafeng@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com>
This commit is contained in:
@@ -1139,6 +1139,8 @@ def enable_batch_invariant_mode():
|
||||
|
||||
def override_envs_for_invariance():
|
||||
os.environ["VLLM_ALLREDUCE_USE_SYMM_MEM"] = "0"
|
||||
# Only the 1-stage kernel has a size- and rank-independent reduction order.
|
||||
os.environ["VLLM_CUSTOM_ALLREDUCE_ALGO"] = "1stage"
|
||||
|
||||
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user