[EPLB] Make async EPLB default (#43219)

Signed-off-by: Markov Ilya <markovilya19@gmail.com>
Co-authored-by: Markov Ilya <markovilya19@gmail.com>
Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
Ilya Markov
2026-05-29 18:07:16 +00:00
committed by GitHub
co-authored by Markov Ilya Tyler Michael Smith
parent 84b2a8a7e7
commit 4aaba00f92
7 changed files with 15 additions and 15 deletions
@@ -49,6 +49,7 @@ for BACK in "${BACKENDS[@]}"; do
--data-parallel-size 2 \ --data-parallel-size 2 \
--enable-expert-parallel \ --enable-expert-parallel \
--enable-eplb \ --enable-eplb \
--eplb-config '{"use_async": false}' \
--trust-remote-code \ --trust-remote-code \
--max-model-len 2048 \ --max-model-len 2048 \
--all2all-backend "$BACK" \ --all2all-backend "$BACK" \
@@ -48,7 +48,7 @@ for BACK in "${BACKENDS[@]}"; do
--enforce-eager \ --enforce-eager \
--enable-eplb \ --enable-eplb \
--all2all-backend "$BACK" \ --all2all-backend "$BACK" \
--eplb-config '{"window_size":10, "step_interval":100, "num_redundant_experts":0, "log_balancedness":true}' \ --eplb-config '{"window_size":10, "step_interval":100, "num_redundant_experts":0, "log_balancedness":true, "use_async":false}' \
--tensor-parallel-size "${TENSOR_PARALLEL_SIZE}" \ --tensor-parallel-size "${TENSOR_PARALLEL_SIZE}" \
--data-parallel-size "${DATA_PARALLEL_SIZE}" \ --data-parallel-size "${DATA_PARALLEL_SIZE}" \
--enable-expert-parallel \ --enable-expert-parallel \
+3 -3
View File
@@ -1484,7 +1484,7 @@ steps:
commands: commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
- label: DeepSeek V2-Lite Accuracy (4xH100-4xMI300) # TBD - label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100-4xMI300) # TBD
timeout_in_minutes: 180 timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
agent_pool: mi300_4 agent_pool: mi300_4
@@ -1526,7 +1526,7 @@ steps:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn - export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4 - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
- label: Qwen3-30B-A3B-FP8-block Accuracy (4xH100-4xMI300) # TBD - label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100-4xMI300) # TBD
timeout_in_minutes: 180 timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
agent_pool: mi300_4 agent_pool: mi300_4
@@ -2895,7 +2895,7 @@ steps:
commands: commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt
- label: Qwen3-30B-A3B-FP8-block Accuracy (B200-MI355) # TBD - label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (B200-MI355) # TBD
timeout_in_minutes: 180 timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
agent_pool: mi355_2 agent_pool: mi355_2
+6 -6
View File
@@ -2,8 +2,8 @@ group: E2E Integration
depends_on: depends_on:
- image-build - image-build
steps: steps:
- label: DeepSeek V2-Lite Accuracy - label: DeepSeek V2-Lite Sync EPLB Accuracy
key: deepseek-v2-lite-accuracy key: deepseek-v2-lite-sync-eplb-accuracy
timeout_in_minutes: 60 timeout_in_minutes: 60
device: h100 device: h100
optional: true optional: true
@@ -12,8 +12,8 @@ steps:
commands: commands:
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010 - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
- label: Qwen3-30B-A3B-FP8-block Accuracy - label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy
key: qwen3-30b-a3b-fp8-block-accuracy key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy
timeout_in_minutes: 60 timeout_in_minutes: 60
device: h100 device: h100
optional: true optional: true
@@ -22,8 +22,8 @@ steps:
commands: commands:
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
- label: Qwen3-30B-A3B-FP8-block Accuracy (B200) - label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (B200)
key: qwen3-30b-a3b-fp8-block-accuracy-b200 key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-b200
timeout_in_minutes: 60 timeout_in_minutes: 60
device: b200-k8s device: b200-k8s
optional: true optional: true
+1 -1
View File
@@ -151,7 +151,7 @@ Configure EPLB with the `--eplb-config` argument, which accepts a JSON string. T
| `step_interval` | Frequency of rebalancing (every N engine steps) | 3000 | | `step_interval` | Frequency of rebalancing (every N engine steps) | 3000 |
| `log_balancedness` | Log balancedness metrics (avg tokens per expert ÷ max tokens per expert) | `false` | | `log_balancedness` | Log balancedness metrics (avg tokens per expert ÷ max tokens per expert) | `false` |
| `num_redundant_experts` | Additional global experts per EP rank beyond equal distribution | `0` | | `num_redundant_experts` | Additional global experts per EP rank beyond equal distribution | `0` |
| `use_async` | Use non-blocking EPLB for reduced latency overhead | `false` | | `use_async` | Use non-blocking EPLB for reduced latency overhead | `true` |
| `policy` | The policy type for expert parallel load balancing | `"default"` | | `policy` | The policy type for expert parallel load balancing | `"default"` |
| `communicator` | Backend for expert weight transfers: `"torch_nccl"`, `"torch_gloo"`, `"pynccl"`, `"nixl"`, or `null` (auto) | `null` | | `communicator` | Backend for expert weight transfers: `"torch_nccl"`, `"torch_gloo"`, `"pynccl"`, `"nixl"`, or `null` (auto) | `null` |
+2 -3
View File
@@ -15,7 +15,7 @@ def get_model_args(
spec_method: str, spec_method: str,
tp_size: int, tp_size: int,
model_max_len: int, model_max_len: int,
use_async: bool = False, use_async: bool = True,
) -> dict: ) -> dict:
speculative_config = { speculative_config = {
"method": spec_method, "method": spec_method,
@@ -28,9 +28,8 @@ def get_model_args(
"window_size": 128, "window_size": 128,
"step_interval": 1024, "step_interval": 1024,
"log_balancedness": False, "log_balancedness": False,
"use_async": use_async,
} }
if use_async:
eplb_config["use_async"] = True
model_args = { model_args = {
"pretrained": model_name, "pretrained": model_name,
"dtype": "auto", "dtype": "auto",
+1 -1
View File
@@ -78,7 +78,7 @@ class EPLBConfig:
""" """
Interval for logging the balancedness. Interval for logging the balancedness.
""" """
use_async: bool = False use_async: bool = True
""" """
Whether to use non-blocking EPLB. Whether to use non-blocking EPLB.
""" """