[PP][XPU]Add the flag to control microbatch feature on MRV2+PP (#55145)
Signed-off-by: yisheng <yi.sheng@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -63,6 +63,7 @@ if TYPE_CHECKING:
|
||||
VLLM_USE_RAY_COMPILED_DAG_CHANNEL_TYPE: Literal["auto", "nccl", "shm"] = "auto"
|
||||
VLLM_USE_RAY_COMPILED_DAG_OVERLAP_COMM: bool = False
|
||||
VLLM_USE_RAY_WRAPPED_PP_COMM: bool = True
|
||||
VLLM_XPU_PP_MICROBATCH: bool = True
|
||||
VLLM_USE_RAY_V2_EXECUTOR_BACKEND: bool = False
|
||||
VLLM_DISTRIBUTED_USE_SPLIT_GROUP: bool = False
|
||||
VLLM_XLA_USE_SPMD: bool = False
|
||||
@@ -924,6 +925,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_USE_RAY_WRAPPED_PP_COMM": lambda: bool(
|
||||
int(os.getenv("VLLM_USE_RAY_WRAPPED_PP_COMM", "1"))
|
||||
),
|
||||
# Using a flag to control microbatch on XPU device, users on XPU device can
|
||||
# decide to disable it when they needed.
|
||||
"VLLM_XPU_PP_MICROBATCH": lambda: bool(
|
||||
int(os.getenv("VLLM_XPU_PP_MICROBATCH", "1"))
|
||||
),
|
||||
# When True and distributed_executor_backend="ray", use RayExecutorV2
|
||||
# (MQ-based) instead of RayDistributedExecutor (compiled-graph backend).
|
||||
"VLLM_USE_RAY_V2_EXECUTOR_BACKEND": lambda: bool(
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
@@ -15,6 +17,10 @@ class AsyncScheduler(Scheduler):
|
||||
# reusable read-only placeholder list for speculative decoding.
|
||||
self._spec_token_placeholders: list[int] = [-1] * self.num_spec_tokens
|
||||
self.pp_size = self.parallel_config.pipeline_parallel_size
|
||||
# Only XPU can disable PP decode microbatching via VLLM_XPU_PP_MICROBATCH.
|
||||
self.decode_stagger = self.pp_size
|
||||
if current_platform.is_xpu() and not envs.VLLM_XPU_PP_MICROBATCH:
|
||||
self.decode_stagger = 1
|
||||
|
||||
def _update_after_schedule(self, scheduler_output: SchedulerOutput) -> None:
|
||||
super()._update_after_schedule(scheduler_output)
|
||||
@@ -46,7 +52,9 @@ class AsyncScheduler(Scheduler):
|
||||
if self.use_v2_model_runner:
|
||||
# Set the next step index in which this request is eligible to be
|
||||
# scheduled for decode (for PP microbatching).
|
||||
request.next_decode_eligible_step = self.current_step + self.pp_size
|
||||
request.next_decode_eligible_step = (
|
||||
self.current_step + self.decode_stagger
|
||||
)
|
||||
|
||||
def _update_request_with_output(
|
||||
self, request: Request, new_token_ids: list[int], is_stale: bool = False
|
||||
|
||||
@@ -8,6 +8,7 @@ from dataclasses import dataclass
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -71,8 +72,12 @@ class PPHandler:
|
||||
# pushed by step T's `receive` is consumed pp_size steps later. Pre-seeded
|
||||
# with pp_size None placeholders so the first pp_size consumes are no-ops.
|
||||
# None means no postprocess is pending for that step (broadcast skipped).
|
||||
# Only XPU can disable microbatching via VLLM_XPU_PP_MICROBATCH.
|
||||
ring_depth = get_pp_group().world_size
|
||||
if current_platform.is_xpu() and not envs.VLLM_XPU_PP_MICROBATCH:
|
||||
ring_depth = 1
|
||||
self.queue: deque[PendingRecv | None] = (
|
||||
deque() if self.is_last_rank else deque([None] * get_pp_group().world_size)
|
||||
deque() if self.is_last_rank else deque([None] * ring_depth)
|
||||
)
|
||||
|
||||
# Per req-index generation counter, incremented every time a request
|
||||
|
||||
Reference in New Issue
Block a user