[feature] extend DBO to XBO (#30120)
Signed-off-by: jiangkuaixue123 <jiangxiaozhou111@163.com> Co-authored-by: root <root@hk01dgx028.cm.cluster>
This commit is contained in:
@@ -7,10 +7,15 @@ import torch
|
||||
|
||||
from vllm import forward_context
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_THREAD_ID_TO_CONTEXT: dict = {}
|
||||
_CURRENT_CONTEXTS: list[Optional["UBatchContext"]] = [None, None]
|
||||
# Here we hardcode the number of microbatches to 2 for default.
|
||||
_NUM_UBATCHES: int = 2
|
||||
_CURRENT_CONTEXTS: list[Optional["UBatchContext"]] = []
|
||||
|
||||
|
||||
class UBatchContext:
|
||||
@@ -48,6 +53,7 @@ class UBatchContext:
|
||||
global _CURRENT_CONTEXTS, _THREAD_ID_TO_CONTEXT
|
||||
_THREAD_ID_TO_CONTEXT[threading.get_ident()] = self.id
|
||||
_CURRENT_CONTEXTS[self.id] = self
|
||||
# _NUM_UBATCHES is set in make_ubatch_contexts
|
||||
self.ready_barrier.wait()
|
||||
|
||||
self.cpu_wait_event.wait()
|
||||
@@ -181,7 +187,7 @@ dbo_switch_to_compute_sync = _register_ubatch_function(
|
||||
def dbo_register_recv_hook(recv_hook):
|
||||
if len(_THREAD_ID_TO_CONTEXT) > 0:
|
||||
ctx_idx = _THREAD_ID_TO_CONTEXT[threading.get_ident()]
|
||||
next_ctx = _CURRENT_CONTEXTS[(ctx_idx + 1) % 2]
|
||||
next_ctx = _CURRENT_CONTEXTS[(ctx_idx + 1) % _NUM_UBATCHES]
|
||||
next_ctx.recv_hook = recv_hook
|
||||
|
||||
|
||||
@@ -202,7 +208,14 @@ def make_ubatch_contexts(
|
||||
ready_barrier: threading.Barrier,
|
||||
schedule: str = "default",
|
||||
) -> list[UBatchContext]:
|
||||
assert num_micro_batches == 2, "only been tested with 2 micro-batches"
|
||||
global _NUM_UBATCHES, _CURRENT_CONTEXTS
|
||||
assert num_micro_batches > 1, "num_micro_batches must be greater than 1"
|
||||
|
||||
_NUM_UBATCHES = num_micro_batches
|
||||
# Ensure the global context list is large enough
|
||||
if len(_CURRENT_CONTEXTS) < num_micro_batches:
|
||||
_CURRENT_CONTEXTS.extend([None] * (num_micro_batches - len(_CURRENT_CONTEXTS)))
|
||||
|
||||
"""
|
||||
Create a context manager for micro-batching synchronization.
|
||||
"""
|
||||
@@ -210,8 +223,6 @@ def make_ubatch_contexts(
|
||||
gpu_comm_done_events = [torch.Event() for _ in range(num_micro_batches)]
|
||||
gpu_compute_done_events = [torch.Event() for _ in range(num_micro_batches)]
|
||||
|
||||
assert len(forward_contexts) == 2
|
||||
|
||||
ctxs = []
|
||||
for i in range(num_micro_batches):
|
||||
ctx = UBatchContext(
|
||||
|
||||
Reference in New Issue
Block a user