[BugFix] Make PD work with Ray (#21072)
Signed-off-by: Kourosh Hakhamaneshi <kourosh@anyscale.com>
This commit is contained in:
@@ -15,9 +15,7 @@ from vllm.config import VllmConfig
|
||||
from vllm.distributed import (ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
set_custom_all_reduce)
|
||||
from vllm.distributed.kv_transfer import (ensure_kv_transfer_initialized,
|
||||
get_kv_transfer_group,
|
||||
has_kv_transfer_group)
|
||||
from vllm.distributed.kv_transfer import ensure_kv_transfer_initialized
|
||||
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -335,25 +333,17 @@ class Worker(WorkerBase):
|
||||
assert isinstance(output, IntermediateTensors)
|
||||
get_pp_group().send_tensor_dict(output.tensors,
|
||||
all_gather_group=get_tp_group())
|
||||
output = EMPTY_MODEL_RUNNER_OUTPUT
|
||||
|
||||
# In case of PP with kv transfer, we need to pass through the
|
||||
# finished_sending and finished_recving buffers.
|
||||
empty_output = EMPTY_MODEL_RUNNER_OUTPUT
|
||||
if output.finished_sending or output.finished_recving:
|
||||
empty_output = copy.copy(empty_output)
|
||||
empty_output.finished_sending = output.finished_sending
|
||||
empty_output.finished_recving = output.finished_recving
|
||||
output = empty_output
|
||||
|
||||
assert isinstance(output, ModelRunnerOutput)
|
||||
if has_kv_transfer_group():
|
||||
finished_sending, finished_recving = (
|
||||
get_kv_transfer_group().get_finished(
|
||||
scheduler_output.finished_req_ids))
|
||||
if finished_sending or finished_recving:
|
||||
if output is EMPTY_MODEL_RUNNER_OUTPUT:
|
||||
output = copy.copy(EMPTY_MODEL_RUNNER_OUTPUT)
|
||||
output.finished_sending = finished_sending
|
||||
output.finished_recving = finished_recving
|
||||
|
||||
# Clear KVConnector state for this step.
|
||||
get_kv_transfer_group().clear_connector_metadata()
|
||||
|
||||
# with a connector, the scheduler expects output from all workers
|
||||
return output
|
||||
|
||||
# return output only from the driver worker
|
||||
return output if self.is_driver_worker else None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user