fix: NIXL connector transfers partial block to pass full multi-modal context (#21074)

Signed-off-by: GuanLuo <gluo@nvidia.com>
This commit is contained in:
GuanLuo
2025-08-11 09:40:55 -07:00
committed by GitHub
parent f7dcce7a4a
commit 16fb668b61
4 changed files with 130 additions and 41 deletions
@@ -29,7 +29,7 @@ from vllm.distributed.utils import divide
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from vllm.platforms import _Backend, current_platform
from vllm.utils import make_zmq_path, make_zmq_socket, round_down
from vllm.utils import make_zmq_path, make_zmq_socket
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.request import RequestStatus
@@ -275,10 +275,7 @@ class NixlConnectorScheduler:
if params is not None and params.get("do_remote_prefill"):
# Remote prefill: get all prompt blocks from remote.
assert num_computed_tokens % self.block_size == 0
rounded_num_prompt_tokens = round_down(
len(request.prompt_token_ids), self.block_size)
count = max(rounded_num_prompt_tokens - num_computed_tokens, 0)
count = len(request.prompt_token_ids) - num_computed_tokens
if count > 0:
return count, True
@@ -301,18 +298,16 @@ class NixlConnectorScheduler:
# NOTE: when accelerator is not directly supported by Nixl,
# prefilled blocks need to be saved to host memory before transfer.
# figure out full computed blocks to save
# save all blocks
block_ids = blocks.get_block_ids()[0]
all_full = request.num_tokens % self.block_size == 0
full_block_ids = (block_ids if all_full else block_ids[:-1])
# TODO: skip the blocks that are already in the host xfer buffer.
# Currently, the host xfer buffer block is 1-to-1 mapped to device
# kv blocks, so host blocks won't be flushed as long as its device
# block is not overwritten; and it will be safe to skip saving them
# to host xfer buffer.
if full_block_ids:
if block_ids:
self._reqs_need_save[request.request_id] = \
(request, full_block_ids)
(request, block_ids)
elif params.get("do_remote_prefill"):
if params.get("remote_block_ids"):
if all(p in params for p in ("remote_engine_id", "remote_host",
@@ -401,12 +396,9 @@ class NixlConnectorScheduler:
or request.status != RequestStatus.FINISHED_LENGTH_CAPPED):
return False, None
# Get computed blocks.
all_full = request.num_computed_tokens % self.block_size == 0
computed_block_ids = block_ids if all_full else block_ids[:-1]
# If prompt < block_size, no xfer so free blocks immediately.
delay_free_blocks = len(computed_block_ids) > 0
# TODO: check whether block_ids actually ever be 0. If not we could
# remove the conditional below
delay_free_blocks = len(block_ids) > 0
if delay_free_blocks:
# Prefill request on remote. It will be read from D upon completion
@@ -416,7 +408,7 @@ class NixlConnectorScheduler:
return delay_free_blocks, dict(
do_remote_prefill=True,
do_remote_decode=False,
remote_block_ids=computed_block_ids,
remote_block_ids=block_ids,
remote_engine_id=self.engine_id,
remote_host=self.side_channel_host,
remote_port=self.side_channel_port,