[BugFix] Fix handling of num_computed_tokens with connector (#18232)

Signed-off-by: Nick Hill <nhill@redhat.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@gmail.com>
This commit is contained in:
Nick Hill
2025-05-19 09:03:25 -07:00
committed by GitHub
co-authored by Nicolò Lucchesi
parent 43b5f61dce
commit 1b15df2546
2 changed files with 29 additions and 16 deletions
@@ -209,7 +209,17 @@ class NixlConnectorScheduler:
rounded_num_prompt_tokens = round_down(
len(request.prompt_token_ids), self.block_size)
count = max(rounded_num_prompt_tokens - num_computed_tokens, 0)
return count, count > 0
if count > 0:
return count, True
# NOTE: if count is 0 here, we have less than block_size
# tokens to pull after subtracting the local prefix cache hit.
# The remote only sends fully computed blocks, so there is
# nothing to transfer but we still need to notify the
# prefill worker so that the remote blocks are freed.
if all(p in params for p in ("remote_engine_id", "remote_host",
"remote_port")):
self._reqs_need_recv[request.request_id] = (request, [])
# No remote prefill for this request.
return 0, False
@@ -225,10 +235,6 @@ class NixlConnectorScheduler:
num_external_tokens, params)
if params is not None and params.get("do_remote_prefill"):
# NOTE(rob): if prompt < block_size, no remote blocks
# since the remote only sends fully computed blocks, so
# skip recving for this request. num_external_tokens
# should be 0 if there are no remote blocks.
if params.get("remote_block_ids"):
if all(p in params for p in ("remote_engine_id", "remote_host",
"remote_port")):