[BugFix] Fix handling of num_computed_tokens with connector (#18232)
Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@gmail.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
43b5f61dce
commit
1b15df2546
@@ -209,7 +209,17 @@ class NixlConnectorScheduler:
|
||||
rounded_num_prompt_tokens = round_down(
|
||||
len(request.prompt_token_ids), self.block_size)
|
||||
count = max(rounded_num_prompt_tokens - num_computed_tokens, 0)
|
||||
return count, count > 0
|
||||
if count > 0:
|
||||
return count, True
|
||||
|
||||
# NOTE: if count is 0 here, we have less than block_size
|
||||
# tokens to pull after subtracting the local prefix cache hit.
|
||||
# The remote only sends fully computed blocks, so there is
|
||||
# nothing to transfer but we still need to notify the
|
||||
# prefill worker so that the remote blocks are freed.
|
||||
if all(p in params for p in ("remote_engine_id", "remote_host",
|
||||
"remote_port")):
|
||||
self._reqs_need_recv[request.request_id] = (request, [])
|
||||
|
||||
# No remote prefill for this request.
|
||||
return 0, False
|
||||
@@ -225,10 +235,6 @@ class NixlConnectorScheduler:
|
||||
num_external_tokens, params)
|
||||
|
||||
if params is not None and params.get("do_remote_prefill"):
|
||||
# NOTE(rob): if prompt < block_size, no remote blocks
|
||||
# since the remote only sends fully computed blocks, so
|
||||
# skip recving for this request. num_external_tokens
|
||||
# should be 0 if there are no remote blocks.
|
||||
if params.get("remote_block_ids"):
|
||||
if all(p in params for p in ("remote_engine_id", "remote_host",
|
||||
"remote_port")):
|
||||
|
||||
Reference in New Issue
Block a user