[Bugfix][NIXL] Fix Mamba prefill truncation ordering (#53523)
Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Roger Wang
parent
cbe3966f9c
commit
d4c4ceb31e
@@ -194,6 +194,9 @@ class NixlBaseConnectorScheduler:
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
"""Track a request that may need heartbeats."""
|
||||
params = request.kv_transfer_params
|
||||
if params is not None and params.get("do_remote_decode") and self._has_mamba:
|
||||
self._truncate_mamba_request_for_prefill(request)
|
||||
|
||||
# NOTE (NickLucche) This excludes request meant for P, ie heartbeats are
|
||||
# effectively disabled for Bidirectional KV transfer.
|
||||
if params is None or not params.get("do_remote_prefill"):
|
||||
|
||||
@@ -65,9 +65,6 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
if count > 0:
|
||||
return count, True
|
||||
|
||||
if params is not None and params.get("do_remote_decode") and self._has_mamba:
|
||||
self._truncate_mamba_request_for_prefill(request)
|
||||
|
||||
if (
|
||||
params is not None
|
||||
and params.get("do_remote_decode")
|
||||
|
||||
@@ -123,9 +123,6 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
|
||||
if count > 0:
|
||||
return count, True
|
||||
|
||||
if params is not None and params.get("do_remote_decode") and self._has_mamba:
|
||||
self._truncate_mamba_request_for_prefill(request)
|
||||
|
||||
return 0, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
|
||||
Reference in New Issue
Block a user