[Bugfix][NIXL] Fix Mamba prefill truncation ordering (#53523)

Signed-off-by: Ziming Huang <zelda.huanghuang@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Ziming Huang
2026-08-24 19:13:27 -07:00
committed by GitHub
co-authored by Roger Wang
parent cbe3966f9c
commit d4c4ceb31e
5 changed files with 33 additions and 15 deletions
@@ -194,6 +194,9 @@ class NixlBaseConnectorScheduler:
def on_new_request(self, request: "Request") -> None:
"""Track a request that may need heartbeats."""
params = request.kv_transfer_params
if params is not None and params.get("do_remote_decode") and self._has_mamba:
self._truncate_mamba_request_for_prefill(request)
# NOTE (NickLucche) This excludes request meant for P, ie heartbeats are
# effectively disabled for Bidirectional KV transfer.
if params is None or not params.get("do_remote_prefill"):
@@ -65,9 +65,6 @@ class NixlPullConnectorScheduler(NixlBaseConnectorScheduler):
if count > 0:
return count, True
if params is not None and params.get("do_remote_decode") and self._has_mamba:
self._truncate_mamba_request_for_prefill(request)
if (
params is not None
and params.get("do_remote_decode")
@@ -123,9 +123,6 @@ class NixlPushConnectorScheduler(NixlBaseConnectorScheduler):
if count > 0:
return count, True
if params is not None and params.get("do_remote_decode") and self._has_mamba:
self._truncate_mamba_request_for_prefill(request)
return 0, False
def update_state_after_alloc(