[Bugfix][KV Connector][ROCm] Fix MoRI-IO KV block-offset for speculative decoding (MTP) (#55053)

Signed-off-by: Mohit Deopujari <Mohit.Deopujari@amd.com>
Signed-off-by: MohitAMD <mohit.deopujari@amd.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
MohitAMD
2026-10-07 11:33:30 -05:00
committed by GitHub
co-authored by Cursor
parent c741bfca70
commit 240ee8de9e
4 changed files with 452 additions and 57 deletions
@@ -581,12 +581,31 @@ class MoRIIOConnectorScheduler:
else (0, self.block_size)
for g in kv_cache_config.transfer_groups
]
# add 1 to conservatively account for boundary overlap eg window isn't fully
# aligned with blocks.
# +1 conservatively covers window/block boundary overlap.
self.blocks_per_sw = [
cdiv(n_tokens, block_size) + 1 if n_tokens else 0
for n_tokens, block_size in sw_sizes_tokens
]
# Per-group block size over transfer_groups (matching blocks_per_sw and
# the connector's block payload), used to clamp each full-attention group
# to its prompt length on the WRITE save path.
self._group_block_sizes = [
getattr(g.kv_cache_spec, "block_size", self.block_size)
for g in kv_cache_config.transfer_groups
]
# The WRITE-mode prompt-block clamp only covers full-attention groups, so
# MoRIIO + speculative decoding is not yet safe on sliding-window /
# hybrid models. TODO: lift once SWA + spec is validated end-to-end.
if (
self.mode == MoRIIOMode.WRITE
and vllm_config.speculative_config is not None
and any(bps != 0 for bps in self.blocks_per_sw)
):
raise NotImplementedError(
"MoRIIO with speculative decoding is not yet supported for "
"sliding-window / hybrid models: the prompt-block clamp is only "
"validated for full-attention KV groups."
)
# In WRITE mode and chunked prefill, we perform the write after the last chunk.
# Hence we need to track once we are in the last chunk. We do this by computing
# len(block_ids[g]) * block_size[g] for a full attn group g, as hybrid groups
@@ -1178,6 +1197,45 @@ class MoRIIOConnectorScheduler:
params["do_remote_prefill"] = False
def _clamp_to_prompt_blocks(self, req: "Request", block_ids: BlockIds) -> BlockIds:
"""Drop trailing speculative-lookahead blocks the consumer never
allocated, keeping each full-attention group's prompt blocks.
Args:
req: Request being saved.
block_ids: Per-KV-cache-group local block ids.
Returns:
``block_ids`` with every full-attention group truncated to
``ceil(num_prompt_tokens / block_size)`` leading blocks;
sliding-window groups are left unchanged.
"""
clamped = list(block_ids)
changed = False
for g, group_blocks in enumerate(block_ids):
if self.blocks_per_sw[g] != 0:
continue
num_prompt_blocks = math.ceil(
req.num_prompt_tokens / self._group_block_sizes[g]
)
if len(group_blocks) > num_prompt_blocks:
clamped[g] = group_blocks[:num_prompt_blocks]
changed = True
return clamped if changed else block_ids
def _is_final_prefill_chunk(
self, req: "Request", scheduler_output: SchedulerOutput
) -> bool:
"""Whether this step completes the prompt.
``build_connector_meta`` runs before ``_update_after_schedule``, so
``num_computed_tokens`` excludes the current chunk; add this step's
scheduled tokens. Token progress is lookahead-safe unlike block counts.
"""
num_scheduled = scheduler_output.num_scheduled_tokens.get(req.request_id, 0)
return req.num_computed_tokens + num_scheduled >= req.num_prompt_tokens
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
@@ -1191,38 +1249,36 @@ class MoRIIOConnectorScheduler:
# It places the request metadata into the saving queue.
for i, req_id in enumerate(scheduler_output.scheduled_cached_reqs.req_ids):
# Non-disagg requests aren't registered here; skip them
# instead of KeyError-ing.
if req_id not in self._reqs_need_pending_save:
continue
req, existing_blocks = self._reqs_need_pending_save[req_id]
# The final chunk may allocate no new blocks (the previous chunk
# reserved speculative-lookahead slack), so new_block_ids is None;
# still fall through to the final-chunk check below.
new_block_ids = scheduler_output.scheduled_cached_reqs.new_block_ids[i]
if new_block_ids is not None:
# A non-disagg request (no kv_transfer_params, e.g. smoke
# test) is never registered in _reqs_need_pending_save;
# indexing it unconditionally would KeyError and crash the
# EngineCore. Skip it silently.
if req_id not in self._reqs_need_pending_save:
continue
req, existing_blocks = self._reqs_need_pending_save[req_id]
updated_blocks = [
existing_blocks = [
existing_blocks[g] + new_block_ids[g]
for g in range(len(new_block_ids))
]
self._reqs_need_pending_save[req_id] = (req, updated_blocks)
saved_tokens = (
len(updated_blocks[self._full_attn_group_idx])
* self._full_attn_block_size
)
if saved_tokens >= req.num_prompt_tokens:
# Final chunk: live kv_transfer_params may be cleared,
# so prefer the snapshot from update_state_after_alloc.
kv_params = self._req_kv_params.pop(
req_id, req.kv_transfer_params or {}
)
meta.add_new_req(
request_id=req_id,
local_block_ids=self._reqs_need_pending_save[req_id][1],
kv_transfer_params=kv_params,
write_mode=True,
)
del self._reqs_need_pending_save[req_id]
self._reqs_need_pending_save[req_id] = (req, existing_blocks)
if not self._is_final_prefill_chunk(req, scheduler_output):
continue
# Live kv_transfer_params may be cleared; prefer the
# update_state_after_alloc snapshot.
kv_params = self._req_kv_params.pop(
req_id, req.kv_transfer_params or {}
)
save_block_ids = self._clamp_to_prompt_blocks(req, existing_blocks)
meta.add_new_req(
request_id=req_id,
local_block_ids=save_block_ids,
kv_transfer_params=kv_params,
write_mode=True,
)
del self._reqs_need_pending_save[req_id]
# Loop through scheduled reqs and convert to ReqMeta.
for req_id, (req, block_ids) in self._reqs_need_recv.items():
@@ -1253,16 +1309,13 @@ class MoRIIOConnectorScheduler:
for req_id, (req, block_ids) in self._reqs_need_save.items():
kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {})
tokens_covered = (
len(block_ids[self._full_attn_group_idx]) * self._full_attn_block_size
)
if req.num_prompt_tokens > tokens_covered:
# not last chunk prefill
if not self._is_final_prefill_chunk(req, scheduler_output):
self._reqs_need_pending_save[req_id] = (req, block_ids)
continue
save_block_ids = self._clamp_to_prompt_blocks(req, block_ids)
meta.add_new_req(
request_id=req_id,
local_block_ids=block_ids,
local_block_ids=save_block_ids,
kv_transfer_params=kv_params,
write_mode=True,
)
@@ -377,10 +377,8 @@ def iter_layer_registration_regions(
)
region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len
if geometry.regions_per_block == 1:
# With padded or interleaved pages the block stride exceeds the
# meaningful block_len; register the strided span. The span ends with
# the last block's meaningful bytes, not a whole stride, so it never
# runs past the backing allocation of a strided layer view.
# Padded/interleaved pages stride past block_len: register the strided
# span, ending at the last block's bytes so it stays in-allocation.
block_stride_bytes = geometry.block_stride * kv_cache.element_size()
region_len = max(
region_len,
@@ -431,12 +429,8 @@ def compute_block_transfer_offsets(
[list[int], list[int], list[int]], tuple[list[int], list[int], list[int]]
] = merge_contiguous_offsets,
) -> tuple[list[int], list[int], list[int]]:
# A shorter (or empty) local list is the READ-mode "drop the transfer, just
# free the prefill blocks" case (full-prefix-hit / aborted-before-scheduled):
# decode pulls fewer blocks than the prefill holds. The zip loop below pairs
# local[i]<->remote[i] and sizes by len(local), so a short local transfers
# only what decode allocated and an empty local is a no-op. A longer local
# list is a genuine bug and still fails loudly.
# A shorter/empty local list transfers only what decode allocated (READ-mode
# partial or full-prefix-hit); a longer local list is a genuine bug.
if len(local_block_ids) > len(remote_block_ids):
raise ValueError(
"local_block_ids longer than remote_block_ids: "