[Bugfix][KV Connector][ROCm] Fix MoRI-IO KV block-offset for speculative decoding (MTP) (#55053)
Signed-off-by: Mohit Deopujari <Mohit.Deopujari@amd.com> Signed-off-by: MohitAMD <mohit.deopujari@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -581,12 +581,31 @@ class MoRIIOConnectorScheduler:
|
||||
else (0, self.block_size)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
]
|
||||
# add 1 to conservatively account for boundary overlap eg window isn't fully
|
||||
# aligned with blocks.
|
||||
# +1 conservatively covers window/block boundary overlap.
|
||||
self.blocks_per_sw = [
|
||||
cdiv(n_tokens, block_size) + 1 if n_tokens else 0
|
||||
for n_tokens, block_size in sw_sizes_tokens
|
||||
]
|
||||
# Per-group block size over transfer_groups (matching blocks_per_sw and
|
||||
# the connector's block payload), used to clamp each full-attention group
|
||||
# to its prompt length on the WRITE save path.
|
||||
self._group_block_sizes = [
|
||||
getattr(g.kv_cache_spec, "block_size", self.block_size)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
]
|
||||
# The WRITE-mode prompt-block clamp only covers full-attention groups, so
|
||||
# MoRIIO + speculative decoding is not yet safe on sliding-window /
|
||||
# hybrid models. TODO: lift once SWA + spec is validated end-to-end.
|
||||
if (
|
||||
self.mode == MoRIIOMode.WRITE
|
||||
and vllm_config.speculative_config is not None
|
||||
and any(bps != 0 for bps in self.blocks_per_sw)
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"MoRIIO with speculative decoding is not yet supported for "
|
||||
"sliding-window / hybrid models: the prompt-block clamp is only "
|
||||
"validated for full-attention KV groups."
|
||||
)
|
||||
# In WRITE mode and chunked prefill, we perform the write after the last chunk.
|
||||
# Hence we need to track once we are in the last chunk. We do this by computing
|
||||
# len(block_ids[g]) * block_size[g] for a full attn group g, as hybrid groups
|
||||
@@ -1178,6 +1197,45 @@ class MoRIIOConnectorScheduler:
|
||||
|
||||
params["do_remote_prefill"] = False
|
||||
|
||||
def _clamp_to_prompt_blocks(self, req: "Request", block_ids: BlockIds) -> BlockIds:
|
||||
"""Drop trailing speculative-lookahead blocks the consumer never
|
||||
allocated, keeping each full-attention group's prompt blocks.
|
||||
|
||||
Args:
|
||||
req: Request being saved.
|
||||
block_ids: Per-KV-cache-group local block ids.
|
||||
|
||||
Returns:
|
||||
``block_ids`` with every full-attention group truncated to
|
||||
``ceil(num_prompt_tokens / block_size)`` leading blocks;
|
||||
sliding-window groups are left unchanged.
|
||||
|
||||
"""
|
||||
clamped = list(block_ids)
|
||||
changed = False
|
||||
for g, group_blocks in enumerate(block_ids):
|
||||
if self.blocks_per_sw[g] != 0:
|
||||
continue
|
||||
num_prompt_blocks = math.ceil(
|
||||
req.num_prompt_tokens / self._group_block_sizes[g]
|
||||
)
|
||||
if len(group_blocks) > num_prompt_blocks:
|
||||
clamped[g] = group_blocks[:num_prompt_blocks]
|
||||
changed = True
|
||||
return clamped if changed else block_ids
|
||||
|
||||
def _is_final_prefill_chunk(
|
||||
self, req: "Request", scheduler_output: SchedulerOutput
|
||||
) -> bool:
|
||||
"""Whether this step completes the prompt.
|
||||
|
||||
``build_connector_meta`` runs before ``_update_after_schedule``, so
|
||||
``num_computed_tokens`` excludes the current chunk; add this step's
|
||||
scheduled tokens. Token progress is lookahead-safe unlike block counts.
|
||||
"""
|
||||
num_scheduled = scheduler_output.num_scheduled_tokens.get(req.request_id, 0)
|
||||
return req.num_computed_tokens + num_scheduled >= req.num_prompt_tokens
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
@@ -1191,38 +1249,36 @@ class MoRIIOConnectorScheduler:
|
||||
# It places the request metadata into the saving queue.
|
||||
|
||||
for i, req_id in enumerate(scheduler_output.scheduled_cached_reqs.req_ids):
|
||||
# Non-disagg requests aren't registered here; skip them
|
||||
# instead of KeyError-ing.
|
||||
if req_id not in self._reqs_need_pending_save:
|
||||
continue
|
||||
req, existing_blocks = self._reqs_need_pending_save[req_id]
|
||||
# The final chunk may allocate no new blocks (the previous chunk
|
||||
# reserved speculative-lookahead slack), so new_block_ids is None;
|
||||
# still fall through to the final-chunk check below.
|
||||
new_block_ids = scheduler_output.scheduled_cached_reqs.new_block_ids[i]
|
||||
|
||||
if new_block_ids is not None:
|
||||
# A non-disagg request (no kv_transfer_params, e.g. smoke
|
||||
# test) is never registered in _reqs_need_pending_save;
|
||||
# indexing it unconditionally would KeyError and crash the
|
||||
# EngineCore. Skip it silently.
|
||||
if req_id not in self._reqs_need_pending_save:
|
||||
continue
|
||||
req, existing_blocks = self._reqs_need_pending_save[req_id]
|
||||
updated_blocks = [
|
||||
existing_blocks = [
|
||||
existing_blocks[g] + new_block_ids[g]
|
||||
for g in range(len(new_block_ids))
|
||||
]
|
||||
self._reqs_need_pending_save[req_id] = (req, updated_blocks)
|
||||
saved_tokens = (
|
||||
len(updated_blocks[self._full_attn_group_idx])
|
||||
* self._full_attn_block_size
|
||||
)
|
||||
if saved_tokens >= req.num_prompt_tokens:
|
||||
# Final chunk: live kv_transfer_params may be cleared,
|
||||
# so prefer the snapshot from update_state_after_alloc.
|
||||
kv_params = self._req_kv_params.pop(
|
||||
req_id, req.kv_transfer_params or {}
|
||||
)
|
||||
meta.add_new_req(
|
||||
request_id=req_id,
|
||||
local_block_ids=self._reqs_need_pending_save[req_id][1],
|
||||
kv_transfer_params=kv_params,
|
||||
write_mode=True,
|
||||
)
|
||||
del self._reqs_need_pending_save[req_id]
|
||||
self._reqs_need_pending_save[req_id] = (req, existing_blocks)
|
||||
if not self._is_final_prefill_chunk(req, scheduler_output):
|
||||
continue
|
||||
# Live kv_transfer_params may be cleared; prefer the
|
||||
# update_state_after_alloc snapshot.
|
||||
kv_params = self._req_kv_params.pop(
|
||||
req_id, req.kv_transfer_params or {}
|
||||
)
|
||||
save_block_ids = self._clamp_to_prompt_blocks(req, existing_blocks)
|
||||
meta.add_new_req(
|
||||
request_id=req_id,
|
||||
local_block_ids=save_block_ids,
|
||||
kv_transfer_params=kv_params,
|
||||
write_mode=True,
|
||||
)
|
||||
del self._reqs_need_pending_save[req_id]
|
||||
|
||||
# Loop through scheduled reqs and convert to ReqMeta.
|
||||
for req_id, (req, block_ids) in self._reqs_need_recv.items():
|
||||
@@ -1253,16 +1309,13 @@ class MoRIIOConnectorScheduler:
|
||||
|
||||
for req_id, (req, block_ids) in self._reqs_need_save.items():
|
||||
kv_params = self._req_kv_params.get(req_id, req.kv_transfer_params or {})
|
||||
tokens_covered = (
|
||||
len(block_ids[self._full_attn_group_idx]) * self._full_attn_block_size
|
||||
)
|
||||
if req.num_prompt_tokens > tokens_covered:
|
||||
# not last chunk prefill
|
||||
if not self._is_final_prefill_chunk(req, scheduler_output):
|
||||
self._reqs_need_pending_save[req_id] = (req, block_ids)
|
||||
continue
|
||||
save_block_ids = self._clamp_to_prompt_blocks(req, block_ids)
|
||||
meta.add_new_req(
|
||||
request_id=req_id,
|
||||
local_block_ids=block_ids,
|
||||
local_block_ids=save_block_ids,
|
||||
kv_transfer_params=kv_params,
|
||||
write_mode=True,
|
||||
)
|
||||
|
||||
@@ -377,10 +377,8 @@ def iter_layer_registration_regions(
|
||||
)
|
||||
region_len = geometry.num_blocks * geometry.regions_per_block * geometry.block_len
|
||||
if geometry.regions_per_block == 1:
|
||||
# With padded or interleaved pages the block stride exceeds the
|
||||
# meaningful block_len; register the strided span. The span ends with
|
||||
# the last block's meaningful bytes, not a whole stride, so it never
|
||||
# runs past the backing allocation of a strided layer view.
|
||||
# Padded/interleaved pages stride past block_len: register the strided
|
||||
# span, ending at the last block's bytes so it stays in-allocation.
|
||||
block_stride_bytes = geometry.block_stride * kv_cache.element_size()
|
||||
region_len = max(
|
||||
region_len,
|
||||
@@ -431,12 +429,8 @@ def compute_block_transfer_offsets(
|
||||
[list[int], list[int], list[int]], tuple[list[int], list[int], list[int]]
|
||||
] = merge_contiguous_offsets,
|
||||
) -> tuple[list[int], list[int], list[int]]:
|
||||
# A shorter (or empty) local list is the READ-mode "drop the transfer, just
|
||||
# free the prefill blocks" case (full-prefix-hit / aborted-before-scheduled):
|
||||
# decode pulls fewer blocks than the prefill holds. The zip loop below pairs
|
||||
# local[i]<->remote[i] and sizes by len(local), so a short local transfers
|
||||
# only what decode allocated and an empty local is a no-op. A longer local
|
||||
# list is a genuine bug and still fails loudly.
|
||||
# A shorter/empty local list transfers only what decode allocated (READ-mode
|
||||
# partial or full-prefix-hit); a longer local list is a genuine bug.
|
||||
if len(local_block_ids) > len(remote_block_ids):
|
||||
raise ValueError(
|
||||
"local_block_ids longer than remote_block_ids: "
|
||||
|
||||
Reference in New Issue
Block a user