[Spec Decode] Fix hidden-state extraction block size for hybrid verifiers (#46301)
Signed-off-by: Igor Margulis <igor.margulis@intel.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Cursor
mgoin
parent
dc148dc4d7
commit
d8f483dc30
@@ -83,7 +83,10 @@ def basic_cache(
|
||||
kv_cache: torch.Tensor, # shape: [num_blocks, block_size, num_heads, head_size]
|
||||
slot_mapping: torch.Tensor, # shape: [seq_len]
|
||||
):
|
||||
# Padding slots are -1; redirect them to the null block (block 0, never
|
||||
# allocated to a request) so the scatter stays branch-free and sync-free.
|
||||
block_size = kv_cache.shape[1]
|
||||
slot_mapping = slot_mapping.clamp_min(0)
|
||||
kv_cache[slot_mapping // block_size, slot_mapping % block_size] = to_cache
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user