[Kimi-K3] support DCP partial prefix cache hit (#50493)
Signed-off-by: Summer Yang <girasoleyang@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Yifan Qiao
parent
0e8989b416
commit
0db502c8d8
@@ -116,8 +116,15 @@ class BlockTables:
|
||||
bpk = self.blocks_per_kv_block[i]
|
||||
if bpk > 1:
|
||||
block_ids = [b * bpk + k for b in block_ids for k in range(bpk)]
|
||||
end = start + len(block_ids)
|
||||
row_capacity = self.block_tables[i].gpu.shape[1]
|
||||
if end > row_capacity:
|
||||
raise RuntimeError(
|
||||
f"Block table write for request {req_index}, group {i} exceeds "
|
||||
f"row capacity ({end} > {row_capacity})"
|
||||
)
|
||||
self.block_tables[i].stage_write(req_index, start, block_ids)
|
||||
self.num_blocks.np[i, req_index] = start + len(block_ids)
|
||||
self.num_blocks.np[i, req_index] = end
|
||||
|
||||
def apply_staged_writes(self) -> None:
|
||||
if self.num_kv_cache_groups == 0:
|
||||
|
||||
Reference in New Issue
Block a user