DCP supports hybrid attention (#40996)
Signed-off-by: YanXu <yancey.yx@alibaba-inc.com> Signed-off-by: Jingyi Yang <girasoleyang@gmail.com> Co-authored-by: Jingyi Yang <girasoleyang@gmail.com>
This commit is contained in:
@@ -29,13 +29,17 @@ def _compute_slot_mapping_kernel_impl(
|
||||
block_table_stride: int, # max_num_blocks_per_req
|
||||
block_size: int,
|
||||
slot_mapping: torch.Tensor, # [max_num_tokens], int64
|
||||
TOTAL_CP_WORLD_SIZE: int,
|
||||
TOTAL_CP_RANK: int,
|
||||
CP_KV_CACHE_INTERLEAVE_SIZE: int,
|
||||
PAD_ID: int,
|
||||
BLOCK_SIZE: int,
|
||||
KV_CACHE_BLOCK_SIZE: int | None = None,
|
||||
BLOCKS_PER_KV_BLOCK: int = 1,
|
||||
TOTAL_CP_WORLD_SIZE: int = 1,
|
||||
TOTAL_CP_RANK: int = 0,
|
||||
CP_KV_CACHE_INTERLEAVE_SIZE: int = 1,
|
||||
PAD_ID: int = -1,
|
||||
BLOCK_SIZE: int = 1024,
|
||||
) -> None:
|
||||
assert TOTAL_CP_WORLD_SIZE == 1, "Context Parallelism is not supported on CPU."
|
||||
if BLOCKS_PER_KV_BLOCK != 1:
|
||||
assert block_size * BLOCKS_PER_KV_BLOCK == KV_CACHE_BLOCK_SIZE
|
||||
torch.ops._C.compute_slot_mapping_kernel_impl(
|
||||
query_start_loc,
|
||||
positions,
|
||||
|
||||
Reference in New Issue
Block a user