[Bugfix] Fix M-RoPE position calculation when chunked prefill is enabled (#10388)

Signed-off-by: imkero <kerorek@outlook.com>
This commit is contained in:
电脑星人
2024-11-17 02:10:00 +08:00
committed by GitHub
parent b98d89efd4
commit 361c29e174
3 changed files with 135 additions and 5 deletions
@@ -847,6 +847,7 @@ class MRotaryEmbedding(RotaryEmbedding):
vision_end_token_id: int,
spatial_merge_size: int,
context_len: int = 0,
seq_len: Optional[int] = None,
) -> Tuple[List[List[int]], int]:
"""Get mrope input positions and delta value."""
@@ -921,7 +922,7 @@ class MRotaryEmbedding(RotaryEmbedding):
torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
llm_positions = llm_positions[:, context_len:]
llm_positions = llm_positions[:, context_len:seq_len]
mrope_position_delta = (llm_positions.max() + 1 -
len(input_tokens)).item()