[Core] Refactor Attention Take 2 (#3462)
This commit is contained in:
@@ -34,19 +34,19 @@ def test_prepare_prompt(batch_size):
|
||||
expected_selected_token_indices.append(selected_token_start_idx +
|
||||
prompt_len - 1)
|
||||
selected_token_start_idx += prompt_len
|
||||
(input_tokens, input_positions, input_metadata, return_prompt_lens, _, _,
|
||||
_, _) = (model_runner._prepare_prompt(seq_group_metadata_list))
|
||||
(input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
|
||||
_) = (model_runner._prepare_prompt(seq_group_metadata_list))
|
||||
assert return_prompt_lens == prompt_lens
|
||||
|
||||
# Verify input metadata is correct for prompts.
|
||||
device = model_runner.device
|
||||
assert input_metadata.is_prompt is True
|
||||
assert torch.allclose(input_metadata.prompt_lens_tensor,
|
||||
assert attn_metadata.is_prompt is True
|
||||
assert torch.allclose(attn_metadata.prompt_lens_tensor,
|
||||
torch.tensor(prompt_lens, device=device))
|
||||
assert input_metadata.prompt_lens == prompt_lens
|
||||
assert input_metadata.num_prompt_tokens == sum(prompt_lens)
|
||||
assert input_metadata.num_generation_tokens == 0
|
||||
assert input_metadata.max_seq_len == max(prompt_lens)
|
||||
assert attn_metadata.prompt_lens == prompt_lens
|
||||
assert attn_metadata.num_prompt_tokens == sum(prompt_lens)
|
||||
assert attn_metadata.num_generation_tokens == 0
|
||||
assert attn_metadata.max_prompt_len == max(prompt_lens)
|
||||
|
||||
# Test subquery start locs.
|
||||
start_idx = 0
|
||||
@@ -55,7 +55,7 @@ def test_prepare_prompt(batch_size):
|
||||
start_idx += prompt_len
|
||||
start_loc.append(start_idx)
|
||||
assert torch.allclose(
|
||||
input_metadata.subquery_start_loc,
|
||||
attn_metadata.subquery_start_loc,
|
||||
torch.tensor(start_loc, dtype=torch.int32, device=device))
|
||||
|
||||
# Test seq start locs. Note that for normal prefill it is
|
||||
@@ -67,22 +67,22 @@ def test_prepare_prompt(batch_size):
|
||||
seq_start_loc.append(start_idx)
|
||||
|
||||
assert torch.allclose(
|
||||
input_metadata.seq_start_loc,
|
||||
attn_metadata.seq_start_loc,
|
||||
torch.tensor(start_loc, dtype=torch.int32, device=device))
|
||||
assert input_metadata.max_context_len is None
|
||||
assert attn_metadata.max_context_len is None
|
||||
assert torch.allclose(
|
||||
input_metadata.context_lens,
|
||||
torch.zeros(input_metadata.context_lens.shape[0],
|
||||
attn_metadata.context_lens,
|
||||
torch.zeros(attn_metadata.context_lens.shape[0],
|
||||
dtype=torch.int,
|
||||
device=device))
|
||||
|
||||
expected = torch.tensor([[] for _ in range(len(seq_group_metadata_list))],
|
||||
dtype=torch.int32,
|
||||
device=model_runner.device)
|
||||
assert torch.allclose(input_metadata.block_tables, expected)
|
||||
assert torch.allclose(attn_metadata.block_tables, expected)
|
||||
# Cuda graph should not be used for prerill.
|
||||
assert input_metadata.use_cuda_graph is False
|
||||
assert input_metadata.kv_cache_dtype == "auto"
|
||||
assert attn_metadata.use_cuda_graph is False
|
||||
assert attn_metadata.kv_cache_dtype == "auto"
|
||||
|
||||
assert input_tokens.shape == (sum(prompt_lens), )
|
||||
assert input_positions.shape == (sum(prompt_lens), )
|
||||
@@ -140,34 +140,34 @@ def test_prepare_decode_cuda_graph(batch_size):
|
||||
block_tables={0: [1]},
|
||||
))
|
||||
|
||||
input_tokens, input_positions, input_metadata, _, _, _ = (
|
||||
input_tokens, input_positions, attn_metadata, _, _, _ = (
|
||||
model_runner._prepare_decode(seq_group_metadata_list))
|
||||
|
||||
expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
|
||||
# Verify input metadata is correct for prompts.
|
||||
device = model_runner.device
|
||||
assert input_metadata.is_prompt is False
|
||||
assert input_metadata.prompt_lens is None
|
||||
assert input_metadata.num_prompt_tokens == 0
|
||||
assert input_metadata.num_generation_tokens == expected_bs
|
||||
assert input_metadata.max_seq_len is None
|
||||
assert input_metadata.subquery_start_loc is None
|
||||
assert input_metadata.seq_start_loc is None
|
||||
assert input_metadata.max_context_len == max(prompt_lens)
|
||||
assert attn_metadata.is_prompt is False
|
||||
assert attn_metadata.prompt_lens is None
|
||||
assert attn_metadata.num_prompt_tokens == 0
|
||||
assert attn_metadata.num_generation_tokens == expected_bs
|
||||
assert attn_metadata.max_prompt_len is None
|
||||
assert attn_metadata.subquery_start_loc is None
|
||||
assert attn_metadata.seq_start_loc is None
|
||||
assert attn_metadata.max_context_len == max(prompt_lens)
|
||||
assert torch.allclose(
|
||||
input_metadata.context_lens[:len(prompt_lens)],
|
||||
attn_metadata.context_lens[:len(prompt_lens)],
|
||||
torch.tensor(prompt_lens, dtype=torch.int, device=device))
|
||||
|
||||
# block table's first index corresponds to each batch, meaning in
|
||||
# decoding it is each token.
|
||||
assert input_metadata.block_tables.shape[0] == len(input_tokens)
|
||||
assert attn_metadata.block_tables.shape[0] == len(input_tokens)
|
||||
# Block table's second dim correspondsd to each token's block number.
|
||||
# It is padded up to
|
||||
assert input_metadata.block_tables.shape[1] == (
|
||||
assert attn_metadata.block_tables.shape[1] == (
|
||||
model_runner.get_max_block_per_batch())
|
||||
# Cuda graph should not be used for prerill.
|
||||
assert input_metadata.use_cuda_graph is True
|
||||
assert input_metadata.kv_cache_dtype == "auto"
|
||||
assert attn_metadata.use_cuda_graph is True
|
||||
assert attn_metadata.kv_cache_dtype == "auto"
|
||||
|
||||
assert input_tokens.shape == (expected_bs, )
|
||||
assert input_positions.shape == (expected_bs, )
|
||||
|
||||
Reference in New Issue
Block a user