[Attention][Cleanup] Remove tree attention (#42121)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import ast
|
||||
from importlib.util import find_spec
|
||||
from typing import Any, cast
|
||||
|
||||
@@ -29,10 +28,6 @@ from vllm.platforms import current_platform
|
||||
from vllm.utils.platform_utils import is_pin_memory_available
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.backends.tree_attn import (
|
||||
TreeAttentionMetadata,
|
||||
TreeAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadata
|
||||
from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, UniformTypeKVCacheSpecs
|
||||
@@ -284,29 +279,6 @@ class SpecDecodeBaseProposer:
|
||||
|
||||
self.allowed_attn_types = tuple(rocm_types)
|
||||
|
||||
# Parse the speculative token tree.
|
||||
spec_token_tree = self.speculative_config.speculative_token_tree
|
||||
assert spec_token_tree is not None
|
||||
self.tree_choices: list[tuple[int, ...]] = ast.literal_eval(spec_token_tree)
|
||||
tree_depth = len(self.tree_choices[-1])
|
||||
# Precompute per-level properties of the tree.
|
||||
num_drafts_per_level = [0] * tree_depth
|
||||
for node in self.tree_choices:
|
||||
num_drafts_per_level[len(node) - 1] += 1
|
||||
self.cu_drafts_per_level = [num_drafts_per_level[0]]
|
||||
self.child_drafts_per_level = [num_drafts_per_level[0]]
|
||||
for level in range(1, tree_depth):
|
||||
self.cu_drafts_per_level.append(
|
||||
self.cu_drafts_per_level[-1] + num_drafts_per_level[level]
|
||||
)
|
||||
self.child_drafts_per_level.append(
|
||||
num_drafts_per_level[level] // num_drafts_per_level[level - 1]
|
||||
)
|
||||
# Precompute draft position offsets in flattened tree.
|
||||
self.tree_draft_pos_offsets = torch.arange(
|
||||
1, len(self.tree_choices) + 1, device=device, dtype=torch.int32
|
||||
).repeat(self.max_batch_size, 1)
|
||||
|
||||
def _raise_if_padded_drafter_batch_disabled(self):
|
||||
if self.speculative_config.disable_padded_drafter_batch:
|
||||
raise NotImplementedError(
|
||||
@@ -512,20 +484,6 @@ class SpecDecodeBaseProposer:
|
||||
# (which read via _get_positions) use the correct values.
|
||||
self.positions[:batch_size] = positions
|
||||
|
||||
if any(isinstance(md, TreeAttentionMetadata) for md in per_group_attn_metadata):
|
||||
# Draft using tree attention - requires full logits for top-k
|
||||
logits = self.model.compute_logits(sample_hidden_states)
|
||||
draft_token_ids_list = self.propose_tree(
|
||||
batch_size=batch_size,
|
||||
logits=logits,
|
||||
positions=positions,
|
||||
hidden_states=hidden_states,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
slot_mappings=slot_mappings,
|
||||
)
|
||||
# [batch_size, num_tree_tokens]
|
||||
return torch.cat(draft_token_ids_list, dim=1)
|
||||
|
||||
draft_token_ids = self._greedy_sample(sample_hidden_states)
|
||||
|
||||
if self.allowed_attn_types is not None:
|
||||
@@ -1014,178 +972,6 @@ class SpecDecodeBaseProposer:
|
||||
num_rejected_tokens_gpu,
|
||||
)
|
||||
|
||||
def propose_tree(
|
||||
self,
|
||||
batch_size: int,
|
||||
# [num_tokens, vocab_size]
|
||||
logits: torch.Tensor,
|
||||
# [num_tokens]
|
||||
positions: torch.Tensor,
|
||||
# [num_tokens, hidden_size]
|
||||
hidden_states: torch.Tensor,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
slot_mappings: dict[str, torch.Tensor]
|
||||
| list[dict[str, torch.Tensor]]
|
||||
| None = None,
|
||||
) -> list[torch.Tensor]:
|
||||
tree_attn_metadata_builder = self.draft_attn_groups[0].get_metadata_builder()
|
||||
assert isinstance(tree_attn_metadata_builder, TreeAttentionMetadataBuilder)
|
||||
|
||||
total_num_drafts = self.cu_drafts_per_level[0]
|
||||
level_num_drafts = total_num_drafts
|
||||
# Sample a draft token for each child at the tree root level.
|
||||
num_children = self.child_drafts_per_level[0]
|
||||
if num_children == 1:
|
||||
draft_token_ids = logits.argmax(dim=-1).view(batch_size, -1)
|
||||
else:
|
||||
draft_token_ids = torch.topk(logits, num_children, dim=-1).indices.view(
|
||||
batch_size, -1
|
||||
)
|
||||
draft_token_ids_list = [draft_token_ids]
|
||||
draft_hidden_states = hidden_states.view(batch_size, 1, -1)
|
||||
|
||||
# Initialize empty tensors for concatenation with the level outputs.
|
||||
tree_input_ids = torch.empty(
|
||||
0, device=self.input_ids.device, dtype=self.input_ids.dtype
|
||||
)
|
||||
tree_positions = torch.empty(
|
||||
0, device=self.positions.device, dtype=self.positions.dtype
|
||||
)
|
||||
tree_hidden_states = torch.empty(
|
||||
0, device=self.hidden_states.device, dtype=self.hidden_states.dtype
|
||||
)
|
||||
# Precompute the draft token positions.
|
||||
flattened_draft_positions = (
|
||||
positions.view(batch_size, -1) + self.tree_draft_pos_offsets[:batch_size, :]
|
||||
)
|
||||
tree_depth = len(self.cu_drafts_per_level)
|
||||
for level in range(tree_depth - 1):
|
||||
# Get draft positions for RoPE.
|
||||
draft_positions = positions + (level + 1)
|
||||
exceeds_max_model_len = (positions + total_num_drafts) >= self.max_model_len
|
||||
# Mask out the position ids that exceed the max model length.
|
||||
# Otherwise, we may get out-of-range error in RoPE.
|
||||
draft_positions = torch.where(
|
||||
exceeds_max_model_len,
|
||||
0,
|
||||
draft_positions,
|
||||
).view(batch_size, -1)
|
||||
|
||||
if level_num_drafts > 1:
|
||||
# Repeat the positions for each draft at this level.
|
||||
draft_positions = draft_positions.repeat_interleave(
|
||||
level_num_drafts, dim=1
|
||||
)
|
||||
|
||||
if num_children > 1:
|
||||
# Repeat draft hidden states for each child.
|
||||
draft_hidden_states = draft_hidden_states.repeat_interleave(
|
||||
num_children, dim=1
|
||||
)
|
||||
|
||||
# Concatenate the draft tokens, positions, and hidden states.
|
||||
tree_input_ids = torch.cat([tree_input_ids, draft_token_ids], dim=1)
|
||||
tree_positions = torch.cat([tree_positions, draft_positions], dim=1)
|
||||
tree_hidden_states = torch.cat(
|
||||
[tree_hidden_states, draft_hidden_states], dim=1
|
||||
)
|
||||
|
||||
# Build new attention metadata for the next level of drafts.
|
||||
# This is necessary to support tree attention.
|
||||
query_len = total_num_drafts
|
||||
common_attn_metadata = replace(
|
||||
common_attn_metadata,
|
||||
query_start_loc=query_len * self.arange[: batch_size + 1],
|
||||
seq_lens=common_attn_metadata.seq_lens + level_num_drafts,
|
||||
num_actual_tokens=batch_size * query_len,
|
||||
max_query_len=query_len,
|
||||
)
|
||||
attn_metadata = tree_attn_metadata_builder.build_for_drafting(
|
||||
common_attn_metadata=common_attn_metadata, draft_index=level + 1
|
||||
)
|
||||
|
||||
# Apply new attention metadata to all draft layers.
|
||||
per_layer_attn_metadata = {}
|
||||
for attn_group in self.draft_attn_groups:
|
||||
for layer_name in attn_group.layer_names:
|
||||
per_layer_attn_metadata[layer_name] = attn_metadata
|
||||
|
||||
# Consider max model length.
|
||||
attn_metadata.max_seq_len = min(
|
||||
attn_metadata.max_seq_len, self.max_model_len
|
||||
)
|
||||
# For the requests that exceed the max model length, we set the
|
||||
# sequence length to 1 to minimize their overheads in attention.
|
||||
attn_metadata.seq_lens.masked_fill_(exceeds_max_model_len, 1)
|
||||
|
||||
# Compute the slot mapping.
|
||||
block_size = tree_attn_metadata_builder.kv_cache_spec.block_size
|
||||
query_positions = flattened_draft_positions[:, level : level + query_len]
|
||||
block_numbers = query_positions // block_size
|
||||
block_ids = attn_metadata.block_table.gather(dim=1, index=block_numbers)
|
||||
slot_mapping = block_ids * block_size + query_positions % block_size
|
||||
# Mask out the slot mappings that exceed the max model length.
|
||||
# Otherwise, the KV cache will be inadvertently updated with the
|
||||
# padding tokens.
|
||||
slot_mapping[exceeds_max_model_len] = PADDING_SLOT_ID
|
||||
attn_metadata.slot_mapping = slot_mapping.view(-1)
|
||||
|
||||
# Copy inputs to buffer for cudagraph.
|
||||
num_tokens = attn_metadata.num_actual_tokens
|
||||
input_ids = tree_input_ids.view(-1)
|
||||
self.input_ids[:num_tokens] = input_ids
|
||||
self.positions[:num_tokens] = tree_positions.view(-1)
|
||||
self.hidden_states[:num_tokens] = tree_hidden_states.view(num_tokens, -1)
|
||||
|
||||
cudagraph_runtime_mode, batch_desc = self.cudagraph_dispatcher.dispatch(
|
||||
num_tokens
|
||||
)
|
||||
num_input_tokens = batch_desc.num_tokens
|
||||
# Run the model.
|
||||
with set_forward_context(
|
||||
per_layer_attn_metadata,
|
||||
self.vllm_config,
|
||||
num_tokens=num_input_tokens,
|
||||
cudagraph_runtime_mode=cudagraph_runtime_mode,
|
||||
slot_mapping=self._get_slot_mapping(
|
||||
num_input_tokens, attn_metadata.slot_mapping
|
||||
),
|
||||
):
|
||||
last_hidden_states, hidden_states = self.model(
|
||||
input_ids=self.input_ids[:num_input_tokens],
|
||||
positions=self.positions[:num_input_tokens],
|
||||
hidden_states=self.hidden_states[:num_input_tokens],
|
||||
inputs_embeds=None,
|
||||
)
|
||||
|
||||
# Get the output hidden states for the draft tokens.
|
||||
draft_hidden_states = hidden_states[:num_tokens].view(
|
||||
batch_size, query_len, -1
|
||||
)[:, -level_num_drafts:]
|
||||
draft_last_hidden_states = last_hidden_states[:num_tokens].view(
|
||||
batch_size, query_len, -1
|
||||
)[:, -level_num_drafts:]
|
||||
|
||||
# Get the output logits for the draft tokens.
|
||||
logits = self.model.compute_logits(
|
||||
draft_last_hidden_states.reshape(batch_size * level_num_drafts, -1)
|
||||
)
|
||||
|
||||
# Sample a draft token for each child at the next tree level.
|
||||
num_children = self.child_drafts_per_level[level + 1]
|
||||
if num_children == 1:
|
||||
draft_token_ids = logits.argmax(dim=-1).view(batch_size, -1)
|
||||
else:
|
||||
draft_token_ids = torch.topk(logits, num_children, dim=-1).indices.view(
|
||||
batch_size, -1
|
||||
)
|
||||
draft_token_ids_list.append(draft_token_ids)
|
||||
|
||||
# Update the # drafts counters for the next tree level.
|
||||
level_num_drafts = self.cu_drafts_per_level[level + 1] - total_num_drafts
|
||||
total_num_drafts = self.cu_drafts_per_level[level + 1]
|
||||
return draft_token_ids_list
|
||||
|
||||
def prepare_inputs(
|
||||
self,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
|
||||
Reference in New Issue
Block a user