[Attention][Cleanup] Remove tree attention (#42121)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-05-08 18:36:19 -07:00
committed by GitHub
parent be0dcc29dc
commit b1728c1e66
11 changed files with 6 additions and 1399 deletions
-214
View File
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import ast
from importlib.util import find_spec
from typing import Any, cast
@@ -29,10 +28,6 @@ from vllm.platforms import current_platform
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.backends.tree_attn import (
TreeAttentionMetadata,
TreeAttentionMetadataBuilder,
)
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadata
from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher
from vllm.v1.kv_cache_interface import KVCacheConfig, UniformTypeKVCacheSpecs
@@ -284,29 +279,6 @@ class SpecDecodeBaseProposer:
self.allowed_attn_types = tuple(rocm_types)
# Parse the speculative token tree.
spec_token_tree = self.speculative_config.speculative_token_tree
assert spec_token_tree is not None
self.tree_choices: list[tuple[int, ...]] = ast.literal_eval(spec_token_tree)
tree_depth = len(self.tree_choices[-1])
# Precompute per-level properties of the tree.
num_drafts_per_level = [0] * tree_depth
for node in self.tree_choices:
num_drafts_per_level[len(node) - 1] += 1
self.cu_drafts_per_level = [num_drafts_per_level[0]]
self.child_drafts_per_level = [num_drafts_per_level[0]]
for level in range(1, tree_depth):
self.cu_drafts_per_level.append(
self.cu_drafts_per_level[-1] + num_drafts_per_level[level]
)
self.child_drafts_per_level.append(
num_drafts_per_level[level] // num_drafts_per_level[level - 1]
)
# Precompute draft position offsets in flattened tree.
self.tree_draft_pos_offsets = torch.arange(
1, len(self.tree_choices) + 1, device=device, dtype=torch.int32
).repeat(self.max_batch_size, 1)
def _raise_if_padded_drafter_batch_disabled(self):
if self.speculative_config.disable_padded_drafter_batch:
raise NotImplementedError(
@@ -512,20 +484,6 @@ class SpecDecodeBaseProposer:
# (which read via _get_positions) use the correct values.
self.positions[:batch_size] = positions
if any(isinstance(md, TreeAttentionMetadata) for md in per_group_attn_metadata):
# Draft using tree attention - requires full logits for top-k
logits = self.model.compute_logits(sample_hidden_states)
draft_token_ids_list = self.propose_tree(
batch_size=batch_size,
logits=logits,
positions=positions,
hidden_states=hidden_states,
common_attn_metadata=common_attn_metadata,
slot_mappings=slot_mappings,
)
# [batch_size, num_tree_tokens]
return torch.cat(draft_token_ids_list, dim=1)
draft_token_ids = self._greedy_sample(sample_hidden_states)
if self.allowed_attn_types is not None:
@@ -1014,178 +972,6 @@ class SpecDecodeBaseProposer:
num_rejected_tokens_gpu,
)
def propose_tree(
self,
batch_size: int,
# [num_tokens, vocab_size]
logits: torch.Tensor,
# [num_tokens]
positions: torch.Tensor,
# [num_tokens, hidden_size]
hidden_states: torch.Tensor,
common_attn_metadata: CommonAttentionMetadata,
slot_mappings: dict[str, torch.Tensor]
| list[dict[str, torch.Tensor]]
| None = None,
) -> list[torch.Tensor]:
tree_attn_metadata_builder = self.draft_attn_groups[0].get_metadata_builder()
assert isinstance(tree_attn_metadata_builder, TreeAttentionMetadataBuilder)
total_num_drafts = self.cu_drafts_per_level[0]
level_num_drafts = total_num_drafts
# Sample a draft token for each child at the tree root level.
num_children = self.child_drafts_per_level[0]
if num_children == 1:
draft_token_ids = logits.argmax(dim=-1).view(batch_size, -1)
else:
draft_token_ids = torch.topk(logits, num_children, dim=-1).indices.view(
batch_size, -1
)
draft_token_ids_list = [draft_token_ids]
draft_hidden_states = hidden_states.view(batch_size, 1, -1)
# Initialize empty tensors for concatenation with the level outputs.
tree_input_ids = torch.empty(
0, device=self.input_ids.device, dtype=self.input_ids.dtype
)
tree_positions = torch.empty(
0, device=self.positions.device, dtype=self.positions.dtype
)
tree_hidden_states = torch.empty(
0, device=self.hidden_states.device, dtype=self.hidden_states.dtype
)
# Precompute the draft token positions.
flattened_draft_positions = (
positions.view(batch_size, -1) + self.tree_draft_pos_offsets[:batch_size, :]
)
tree_depth = len(self.cu_drafts_per_level)
for level in range(tree_depth - 1):
# Get draft positions for RoPE.
draft_positions = positions + (level + 1)
exceeds_max_model_len = (positions + total_num_drafts) >= self.max_model_len
# Mask out the position ids that exceed the max model length.
# Otherwise, we may get out-of-range error in RoPE.
draft_positions = torch.where(
exceeds_max_model_len,
0,
draft_positions,
).view(batch_size, -1)
if level_num_drafts > 1:
# Repeat the positions for each draft at this level.
draft_positions = draft_positions.repeat_interleave(
level_num_drafts, dim=1
)
if num_children > 1:
# Repeat draft hidden states for each child.
draft_hidden_states = draft_hidden_states.repeat_interleave(
num_children, dim=1
)
# Concatenate the draft tokens, positions, and hidden states.
tree_input_ids = torch.cat([tree_input_ids, draft_token_ids], dim=1)
tree_positions = torch.cat([tree_positions, draft_positions], dim=1)
tree_hidden_states = torch.cat(
[tree_hidden_states, draft_hidden_states], dim=1
)
# Build new attention metadata for the next level of drafts.
# This is necessary to support tree attention.
query_len = total_num_drafts
common_attn_metadata = replace(
common_attn_metadata,
query_start_loc=query_len * self.arange[: batch_size + 1],
seq_lens=common_attn_metadata.seq_lens + level_num_drafts,
num_actual_tokens=batch_size * query_len,
max_query_len=query_len,
)
attn_metadata = tree_attn_metadata_builder.build_for_drafting(
common_attn_metadata=common_attn_metadata, draft_index=level + 1
)
# Apply new attention metadata to all draft layers.
per_layer_attn_metadata = {}
for attn_group in self.draft_attn_groups:
for layer_name in attn_group.layer_names:
per_layer_attn_metadata[layer_name] = attn_metadata
# Consider max model length.
attn_metadata.max_seq_len = min(
attn_metadata.max_seq_len, self.max_model_len
)
# For the requests that exceed the max model length, we set the
# sequence length to 1 to minimize their overheads in attention.
attn_metadata.seq_lens.masked_fill_(exceeds_max_model_len, 1)
# Compute the slot mapping.
block_size = tree_attn_metadata_builder.kv_cache_spec.block_size
query_positions = flattened_draft_positions[:, level : level + query_len]
block_numbers = query_positions // block_size
block_ids = attn_metadata.block_table.gather(dim=1, index=block_numbers)
slot_mapping = block_ids * block_size + query_positions % block_size
# Mask out the slot mappings that exceed the max model length.
# Otherwise, the KV cache will be inadvertently updated with the
# padding tokens.
slot_mapping[exceeds_max_model_len] = PADDING_SLOT_ID
attn_metadata.slot_mapping = slot_mapping.view(-1)
# Copy inputs to buffer for cudagraph.
num_tokens = attn_metadata.num_actual_tokens
input_ids = tree_input_ids.view(-1)
self.input_ids[:num_tokens] = input_ids
self.positions[:num_tokens] = tree_positions.view(-1)
self.hidden_states[:num_tokens] = tree_hidden_states.view(num_tokens, -1)
cudagraph_runtime_mode, batch_desc = self.cudagraph_dispatcher.dispatch(
num_tokens
)
num_input_tokens = batch_desc.num_tokens
# Run the model.
with set_forward_context(
per_layer_attn_metadata,
self.vllm_config,
num_tokens=num_input_tokens,
cudagraph_runtime_mode=cudagraph_runtime_mode,
slot_mapping=self._get_slot_mapping(
num_input_tokens, attn_metadata.slot_mapping
),
):
last_hidden_states, hidden_states = self.model(
input_ids=self.input_ids[:num_input_tokens],
positions=self.positions[:num_input_tokens],
hidden_states=self.hidden_states[:num_input_tokens],
inputs_embeds=None,
)
# Get the output hidden states for the draft tokens.
draft_hidden_states = hidden_states[:num_tokens].view(
batch_size, query_len, -1
)[:, -level_num_drafts:]
draft_last_hidden_states = last_hidden_states[:num_tokens].view(
batch_size, query_len, -1
)[:, -level_num_drafts:]
# Get the output logits for the draft tokens.
logits = self.model.compute_logits(
draft_last_hidden_states.reshape(batch_size * level_num_drafts, -1)
)
# Sample a draft token for each child at the next tree level.
num_children = self.child_drafts_per_level[level + 1]
if num_children == 1:
draft_token_ids = logits.argmax(dim=-1).view(batch_size, -1)
else:
draft_token_ids = torch.topk(logits, num_children, dim=-1).indices.view(
batch_size, -1
)
draft_token_ids_list.append(draft_token_ids)
# Update the # drafts counters for the next tree level.
level_num_drafts = self.cu_drafts_per_level[level + 1] - total_num_drafts
total_num_drafts = self.cu_drafts_per_level[level + 1]
return draft_token_ids_list
def prepare_inputs(
self,
common_attn_metadata: CommonAttentionMetadata,