[V0 Deprecation] Remove V0 Sequence class & Sampler (#25332)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Signed-off-by: Woosuk Kwon <woosuk@thinkingmachines.ai>
This commit is contained in:
@@ -2,18 +2,15 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.sampling_metadata import SamplingMetadata
|
||||
|
||||
from .utils import maybe_prefix
|
||||
|
||||
@@ -105,8 +102,10 @@ class Medusa(nn.Module):
|
||||
return [block(hidden_states) for block in self.blocks]
|
||||
|
||||
def compute_logits(
|
||||
self, hidden_states: list[torch.Tensor],
|
||||
sampling_metadata: SamplingMetadata) -> list[torch.Tensor]:
|
||||
self,
|
||||
hidden_states: list[torch.Tensor],
|
||||
sampling_metadata,
|
||||
) -> list[torch.Tensor]:
|
||||
logits_lst: list[torch.Tensor] = []
|
||||
|
||||
for hs, lm_head in zip(hidden_states, self.lm_heads):
|
||||
@@ -130,57 +129,6 @@ class Medusa(nn.Module):
|
||||
|
||||
return logits_lst
|
||||
|
||||
def sample(
|
||||
self,
|
||||
logits: list[torch.Tensor],
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> list[SamplerOutput]:
|
||||
logits = torch.stack(logits, dim=0).float()
|
||||
logprobs = torch.log_softmax(logits, dim=-1)
|
||||
token_ids = logits.argmax(-1) # support only top-1 for now
|
||||
probs = torch.softmax(logits, dim=-1)
|
||||
|
||||
token_id_list = []
|
||||
token_prob_list = []
|
||||
token_logprob_list = []
|
||||
|
||||
for idx, seq_group in enumerate(sampling_metadata.seq_groups):
|
||||
token_id_list.append(token_ids[:, seq_group.sample_indices])
|
||||
token_prob_list.append(probs[:, seq_group.sample_indices])
|
||||
token_logprob_list.append(logprobs[:, seq_group.sample_indices])
|
||||
|
||||
outputs: list[Optional[SamplerOutput]] = []
|
||||
for idx in range(len(sampling_metadata.seq_groups)):
|
||||
outputs.append(
|
||||
SamplerOutput(
|
||||
outputs=None,
|
||||
sampled_token_probs=token_prob_list[idx].squeeze(1),
|
||||
logprobs=token_logprob_list[idx].squeeze(1),
|
||||
sampled_token_ids=token_id_list[idx].squeeze(1),
|
||||
))
|
||||
|
||||
return outputs
|
||||
|
||||
def generate_proposals(
|
||||
self,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> Optional[list[SamplerOutput]]:
|
||||
# During preemption, we may receive an empty tensor (batch_size=0)
|
||||
if previous_hidden_states.size(0) == 0:
|
||||
# Return None to signal the Top1Proposer that no proposals
|
||||
# were generated for this batch, allowing it to handle this
|
||||
# special case appropriately
|
||||
return None
|
||||
|
||||
return self.sample(
|
||||
logits=self.compute_logits(
|
||||
hidden_states=self.forward(previous_hidden_states),
|
||||
sampling_metadata=sampling_metadata,
|
||||
),
|
||||
sampling_metadata=sampling_metadata,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str,
|
||||
torch.Tensor]]) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
|
||||
@@ -8,9 +8,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor import SamplingMetadata
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead, VocabParallelEmbedding)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
@@ -141,55 +139,57 @@ class MLPSpeculator(nn.Module):
|
||||
self.config = config
|
||||
self.logits_processor = LogitsProcessor(config.vocab_size,
|
||||
config.vocab_size, 1.0)
|
||||
self.sampler = get_sampler()
|
||||
|
||||
def generate_proposals(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
previous_hidden_states: torch.Tensor,
|
||||
num_predict_tokens: int,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> list[SamplerOutput]:
|
||||
if num_predict_tokens > self.max_speculative_tokens:
|
||||
raise ValueError(f"Max speculative tokens for model is "
|
||||
f"{self.max_speculative_tokens}, but "
|
||||
f"{num_predict_tokens} were requested")
|
||||
# NOTE(woosuk): This method is commented out because it is old code
|
||||
# using V0. We should either port it to V1 or remove it.
|
||||
|
||||
# b x 1 x d
|
||||
previous_hidden_states = previous_hidden_states.unsqueeze(1)
|
||||
# def generate_proposals(
|
||||
# self,
|
||||
# input_ids: torch.Tensor,
|
||||
# previous_hidden_states: torch.Tensor,
|
||||
# num_predict_tokens: int,
|
||||
# sampling_metadata: SamplingMetadata,
|
||||
# ) -> list[SamplerOutput]:
|
||||
# if num_predict_tokens > self.max_speculative_tokens:
|
||||
# raise ValueError(f"Max speculative tokens for model is "
|
||||
# f"{self.max_speculative_tokens}, but "
|
||||
# f"{num_predict_tokens} were requested")
|
||||
|
||||
if self.scale_input:
|
||||
previous_hidden_states = self.ln0(previous_hidden_states) / SQRT2
|
||||
# # b x 1 x d
|
||||
# previous_hidden_states = previous_hidden_states.unsqueeze(1)
|
||||
|
||||
# b x 1
|
||||
last_tokens = input_ids.unsqueeze(1)
|
||||
# if self.scale_input:
|
||||
# previous_hidden_states = self.ln0(previous_hidden_states) / SQRT2
|
||||
|
||||
next_tokens = []
|
||||
# # b x 1
|
||||
# last_tokens = input_ids.unsqueeze(1)
|
||||
|
||||
for head_index in range(num_predict_tokens):
|
||||
# next_tokens = []
|
||||
|
||||
# Project and predict
|
||||
z = self.emb[head_index](last_tokens) # b k d
|
||||
states = self.proj[head_index](previous_hidden_states)
|
||||
# for head_index in range(num_predict_tokens):
|
||||
|
||||
# Weighted add of state_weight*state and emb_weight*z
|
||||
# Let subsequent LN take care of denominator
|
||||
# state_weight is close to 1, so shouldn't be any precision issues
|
||||
states.add_(z, alpha=self.emb_weight / self.state_weight)
|
||||
# # Project and predict
|
||||
# z = self.emb[head_index](last_tokens) # b k d
|
||||
# states = self.proj[head_index](previous_hidden_states)
|
||||
|
||||
states = self.activation(self.ln[head_index](states)) # b k d
|
||||
previous_hidden_states = states
|
||||
# TODO: not yet supporting top_k_tokens_per_head
|
||||
states = states.flatten(0, 1)
|
||||
# # Weighted add of state_weight*state and emb_weight*z
|
||||
# # Let subsequent LN take care of denominator
|
||||
# # state_weight is close to 1, so shouldn't be any precision issues
|
||||
# states.add_(z, alpha=self.emb_weight / self.state_weight)
|
||||
|
||||
logits = self.logits_processor(self.head[head_index], states,
|
||||
sampling_metadata)
|
||||
# states = self.activation(self.ln[head_index](states)) # b k d
|
||||
# previous_hidden_states = states
|
||||
# # TODO: not yet supporting top_k_tokens_per_head
|
||||
# states = states.flatten(0, 1)
|
||||
|
||||
output = self.sampler(logits, sampling_metadata)
|
||||
last_tokens = output.sampled_token_ids
|
||||
next_tokens.append(output)
|
||||
# logits = self.logits_processor(self.head[head_index], states,
|
||||
# sampling_metadata)
|
||||
|
||||
return next_tokens
|
||||
# output = self.sampler(logits, sampling_metadata)
|
||||
# last_tokens = output.sampled_token_ids
|
||||
# next_tokens.append(output)
|
||||
|
||||
# return next_tokens
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str,
|
||||
torch.Tensor]]) -> set[str]:
|
||||
|
||||
@@ -697,16 +697,12 @@ class Phi4FlashForCausalLM(nn.Module, HasInnerState, IsHybrid, SupportsV0Only):
|
||||
hidden_states: torch.Tensor,
|
||||
sampling_metadata: SamplingMetadata,
|
||||
) -> Optional[torch.Tensor]:
|
||||
# If the shape is the same, it means that we have already
|
||||
# prune hidden states manually.
|
||||
prune_hidden_states = hidden_states.size(
|
||||
0) != sampling_metadata.selected_token_indices.size(0)
|
||||
processed_logits = self.logits_processor(
|
||||
self.lm_head,
|
||||
hidden_states,
|
||||
sampling_metadata,
|
||||
self.embedding_bias,
|
||||
prune_hidden_states=prune_hidden_states)
|
||||
)
|
||||
return processed_logits
|
||||
|
||||
def load_weights(
|
||||
|
||||
Reference in New Issue
Block a user