[V0 Deprecation] Remove pooling model support in V0 (#23434)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Signed-off-by: Max de Bayser <mbayser@br.ibm.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
co-authored by
Woosuk Kwon
parent
934bebf192
commit
2554b27baa
@@ -13,17 +13,12 @@ import torch.nn.functional as F
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config import ModelConfig, PoolerConfig
|
||||
from vllm.model_executor.pooling_metadata import ( # noqa: E501
|
||||
PoolingMetadata as V0PoolingMetadata)
|
||||
from vllm.model_executor.pooling_metadata import PoolingTensors
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.sequence import PoolerOutput, PoolingSequenceGroupOutput
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils import current_stream, resolve_obj_by_qualname
|
||||
from vllm.v1.pool.metadata import PoolingCursor
|
||||
from vllm.v1.pool.metadata import PoolingMetadata as V1PoolingMetadata
|
||||
from vllm.v1.pool.metadata import PoolingCursor, PoolingMetadata
|
||||
|
||||
PoolingMetadata = Union[V0PoolingMetadata, V1PoolingMetadata]
|
||||
PoolingFn = Callable[
|
||||
[Union[torch.Tensor, list[torch.Tensor]], PoolingMetadata],
|
||||
Union[torch.Tensor, list[torch.Tensor]]]
|
||||
@@ -127,36 +122,23 @@ def get_prompt_lens(
|
||||
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> torch.Tensor:
|
||||
if isinstance(pooling_metadata, V1PoolingMetadata):
|
||||
return pooling_metadata.prompt_lens
|
||||
|
||||
return PoolingTensors.from_pooling_metadata(
|
||||
pooling_metadata, hidden_states[0].device).prompt_lens
|
||||
return pooling_metadata.prompt_lens
|
||||
|
||||
|
||||
def get_prompt_token_ids(
|
||||
pooling_metadata: PoolingMetadata) -> list[torch.Tensor]:
|
||||
if isinstance(pooling_metadata, V1PoolingMetadata):
|
||||
assert pooling_metadata.prompt_token_ids is not None, (
|
||||
"Please set `requires_token_ids=True` in `get_pooling_updates`")
|
||||
|
||||
return [
|
||||
pooling_metadata.prompt_token_ids[i, :num]
|
||||
for i, num in enumerate(pooling_metadata.prompt_lens)
|
||||
]
|
||||
assert pooling_metadata.prompt_token_ids is not None, (
|
||||
"Please set `requires_token_ids=True` in `get_pooling_updates`")
|
||||
|
||||
return [
|
||||
torch.tensor(seq_data_i.prompt_token_ids)
|
||||
for seq_data_i in pooling_metadata.seq_data.values()
|
||||
pooling_metadata.prompt_token_ids[i, :num]
|
||||
for i, num in enumerate(pooling_metadata.prompt_lens)
|
||||
]
|
||||
|
||||
|
||||
def get_pooling_params(
|
||||
pooling_metadata: PoolingMetadata) -> list[PoolingParams]:
|
||||
if isinstance(pooling_metadata, V0PoolingMetadata):
|
||||
pooling_params = [p for _, p in pooling_metadata.seq_groups]
|
||||
else:
|
||||
pooling_params = pooling_metadata.pooling_params
|
||||
pooling_params = pooling_metadata.pooling_params
|
||||
return pooling_params
|
||||
|
||||
|
||||
|
||||
@@ -24,9 +24,9 @@ from vllm.model_executor.layers.pooler import (ClassifierPooler,
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding)
|
||||
from vllm.model_executor.pooling_metadata import PoolingMetadata
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .interfaces import SupportsCrossEncoding, SupportsQuant
|
||||
from .interfaces_base import default_pooling_type
|
||||
|
||||
@@ -15,10 +15,10 @@ from vllm.model_executor.layers.pooler import (DispatchPooler, Pooler,
|
||||
build_output, get_prompt_lens,
|
||||
get_prompt_token_ids)
|
||||
from vllm.model_executor.models.llama import LlamaForCausalLM
|
||||
from vllm.model_executor.pooling_metadata import PoolingMetadata
|
||||
from vllm.sequence import PoolerOutput
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .interfaces_base import default_pooling_type
|
||||
|
||||
|
||||
@@ -22,9 +22,9 @@ from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.model_executor.pooling_metadata import PoolingMetadata
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .interfaces import SupportsCrossEncoding
|
||||
from .interfaces_base import default_pooling_type
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.utils import is_pin_memory_available
|
||||
from vllm.v1.pool.metadata import PoolingCursor, build_pooling_cursor
|
||||
|
||||
|
||||
class PoolingMetadata:
|
||||
"""Metadata for pooling operations in the Pooler layer.
|
||||
|
||||
This class holds the necessary information for pooling operations,
|
||||
providing context for how to perform pooling and other related operations.
|
||||
|
||||
Attributes:
|
||||
seq_groups: List of (seq_ids, pooling_params).
|
||||
seq_data: A mapping of sequence ID to additional sequence data.
|
||||
prompt_lens: List of the lengths of each prompt.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
seq_groups: list[tuple[list[int], PoolingParams]],
|
||||
seq_data: dict[int, Any], # Specific data related to sequences
|
||||
prompt_lens: list[int],
|
||||
pooling_cursor: Optional[PoolingCursor] = None) -> None:
|
||||
self.seq_groups = seq_groups
|
||||
self.seq_data = seq_data
|
||||
self.prompt_lens = prompt_lens
|
||||
self.pooling_cursor: Optional[PoolingCursor] = pooling_cursor
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return ("PoolingMetadata("
|
||||
f"seq_groups={self.seq_groups}, "
|
||||
f"seq_data={self.seq_data}, "
|
||||
f"prompt_lens={self.prompt_lens})")
|
||||
|
||||
def __getitem__(self, indices: slice):
|
||||
return PoolingMetadata(
|
||||
seq_groups=self.seq_groups[indices],
|
||||
seq_data=dict(list(self.seq_data.items())[indices]),
|
||||
prompt_lens=self.prompt_lens[indices],
|
||||
pooling_cursor=None
|
||||
if self.pooling_cursor is None else self.pooling_cursor[indices],
|
||||
)
|
||||
|
||||
def build_pooling_cursor(self, num_scheduled_tokens: list[int],
|
||||
device: torch.device):
|
||||
prompt_lens = torch.tensor(self.prompt_lens, device="cpu")
|
||||
self.pooling_cursor = build_pooling_cursor(num_scheduled_tokens,
|
||||
prompt_lens,
|
||||
device=device)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PoolingTensors:
|
||||
"""Tensors for pooling."""
|
||||
|
||||
prompt_lens: torch.Tensor
|
||||
|
||||
@classmethod
|
||||
def from_pooling_metadata(
|
||||
cls,
|
||||
pooling_metadata: "PoolingMetadata",
|
||||
device: torch.device,
|
||||
) -> "PoolingTensors":
|
||||
"""
|
||||
Create PoolingTensors from PoolingMetadata.
|
||||
|
||||
Args:
|
||||
pooling_metadata: PoolingMetadata instance to convert.
|
||||
device: Device to store the tensors.
|
||||
"""
|
||||
# Convert prompt lengths to tensor
|
||||
pin_memory = is_pin_memory_available()
|
||||
|
||||
prompt_lens_t = torch.tensor(
|
||||
pooling_metadata.prompt_lens,
|
||||
device="cpu",
|
||||
dtype=torch.long,
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
|
||||
return cls(prompt_lens=prompt_lens_t.to(device=device,
|
||||
non_blocking=True), )
|
||||
Reference in New Issue
Block a user